[
  {
    "path": ".deepsource.toml",
    "content": "version = 1\n\ntest_patterns = [\"test/**/test_*.rb\"]\n\nexclude_patterns = [\n    \"bin/**\",\n    \"docs/**\",\n    \"example/**\"\n]\n\n[[analyzers]]\nname = \"ruby\"\nenabled = true\n"
  },
  {
    "path": ".github/DISCUSSION_TEMPLATE/q-a-japanese.yml",
    "content": "title: \"[QA (Japanese)]\"\nlabels: [\"Q&A (Japanese)\"]\nbody:\n  - type: markdown\n    attributes:\n      value: |\n        日本語で気軽に質問するためのカテゴリです。もし他の人が困っているのを見つけたらぜひ回答してあげてください。\n  - type: textarea\n    id: question\n    attributes:\n      label: やりたいこと\n      description: |\n        何について困っているのかを書いてください。試したことや実際の結果を示してください。\n        期待する挙動と実際の結果の違いがあればそれも書くのをおすすめします。\n    validations:\n      required: true\n  - type: textarea\n    id: configuration\n    attributes:\n      label: 設定した内容\n      description: |\n        どのような設定をして期待する挙動を実現しようとしたのかを書いてください。(例: fluentd.confの内容を貼り付ける)\n      render: apache\n  - type: textarea\n    id: logs\n    attributes:\n      label: ログの内容\n      description: |\n        Fluentdのログを提示してください。エラーログがあると回答の助けになります。(例: fluentd.logの内容を貼り付ける)\n      render: shell\n  - type: textarea\n    id: environment\n    attributes:\n      label: 環境について\n      description: |\n        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`\n        - Operating system: `cat /etc/os-release`\n        - Kernel version: `uname -r`\n\n        どんな環境で困っているかの情報がないと、再現できないため誰も回答できないことがあります。\n        必要な情報を記入することをおすすめします。\n      value: |\n        - Fluentd version:\n        - TD Agent version:\n        - Fluent Package version:\n        - Docker image (tag):\n        - Operating system:\n        - Kernel version:\n      render: markdown\n"
  },
  {
    "path": ".github/DISCUSSION_TEMPLATE/q-a.yml",
    "content": "title: \"[Q&A]\"\nlabels: [\"Q&A\"]\nbody:\n  - type: markdown\n    attributes:\n      value: |\n        It is recommended to support each other.\n  - type: textarea\n    id: question\n    attributes:\n      label: What is a problem?\n      description: |\n        A clear and concise description of what you want to happen.\n        What exactly did you do (or not do) that was effective (or ineffective)?\n    validations:\n      required: true\n  - type: textarea\n    id: configuration\n    attributes:\n      label: Describe the configuration of Fluentd\n      description: |\n        If there is the actual configuration of Fluentd, it will help.\n  - type: textarea\n    id: logs\n    attributes:\n      label: Describe the logs of Fluentd\n      description: |\n        If there are error logs of Fluentd, it will help.\n  - type: textarea\n    id: environment\n    attributes:\n      label: Environment\n      description: |\n        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`\n        - Operating system: `cat /etc/os-release`\n        - Kernel version: `uname -r`\n\n        Please describe your environment information. If will help to support.\n      value: |\n        - Fluentd version:\n        - TD Agent version:\n        - Fluent Package version:\n        - Docker image (tag):\n        - Operating system:\n        - Kernel version:\n      render: markdown\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/bug_report.yml",
    "content": "name: Bug Report\ndescription: Create a report with a procedure for reproducing the bug\nlabels: \"waiting-for-triage\"\nbody:\n  - type: markdown\n    attributes:\n      value: |\n        Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.\n  - type: textarea\n    id: description\n    attributes:\n      label: Describe the bug\n      description: A clear and concise description of what the bug is\n    validations:\n      required: true\n  - type: textarea\n    id: reproduce\n    attributes:\n      label: To Reproduce\n      description: Steps to reproduce the behavior\n    validations:\n      required: true\n  - type: textarea\n    id: expected\n    attributes:\n      label: Expected behavior\n      description: A clear and concise description of what you expected to happen\n    validations:\n      required: true\n  - type: textarea\n    id: environment\n    attributes:\n      label: Your Environment\n      description: |\n        - Fluentd or its package version: `fluentd --version` (Fluentd, fluent-package) or `td-agent --version` (td-agent)\n        - Operating system: `cat /etc/os-release`\n        - Kernel version: `uname -r`\n\n        Tip: If you hit the problem with older fluentd version, try latest version first.\n      value: |\n        - Fluentd version:\n        - Package version:\n        - Operating system:\n        - Kernel version:\n      render: markdown\n    validations:\n      required: true\n  - type: textarea\n    id: configuration\n    attributes:\n      label: Your Configuration\n      description: |\n        Write your configuration here. Minimum reproducible fluentd.conf is recommended.\n      render: apache\n    validations:\n      required: true\n  - type: textarea\n    id: logs\n    attributes:\n      label: Your Error Log\n      description: Write your ALL error log here\n      render: shell\n    validations:\n      required: true\n  - type: textarea\n    id: addtional-context\n    attributes:\n      label: Additional context\n      description: Add any other context about the problem here.\n    validations:\n      required: false\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/config.yml",
    "content": "blank_issues_enabled: false\ncontact_links:\n  - name: Ask a Question\n    url: https://github.com/fluent/fluentd/discussions\n    about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions\n  - name: Feedback a Fluentd Use-Case/Testimonial\n    url: https://github.com/fluent/fluentd-website/issues/new?template=testimonials.yml\n    about: Feedback your Fluentd use-case/testimonial, How do you use Fluentd in your service?\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/feature_request.yml",
    "content": "name: Feature request\ndescription: Suggest an idea for this project\nlabels: \"waiting-for-triage\"\nbody:\n  - type: markdown\n    attributes:\n      value: |\n        Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.\n  - type: textarea\n    id: description\n    attributes:\n      label: Is your feature request related to a problem? Please describe.\n      description: |\n        A clear and concise description of what the problem is.\n        Ex. I'm always frustrated when [...]\n    validations:\n      required: true\n  - type: textarea\n    id: solution\n    attributes:\n      label: Describe the solution you'd like\n      description: A clear and concise description of what you want to happen.\n    validations:\n      required: true\n  - type: textarea\n    id: alternative\n    attributes:\n      label: Describe alternatives you've considered\n      description: A clear and concise description of any alternative solutions or features you've considered.\n    validations:\n      required: true\n  - type: textarea\n    id: addtional-context\n    attributes:\n      label: Additional context\n      description: Add any other context or screenshots about the feature request here.\n    validations:\n      required: false\n\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE.md",
    "content": "Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.\n\n**Got a question or problem?**\n\nRESOURCES of [Official site](https://www.fluentd.org/) and [Fluentd documentation](https://docs.fluentd.org/) may help you.\n\nIf you have further questions about Fluentd and plugins, please direct these to [Mailing List](https://groups.google.com/forum/#!forum/fluentd).\nDon't use Github issue for asking questions. Here are examples:\n\n- I installed xxx plugin but it doesn't work. Why?\n- Fluentd starts but logs are not sent to xxx. Am I wrong?\n- I want to do xxx. How to realize it with plugins?\n\nWe may close such questions to keep clear repository for developers and users.\nGithub issue is mainly for submitting a bug report or feature request. See below.\n\nIf you can't judge your case is a bug or not, use mailing list or slack first.\n"
  },
  {
    "path": ".github/PULL_REQUEST_TEMPLATE.md",
    "content": "<!--\nThank you for contributing to Fluentd!\nYour commits need to follow DCO: https://probot.github.io/apps/dco/\nAnd please provide the following information to help us make the most of your pull request:\n-->\n\n**Which issue(s) this PR fixes**: \nFixes #\n\n**What this PR does / why we need it**: \n\n**Docs Changes**:\n\n**Release Note**: \n"
  },
  {
    "path": ".github/dependabot.yml",
    "content": "version: 2\nupdates:\n  - package-ecosystem: 'github-actions'\n    directory: '/'\n    schedule:\n      interval: 'weekly'\n"
  },
  {
    "path": ".github/workflows/backport.yml",
    "content": "name: Backport Pull Requests\n\non:\n  schedule:\n    # Sun 10:00 (JST)\n    - cron: '0 1 * * 0'\n  workflow_dispatch:\n\npermissions:\n  contents: read\n\nconcurrency:\n  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}\n  cancel-in-progress: true\n\njobs:\n  test:\n    permissions:\n      contents: write\n      pull-requests: write\n    runs-on: ubuntu-latest\n    continue-on-error: false\n    strategy:\n      fail-fast: false\n      matrix:\n        ruby-version: ['3.4']\n        task: ['backport:v1_19']\n\n    name: Backport PR ( ${{ matrix.task }} )\n    steps:\n      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n        with:\n          # fetch all history for all branches to execute cherry-pick\n          fetch-depth: 0\n      - name: Set up Ruby\n        uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0\n        with:\n          ruby-version: ${{ matrix.ruby-version }}\n          bundler-cache: true\n      - name: Set up git identity\n        run: |\n          git config user.name \"github-actions[bot]\"\n          git config user.email \"github-actions[bot]@users.noreply.github.com\"\n      - name: Run backport task ( ${{ matrix.task }} )\n        shell: bash\n        env:\n          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: |\n          bundle exec rake ${{ matrix.task }}\n"
  },
  {
    "path": ".github/workflows/benchmark.yml",
    "content": "name: Benchmark\n\non:\n  push:\n    branches: [master]\n  pull_request:\n    branches: [master]\n  workflow_dispatch:\n\npermissions: read-all\n\nconcurrency:\n  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}\n  cancel-in-progress: true\n\njobs:\n  test:\n    runs-on: ${{ matrix.os }}\n    continue-on-error: false\n    strategy:\n      fail-fast: false\n      matrix:\n        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']\n        ruby-version: ['4.0']\n\n    name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}\n    steps:\n      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n      - name: Set up Ruby\n        uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0\n        with:\n          ruby-version: ${{ matrix.ruby-version }}\n      - name: Install dependencies\n        run: bundle install\n      - name: Run Benchmark\n        shell: bash # Ensure to use bash shell on all platforms\n        run: |\n          bundle exec rake benchmark:run:in_tail | tee -a $GITHUB_STEP_SUMMARY\n"
  },
  {
    "path": ".github/workflows/rubocop.yml",
    "content": "name: RucoCop Security & Performance Check\n\non:\n  push:\n    paths-ignore:\n      - '*.md'\n      - 'lib/fluent/version.rb'\n  pull_request:\n    paths-ignore:\n      - '*.md'\n      - 'lib/fluent/version.rb'\n  workflow_dispatch:\n\nconcurrency:\n  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}\n  cancel-in-progress: true\n\npermissions: read-all\n\njobs:\n  rubocop:\n    runs-on: ubuntu-latest\n    continue-on-error: false\n    strategy:\n      fail-fast: false\n      matrix:\n        ruby-version: ['4.0']\n    name: Ruby ${{ matrix.ruby-version }}\n    steps:\n      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n      - name: Set up Ruby\n        uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0\n        with:\n          ruby-version: ${{ matrix.ruby-version }}\n      - name: Install dependencies\n        run: |\n          bundle install\n          gem install rubocop-performance\n      - name: Run RuboCop\n        run: rubocop\n"
  },
  {
    "path": ".github/workflows/scorecards.yml",
    "content": "name: Scorecard supply-chain security\non:\n  # For Branch-Protection check. Only the default branch is supported. See\n  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection\n  branch_protection_rule:\n  # To guarantee Maintained check is occasionally updated. See\n  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained\n  schedule:\n    - cron: '38 12 * * 2'\n  push:\n    branches: [ \"master\" ]\n\n# Declare default permissions as read only.\npermissions: read-all\n\njobs:\n  analysis:\n    name: Scorecard analysis\n    runs-on: ubuntu-latest\n    permissions:\n      # Needed to upload the results to code-scanning dashboard.\n      security-events: write\n      # Needed to publish results and get a badge (see publish_results below).\n      id-token: write\n      # Uncomment the permissions below if installing in a private repository.\n      # contents: read\n      # actions: read\n\n    steps:\n      - name: \"Checkout code\"\n        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n        with:\n          persist-credentials: false\n\n      - name: \"Run analysis\"\n        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3\n        with:\n          results_file: results.sarif\n          results_format: sarif\n          # (Optional) \"write\" PAT token. Uncomment the `repo_token` line below if:\n          # - you want to enable the Branch-Protection check on a *public* repository, or\n          # - you are installing Scorecard on a *private* repository\n          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional.\n          # repo_token: ${{ secrets.SCORECARD_TOKEN }}\n\n          # Public repositories:\n          #   - Publish results to OpenSSF REST API for easy access by consumers\n          #   - Allows the repository to include the Scorecard badge.\n          #   - See https://github.com/ossf/scorecard-action#publishing-results.\n          # For private repositories:\n          #   - `publish_results` will always be set to `false`, regardless\n          #     of the value entered here.\n          publish_results: true\n\n      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF\n      # format to the repository Actions tab.\n      - name: \"Upload artifact\"\n        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0\n        with:\n          name: SARIF file\n          path: results.sarif\n          retention-days: 5\n\n      # Upload the results to GitHub's code scanning dashboard (optional).\n      # Commenting out will disable upload of results to your repo's Code Scanning dashboard\n      - name: \"Upload to code-scanning\"\n        uses: github/codeql-action/upload-sarif@b1bff81932f5cdfc8695c7752dcee935dcd061c8 # v4.33.0\n        with:\n          sarif_file: results.sarif\n"
  },
  {
    "path": ".github/workflows/stale-actions.yml",
    "content": "name: \"Mark or close stale issues and PRs\"\non:\n  schedule:\n  - cron: \"00 10 * * *\"\n\npermissions: read-all\n\njobs:\n  stale:\n    runs-on: ubuntu-latest\n    permissions:\n      issues: write\n      pull-requests: write\n    steps:\n    - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0\n      with:\n        repo-token: ${{ secrets.GITHUB_TOKEN }}\n        days-before-stale: 30\n        days-before-close: 7\n        stale-issue-message: \"This issue has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this issue will be closed in 7 days\"\n        stale-pr-message: \"This PR has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this PR will be closed in 7 days\"\n        close-issue-message: \"This issue was automatically closed because of stale in 7 days\"\n        close-pr-message: \"This PR was automatically closed because of stale in 7 days\"\n        stale-pr-label: \"stale\"\n        stale-issue-label: \"stale\"\n        exempt-issue-labels: \"waiting-for-triage,bug,enhancement,feature request,pending,work-in-progress,v1,v2\"\n        exempt-pr-labels: \"waiting-for-triage,bug,enhancement,feature request,pending,work-in-progress,v1,v2\"\n        exempt-all-assignees: true\n        exempt-all-milestones: true\n"
  },
  {
    "path": ".github/workflows/test-ruby-head.yml",
    "content": "name: Test with Ruby head\n\non:\n  schedule:\n    - cron: '11 14 * * 0'\n  workflow_dispatch:\n\npermissions: read-all\n\njobs:\n  test:\n    runs-on: ${{ matrix.os }}\n    continue-on-error: false\n    strategy:\n      fail-fast: false\n      matrix:\n        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']\n        ruby-version: ['head']\n    env:\n      RUBYOPT: \"--disable-frozen_string_literal\"\n    name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}\n    steps:\n      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n      - name: Set up Ruby\n        uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0\n        with:\n          ruby-version: ${{ matrix.ruby-version }}\n      - name: Install addons\n        if: ${{ matrix.os == 'ubuntu-latest' }}\n        run: sudo apt-get install libgmp3-dev libcap-ng-dev\n      - name: Install dependencies\n        run: bundle install\n      - name: Run tests\n        run: bundle exec rake test TESTOPTS=\"-v --report-slow-tests --no-show-detail-immediately\"\n"
  },
  {
    "path": ".github/workflows/test.yml",
    "content": "name: Test\n\non:\n  push:\n    branches: [master]\n    paths-ignore:\n      - '*.md'\n      - 'lib/fluent/version.rb'\n  pull_request:\n    branches: [master]\n    paths-ignore:\n      - '*.md'\n      - 'lib/fluent/version.rb'\n  workflow_dispatch:\n\nconcurrency:\n  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}\n  cancel-in-progress: true\n\npermissions: read-all\n\njobs:\n  test:\n    runs-on: ${{ matrix.os }}\n    continue-on-error: false\n    strategy:\n      fail-fast: false\n      matrix:\n        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']\n        ruby-version: ['4.0', '3.4', '3.3', '3.2']\n    env:\n      RUBYOPT: \"--disable-frozen_string_literal\"\n    name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}\n    steps:\n      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n      - name: Set up Ruby\n        uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0\n        with:\n          ruby-version: ${{ matrix.ruby-version }}\n      - name: Install addons\n        if: ${{ matrix.os == 'ubuntu-latest' }}\n        run: sudo apt-get install libgmp3-dev libcap-ng-dev\n      - name: Install dependencies\n        run: bundle install\n      - name: Run tests\n        run: bundle exec rake test TESTOPTS=\"-v --report-slow-tests --no-show-detail-immediately\"\n\n  test-windows-service:\n    runs-on: windows-latest\n    strategy:\n      fail-fast: false\n      matrix:\n        ruby-version: ['4.0', '3.4', '3.3', '3.2']\n    name: Windows service (Ruby ${{ matrix.ruby-version }})\n    steps:\n      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n      - name: Set up Ruby\n        uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0\n        with:\n          ruby-version: ${{ matrix.ruby-version }}\n      - name: Install dependencies\n        run: |\n          bundle install\n          rake install\n      - name: Run tests\n        run: test\\scripts\\windows_service_test.ps1\n"
  },
  {
    "path": ".gitignore",
    "content": "Gemfile.lock\nINSTALL\nNEWS\nMakefile\nMakefile.in\nREADME\nac\naclocal.m4\nautom4te.cache\nconfdefs.h\nconfig.log\nconfig.status\nconfigure\ndeps/\nfluent-cat\nfluent-gem\nfluentd\npkg/*\ntmp/*\ntest/tmp/*\ntest/config/tmp/*\nmake_dist.sh\nGemfile.local\n.ruby-version\n*.swp\ncoverage/*\n.vagrant/\ncov-int/\ncov-fluentd.tar.gz\n.vscode\n.idea/\n.bundle/\n/vendor/\n"
  },
  {
    "path": ".rubocop.yml",
    "content": "plugins:\n  - rubocop-performance\n\nAllCops:\n  Exclude:\n    - 'vendor/**/*'\n  NewCops: enable\n  SuggestExtensions: false\n  TargetRubyVersion: 3.4\n\n#\n# Policy: Check Security & Performance in primary use-cases\n# (Disable most of cosmetic rules)\n#\n\nLint:\n  Enabled: false\n\nStyle:\n  Enabled: false\n\nGemspec:\n  Enabled: false\n\nNaming:\n  Enabled: false\n\nLayout:\n  Enabled: false\n\nMetrics:\n  Enabled: false\n\nSecurity:\n  Enabled: true\n\nPerformance:\n  Enabled: true\n\n#\n# False positive or exceptional cases\n#\n\n# False positive because it's intentional\nSecurity/Open:\n  Exclude:\n    - lib/fluent/plugin/buffer/chunk.rb\n  Enabled: true\n\n# False positive because it's intentional\nSecurity/Eval:\n  Exclude:\n    - lib/fluent/config/dsl.rb\n    - lib/fluent/plugin.rb\n    - lib/fluent/plugin/in_debug_agent.rb\n  Enabled: true\n\n# False positive because send method must accept literals.\nPerformance/StringIdentifierArgument:\n  Exclude:\n    - test/plugin/test_in_tcp.rb\n    - test/plugin/test_in_udp.rb\n    - test/counter/test_server.rb\n    - test/plugin/test_out_forward.rb\n    - lib/fluent/plugin/out_forward.rb\n  Enabled: true\n\nPerformance/StringInclude:\n  Exclude:\n    - 'test/**/*'\n    # It was not improved with String#include?\n    - lib/fluent/command/plugin_config_formatter.rb\n  Enabled: true\n\n# False positive for include? against constant ranges.\n# Almost same between include? and cover?.\n# See https://github.com/rubocop/rubocop-jp/issues/20\nPerformance/RangeInclude:\n   Exclude:\n     - lib/fluent/plugin/parser_multiline.rb\n   Enabled: true\n\n# Allow using &method(:func)\nPerformance/MethodObjectAsBlock:\n  Exclude:\n    - 'test/**/*'\n  Enabled: false\n\n# Allow block.call\nPerformance/RedundantBlockCall:\n  Exclude:\n    - 'test/**/*'\n    - 'lib/fluent/plugin_helper/*.rb'\n    - 'lib/fluent/plugin/*.rb'\n    - 'lib/fluent/compat/*.rb'\n    - 'lib/fluent/config/*.rb'\n    - 'lib/fluent/*.rb'\n  Enabled: true\n\n#\n# TODO: low priority to be fixed\n#\nPerformance/ConstantRegexp:\n   Exclude:\n     - 'test/**/*'\n   Enabled: true\n\nPerformance/Sum:\n  Exclude:\n    - 'test/**/*'\n  Enabled: true\n\nPerformance/CollectionLiteralInLoop:\n  Exclude:\n    - 'test/**/*'\n  Enabled: true\n"
  },
  {
    "path": "ADOPTERS.md",
    "content": "# Fluentd Adopters\n\nFluentd is widely used by hundred of companies, please refer to the testimonial section of our project website  to learn more about it:\n\nhttps://www.fluentd.org/testimonials\n"
  },
  {
    "path": "AUTHORS",
    "content": "FURUHASHI Sadayuki <frsyuki _at_ gmail.com>\nNAKAGAWA Masahiro <repeatedly _at_ gmail.com>\n"
  },
  {
    "path": "CHANGELOG.md",
    "content": "# v1.20\n\n## Release v1.20.0 - TBD\n\n### Bug fix\n\n* http_server helper: Fix IPv6 bind address support in URI construction\n  * Fixed `URI::InvalidURIError` when binding to IPv6 addresses (e.g., `::`, `::1`)\n  * IPv6 addresses are now properly bracketed in URIs per RFC 3986 (e.g., `http://[::]:24231`)\n  * Handles pre-bracketed addresses correctly to avoid double-bracketing\n  * Affects all plugins using http_server helper with IPv6 bind addresses\n\n# v1.19\n\n## Release v1.19.2 - 2026/02/13\n\n### Bug Fix\n\n* out_forward: add timeout to establish_connection to prevent infinite loop https://github.com/fluent/fluentd/pull/5138\n* gem: use latest net-http to solve IPv6 addr error https://github.com/fluent/fluentd/pull/5192\n* in_tail: fix error when files without read permission are included in glob patterns https://github.com/fluent/fluentd/pull/5222\n* command/fluentd: load win32/registry when edit registry for Ruby 4.0 https://github.com/fluent/fluentd/pull/5221\n* plugin_helper/http_server: Ensure request body is closed to prevent socket leaks in POST requests https://github.com/fluent/fluentd/pull/5234\n* config: fix duplicate config file loading in config_include_dir https://github.com/fluent/fluentd/pull/5235\n* gem: add ostruct gem as dependency for Ruby 4.0 https://github.com/fluent/fluentd/pull/5251\n\n### Misc\n\n* config: warn when backed-up config file will be included https://github.com/fluent/fluentd/pull/5252\n* filter_record_transformer: use cgi/escape to avoid Ruby 4.0 deprecation https://github.com/fluent/fluentd/pull/5256\n* CI fixes\n  * https://github.com/fluent/fluentd/pull/5257\n  * https://github.com/fluent/fluentd/pull/5229\n  * https://github.com/fluent/fluentd/pull/5225\n  * https://github.com/fluent/fluentd/pull/5186\n  * https://github.com/fluent/fluentd/pull/5184\n  * https://github.com/fluent/fluentd/pull/5176\n\n## Release v1.19.1 - 2025/11/06\n\n### Bug Fix\n\n* YAML config: Supports parsing array format https://github.com/fluent/fluentd/pull/5139\n\n### Misc\n\n* gem: fix uri gem version to keep IPv6 tests https://github.com/fluent/fluentd/pull/5144\n* CI fixes\n  * https://github.com/fluent/fluentd/pull/5055\n  * https://github.com/fluent/fluentd/pull/5057\n  * https://github.com/fluent/fluentd/pull/5063\n  * https://github.com/fluent/fluentd/pull/5064\n  * https://github.com/fluent/fluentd/pull/5077\n  * https://github.com/fluent/fluentd/pull/5078\n  * https://github.com/fluent/fluentd/pull/5136\n  * https://github.com/fluent/fluentd/pull/5140\n\n## Release v1.19.0 - 2025/07/30\n\n### Enhancement\n\nNew features:\n\n* Add zstd compression support https://github.com/fluent/fluentd/pull/4657\n  * Buffer: add `zstd` to `compress` option.\n  * out_file: add `zstd` to `compress` option.\n  * out_forward: add `zstd` to `compress` option. (Experimental)\n  * in_forward: support `zstd` format.\n* buffer: add feature to evacuate chunk files when retry limit\n  https://github.com/fluent/fluentd/pull/4986\n* out_http: TLS1.3 support\n  https://github.com/fluent/fluentd/pull/4859\n* out_stdout: support output to STDOUT independently of Fluentd logger by setting `use_logger` to `false`\n  https://github.com/fluent/fluentd/pull/5036\n* out_file: add symlink_path_use_relative option to use relative path instead of absolute path in symlink_path\n  https://github.com/fluent/fluentd/pull/4904\n* System configuration: Add forced_stacktrace_level to force the log level of stacktraces.\n  https://github.com/fluent/fluentd/pull/5008\n* System configuration: support built-in config files\n  https://github.com/fluent/fluentd/pull/4893\n\nMetrics:\n\n* metrics: enable input metrics by default\n  https://github.com/fluent/fluentd/pull/4966\n* in_tail: add \"tracked_file_count\" metrics to see how many log files are being tracked\n  https://github.com/fluent/fluentd/pull/4980\n* output: add metrics for number of writing events in secondary\n  https://github.com/fluent/fluentd/pull/4971\n* output: add metrics for dropped oldest chunk count\n  https://github.com/fluent/fluentd/pull/4981\n\nOthers:\n\n* in_forward: enable skip_invalid_event by default not to process broken data\n  https://github.com/fluent/fluentd/pull/5003\n* buf_file: reinforce buffer file corruption check\n  https://github.com/fluent/fluentd/pull/4998\n* in_http: allow empty Origin header requests to pass CORS checks\n  https://github.com/fluent/fluentd/pull/4866\n* in_tail: add warning for directory permission\n  https://github.com/fluent/fluentd/pull/4865\n* Add logging for errors about loading dependencies on startup\n  https://github.com/fluent/fluentd/pull/4858\n* Performance improvements\n  * https://github.com/fluent/fluentd/pull/4759\n    https://github.com/fluent/fluentd/pull/4760\n    https://github.com/fluent/fluentd/pull/4763\n    https://github.com/fluent/fluentd/pull/4764\n    https://github.com/fluent/fluentd/pull/4769\n    https://github.com/fluent/fluentd/pull/4813\n    https://github.com/fluent/fluentd/pull/4817\n    https://github.com/fluent/fluentd/pull/4835\n    https://github.com/fluent/fluentd/pull/4845\n    https://github.com/fluent/fluentd/pull/4881\n    https://github.com/fluent/fluentd/pull/4884\n    https://github.com/fluent/fluentd/pull/4886\n    https://github.com/fluent/fluentd/pull/4930\n    https://github.com/fluent/fluentd/pull/4931\n    https://github.com/fluent/fluentd/pull/4932\n    https://github.com/fluent/fluentd/pull/4933\n    https://github.com/fluent/fluentd/pull/4934\n    https://github.com/fluent/fluentd/pull/4995\n\n### Bug Fix\n\n* in_tail: fixed where specifying only encoding parameter might cause data corruption (affects since v0.14.12).\n  https://github.com/fluent/fluentd/pull/5010\n* formatter_csv: fix memory leak\n  https://github.com/fluent/fluentd/pull/4864\n* server plugin helper: ensure to close all connections at shutdown\n  https://github.com/fluent/fluentd/pull/5026\n* Fixed a bug where the default `umask` was not set to `0` when using `--daemon` (td-agent, fluent-package) since v1.14.6.\n  https://github.com/fluent/fluentd/pull/4836\n* `--umask` command line option: Fixed so that it is applied when Fluentd runs with `--daemon` (fluent-package) as well as when Fluentd runs with `--no-supervisor`.\n  https://github.com/fluent/fluentd/pull/4836\n* Windows: Stop the service when the supervisor is dead\n  https://github.com/fluent/fluentd/pull/4909\n* Windows: Fixed an issue where stopping the service immediately after startup could leave the processes.\n  https://github.com/fluent/fluentd/pull/4782\n* Windows: Fixed an issue where stopping service sometimes can not be completed forever.\n  https://github.com/fluent/fluentd/pull/4782\n\n### Misc\n\n* in_monitor_agent: stop using CGI.parse due to support Ruby 3.5\n  https://github.com/fluent/fluentd/pull/4962\n* HTTP server plugin helper: stop using CGI.parse due to support Ruby 3.5\n  https://github.com/fluent/fluentd/pull/4962\n* config: change inspect format\n  https://github.com/fluent/fluentd/pull/4914\n* console_adapter: support console gem v1.30\n  https://github.com/fluent/fluentd/pull/4857\n* gemspec: fix io-event and io-stream version to avoid unstable behavior on Windows\n  https://github.com/fluent/fluentd/pull/5042\n* in_http: replace WEBrick::HTTPUtils.parse_query with URI\n  Note that at least, this makes it unable to use ; delimiter.\n  https://github.com/fluent/fluentd/pull/4900\n* http_server: stop fallback to WEBrick\n  https://github.com/fluent/fluentd/pull/4899\n* metrics: add getter method automatically\n  https://github.com/fluent/fluentd/pull/4978\n* http_server helper: add `header` method for `Request`\n  https://github.com/fluent/fluentd/pull/4903\n* multi_output: fix metrics name\n  https://github.com/fluent/fluentd/pull/4979\n* plugin_id: fix typo\n  https://github.com/fluent/fluentd/pull/4964\n* CI fixes\n  * https://github.com/fluent/fluentd/pull/4728\n    https://github.com/fluent/fluentd/pull/4730\n    https://github.com/fluent/fluentd/pull/4746\n    https://github.com/fluent/fluentd/pull/4747\n    https://github.com/fluent/fluentd/pull/4748\n    https://github.com/fluent/fluentd/pull/4750\n    https://github.com/fluent/fluentd/pull/4755\n    https://github.com/fluent/fluentd/pull/4820\n    https://github.com/fluent/fluentd/pull/4874\n    https://github.com/fluent/fluentd/pull/4877\n* Fixes RuboCop's remarks\n  https://github.com/fluent/fluentd/pull/4928\n* CI: Add benchmark scripts\n  https://github.com/fluent/fluentd/pull/4989\n\n# v1.18\n\n## Release v1.18.0 - 2024/11/29\n\n### Enhancement\n\n* Add zero-downtime-restart feature for non-Windows\n  https://github.com/fluent/fluentd/pull/4624\n* Add with-source-only feature\n  https://github.com/fluent/fluentd/pull/4661\n  * `fluentd` command: Add `--with-source-only` option\n  * System configuration: Add `with_source_only` option\n* Embedded plugin: Add `out_buffer` plugin, which can be used for buffering and relabeling events\n  https://github.com/fluent/fluentd/pull/4661\n* Config File Syntax: Extend Embedded Ruby Code support for Hashes and Arrays\n  https://github.com/fluent/fluentd/pull/4580\n  * Example: `key {\"foo\":\"#{1 + 1}\"} => key {\"foo\":\"2\"}`\n  * Please note that this is not backward compatible, although we assume that this will never affect to actual existing configs.\n  * In case the behavior changes unintentionally, you can disable this feature by surrounding the entire value with single quotes.\n    * `key '{\"foo\":\"#{1 + 1}\"}' => key {\"foo\":\"#{1 + 1}\"}`\n* transport tls: Use SSL_VERIFY_NONE by default\n  https://github.com/fluent/fluentd/pull/4718\n* transport tls: Add ensure_fips option to ensure FIPS compliant mode\n  https://github.com/fluent/fluentd/pull/4720\n* plugin_helper/server: Add receive_buffer_size parameter in transport section\n  https://github.com/fluent/fluentd/pull/4649\n* filter_parser: Now able to handle multiple parsed results\n  https://github.com/fluent/fluentd/pull/4620\n* in_http: add `add_tag_prefix` option\n  https://github.com/fluent/fluentd/pull/4655\n* System configuration: add `path` option in `log` section\n  https://github.com/fluent/fluentd/pull/4604\n\n### Bug Fix\n\n* command: fix NoMethodError of --daemon under Windows\n  https://github.com/fluent/fluentd/pull/4716\n* `fluentd` command: fix `--plugin` (`-p`) option not to overwrite default value\n  https://github.com/fluent/fluentd/pull/4605\n\n### Misc\n\n* http_server: Ready to support Async 2.0 gem\n  https://github.com/fluent/fluentd/pull/4619\n* Minor code refactoring\n  * https://github.com/fluent/fluentd/pull/4641\n* CI fixes\n  * https://github.com/fluent/fluentd/pull/4638\n  * https://github.com/fluent/fluentd/pull/4644\n  * https://github.com/fluent/fluentd/pull/4675\n  * https://github.com/fluent/fluentd/pull/4676\n  * https://github.com/fluent/fluentd/pull/4677\n  * https://github.com/fluent/fluentd/pull/4686\n\n# v1.17\n\n## Release v1.17.1 - 2024/08/19\n\n### Enhancement\n\n* out_http: Add `compress gzip` option\n  https://github.com/fluent/fluentd/pull/4528\n* in_exec: Add `encoding` option to handle non-ascii characters\n  https://github.com/fluent/fluentd/pull/4533\n* in_tail: Add throttling metrics\n  https://github.com/fluent/fluentd/pull/4578\n* compat: Improve method call performance\n  https://github.com/fluent/fluentd/pull/4588\n* in_sample: Add `reuse_record` parameter to reuse the sample data\n  https://github.com/fluent/fluentd/pull/4586\n  * `in_sample` has changed to copy sample data by default to avoid the impact of destructive changes by subsequent plugins.\n  * This increases the load when generating large amounts of sample data.\n  * You can use this new parameter to have the same performance as before.\n\n### Bug Fix\n\n* logger: Fix LoadError with console gem v1.25\n  https://github.com/fluent/fluentd/pull/4492\n* parser_json: Fix wrong LoadError warning\n  https://github.com/fluent/fluentd/pull/4522\n* in_tail: Fix an issue where a large single line could consume a large amount of memory even though `max_line_size` is set\n  https://github.com/fluent/fluentd/pull/4530\n* yaml_parser: Support $log_level element\n  https://github.com/fluent/fluentd/pull/4482\n\n### Misc\n\n* Comment out inappropriate default configuration about out_forward\n  https://github.com/fluent/fluentd/pull/4523\n* gemspec: Remove unnecessary files from released gem\n  https://github.com/fluent/fluentd/pull/4534\n* plugin-generator: Update gemspec to remove unnecessary files\n  https://github.com/fluent/fluentd/pull/4535\n* Suppress non-parenthesis warnings\n  https://github.com/fluent/fluentd/pull/4594\n* Fix FrozenError in http_server plugin helper\n  https://github.com/fluent/fluentd/pull/4598\n* Add logger gem dependency for Ruby 3.5\n  https://github.com/fluent/fluentd/pull/4589\n* out_file: Add warn message for symlink_path setting\n  https://github.com/fluent/fluentd/pull/4502\n\n## Release v1.17.0 - 2024/04/30\n\n### Enhancement\n\n* in_http: Recognize CSP reports as JSON data\n  https://github.com/fluent/fluentd/pull/4282\n* out_http: Add option to reuse connections\n  https://github.com/fluent/fluentd/pull/4330\n* in_tail: Expand glob capability for square brackets and one character matcher\n  https://github.com/fluent/fluentd/pull/4401\n* out_http: Support AWS Signature Version 4 authentication\n  https://github.com/fluent/fluentd/pull/4459\n\n### Bug Fix\n\n* Make sure `parser_json` and `parser_msgpack` return `Hash`.\n  Make `parser_json` and `parser_msgpack` accept only `Hash` or `Array` of `Hash`.\n  https://github.com/fluent/fluentd/pull/4474\n* filter_parser: Add error event for multiple parsed results\n  https://github.com/fluent/fluentd/pull/4478\n\n### Misc\n\n* Raise minimum required ruby version\n  https://github.com/fluent/fluentd/pull/4288\n* Require missing dependent gems as of Ruby 3.4-dev\n  https://github.com/fluent/fluentd/pull/4411\n* Minor code refactoring\n  https://github.com/fluent/fluentd/pull/4294\n  https://github.com/fluent/fluentd/pull/4299\n  https://github.com/fluent/fluentd/pull/4302\n  https://github.com/fluent/fluentd/pull/4320\n* CI fixes\n  https://github.com/fluent/fluentd/pull/4369\n  https://github.com/fluent/fluentd/pull/4433\n  https://github.com/fluent/fluentd/pull/4452\n  https://github.com/fluent/fluentd/pull/4477\n* github: unify YAML file extension to .yml\n  https://github.com/fluent/fluentd/pull/4429\n\n# v1.16\n\n## Release v1.16.10 - 2025/09/12\n\n### Bug Fix\n\n* server plugin helper: ensure to close all connections at shutdown\n  https://github.com/fluent/fluentd/pull/5088\n\n### Misc\n\n* CI improvemnts\n  https://github.com/fluent/fluentd/pull/5083\n  https://github.com/fluent/fluentd/pull/5085\n  https://github.com/fluent/fluentd/pull/5086\n  https://github.com/fluent/fluentd/pull/5091\n  https://github.com/fluent/fluentd/pull/5092\n\n## Release v1.16.9 - 2025/05/14\n\n### Bug Fix\n\n* winsvc: Fix bug where service accidentally stops after starting.\n  The previous version (v1.16.8) should not be used for Windows Service.\n  https://github.com/fluent/fluentd/pull/4955\n\n### Misc\n\n* CI improvements\n  https://github.com/fluent/fluentd/pull/4956\n\n## Release v1.16.8 - 2025/05/01\n\n**This version has a critical bug about Windows Service. Do not use this version.**\n(https://github.com/fluent/fluentd/pull/4955)\n\n### Bug Fix\n\n* winsvc: Stop the service when the supervisor is dead\n  https://github.com/fluent/fluentd/pull/4942\n* formatter_csv: Fix memory leak\n  https://github.com/fluent/fluentd/pull/4920\n\n### Misc\n\n* Add fiddle as dependency gem for Ruby 3.5 on Windows\n  https://github.com/fluent/fluentd/pull/4919\n* Refactoring code\n  https://github.com/fluent/fluentd/pull/4921\n  https://github.com/fluent/fluentd/pull/4922\n  https://github.com/fluent/fluentd/pull/4926\n  https://github.com/fluent/fluentd/pull/4943\n* CI improvements\n  https://github.com/fluent/fluentd/pull/4821\n  https://github.com/fluent/fluentd/pull/4850\n  https://github.com/fluent/fluentd/pull/4851\n  https://github.com/fluent/fluentd/pull/4862\n  https://github.com/fluent/fluentd/pull/4915\n  https://github.com/fluent/fluentd/pull/4923\n  https://github.com/fluent/fluentd/pull/4925\n  https://github.com/fluent/fluentd/pull/4927\n\n## Release v1.16.7 - 2025/01/29\n\n### Bug Fix\n\n* Windows: Fix `NoMethodError` of `--daemon` option\n  https://github.com/fluent/fluentd/pull/4796\n* Windows: Fixed an issue where stopping the service immediately after startup could leave the processes\n  https://github.com/fluent/fluentd/pull/4782\n* Windows: Fixed an issue where stopping service sometimes can not be completed forever\n  https://github.com/fluent/fluentd/pull/4782\n\n### Misc\n\n* Windows: Add workaround for unexpected exception\n  https://github.com/fluent/fluentd/pull/4747\n* README: remove deprecated google analytics beacon\n  https://github.com/fluent/fluentd/pull/4797\n* CI improvements\n  https://github.com/fluent/fluentd/pull/4723\n  https://github.com/fluent/fluentd/pull/4788\n  https://github.com/fluent/fluentd/pull/4789\n  https://github.com/fluent/fluentd/pull/4790\n  https://github.com/fluent/fluentd/pull/4791\n  https://github.com/fluent/fluentd/pull/4793\n  https://github.com/fluent/fluentd/pull/4794\n  https://github.com/fluent/fluentd/pull/4795\n  https://github.com/fluent/fluentd/pull/4798\n  https://github.com/fluent/fluentd/pull/4799\n  https://github.com/fluent/fluentd/pull/4800\n  https://github.com/fluent/fluentd/pull/4801\n  https://github.com/fluent/fluentd/pull/4803\n\n## Release v1.16.6 - 2024/08/16\n\n### Bug Fix\n\n* YAML config syntax: Fix issue where `$log_level` element was not supported correctly\n  https://github.com/fluent/fluentd/pull/4486\n* parser_json: Fix wrong LoadError warning\n  https://github.com/fluent/fluentd/pull/4592\n* `fluentd` command: Fix `--plugin` (`-p`) option not to overwrite default value\n  https://github.com/fluent/fluentd/pull/4605\n\n### Misc\n\n* out_file: Add warn message for symlink_path setting\n  https://github.com/fluent/fluentd/pull/4512\n* Keep console gem v1.23 to avoid LoadError\n  https://github.com/fluent/fluentd/pull/4510\n\n## Release v1.16.5 - 2024/03/27\n\n### Bug Fix\n\n* Buffer: Fix emit error of v1.16.4 sometimes failing to process large data\n  exceeding chunk size limit\n  https://github.com/fluent/fluentd/pull/4447\n\n## Release v1.16.4 - 2024/03/14\n\n### Bug Fix\n\n* Fix to avoid processing discarded chunks in write_step_by_step.\n  It fixes not to raise pile of IOError when many `chunk\n  bytes limit exceeds` errors are occurred.\n  https://github.com/fluent/fluentd/pull/4342\n* in_tail: Fix tail watchers in `rotate_wait` state not being managed.\n  https://github.com/fluent/fluentd/pull/4334\n\n### Misc\n\n* buffer: Avoid unnecessary log processing. It will improve performance.\n  https://github.com/fluent/fluentd/pull/4331\n\n## Release v1.16.3 - 2023/11/14\n\n### Bug Fix\n\n* in_tail: Fix a stall bug on !follow_inode case\n  https://github.com/fluent/fluentd/pull/4327\n* in_tail: add warning for silent stop on !follow_inodes case\n  https://github.com/fluent/fluentd/pull/4339\n* Buffer: Fix NoMethodError with empty unstaged chunk arrays\n  https://github.com/fluent/fluentd/pull/4303\n* Fix for rotate_age where Fluentd passes as Symbol\n  https://github.com/fluent/fluentd/pull/4311\n\n## Release v1.16.2 - 2023/07/14\n\n### Bug Fix\n\n* in_tail: Fix new watcher is wrongly detached on rotation when `follow_inodes`,\n  which causes stopping tailing the file\n  https://github.com/fluent/fluentd/pull/4208\n* in_tail: Prevent wrongly unwatching when `follow_inodes`, which causes log\n  duplication\n  https://github.com/fluent/fluentd/pull/4237\n* in_tail: Fix warning log about overwriting entry when `follow_inodes`\n  https://github.com/fluent/fluentd/pull/4214\n* in_tail: Ensure to discard TailWatcher with missing target when `follow_inodes`\n  https://github.com/fluent/fluentd/pull/4239\n* MessagePackFactory: Make sure to reset local unpacker to prevent received\n  broken data from affecting other receiving data\n  https://github.com/fluent/fluentd/pull/4178\n* Fix failure to launch Fluentd on Windows when the log path isn't specified in\n  the command line\n  https://github.com/fluent/fluentd/pull/4188\n* logger: Prevent growing cache size of `ignore_same_log_interval` unlimitedly\n  https://github.com/fluent/fluentd/pull/4229\n* Update sigdump to 0.2.5 to fix wrong value of object counts\n  https://github.com/fluent/fluentd/pull/4225\n\n### Misc\n\n* in_tail: Check detaching inode when `follow_inodes`\n  https://github.com/fluent/fluentd/pull/4191\n* in_tail: Add debug log for pos file compaction\n  https://github.com/fluent/fluentd/pull/4228\n* Code improvements detected by RuboCop Performance\n  https://github.com/fluent/fluentd/pull/4201\n  https://github.com/fluent/fluentd/pull/4210\n* Add notice for unused argument `unpacker` of `ChunkMessagePackEventStreamer.each`\n  https://github.com/fluent/fluentd/pull/4159\n\n## Release v1.16.1 - 2023/04/17\n\n### Enhancement\n\n* in_tcp: Add `message_length_limit` to drop large incoming data\n  https://github.com/fluent/fluentd/pull/4137\n\n### Bug Fix\n\n* Fix NameError of `SecondaryFileOutput` when setting secondary other than\n  `out_secondary_file`\n  https://github.com/fluent/fluentd/pull/4124\n* Server helper: Suppress error of `UDPServer` over `max_bytes` on Windows\n  https://github.com/fluent/fluentd/pull/4131\n* Buffer: Fix that `compress` setting causes unexpected error when receiving\n  already compressed MessagePack\n  https://github.com/fluent/fluentd/pull/4147\n\n### Misc\n\n* Update MAINTAINERS.md\n  https://github.com/fluent/fluentd/pull/4119\n* Update security policy\n  https://github.com/fluent/fluentd/pull/4123\n* Plugin template: Remove unnecessary code\n  https://github.com/fluent/fluentd/pull/4128\n* Revive issue auto closer\n  https://github.com/fluent/fluentd/pull/4116\n* Fix a link for the repository of td-agent\n  https://github.com/fluent/fluentd/pull/4145\n* in_udp: add test of message_length_limit\n  https://github.com/fluent/fluentd/pull/4117\n* Fix a typo of an argument of `Fluent::EventStream#each`\n  https://github.com/fluent/fluentd/pull/4148\n* Test in_tcp: Fix undesirable way to assert logs\n  https://github.com/fluent/fluentd/pull/4138\n\n## Release v1.16.0 - 2023/03/29\n\n### Enhancement\n\n* in_tcp: Add `send_keepalive_packet` option\n  https://github.com/fluent/fluentd/pull/3961\n* buffer: backup broken file chunk\n  https://github.com/fluent/fluentd/pull/4025\n* Add warning messages for restoring buffer with `flush_at_shutdown true`\n  https://github.com/fluent/fluentd/pull/4027\n* Add logs for time period of restored buffer possibly broken\n  https://github.com/fluent/fluentd/pull/4028\n\n### Bug Fix\n\n* http_server_helper: Fix format of log messages originating from Async gem\n  https://github.com/fluent/fluentd/pull/3987\n* Change to not generate a sigdump file after receiving a `SIGTERM` signal on\n  non-Windows\n  https://github.com/fluent/fluentd/pull/4034\n  https://github.com/fluent/fluentd/pull/4043\n* out_forward: fix error of ack handling conflict on stopping with\n  `require_ack_response` enabled\n  https://github.com/fluent/fluentd/pull/4030\n* Fix problem that some `system` configs are not reflected\n  https://github.com/fluent/fluentd/pull/4064\n  https://github.com/fluent/fluentd/pull/4065\n  https://github.com/fluent/fluentd/pull/4086\n  https://github.com/fluent/fluentd/pull/4090\n  https://github.com/fluent/fluentd/pull/4096\n* Fix bug that the logger outputs some initial log messages without applying\n  some settings such as `format`\n  https://github.com/fluent/fluentd/pull/4091\n* Windows: Fix a bug that the wrong log file is reopened with log rotate setting\n  when flushing or graceful reloading\n  https://github.com/fluent/fluentd/pull/4054\n* Fix race condition of out_secondary_file\n  https://github.com/fluent/fluentd/pull/4081\n* Suppress warning using different secondary for out_secondary_file\n  https://github.com/fluent/fluentd/pull/4087\n* Fix value of `system_config.workers` at `run_configure`.\n  Change argument type of `Fluent::Plugin::Base::configure()` to\n  `Fluent::Config::Element` only.\n  https://github.com/fluent/fluentd/pull/4066\n* Fix bug that Fluentd sometimes tries to use an unavailable port and fails to\n  start on Windows\n  https://github.com/fluent/fluentd/pull/4092\n\n### Misc\n\n* Add method for testing `filtered_with_time`\n  https://github.com/fluent/fluentd/pull/3899\n* Replace `$$` with `Process.pid`\n  https://github.com/fluent/fluentd/pull/4040\n* Relax required webric gem version\n  https://github.com/fluent/fluentd/pull/4061\n* CI fixes to support Ruby 3.2\n  https://github.com/fluent/fluentd/pull/3968\n  https://github.com/fluent/fluentd/pull/3996\n  https://github.com/fluent/fluentd/pull/3997\n* Other CI fixes\n  https://github.com/fluent/fluentd/pull/3969\n  https://github.com/fluent/fluentd/pull/3990\n  https://github.com/fluent/fluentd/pull/4013\n  https://github.com/fluent/fluentd/pull/4033\n  https://github.com/fluent/fluentd/pull/4044\n  https://github.com/fluent/fluentd/pull/4050\n  https://github.com/fluent/fluentd/pull/4062\n  https://github.com/fluent/fluentd/pull/4074\n  https://github.com/fluent/fluentd/pull/4082\n  https://github.com/fluent/fluentd/pull/4085\n* Update MAINTAINERS.md\n  https://github.com/fluent/fluentd/pull/4026\n  https://github.com/fluent/fluentd/pull/4069\n\n# v1.15\n\n## Release v1.15.3 - 2022/11/02\n\n### Bug Fix\n\n* Support glob for `!include` directive in YAML config format\n  https://github.com/fluent/fluentd/pull/3917\n* Remove meaningless oj options\n  https://github.com/fluent/fluentd/pull/3929\n* Fix log initializer to correctly create per-process files on Windows\n  https://github.com/fluent/fluentd/pull/3939\n* out_file: Fix the multi-worker check with `<worker 0-N>` directive\n  https://github.com/fluent/fluentd/pull/3942\n\n### Misc\n\n* Fix broken tests on Ruby 3.2\n  https://github.com/fluent/fluentd/pull/3883\n  https://github.com/fluent/fluentd/pull/3922\n\n## Release v1.15.2 - 2022/08/22\n\n### Enhancement\n\n* Add a new system configuration `enable_jit`\n  https://github.com/fluent/fluentd/pull/3857\n\n### Bug Fix\n\n* out_file: Fix append mode with `--daemon` flag\n  https://github.com/fluent/fluentd/pull/3864\n* child_process: Plug file descriptor leak\n  https://github.com/fluent/fluentd/pull/3844\n\n### Misc\n\n* Drop win32-api gem to support Ruby 3.2\n  https://github.com/fluent/fluentd/pull/3849\n  https://github.com/fluent/fluentd/pull/3866\n\n## Release v1.15.1 - 2022/07/27\n\n### Bug Fix\n\n* Add support for concurrent append in out_file\n  https://github.com/fluent/fluentd/pull/3808\n\n### Misc\n\n* in_tail: Show more information on skipping update_watcher\n  https://github.com/fluent/fluentd/pull/3829\n\n## Release v1.15.0 - 2022/06/29\n\n### Enhancement\n\n* in_tail: Add log throttling in files based on group rules\n  https://github.com/fluent/fluentd/pull/3535\n  https://github.com/fluent/fluentd/pull/3771\n* Add `dump` command to fluent-ctl\n  https://github.com/fluent/fluentd/pull/3680\n* Handle YAML configuration format on configuration file\n  https://github.com/fluent/fluentd/pull/3712\n* Add `restart_worker_interval` parameter in `<system>` directive to set\n  interval to restart workers that has stopped for some reason.\n  https://github.com/fluent/fluentd/pull/3768\n\n### Bug fixes\n\n* out_forward: Fix to update timeout of cached sockets\n  https://github.com/fluent/fluentd/pull/3711\n* in_tail: Fix a possible crash on file rotation when `follow_inodes true`\n  https://github.com/fluent/fluentd/pull/3754\n* output: Fix a possible crash of flush thread\n  https://github.com/fluent/fluentd/pull/3755\n* in_tail: Fix crash bugs on Ruby 3.1 on Windows\n  https://github.com/fluent/fluentd/pull/3766\n* in_tail: Fix a bug that in_tail cannot open non-ascii path on Windows\n  https://github.com/fluent/fluentd/pull/3774\n* Fix a bug that fluentd doesn't release its own log file even after rotated by\n  external tools\n  https://github.com/fluent/fluentd/pull/3782\n\n### Misc\n\n* in_tail: Simplify TargetInfo related code\n  https://github.com/fluent/fluentd/pull/3489\n* Fix a wrong issue number in CHANGELOG\n  https://github.com/fluent/fluentd/pull/3700\n* server helper: Add comments to linger_timeout behavior about Windows\n  https://github.com/fluent/fluentd/pull/3701\n* service_discovery: Fix typo\n  https://github.com/fluent/fluentd/pull/3724\n* test: Fix unstable tests and warnings\n  https://github.com/fluent/fluentd/pull/3745\n  https://github.com/fluent/fluentd/pull/3753\n  https://github.com/fluent/fluentd/pull/3767\n  https://github.com/fluent/fluentd/pull/3783\n  https://github.com/fluent/fluentd/pull/3784\n  https://github.com/fluent/fluentd/pull/3785\n  https://github.com/fluent/fluentd/pull/3787\n\n# v1.14\n\n## Release v1.14.6 - 2022/03/31\n\n### Enhancement\n\n* Enable server plugins to specify socket-option `SO_LINGER`\n  https://github.com/fluent/fluentd/pull/3644\n* Add `--umask` command line parameter\n  https://github.com/fluent/fluentd/pull/3671\n  https://github.com/fluent/fluentd/pull/3679\n\n### Bug fixes\n\n* Fix metric name typo\n  https://github.com/fluent/fluentd/pull/3630\n  https://github.com/fluent/fluentd/pull/3673\n* Apply modifications in pipeline to the records being passed to `@ERROR` label\n  https://github.com/fluent/fluentd/pull/3631\n* Fix wrong calculation of retry interval\n  https://github.com/fluent/fluentd/pull/3640\n  https://github.com/fluent/fluentd/pull/3649\n  https://github.com/fluent/fluentd/pull/3685\n  https://github.com/fluent/fluentd/pull/3686\n* Support IPv6 address for `rpc_endpoint` in `system` config\n  https://github.com/fluent/fluentd/pull/3641\n\n### Misc\n\n* CI: Support Ruby 3.1 except Windows\n  https://github.com/fluent/fluentd/pull/3619\n* Switch to GitHub Discussions\n  https://github.com/fluent/fluentd/pull/3654\n* Fix CHANGELOG.md heading styles\n  https://github.com/fluent/fluentd/pull/3648\n* Declare `null_value_pattern` as `regexp`\n  https://github.com/fluent/fluentd/pull/3650\n\n## Release v1.14.5 - 2022/02/09\n\n### Enhancement\n\n* Add support for \"application/x-ndjson\" to `in_http`\n  https://github.com/fluent/fluentd/pull/3616\n* Add support for ucrt binary for Windows\n  https://github.com/fluent/fluentd/pull/3613\n\n### Bug fixes\n\n* Don't retry when `retry_max_times == 0`\n  https://github.com/fluent/fluentd/pull/3608\n* Fix hang-up issue during TLS handshake in `out_forward`\n  https://github.com/fluent/fluentd/pull/3601\n* Bump up required ServerEngine to v2.2.5\n  https://github.com/fluent/fluentd/pull/3599\n* Fix \"invalid byte sequence is replaced\" warning on Kubernetes\n  https://github.com/fluent/fluentd/pull/3596\n* Fix \"ArgumentError: unknown keyword: :logger\" on Windows with Ruby 3.1\n  https://github.com/fluent/fluentd/pull/3592\n\n## Release v1.14.4 - 2022/01/06\n\n### Enhancement\n\n* `in_tail`: Add option to skip long lines (`max_line_size`)\n  https://github.com/fluent/fluentd/pull/3565\n\n### Bug fix\n\n* Incorrect BufferChunkOverflowError when each event size is < `chunk_limit_size`\n  https://github.com/fluent/fluentd/pull/3560\n* On macOS with Ruby 2.7/3.0, `out_file` fails to write events if `append` is true.\n  https://github.com/fluent/fluentd/pull/3579\n* test: Fix unstable test cases\n  https://github.com/fluent/fluentd/pull/3574\n  https://github.com/fluent/fluentd/pull/3577\n\n## Release v1.14.3 - 2021/11/26\n\n### Enhancement\n\n* Changed to accept `http_parser.rb` 0.8.0.\n  `http_parser.rb` 0.8.0 is ready for Ractor.\n  https://github.com/fluent/fluentd/pull/3544\n\n### Bug fix\n\n* in_tail: Fixed a bug that no new logs are read when\n  `enable_stat_watcher true` and `enable_watch_timer false` is set.\n  https://github.com/fluent/fluentd/pull/3541\n* in_tail: Fixed a bug that the beginning and initial lines are lost\n  after startup when `read_from_head false` and path includes wildcard '*'.\n  https://github.com/fluent/fluentd/pull/3542\n* Fixed a bug that processing messages were lost when\n  BufferChunkOverflowError was thrown even though only a specific\n  message size exceeds chunk_limit_size.\n  https://github.com/fluent/fluentd/pull/3553\n  https://github.com/fluent/fluentd/pull/3562\n\n### Misc\n\n* Bump up required version of `win32-service` gem.\n  newer version is required to implement additional `fluent-ctl` commands.\n  https://github.com/fluent/fluentd/pull/3556\n\n## Release v1.14.2 - 2021/10/29\n\nIMPORTANT: This release contain the fix for CVE-2021-41186 -\nReDoS vulnerability in `parser_apache2`.\nThis vulnerability is affected from Fluentd v0.14.14 to v1.14.1.\nWe recommend to upgrade Fluentd to v1.14.2 or use patched version of\n`parser_apache2` plugin.\n\n### Enhancement\n\n* fluent-cat: Add `--event-time` option to send specified event time for testing.\n  https://github.com/fluent/fluentd/pull/3528\n\n### Bug fix\n\n* Fixed to generate correct epoch timestamp even after switching Daylight Saving Time\n  https://github.com/fluent/fluentd/pull/3524\n* Fixed ReDoS vulnerability in parser_apache2.\n  This vulnerability is caused by a certain pattern of a broken apache log.\n\n## Release v1.14.1 - 2021/09/29\n\n### Enhancement\n\n* in_tail: Added file related metrics.\n  These metrics should be collected same as fluent-bit's in_tail.\n  https://github.com/fluent/fluentd/pull/3504\n* out_forward: Changed to use metrics mechanism for node statistics\n  https://github.com/fluent/fluentd/pull/3506\n\n### Bug fix\n\n* in_tail: Fixed a crash bug that it raise undefined method of eof? error.\n  This error may happen only when `read_bytes_limit_per_second` was specified.\n  https://github.com/fluent/fluentd/pull/3500\n* out_forward: Fixed a bug that node statistics information is not included correctly.\n  https://github.com/fluent/fluentd/pull/3503\n  https://github.com/fluent/fluentd/pull/3507\n* Fixed a error when using `@include` directive\n  It was occurred when http/https scheme URI is used in `@include` directive with Ruby 3.\n  https://github.com/fluent/fluentd/pull/3517\n* out_copy: Fixed to suppress a wrong warning for `ignore_if_prev_success`\n  It didn't work even if a user set it.\n  https://github.com/fluent/fluentd/pull/3515\n* Fixed not to output nanoseconds field of next retry time in warning log\n  Then, inappropriate labels in log are also fixed. (retry_time -> retry_times,\n  next_retry_seconds -> next_retry_time)\n  https://github.com/fluent/fluentd/pull/3518\n\n## Release v1.14.0 - 2021/08/30\n\n### Enhancement\n\n* Added `enable_input_metrics`, `enable_size_metrics` system\n  configuration parameter\n  This feature might need to pay higher CPU cost, so input event metrics\n  features are disabled by default. These features are also enabled by\n  `--enable-input-metrics`,`--enable-size-metrics` command line\n  option.\n  https://github.com/fluent/fluentd/pull/3440\n* Added reserved word `@ROOT` for getting root router.\n  This is incompatible change. Do not use `@ROOT` for label name.\n  https://github.com/fluent/fluentd/pull/3358\n* in_syslog: Added `send_keepalive_packet` option\n  https://github.com/fluent/fluentd/pull/3474\n* in_http: Added `cors_allow_credentials` option.\n  This option tells browsers whether to expose the response to\n  frontend when the credentials mode is \"include\".\n  https://github.com/fluent/fluentd/pull/3481\n  https://github.com/fluent/fluentd/pull/3491\n\n### Bug fix\n\n* in_tail: Fixed a bug that deleted paths are not removed\n  from pos file by file compaction at start up\n  https://github.com/fluent/fluentd/pull/3467\n* in_tail: Revived a warning message of retrying unaccessible file\n  https://github.com/fluent/fluentd/pull/3478\n* TLSServer: Fixed a crash bug on logging peer host name errors\n  https://github.com/fluent/fluentd/pull/3483\n\n### Misc\n\n* Added metrics plugin mechanism\n  The implementations is changed to use metrics plugin.\n  In the future, 3rd party plugin will be able to handle these metrics.\n  https://github.com/fluent/fluentd/pull/3471\n  https://github.com/fluent/fluentd/pull/3473\n  https://github.com/fluent/fluentd/pull/3479\n  https://github.com/fluent/fluentd/pull/3484\n\n# v1.13\n\n## Release v1.13.3 - 2021/07/27\n\n### Bug fix\n\n* in_tail: Care DeletePending state on Windows\n  https://github.com/fluent/fluentd/pull/3457\n  https://github.com/fluent/fluentd/pull/3460\n* in_tail: Fix some pos_file bugs.\n  Avoid deleting pos_file entries unexpectedly when both\n  `pos_file_compaction_interval` and `follow_inode` are enabled.\n  Use `bytesize` instead of `size` for path length.\n  https://github.com/fluent/fluentd/pull/3459\n* in_tail: Fix detecting rotation twice on `follow_inode`.\n  https://github.com/fluent/fluentd/pull/3466\n\n### Misc\n\n* Remove needless spaces in a sample config file\n  https://github.com/fluent/fluentd/pull/3456\n\n## Release v1.13.2 - 2021/07/12\n\n### Enhancement\n\n* fluent-plugin-generate: Storage plugin was supported.\n  https://github.com/fluent/fluentd/pull/3426\n* parser_json: Added support to customize configuration of oj options.\n  Use `FLUENT_OJ_OPTION_BIGDECIMAL_LOAD`, `FLUENT_OJ_OPTION_MAX_NESTING`,\n  `FLUENT_OJ_OPTION_MODE`, and `FLUENT_OJ_OPTION_USE_TO_JSON` environment\n  variable to configure it.\n  https://github.com/fluent/fluentd/pull/3315\n\n### Bug fix\n\n* binlog_reader: Fixed a crash bug by missing \"fluent/env\" dependency.\n  https://github.com/fluent/fluentd/pull/3443\n* Fixed a crash bug on outputting log at the early stage when parsing\n  config file.  This is a regression since v1.13.0. If you use invalid\n  '@' prefix parameter, remove it as a workaround.\n  https://github.com/fluent/fluentd/pull/3451\n* in_tail: Fixed a bug that when rotation is occurred, remaining lines\n  will be discarded if the throttling feature is enabled.\n  https://github.com/fluent/fluentd/pull/3390\n* fluent-plugin-generate: Fixed a crash bug during gemspec generation.\n  It was unexpectedly introduced by #3305, thus this bug was a\n  regression since 1.12.3.\n  https://github.com/fluent/fluentd/pull/3444\n\n### Misc\n\n* Fixed the runtime dependency version of http_parse.rb to 0.7.0.\n  It was fixed because false positive detection is occurred frequently\n  by security scanning tools.\n  https://github.com/fluent/fluentd/pull/3450\n\n## Release v1.13.1 - 2021/06/25\n\n### Bug fix\n\n* out_forward: Fixed a race condition on handshake\n  It's caused by using a same unpacker from multiple threads.\n  https://github.com/fluent/fluentd/pull/3405\n  https://github.com/fluent/fluentd/pull/3406\n* in_tail: Fixed to remove too much verbose debugging logs\n  It was unexpectedly introduced by #3185 log throttling feature.\n  https://github.com/fluent/fluentd/pull/3418\n* Fixed not to echo back the provides path as is on a 404 error\n  There was a potential cross-site scripting vector even though\n  it is quite difficult to exploit.\n  https://github.com/fluent/fluentd/pull/3427\n\n### Misc\n\n* Pretty print for Fluent::Config::Section has been supported\n  for debugging\n  https://github.com/fluent/fluentd/pull/3398\n* CI: Dropped to run CI for Ruby 2.5\n  https://github.com/fluent/fluentd/pull/3412\n\n## Release v1.13.0 - 2021/05/29\n\n### Enhancement\n\n* in_tail: Handle log throttling per file feature\n  https://github.com/fluent/fluentd/pull/3185\n  https://github.com/fluent/fluentd/pull/3364\n  https://github.com/fluent/fluentd/pull/3379\n* Extend to support service discovery manager in simpler way\n  https://github.com/fluent/fluentd/pull/3299\n  https://github.com/fluent/fluentd/pull/3362\n* in_http: HTTP GET requests has been supported\n  https://github.com/fluent/fluentd/pull/3373\n* The log rotate settings in system configuration has been supported\n  https://github.com/fluent/fluentd/pull/3352\n\n### Bug fix\n\n* Fix to disable `trace_instruction` when\n  `RubyVM::InstructionSequence` is available. It improves\n  compatibility with `truffleruby` some extent.\n  https://github.com/fluent/fluentd/pull/3376\n* in_tail: Safely skip files which are used by another process on\n  Windows. It improves exception handling about\n  `ERROR_SHARING_VIOLATION` on Windows.\n  https://github.com/fluent/fluentd/pull/3378\n* fluent-cat: the issue resending secondary file in specific format\n  has been fixed\n  https://github.com/fluent/fluentd/pull/3368\n* in_tail: Shutdown immediately & safely even if reading huge files\n  Note that `skip_refresh_on_startup` must be enabled.\n  https://github.com/fluent/fluentd/pull/3380\n\n### Misc\n\n* example: Change a path to backup_path in counter_server correctly\n  https://github.com/fluent/fluentd/pull/3359\n* README: Update link to community forum to discuss.fluentd.org\n  https://github.com/fluent/fluentd/pull/3360\n\n# v1.12\n\n## Release v1.12.4 - 2021/05/26\n\n### Bug fix\n\n* in_tail: Fix a bug that refresh_watcher fails to handle file rotations\n  https://github.com/fluent/fluentd/pull/3393\n\n## Release v1.12.3 - 2021/04/23\n\n### Enhancement\n\n* plugin_helper: Allow TLS to use keep-alive socket option\n  https://github.com/fluent/fluentd/pull/3308\n\n### Bug fix\n\n* parser_csv, parser_syslog: Fix a naming conflict on parser_type\n  https://github.com/fluent/fluentd/pull/3302\n* in_tail: Fix incorrect error code & message on Windows\n  https://github.com/fluent/fluentd/pull/3325\n  https://github.com/fluent/fluentd/pull/3329\n  https://github.com/fluent/fluentd/pull/3331\n  https://github.com/fluent/fluentd/pull/3337\n* in_tail: Fix a crash bug on catching a short-lived log\n  https://github.com/fluent/fluentd/pull/3328\n* storage_local: Fix position file corruption issue on concurrent gracefulReloads\n  https://github.com/fluent/fluentd/pull/3335\n* Fix incorrect warnings about ${chunk_id} with out_s3\n  https://github.com/fluent/fluentd/pull/3339\n* TLS Server: Add peer information to error log message\n  https://github.com/fluent/fluentd/pull/3330\n\n### Misc\n\n* fluent-plugin-generate: add note about plugin name\n  https://github.com/fluent/fluentd/pull/3303\n* fluent-plugin-generate: Use same depended gem version with fluentd\n  https://github.com/fluent/fluentd/pull/3305\n* Fix some broken unit tests and improve CI's stability\n  https://github.com/fluent/fluentd/pull/3304\n  https://github.com/fluent/fluentd/pull/3307\n  https://github.com/fluent/fluentd/pull/3312\n  https://github.com/fluent/fluentd/pull/3313\n  https://github.com/fluent/fluentd/pull/3314\n  https://github.com/fluent/fluentd/pull/3316\n  https://github.com/fluent/fluentd/pull/3336\n* Permit to install with win32-service 2.2.0 on Windows\n  https://github.com/fluent/fluentd/pull/3343\n\n## Release v1.12.2 - 2021/03/29\n\n### Enhancement\n\n* out_copy: Add ignore_if_prev_successes\n  https://github.com/fluent/fluentd/pull/3190\n  https://github.com/fluent/fluentd/pull/3287\n* Support multiple kind of timestamp format\n  https://github.com/fluent/fluentd/pull/3252\n* formatter_ltsv: suppress delimiters in output\n  https://github.com/fluent/fluentd/pull/1666\n  https://github.com/fluent/fluentd/pull/3288\n  https://github.com/fluent/fluentd/pull/3289\n\n### Bug fix\n\n* in_tail: Expect ENOENT during stat\n  https://github.com/fluent/fluentd/pull/3275\n* out_forward: Prevent transferring duplicate logs on restart\n  https://github.com/fluent/fluentd/pull/3267\n  https://github.com/fluent/fluentd/pull/3285\n* in_tail: Handle to send rotated logs when mv is used for rotating\n  https://github.com/fluent/fluentd/pull/3294\n* fluent-plugin-config-format: Fill an uninitialized instance variable\n  https://github.com/fluent/fluentd/pull/3297\n* Fix MessagePackEventStream issue with Enumerable methods\n  https://github.com/fluent/fluentd/pull/2116\n\n### Misc\n\n* Add webrick to support Ruby 3.0\n  https://github.com/fluent/fluentd/pull/3257\n* Suggest Discourse instead of Google Groups\n  https://github.com/fluent/fluentd/pull/3261\n* Update MAINTAINERS.md\n  https://github.com/fluent/fluentd/pull/3282\n* Introduce DeepSource to check code quality\n  https://github.com/fluent/fluentd/pull/3286\n  https://github.com/fluent/fluentd/pull/3259\n  https://github.com/fluent/fluentd/pull/3291\n* Migrate to GitHub Actions and stabilize tests\n  https://github.com/fluent/fluentd/pull/3266\n  https://github.com/fluent/fluentd/pull/3268\n  https://github.com/fluent/fluentd/pull/3281\n  https://github.com/fluent/fluentd/pull/3283\n  https://github.com/fluent/fluentd/pull/3290\n\n## Release v1.12.1 - 2021/02/18\n\n### Enhancement\n\n* out_http: Add `headers_from_placeholders` parameter\n  https://github.com/fluent/fluentd/pull/3241\n* fluent-plugin-config-format: Add `--table` option to use markdown table\n  https://github.com/fluent/fluentd/pull/3240\n* Add `--disable-shared-socket`/`disable_shared_socket` to disable ServerEngine's shared socket setup\n  https://github.com/fluent/fluentd/pull/3250\n\n### Bug fix\n\n* ca_generate: Fix creating TLS certification files which include broken extensions\n  https://github.com/fluent/fluentd/pull/3246\n* test: Drop TLS 1.1 tests\n  https://github.com/fluent/fluentd/pull/3256\n* Remove old gem constraints to support Ruby 3\n\n### Misc\n\n* Use GitHub Actions\n  https://github.com/fluent/fluentd/pull/3233\n  https://github.com/fluent/fluentd/pull/3255\n\n## Release v1.12.0 - 2021/01/05\n\n### New feature\n\n* in_tail: Add `follow_inode` to support log rotation with wild card\n  https://github.com/fluent/fluentd/pull/3182\n* in_tail: Handle linux capability\n  https://github.com/fluent/fluentd/pull/3155\n  https://github.com/fluent/fluentd/pull/3162\n* windows: Add win32 events alternative to unix signals\n  https://github.com/fluent/fluentd/pull/3131\n\n### Enhancement\n\n* buffer: Enable metadata comparison optimization on all platforms\n  https://github.com/fluent/fluentd/pull/3095\n* fluent-plugin-config-formatter: Handle `service_discovery` type\n  https://github.com/fluent/fluentd/pull/3178\n* in_http: Add `add_query_params` parameter to add query params to event record\n  https://github.com/fluent/fluentd/pull/3197\n* inject: Support `unixtime_micros` and `unixtime_nanos` in `time_type`\n  https://github.com/fluent/fluentd/pull/3220\n* Refactoring code\n  https://github.com/fluent/fluentd/pull/3167\n  https://github.com/fluent/fluentd/pull/3170\n  https://github.com/fluent/fluentd/pull/3180\n  https://github.com/fluent/fluentd/pull/3196\n  https://github.com/fluent/fluentd/pull/3213\n  https://github.com/fluent/fluentd/pull/3222\n\n### Bug fix\n\n* output: Prevent retry.step from being called too many times in a short time\n  https://github.com/fluent/fluentd/pull/3203\n\n# v1.11\n\n## Release v1.11.5 - 2020/11/06\n\n### Enhancement\n\n* formatter: Provide `newline` parameter to support `CRLF`\n  https://github.com/fluent/fluentd/pull/3152\n* out_http: adding support for intermediate certificates\n  https://github.com/fluent/fluentd/pull/3146\n* Update serverengine dependency to 2.2.2 or later\n\n### Bug fix\n\n* Fix a bug that windows service isn't stopped gracefully\n  https://github.com/fluent/fluentd/pull/3156\n\n## Release v1.11.4 - 2020/10/13\n\n### Enhancement\n\n* inject: Support `unixtime_millis` in `time_type` parameter\n  https://github.com/fluent/fluentd/pull/3145\n\n### Bug fix\n\n* out_http: Fix broken data with `json_array true`\n  https://github.com/fluent/fluentd/pull/3144\n* output: Fix wrong logging issue for `${chunk_id}`\n  https://github.com/fluent/fluentd/pull/3134\n\n## Release v1.11.3 - 2020/09/30\n\n### Enhancement\n\n* in_exec: Add `connect_mode` parameter to read stderr\n  https://github.com/fluent/fluentd/pull/3108\n* parser_json: Improve the performance\n  https://github.com/fluent/fluentd/pull/3109\n* log: Add `ignore_same_log_interval` parameter\n  https://github.com/fluent/fluentd/pull/3119\n* Upgrade win32 gems\n  https://github.com/fluent/fluentd/pull/3100\n* Refactoring code\n  https://github.com/fluent/fluentd/pull/3094\n  https://github.com/fluent/fluentd/pull/3118\n\n### Bug fix\n\n* buffer: Fix calculation of timekey stats\n  https://github.com/fluent/fluentd/pull/3018\n* buffer: fix binmode usage for prevent gc\n  https://github.com/fluent/fluentd/pull/3138\n\n## Release v1.11.2 - 2020/08/04\n\n### Enhancement\n\n* `in_dummy` renamed to `in_sample`\n  https://github.com/fluent/fluentd/pull/3065\n* Allow regular expression in filter/match directive\n  https://github.com/fluent/fluentd/pull/3071\n* Refactoring code\n  https://github.com/fluent/fluentd/pull/3051\n\n### Bug fix\n\n* buffer: Fix log message for `chunk_limit_records` case\n  https://github.com/fluent/fluentd/pull/3079\n* buffer: Fix timekey optimization for non-windows platform\n  https://github.com/fluent/fluentd/pull/3092\n* cert: Raise an error for broken certificate file\n  https://github.com/fluent/fluentd/pull/3086\n* cert: Set TLS ciphers list correctly on older OpenSSL\n  https://github.com/fluent/fluentd/pull/3093\n\n## Release v1.11.1 - 2020/06/22\n\n### Enhancement\n\n* in_http: Add `dump_error_log` parameter\n  https://github.com/fluent/fluentd/pull/3035\n* in_http: Improve time field handling\n  https://github.com/fluent/fluentd/pull/3046\n* Refactoring code\n  https://github.com/fluent/fluentd/pull/3047\n\n### Bug fix\n\n* in_tail: Use actual path instead of based pattern for ignore list\n  https://github.com/fluent/fluentd/pull/3042\n* child_process helper: Fix child process failure due to SIGPIPE if the command uses stdout\n  https://github.com/fluent/fluentd/pull/3044\n\n## Release v1.11.0 - 2020/06/04\n\n### New feature\n\n* in_unix: Use v1 API\n  https://github.com/fluent/fluentd/pull/2992\n\n### Enhancement\n\n* parser_syslog: Support any `time_format` for RFC3164 string parser\n  https://github.com/fluent/fluentd/pull/3014\n* parser_syslog: Add new parser for RFC5424\n  https://github.com/fluent/fluentd/pull/3015\n* Refactoring code\n  https://github.com/fluent/fluentd/pull/3019\n\n### Bug fix\n\n* in_gc_stat: Add `use_symbol_keys` parameter to emit string key record\n  https://github.com/fluent/fluentd/pull/3008\n\n# v1.10\n\n## Release v1.10.4 - 2020/05/12\n\n### Enhancement\n\n* out_http: Support single json array payload\n  https://github.com/fluent/fluentd/pull/2973\n* Refactoring\n  https://github.com/fluent/fluentd/pull/2988\n\n### Bug fix\n\n* supervisor: Call `File.umask(0)` for standalone worker\n  https://github.com/fluent/fluentd/pull/2987\n* out_forward: Fix ZeroDivisionError issue with `weight 0`\n  https://github.com/fluent/fluentd/pull/2989\n\n## Release v1.10.3 - 2020/05/01\n\n### Enhancement\n\n* record_accessor: Add `set` method\n  https://github.com/fluent/fluentd/pull/2977\n* config: Ruby DSL format is deprecated\n  https://github.com/fluent/fluentd/pull/2958\n* Refactor code\n  https://github.com/fluent/fluentd/pull/2961\n  https://github.com/fluent/fluentd/pull/2962\n  https://github.com/fluent/fluentd/pull/2965\n  https://github.com/fluent/fluentd/pull/2966\n  https://github.com/fluent/fluentd/pull/2978\n\n### Bug fix\n\n* out_forward: Disable `linger_timeout` setting on Windows\n  https://github.com/fluent/fluentd/pull/2959\n* out_forward: Fix warning of service discovery manager when fluentd stops\n  https://github.com/fluent/fluentd/pull/2974\n\n## Release v1.10.2 - 2020/04/15\n\n### Enhancement\n\n* out_copy: Add plugin_id to log message\n  https://github.com/fluent/fluentd/pull/2934\n* socket: Allow cert chains in mutual auth\n  https://github.com/fluent/fluentd/pull/2930\n* system: Add ignore_repeated_log_interval parameter\n  https://github.com/fluent/fluentd/pull/2937\n* windows: Allow to launch fluentd from whitespace included path\n  https://github.com/fluent/fluentd/pull/2920\n* Refactor code\n  https://github.com/fluent/fluentd/pull/2935\n  https://github.com/fluent/fluentd/pull/2936\n  https://github.com/fluent/fluentd/pull/2938\n  https://github.com/fluent/fluentd/pull/2939\n  https://github.com/fluent/fluentd/pull/2946\n\n### Bug fix\n\n* in_syslog: Fix octet-counting mode bug\n  https://github.com/fluent/fluentd/pull/2942\n* out_forward: Create timer for purging obsolete sockets when keepalive_timeout is not set\n  https://github.com/fluent/fluentd/pull/2943\n* out_forward: Need authentication when sending tcp heartbeat with keepalive\n  https://github.com/fluent/fluentd/pull/2945\n* command: Fix fluent-debug start failure\n  https://github.com/fluent/fluentd/pull/2948\n* command: Fix regression of supervisor's worker and `--daemon` combo\n  https://github.com/fluent/fluentd/pull/2950\n\n## Release v1.10.1 - 2020/04/02\n\n### Enhancement\n\n* command: `--daemon` and `--no-supervisor` now work together\n  https://github.com/fluent/fluentd/pull/2912\n* Refactor code\n  https://github.com/fluent/fluentd/pull/2913\n\n### Bug fix\n\n* in_tail: `Fix pos_file_compaction_interval` parameter type\n  https://github.com/fluent/fluentd/pull/2921\n* in_tail: Fix seek position update after compaction\n  https://github.com/fluent/fluentd/pull/2922\n* parser_syslog: Fix regression in the `with_priority` and RFC5424 case\n  https://github.com/fluent/fluentd/pull/2923\n\n### Misc\n\n* Add document for security audit\n  https://github.com/fluent/fluentd/pull/2911\n\n## Release v1.10.0 - 2020/03/24\n\n### New feature\n\n* sd plugin: Add SRV record plugin\n  https://github.com/fluent/fluentd/pull/2876\n\n### Enhancement\n\n* server: Add `cert_verifier` parameter for TLS transport\n  https://github.com/fluent/fluentd/pull/2888\n* parser_syslog: Support customized time format\n  https://github.com/fluent/fluentd/pull/2886\n* in_dummy: Delete `suspend` parameter\n  https://github.com/fluent/fluentd/pull/2897\n* Refactor code\n  https://github.com/fluent/fluentd/pull/2858\n  https://github.com/fluent/fluentd/pull/2862\n  https://github.com/fluent/fluentd/pull/2864\n  https://github.com/fluent/fluentd/pull/2869\n  https://github.com/fluent/fluentd/pull/2870\n  https://github.com/fluent/fluentd/pull/2874\n  https://github.com/fluent/fluentd/pull/2881\n  https://github.com/fluent/fluentd/pull/2885\n  https://github.com/fluent/fluentd/pull/2894\n  https://github.com/fluent/fluentd/pull/2896\n  https://github.com/fluent/fluentd/pull/2898\n  https://github.com/fluent/fluentd/pull/2899\n  https://github.com/fluent/fluentd/pull/2900\n  https://github.com/fluent/fluentd/pull/2901\n  https://github.com/fluent/fluentd/pull/2906\n\n### Bug fix\n\n* out_forward: windows: Permit to specify `linger_timeout`\n  https://github.com/fluent/fluentd/pull/2868\n* parser_syslog: Fix syslog format detection\n  https://github.com/fluent/fluentd/pull/2879\n* buffer: Fix `available_buffer_space_ratio` calculation\n  https://github.com/fluent/fluentd/pull/2882\n* tls: Support CRLF based X.509 certificates\n  https://github.com/fluent/fluentd/pull/2890\n* msgpack_factory mixin: Fix performance penalty for deprecation log\n  https://github.com/fluent/fluentd/pull/2903\n\n\n# v1.9\n\n## Release v1.9.3 - 2020/03/05\n\n### Enhancement\n\n* in_tail: Emit buffered lines as `unmatched_line` at shutdown phase when `emit_unmatched_lines true`\n  https://github.com/fluent/fluentd/pull/2837\n* Specify directory mode explicitly\n  https://github.com/fluent/fluentd/pull/2827\n* server helper: Change SSLError log level to warn in accept\n  https://github.com/fluent/fluentd/pull/2861\n* Refactor code\n  https://github.com/fluent/fluentd/pull/2829\n  https://github.com/fluent/fluentd/pull/2830\n  https://github.com/fluent/fluentd/pull/2832\n  https://github.com/fluent/fluentd/pull/2836\n  https://github.com/fluent/fluentd/pull/2838\n  https://github.com/fluent/fluentd/pull/2842\n  https://github.com/fluent/fluentd/pull/2843\n\n### Bug fix\n\n* buffer: Add seq to metadata that it can be unique\n  https://github.com/fluent/fluentd/pull/2824\n  https://github.com/fluent/fluentd/pull/2853\n* buffer: Use `Tempfile` as binmode for decompression\n  https://github.com/fluent/fluentd/pull/2847\n\n### Misc\n\n* Add `.idea` to git ignore file\n  https://github.com/fluent/fluentd/pull/2834\n* appveyor: Fix tests\n  https://github.com/fluent/fluentd/pull/2853\n  https://github.com/fluent/fluentd/pull/2855\n* Update pem for test\n  https://github.com/fluent/fluentd/pull/2839\n\n## Release v1.9.2 - 2020/02/13\n\n### Enhancement\n\n* in_tail: Add `pos_file_compaction_interval` parameter for auto compaction\n  https://github.com/fluent/fluentd/pull/2805\n* command: Use given encoding when RUBYOPT has `-E`\n  https://github.com/fluent/fluentd/pull/2814\n\n### Bug fix\n\n* command: Accept RUBYOPT with two or more options\n  https://github.com/fluent/fluentd/pull/2807\n* command: Fix infinite loop bug when RUBYOPT is invalid\n  https://github.com/fluent/fluentd/pull/2813\n* log: serverengine's log should be formatted with the same format of fluentd\n  https://github.com/fluent/fluentd/pull/2812\n* in_http: Fix `NoMethodError` when `OPTIONS` request doesn't have 'Origin' header\n  https://github.com/fluent/fluentd/pull/2823\n* parser_syslog: Improved for parsing RFC5424 structured data in `parser_syslog`\n  https://github.com/fluent/fluentd/pull/2816\n\n## Release v1.9.1 - 2020/01/31\n\n### Enhancement\n\n* http_server helper: Support HTTPS\n  https://github.com/fluent/fluentd/pull/2787\n* in_tail: Add `path_delimiter` to split with any char\n  https://github.com/fluent/fluentd/pull/2796\n* in_tail: Remove an entry from PositionFile when it is unwatched\n  https://github.com/fluent/fluentd/pull/2803\n* out_http: Add warning for `retryable_response_code`\n  https://github.com/fluent/fluentd/pull/2809\n* parser_syslog: Add multiline RFC5424 support\n  https://github.com/fluent/fluentd/pull/2767\n* Add TLS module to unify TLS related code\n  https://github.com/fluent/fluentd/pull/2802\n\n### Bug fix\n\n* output: Add `EncodingError` to unrecoverable errors\n  https://github.com/fluent/fluentd/pull/2808\n* tls: Fix TLS version handling in secure mode\n  https://github.com/fluent/fluentd/pull/2802\n\n## Release v1.9.0 - 2020/01/22\n\n### New feature\n\n* New light-weight config reload mechanism\n  https://github.com/fluent/fluentd/pull/2716\n* Drop ruby 2.1/2.2/2.3 support\n  https://github.com/fluent/fluentd/pull/2750\n\n### Enhancement\n\n* output: Show better message for secondary warning\n  https://github.com/fluent/fluentd/pull/2751\n* Use `ext_monitor` gem if it is installed. For ruby 2.6 or earlier\n  https://github.com/fluent/fluentd/pull/2670\n* Support Ruby's Time class in msgpack serde\n  https://github.com/fluent/fluentd/pull/2775\n* Clean up code/test\n  https://github.com/fluent/fluentd/pull/2753\n  https://github.com/fluent/fluentd/pull/2763\n  https://github.com/fluent/fluentd/pull/2764\n  https://github.com/fluent/fluentd/pull/2780\n\n### Bug fix\n\n* buffer: Disable the optimization of Metadata instance comparison on Windows\n  https://github.com/fluent/fluentd/pull/2778\n* output/buffer: Fix stage size computation\n  https://github.com/fluent/fluentd/pull/2734\n* server: Ignore Errno::EHOSTUNREACH in TLS accept to avoid fluentd restart\n  https://github.com/fluent/fluentd/pull/2773\n* server: Fix IPv6 dual stack mode issue for udp socket\n  https://github.com/fluent/fluentd/pull/2781\n* config: Support @include/include directive for spaces included path\n  https://github.com/fluent/fluentd/pull/2780\n\n\n# v1.8\n\n## Release v1.8.1 - 2019/12/26\n\n### Enhancement\n\n* in_tail: Add `path_timezone` parameter to format `path` with the specified timezone\n  https://github.com/fluent/fluentd/pull/2719\n* out_copy: Add `copy_mode` parameter. `deep_copy` parameter is now deprecated.\n  https://github.com/fluent/fluentd/pull/2747\n* supervisor: Add deprecated log for `inline_config`\n  https://github.com/fluent/fluentd/pull/2746\n\n### Bug fixes\n\n* parser_ltsv: Prevent garbage result by checking `label_delimiter`\n  https://github.com/fluent/fluentd/pull/2748\n\n## Release v1.8.0 - 2019/12/11\n\n### New feature\n\n* Add service discovery plugin and `out_forward` use it\n  https://github.com/fluent/fluentd/pull/2541\n* config: Add strict mode and support `default`/`nil` value in ruby embedded mode\n  https://github.com/fluent/fluentd/pull/2685\n\n### Enhancement\n\n* formatter_csv: Support nested fields\n  https://github.com/fluent/fluentd/pull/2643\n* record_accessor helper: Make code simple and bit faster\n  https://github.com/fluent/fluentd/pull/2660\n* Relax tzinfo dependency to accept v1\n  https://github.com/fluent/fluentd/pull/2673\n* log: Deprecate top-level match for capturing fluentd logs\n  https://github.com/fluent/fluentd/pull/2689\n* in_monitor_agent: Expose Fluentd version in REST API\n  https://github.com/fluent/fluentd/pull/2706\n* time: Accept localtime xor utc\n  https://github.com/fluent/fluentd/pull/2720\n  https://github.com/fluent/fluentd/pull/2731\n* formatter_stdout: Make time_format configurable in stdout format\n  https://github.com/fluent/fluentd/pull/2721\n* supervisor: create log directory when it doesn't exists\n  https://github.com/fluent/fluentd/pull/2732\n* clean up internal classes / methods / code\n  https://github.com/fluent/fluentd/pull/2647\n  https://github.com/fluent/fluentd/pull/2648\n  https://github.com/fluent/fluentd/pull/2653\n  https://github.com/fluent/fluentd/pull/2654\n  https://github.com/fluent/fluentd/pull/2657\n  https://github.com/fluent/fluentd/pull/2667\n  https://github.com/fluent/fluentd/pull/2674\n  https://github.com/fluent/fluentd/pull/2677\n  https://github.com/fluent/fluentd/pull/2680\n  https://github.com/fluent/fluentd/pull/2709\n  https://github.com/fluent/fluentd/pull/2730\n\n### Bug fixes\n\n* output: Fix warning printed when chunk key placeholder not replaced\n  https://github.com/fluent/fluentd/pull/2523\n  https://github.com/fluent/fluentd/pull/2733\n* Fix dry-run mode\n  https://github.com/fluent/fluentd/pull/2651\n* suppress warning\n  https://github.com/fluent/fluentd/pull/2652\n* suppress keyword argument warning for ruby2.7\n  https://github.com/fluent/fluentd/pull/2664\n* RPC: Fix debug log text\n  https://github.com/fluent/fluentd/pull/2666\n* time: Properly show class names in error message\n  https://github.com/fluent/fluentd/pull/2671\n* Fix a potential bug that ThreadError may occur on SIGUSR1\n  https://github.com/fluent/fluentd/pull/2678\n* server helper: Ignore ECONNREFUSED in TLS accept to avoid fluentd restart\n  https://github.com/fluent/fluentd/pull/2695\n* server helper: Fix IPv6 dual stack mode issue for tcp socket.\n  https://github.com/fluent/fluentd/pull/2697\n* supervisor: Fix inline config handling\n  https://github.com/fluent/fluentd/pull/2708\n* Fix typo\n  https://github.com/fluent/fluentd/pull/2710\n  https://github.com/fluent/fluentd/pull/2714\n\n# v1.7\n\n## Release v1.7.4 - 2019/10/24\n\n### Enhancement\n\n* in_http: Add `use_204_response` parameter to return proper 204 response instead of 200.\n  fluentd v2 will change this parameter to `true`.\n  https://github.com/fluent/fluentd/pull/2640\n\n### Bug fixes\n\n* child_process helper: fix stderr blocking for discard case\n  https://github.com/fluent/fluentd/pull/2649\n* log: Fix log rotation handling on Windows\n  https://github.com/fluent/fluentd/pull/2663\n\n## Release v1.7.3 - 2019/10/01\n\n### Enhancement\n\n* in_syslog: Replace priority_key with severity_key\n  https://github.com/fluent/fluentd/pull/2636\n\n### Bug fixes\n\n* out_forward: Fix nil error after purge obsoleted sockets in socket cache\n  https://github.com/fluent/fluentd/pull/2635\n* fix typo in ChangeLog\n  https://github.com/fluent/fluentd/pull/2633\n\n## Release v1.7.2 - 2019/09/19\n\n### Enhancement\n\n* in_tcp: Add security/client to restrict access\n  https://github.com/fluent/fluentd/pull/2622\n\n### Bug fixes\n\n* buf_file/buf_file_single: fix to handle compress data during restart\n  https://github.com/fluent/fluentd/pull/2620\n* plugin: Use `__send__` to avoid conflict with user defined `send`\n  https://github.com/fluent/fluentd/pull/2614\n* buffer: reject invalid timekey at configure phase\n  https://github.com/fluent/fluentd/pull/2615\n\n\n## Release v1.7.1 - 2019/09/08\n\n### Enhancement\n\n* socket helper/out_forward: Support Windows certstore to load certificates\n  https://github.com/fluent/fluentd/pull/2601\n* parser_syslog: Add faster parser for rfc3164 message\n  https://github.com/fluent/fluentd/pull/2599\n\n### Bug fixes\n\n* buf_file/buf_file_single: fix to ignore placeholder based path.\n  https://github.com/fluent/fluentd/pull/2594\n* server helper: Ignore ETIMEDOUT error in SSL_accept\n  https://github.com/fluent/fluentd/pull/2595\n* buf_file: ensure to remove metadata after buffer creation failure\n  https://github.com/fluent/fluentd/pull/2598\n* buf_file_single: fix duplicated path setting check\n  https://github.com/fluent/fluentd/pull/2600\n* fix msgpack-ruby dependency to use recent feature\n  https://github.com/fluent/fluentd/pull/2606\n\n\n## Release v1.7.0 - 2019/08/20\n\n### New feature\n\n* buffer: Add file_single buffer plugin\n  https://github.com/fluent/fluentd/pull/2579\n* output: Add http output plugin\n  https://github.com/fluent/fluentd/pull/2488\n\n### Enhancement\n\n* buffer: Improve the performance of buffer routine\n  https://github.com/fluent/fluentd/pull/2560\n  https://github.com/fluent/fluentd/pull/2563\n  https://github.com/fluent/fluentd/pull/2564\n* output: Use Mutex instead of Monitor\n  https://github.com/fluent/fluentd/pull/2561\n* event: Add `OneEventStrea#empty?` method\n  https://github.com/fluent/fluentd/pull/2565\n* thread: Set thread name for ruby 2.3 or later\n  https://github.com/fluent/fluentd/pull/2574\n* core: Cache msgpack packer/unpacker to avoid the object allocation\n  https://github.com/fluent/fluentd/pull/2559\n* time: Use faster way to get sec and nsec\n  https://github.com/fluent/fluentd/pull/2557\n* buf_file: Reduce IO flush by removing `IO#truncate`\n  https://github.com/fluent/fluentd/pull/2551\n* in_tcp: Improve the performance for multiple event case\n  https://github.com/fluent/fluentd/pull/2567\n* in_syslog: support `source_hostname_key` and `source_address_key` for unmatched event\n  https://github.com/fluent/fluentd/pull/2553\n* formatter_csv: Improve the format performance.\n  https://github.com/fluent/fluentd/pull/2529\n* parser_csv: Add fast parser for typical cases\n  https://github.com/fluent/fluentd/pull/2535\n* out_forward: Refactor code\n  https://github.com/fluent/fluentd/pull/2516\n  https://github.com/fluent/fluentd/pull/2532\n\n### Bug fixes\n\n* output: fix data lost on decompression\n  https://github.com/fluent/fluentd/pull/2547\n* out_exec_filter: fix non-ascii encoding issue\n  https://github.com/fluent/fluentd/pull/2539\n* in_tail: Don't call parser's configure twice\n  https://github.com/fluent/fluentd/pull/2569\n* Fix unused message handling for <section> parameters\n  https://github.com/fluent/fluentd/pull/2578\n* Fix comment/message typos\n  https://github.com/fluent/fluentd/pull/2549\n  https://github.com/fluent/fluentd/pull/2554\n  https://github.com/fluent/fluentd/pull/2556\n  https://github.com/fluent/fluentd/pull/2566\n  https://github.com/fluent/fluentd/pull/2573\n  https://github.com/fluent/fluentd/pull/2576\n  https://github.com/fluent/fluentd/pull/2583\n\n# v1.6\n\n## Release v1.6.3 - 2019/07/29\n\n### Enhancement\n\n* in_syslog: Add `emit_unmatched_lines` parameter\n  https://github.com/fluent/fluentd/pull/2499\n* buf_file: Add `path_suffix` parameter\n  https://github.com/fluent/fluentd/pull/2524\n* in_tail: Improve the performance of split lines\n  https://github.com/fluent/fluentd/pull/2527\n\n### Bug fixes\n\n* http_server: Fix re-define render_json method\n  https://github.com/fluent/fluentd/pull/2517\n\n## Release v1.6.2 - 2019/07/11\n\n### Bug fixes\n\n* http_server helper: Add title argument to support multiple servers\n  https://github.com/fluent/fluentd/pull/2493\n\n## Release v1.6.1 - 2019/07/10\n\n### Enhancement\n\n* socket/cert: Support all private keys OpenSSL supports, not only RSA.\n  https://github.com/fluent/fluentd/pull/2487\n* output/buffer: Improve statistics method performance\n  https://github.com/fluent/fluentd/pull/2491\n\n### Bug fixes\n\n* plugin_config_formatter: update new doc URL\n  https://github.com/fluent/fluentd/pull/2481\n* out_forward: Avoid zero division error when there are no available nodes\n  https://github.com/fluent/fluentd/pull/2482\n\n## Release v1.6.0 - 2019/07/01\n\n### New feature\n\n* plugin: Add http_server helper and in_monitor_agent use it\n  https://github.com/fluent/fluentd/pull/2447\n\n### Enhancement\n\n* in_monitor_agent: Add more metrics for buffer/output\n  https://github.com/fluent/fluentd/pull/2450\n* time/plugin: Add `EventTime#to_time` method for fast conversion\n  https://github.com/fluent/fluentd/pull/2469\n* socket helper/out_forward: Add connect_timeout parameter\n  https://github.com/fluent/fluentd/pull/2467\n* command: Add `--conf-encoding` option\n  https://github.com/fluent/fluentd/pull/2453\n* parser_none: Small performance optimization\n  https://github.com/fluent/fluentd/pull/2455\n\n### Bug fixes\n\n* cert: Fix cert match pattern\n  https://github.com/fluent/fluentd/pull/2466\n* output: Fix forget to increment rollback count\n  https://github.com/fluent/fluentd/pull/2462\n\n# v1.5\n\n## Release v1.5.2 - 2019/06/13\n\n### Bug fixes\n\n* out_forward: Fix duplicated handshake bug in keepalive\n  https://github.com/fluent/fluentd/pull/2456\n\n## Release v1.5.1 - 2019/06/05\n\n### Enhancement\n\n* in_tail: Increase read block size to reduce IO call\n  https://github.com/fluent/fluentd/pull/2418\n* in_monitor_agent: Refactor code\n  https://github.com/fluent/fluentd/pull/2422\n\n### Bug fixes\n\n* out_forward: Fix socket handling of keepalive\n  https://github.com/fluent/fluentd/pull/2434\n* parser: Fix the use of name based timezone\n  https://github.com/fluent/fluentd/pull/2421\n* in_monitor_agent: Fix debug parameter handling\n  https://github.com/fluent/fluentd/pull/2423\n* command: Fix error handling of log rotation age option\n  https://github.com/fluent/fluentd/pull/2427\n* command: Fix ERB warning for ruby 2.6 or later\n  https://github.com/fluent/fluentd/pull/2430\n\n## Release v1.5.0 - 2019/05/18\n\n### New feature\n\n* out_forward: Support keepalive feature\n  https://github.com/fluent/fluentd/pull/2393\n* in_http: Support TLS via server helper\n  https://github.com/fluent/fluentd/pull/2395\n* in_syslog: Support TLS via server helper\n  https://github.com/fluent/fluentd/pull/2399\n\n### Enhancement\n\n* in_syslog: Add delimiter parameter\n  https://github.com/fluent/fluentd/pull/2378\n* in_forward: Add tag/add_tag_prefix parameters\n  https://github.com/fluent/fluentd/pull/2396\n* parser_json: Add stream_buffer_size parameter for yajl\n  https://github.com/fluent/fluentd/pull/2381\n* command: Add deprecated message to show-plugin-config option\n  https://github.com/fluent/fluentd/pull/2401\n* storage_local: Ignore empty file. Call sync after write for XFS.\n  https://github.com/fluent/fluentd/pull/2409\n\n### Bug fixes\n\n* out_forward: Don't use SO_LINGER on SSL/TLS WinSock\n  https://github.com/fluent/fluentd/pull/2398\n* server helper: Fix recursive lock issue in TLSServer\n  https://github.com/fluent/fluentd/pull/2341\n* Fix typo\n  https://github.com/fluent/fluentd/pull/2369\n\n# v1.4\n\n## Release v1.4.2 - 2019/04/02\n\n### Enhancements\n\n* in_http: subdomain support in CORS domain\n  https://github.com/fluent/fluentd/pull/2337\n* in_monitor_agent: Expose current timekey list as a buffer metrics\n  https://github.com/fluent/fluentd/pull/2343\n* in_tcp/in_udp: Add source_address_key parameter\n  https://github.com/fluent/fluentd/pull/2347\n* in_forward: Add send_keepalive_packet parameter to check the remote connection is available or not\n  https://github.com/fluent/fluentd/pull/2352\n\n### Bug fixes\n\n* out_exec_filter: Fix typo of child_respawn description\n  https://github.com/fluent/fluentd/pull/2341\n* in_tail: Create parent directories for symlink\n  https://github.com/fluent/fluentd/pull/2353\n* in_tail: Fix encoding duplication check for non-specified case\n  https://github.com/fluent/fluentd/pull/2361\n* log: Fix time format handling of plugin logger when log format is JSON\n  https://github.com/fluent/fluentd/pull/2356\n\n## Release v1.4.1 - 2019/03/18\n\n### Enhancements\n\n* system: Add worker_id to process_name when workers is larger than 1\n  https://github.com/fluent/fluentd/pull/2321\n* parser_regexp: Check named captures. When no named captures, configuration error is raised\n  https://github.com/fluent/fluentd/pull/2331\n\n### Bug fixes\n\n* out_forward: Make tls_client_private_key_passphrase secret\n  https://github.com/fluent/fluentd/pull/2324\n* in_syslog: Check message length when read from buffer in octet counting\n  https://github.com/fluent/fluentd/pull/2323\n\n## Release v1.4.0 - 2019/02/24\n\n### New features\n\n* multiprocess: Support <worker N-M> syntax\n  https://github.com/fluent/fluentd/pull/2292\n* output: Work <secondary> and retry_forever together\n  https://github.com/fluent/fluentd/pull/2276\n* out_file: Support placeholders in symlink_path\n  https://github.com/fluent/fluentd/pull/2254\n\n### Enhancements\n\n* output: Add MessagePack unpacker error to unrecoverable error list\n  https://github.com/fluent/fluentd/pull/2301\n* output: Reduce flush delay when large timekey and small timekey_wait are specified\n  https://github.com/fluent/fluentd/pull/2291\n* config: Support embedded ruby code in section argument.\n  https://github.com/fluent/fluentd/pull/2295\n* in_tail: Improve encoding parameter handling\n  https://github.com/fluent/fluentd/pull/2305\n* in_tcp/in_udp: Add <parse> section check\n  https://github.com/fluent/fluentd/pull/2267\n\n### Bug fixes\n\n* server: Ignore IOError and related errors in UDP\n  https://github.com/fluent/fluentd/pull/2310\n* server: Ignore EPIPE in TLS accept to avoid fluentd restart\n  https://github.com/fluent/fluentd/pull/2253\n\n# v1.3\n\n## Release v1.3.3 - 2019/01/06\n\n### Enhancements\n\n* parser_syslog: Use String#squeeze for performance improvement\n  https://github.com/fluent/fluentd/pull/2239\n* parser_syslog: Support RFC5424 timestamp without subseconds\n  https://github.com/fluent/fluentd/pull/2240\n\n### Bug fixes\n\n* server: Ignore ECONNRESET in TLS accept to avoid fluentd restart\n  https://github.com/fluent/fluentd/pull/2243\n* log: Fix plugin logger ignores fluentd log event setting\n  https://github.com/fluent/fluentd/pull/2252\n\n## Release v1.3.2 - 2018/12/10\n\n### Enhancements\n\n* out_forward: Support mutual TLS\n  https://github.com/fluent/fluentd/pull/2187\n* out_file: Create `pos_file` directory if it doesn't exist\n  https://github.com/fluent/fluentd/pull/2223\n\n### Bug fixes\n\n* output: Fix logs during retry\n  https://github.com/fluent/fluentd/pull/2203\n\n## Release v1.3.1 - 2018/11/27\n\n### Enhancements\n\n* out_forward: Separate parameter names for certificate\n  https://github.com/fluent/fluentd/pull/2181\n  https://github.com/fluent/fluentd/pull/2190\n* out_forward: Add `verify_connection_at_startup` parameter to check connection setting at startup phase\n  https://github.com/fluent/fluentd/pull/2184\n* config: Check right slash position in regexp type\n  https://github.com/fluent/fluentd/pull/2176\n* parser_nginx: Support multiple IPs in `http_x_forwarded_for` field\n  https://github.com/fluent/fluentd/pull/2171\n\n### Bug fixes\n\n* fluent-cat: Fix retry limit handling\n  https://github.com/fluent/fluentd/pull/2193\n* record_accessor helper: Delete top level field with bracket style\n  https://github.com/fluent/fluentd/pull/2192\n* filter_record_transformer: Keep `class` method to avoid undefined method error\n  https://github.com/fluent/fluentd/pull/2186\n\n## Release v1.3.0 - 2018/11/10\n\n### New features\n\n* output: Change thread execution control\n  https://github.com/fluent/fluentd/pull/2170\n* in_syslog: Support octet counting frame\n  https://github.com/fluent/fluentd/pull/2147\n* Use `flush_thread_count` value for `queued_chunks_limit_size` when `queued_chunks_limit_size` is not specified\n  https://github.com/fluent/fluentd/pull/2173\n\n### Enhancements\n\n* output: Show backtrace for unrecoverable errors\n  https://github.com/fluent/fluentd/pull/2149\n* in_http: Implement support for CORS preflight requests\n  https://github.com/fluent/fluentd/pull/2144\n\n### Bug fixes\n\n* server: Fix deadlock between on_writable and close in sockets\n  https://github.com/fluent/fluentd/pull/2165\n* output: show correct error when wrong plugin is specified for secondary\n  https://github.com/fluent/fluentd/pull/2169\n\n# v1.2\n\n## Release v1.2.6 - 2018/10/03\n\n### Enhancements\n\n* output: Add `disable_chunk_backup` for ignore broken chunks.\n  https://github.com/fluent/fluentd/pull/2117\n* parser_syslog: Improve regexp for RFC5424\n  https://github.com/fluent/fluentd/pull/2141\n* in_http: Allow specifying the wildcard '*' as the CORS domain\n  https://github.com/fluent/fluentd/pull/2139\n\n### Bug fixes\n\n* in_tail: Prevent thread switching in the interval between seek and read/write operations to pos_file\n  https://github.com/fluent/fluentd/pull/2118\n* parser: Handle LoadError properly for oj\n  https://github.com/fluent/fluentd/pull/2140\n\n## Release v1.2.5 - 2018/08/22\n\n### Bug fixes\n\n* in_tail: Fix resource leak by file rotation\n  https://github.com/fluent/fluentd/pull/2105\n* fix typos\n\n## Release v1.2.4 - 2018/08/01\n\n### Bug fixes\n\n* output: Consider timezone when calculate timekey\n  https://github.com/fluent/fluentd/pull/2054\n* output: Fix bug in suppress_emit_error_log_interval\n  https://github.com/fluent/fluentd/pull/2069\n* server-helper: Fix connection leak by close timing issue.\n  https://github.com/fluent/fluentd/pull/2087\n\n## Release v1.2.3 - 2018/07/10\n\n### Enhancements\n\n* in_http: Consider `<parse>` parameters in batch mode\n  https://github.com/fluent/fluentd/pull/2055\n* in_http: Support gzip payload\n  https://github.com/fluent/fluentd/pull/2060\n* output: Improve compress performance\n  https://github.com/fluent/fluentd/pull/2031\n* in_monitor_agent: Add missing descriptions for configurable options\n  https://github.com/fluent/fluentd/pull/2037\n* parser_syslog: update regex of pid field for conformance to RFC5424 spec\n  https://github.com/fluent/fluentd/pull/2051\n\n### Bug fixes\n\n* in_tail: Fix to rescue Errno::ENOENT for File.mtime()\n  https://github.com/fluent/fluentd/pull/2063\n* fluent-plugin-generate: Fix Parser plugin template\n  https://github.com/fluent/fluentd/pull/2026\n* fluent-plugin-config-format: Fix NoMethodError for some plugins\n  https://github.com/fluent/fluentd/pull/2023\n* config: Don't warn message for reserved parameters in DSL\n  https://github.com/fluent/fluentd/pull/2034\n\n## Release v1.2.2 - 2018/06/12\n\n### Enhancements\n\n* filter_parser: Add remove_key_name_field parameter\n  https://github.com/fluent/fluentd/pull/2012\n* fluent-plugin-config-format: Dump config_argument\n  https://github.com/fluent/fluentd/pull/2003\n\n### Bug fixes\n\n* in_tail: Change pos file entry handling to avoid read conflict for other plugins\n  https://github.com/fluent/fluentd/pull/1963\n* buffer: Wait for all chunks being purged before deleting @queued_num items\n  https://github.com/fluent/fluentd/pull/2016\n\n## Release v1.2.1 - 2018/05/23\n\n### Enhancements\n\n* Counter: Add wait API to client\n  https://github.com/fluent/fluentd/pull/1997\n\n### Bug fixes\n\n* in_tcp/in_udp: Fix source_hostname_key to set hostname correctly\n  https://github.com/fluent/fluentd/pull/1976\n* in_monitor_agent: Fix buffer_total_queued_size calculation\n  https://github.com/fluent/fluentd/pull/1990\n* out_file: Temporal fix for broken gzipped files with gzip and append\n  https://github.com/fluent/fluentd/pull/1995\n* test: Fix unstable backup test\n  https://github.com/fluent/fluentd/pull/1979\n* gemspec: Remove deprecated has_rdoc\n\n## Release v1.2.0 - 2018/04/30\n\n### New Features\n\n* New Counter API\n  https://github.com/fluent/fluentd/pull/1857\n* output: Backup for broken chunks\n  https://github.com/fluent/fluentd/pull/1952\n* filter_grep: Support for `<and>` and `<or>` sections\n  https://github.com/fluent/fluentd/pull/1897\n* config: Support `regexp` type in configuration parameter\n  https://github.com/fluent/fluentd/pull/1927\n\n### Enhancements\n\n* parser_nginx: Support optional `http-x-forwarded-for` field\n  https://github.com/fluent/fluentd/pull/1932\n* filter_grep: Improve the performance\n  https://github.com/fluent/fluentd/pull/1940\n\n### Bug fixes\n\n* log: Fix unexpected implementation bug when log rotation setting is applied\n  https://github.com/fluent/fluentd/pull/1957\n* server helper: Close invalid socket when ssl error happens on reading\n  https://github.com/fluent/fluentd/pull/1942\n* output: Buffer chunk's unique id should be formatted as hex in the log\n\n# v1.1\n\n## Release v1.1.3 - 2018/04/03\n\n### Enhancements\n\n* output: Support negative index for tag placeholders\n  https://github.com/fluent/fluentd/pull/1908\n* buffer: Add queued_chunks_limit_size to control the number of queued chunks\n  https://github.com/fluent/fluentd/pull/1916\n* time: Make Fluent::EventTime human readable for inspect\n  https://github.com/fluent/fluentd/pull/1915\n\n### Bug fixes\n\n* output: Delete empty queued_num field after purging chunks\n  https://github.com/fluent/fluentd/pull/1919\n* fluent-debug: Fix usage message of fluent-debug command\n  https://github.com/fluent/fluentd/pull/1920\n* out_forward: The node should be disabled when TLS socket for ack returns an error\n  https://github.com/fluent/fluentd/pull/1925\n\n## Release v1.1.2 - 2018/03/18\n\n### Enhancements\n\n* filter_grep: Support pattern starts with character classes with //\n  https://github.com/fluent/fluentd/pull/1887\n\n### Bug fixes\n\n* in_tail: Handle records in the correct order on file rotation\n  https://github.com/fluent/fluentd/pull/1880\n* out_forward: Fix race condition with `<security>` on multi thread environment\n  https://github.com/fluent/fluentd/pull/1893\n* output: Prevent flushing threads consume too much CPU when retry happens\n  https://github.com/fluent/fluentd/pull/1901\n* config: Fix boolean param handling for comment without value\n  https://github.com/fluent/fluentd/pull/1883\n* test: Fix random test failures in test/plugin/test_out_forward.rb\n  https://github.com/fluent/fluentd/pull/1881\n  https://github.com/fluent/fluentd/pull/1890\n* command: Fix typo in binlog_reader\n  https://github.com/fluent/fluentd/pull/1898\n\n## Release v1.1.1 - 2018/03/05\n\n### Enhancements\n\n* in_debug_agent: Support multi worker environment\n  https://github.com/fluent/fluentd/pull/1869\n* in_forward: Improve SSL setup to support mutual TLS\n  https://github.com/fluent/fluentd/pull/1861\n* buf_file: Skip and delete broken file chunks to avoid unsuccessful retry in resume\n  https://github.com/fluent/fluentd/pull/1874\n* command: Show fluentd version for debug purpose\n  https://github.com/fluent/fluentd/pull/1839\n\n### Bug fixes\n\n* in_forward: Do not close connection until write is complete on failed auth PONG\n  https://github.com/fluent/fluentd/pull/1835\n* in_tail: Fix IO event race condition during shutdown\n  https://github.com/fluent/fluentd/pull/1876\n* in_http: Emit event time instead of raw time value in batch\n  https://github.com/fluent/fluentd/pull/1850\n* parser_json: Add EncodingError to rescue list for oj 3.x.\n  https://github.com/fluent/fluentd/pull/1875\n* config: Fix config_param for string type with frozen string\n  https://github.com/fluent/fluentd/pull/1838\n* timer: Fix a bug to leak non-repeating timer watchers\n  https://github.com/fluent/fluentd/pull/1864\n\n## Release v1.1.0 - 2018/01/17\n\n### New features / Enhancements\n\n* config: Add hostname and worker_id short-cut\n  https://github.com/fluent/fluentd/pull/1814\n* parser_ltsv: Add delimiter_pattern parameter\n  https://github.com/fluent/fluentd/pull/1802\n* record_accessor helper: Support nested field deletion\n  https://github.com/fluent/fluentd/pull/1800\n* record_accessor helper: Expose internal instance `@keys` variable\n  https://github.com/fluent/fluentd/pull/1808\n* log: Improve Log#on_xxx API performance\n  https://github.com/fluent/fluentd/pull/1809\n* time: Improve time formatting performance\n  https://github.com/fluent/fluentd/pull/1796\n* command: Port certificates generating command from secure-forward\n  https://github.com/fluent/fluentd/pull/1818\n\n### Bug fixes\n\n* server helper: Fix TCP + TLS degradation\n  https://github.com/fluent/fluentd/pull/1805\n* time: Fix the method for TimeFormatter#call\n  https://github.com/fluent/fluentd/pull/1813\n\n# v1.0\n\n## Release v1.0.2 - 2017/12/17\n\n### New features / Enhancements\n\n* Use dig_rb instead of ruby_dig to support dig method in more objects\n  https://github.com/fluent/fluentd/pull/1794\n\n## Release v1.0.1 - 2017/12/14\n\n### New features / Enhancements\n\n* in_udp: Add receive_buffer_size parameter\n  https://github.com/fluent/fluentd/pull/1788\n* in_tail: Add enable_stat_watcher option to disable inotify events\n  https://github.com/fluent/fluentd/pull/1775\n* Relax strptime gem version\n\n### Bug fixes\n\n* in_tail: Properly handle moved back and truncated case\n  https://github.com/fluent/fluentd/pull/1793\n* out_forward: Rebuild weight array to apply server setting properly\n  https://github.com/fluent/fluentd/pull/1784\n* fluent-plugin-config-formatter: Use v1.0 for URL\n  https://github.com/fluent/fluentd/pull/1781\n\n## Release v1.0.0 - 2017/12/6\n\nSee [CNCF announcement](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)\n\n### New features / Enhancements\n\n* out_copy: Support ignore_error argument in `<store>`\n  https://github.com/fluent/fluentd/pull/1764\n* server helper: Improve resource usage of TLS transport\n  https://github.com/fluent/fluentd/pull/1764\n* Disable tracepoint feature to omit unnecessary insts\n  https://github.com/fluent/fluentd/pull/1764\n\n### Bug fixes\n\n* out_forward: Don't update retry state when failed to get ack response.\n  https://github.com/fluent/fluentd/pull/1686\n* plugin: Combine before_shutdown and shutdown call in one sequence.\n  https://github.com/fluent/fluentd/pull/1763\n* Add description to parsers\n  https://github.com/fluent/fluentd/pull/1776\n  https://github.com/fluent/fluentd/pull/1777\n  https://github.com/fluent/fluentd/pull/1778\n  https://github.com/fluent/fluentd/pull/1779\n  https://github.com/fluent/fluentd/pull/1780\n* filter_parser: Add parameter description\n  https://github.com/fluent/fluentd/pull/1773\n* plugin: Combine before_shutdown and shutdown call in one sequence.\n  https://github.com/fluent/fluentd/pull/1763\n\n# v0.14\n\n## Release v0.14.25 - 2017/11/29\n\n### New features / Enhancements\n\n* Disable tracepoint feature to omit unnecessary insts\n  https://github.com/fluent/fluentd/pull/1764\n\n### Bug fixes\n\n* out_forward: Don't update retry state when failed to get ack response.\n  https://github.com/fluent/fluentd/pull/1686\n* plugin: Combine before_shutdown and shutdown call in one sequence.\n  https://github.com/fluent/fluentd/pull/1763\n\n## Release v0.14.24 - 2017/11/24\n\n### New features / Enhancements\n\n* plugin-config-formatter: Add link to plugin helper result\n  https://github.com/fluent/fluentd/pull/1753\n* server helper: Refactor code\n  https://github.com/fluent/fluentd/pull/1759\n\n### Bug fixes\n\n* supervisor: Don't call change_privilege twice\n  https://github.com/fluent/fluentd/pull/1757\n\n## Release v0.14.23 - 2017/11/15\n\n### New features / Enhancements\n\n* in_udp: Add remove_newline parameter\n  https://github.com/fluent/fluentd/pull/1747\n\n### Bug fixes\n\n* buffer: Lock buffers in order of metadata\n  https://github.com/fluent/fluentd/pull/1722\n* in_tcp: Fix log corruption under load.\n  https://github.com/fluent/fluentd/pull/1729\n* out_forward: Fix elapsed time miscalculation in tcp heartbeat\n  https://github.com/fluent/fluentd/pull/1738\n* supervisor: Fix worker pid handling during worker restart\n  https://github.com/fluent/fluentd/pull/1739\n* in_tail: Skip setup failed watcher to avoid resource leak and log bloat\n  https://github.com/fluent/fluentd/pull/1742\n* agent: Add error location to emit error logs\n  https://github.com/fluent/fluentd/pull/1746\n* command: Consider hyphen and underscore in fluent-plugin-generate arguments\n  https://github.com/fluent/fluentd/pull/1751\n\n## Release v0.14.22 - 2017/11/01\n\n### New features / Enhancements\n\n* formatter_tsv: Add add_newline parameter\n  https://github.com/fluent/fluentd/pull/1691\n* out_file/out_secondary_file: Support ${chunk_id} placeholder. This includes extract_placeholders API change\n  https://github.com/fluent/fluentd/pull/1708\n* record_accessor: Support double quotes in bracket notation\n  https://github.com/fluent/fluentd/pull/1716\n* log: Show running ruby version in startup log\n  https://github.com/fluent/fluentd/pull/1717\n* log: Log message when chunk is created\n  https://github.com/fluent/fluentd/pull/1718\n* in_tail: Add pos_file duplication check\n  https://github.com/fluent/fluentd/pull/1720\n\n### Bug fixes\n\n* parser_apache2: Delay time parser initialization\n  https://github.com/fluent/fluentd/pull/1690\n* cert_option: Improve generated certificates' conformance to X.509 specification\n  https://github.com/fluent/fluentd/pull/1714\n* buffer: Always lock chunks first to avoid deadlock\n  https://github.com/fluent/fluentd/pull/1721\n\n## Release v0.14.21 - 2017/09/07\n\n### New features / Enhancements\n\n* filter_parser: Support record_accessor in key_name\n  https://github.com/fluent/fluentd/pull/1654\n* buffer: Support record_accessor in chunk keys\n  https://github.com/fluent/fluentd/pull/1662\n\n### Bug fixes\n\n* compat_parameters: Support all syslog parser parameters\n  https://github.com/fluent/fluentd/pull/1650\n* filter_record_transformer: Don't create new keys if the original record doesn't have `keep_keys` keys\n  https://github.com/fluent/fluentd/pull/1663\n* in_tail: Fix the error when 'tag *' is configured\n  https://github.com/fluent/fluentd/pull/1664\n* supervisor: Clear previous worker pids when receive kill signals.\n  https://github.com/fluent/fluentd/pull/1683\n\n## Release v0.14.20 - 2017/07/31\n\n### New features / Enhancements\n\n* plugin: Add record_accessor plugin helper\n  https://github.com/fluent/fluentd/pull/1637\n* log: Add format and time_format parameters to `<system>` setting\n  https://github.com/fluent/fluentd/pull/1644\n\n### Bug fixes\n\n* buf_file: Improve file handling to mitigate broken meta file\n  https://github.com/fluent/fluentd/pull/1628\n* in_syslog: Fix the description of resolve_hostname parameter\n  https://github.com/fluent/fluentd/pull/1633\n* process: Fix signal handling. Send signal to all workers\n  https://github.com/fluent/fluentd/pull/1642\n* output: Fix error message typo\n  https://github.com/fluent/fluentd/pull/1643\n\n## Release v0.14.19 - 2017/07/12\n\n### New features / Enhancements\n\n* in_syslog: More characters are available in tag part of syslog format\n  https://github.com/fluent/fluentd/pull/1610\n* in_syslog: Add resolve_hostname parameter\n  https://github.com/fluent/fluentd/pull/1616\n* filter_grep: Support new configuration format by config_section\n  https://github.com/fluent/fluentd/pull/1611\n\n### Bug fixes\n\n* output: Fix race condition of retry state in flush thread\n  https://github.com/fluent/fluentd/pull/1623\n* test: Fix typo in test_in_tail.rb\n  https://github.com/fluent/fluentd/pull/1622\n\n## Release v0.14.18 - 2017/06/21\n\n### New features / Enhancements\n\n* parser: Add rfc5424 regex without priority\n  https://github.com/fluent/fluentd/pull/1600\n\n### Bug fixes\n\n* in_tail: Fix timing issue that the excluded_path doesn't apply.\n  https://github.com/fluent/fluentd/pull/1597\n* config: Fix broken UTF-8 encoded configuration file handling\n  https://github.com/fluent/fluentd/pull/1592\n* out_forward: Don't stop heartbeat when error happen\n  https://github.com/fluent/fluentd/pull/1602\n* Fix command name typo in plugin template\n  https://github.com/fluent/fluentd/pull/1603\n\n## Release v0.14.17 - 2017/05/29\n\n### New features / Enhancements\n\n* in_tail: Add ignore_repeated_permission_error\n  https://github.com/fluent/fluentd/pull/1574\n* server: Accept private key for TLS server without passphrase\n  https://github.com/fluent/fluentd/pull/1575\n* config: Validate workers option on standalone mode\n  https://github.com/fluent/fluentd/pull/1577\n\n### Bug fixes\n\n* config: Mask all secret parameters in worker section\n  https://github.com/fluent/fluentd/pull/1580\n* out_forward: Fix ack handling\n  https://github.com/fluent/fluentd/pull/1581\n* plugin-config-format: Fix markdown format generator\n  https://github.com/fluent/fluentd/pull/1585\n\n## Release v0.14.16 - 2017/05/13\n\n### New features / Enhancements\n\n* config: Allow null byte in double-quoted string\n  https://github.com/fluent/fluentd/pull/1552\n* parser: Support %iso8601 special case for time_format\n  https://github.com/fluent/fluentd/pull/1562\n\n### Bug fixes\n\n* out_forward: Call proper method for each connection type\n  https://github.com/fluent/fluentd/pull/1560\n* in_monitor_agent: check variable buffer is a Buffer instance\n  https://github.com/fluent/fluentd/pull/1556\n* log: Add missing '<<' method to delegators\n  https://github.com/fluent/fluentd/pull/1558\n* command: uninitialized constant Fluent::Engine in fluent-binlog-reader\n  https://github.com/fluent/fluentd/pull/1568\n\n## Release v0.14.15 - 2017/04/23\n\n### New features / Enhancements\n\n* Add `<worker N>` directive\n  https://github.com/fluent/fluentd/pull/1507\n* in_tail: Do not warn that directories are unreadable in the in_tail plugin\n  https://github.com/fluent/fluentd/pull/1540\n* output: Add formatted_to_msgpack_binary? to Output plugin API\n  https://github.com/fluent/fluentd/pull/1547\n* windows: Allow the Windows Service name Fluentd runs as to be configurable\n  https://github.com/fluent/fluentd/pull/1548\n\n### Bug fixes\n\n* in_http: Fix X-Forwarded-For header handling. Accept multiple headers\n  https://github.com/fluent/fluentd/pull/1535\n* Fix backward compatibility with Fluent::DetachProcess and Fluent::DetachMultiProcess\n  https://github.com/fluent/fluentd/pull/1522\n* fix typo\n  https://github.com/fluent/fluentd/pull/1521\n  https://github.com/fluent/fluentd/pull/1523\n  https://github.com/fluent/fluentd/pull/1544\n* test: Fix out_file test with timezone\n  https://github.com/fluent/fluentd/pull/1546\n* windows: Quote the file path to the Ruby bin directory when starting fluentd as a windows service\n  https://github.com/fluent/fluentd/pull/1536\n\n## Release v0.14.14 - 2017/03/23\n\n### New features / Enhancements\n\n* in_http: Support 'application/msgpack` header\n  https://github.com/fluent/fluentd/pull/1498\n* in_udp: Add message_length_limit parameter for parameter name consistency with in_syslog\n  https://github.com/fluent/fluentd/pull/1515\n* in_monitor_agent: Start one HTTP server per worker on sequential port numbers\n  https://github.com/fluent/fluentd/pull/1493\n* in_tail: Skip the refresh of watching list on startup\n  https://github.com/fluent/fluentd/pull/1487\n* filter_parser: filter_parser: Add emit_invalid_record_to_error parameter\n  https://github.com/fluent/fluentd/pull/1494\n* parser_syslog: Support RFC5424 syslog format\n  https://github.com/fluent/fluentd/pull/1492\n* parser: Allow escape sequence in Apache access log\n  https://github.com/fluent/fluentd/pull/1479\n* config: Add actual value in the placeholder error message\n  https://github.com/fluent/fluentd/pull/1497\n* log: Add Fluent::Log#<< to support some SDKs\n  https://github.com/fluent/fluentd/pull/1478\n\n### Bug fixes\n\n* Fix cleanup resource\n  https://github.com/fluent/fluentd/pull/1483\n* config: Set encoding forcefully to avoid UndefinedConversionError\n  https://github.com/fluent/fluentd/pull/1477\n* Fix Input and Output deadlock when buffer is full during startup\n  https://github.com/fluent/fluentd/pull/1502\n* config: Fix log_level handling in `<system>`\n  https://github.com/fluent/fluentd/pull/1501\n* Fix typo in root agent error log\n  https://github.com/fluent/fluentd/pull/1491\n* storage: Fix a bug storage_create cannot accept hash as `conf` keyword argument\n  https://github.com/fluent/fluentd/pull/1482\n\n## Release v0.14.13 - 2017/02/17\n\n### New features / Enhancements\n\n* in_tail: Add 'limit_recently_modified' to limit watch files.\n  https://github.com/fluent/fluentd/pull/1474\n* configuration: Improve 'flush_interval' handling for better message and backward compatibility\n  https://github.com/fluent/fluentd/pull/1442\n* command: Add 'fluent-plugin-generate' command\n  https://github.com/fluent/fluentd/pull/1427\n* output: Skip record when 'Output#format' returns nil\n  https://github.com/fluent/fluentd/pull/1469\n\n### Bug fixes\n\n* output: Secondary calculation should consider 'retry_max_times'\n  https://github.com/fluent/fluentd/pull/1452\n* Fix regression of deprecated 'process' module\n  https://github.com/fluent/fluentd/pull/1443\n* Fix missing parser_regex require\n  https://github.com/fluent/fluentd/issues/1458\n  https://github.com/fluent/fluentd/pull/1453\n* Keep 'Fluent::BufferQueueLimitError' for existing plugins\n  https://github.com/fluent/fluentd/pull/1456\n* in_tail: Untracked files should be removed from watching list to avoid memory bloat\n  https://github.com/fluent/fluentd/pull/1467\n* in_tail: directories should be skipped when the ** pattern is used\n  https://github.com/fluent/fluentd/pull/1464\n* record_transformer: Revert \"Use BasicObject for cleanroom\" for `enable_ruby` regression.\n  https://github.com/fluent/fluentd/pull/1461\n* buf_file: handle \"Too many open files\" error to keep buffer and metadata pair\n  https://github.com/fluent/fluentd/pull/1468\n\n## Release v0.14.12 - 2017/01/30\n\n### New features / Enhancements\n* Support multi process workers by `workers` option\n  https://github.com/fluent/fluentd/pull/1386\n* Support TLS transport security layer by server plugin helper, and forward input/output plugins\n  https://github.com/fluent/fluentd/pull/1423\n* Update internal log event handling to route log events to `@FLUENT_LOG` label if configured, suppress log events in startup/shutdown in default\n  https://github.com/fluent/fluentd/pull/1405\n* Rename buffer plugin chunk limit parameters for consistency\n  https://github.com/fluent/fluentd/pull/1412\n* Encode string values from configuration files in UTF8\n  https://github.com/fluent/fluentd/pull/1411\n* Reorder plugin load paths to load rubygem plugins earlier than built-in plugins to overwrite them\n  https://github.com/fluent/fluentd/pull/1410\n* Clock API to control internal thread control\n  https://github.com/fluent/fluentd/pull/1425\n* Validate `config_param` options to restrict unexpected specifications\n  https://github.com/fluent/fluentd/pull/1437\n* formatter: Add `add_newline` option to get formatted lines without newlines\n  https://github.com/fluent/fluentd/pull/1420\n* in_forward: Add `ignore_network_errors_at_startup` option for automated cluster deployment\n  https://github.com/fluent/fluentd/pull/1399\n* in_forward: Close listening socket in #stop, not to accept new connection request in early stage of shutdown\n  https://github.com/fluent/fluentd/pull/1401\n* out_forward: Ensure to pack values in `str` type of msgpack\n  https://github.com/fluent/fluentd/pull/1413\n* in_tail: Add `emit_unmatched_lines` to capture lines which unmatch configured regular expressions\n  https://github.com/fluent/fluentd/pull/1421\n* in_tail: Add `open_on_every_update` to read lines from files opened in exclusive mode on Windows platform\n  https://github.com/fluent/fluentd/pull/1409\n* in_monitor_agent: Add `with_ivars` query parameter to get instance variables only for specified instance variables\n  https://github.com/fluent/fluentd/pull/1393\n* storage_local: Generate file store path using `usage`, with `root_dir` configuration\n  https://github.com/fluent/fluentd/pull/1438\n* Improve test stability\n  https://github.com/fluent/fluentd/pull/1426\n\n### Bug fixes\n* Fix bug to ignore command line options: `--rpc-endpoint`, `--suppress-config-dump`, etc\n  https://github.com/fluent/fluentd/pull/1398\n* Fix bug to block infinitely in shutdown when buffer is full and `overflow_action` is `block`\n  https://github.com/fluent/fluentd/pull/1396\n* buf_file: Fix bug not to use `root_dir` even if configured correctly\n  https://github.com/fluent/fluentd/pull/1417\n* filter_record_transformer: Fix to use BasicObject for clean room\n  https://github.com/fluent/fluentd/pull/1415\n* filter_record_transformer: Fix bug that `remove_keys` doesn't work with `renew_time_key`\n  https://github.com/fluent/fluentd/pull/1433\n* in_monitor_agent: Fix bug to crash with NoMethodError for some output plugins\n  https://github.com/fluent/fluentd/pull/1365\n\n## Release v0.14.11 - 2016/12/26\n\n### New features / Enhancements\n* Add \"root_dir\" parameter in `<system>` directive to configure server root directory, used for buffer/storage paths\n  https://github.com/fluent/fluentd/pull/1374\n* Fix not to restart Fluentd processes when unrecoverable errors occur\n  https://github.com/fluent/fluentd/pull/1359\n* Show warnings in log when output flush operation takes longer time than threshold\n  https://github.com/fluent/fluentd/pull/1370\n* formatter_csv: Raise configuration error when no field names are specified\n  https://github.com/fluent/fluentd/pull/1369\n* in_syslog: Update implementation to use plugin helpers\n  https://github.com/fluent/fluentd/pull/1382\n* in_forward: Add a configuration parameter \"source_address_key\"\n  https://github.com/fluent/fluentd/pull/1382\n* in_monitor_agent: Add a parameter \"include_retry\" to get detail retry status\n  https://github.com/fluent/fluentd/pull/1387\n* Add Ruby 2.4 into supported ruby versions\n\n### Bug fixes\n* Fix to set process name of supervisor process\n  https://github.com/fluent/fluentd/pull/1380\n* in_forward: Fix a bug not to handle \"require_ack_response\" correctly\n  https://github.com/fluent/fluentd/pull/1389\n\n\n## Release v0.14.10 - 2016/12/14\n\n### New features / Enhancement\n\n* Add socket/server plugin helper to write TCP/UDP clients/servers as Fluentd plugin\n  https://github.com/fluent/fluentd/pull/1312\n  https://github.com/fluent/fluentd/pull/1350\n  https://github.com/fluent/fluentd/pull/1356\n  https://github.com/fluent/fluentd/pull/1362\n* Fix to raise errors when injected hostname is also specified as chunk key\n  https://github.com/fluent/fluentd/pull/1357\n* in_tail: Optimize to read lines from file\n  https://github.com/fluent/fluentd/pull/1325\n* in_monitor_agent: Add new parameter \"include_config\"(default: true)\n  https://github.com/fluent/fluentd/pull/1317\n* in_syslog: Add \"priority_key\" and \"facility_key\" options\n  https://github.com/fluent/fluentd/pull/1351\n* filter_record_transformer: Remove obsoleted syntax like \"${message}\" and not to dump records in logs\n  https://github.com/fluent/fluentd/pull/1328\n* Add an option \"--time-as-integer\" to fluent-cat command to send events from v0.14 fluent-cat to v0.12 fluentd\n  https://github.com/fluent/fluentd/pull/1349\n\n### Bug fixes\n\n* Specify correct Oj options for newer versions (Oj 2.18.0 or later)\n  https://github.com/fluent/fluentd/pull/1331\n* TimeSlice output plugins (in v0.12 style) raise errors when \"utc\" parameter is specified\n  https://github.com/fluent/fluentd/pull/1319\n* Parser plugins cannot use options for regular expressions\n  https://github.com/fluent/fluentd/pull/1326\n* Fix bugs not to raise errors to use logger in v0.12 plugins\n  https://github.com/fluent/fluentd/pull/1344\n  https://github.com/fluent/fluentd/pull/1332\n* Fix bug about shutting down Fluentd in Windows\n  https://github.com/fluent/fluentd/pull/1367\n* in_tail: Close files explicitly in tests\n  https://github.com/fluent/fluentd/pull/1327\n* out_forward: Fix bug not to convert buffer configurations into v0.14 parameters\n  https://github.com/fluent/fluentd/pull/1337\n* out_forward: Fix bug to raise error when \"expire_dns_cache\" is specified\n  https://github.com/fluent/fluentd/pull/1346\n* out_file: Fix bug to raise error about buffer chunking when it's configured as secondary\n  https://github.com/fluent/fluentd/pull/1338\n\n## Release v0.14.9 - 2016/11/15\n\n### New features / Enhancement\n\n* filter_parser: Port fluent-plugin-parser into built-in plugin\n  https://github.com/fluent/fluentd/pull/1191\n* parser/formatter plugin helpers with default @type in plugin side\n  https://github.com/fluent/fluentd/pull/1267\n* parser: Reconstruct Parser related classes\n  https://github.com/fluent/fluentd/pull/1286\n* filter_record_transformer: Remove old behaviours\n  https://github.com/fluent/fluentd/pull/1311\n* Migrate some built-in plugins into v0.14 API\n  https://github.com/fluent/fluentd/pull/1257 (out_file)\n  https://github.com/fluent/fluentd/pull/1297 (out_exec, out_exec_filter)\n  https://github.com/fluent/fluentd/pull/1306 (in_forward, out_forward)\n  https://github.com/fluent/fluentd/pull/1308 (in_http)\n* test: Improve test drivers\n  https://github.com/fluent/fluentd/pull/1302\n  https://github.com/fluent/fluentd/pull/1305\n\n### Bug fixes\n\n* log: Avoid name conflict between Fluent::Logger\n  https://github.com/fluent/fluentd/pull/1274\n* fluent-cat: Fix fluent-cat command to send sub-second precision time\n  https://github.com/fluent/fluentd/pull/1277\n* config: Fix a bug not to overwrite default value with nil\n  https://github.com/fluent/fluentd/pull/1296\n* output: Fix timezone for compat timesliced output plugins\n  https://github.com/fluent/fluentd/pull/1307\n* out_forward: fix not to raise error when out_forward is initialized as secondary\n  https://github.com/fluent/fluentd/pull/1313\n* output: Event router for secondary output\n  https://github.com/fluent/fluentd/pull/1283\n* test: fix to return the block value as expected by many rubyists\n  https://github.com/fluent/fluentd/pull/1284\n\n## Release v0.14.8 - 2016/10/13\n\n### Bug fixes\n\n* Add msgpack_each to buffer chunks in compat-layer output plugins\n  https://github.com/fluent/fluentd/pull/1273\n\n## Release v0.14.7 - 2016/10/07\n\n### New features / Enhancement\n\n* Support data compression in buffer plugins\n  https://github.com/fluent/fluentd/pull/1172\n* in_forward: support to transfer compressed data\n  https://github.com/fluent/fluentd/pull/1179\n* out_stdout: fix to show nanosecond resolution time\n  https://github.com/fluent/fluentd/pull/1249\n* Add option to rotate Fluentd daemon's log\n  https://github.com/fluent/fluentd/pull/1235\n* Add extract plugin helper, with symmetric time parameter support in parser/formatter and inject/extract\n  https://github.com/fluent/fluentd/pull/1207\n* Add a feature to parse/format numeric time (unix time [+ subsecond value])\n  https://github.com/fluent/fluentd/pull/1254\n* Raise configuration errors for inconsistent `<label>` configurations\n  https://github.com/fluent/fluentd/pull/1233\n* Fix to instantiate an unconfigured section even for multi: true\n  https://github.com/fluent/fluentd/pull/1210\n* Add validators of placeholders for buffering key extraction\n  https://github.com/fluent/fluentd/pull/1255\n* Fix to show log messages about filter optimization only when needed\n  https://github.com/fluent/fluentd/pull/1227\n* Add some features to write plugins more easily\n  https://github.com/fluent/fluentd/pull/1256\n* Add a tool to load dumped events from file\n  https://github.com/fluent/fluentd/pull/1165\n\n### Bug fixes\n\n* Fix Oj's default option to encode/decode JSON in the same way with Yajl\n  https://github.com/fluent/fluentd/pull/1147\n  https://github.com/fluent/fluentd/pull/1239\n* Fix to raise correct configuration errors\n  https://github.com/fluent/fluentd/pull/1223\n* Fix a bug to call `shutdown` method (and some others) twice\n  https://github.com/fluent/fluentd/pull/1242\n* Fix to enable `chunk.each` only when it's encoded by msgpack\n  https://github.com/fluent/fluentd/pull/1263\n* Fix a bug not to stop enqueue/flush threads correctly\n  https://github.com/fluent/fluentd/pull/1264\n* out_forward: fix a bug that UDP heartbeat doesn't work\n  https://github.com/fluent/fluentd/pull/1238\n* out_file: fix a crash bug when v0.14 enables symlink and resumes existing buffer file chunk generated by v0.12\n  https://github.com/fluent/fluentd/pull/1234\n* in_monitor_agent: fix compatibility problem between outputs of v0.12 and v0.14\n  https://github.com/fluent/fluentd/pull/1232\n* in_tail: fix a bug to crash to read large amount logs\n  https://github.com/fluent/fluentd/pull/1259\n  https://github.com/fluent/fluentd/pull/1261\n\n## Release v0.14.6 - 2016/09/07\n\n### Bug fixes\n\n* in_tail: Add a missing parser_multiline require\n  https://github.com/fluent/fluentd/pull/1212\n* forward: Mark secret parameters of forward plugins as secret\n  https://github.com/fluent/fluentd/pull/1209\n\n## Release v0.14.5 - 2016/09/06\n\n### New features / Enhancement\n\n* Add authentication / authorization feature to forward protocol and in/out_forward plugins\n  https://github.com/fluent/fluentd/pull/1136\n* Add a new plugin to dump buffers in retries as secondary plugin\n  https://github.com/fluent/fluentd/pull/1154\n* Merge out_buffered_stdout and out_buffered_null into out_stdout and out_null\n  https://github.com/fluent/fluentd/pull/1200\n\n### Bug fixes\n\n* Raise configuration errors to clarify what's wrong when \"@type\" is missing\n  https://github.com/fluent/fluentd/pull/1202\n* Fix the bug not to launch Fluentd when v0.12 MultiOutput plugin is configured\n  https://github.com/fluent/fluentd/pull/1206\n\n## Release v0.14.4 - 2016/08/31\n\n### New features / Enhancement\n\n* Add a method to Filter API to update time of events\n  https://github.com/fluent/fluentd/pull/1140\n* Improve performance of filter pipeline\n  https://github.com/fluent/fluentd/pull/1145\n* Fix to suppress not to warn about different plugins for primary and secondary without any problems\n  https://github.com/fluent/fluentd/pull/1153\n* Add deprecated/obsoleted options to config_param to show removed/warned parameters\n  https://github.com/fluent/fluentd/pull/1186\n* in_forward: Add a feature source_hostname_key to inject source hostname into records\n  https://github.com/fluent/fluentd/pull/807\n* in_tail: Add a feature from_encoding to specify both encoding from and to\n  https://github.com/fluent/fluentd/pull/1067\n* filter_record_transformer: Fix to prevent overwriting reserved placeholder keys\n  https://github.com/fluent/fluentd/pull/1176\n* Migrate some built-in plugins into v0.14 API\n  https://github.com/fluent/fluentd/pull/1149\n  https://github.com/fluent/fluentd/pull/1151\n* Update dependencies\n  https://github.com/fluent/fluentd/pull/1193\n\n### Bug fixes\n\n* Fix to start/stop/restart Fluentd processes correctly on Windows environment\n  https://github.com/fluent/fluentd/pull/1171\n  https://github.com/fluent/fluentd/pull/1192\n* Fix to handle Windows events correctly in winsvc.rb\n  https://github.com/fluent/fluentd/pull/1155\n  https://github.com/fluent/fluentd/pull/1170\n* Fix not to continue to restart workers for configuration errors\n  https://github.com/fluent/fluentd/pull/1183\n* Fix output threads to start enqueue/flush buffers until plugins' start method ends\n  https://github.com/fluent/fluentd/pull/1190\n* Fix a bug not to set umask 0\n  https://github.com/fluent/fluentd/pull/1152\n* Fix resource leak on one-shot timers\n  https://github.com/fluent/fluentd/pull/1178\n* Fix to call plugin helper methods in configure\n  https://github.com/fluent/fluentd/pull/1184\n* Fix a bug to count event size\n  https://github.com/fluent/fluentd/pull/1164/files\n* Fix to require missed compat modules\n  https://github.com/fluent/fluentd/pull/1168\n* Fix to start properly for plugins under MultiOutput\n  https://github.com/fluent/fluentd/pull/1167\n* Fix test drivers to set class name into plugin instances\n  https://github.com/fluent/fluentd/pull/1069\n* Fix tests not to use mocks for Time (improve test stabilization)\n  https://github.com/fluent/fluentd/pull/1194\n\n## Release 0.14.3 - 2016/08/30\n\n* Fix the dependency for ServerEngine 1.x\n\n## Release 0.14.2 - 2016/08/09\n\n### New features / Enhancement\n\n* Fix to split large event stream into some/many chunks in buffers\n  https://github.com/fluent/fluentd/pull/1062\n* Add parser and filter support in compat_parameters plugin helper\n  https://github.com/fluent/fluentd/pull/1079\n* Add a RPC call to flush buffers and stop workers\n  https://github.com/fluent/fluentd/pull/1134\n* Update forward protocol to pass the number of events in a payload\n  https://github.com/fluent/fluentd/pull/1137\n* Improve performance of some built-in formatter plugins\n  https://github.com/fluent/fluentd/pull/1082\n  https://github.com/fluent/fluentd/pull/1086\n* Migrate some built-in plugins and plugin util modules into v0.14 API\n  https://github.com/fluent/fluentd/pull/1058\n  https://github.com/fluent/fluentd/pull/1061\n  https://github.com/fluent/fluentd/pull/1076\n  https://github.com/fluent/fluentd/pull/1078\n  https://github.com/fluent/fluentd/pull/1081\n  https://github.com/fluent/fluentd/pull/1083\n  https://github.com/fluent/fluentd/pull/1091\n* Register RegExpParser as a parser plugin explicitly\n  https://github.com/fluent/fluentd/pull/1094\n* Add delimiter option to CSV parser\n  https://github.com/fluent/fluentd/pull/1108\n* Add an option to receive longer udp syslog messages\n  https://github.com/fluent/fluentd/pull/1127\n* Add a option to suspend internal status in dummy plugin\n  https://github.com/fluent/fluentd/pull/900\n* Add a feature to capture filtered records in test driver for Filter plugins\n  https://github.com/fluent/fluentd/pull/1077\n* Add some utility methods to plugin test drivers\n  https://github.com/fluent/fluentd/pull/1114\n\n### Bug fixes\n\n* Fix bug to read non buffer-chunk files as buffer chunks when Fluentd resumed\n  https://github.com/fluent/fluentd/pull/1124\n* Fix bug not to load Filter plugins which are specified in configurations\n  https://github.com/fluent/fluentd/pull/1118\n* Fix bug to ignore `-p` option to specify directories of plugins\n  https://github.com/fluent/fluentd/pull/1133\n* Fix bug to overwrite base class configuration section definitions by subclasses\n  https://github.com/fluent/fluentd/pull/1119\n* Fix to stop Fluentd worker process by Ctrl-C when --no-supervisor specified\n  https://github.com/fluent/fluentd/pull/1089\n* Fix regression about RPC call to reload configuration\n  https://github.com/fluent/fluentd/pull/1093\n* Specify to ensure Oj JSON parser to use strict mode\n  https://github.com/fluent/fluentd/pull/1147\n* Fix unexisting path handling in Windows environment\n  https://github.com/fluent/fluentd/pull/1104\n\n## Release 0.14.1 - 2016/06/30\n\n### New features / Enhancement\n\n* Add plugin helpers for parsers and formatters\n  https://github.com/fluent/fluentd/pull/1023\n* Extract some mixins into compat modules\n  https://github.com/fluent/fluentd/pull/1044\n  https://github.com/fluent/fluentd/pull/1052\n* Add utility methods for tests and test drivers\n  https://github.com/fluent/fluentd/pull/1047\n* Migrate some built-in plugins to v0.14 APIs\n  https://github.com/fluent/fluentd/pull/1049\n  https://github.com/fluent/fluentd/pull/1057\n  https://github.com/fluent/fluentd/pull/1060\n  https://github.com/fluent/fluentd/pull/1064\n* Add support of X-Forwarded-For header in in_http plugin\n  https://github.com/fluent/fluentd/pull/1051\n* Warn not to create too many staged chunks at configure\n  https://github.com/fluent/fluentd/pull/1054\n* Add a plugin helper to inject tag/time/hostname\n  https://github.com/fluent/fluentd/pull/1063\n\n### Bug fixes\n\n* Fix in_monitor_agent for v0.14 plugins\n  https://github.com/fluent/fluentd/pull/1003\n* Fix to call #format_stream of plugins themselves when RecordFilter mixin included\n  https://github.com/fluent/fluentd/pull/1005\n* Fix shutdown sequence to wait force flush\n  https://github.com/fluent/fluentd/pull/1009\n* Fix a deadlock bug in shutdown\n  https://github.com/fluent/fluentd/pull/1010\n* Fix to require DetachProcessMixin in default for compat plugins\n  https://github.com/fluent/fluentd/pull/1014\n* Fix to overwrite configure_proxy name only for root sections for debugging\n  https://github.com/fluent/fluentd/pull/1015\n* Rename file for in_unix plugin\n  https://github.com/fluent/fluentd/pull/1017\n* Fix a bug not to create pid file when daemonized\n  https://github.com/fluent/fluentd/pull/1021\n* Fix wrong DEFAULT_PLUGIN_PATH\n  https://github.com/fluent/fluentd/pull/1028\n* Fix a bug not to use primary plugin type for secondary in default\n  https://github.com/fluent/fluentd/pull/1032\n* Add --run-worker option to distinguish to run as worker without supervisor\n  https://github.com/fluent/fluentd/pull/1033\n* Fix regression of fluent-debug command\n  https://github.com/fluent/fluentd/pull/1046\n* Update windows-pr dependency to 1.2.5\n  https://github.com/fluent/fluentd/pull/1065\n* Fix supervisor to pass RUBYOPT to worker processes\n  https://github.com/fluent/fluentd/pull/1066\n\n## Release 0.14.0 - 2016/05/25\n\n### New features / Enhancement\n\nThis list includes changes of 0.14.0.pre.1 and release candidates.\n\n* Update supported Ruby version to 2.1 or later\n  https://github.com/fluent/fluentd/pull/692\n* Sub-second event time support\n  https://github.com/fluent/fluentd/pull/653\n* Windows support and supervisor improvement\n  https://github.com/fluent/fluentd/pull/674\n  https://github.com/fluent/fluentd/pull/831\n  https://github.com/fluent/fluentd/pull/880\n* Add New plugin API\n  https://github.com/fluent/fluentd/pull/800\n  https://github.com/fluent/fluentd/pull/843\n  https://github.com/fluent/fluentd/pull/866\n  https://github.com/fluent/fluentd/pull/905\n  https://github.com/fluent/fluentd/pull/906\n  https://github.com/fluent/fluentd/pull/917\n  https://github.com/fluent/fluentd/pull/928\n  https://github.com/fluent/fluentd/pull/943\n  https://github.com/fluent/fluentd/pull/964\n  https://github.com/fluent/fluentd/pull/965\n  https://github.com/fluent/fluentd/pull/972\n  https://github.com/fluent/fluentd/pull/983\n* Add standard chunking format\n  https://github.com/fluent/fluentd/pull/914\n* Add Compatibility layer for v0.12 plugins\n  https://github.com/fluent/fluentd/pull/912\n  https://github.com/fluent/fluentd/pull/969\n  https://github.com/fluent/fluentd/pull/974\n  https://github.com/fluent/fluentd/pull/992\n  https://github.com/fluent/fluentd/pull/999\n* Add Plugin Storage API\n  https://github.com/fluent/fluentd/pull/864\n  https://github.com/fluent/fluentd/pull/910\n* Enforce to use router.emit instead of Engine.emit\n  https://github.com/fluent/fluentd/pull/883\n* log: Show plugin name and id in logs\n  https://github.com/fluent/fluentd/pull/860\n* log: Dump configurations with v1 syntax in logs\n  https://github.com/fluent/fluentd/pull/867\n* log: Dump errors with class in logs\n  https://github.com/fluent/fluentd/pull/899\n* config: Add simplified syntax for configuration values of hash and array\n  https://github.com/fluent/fluentd/pull/875\n* config: Add 'init' option to config_section to initialize section objects\n  https://github.com/fluent/fluentd/pull/877\n* config: Support multiline string in quoted strings\n  https://github.com/fluent/fluentd/pull/929\n* config: Add optional arguments on Element#elements to select child elements\n  https://github.com/fluent/fluentd/pull/948\n* config: Show deprecated warnings for reserved parameters\n  https://github.com/fluent/fluentd/pull/971\n* config: Make the detach process forward interval configurable\n  https://github.com/fluent/fluentd/pull/982\n* in_tail: Add 'path_key' option to inject tailing path\n  https://github.com/fluent/fluentd/pull/951\n* Remove in_status plugin\n  https://github.com/fluent/fluentd/pull/690\n\n### Bug fixes\n\n* config: Enum list must be of symbols\n  https://github.com/fluent/fluentd/pull/821\n* config: Fix to dup values in default\n  https://github.com/fluent/fluentd/pull/827\n* config: Fix problems about overwriting subsections\n  https://github.com/fluent/fluentd/pull/844\n  https://github.com/fluent/fluentd/pull/981\n* log: Serialize Fluent::EventTime as Integer in JSON\n  https://github.com/fluent/fluentd/pull/904\n* out_forward: Add missing error class and tests for it\n  https://github.com/fluent/fluentd/pull/922\n\n### Internal fix / Refactoring\n\n* Fix dependencies between files\n  https://github.com/fluent/fluentd/pull/799\n  https://github.com/fluent/fluentd/pull/808\n  https://github.com/fluent/fluentd/pull/823\n  https://github.com/fluent/fluentd/pull/824\n  https://github.com/fluent/fluentd/pull/825\n  https://github.com/fluent/fluentd/pull/826\n  https://github.com/fluent/fluentd/pull/828\n  https://github.com/fluent/fluentd/pull/859\n  https://github.com/fluent/fluentd/pull/892\n* Separate PluginId from config\n  https://github.com/fluent/fluentd/pull/832\n* Separate MessagePack factory from Engine\n  https://github.com/fluent/fluentd/pull/871\n* Register plugins to registry\n  https://github.com/fluent/fluentd/pull/838\n* Move TypeConverter mixin to mixin.rb\n  https://github.com/fluent/fluentd/pull/842\n* Override default configurations by `<system>`\n  https://github.com/fluent/fluentd/pull/854\n* Suppress Ruby level warnings\n  https://github.com/fluent/fluentd/pull/846\n  https://github.com/fluent/fluentd/pull/852\n  https://github.com/fluent/fluentd/pull/890\n  https://github.com/fluent/fluentd/pull/946\n  https://github.com/fluent/fluentd/pull/955\n  https://github.com/fluent/fluentd/pull/966\n\nSee https://github.com/fluent/fluentd/blob/v0.12/CHANGELOG.md for v0.12 changelog\n"
  },
  {
    "path": "CONTRIBUTING.md",
    "content": "# Contributing to Fluentd\n\nWe'd love your contribution. Here are the guidelines!\n\n## Got a question or problem?\n\nRESOURCES of [Official site](https://www.fluentd.org/) and [Fluentd documentation](https://docs.fluentd.org/) may help you.\n\nIf you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://github.com/fluent/fluentd/discussions).\nDon't use Github issue for asking questions. Here are examples:\n\n- I installed xxx plugin but it doesn't work. Why?\n- Fluentd starts but logs are not sent to xxx. Am I wrong?\n- I want to do xxx. How to realize it with plugins?\n\nWe may close such questions to keep clear repository for developers and users.\nGithub issue is mainly for submitting a bug report or feature request. See below.\n\nIf you can't judge your case is a bug or not, use community forum or slack first.\n\n## Found a bug?\n\nIf you find a bug of Fluentd or a mistake in the documentation, you can help us by\nsubmitting an issue to Fluentd. Even better you can submit a Pull Request with a fix.\n\n* **Fluentd**: Use [fluentd](https://github.com/fluent/fluentd) repository. Fill issue template.\n* **Documentation**: Use [fluentd documentation](https://github.com/fluent/fluentd-docs-gitbook) repository.\n\nIf you find a bug of 3rd party plugins, please submit an issue to each plugin repository.\nAnd use [fluent-package-builder](https://github.com/fluent/fluent-package-builder) repository for td-agent related issues.\n\nNote: Before report the issue, check latest version first. Sometimes users report fixed bug with older version.\n\n## Patch Guidelines\n\nHere are some things that would increase a chance that your patch is accepted:\n\n* Write tests.\n* Run tests before send Pull Request by `bundle exec rake test`\n* Write a [good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).\n  * Fluentd repositories needs [DCO](https://github.com/apps/dco) on PR. Please add `Signed-off-by` to the commit(See DCO link for more detail).\n\nThere are some patches which are hard to write tests, e.g. process handling, concurrency issue or etc.\nIn such case, please don't hesitate to submit a Pull Request.\nWe can discuss how to manage a patch on Pull Request :)\n"
  },
  {
    "path": "GOVERNANCE.md",
    "content": "# Fluentd Governance\n\n## Principles\n\nThe Fluentd community adheres to the following principles:\n\n- Open: Fluentd is open source. See repository guidelines and CLA, below.\n- Welcoming and respectful: See Code of Conduct, below.\n- Transparent and accessible: Work and collaboration are done in public.\n- Merit: Ideas and contributions are accepted according to their technical merit and alignment with project objectives, scope, and design principles.\n\n## Voting\n\nThe Fluentd project employs \"organization voting\" to ensure no single organization can dominate the project.\n\nIndividuals not associated with or employed by a company or organization are allowed one organization vote. Each company or organization (regardless of the number of maintainers associated with or employed by that company/organization) receives one organization vote.\n\nIn other words, if two maintainers are employed by Company X, two by Company Y, two by Company Z, and one maintainer is an un-affiliated individual, a total of four \"organization votes\" are possible; one for X, one for Y, one for Z, and one for the un-affiliated individual.\n\nAny maintainer from an organization may cast the vote for that organization.\n\nFor formal votes, a specific statement of what is being voted on should be added to the relevant github issue or PR, and a link to that issue or PR added to the maintainers meeting agenda document. Maintainers should indicate their yes/no vote on that issue or PR, and after a suitable period of time, the votes will be tallied and the outcome noted.\n\n## Changes in Maintainership\n\nNew maintainers are proposed by an existing maintainer and are elected by a 2/3 majority organization vote.\n\nMaintainers can be removed by a 2/3 majority organization vote.\n\n## Github Project Administration\n\nMaintainers will be added to the __fluent__ GitHub organization and added to the GitHub cni-maintainers team, and made a GitHub maintainer of that team.\n\nAfter 6 months a maintainer will be made an \"owner\" of the GitHub organization.\n\n## Projects\n\nThe fluent organization is open to receive new sub-projects under it umbrella. To apply a project as part of the __fluent__ organization, it has to met the following criteria:\n\n- Licensed under the terms of the Apache License v2.0\n- Project has been active for at least one year since it inception\n- More than 2 contributors\n- Related to one or more scopes of Fluentd ecosystem:\n  - Data collection\n  - Log management\n  - Metering\n- Be supported by 2/3 majority of organization\n\nThe submission process starts as a Pull Request on Fluentd repository with the required information mentioned above. Once a project is accepted, it's considered a __CNCF sub-project under the umbrella of Fluentd__\n\n## Code of Conduct\n\nFluentd follows the CNCF Code of Conduct:\n\nhttps://github.com/cncf/foundation/blob/master/code-of-conduct.md\n"
  },
  {
    "path": "Gemfile",
    "content": "source 'https://rubygems.org/'\n\ngemspec\n\ngem 'benchmark'\n\nlocal_gemfile = File.join(File.dirname(__FILE__), \"Gemfile.local\")\nif File.exist?(local_gemfile)\n  puts \"Loading Gemfile.local ...\" if $DEBUG # `ruby -d` or `bundle -v`\n  instance_eval File.read(local_gemfile)\nend\n"
  },
  {
    "path": "GithubWorkflow.md",
    "content": "# Github workflow for contributing to fluentd\n\nTable of Contents\n\n* [Fork a repository](#fork-a-repository)\n* [Clone fork repository to local](#clone-fork-repository-to-local)\n* [Create a branch to add a new feature or fix issues](#create-a-branch-to-add-a-new-feature-or-fix-issues)\n* [Commit and Push](#commit-and-push)\n* [Create a Pull Request](#create-a-pull-request)\n\n\nThe [fluentd](https://github.com/fluent/fluentd.git) code is hosted on Github (https://github.com/fluent/fluentd). The repository is called `upstream`. Contributors will develop and commit their changes in a clone of upstream repository. Then contributors push their change to their forked repository (`origin`) and create a Pull Request (PR), the PR will be merged to `upstream` repository if it meets the all the necessary requirements.\t\t\n\n## Fork a repository\n\n Go to https://github.com/fluent/fluentd then hit the `Fork` button to fork your own copy of repository **fluentd** to your github account.\n\n## Clone the forked repository to local\n\nClone the forked repo in [above step](#fork-a-repository) to your local working directory:\n```sh\n$ git clone https://github.com/$your_github_account/fluentd.git   \n```\n\nKeep your fork in sync with the main repo, add an `upstream` remote:\n```sh\n$ cd fluentd\n$ git remote add upstream https://github.com/fluentd/fluentd.git\n$ git remote -v\n\norigin  https://github.com/$your_github_account/fluentd.git (fetch)\norigin  https://github.com/$your_github_account/fluentd.git (push)\nupstream        https://github.com/fluentd/fluentd.git (fetch)\nupstream        https://github.com/fluentd/fluentd.git (push)\n```\n\nSync your local `master` branch:\n```sh\n$ git checkout master\n$ git pull origin master\n$ git fetch upstream\n$ git rebase upstream/master\n```\n\n## Create a branch to add a new feature or fix issues\n\nBefore making any change, create a new branch:\n```sh\n$ git checkout master\n$ git pull upstream master\n$ git checkout -b new-feature\n```\n\n## Commit and Push\n\nMake any change on the branch `new-feature`  then build and test your codes.  \nInclude in what will be committed:\n```sh\n$ git add <file>\n```\n\nCommit your changes with `sign-offs`\n```sh\n$ git commit -s\n```\n\nEnter your commit message to describe the changes. See the tips for a good commit message at [here](https://chris.beams.io/posts/git-commit/).  \nLikely you go back and edit/build/test some more then `git commit --amend`  \n\nPush your branch `new-feature` to your forked repository:\n```sh\n$ git push -u origin new-feature\n```\n\n## Create a Pull Request\n\n* Go to your fork at https://github.com/$your_github_account/fluentd\n* Create a Pull Request from the branch you recently pushed by hitting the button `Compare & pull request` next to branch name.\n"
  },
  {
    "path": "LICENSE",
    "content": "\n                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright 2011-2018 Fluentd Authors\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n"
  },
  {
    "path": "MAINTAINERS.md",
    "content": "# Fluentd Maintainers\n\n- [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/)\n- [Okkez](https://github.com/okkez)\n- [Hiroshi Hatake](https://github.com/cosmo0920), [Chronosphere](https://chronosphere.io/)\n- [Masahiro Nakagawa](https://github.com/repeatedly)\n- [Satoshi Tagomori](https://github.com/tagomoris)\n- [Toru Takahashi](https://github.com/toru-takahashi), [Treasure Data](https://www.treasuredata.com/)\n- [Eduardo Silva](https://github.com/edsiper), [Chronosphere](https://chronosphere.io/)\n- [Fujimoto Seiji](https://github.com/fujimots)\n- [Takuro Ashie](https://github.com/ashie), [ClearCode](https://www.clear-code.com/)\n- [Kentaro Hayashi](https://github.com/kenhys), [ClearCode](https://www.clear-code.com/)\n- [Daijiro Fukuda](https://github.com/daipom), [ClearCode](https://www.clear-code.com/)\n\n## Sub Projects: Fluent Bit\n\nFluent Bit maintainers list can be found here:\n\n- https://github.com/fluent/fluent-bit/blob/master/MAINTAINERS.md\n"
  },
  {
    "path": "README.md",
    "content": "Fluentd: Open-Source Log Collector\n===================================\n\n[![Test](https://github.com/fluent/fluentd/actions/workflows/test.yml/badge.svg)](https://github.com/fluent/fluentd/actions/workflows/test.yml)\n[![Test with Ruby head](https://github.com/fluent/fluentd/actions/workflows/test-ruby-head.yml/badge.svg)](https://github.com/fluent/fluentd/actions/workflows/test-ruby-head.yml)\n[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)\n[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/fluent/fluentd/badge)](https://scorecard.dev/viewer/?uri=github.com/fluent/fluentd)\n\n[Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).\n\n<p align=\"center\">\n<img src=\"https://www.fluentd.org/images/fluentd-architecture.png\" width=\"500px\"/>\n</p>\n\n## Quick Start\n\n    $ gem install fluentd\n    $ fluentd -s conf\n    $ fluentd -c conf/fluent.conf &\n    $ echo '{\"json\":\"message\"}' | fluent-cat debug.test\n\n## Development\n\n### Branch\n\n- master: For v1 development.\n- v0.12: For v0.12. This is deprecated version. we already stopped supporting (See https://www.fluentd.org/blog/drop-schedule-announcement-in-2019).\n\n### Prerequisites\n\n- Ruby 3.2 or later\n- git\n\n`git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.\n\n### Install dependent gems\n\nUse bundler:\n\n    $ gem install bundler\n    $ bundle install --path vendor/bundle\n\n### Run test\n\n    $ bundle exec rake test\n\nYou can run specified test via `TEST` environment variable:\n\n    $ bundle exec rake test TEST=test/test_specified_path.rb\n    $ bundle exec rake test TEST=test/test_*.rb\n\n## More Information\n\n- Website: https://www.fluentd.org/\n- Documentation: https://docs.fluentd.org/\n- Project repository: https://github.com/fluent\n- Discussion: https://github.com/fluent/fluentd/discussions\n- Slack / Community: https://slack.fluentd.org\n- Newsletters: https://www.fluentd.org/newsletter\n- Author: [Sadayuki Furuhashi](https://github.com/frsyuki)\n- Copyright: 2011-2021 Fluentd Authors\n- License: Apache License, Version 2.0\n\n## Security\n\nA third party security audit was performed by Cure53, you can see the full report [here](docs/SECURITY_AUDIT.pdf).\n\nSee [SECURITY](SECURITY.md) to contact us about vulnerability.\n\n## Contributors:\n\nPatches contributed by [great developers](https://github.com/fluent/fluentd/contributors).\n"
  },
  {
    "path": "Rakefile",
    "content": "#!/usr/bin/env rake\nrequire \"bundler/gem_tasks\"\n\nrequire 'fileutils'\nrequire 'rake/testtask'\nrequire 'rake/clean'\n\nrequire_relative 'tasks/benchmark'\nrequire_relative 'tasks/backport'\n\ntask test: [:base_test]\n\n# 1. update ChangeLog and lib/fluent/version.rb\n# 2. bundle && bundle exec rake build:all\n# 3. release 3 packages built on pkg/ directory\nnamespace :build do\n  desc 'Build gems for all platforms'\n  task :all do\n    Bundler.with_original_env do\n      %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|\n        ENV['GEM_BUILD_FAKE_PLATFORM'] = name\n        Rake::Task[\"build\"].execute\n      end\n    end\n  end\nend\n\ndesc 'Run test_unit based test'\nRake::TestTask.new(:base_test) do |t|\n  # To run test with dumping all test case names (to find never ending test case)\n  #  $ bundle exec rake test TESTOPTS=-v\n  #\n  # To run test for only one file (or file path pattern)\n  #  $ bundle exec rake base_test TEST=test/test_specified_path.rb\n  #  $ bundle exec rake base_test TEST=test/test_*.rb\n  t.libs << \"test\"\n  t.test_files = if ENV[\"WIN_RAPID\"]\n                   [\"test/test_event.rb\", \"test/test_supervisor.rb\", \"test/plugin_helper/test_event_loop.rb\"]\n                 else\n                   tests = Dir[\"test/**/test_*.rb\"].sort\n                   if Process.uid.zero?\n                     puts \"test_file_util.rb for non-root user env. Disable this test on root environment\"\n                     tests.delete(\"test/plugin/test_file_util.rb\")\n                   end\n                   tests\n                 end\n  t.verbose = true\n  t.warning = true\n  t.ruby_opts = [\"-Eascii-8bit:ascii-8bit\"]\nend\n\ntask :parallel_test do\n  FileUtils.rm_rf('./test/tmp')\n  sh(\"parallel_test ./test/*.rb ./test/plugin/*.rb\")\n  FileUtils.rm_rf('./test/tmp')\nend\n\ndesc 'Run test with simplecov'\ntask :coverage do |t|\n  ENV['SIMPLE_COV'] = '1'\n  Rake::Task[\"test\"].invoke\nend\n\ndesc 'Build Coverity tarball & upload it'\ntask :coverity do\n  # https://scan.coverity.com/projects/fluentd?tab=overview\n  # See \"View Defects\" after sign-in.\n  #\n  # Setup steps:\n  # 1. get coverity build tool and set PATH to bin/: https://scan.coverity.com/download\n  # 2. set environment variables:\n  #    * $COVERITY_USER (your email address)\n  #    * $COVERITY_TOKEN (token for Fluentd project: https://scan.coverity.com/projects/fluentd?tab=project_settings)\n  sh \"cov-build --dir cov-int --no-command --fs-capture-search ./\"\n  sh \"tar czf cov-fluentd.tar.gz cov-int\"\n  user = ENV['COVERITY_USER']\n  token = ENV['COVERITY_TOKEN']\n  sh \"curl --form token=#{token} --form email=#{user} --form file=@cov-fluentd.tar.gz --form version=\\\"Master\\\" --form description=\\\"GIT Master\\\" https://scan.coverity.com/builds?project=Fluentd\"\n  FileUtils.rm_rf(['./cov-int', 'cov-fluentd.tar.gz'])\nend\n\ntask :check_env do\n  unless ENV['GEM_HOST_API_KEY']\n    abort \"Missing required environment variable: GEM_HOST_API_KEY\\nSee https://guides.rubygems.org/api-key-scopes/\"\n  end\nend\nRake::Task[\"release:rubygem_push\"].enhance([\"check_env\"])\n\ntask default: [:test, :build]\n"
  },
  {
    "path": "SECURITY.md",
    "content": "# Security Policy\n\n## Supported Versions\n\n|  Version  |     Supported      |\n|-----------|--------------------|\n| 1.19.x    | :white_check_mark: |\n| 1.18.x    | :x:                |\n| 1.17.x    | :x:                |\n| 1.16.x    | :white_check_mark: |\n| <= 1.15.x | :x:                |\n\n## Reporting a Vulnerability\n\nPlease post your vulnerability report from the following page:\nhttps://github.com/fluent/fluentd/security/advisories\n\n> [!CAUTION]\n> In above contact form, we accept about **ONLY** Fluentd's vulnerability, in contrast, **REJECT** your vulnerability report about Fluentd derivative products such as fluent-package, Fluentd container images and Fluentd kubernetes deamonset images and so on. There are appropriate contact forms for every these derivative products. See the following notice.\n\n> [!NOTE]\n> If you use fluent-package, please check [fluent-package-builder](https://github.com/fluent/fluent-package-builder/blob/master/SECURITY.md) and report it there.\n\n> [!NOTE]\n> If you use a Docker image of Fluentd, please check [Fluentd Docker Image](https://github.com/fluent/fluentd-docker-image/blob/master/SECURITY.md) and report it there.\n"
  },
  {
    "path": "bin/fluent-binlog-reader",
    "content": "#!/usr/bin/env ruby\n# -*- coding: utf-8 -*-\nhere = File.dirname(__FILE__)\n$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))\nrequire 'fluent/command/binlog_reader'\n\nFluentBinlogReader.new.call\n"
  },
  {
    "path": "bin/fluent-ca-generate",
    "content": "#!/usr/bin/env ruby\n\n$LOAD_PATH.unshift(File.join(__dir__, 'lib'))\nrequire 'fluent/command/ca_generate'\n\nFluent::CaGenerate.new.call\n"
  },
  {
    "path": "bin/fluent-cap-ctl",
    "content": "#!/usr/bin/env ruby\n# -*- coding: utf-8 -*-\nhere = File.dirname(__FILE__)\n$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))\nrequire 'fluent/command/cap_ctl'\n\nFluent::CapCtl.new.call"
  },
  {
    "path": "bin/fluent-ctl",
    "content": "#!/usr/bin/env ruby\n\nhere = File.dirname(__FILE__)\n$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))\nrequire 'fluent/command/ctl'\n\nFluent::Ctl.new.call\n"
  },
  {
    "path": "bin/fluent-debug",
    "content": "#!/usr/bin/env ruby\n# -*- coding: utf-8 -*-\nhere = File.dirname(__FILE__)\n$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))\nrequire 'fluent/command/debug'\n"
  },
  {
    "path": "bin/fluent-plugin-config-format",
    "content": "#!/usr/bin/env ruby\n$LOAD_PATH.unshift(File.join(__dir__, 'lib'))\nrequire 'fluent/command/plugin_config_formatter'\n\nFluentPluginConfigFormatter.new.call\n"
  },
  {
    "path": "bin/fluent-plugin-generate",
    "content": "#!/usr/bin/env ruby\n$LOAD_PATH << File.expand_path(File.join(__dir__, '..', 'lib'))\nrequire 'fluent/command/plugin_generator'\n\nFluentPluginGenerator.new.call\n"
  },
  {
    "path": "code-of-conduct.md",
    "content": "## Fluentd Community Code of Conduct\n\nFluentd follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).\n"
  },
  {
    "path": "example/copy_roundrobin.conf",
    "content": "<source>\n  @type  sample\n  @label @test\n  tag    test.copy\n  auto_increment_key id\n</source>\n\n<source>\n  @type sample\n  @label @test\n  tag    test.rr\n  auto_increment_key id\n</source>\n\n<label @test>\n  <match test.copy>\n    @type copy\n    <store>\n      @type stdout\n      output_type json\n    </store>\n    <store>\n      @type stdout\n      output_type ltsv\n    </store>\n  </match>\n\n  <match test.rr>\n    @type roundrobin\n    <store>\n      @type stdout\n      output_type json\n    </store>\n    <store>\n      @type stdout\n      output_type ltsv\n    </store>\n  </match>\n</label>\n"
  },
  {
    "path": "example/counter.conf",
    "content": "<system>\n  <counter_server>\n    scope server1\n    bind 127.0.0.1\n    port 24321\n    backup_path tmp/back\n  </counter_server>\n</system>\n\n<source>\n  @type sample\n  tag \"test.data\"\n  auto_increment_key number\n</source>\n\n<match>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/filter_stdout.conf",
    "content": "<source>\n  @type sample\n  tag sample\n</source>\n\n<filter **>\n  @type stdout\n</filter>\n\n<filter **>\n  @type stdout\n  output_type hash\n</filter>\n\n<filter **>\n  @type stdout\n  format ltsv\n</filter>\n\n<match **>\n  @type null\n</match>\n"
  },
  {
    "path": "example/in_forward.conf",
    "content": "<system>\n  rpc_endpoint 0.0.0.0:24444\n</system>\n\n<source>\n  @type forward\n</source>\n\n<match test>\n  @type stdout\n  # <buffer>\n  #   flush_interval 10s\n  # </buffer>\n</match>\n"
  },
  {
    "path": "example/in_forward_client.conf",
    "content": "<system>\n  rpc_endpoint 0.0.0.0:24444\n</system>\n\n<source>\n  @type forward\n  port 24224\n  bind 0.0.0.0\n  <security>\n    self_hostname input.testing.local\n    shared_key    secure_communication_is_awesome\n    user_auth     yes\n    allow_anonymous_source no\n    <user>\n      username user1\n      password yes_this_is_user1\n    </user>\n    <user>\n      username user2\n      password yes_this_is_really_user2\n    </user>\n    <user>\n      username user3\n      password noooooo_this_may_not_be_user3\n    </user>\n    <client>\n      # host  127.0.0.1\n      network 127.0.0.0/24\n      shared_key using_different_key_makes_us_secure\n      users user1,user2\n    </client>\n  </security>\n</source>\n\n<match {test,test2,test3,test4}>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/in_forward_shared_key.conf",
    "content": "<system>\n  rpc_endpoint 0.0.0.0:24444\n</system>\n\n<source>\n  @type forward\n  <security>\n    self_hostname input.testing.local\n    shared_key    secure_communication_is_awesome\n  </security>\n</source>\n\n<match test>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/in_forward_tls.conf",
    "content": "<source>\n  @type forward\n  port 24224\n  <transport tls>\n    insecure true\n  </transport>\n</source>\n\n<match test>\n  @type stdout\n  # <buffer>\n  #   flush_interval 10s\n  # </buffer>\n</match>\n"
  },
  {
    "path": "example/in_forward_users.conf",
    "content": "<system>\n  rpc_endpoint 0.0.0.0:24444\n</system>\n\n<source>\n  @type forward\n  <security>\n    self_hostname input.testing.local\n    shared_key    secure_communication_is_awesome\n    user_auth     yes\n    <user>\n      username user1\n      password yes_this_is_user1\n    </user>\n    <user>\n      username user2\n      password yes_this_is_really_user2\n    </user>\n  </security>\n</source>\n\n<match {test,test2,test3}>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/in_forward_workers.conf",
    "content": "<system>\n  workers 3\n  root_dir \"#{File.join(Dir.pwd, 'test', 'tmp', 'root')}\"\n</system>\n\n<source>\n  @type forward\n  @id forward_in_1\n</source>\n\n<match test>\n  @type stdout\n  @id stdout_out_1\n  <inject>\n    worker_id_key worker_id\n  </inject>\n  <buffer>\n    @type file\n    flush_interval 1s\n  </buffer>\n</match>\n"
  },
  {
    "path": "example/in_http.conf",
    "content": "<source>\n  @type http\n  bind 0.0.0.0\n  port 9880\n  body_size_limit 32MB\n  keepalive_timeout 10\n  # backlog 0\n  add_http_headers false\n  <parse>\n    @type json\n  </parse>\n</source>\n\n<match test>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/in_out_forward.conf",
    "content": "<source>\n  @type forward\n  port 24224\n</source>\n\n<match test.**>\n  @type forward\n  buffer_type file\n  buffer_path /tmp/fluentd.forward.buffer\n  num_threads 10\n  flush_interval 1s\n  <server>\n    host 127.0.0.1\n    port 24225\n  </server>\n</match>\n\n"
  },
  {
    "path": "example/in_sample_blocks.conf",
    "content": "<source>\n  @type sample\n  tag sample\n  rate 100\n  sample {\"message\":\"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay\"}\n</source>\n\n<match sample>\n  @type null\n  never_flush true\n  <buffer>\n    @type memory\n    overflow_action block\n    chunk_limit_size 1k\n    total_limit_size 2k\n  </buffer>\n</match>\n"
  },
  {
    "path": "example/in_sample_with_compression.conf",
    "content": "<source>\n  @type sample\n  @label @main\n  tag \"test.data\"\n  size 2\n  rate 10\n  sample {\"message\":\"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay\"}\n  auto_increment_key number\n</source>\n\n<label @main>\n  <match test.data>\n    @type stdout\n    <buffer>\n      @type file\n      path \"#{Dir.pwd}/compressed_buffers\"\n      flush_at_shutdown false\n      chunk_limit_size 1m\n      flush_interval 10s\n      compress gzip\n    </buffer>\n  </match>\n</label>\n"
  },
  {
    "path": "example/in_syslog.conf",
    "content": "<source>\n  @type syslog\n  bind 0.0.0.0\n  port 5140\n  tag test\n  protocol_type udp\n  include_source_host false\n  source_host_key source_host\n  # format ...\n  # with_priority true\n</source>\n\n<match test>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/in_tail.conf",
    "content": "<source>\n  @type tail\n  format none\n  path /var/log/fluentd_test.log\n  pos_file /var/log/fluentd_test.pos\n  tag test\n  rotate_wait 5\n  read_from_head true\n  refresh_interval 60\n</source>\n\n<match test>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/in_tcp.conf",
    "content": "<source>\n  @type tcp\n  format none\n  bind 0.0.0.0\n  port 5170\n  delimiter \\n\n  source_host_key \"host\"\n  tag test\n</source>\n\n<match test>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/in_udp.conf",
    "content": "<source>\n  @type udp\n  format none\n  bind 0.0.0.0\n  port 5160\n  body_size_limit 4KB\n  source_host_key \"host\"\n  tag test\n</source>\n\n<match test>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/logevents.conf",
    "content": "<source>\n  @type sample\n  @label @samplelog\n  tag \"data\"\n  sample {\"message\":\"yay\"}\n</source>\n<label @samplelog>\n  <match **>\n    @type stdout\n  </match>\n</label>\n<label @FLUENT_LOG>\n  <match fluent.debug fluent.info fluent.warn fluent.error fluent.fatal>\n    @type stdout\n    <inject>\n      hostname_key \"host\"\n    </inject>\n  </match>\n  # <match fluent.{info,warn,error,fatal}>\n  #   @type stdout\n  #   <inject>\n  #     hostname_key \"host\"\n  #   </inject>\n  # </match>\n</label>\n"
  },
  {
    "path": "example/multi_filters.conf",
    "content": "# This example is to measure optimized filter pipeline performance.\n\n<source>\n  @type sample\n  tag   test\n  size 1000\n</source>\n\n<filter test>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<filter test>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<filter test>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<filter test>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<filter test>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<filter test>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<filter test>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<filter test>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<filter test>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<filter test>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<match test>\n  @type buffered_null\n</match>\n"
  },
  {
    "path": "example/out_copy.conf",
    "content": "<source>\n  @type forward\n</source>\n\n<match test>\n  @type copy\n  deep_copy false\n  <store>\n    @type stdout\n  </store>\n  <store>\n    @type file\n    path /var/log/fluentd/out_file_test\n    format json\n    buffer_type memory\n    # compress gzip\n    # symlink_path /path/to/symlink\n    append false\n  </store>\n</match>\n"
  },
  {
    "path": "example/out_exec_filter.conf",
    "content": "<source>\n  @type sample\n  @label @exec\n  tag exec_input\n  rate 10\n  auto_increment_key num\n  sample {\"data\":\"mydata\"}\n</source>\n\n<label @exec>\n  <match exec_input>\n    @type exec_filter\n    @label @stdout\n    tag result\n    command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + \"\\t\" + Process.pid.to_s}; 1000.times{ proc.call }'\n    num_children 3\n    child_respawn -1\n    <inject>\n      time_key time\n      time_type float\n    </inject>\n    <format>\n      @type tsv\n      keys data, num, time\n    </format>\n    <parse>\n      @type tsv\n      keys data, num, time, pid\n    </parse>\n    <extract>\n      time_key time\n      time_type float\n    </extract>\n  </match>\n</label>\n\n<label @stdout>\n  <match result>\n    @type stdout\n  </match>\n</label>\n\n"
  },
  {
    "path": "example/out_file.conf",
    "content": "<source>\n  @type forward\n</source>\n\n<match test>\n  @type file\n  path /var/log/fluentd/out_file_test\n  format json\n  buffer_type memory\n  # compress gzip\n  # symlink_path /path/to/symlink\n  append false\n</match>\n"
  },
  {
    "path": "example/out_forward.conf",
    "content": "<source>\n  @type sample\n  tag test\n</source>\n\n<match test>\n  @type forward\n\n  <server>\n    # first server\n    host 127.0.0.1\n    port 24224\n  </server>\n  # <server>\n  #   # second server\n  #   host localhost\n  #   port 24225\n  # </server>\n  # <server>\n  #   # second server\n  #   host localhost\n  #   port 24226\n  #   standby\n  # </server>\n\n  flush_interval 0\n  send_timeout 60\n  heartbeat_type udp\n  heartbeat_interval 1\n  recover_wait 10\n  hard_timeout 60\n  expire_dns_cache nil\n  phi_threshold 16\n  phi_failure_detector true\n</match>\n"
  },
  {
    "path": "example/out_forward_buf_file.conf",
    "content": "<source>\n    @type sample\n    tag test\n</source>\n\n<source>\n    @type monitor_agent\n    emit_interval 5\n</source>\n\n<match test>\n    @type forward\n    buffer_path /tmp/fluentd.forward\n    buffer_type file\n    flush_interval 5\n    send_timeout 60\n    heartbeat_type tcp\n    heartbeat_interval 1\n    <server>\n        host 127.0.0.1\n        port 24224\n    </server>\n</match>\n"
  },
  {
    "path": "example/out_forward_client.conf",
    "content": "<source>\n  @type sample\n  tag test\n</source>\n<source>\n  @type sample\n  tag test2\n</source>\n<source>\n  @type sample\n  tag test3\n</source>\n<source>\n  @type sample\n  tag test4\n</source>\n<source>\n  @type sample\n  tag test5\n</source>\n\n<match test>\n  @type forward\n  flush_interval 0\n  <security>\n    self_hostname output.testing.local\n    shared_key    secure_communication_is_awesome\n  </security>\n  <server>\n    host 127.0.0.1\n    port 24224\n    username user1\n    password yes_this_is_user1\n    shared_key using_different_key_makes_us_secure\n  </server>\n</match>\n\n<match test2>\n  @type forward\n  flush_interval 0\n  <security>\n    self_hostname output-alt1.testing.local\n    shared_key    using_different_key_makes_us_secure\n  </security>\n  <server>\n    host 127.0.0.1\n    port 24224\n    username user1\n    password yes_this_is_user1\n  </server>\n  <server>\n    host 127.0.0.1\n    port 24224\n    username user2\n    password yes_this_is_really_user2\n  </server>\n</match>\n\n<match test3>\n  @type forward\n  flush_interval 0\n  <security>\n    self_hostname output-fail1.testing.local\n    # default key: fail\n    shared_key    secure_communication_is_awesome\n  </security>\n  <server>\n    host 127.0.0.1\n    port 24224\n    username user1\n    password yes_this_is_user1\n    # [warn]: Shared key mismatch address=\"127.0.0.1\" hostname=\"output-fail1.testing.local\"\n  </server>\n</match>\n\n<match test4>\n  @type forward\n  flush_interval 0\n  <security>\n    self_hostname output-fail2.testing.local\n    shared_key    using_different_key_makes_us_secure\n  </security>\n  <server>\n    host 127.0.0.1\n    port 24224\n    username user3\n    # user3 (user denied): fail\n    password noooooo_this_may_not_be_user3\n    # [warn]: Authentication failed address=\"127.0.0.1\" hostname=\"output-fail2.testing.local\" username=\"user3\"\n  </server>\n</match>\n\n<match test5>\n  @type forward\n  flush_interval 0\n  <security>\n    self_hostname output-fail3.testing.local\n    shared_key    using_different_key_makes_us_secure\n  </security>\n  <server>\n    # another ip (host rejected): fail\n    # This pattern will work only with Ruby 2.3\n    host \"#{Socket.getifaddrs.select{|i| i.addr.ipv4? }.reject{|i| i.addr.ip_address == '127.0.0.1' }.first.addr.ip_address}\"\n    port 24224\n    username user1\n    password yes_this_is_user1\n    # [warn]: Anonymous client disallowed address=\"192.168.1.75\" hostname=\"output-fail3.testing.local\"\n  </server>\n</match>\n"
  },
  {
    "path": "example/out_forward_heartbeat_none.conf",
    "content": "<source>\n  @type sample\n  tag test\n</source>\n\n<match test>\n  @type forward\n  heartbeat_type none\n  <server>\n    host 127.0.0.1\n    port 24224\n  </server>\n  <buffer>\n    flush_mode immediate\n  </buffer>\n</match>\n"
  },
  {
    "path": "example/out_forward_sd.conf",
    "content": "<source>\n  @type sample\n  tag test\n</source>\n\n<match test>\n  @type forward\n\n  <service_discovery>\n    @type file\n    path \"#{Dir.pwd}/example/sd.yaml\"\n  </service_discovery>\n\n  <buffer>\n    flush_interval 1\n  </buffer>\n</match>\n"
  },
  {
    "path": "example/out_forward_shared_key.conf",
    "content": "<source>\n  @type sample\n  tag test\n</source>\n<source>\n  @type sample\n  tag test2\n</source>\n\n<match test>\n  @type forward\n  flush_interval 0\n  <security>\n    self_hostname output.testing.local\n    shared_key    secure_communication_is_awesome\n  </security>\n  <server>\n    host 127.0.0.1\n    port 24224\n  </server>\n</match>\n\n<match test2>\n  @type forward\n  flush_interval 0\n  <security>\n    self_hostname output-fail.testing.local\n    shared_key    secure_communication_is_not_awesome\n    # input plugin shows warning for wrong shared_key\n    # 2016-08-08 16:27:00 +0900 [warn]: Shared key mismatch address=\"127.0.0.1\" hostname=\"output-fail.testing.local\"\n  </security>\n  <server>\n    host 127.0.0.1\n    port 24224\n  </server>\n</match>\n"
  },
  {
    "path": "example/out_forward_tls.conf",
    "content": "<source>\n  @type sample\n  tag test\n</source>\n\n<match test>\n  @type forward\n  transport tls\n  tls_insecure_mode true\n  <server>\n    # first server\n    host 127.0.0.1\n    port 24224\n  </server>\n  <buffer>\n    flush_interval 0\n  </buffer>\n</match>\n"
  },
  {
    "path": "example/out_forward_users.conf",
    "content": "<source>\n  @type sample\n  tag test\n</source>\n<source>\n  @type sample\n  tag test2\n</source>\n<source>\n  @type sample\n  tag test3\n</source>\n\n<match test>\n  @type forward\n  flush_interval 0\n  <security>\n    self_hostname output.testing.local\n    shared_key    secure_communication_is_awesome\n  </security>\n  <server>\n    host 127.0.0.1\n    port 24224\n    username user1\n    password yes_this_is_user1\n  </server>\n</match>\n\n<match test2>\n  @type forward\n  flush_interval 0\n  <security>\n    self_hostname output-alt1.testing.local\n    shared_key    secure_communication_is_awesome\n  </security>\n  <server>\n    host 127.0.0.1\n    port 24224\n    username user1\n    password yes_this_is_user1\n  </server>\n  <server>\n    host 127.0.0.1\n    port 24224\n    username user2\n    password yes_this_is_really_user2\n  </server>\n</match>\n\n<match test3>\n  @type forward\n  flush_interval 0\n  <security>\n    self_hostname output-fail.testing.local\n    shared_key    secure_communication_is_awesome\n  </security>\n  <server>\n    host 127.0.0.1\n    port 24224\n    username user3\n    password no_there_are_not_such_user\n    # input plugin warns authentication erro:\n    # [warn]: Authentication failed address=\"127.0.0.1\" hostname=\"output-fail.testing.local\" username=\"user3\"\n  </server>\n</match>\n"
  },
  {
    "path": "example/out_null.conf",
    "content": "#\n# bundle exec bin/fluentd -c example/out_buffered_null.conf\n#   (+ --emit-error-log-interval 10)\n<source>\n  @type sample\n  tag sample\n  rate 500000000\n  sample [\n    {\"message\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"},\n    {\"message\": \"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"},\n    {\"message\": \"ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"}\n  ]\n</source>\n\n<match sample.**>\n  @type null\n  <buffer>\n    flush_interval 60s\n    chunk_limit_size 1k\n    total_limit_size 4k\n  </buffer>\n</match>\n\n<label error_log>\n  <match **>\n    @type stdout\n    # <buffer>\n    #   flush_interval 1s\n    # </buffer>\n  </match>\n</label>\n\n<match fluent.**>\n  @type relabel\n  @label error_log\n</match>\n"
  },
  {
    "path": "example/sd.yaml",
    "content": "- 'host': 127.0.0.1\n  'port': 24224\n  'weight': 1\n  'name': server1\n- 'host': 127.0.0.1\n  'port': 24225\n  'weight': 1\n  'name': server2\n"
  },
  {
    "path": "example/secondary_file.conf",
    "content": "<system>\n  rpc_endpoint 0.0.0.0:24444\n</system>\n\n<source>\n  @type sample\n  tag test\n</source>\n\n<source>\n  @type forward\n  @label @raw\n</source>\n\n<label @raw>\n  <match>\n    @type stdout\n  </match>\n</label>\n\n<match test>\n  @type forward\n\n  <buffer time,tag,message>\n    @type memory\n    timekey 2s\n    timekey_wait 1s\n    flush_mode interval\n    flush_interval 1s\n  </buffer>\n\n  <server>\n    host 0.0.0.0\n    port 24224\n  </server>\n\n  <secondary>\n    @type secondary_file\n    directory log/secondary/\n    basename ${tag}_%Y%m%d%L_${message}\n  </secondary>\n</match>\n"
  },
  {
    "path": "example/suppress_config_dump.conf",
    "content": "<system>\n  suppress_config_dump false\n</system>\n\n<match data.*>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/v0_12_filter.conf",
    "content": "# An example config to use filter plugins.\n# THIS FEATURE IS SUPPORTED FOR v0.12 AND ABOVE.\n\n# in_forward to generate events to be tested.\n# You can send an arbitrary event with an arbitrary tag.\n# For example, the following command creates the event\n# {\"message\":\"hello world\"} with tag = foo\n#\n# $ echo '{\"message\":\"hello world\"}' | fluent-cat foo\n\n<source>\n  @type forward\n  port 24224\n</source>\n\n# For all events with the tag \"foo\", filter it out\n# UNLESS the value of the \"message\" field matches /keep this/\n#\n# - {\"message\":\"keep this please\"} is kept.\n# - {\"message\":\"Do not keep\"} is filtered out.\n# - {\"messag22\":\"keep this please\"} is filtered out.\n\n<filter foo>\n  @type grep\n  regexp1  message keep this\n</filter>\n\n# Matches the events that was kept by the above filter\n<match foo>\n  @type stdout\n</match>\n\n# For all events with the tag \"bar\", add the machine's hostname with\n# the key \"hostname\" BEFORE forwarding to another instance of Fluentd\n# at 123.4.2.4:24224.\n\n<filter bar>\n  @type record_transformer\n  <record>\n    hostname ${hostname}\n  </record>\n</filter>\n\n# By the time it is getting matched here, the event has\n# the \"hostname\" field.\n<match bar>\n  @type forward\n  <server>\n    host 123.4.2.4\n    port 24225\n  </server>\n</match>\n\n# Composing two filters. For all events with the tag foo.bar,\n#\n# 1. The first filter filters out all events that has the field \"hello\"\n# 2. Then, for those events, we downcase the value of the \"name\" field.\n#\n# - {\"name\":\"SADA\", \"hello\":100} gets filtered out\n# - {\"name\":\"SADA\"} becomes {\"name\":\"sada\"}\n# - {\"last_name\":\"FURUHASHI\"} throws an error because it has no field called \"name\"\n\n<filter foo.bar>\n  @type grep\n  exclude1 hello .\n</filter>\n\n<filter foo.bar>\n  @type record_transformer\n  enable_ruby true\n  <record>\n    name ${name.downcase}\n  </record>\n</filter>\n\n<match foo.bar>\n  @type stdout\n</match>\n"
  },
  {
    "path": "example/v1_literal_example.conf",
    "content": "<section1>\n  key1  'text'     # text\n  key2  '\\''       # ' (1 char)\n  key3  '\\\\'       # \\ (1 char)\n  key4  '\\t'       # \\t (2 char)\n  key5  '\\['       # \\[ (2 char)\n  key6  '\\\\['      # \\[ (2 char)\n  key7  '#t'       # #t (2 char)\n  key8  '\\#{test}' # \\#{test} (8 char)\n  key9  '#{test}'  # #{test} (7 char)\n  key10 '\\[(?<time>[^\\]]*)\\] (?<message>.*)'    # \\[(?<time>[^\\]]*\\] (?<message>.*)\n</section1>\n<section2>\n  key1  \"text\"     # text\n  key2  \"\\\"\"       # \" (1 char)\n  key3  \"\\\\\"       # \\ (1 char)\n  key4  \"\\t\"       # TAB (1 char)\n  key5  \"\\[\"       # [ (1 char)\n  key6  \"\\\\[\"      # \\[ (2 char)\n  key7  \"#t\"       # #t (2 char)\n  key8  \"\\#{test}\" # #{test} (7 char)\n  key9  \"#{test}\"  # replaced by eval('test')\n  key10 \"\\\\[(?<time>[^\\\\]]*)\\\\] (?<message>.*)\" # \\[(?<time>[^\\]]*\\] (?<message>.*)\n</section2>\n<section3>\n  key1  text     # text\n  key2  \\        # \\ (1 char)\n  key3  \\\\       # \\\\ (2 char)\n  key4  \\t       # \\t (2 char)\n  key5  \\[       # \\[ (2 char)\n  key6  \\\\[      # \\\\[ (3 char)\n  key7  #t       # #t (2 char)\n  key8  \\#{test} # \\#{test} (8 char)\n  key9  #{test}  # #{test} (7 char)\n  key10 \\[(?<time>[^\\]]*)\\] (?<message>.*)    # \\[(?<time>[^\\]]*\\] (?<message>.*)\n</section3>\n"
  },
  {
    "path": "example/worker_section.conf",
    "content": "<system>\n  workers 4\n  root_dir /path/fluentd/root\n</system>\n\n<source> # top-level sections works on all workers in parallel\n  @type forward\n  port 24224\n</source>\n\n<match all> # this sections also works on all workers in parallel\n  @type stdout\n  <inject>\n    worker_id_key worker_id\n  </inject>\n</match>\n\n<worker 0> # this section works only on first worker process\n  <source>\n    @type tail\n    format none\n    path /var/log/fluentd_test.log\n    pos_file /var/log/fluentd_test.pos\n    tag tail\n    rotate_wait 5\n    read_from_head true\n    refresh_interval 60\n  </source>\n\n  <match tail>\n    @type stdout\n    <inject>\n      worker_id_key worker_id\n    </inject>\n  </match>\n</worker>\n"
  },
  {
    "path": "fluent.conf",
    "content": "# In v1 configuration, type and id are @ prefix parameters.\n# @type and @id are recommended. type and id are still available for backward compatibility\n\n## built-in TCP input\n## $ echo <json> | fluent-cat <tag>\n<source>\n  @type forward\n  @id forward_input\n</source>\n\n## built-in UNIX socket input\n#<source>\n#  @type unix\n#</source>\n\n# HTTP input\n# http://localhost:8888/<tag>?json=<json>\n<source>\n  @type http\n  @id http_input\n\n  port 8888\n</source>\n\n## File input\n## read apache logs with tag=apache.access\n#<source>\n#  @type tail\n#  format apache\n#  path /var/log/httpd-access.log\n#  tag apache.access\n#</source>\n\n## Mutating event filter\n## Add hostname and tag fields to apache.access tag events\n#<filter apache.access>\n#  @type record_transformer\n#  <record>\n#    hostname ${hostname}\n#    tag ${tag}\n#  </record>\n#</filter>\n\n## Selecting event filter\n## Remove unnecessary events from apache prefixed tag events\n#<filter apache.**>\n#  @type grep\n#  include1 method GET # pass only GET in 'method' field\n#  exclude1 message debug # remove debug event\n#</filter>\n\n# Listen HTTP for monitoring\n# http://localhost:24220/api/plugins\n# http://localhost:24220/api/plugins?type=TYPE\n# http://localhost:24220/api/plugins?tag=MYTAG\n<source>\n  @type monitor_agent\n  @id monitor_agent_input\n\n  port 24220\n</source>\n\n# Listen DRb for debug\n<source>\n  @type debug_agent\n  @id debug_agent_input\n\n  bind 127.0.0.1\n  port 24230\n</source>\n\n## match tag=apache.access and write to file\n#<match apache.access>\n#  @type file\n#  path /var/log/fluent/access\n#</match>\n\n## match tag=debug.** and dump to console\n<match debug.**>\n  @type stdout\n  @id stdout_output\n</match>\n\n## match tag=system.** and forward to another fluent server\n#<match system.**>\n#  @type forward\n#  @id forward_output\n#\n#  <server>\n#    host 192.168.0.11\n#  </server>\n#  <secondary>\n#    <server>\n#      host 192.168.0.12\n#    </server>\n#  </secondary>\n#</match>\n\n## match tag=myapp.** and forward and write to file\n#<match myapp.**>\n#  @type copy\n#  <store>\n#    @type forward\n#    buffer_type file\n#    buffer_path /var/log/fluent/myapp-forward\n#    retry_limit 50\n#    flush_interval 10s\n#    <server>\n#      host 192.168.0.13\n#    </server>\n#  </store>\n#  <store>\n#    @type file\n#    path /var/log/fluent/myapp\n#  </store>\n#</match>\n\n## match fluent's internal events\n#<match fluent.**>\n#  @type null\n#</match>\n\n## match not matched logs and write to file\n#<match **>\n#  @type file\n#  path /var/log/fluent/else\n#  compress gz\n#</match>\n\n## Label: For handling complex event routing\n#<label @STAGING>\n#  <match system.**>\n#    @type forward\n#    @id staging_forward_output\n#    <server>\n#      host 192.168.0.101\n#    </server>\n#  </match>\n#</label>\n"
  },
  {
    "path": "fluentd.gemspec",
    "content": "require File.expand_path('../lib/fluent/version', __FILE__)\n\nGem::Specification.new do |gem|\n  gem.name          = \"fluentd\"\n  gem.version       = Fluent::VERSION # see lib/fluent/version.rb\n\n  gem.authors       = [\"Sadayuki Furuhashi\"]\n  gem.email         = [\"frsyuki@gmail.com\"]\n  gem.description   = %q{Fluentd is an open source data collector designed to scale and simplify log management. It can collect, process and ship many kinds of data in near real-time.}\n  gem.summary       = %q{Fluentd event collector}\n  gem.homepage      = \"https://www.fluentd.org/\"\n\n  gem.files         = Dir.chdir(__dir__) do\n    `git ls-files -z`.split(\"\\x0\").reject do |f|\n      (File.expand_path(f) == __FILE__) ||\n        f.start_with?(*%w[tasks/ test/ .git Gemfile])\n    end\n  end\n  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }\n  gem.require_paths = [\"lib\"]\n  gem.license = \"Apache-2.0\"\n\n  gem.metadata[\"homepage_uri\"] = gem.homepage\n  gem.metadata[\"source_code_uri\"] = \"https://github.com/fluent/fluentd\"\n  gem.metadata[\"changelog_uri\"] = \"https://github.com/fluent/fluentd/blob/master/CHANGELOG.md\"\n  gem.metadata[\"bug_tracker_uri\"] = \"https://github.com/fluent/fluentd/issues\"\n\n  gem.required_ruby_version = '>= 3.2'\n\n  gem.add_runtime_dependency(\"bundler\")\n  gem.add_runtime_dependency(\"msgpack\", [\">= 1.3.1\", \"< 2.0.0\"])\n  gem.add_runtime_dependency(\"yajl-ruby\", [\"~> 1.0\"])\n  gem.add_runtime_dependency(\"cool.io\", [\">= 1.4.5\", \"< 2.0.0\"])\n  gem.add_runtime_dependency(\"serverengine\", [\">= 2.3.2\", \"< 3.0.0\"])\n  gem.add_runtime_dependency(\"http_parser.rb\", [\">= 0.5.1\", \"< 0.9.0\"])\n  gem.add_runtime_dependency(\"sigdump\", [\"~> 0.2.5\"])\n  gem.add_runtime_dependency(\"tzinfo\", [\">= 1.0\", \"< 3.0\"])\n  gem.add_runtime_dependency(\"tzinfo-data\", [\"~> 1.0\"])\n  gem.add_runtime_dependency(\"strptime\", [\">= 0.2.4\", \"< 1.0.0\"])\n  gem.add_runtime_dependency(\"webrick\", [\"~> 1.4\"])\n  gem.add_runtime_dependency(\"zstd-ruby\", [\"~> 1.5\"])\n  gem.add_runtime_dependency(\"uri\", '~> 1.0')\n  gem.add_runtime_dependency(\"net-http\", '~> 0.8')\n  gem.add_runtime_dependency(\"async-http\", \"~> 0.86\")\n\n  # gems that aren't default gems as of Ruby 3.4\n  gem.add_runtime_dependency(\"base64\", [\"~> 0.2\"])\n  gem.add_runtime_dependency(\"csv\", [\"~> 3.2\"])\n  gem.add_runtime_dependency(\"drb\", [\"~> 2.2\"])\n\n  # gems that aren't default gems as of Ruby 3.5\n  gem.add_runtime_dependency(\"logger\", [\"~> 1.6\"])\n\n  # gems that aren't default gems as of Ruby 4.0\n  gem.add_runtime_dependency(\"ostruct\", [\"~> 0.6\"])\n\n  # build gem for a certain platform. see also Rakefile\n  fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s\n  gem.platform = fake_platform unless fake_platform.empty?\n  if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)\n    gem.add_runtime_dependency(\"win32-service\", [\"~> 2.3.0\"])\n    gem.add_runtime_dependency(\"win32-ipc\", [\"~> 0.7.0\"])\n    gem.add_runtime_dependency(\"win32-event\", [\"~> 0.6.3\"])\n    gem.add_runtime_dependency(\"certstore_c\", [\"~> 0.1.7\"])\n\n    # gems that aren't default gems as of Ruby 3.5\n    gem.add_runtime_dependency(\"fiddle\", [\"~> 1.1\"])\n  end\n\n  gem.add_development_dependency(\"rake\", [\"~> 13.0\"])\n  gem.add_development_dependency(\"flexmock\", [\"~> 2.0\"])\n  gem.add_development_dependency(\"parallel_tests\", [\"~> 0.15.3\"])\n  gem.add_development_dependency(\"simplecov\", [\"~> 0.7\"])\n  gem.add_development_dependency(\"rr\", [\"~> 3.0\"])\n  # timecop v0.9.9 supports `Process.clock_gettime`. It breaks some tests.\n  # (https://github.com/fluent/fluentd/pull/4521)\n  gem.add_development_dependency(\"timecop\", [\"< 0.9.9\"])\n  gem.add_development_dependency(\"test-unit\", [\"~> 3.3\"])\n  gem.add_development_dependency(\"test-unit-rr\", [\"~> 1.0\"])\n  gem.add_development_dependency(\"oj\", [\">= 2.14\", \"< 4\"])\n  gem.add_development_dependency(\"console\", \"~> 1.30\")\n  gem.add_development_dependency(\"aws-sigv4\", [\"~> 1.8\"])\n  gem.add_development_dependency(\"aws-sdk-core\", [\"~> 3.191\"])\n  gem.add_development_dependency(\"rexml\", [\"~> 3.2\"])\nend\n"
  },
  {
    "path": "lib/fluent/agent.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/configurable'\nrequire 'fluent/plugin'\nrequire 'fluent/output'\nrequire 'fluent/match'\n\nmodule Fluent\n  #\n  # Agent is a resource unit who manages emittable plugins\n  #\n  # Next step: `fluentd/root_agent.rb`\n  # Next step: `fluentd/label.rb`\n  #\n  class Agent\n    include Configurable\n\n    def initialize(log:)\n      super()\n\n      @context = nil\n      @outputs = []\n      @filters = []\n\n      @lifecycle_control_list = nil\n      # lifecycle_control_list is the list of plugins in this agent, and ordered\n      # from plugins which DOES emit, then DOESN'T emit\n      # (input -> output w/ router -> filter -> output w/o router)\n      # for start: use this order DESC\n      #   (because plugins which appears later in configurations will receive events from plugins which appears earlier)\n      # for stop/before_shutdown/shutdown/after_shutdown/close/terminate: use this order ASC\n      @lifecycle_cache = nil\n\n      @log = log\n      @event_router = EventRouter.new(NoMatchMatch.new(log), self)\n      @error_collector = nil\n    end\n\n    attr_reader :log\n    attr_reader :outputs\n    attr_reader :filters\n    attr_reader :context\n    attr_reader :event_router\n    attr_reader :error_collector\n\n    def configure(conf)\n      super\n\n      # initialize <match> and <filter> elements\n      conf.elements('filter', 'match').each { |e|\n        if !Fluent::Engine.supervisor_mode && e.for_another_worker?\n          next\n        end\n        pattern = e.arg.empty? ? '**' : e.arg\n        type = e['@type']\n        raise ConfigError, \"Missing '@type' parameter on <#{e.name}> directive\" unless type\n        if e.name == 'filter'\n          add_filter(type, pattern, e)\n        else\n          add_match(type, pattern, e)\n        end\n      }\n    end\n\n    def lifecycle_control_list\n      return @lifecycle_control_list if @lifecycle_control_list\n\n      lifecycle_control_list = {\n        input: [],\n        output_with_router: [],\n        filter: [],\n        output: [],\n      }\n      if self.respond_to?(:inputs)\n        inputs.each do |i|\n          lifecycle_control_list[:input] << i\n        end\n      end\n      outputs.each do |o|\n        if o.has_router?\n          lifecycle_control_list[:output_with_router] << o\n        else\n          lifecycle_control_list[:output] << o\n        end\n      end\n      filters.each do |f|\n        lifecycle_control_list[:filter] << f\n      end\n\n      @lifecycle_control_list = lifecycle_control_list\n    end\n\n    def lifecycle(desc: false)\n      kind_list = if desc\n                    [:output, :filter, :output_with_router]\n                  else\n                    [:output_with_router, :filter, :output]\n                  end\n      kind_list.each do |kind|\n        list = if desc\n                 lifecycle_control_list[kind].reverse\n               else\n                 lifecycle_control_list[kind]\n               end\n        display_kind = (kind == :output_with_router ? :output : kind)\n        list.each do |instance|\n          yield instance, display_kind\n        end\n      end\n    end\n\n    def add_match(type, pattern, conf)\n      log_type = conf.for_this_worker? ? :default : :worker0\n      log.info log_type, \"adding match#{@context.nil? ? '' : \" in #{@context}\"}\", pattern: pattern, type: type\n\n      output = Plugin.new_output(type)\n      output.context_router = @event_router\n      output.configure(conf)\n      @outputs << output\n      if output.respond_to?(:outputs) && output.respond_to?(:multi_output?) && output.multi_output?\n        # TODO: ruby 2.3 or later: replace `output.respond_to?(:multi_output?) && output.multi_output?` with output&.multi_output?\n        outputs = if output.respond_to?(:static_outputs)\n                    output.static_outputs\n                  else\n                    output.outputs\n                  end\n        @outputs.push(*outputs)\n      end\n      @event_router.add_rule(pattern, output)\n\n      output\n    end\n\n    def add_filter(type, pattern, conf)\n      log_type = conf.for_this_worker? ? :default : :worker0\n      log.info log_type, \"adding filter#{@context.nil? ? '' : \" in #{@context}\"}\", pattern: pattern, type: type\n\n      filter = Plugin.new_filter(type)\n      filter.context_router = @event_router\n      filter.configure(conf)\n      @filters << filter\n      @event_router.add_rule(pattern, filter)\n\n      filter\n    end\n\n    # For handling invalid record\n    def emit_error_event(tag, time, record, error)\n    end\n\n    def handle_emits_error(tag, es, error)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/capability.rb",
    "content": "#\n# Fluent\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire \"fluent/env\"\n\nif Fluent.linux?\n  begin\n    require 'capng'\n  rescue LoadError\n  end\nend\n\nmodule Fluent\n  if defined?(CapNG)\n    class Capability\n      def initialize(target = nil, pid = nil)\n        @capng = CapNG.new(target, pid)\n      end\n\n      def usable?\n        true\n      end\n\n      def apply(select_set)\n        @capng.apply(select_set)\n      end\n\n      def clear(select_set)\n        @capng.clear(select_set)\n      end\n\n      def have_capability?(type, capability)\n        @capng.have_capability?(type, capability)\n      end\n\n      def update(action, type, capability_or_capability_array)\n        @capng.update(action, type, capability_or_capability_array)\n      end\n\n      def have_capabilities?(select_set)\n        @capng.have_capabilities?(select_set)\n      end\n    end\n  else\n    class Capability\n      def initialize(target = nil, pid = nil)\n      end\n\n      def usable?\n        false\n      end\n\n      def apply(select_set)\n        false\n      end\n\n      def clear(select_set)\n        false\n      end\n\n      def have_capability?(type, capability)\n        false\n      end\n\n      def update(action, type, capability_or_capability_array)\n        false\n      end\n\n      def have_capabilities?(select_set)\n        false\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/clock.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Clock\n    CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC\n\n    @@block_level = 0\n    @@frozen_clock = nil\n\n    def self.now\n      @@frozen_clock || now_raw\n    end\n\n    def self.freeze(dst = nil, &block)\n      return freeze_block(dst, &block) if block_given?\n\n      dst = dst_clock_from_time(dst) if dst.is_a?(Time)\n      @@frozen_clock = dst || now_raw\n    end\n\n    def self.return\n      raise \"invalid return while running code in blocks\" if @@block_level > 0\n      @@frozen_clock = nil\n    end\n\n    # internal use\n\n    def self.now_raw\n      Process.clock_gettime(CLOCK_ID)\n    end\n\n    def self.real_now(unit = :second)\n      Process.clock_gettime(Process::CLOCK_REALTIME, unit)\n    end\n\n    def self.dst_clock_from_time(time)\n      diff_sec = Time.now - time\n      now_raw - diff_sec\n    end\n\n    def self.freeze_block(dst)\n      dst = dst_clock_from_time(dst) if dst.is_a?(Time)\n      pre_frozen_clock = @@frozen_clock\n      @@frozen_clock = dst || now_raw\n      @@block_level += 1\n      yield\n    ensure\n      @@block_level -= 1\n      @@frozen_clock = pre_frozen_clock\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/command/binlog_reader.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'optparse'\nrequire 'msgpack'\n\nrequire 'fluent/msgpack_factory'\nrequire 'fluent/formatter'\nrequire 'fluent/plugin'\nrequire 'fluent/config/element'\nrequire 'fluent/engine'\nrequire 'fluent/version'\n\nclass FluentBinlogReader\n  SUBCOMMAND = %w(cat head formats)\n  HELP_TEXT = <<HELP\nUsage: fluent-binlog-reader <command> [<args>]\n\nCommands of fluent-binlog-reader:\n   cat     :     Read files sequentially, writing them to standard output.\n   head    :     Display the beginning of a text file.\n   formats :     Display plugins that you can use.\n\nSee 'fluent-binlog-reader <command> --help' for more information on a specific command.\nHELP\n\n  def initialize(argv = ARGV)\n    @argv = argv\n  end\n\n  def call\n    command_class = BinlogReaderCommand.const_get(command)\n    command_class.new(@argv).call\n  end\n\n  private\n\n  def command\n    command = @argv.shift\n    if command\n      if command == '--version'\n        puts \"#{File.basename($PROGRAM_NAME)} #{Fluent::VERSION}\"\n        exit 0\n      elsif !SUBCOMMAND.include?(command)\n        usage \"'#{command}' is not supported: Required subcommand : #{SUBCOMMAND.join(' | ')}\"\n      end\n    else\n      usage \"Required subcommand : #{SUBCOMMAND.join(' | ')}\"\n    end\n\n    command.split('_').map(&:capitalize).join('')\n  end\n\n  def usage(msg = nil)\n    puts HELP_TEXT\n    puts \"Error: #{msg}\" if msg\n    exit 1\n  end\nend\n\nmodule BinlogReaderCommand\n  class Base\n    def initialize(argv = ARGV)\n      @argv = argv\n\n      @options = { plugin: [] }\n      @opt_parser = OptionParser.new do |opt|\n        opt.version = Fluent::VERSION\n        opt.separator 'Options:'\n\n        opt.on('-p DIR', '--plugin', 'add library directory path') do |v|\n          @options[:plugin] << v\n        end\n      end\n    end\n\n    def call\n      raise NotImplementedError, 'BUG: command  MUST implement this method'\n    end\n\n    private\n\n    def usage(msg = nil)\n      puts @opt_parser.to_s\n      puts \"Error: #{msg}\" if msg\n      exit 1\n    end\n\n    def parse_options!\n      @opt_parser.parse!(@argv)\n\n      unless @options[:plugin].empty?\n        if dir = @options[:plugin].find { |d| !Dir.exist?(d) }\n          usage \"Directory #{dir} doesn't exist\"\n        else\n          @options[:plugin].each do |d|\n            Fluent::Plugin.add_plugin_dir(d)\n          end\n        end\n      end\n    rescue => e\n      usage e\n    end\n  end\n\n  module Formattable\n    DEFAULT_OPTIONS = {\n      format: :out_file\n    }\n\n    def initialize(argv = ARGV)\n      super\n      @options.merge!(DEFAULT_OPTIONS)\n      configure_option_parser\n    end\n\n    private\n\n    def configure_option_parser\n      @options[:config_params] = {}\n\n      @opt_parser.banner = \"Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file\"\n\n      @opt_parser.on('-f TYPE', '--format', 'configure output format') do |v|\n        @options[:format] = v.to_sym\n      end\n\n      @opt_parser.on('-e KEY=VALUE', 'configure formatter config params') do |v|\n        key, value = v.split('=')\n        usage \"#{v} is invalid. valid format is like `key=value`\" unless value\n        @options[:config_params].merge!(key => value)\n      end\n    end\n\n    def lookup_formatter(format, params)\n      conf = Fluent::Config::Element.new('ROOT', '', params, [])\n      formatter = Fluent::Plugin.new_formatter(format)\n\n      if formatter.respond_to?(:configure)\n        formatter.configure(conf)\n      end\n      formatter\n    rescue => e\n      usage e\n    end\n  end\n\n  class Head < Base\n    include Formattable\n\n    DEFAULT_HEAD_OPTIONS = {\n      count: 5\n    }\n\n    def initialize(argv = ARGV)\n      super\n      @options.merge!(default_options)\n      parse_options!\n    end\n\n    def call\n      @formatter = lookup_formatter(@options[:format], @options[:config_params])\n\n      File.open(@path, 'rb') do |io|\n        i = 1\n        Fluent::MessagePackFactory.unpacker(io).each do |(time, record)|\n          print @formatter.format(@path, time, record) # path is used for tag\n          break if @options[:count] && i == @options[:count]\n          i += 1\n        end\n      end\n    end\n\n    private\n\n    def default_options\n      DEFAULT_HEAD_OPTIONS\n    end\n\n    def parse_options!\n      @opt_parser.on('-n COUNT', 'Set the number of lines to display') do |v|\n        @options[:count] = v.to_i\n        usage \"illegal line count -- #{@options[:count]}\" if @options[:count] < 1\n      end\n\n      super\n\n      usage 'Path is required' if @argv.empty?\n      @path = @argv.first\n      usage \"#{@path} is not found\" unless File.exist?(@path)\n    end\n  end\n\n  class Cat < Head\n    DEFAULT_CAT_OPTIONS = {\n      count: nil                # Overwrite DEFAULT_HEAD_OPTIONS[:count]\n    }\n\n    def default_options\n      DEFAULT_CAT_OPTIONS\n    end\n  end\n\n  class Formats < Base\n    def initialize(argv = ARGV)\n      super\n      parse_options!\n    end\n\n    def call\n      prefix = Fluent::Plugin::FORMATTER_REGISTRY.dir_search_prefix || 'formatter_'\n\n      plugin_dirs = @options[:plugin]\n      unless plugin_dirs.empty?\n        plugin_dirs.each do |d|\n          Dir.glob(\"#{d}/#{prefix}*.rb\").each do |path|\n            require File.absolute_path(path)\n          end\n        end\n      end\n\n      $LOAD_PATH.map do |lp|\n        Dir.glob(\"#{lp}/#{prefix}*.rb\").each do |path|\n          require path\n        end\n      end\n\n      puts Fluent::Plugin::FORMATTER_REGISTRY.map.keys\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/command/bundler_injection.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'rbconfig'\n\nif ENV['BUNDLE_BIN_PATH']\n  puts 'error: You seem to use `bundle exec` already.'\n  exit 1\nelse\n  begin\n    bundle_bin = Gem::Specification.find_by_name('bundler').bin_file('bundle')\n  rescue Gem::LoadError => e\n    puts \"error: #{e}\"\n    exit 1\n  end\n  ruby_bin = RbConfig.ruby\n  system(\"#{ruby_bin} #{bundle_bin} install\")\n  unless $?.success?\n    exit $?.exitstatus\n  end\n\n  cmdline = [\n    ruby_bin,\n    bundle_bin,\n    'exec',\n    ruby_bin,\n    File.expand_path(File.join(File.dirname(__FILE__), 'fluentd.rb')),\n  ] + ARGV\n\n  exec(*cmdline)\n  exit! 127\nend\n"
  },
  {
    "path": "lib/fluent/command/ca_generate.rb",
    "content": "require 'openssl'\nrequire 'optparse'\nrequire 'fileutils'\nrequire 'fluent/version'\n\nmodule Fluent\n  class CaGenerate\n    DEFAULT_OPTIONS = {\n      private_key_length: 2048,\n      cert_country:  'US',\n      cert_state:    'CA',\n      cert_locality: 'Mountain View',\n      cert_common_name: 'Fluentd Forward CA',\n    }\n    HELP_TEXT = <<HELP\nUsage: fluent-ca-generate DIR_PATH PRIVATE_KEY_PASSPHRASE [--country COUNTRY] [--state STATE] [--locality LOCALITY] [--common-name COMMON_NAME]\nHELP\n\n    def initialize(argv = ARGV)\n      @argv = argv\n      @options = {}\n      @opt_parser = OptionParser.new\n      configure_option_parser\n      @options.merge!(DEFAULT_OPTIONS)\n      parse_options!\n    end\n\n    def usage(msg = nil)\n      puts HELP_TEXT\n      puts \"Error: #{msg}\" if msg\n      exit 1\n    end\n\n    def call\n      ca_dir, passphrase, = @argv[0..1]\n\n      unless ca_dir && passphrase\n        puts \"#{HELP_TEXT}\"\n        puts ''\n        exit 1\n      end\n\n      FileUtils.mkdir_p(ca_dir)\n\n      cert, key = Fluent::CaGenerate.generate_ca_pair(@options)\n\n      key_data = key.export(OpenSSL::Cipher.new('aes256'), passphrase)\n      File.open(File.join(ca_dir, 'ca_key.pem'), 'w') do |file|\n        file.write key_data\n      end\n      File.open(File.join(ca_dir, 'ca_cert.pem'), 'w') do |file|\n        file.write cert.to_pem\n      end\n\n      puts \"successfully generated: ca_key.pem, ca_cert.pem\"\n      puts \"copy and use ca_cert.pem to client(out_forward)\"\n    end\n\n    def self.certificates_from_file(path)\n      data = File.read(path)\n      pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\\n(?:[^-]*\\n)+-+END CERTIFICATE-+\\n', Regexp::MULTILINE)\n      list = []\n      data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match)}\n      list\n    end\n\n    def self.generate_ca_pair(opts={})\n      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])\n\n      issuer = subject = OpenSSL::X509::Name.new\n      subject.add_entry('C', opts[:cert_country])\n      subject.add_entry('ST', opts[:cert_state])\n      subject.add_entry('L', opts[:cert_locality])\n      subject.add_entry('CN', opts[:cert_common_name])\n\n      digest = OpenSSL::Digest::SHA256.new\n\n      factory = OpenSSL::X509::ExtensionFactory.new\n\n      cert = OpenSSL::X509::Certificate.new\n      cert.not_before = Time.at(0)\n      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after\n      cert.public_key = key\n      cert.serial = 1\n      cert.issuer = issuer\n      cert.subject = subject\n      cert.add_extension(factory.create_extension('basicConstraints', 'CA:TRUE'))\n      cert.sign(key, digest)\n\n      return cert, key\n    end\n\n    def self.generate_server_pair(opts={})\n      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])\n\n      ca_key = OpenSSL::PKey::read(File.read(opts[:ca_key_path]), opts[:ca_key_passphrase])\n      ca_cert = OpenSSL::X509::Certificate.new(File.read(opts[:ca_cert_path]))\n      issuer = ca_cert.issuer\n\n      subject = OpenSSL::X509::Name.new\n      subject.add_entry('C', opts[:country])\n      subject.add_entry('ST', opts[:state])\n      subject.add_entry('L', opts[:locality])\n      subject.add_entry('CN', opts[:common_name])\n\n      digest = OpenSSL::Digest::SHA256.new\n\n      cert = OpenSSL::X509::Certificate.new\n      cert.not_before = Time.at(0)\n      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after\n      cert.public_key = key\n      cert.serial = 2\n      cert.issuer = issuer\n      cert.subject = subject\n\n      factory = OpenSSL::X509::ExtensionFactory.new\n      server_cert.add_extension(factory.create_extension('basicConstraints', 'CA:FALSE'))\n      server_cert.add_extension(factory.create_extension('nsCertType', 'server'))\n\n      cert.sign ca_key, digest\n\n      return cert, key\n    end\n\n    def self.generate_self_signed_server_pair(opts={})\n      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])\n\n      issuer = subject = OpenSSL::X509::Name.new\n      subject.add_entry('C', opts[:country])\n      subject.add_entry('ST', opts[:state])\n      subject.add_entry('L', opts[:locality])\n      subject.add_entry('CN', opts[:common_name])\n\n      digest = OpenSSL::Digest::SHA256.new\n\n      cert = OpenSSL::X509::Certificate.new\n      cert.not_before = Time.at(0)\n      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after\n      cert.public_key = key\n      cert.serial = 1\n      cert.issuer = issuer\n      cert.subject  = subject\n      cert.sign(key, digest)\n\n      return cert, key\n    end\n\n    private\n\n    def configure_option_parser\n      @opt_parser.banner = HELP_TEXT\n      @opt_parser.version = Fluent::VERSION\n\n      @opt_parser.on('--key-length [KEY_LENGTH]',\n                     \"configure key length. (default: #{DEFAULT_OPTIONS[:private_key_length]})\") do |v|\n        @options[:private_key_length] = v.to_i\n      end\n\n      @opt_parser.on('--country [COUNTRY]',\n                     \"configure country. (default: #{DEFAULT_OPTIONS[:cert_country]})\") do |v|\n        @options[:cert_country] = v.upcase\n      end\n\n      @opt_parser.on('--state [STATE]',\n                     \"configure state. (default: #{DEFAULT_OPTIONS[:cert_state]})\") do |v|\n        @options[:cert_state] = v\n      end\n\n      @opt_parser.on('--locality [LOCALITY]',\n                     \"configure locality. (default: #{DEFAULT_OPTIONS[:cert_locality]})\") do |v|\n        @options[:cert_locality] = v\n      end\n\n      @opt_parser.on('--common-name [COMMON_NAME]',\n                     \"configure common name (default: #{DEFAULT_OPTIONS[:cert_common_name]})\") do |v|\n        @options[:cert_common_name] = v\n      end\n    end\n\n    def parse_options!\n      @opt_parser.parse!(@argv)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/command/cap_ctl.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'optparse'\nrequire 'fluent/log'\nrequire 'fluent/env'\nrequire 'fluent/capability'\n\nmodule Fluent\n  class CapCtl\n    def prepare_option_parser\n      @op = OptionParser.new\n\n      @op.on('--clear', \"Clear Fluentd Ruby capability\") {|s|\n        @opts[:clear_capabilities] = true\n      }\n\n      @op.on('--add [CAPABILITY1,CAPABILITY2, ...]', \"Add capabilities into Fluentd Ruby\") {|s|\n        @opts[:add_capabilities] = s\n      }\n\n      @op.on('--drop [CAPABILITY1,CAPABILITY2, ...]', \"Drop capabilities from Fluentd Ruby\") {|s|\n        @opts[:drop_capabilities] = s\n      }\n\n      @op.on('--get', \"Get capabilities for Fluentd Ruby\") {|s|\n        @opts[:get_capabilities] = true\n      }\n\n      @op.on('-f', '--file FILE', \"Specify target file to add Linux capabilities\") {|s|\n        @opts[:target_file] = s\n      }\n    end\n\n    def usage(msg)\n      puts @op.to_s\n      puts \"error: #{msg}\" if msg\n      exit 1\n    end\n\n    def initialize(argv = ARGV)\n      @opts = {}\n      @argv = argv\n\n      if Fluent.linux?\n        begin\n          require 'capng'\n\n          @capng = CapNG.new\n        rescue LoadError\n          puts \"Error: capng_c is not loaded. Please install it first.\"\n          exit 1\n        end\n      else\n        puts \"Error: This environment is not supported.\"\n        exit 2\n      end\n\n      prepare_option_parser\n    end\n\n    def call\n      parse_options!(@argv)\n\n      target_file = if !!@opts[:target_file]\n                      @opts[:target_file]\n                    else\n                      File.readlink(\"/proc/self/exe\")\n                    end\n\n      if @opts[:clear_capabilities]\n        clear_capabilities(@opts, target_file)\n      elsif @opts[:add_capabilities]\n        add_capabilities(@opts, target_file)\n      elsif @opts[:drop_capabilities]\n        drop_capabilities(@opts, target_file)\n      end\n      if @opts[:get_capabilities]\n        get_capabilities(@opts, target_file)\n      end\n    end\n\n    def clear_capabilities(opts, target_file)\n      if !!opts[:clear_capabilities]\n        @capng.clear(:caps)\n        ret = @capng.apply_caps_file(target_file)\n        puts \"Clear capabilities #{ret ? 'done' : 'fail'}.\"\n      end\n    end\n\n    def add_capabilities(opts, target_file)\n      if add_caps = opts[:add_capabilities]\n        @capng.clear(:caps)\n        @capng.caps_file(target_file)\n        capabilities = add_caps.split(/\\s*,\\s*/)\n        check_capabilities(capabilities, get_valid_capabilities)\n        ret = @capng.update(:add,\n                            CapNG::Type::EFFECTIVE | CapNG::Type::INHERITABLE | CapNG::Type::PERMITTED,\n                            capabilities)\n        puts \"Updating #{add_caps} #{ret ? 'done' : 'fail'}.\"\n        ret = @capng.apply_caps_file(target_file)\n        puts \"Adding #{add_caps} #{ret ? 'done' : 'fail'}.\"\n      end\n    end\n\n    def drop_capabilities(opts, target_file)\n      if drop_caps = opts[:drop_capabilities]\n        @capng.clear(:caps)\n        @capng.caps_file(target_file)\n        capabilities = drop_caps.split(/\\s*,\\s*/)\n        check_capabilities(capabilities, get_valid_capabilities)\n        ret = @capng.update(:drop,\n                            CapNG::Type::EFFECTIVE | CapNG::Type::INHERITABLE | CapNG::Type::PERMITTED,\n                            capabilities)\n        puts \"Updating #{drop_caps} #{ret ? 'done' : 'fail'}.\"\n        @capng.apply_caps_file(target_file)\n        puts \"Dropping #{drop_caps} #{ret ? 'done' : 'fail'}.\"\n      end\n    end\n\n    def get_capabilities(opts, target_file)\n      if opts[:get_capabilities]\n        @capng.caps_file(target_file)\n        print = CapNG::Print.new\n        puts \"Capabilities in '#{target_file}',\"\n        puts \"Effective:   #{print.caps_text(:buffer, :effective)}\"\n        puts \"Inheritable: #{print.caps_text(:buffer, :inheritable)}\"\n        puts \"Permitted:   #{print.caps_text(:buffer, :permitted)}\"\n      end\n    end\n\n    def get_valid_capabilities\n      capabilities = []\n      cap = CapNG::Capability.new\n      cap.each do |_code, capability|\n        capabilities << capability\n      end\n      capabilities\n    end\n\n    def check_capabilities(capabilities, valid_capabilities)\n      capabilities.each do |capability|\n        unless valid_capabilities.include?(capability)\n          raise ArgumentError, \"'#{capability}' is not valid capability. Valid Capabilities are:  #{valid_capabilities.join(\", \")}\"\n        end\n      end\n    end\n\n    def parse_options!(argv)\n      begin\n        rest = @op.parse(argv)\n\n        if rest.length != 0\n          usage nil\n        end\n      rescue\n        usage $!.to_s\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/command/cat.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'optparse'\nrequire 'fluent/env'\nrequire 'fluent/time'\nrequire 'fluent/msgpack_factory'\nrequire 'fluent/version'\n\nop = OptionParser.new\n\nop.banner += \" <tag>\"\nop.version = Fluent::VERSION\n\nport = 24224\nhost = '127.0.0.1'\nunix = false\nsocket_path = Fluent::DEFAULT_SOCKET_PATH\n\nconfig_path = Fluent::DEFAULT_CONFIG_PATH\nformat = 'json'\nmessage_key = 'message'\ntime_as_integer = false\nretry_limit = 5\nevent_time = nil\n\nop.on('-p', '--port PORT', \"fluent tcp port (default: #{port})\", Integer) {|i|\n  port = i\n}\n\nop.on('-h', '--host HOST', \"fluent host (default: #{host})\") {|s|\n  host = s\n}\n\nop.on('-u', '--unix', \"use unix socket instead of tcp\", TrueClass) {|b|\n  unix = b\n}\n\nop.on('-s', '--socket PATH', \"unix socket path (default: #{socket_path})\") {|s|\n  socket_path = s\n}\n\nop.on('-f', '--format FORMAT', \"input format (default: #{format})\") {|s|\n  format = s\n}\n\nop.on('--json', \"same as: -f json\", TrueClass) {|b|\n  format = 'json'\n}\n\nop.on('--msgpack', \"same as: -f msgpack\", TrueClass) {|b|\n  format = 'msgpack'\n}\n\nop.on('--none', \"same as: -f none\", TrueClass) {|b|\n  format = 'none'\n}\n\nop.on('--message-key KEY', \"key field for none format (default: #{message_key})\") {|s|\n  message_key = s\n}\n\nop.on('--time-as-integer', \"Send time as integer for v0.12 or earlier\", TrueClass) { |b|\n  time_as_integer = true\n}\n\nop.on('--retry-limit N', \"Specify the number of retry limit (default: #{retry_limit})\", Integer) {|n|\n  retry_limit = n\n}\n\nop.on('--event-time TIME_STRING', \"Specify the time expression string (default: nil)\") {|v|\n  event_time = v\n}\n\nsingleton_class.module_eval do\n  define_method(:usage) do |msg|\n    puts op.to_s\n    puts \"error: #{msg}\" if msg\n    exit 1\n  end\nend\n\nbegin\n  op.parse!(ARGV)\n\n  if ARGV.length != 1\n    usage nil\n  end\n\n  tag = ARGV.shift\n\nrescue\n  usage $!.to_s\nend\n\nrequire 'socket'\nrequire 'json'\nrequire 'msgpack'\nrequire 'fluent/ext_monitor_require'\n\nclass Writer\n  include MonitorMixin\n\n  RetryLimitError = Class.new(StandardError)\n\n  class TimerThread\n    def initialize(writer)\n      @writer = writer\n    end\n\n    def start\n      @finish = false\n      @thread = Thread.new(&method(:run))\n    end\n\n    def shutdown\n      @finish = true\n      @thread.join\n    end\n\n    def run\n      until @finish\n        sleep 1\n        @writer.on_timer\n      end\n    end\n  end\n\n  def initialize(tag, connector, time_as_integer: false, retry_limit: 5, event_time: nil)\n    @tag = tag\n    @connector = connector\n    @socket = false\n\n    @socket_time = Time.now.to_i\n    @socket_ttl = 10  # TODO\n    @error_history = []\n\n    @pending = []\n    @pending_limit = 1024  # TODO\n    @retry_wait = 1\n    @retry_limit = retry_limit\n    @time_as_integer = time_as_integer\n    @event_time = event_time\n\n    super()\n  end\n\n  def secondary_record?(record)\n    record.class != Hash &&\n      record.size == 2 &&\n      record.first.class == Fluent::EventTime &&\n      record.last.class == Hash\n  end\n\n  def write(record)\n    unless secondary_record?(record)\n      if record.class != Hash\n        raise ArgumentError, \"Input must be a map (got #{record.class})\"\n      end\n    end\n\n    time = if @event_time\n             Fluent::EventTime.parse(@event_time)\n           else\n             Fluent::EventTime.now\n           end\n    time = time.to_i if @time_as_integer\n    entry = if secondary_record?(record)\n              # Even though secondary contains Fluent::EventTime in record,\n              # fluent-cat just ignore it and set Fluent::EventTime.now instead.\n              # This specification is adopted to keep consistency.\n              [time, record.last]\n            else\n              [time, record]\n            end\n    synchronize {\n      unless write_impl([entry])\n        # write failed\n        @pending.push(entry)\n\n        while @pending.size > @pending_limit\n          # exceeds pending limit; trash oldest record\n          time, record = @pending.shift\n          abort_message(time, record)\n        end\n      end\n    }\n  end\n\n  def on_timer\n    now = Time.now.to_i\n\n    synchronize {\n      unless @pending.empty?\n        # flush pending records\n        if write_impl(@pending)\n          # write succeeded\n          @pending.clear\n        end\n      end\n\n      if @socket && @socket_time + @socket_ttl < now\n        # socket is not used @socket_ttl seconds\n        close\n      end\n    }\n  end\n\n  def close\n    @socket.close\n    @socket = nil\n  end\n\n  def start\n    @timer = TimerThread.new(self)\n    @timer.start\n    self\n  end\n\n  def shutdown\n    @timer.shutdown\n  end\n\n  private\n  def write_impl(array)\n    socket = get_socket\n    unless socket\n      return false\n    end\n\n    begin\n      packer = Fluent::MessagePackFactory.packer\n      socket.write packer.pack([@tag, array])\n      socket.flush\n    rescue\n      $stderr.puts \"write failed: #{$!}\"\n      close\n      return false\n    end\n\n    return true\n  end\n\n  def get_socket\n    unless @socket\n      unless try_connect\n        return nil\n      end\n    end\n\n    @socket_time = Time.now.to_i\n    return @socket\n  end\n\n  def try_connect\n    begin\n      now = Time.now.to_i\n\n      unless @error_history.empty?\n        # wait before re-connecting\n        wait = 1 #@retry_wait * (2 ** (@error_history.size-1))\n        if now <= @socket_time + wait\n          sleep(wait)\n          try_connect\n        end\n      end\n\n      @socket = @connector.call\n      @error_history.clear\n      return true\n\n    rescue RetryLimitError => ex\n      raise ex\n    rescue\n      $stderr.puts \"connect failed: #{$!}\"\n      @error_history << $!\n      @socket_time = now\n\n      if @retry_limit < @error_history.size\n        # abort all pending records\n        @pending.each {|(time, record)|\n          abort_message(time, record)\n        }\n        @pending.clear\n        @error_history.clear\n        raise RetryLimitError, \"exceed retry limit\"\n      else\n        retry\n      end\n    end\n  end\n\n  def abort_message(time, record)\n    $stdout.puts \"!#{time}:#{JSON.generate(record)}\"\n  end\nend\n\n\nif unix\n  connector = Proc.new {\n    UNIXSocket.open(socket_path)\n  }\nelse\n  connector = Proc.new {\n    TCPSocket.new(host, port)\n  }\nend\n\nw = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit, event_time: event_time)\nw.start\n\ncase format\nwhen 'json'\n  begin\n    while line = $stdin.gets\n      record = JSON.parse(line)\n      w.write(record)\n    end\n  rescue\n    $stderr.puts $!\n    exit 1\n  end\n\nwhen 'msgpack'\n  require 'fluent/engine'\n\n  begin\n    u = Fluent::MessagePackFactory.msgpack_unpacker($stdin)\n    u.each {|record|\n      w.write(record)\n    }\n  rescue EOFError\n  rescue\n    $stderr.puts $!\n    exit 1\n  end\n\nwhen 'none'\n  begin\n    while line = $stdin.gets\n      record = { message_key => line.chomp }\n      w.write(record)\n    end\n  rescue\n    $stderr.puts $!\n    exit 1\n  end\n\nelse\n  $stderr.puts \"Unknown format '#{format}'\"\n  exit 1\nend\n"
  },
  {
    "path": "lib/fluent/command/ctl.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'optparse'\nrequire 'fluent/env'\nrequire 'fluent/version'\nif Fluent.windows?\n  require 'win32/event'\n  require 'win32/service'\nend\n\nmodule Fluent\n  class Ctl\n    DEFAULT_OPTIONS = {}\n\n    if Fluent.windows?\n      include Windows::ServiceConstants\n      include Windows::ServiceStructs\n      include Windows::ServiceFunctions\n\n      COMMAND_MAP = {\n        shutdown: \"\",\n        restart: \"HUP\",\n        flush: \"USR1\",\n        reload: \"USR2\",\n        dump: \"CONT\",\n      }\n      WINSVC_CONTROL_CODE_MAP = {\n        shutdown: SERVICE_CONTROL_STOP,\n        # 128 - 255: user-defined control code\n        # See https://docs.microsoft.com/en-us/windows/win32/api/winsvc/nf-winsvc-controlservice\n        restart: 128,\n        flush: 129,\n        reload: SERVICE_CONTROL_PARAMCHANGE,\n        dump: 130,\n      }\n    else\n      COMMAND_MAP = {\n        shutdown: :TERM,\n        restart: :HUP,\n        flush: :USR1,\n        reload: :USR2,\n        dump: :CONT,\n      }\n    end\n\n    def initialize(argv = ARGV)\n      @argv = argv\n      @options = {}\n      @opt_parser = OptionParser.new\n      configure_option_parser\n      @options.merge!(DEFAULT_OPTIONS)\n      parse_options!\n    end\n\n    def help_text\n      text = \"\\n\"\n      if Fluent.windows?\n        text << \"Usage: #{$PROGRAM_NAME} COMMAND [PID_OR_SVCNAME]\\n\"\n      else\n        text << \"Usage: #{$PROGRAM_NAME} COMMAND PID\\n\"\n      end\n      text << \"\\n\"\n      text << \"Commands: \\n\"\n      COMMAND_MAP.each do |key, value|\n        text << \"  #{key}\\n\"\n      end\n      text\n    end\n\n    def usage(msg = nil)\n      puts help_text\n      if msg\n        puts\n        puts \"Error: #{msg}\"\n      end\n      exit 1\n    end\n\n    def call\n      if Fluent.windows?\n        if /^[0-9]+$/.match?(@pid_or_svcname)\n          # Use as PID\n          return call_windows_event(@command, \"fluentd_#{@pid_or_svcname}\")\n        end\n\n        unless call_winsvc_control_code(@command, @pid_or_svcname)\n          puts \"Cannot send control code to #{@pid_or_svcname} service, try to send an event with this name ...\"\n          call_windows_event(@command, @pid_or_svcname)\n        end\n      else\n        call_signal(@command, @pid_or_svcname)\n      end\n    end\n\n    private\n\n    def call_signal(command, pid)\n      signal = COMMAND_MAP[command.to_sym]\n      Process.kill(signal, pid.to_i)\n    end\n\n    def call_winsvc_control_code(command, pid_or_svcname)\n      status = SERVICE_STATUS.new\n\n      begin\n        handle_scm = OpenSCManager(nil, nil, SC_MANAGER_CONNECT)\n        FFI.raise_windows_error('OpenSCManager') if handle_scm == 0\n\n        handle_scs = OpenService(handle_scm, \"fluentdwinsvc\", SERVICE_STOP | SERVICE_PAUSE_CONTINUE | SERVICE_USER_DEFINED_CONTROL)\n        FFI.raise_windows_error('OpenService') if handle_scs == 0\n\n        control_code = WINSVC_CONTROL_CODE_MAP[command.to_sym]\n\n        unless ControlService(handle_scs, control_code, status)\n          FFI.raise_windows_error('ControlService')\n        end\n      rescue => e\n        puts e\n        state = status[:dwCurrentState]\n        return state == SERVICE_STOPPED || state == SERVICE_STOP_PENDING\n      ensure\n        CloseServiceHandle(handle_scs)\n        CloseServiceHandle(handle_scm)\n      end\n\n      return true\n    end\n\n    def call_windows_event(command, pid_or_svcname)\n      prefix = pid_or_svcname\n      event_name = COMMAND_MAP[command.to_sym]\n      suffix = event_name.empty? ? \"\" : \"_#{event_name}\"\n\n      begin\n        event = Win32::Event.open(\"#{prefix}#{suffix}\")\n        event.set\n        event.close\n      rescue Errno::ENOENT\n        puts \"Error: Cannot find the fluentd process with the event name: \\\"#{prefix}\\\"\"\n      end\n    end\n\n    def configure_option_parser\n      @opt_parser.banner = help_text\n      @opt_parser.version = Fluent::VERSION\n    end\n\n    def parse_options!\n      @opt_parser.parse!(@argv)\n\n      @command = @argv[0]\n      @pid_or_svcname = @argv[1] || \"fluentdwinsvc\"\n\n      usage(\"Command isn't specified!\") if @command.nil? || @command.empty?\n      usage(\"Unknown command: #{@command}\") unless COMMAND_MAP.has_key?(@command.to_sym)\n\n      if Fluent.windows?\n        usage(\"PID or SVCNAME isn't specified!\") if @pid_or_svcname.nil? || @pid_or_svcname.empty?\n      else\n        usage(\"PID isn't specified!\") if @pid_or_svcname.nil? || @pid_or_svcname.empty?\n        usage(\"Invalid PID: #{pid}\") unless /^[0-9]+$/.match?(@pid_or_svcname)\n      end\n    end\n  end\nend\n\n"
  },
  {
    "path": "lib/fluent/command/debug.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'optparse'\n\nop = OptionParser.new\n\nhost = '127.0.0.1'\nport = 24230\nunix = nil\n\nop.on('-h', '--host HOST', \"fluent host (default: #{host})\") {|s|\n  host = s\n}\n\nop.on('-p', '--port PORT', \"debug_agent tcp port (default: #{port})\", Integer) {|i|\n  port = i\n}\n\nop.on('-u', '--unix PATH', \"use unix socket instead of tcp\") {|b|\n  unix = b\n}\n\nsingleton_class.module_eval do\n  define_method(:usage) do |msg|\n    puts op.to_s\n    puts \"error: #{msg}\" if msg\n    exit 1\n  end\nend\n\nbegin\n  op.parse!(ARGV)\n\n  if ARGV.length != 0\n    usage nil\n  end\nrescue\n  usage $!.to_s\nend\n\nrequire 'drb/drb'\n\nif unix\n  uri = \"drbunix:#{unix}\"\nelse\n  uri = \"druby://#{host}:#{port}\"\nend\n\nrequire 'fluent/log'\nrequire 'fluent/env'\nrequire 'fluent/engine'\nrequire 'fluent/system_config'\nrequire 'serverengine'\n\ninclude Fluent::SystemConfig::Mixin\n\ndl_opts = {}\ndl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\nlogger = ServerEngine::DaemonLogger.new(STDERR, dl_opts)\n$log = Fluent::Log.new(logger)\nFluent::Engine.init(system_config)\n\nDRb::DRbObject.class_eval do\n  undef_method :methods\n  undef_method :instance_eval\n  undef_method :instance_variables\n  undef_method :instance_variable_get\nend\n\nremote_engine = DRb::DRbObject.new_with_uri(uri)\n\nFluent.module_eval do\n  remove_const(:Engine)\n  const_set(:Engine, remote_engine)\nend\n\ninclude Fluent\n\nputs \"Connected to #{uri}.\"\nputs \"Usage:\"\nputs \"    Fluent::Engine.root_agent.event_router.match('some.tag') : get an output plugin instance\"\nputs \"    Fluent::Engine.root_agent.inputs[i]                      : get input plugin instances\"\nputs \"    Fluent::Plugin::OUTPUT_REGISTRY.lookup(name)             : load output plugin class (use this if you get DRb::DRbUnknown)\"\nputs \"\"\n\nEncoding.default_internal = nil if Encoding.respond_to?(:default_internal)\n\nrequire 'irb'\nIRB.start\n"
  },
  {
    "path": "lib/fluent/command/fluentd.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'optparse'\n\nrequire 'fluent/supervisor'\nrequire 'fluent/log'\nrequire 'fluent/env'\nrequire 'fluent/version'\n\n$fluentdargv = Marshal.load(Marshal.dump(ARGV))\n\nop = OptionParser.new\nop.version = Fluent::VERSION\n\ndefault_opts = Fluent::Supervisor.default_options\ncmd_opts = {}\n\nop.on('-s', \"--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]\", \"install sample configuration file to the directory\") {|s|\n  cmd_opts[:setup_path] = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)\n}\n\nop.on('-c', '--config PATH', \"config file path (default: #{Fluent::DEFAULT_CONFIG_PATH})\") {|s|\n  cmd_opts[:config_path] = s\n}\n\nop.on('--dry-run', \"Check fluentd setup is correct or not\", TrueClass) {|b|\n  cmd_opts[:dry_run] = b\n}\n\nop.on('--show-plugin-config=PLUGIN', \"[DEPRECATED] Show PLUGIN configuration and exit(ex: input:dummy)\") {|plugin|\n  cmd_opts[:show_plugin_config] = plugin\n}\n\nop.on('-p', '--plugin DIR', \"add plugin directory\") {|s|\n  (cmd_opts[:plugin_dirs] ||= default_opts[:plugin_dirs]) << s\n}\n\nop.on('-I PATH', \"add library path\") {|s|\n  $LOAD_PATH << s\n}\n\nop.on('-r NAME', \"load library\") {|s|\n  (cmd_opts[:libs] ||= []) << s\n}\n\nop.on('-d', '--daemon PIDFILE', \"daemonize fluent process\") {|s|\n  cmd_opts[:daemonize] = s\n}\n\nop.on('--under-supervisor', \"run fluent worker under supervisor (this option is NOT for users)\") {\n  cmd_opts[:supervise] = false\n}\n\nop.on('--no-supervisor', \"run fluent worker without supervisor\") {\n  cmd_opts[:supervise] = false\n  cmd_opts[:standalone_worker] = true\n}\n\nop.on('--workers NUM', \"specify the number of workers under supervisor\") { |i|\n  cmd_opts[:workers] = i.to_i\n}\n\nop.on('--user USER', \"change user\") {|s|\n  cmd_opts[:chuser] = s\n}\n\nop.on('--group GROUP', \"change group\") {|s|\n  cmd_opts[:chgroup] = s\n}\n\nop.on('--umask UMASK', \"change umask\") {|s|\n  cmd_opts[:chumask] = s\n}\n\nop.on('-o', '--log PATH', \"log file path\") {|s|\n  cmd_opts[:log_path] = s\n}\n\nop.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|\n  if Fluent::Log::LOG_ROTATE_AGE.include?(age)\n    cmd_opts[:log_rotate_age] = age\n  else\n    begin\n      cmd_opts[:log_rotate_age] = Integer(age)\n    rescue TypeError, ArgumentError\n      usage \"log-rotate-age should be #{ROTATE_AGE.join(', ')} or a number\"\n    end\n  end\n}\n\nop.on('--log-rotate-size BYTES', 'sets the byte size to rotate log files') {|s|\n  cmd_opts[:log_rotate_size] = s.to_i\n}\n\nop.on('--log-event-verbose', 'enable log events during process startup/shutdown') {|b|\n  cmd_opts[:log_event_verbose] = b\n}\n\nop.on('-i', '--inline-config CONFIG_STRING', \"inline config which is appended to the config file on-the-fly\") {|s|\n  cmd_opts[:inline_config] = s\n}\n\nop.on('--emit-error-log-interval SECONDS', \"suppress interval seconds of emit error logs\") {|s|\n  cmd_opts[:suppress_interval] = s.to_i\n}\n\nop.on('--suppress-repeated-stacktrace [VALUE]', \"suppress repeated stacktrace\", TrueClass) {|b|\n  b = true if b.nil?\n  cmd_opts[:suppress_repeated_stacktrace] = b\n}\n\nop.on('--without-source', \"invoke a fluentd without input plugins\", TrueClass) {|b|\n  cmd_opts[:without_source] = b\n}\n\nunless Fluent.windows?\n  op.on('--with-source-only', \"Invoke a fluentd only with input plugins. The data is stored in a temporary buffer. Send SIGWINCH to cancel this mode and process the data (Not supported on Windows).\", TrueClass) {|b|\n    cmd_opts[:with_source_only] = b\n  }\nend\n\nop.on('--config-file-type VALUE', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|\n  if (s == 'yaml') || (s == 'yml')\n    cmd_opts[:config_file_type] = s.to_sym\n  elsif (s == 'guess')\n    cmd_opts[:config_file_type] = s.to_sym\n  else\n    usage '--config-file-type accepts yaml/yml or guess'\n  end\n}\n\nop.on('--use-v1-config', \"Use v1 configuration format (default)\", TrueClass) {|b|\n  cmd_opts[:use_v1_config] = b\n}\n\nop.on('--use-v0-config', \"Use v0 configuration format\", TrueClass) {|b|\n  cmd_opts[:use_v1_config] = !b\n}\n\nop.on('--strict-config-value', \"Parse config values strictly\", TrueClass) {|b|\n  cmd_opts[:strict_config_value] = b\n}\n\nop.on('--enable-input-metrics', \"[DEPRECATED] Enable input plugin metrics on fluentd\", TrueClass) {|b|\n  cmd_opts[:enable_input_metrics] = b\n}\n\nop.on('--disable-input-metrics', \"Disable input plugin metrics on fluentd\", FalseClass) {|b|\n  cmd_opts[:enable_input_metrics] = b\n}\n\nop.on('--enable-size-metrics', \"Enable plugin record size metrics on fluentd\", TrueClass) {|b|\n  cmd_opts[:enable_size_metrics] = b\n}\n\nop.on('-v', '--verbose', \"increase verbose level (-v: debug, -vv: trace)\", TrueClass) {|b|\n  return unless b\n  cur_level = cmd_opts.fetch(:log_level, default_opts[:log_level])\n  cmd_opts[:log_level] = [cur_level - 1, Fluent::Log::LEVEL_TRACE].max\n}\n\nop.on('-q', '--quiet', \"decrease verbose level (-q: warn, -qq: error)\", TrueClass) {|b|\n  return unless b\n  cur_level = cmd_opts.fetch(:log_level, default_opts[:log_level])\n  cmd_opts[:log_level] = [cur_level + 1, Fluent::Log::LEVEL_ERROR].min\n}\n\nop.on('--suppress-config-dump', \"suppress config dumping when fluentd starts\", TrueClass) {|b|\n  cmd_opts[:suppress_config_dump] = b\n}\n\nop.on('-g', '--gemfile GEMFILE', \"Gemfile path\") {|s|\n  cmd_opts[:gemfile] = s\n}\n\nop.on('-G', '--gem-path GEM_INSTALL_PATH', \"Gemfile install path (default: $(dirname $gemfile)/vendor/bundle)\") {|s|\n  cmd_opts[:gem_install_path] = s\n}\n\nop.on('--conf-encoding ENCODING', \"specify configuration file encoding\") { |s|\n  cmd_opts[:conf_encoding] = s\n}\n\nop.on('--disable-shared-socket', \"Don't open shared socket for multiple workers\") { |b|\n  cmd_opts[:disable_shared_socket] = b\n}\n\nif Fluent.windows?\n  cmd_opts.merge!(\n    :winsvc_name => 'fluentdwinsvc',\n    :winsvc_display_name => 'Fluentd Windows Service',\n    :winsvc_desc => 'Fluentd is an event collector system.',\n  )\n\n  op.on('-x', '--signame INTSIGNAME', \"an object name which is used for Windows Service signal (Windows only)\") {|s|\n    cmd_opts[:signame] = s\n  }\n\n  op.on('--reg-winsvc MODE', \"install/uninstall as Windows Service. (i: install, u: uninstall) (Windows only)\") {|s|\n    cmd_opts[:regwinsvc] = s\n  }\n\n  op.on('--[no-]reg-winsvc-auto-start', \"Automatically start the Windows Service at boot. (only effective with '--reg-winsvc i') (Windows only)\") {|s|\n    cmd_opts[:regwinsvcautostart] = s\n  }\n\n  op.on('--[no-]reg-winsvc-delay-start', \"Automatically start the Windows Service at boot with delay. (only effective with '--reg-winsvc i' and '--reg-winsvc-auto-start') (Windows only)\") {|s|\n    cmd_opts[:regwinsvcdelaystart] = s\n  }\n\n  op.on('--reg-winsvc-fluentdopt OPTION', \"specify fluentd option parameters for Windows Service. (Windows only)\") {|s|\n    cmd_opts[:fluentdopt] = s\n  }\n\n  op.on('--winsvc-name NAME', \"The Windows Service name to run as (Windows only)\") {|s|\n    cmd_opts[:winsvc_name] = s\n  }\n\n  op.on('--winsvc-display-name DISPLAY_NAME', \"The Windows Service display name (Windows only)\") {|s|\n    cmd_opts[:winsvc_display_name] = s\n  }\n\n  op.on('--winsvc-desc DESC', \"The Windows Service description (Windows only)\") {|s|\n    cmd_opts[:winsvc_desc] = s\n  }\nend\n\n\nsingleton_class.module_eval do\n  define_method(:usage) do |msg|\n    puts op.to_s\n    puts \"error: #{msg}\" if msg\n    exit 1\n  end\nend\n\nbegin\n  rest = op.parse(ARGV)\n\n  if rest.length != 0\n    usage nil\n  end\nrescue\n  usage $!.to_s\nend\n\nopts = default_opts.merge(cmd_opts)\n\n##\n## Bundler injection\n#\nif ENV['FLUENTD_DISABLE_BUNDLER_INJECTION'] != '1' && gemfile = opts[:gemfile]\n  ENV['BUNDLE_GEMFILE'] = gemfile\n  if path = opts[:gem_install_path]\n    ENV['BUNDLE_PATH'] = path\n  else\n    ENV['BUNDLE_PATH'] = File.expand_path(File.join(File.dirname(gemfile), 'vendor/bundle'))\n  end\n  ENV['FLUENTD_DISABLE_BUNDLER_INJECTION'] = '1'\n  load File.expand_path(File.join(File.dirname(__FILE__), 'bundler_injection.rb'))\nend\n\nif setup_path = opts[:setup_path]\n  require 'fileutils'\n  FileUtils.mkdir_p File.join(setup_path, \"plugin\")\n  confpath = File.join(setup_path, \"fluent.conf\")\n  if File.exist?(confpath)\n    puts \"#{confpath} already exists.\"\n  else\n    File.open(confpath, \"w\") {|f|\n      conf = File.read File.join(File.dirname(__FILE__), \"..\", \"..\", \"..\", \"fluent.conf\")\n      f.write conf\n    }\n    puts \"Installed #{confpath}.\"\n  end\n  exit 0\nend\n\nearly_exit = false\nstart_service = false\nif winsvcinstmode = opts[:regwinsvc]\n  require 'fileutils'\n  require \"win32/service\"\n  require \"win32/registry\"\n  include Win32\n\n  case winsvcinstmode\n  when 'i'\n    binary_path = File.join(File.dirname(__FILE__), \"..\")\n    ruby_path = ServerEngine.ruby_bin_path\n    start_type = Service::DEMAND_START\n    if opts[:regwinsvcautostart]\n      start_type = Service::AUTO_START\n      start_service = true\n    end\n\n\n    Service.create(\n      service_name: opts[:winsvc_name],\n      host: nil,\n      service_type: Service::WIN32_OWN_PROCESS,\n      description: opts[:winsvc_desc],\n      start_type: start_type,\n      error_control: Service::ERROR_NORMAL,\n      binary_path_name: \"\\\"#{ruby_path}\\\" -C \\\"#{binary_path}\\\"  winsvc.rb --service-name #{opts[:winsvc_name]}\",\n      load_order_group: \"\",\n      dependencies: [\"\"],\n      display_name: opts[:winsvc_display_name]\n    )\n\n    if opts[:regwinsvcdelaystart]\n      Service.configure(\n        service_name: opts[:winsvc_name],\n        delayed_start: true\n      )\n    end\n  when 'u'\n    if Service.status(opts[:winsvc_name]).current_state != 'stopped'\n      begin\n        Service.stop(opts[:winsvc_name])\n      rescue => ex\n        puts \"Warning: Failed to stop service: \", ex\n      end\n    end\n    Service.delete(opts[:winsvc_name])\n  else\n    # none\n  end\n  early_exit = true\nend\n\nif fluentdopt = opts[:fluentdopt]\n  require \"win32/registry\"\n  Win32::Registry::HKEY_LOCAL_MACHINE.open(\"SYSTEM\\\\CurrentControlSet\\\\Services\\\\#{opts[:winsvc_name]}\", Win32::Registry::KEY_ALL_ACCESS) do |reg|\n    reg['fluentdopt', Win32::Registry::REG_SZ] = fluentdopt\n  end\n  early_exit = true\nend\n\nif start_service\n  Service.start(opts[:winsvc_name])\nend\n\nexit 0 if early_exit\n\nif opts[:supervise]\n  supervisor = Fluent::Supervisor.new(cmd_opts)\n  supervisor.configure(supervisor: true)\n  supervisor.run_supervisor(dry_run: opts[:dry_run])\nelse\n  if opts[:standalone_worker] && opts[:workers] && opts[:workers] > 1\n    puts \"Error: multi workers is not supported with --no-supervisor\"\n    exit 2\n  end\n  worker = Fluent::Supervisor.new(cmd_opts)\n  worker.configure\n\n  if opts[:daemonize] && opts[:standalone_worker]\n    require 'fluent/daemonizer'\n    args = ARGV.dup\n    i = args.index('--daemon')\n    args.delete_at(i + 1)          # value of --daemon\n    args.delete_at(i)              # --daemon itself\n\n    Fluent::Daemonizer.daemonize(opts[:daemonize], args) do\n      worker.run_worker\n    end\n  else\n    worker.run_worker\n  end\nend\n"
  },
  {
    "path": "lib/fluent/command/plugin_config_formatter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire \"erb\"\nrequire \"optparse\"\nrequire \"pathname\"\nrequire \"fluent/plugin\"\nrequire \"fluent/env\"\nrequire \"fluent/engine\"\nrequire \"fluent/system_config\"\nrequire \"fluent/config/element\"\nrequire 'fluent/version'\n\nclass FluentPluginConfigFormatter\n\n  AVAILABLE_FORMATS = [:markdown, :txt, :json]\n  SUPPORTED_TYPES = [\n    \"input\", \"output\", \"filter\",\n    \"buffer\", \"parser\", \"formatter\", \"storage\",\n    \"service_discovery\"\n  ]\n\n  DOCS_BASE_URL = \"https://docs.fluentd.org/v/1.0\"\n  DOCS_PLUGIN_HELPER_BASE_URL = \"#{DOCS_BASE_URL}/plugin-helper-overview/\"\n\n  def initialize(argv = ARGV)\n    @argv = argv\n\n    @compact = false\n    @format = :markdown\n    @verbose = false\n    @libs = []\n    @plugin_dirs = []\n    @table = false\n    @options = {}\n\n    prepare_option_parser\n  end\n\n  def call\n    parse_options!\n    init_libraries\n    @plugin = Fluent::Plugin.__send__(\"new_#{@plugin_type}\", @plugin_name)\n    dumped_config = {}\n    if @plugin.class.respond_to?(:plugin_helpers)\n      dumped_config[:plugin_helpers] = @plugin.class.plugin_helpers\n    end\n    @plugin.class.ancestors.reverse_each do |plugin_class|\n      next unless plugin_class.respond_to?(:dump_config_definition)\n      unless @verbose\n        next if /::PluginHelper::/.match?(plugin_class.name)\n      end\n      dumped_config_definition = plugin_class.dump_config_definition\n      dumped_config[plugin_class.name] = dumped_config_definition unless dumped_config_definition.empty?\n    end\n    case @format\n    when :txt\n      puts dump_txt(dumped_config)\n    when :markdown\n      puts dump_markdown(dumped_config)\n    when :json\n      puts dump_json(dumped_config)\n    end\n  end\n\n  private\n\n  def dump_txt(dumped_config)\n    dumped = \"\"\n    plugin_helpers = dumped_config.delete(:plugin_helpers)\n    if plugin_helpers && !plugin_helpers.empty?\n      dumped << \"helpers: #{plugin_helpers.join(',')}\\n\"\n    end\n    if @verbose\n      dumped_config.each do |name, config|\n        dumped << \"#{name}\\n\"\n        dumped << dump_section_txt(config)\n      end\n    else\n      configs = dumped_config.values\n      root_section = configs.shift\n      configs.each do |config|\n        root_section.update(config)\n      end\n      dumped << dump_section_txt(root_section)\n    end\n    dumped\n  end\n\n  def dump_section_txt(base_section, level = 0)\n    dumped = \"\"\n    indent = \" \" * level\n    if base_section[:section]\n      sections = []\n      params = base_section\n    else\n      sections, params = base_section.partition {|_name, value| value[:section] }\n    end\n    params.each do |name, config|\n      next if name == :section\n      dumped << \"#{indent}#{name}: #{config[:type]}: (#{config[:default].inspect})\"\n      dumped << \" # #{config[:description]}\" if config.key?(:description)\n      dumped << \"\\n\"\n    end\n    sections.each do |section_name, sub_section|\n      required = sub_section.delete(:required)\n      multi = sub_section.delete(:multi)\n      alias_name = sub_section.delete(:alias)\n      required_label = required ? \"required\" : \"optional\"\n      multi_label = multi ? \"multiple\" : \"single\"\n      alias_label = \"alias: #{alias_name}\"\n      dumped << \"#{indent}<#{section_name}>: #{required_label}, #{multi_label}\"\n      if alias_name\n        dumped << \", #{alias_label}\\n\"\n      else\n        dumped << \"\\n\"\n      end\n      sub_section.delete(:section)\n      dumped << dump_section_txt(sub_section, level + 1)\n    end\n    dumped\n  end\n\n  def dump_markdown(dumped_config)\n    dumped = \"\"\n    plugin_helpers = dumped_config.delete(:plugin_helpers)\n    if plugin_helpers && !plugin_helpers.empty?\n      dumped = \"## Plugin helpers\\n\\n\"\n      plugin_helpers.each do |plugin_helper|\n        dumped << \"* #{plugin_helper_markdown_link(plugin_helper)}\\n\"\n      end\n      dumped << \"\\n\"\n    end\n    dumped_config.each do |name, config|\n      if name == @plugin.class.name\n        dumped << \"## #{name}\\n\\n\"\n        dumped << dump_section_markdown(config)\n      else\n        dumped << \"* See also: #{plugin_overview_markdown_link(name)}\\n\\n\"\n      end\n    end\n    dumped\n  end\n\n  def dump_section_markdown(base_section, level = 0)\n    dumped = \"\"\n    if base_section[:section]\n      sections = []\n      params = base_section\n    else\n      sections, params = base_section.partition {|_name, value| value[:section] }\n    end\n    if @table && (not params.empty?)\n      dumped << \"### Configuration\\n\\n\"\n      dumped << \"|parameter|type|description|default|\\n\"\n      dumped << \"|---|---|---|---|\\n\"\n    end\n    params.each do |name, config|\n      next if name == :section\n      template_name = if @compact\n                        \"param.md-compact.erb\"\n                      elsif @table\n                        \"param.md-table.erb\"\n                      else\n                        \"param.md.erb\"\n                      end\n      template = template_path(template_name).read\n      dumped <<\n        if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+\n          ERB.new(template, trim_mode: \"-\")\n        else\n          ERB.new(template, nil, \"-\")\n        end.result(binding)\n    end\n    dumped << \"\\n\"\n    sections.each do |section_name, sub_section|\n      required = sub_section.delete(:required)\n      multi = sub_section.delete(:multi)\n      alias_name = sub_section.delete(:alias)\n      sub_section.delete(:section)\n      template = template_path(\"section.md.erb\").read\n      dumped <<\n        if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+\n          ERB.new(template, trim_mode: \"-\")\n        else\n          ERB.new(template, nil, \"-\")\n        end.result(binding)\n    end\n    dumped\n  end\n\n  def dump_json(dumped_config)\n    if @compact\n      JSON.generate(dumped_config)\n    else\n      JSON.pretty_generate(dumped_config)\n    end\n  end\n\n  def plugin_helper_url(plugin_helper)\n    \"#{DOCS_PLUGIN_HELPER_BASE_URL}api-plugin-helper-#{plugin_helper}\"\n  end\n\n  def plugin_helper_markdown_link(plugin_helper)\n    \"[#{plugin_helper}](#{plugin_helper_url(plugin_helper)})\"\n  end\n\n  def plugin_overview_url(class_name)\n    plugin_type = class_name.slice(/::(\\w+)\\z/, 1).downcase\n    \"#{DOCS_BASE_URL}/#{plugin_type}#overview\"\n  end\n\n  def plugin_overview_markdown_link(class_name)\n    plugin_type = class_name.slice(/::(\\w+)\\z/, 1)\n    \"[#{plugin_type} Plugin Overview](#{plugin_overview_url(class_name)})\"\n  end\n\n  def usage(message = nil)\n    puts @parser.to_s\n    puts\n    puts \"Error: #{message}\" if message\n    exit(false)\n  end\n\n  def prepare_option_parser\n    @parser = OptionParser.new\n    @parser.version = Fluent::VERSION\n    @parser.banner = <<BANNER\nUsage: #{$0} [options] <type> <name>\n\nOutput plugin config definitions\n\nArguments:\n\\ttype: #{SUPPORTED_TYPES.join(\",\")}\n\\tname: registered plugin name\n\nOptions:\nBANNER\n    @parser.on(\"--verbose\", \"Be verbose\") do\n      @verbose = true\n    end\n    @parser.on(\"-c\", \"--compact\", \"Compact output\") do\n      @compact = true\n    end\n    @parser.on(\"-f\", \"--format=FORMAT\", \"Specify format. (#{AVAILABLE_FORMATS.join(',')})\") do |s|\n      format = s.to_sym\n      usage(\"Unsupported format: #{s}\") unless AVAILABLE_FORMATS.include?(format)\n      @format = format\n    end\n    @parser.on(\"-I PATH\", \"Add PATH to $LOAD_PATH\") do |s|\n      $LOAD_PATH.unshift(s)\n    end\n    @parser.on(\"-r NAME\", \"Load library\") do |s|\n      @libs << s\n    end\n    @parser.on(\"-p\", \"--plugin=DIR\", \"Add plugin directory\") do |s|\n      @plugin_dirs << s\n    end\n    @parser.on(\"-t\", \"--table\", \"Use table syntax to dump parameters\") do\n      @table = true\n    end\n  end\n\n  def parse_options!\n    @parser.parse!(@argv)\n\n    raise \"Must specify plugin type and name\" unless @argv.size == 2\n\n    @plugin_type, @plugin_name = @argv\n    @options = {\n      compact: @compact,\n      format: @format,\n      verbose: @verbose,\n    }\n  rescue => e\n    usage(e)\n  end\n\n  def init_libraries\n    @libs.each do |lib|\n      require lib\n    end\n\n    @plugin_dirs.each do |dir|\n      if Dir.exist?(dir)\n        dir = File.expand_path(dir)\n        Fluent::Plugin.add_plugin_dir(dir)\n      end\n    end\n  end\n\n  def template_path(name)\n    (Pathname(__dir__) + \"../../../templates/plugin_config_formatter/#{name}\").realpath\n  end\nend\n"
  },
  {
    "path": "lib/fluent/command/plugin_generator.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire \"optparse\"\nrequire \"pathname\"\nrequire \"fileutils\"\nrequire \"erb\"\nrequire \"open-uri\"\n\nrequire \"fluent/registry\"\nrequire 'fluent/version'\n\nclass FluentPluginGenerator\n  attr_reader :type, :name\n  attr_reader :license_name\n\n  SUPPORTED_TYPES = [\"input\", \"output\", \"filter\", \"parser\", \"formatter\", \"storage\"]\n\n  def initialize(argv = ARGV)\n    @argv = argv\n    @parser = prepare_parser\n\n    @license_name = \"Apache-2.0\"\n    @overwrite_all = false\n  end\n\n  def call\n    parse_options!\n    FileUtils.mkdir_p(gem_name)\n    Dir.chdir(gem_name) do\n      copy_license\n      template_directory.find do |path|\n        next if path.directory?\n        dest_dir = path.dirname.sub(/\\A#{Regexp.quote(template_directory.to_s)}\\/?/, \"\")\n        dest_file = dest_filename(path)\n        if path.extname == \".erb\"\n          if path.fnmatch?(\"*/plugin/*\")\n            next unless path.basename.fnmatch?(\"*#{type}*\")\n          end\n          template(path, dest_dir + dest_file)\n        else\n          file(path, dest_dir + dest_file)\n        end\n      end\n      pid = spawn(\"git\", \"init\", \".\")\n      Process.wait(pid)\n    end\n  end\n\n  private\n\n  def template_directory\n    (Pathname(__dir__) + \"../../../templates/new_gem\").realpath\n  end\n\n  def template_file(filename)\n    template_directory + filename\n  end\n\n  def template(source, dest)\n    dest.dirname.mkpath\n    contents =\n      if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+\n        ERB.new(source.read, trim_mode: \"-\")\n      else\n        ERB.new(source.read, nil, \"-\")\n      end.result(binding)\n    label = create_label(dest, contents)\n    puts \"\\t#{label} #{dest}\"\n    if label == \"conflict\"\n      return unless overwrite?(dest)\n    end\n    File.write(dest, contents)\n  end\n\n  def file(source, dest)\n    label = create_label(dest, source.read)\n    puts \"\\t#{label} #{dest}\"\n    if label == \"conflict\"\n      return unless overwrite?(dest)\n    end\n    FileUtils.cp(source, dest)\n  end\n\n  def prepare_parser\n    @parser = OptionParser.new\n    @parser.version = Fluent::VERSION\n    @parser.banner = <<BANNER\nUsage: fluent-plugin-generate [options] <type> <name>\n\nGenerate a project skeleton for creating a Fluentd plugin\n\nArguments:\n\\ttype: #{SUPPORTED_TYPES.join(\",\")}\n\\tname: Your plugin name (fluent-plugin- prefix will be added to <name>)\n\nOptions:\nBANNER\n\n    @parser.on(\"--[no-]license=NAME\", \"Specify license name (default: Apache-2.0)\") do |v|\n      @license_name = v || \"no-license\"\n    end\n    @parser\n  end\n\n  def parse_options!\n    @parser.parse!(@argv)\n    unless @argv.size == 2\n      raise ArgumentError, \"Missing arguments\"\n    end\n    @type, @name = @argv\n  rescue => e\n    usage(\"#{e.class}:#{e.message}\")\n  end\n\n  def usage(message = \"\")\n    puts message\n    puts\n    puts @parser.help\n    exit(false)\n  end\n\n  def user_name\n    v = `git config --get user.name`.chomp\n    v.empty? ? \"TODO: Write your name\" : v\n  end\n\n  def user_email\n    v = `git config --get user.email`.chomp\n    v.empty? ? \"TODO: Write your email\" : v\n  end\n\n  def gem_name\n    \"fluent-plugin-#{dash_name}\"\n  end\n\n  def plugin_name\n    underscore_name\n  end\n\n  def gem_file_path\n    File.expand_path(File.join(File.dirname(__FILE__),\n                               \"../../../\",\n                               \"Gemfile\"))\n  end\n\n  def lock_file_path\n    File.expand_path(File.join(File.dirname(__FILE__),\n                               \"../../../\",\n                               \"Gemfile.lock\"))\n  end\n\n  def locked_gem_version(gem_name)\n    if File.exist?(lock_file_path)\n      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)\n      d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version\n    else\n      # fallback even though Fluentd is installed without bundler\n      Gem::Specification.find_by_name(gem_name).version.version\n    end\n  end\n\n  def rake_version\n    locked_gem_version(\"rake\")\n  end\n\n  def test_unit_version\n    locked_gem_version(\"test-unit\")\n  end\n\n  def bundler_version\n    if File.exist?(lock_file_path)\n      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)\n      d.locked_gems.bundler_version.version\n    else\n      # fallback even though Fluentd is installed without bundler\n      Gem::Specification.find_by_name(\"bundler\").version.version\n    end\n  end\n\n  def class_name\n    \"#{capitalized_name}#{type.capitalize}\"\n  end\n\n  def plugin_filename\n    case type\n    when \"input\"\n      \"in_#{underscore_name}.rb\"\n    when \"output\"\n      \"out_#{underscore_name}.rb\"\n    else\n      \"#{type}_#{underscore_name}.rb\"\n    end\n  end\n\n  def test_filename\n    case type\n    when \"input\"\n      \"test_in_#{underscore_name}.rb\"\n    when \"output\"\n      \"test_out_#{underscore_name}.rb\"\n    else\n      \"test_#{type}_#{underscore_name}.rb\"\n    end\n  end\n\n  def dest_filename(path)\n    case path.to_s\n    when %r!\\.gemspec!\n      \"#{gem_name}.gemspec\"\n    when %r!lib/fluent/plugin!\n      plugin_filename\n    when %r!test/plugin!\n      test_filename\n    else\n      path.basename.sub_ext(\"\")\n    end\n  end\n\n  def capitalized_name\n    @capitalized_name ||= name.split(/[-_]/).map(&:capitalize).join\n  end\n\n  def underscore_name\n    @underscore_name ||= name.tr(\"-\", \"_\")\n  end\n\n  def dash_name\n    @dash_name ||= name.tr(\"_\", \"-\")\n  end\n\n  def preamble\n    @license.preamble(user_name)\n  end\n\n  def copy_license\n    # in gem_name directory\n    return unless license_name\n    puts \"License: #{license_name}\"\n    license_class = self.class.lookup_license(license_name)\n    @license = license_class.new\n    Pathname(\"LICENSE\").write(@license.text) unless @license.text.empty?\n  rescue Fluent::ConfigError\n    usage(\"Unknown license: #{license_name}\")\n  rescue => ex\n    usage(\"#{ex.class}: #{ex.message}\")\n  end\n\n  def create_label(dest, contents)\n    if dest.exist?\n      if dest.read == contents\n        \"identical\"\n      else\n        \"conflict\"\n      end\n    else\n      \"create\"\n    end\n  end\n\n  def overwrite?(dest)\n    return true if @overwrite_all\n    loop do\n      print \"Overwrite #{dest}? (enter \\\"h\\\" for help) [Ynaqh]\"\n      answer = $stdin.gets.chomp\n      return true if /\\Ay\\z/i =~ answer || answer.empty?\n      case answer\n      when \"n\"\n        return false\n      when \"a\"\n        @overwrite_all = true\n        return true\n      when \"q\"\n        exit\n      when \"h\"\n        puts <<HELP\n\\tY - yes, overwrite\n\\tn - no, do not overwrite\n\\ta - all, overwrite this and all others\n\\tq - quit, abort\n\\th - help, show this help\nHELP\n      end\n      puts \"Retrying...\"\n    end\n  end\n\n  class NoLicense\n    attr_reader :name, :full_name, :text\n\n    def initialize\n      @name = \"\"\n      @full_name = \"\"\n      @text = \"\"\n    end\n\n    def preamble(usename)\n      \"\"\n    end\n  end\n\n  class ApacheLicense\n    LICENSE_URL = \"http://www.apache.org/licenses/LICENSE-2.0.txt\"\n\n    attr_reader :text\n\n    def initialize\n      @text = \"\"\n      @preamble_source = \"\"\n      @preamble = nil\n      uri = URI.parse(LICENSE_URL)\n      uri.open do |io|\n        @text = io.read\n      end\n      @preamble_source = @text[/^(\\s*Copyright.+)/m, 1]\n    end\n\n    def name\n      \"Apache-2.0\"\n    end\n\n    def full_name\n      \"Apache License, Version 2.0\"\n    end\n\n    def preamble(user_name)\n      @preamble ||= @preamble_source.dup.tap do |source|\n        source.gsub!(/\\[yyyy\\]/, \"#{Date.today.year}-\")\n        source.gsub!(/\\[name of copyright owner\\]/, user_name)\n        source.gsub!(/^ {2}|^$/, \"#\")\n        source.chomp!\n      end\n    end\n  end\n\n  LICENSE_REGISTRY = Fluent::Registry.new(:license, \"\")\n\n  def self.register_license(license, klass)\n    LICENSE_REGISTRY.register(license, klass)\n  end\n\n  def self.lookup_license(license)\n    LICENSE_REGISTRY.lookup(license)\n  end\n\n  {\n    \"no-license\" => NoLicense,\n    \"Apache-2.0\" => ApacheLicense\n  }.each do |license, klass|\n    register_license(license, klass)\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/call_super_mixin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Compat\n    module CallSuperMixin\n      # This mixin is to prepend to 3rd party plugins of v0.12 APIs.\n      # In past, there were not strong rule to call super in #start, #before_shutdown and #shutdown.\n      # But v0.14 API requires to call super in these methods to setup/teardown plugin helpers and others.\n      # This mixin prepends method calls to call super forcedly if checker returns false (it shows Fluent::Plugin::Base#methods wasn't called)\n\n      def self.prepended(klass)\n        @@_super_start ||= {}\n        @@_super_before_shutdown ||= {}\n        @@_super_shutdown ||= {}\n\n        # ancestors[0]: this module\n        # ancestors[1]: prepended class (plugin itself)\n        method_search = ->(ancestors, method){\n          closest = ancestors[2, ancestors.size - 2].index{|m| m.method_defined?(method) }\n          ancestors[2 + closest].instance_method(method)\n        }\n        @@_super_start[klass]           = method_search.call(klass.ancestors, :start) # this returns Fluent::Compat::*#start (or helpers on it)\n        @@_super_before_shutdown[klass] = method_search.call(klass.ancestors, :before_shutdown)\n        @@_super_shutdown[klass]        = method_search.call(klass.ancestors, :shutdown)\n      end\n\n      def start\n        super\n        unless self.started?\n          @@_super_start[self.class].bind_call(self)\n          # #super will reset logdev (especially in test), so this warn should be after calling it\n          log.warn \"super was not called in #start: called it forcedly\", plugin: self.class\n        end\n      end\n\n      def before_shutdown\n        super\n        unless self.before_shutdown?\n          log.warn \"super was not called in #before_shutdown: calling it forcedly\", plugin: self.class\n          @@_super_before_shutdown[self.class].bind_call(self)\n        end\n      end\n\n      def stop\n        klass = self.class\n        @@_super_start.delete(klass)\n        @@_super_before_shutdown.delete(klass)\n        @@_super_shutdown.delete(klass)\n\n        super\n      end\n\n      def shutdown\n        super\n        unless self.shutdown?\n          log.warn \"super was not called in #shutdown: calling it forcedly\", plugin: self.class\n          @@_super_shutdown[self.class].bind_call(self)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/detach_process_mixin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Compat\n    module DetachProcessMixin\n      def detach_process\n        log.warn \"#{__method__} is not supported in this version. ignored.\"\n        yield\n      end\n    end\n\n    module DetachMultiProcessMixin\n      def detach_multi_process\n        log.warn \"#{__method__} is not supported in this version. ignored.\"\n        yield\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/exec_util.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'msgpack'\nrequire 'json'\nrequire 'yajl'\n\nrequire 'fluent/engine'\nrequire 'fluent/plugin'\nrequire 'fluent/parser'\n\nmodule Fluent\n  module Compat\n    module ExecUtil\n      SUPPORTED_FORMAT = {\n        'tsv' => :tsv,\n        'json' => :json,\n        'msgpack' => :msgpack,\n      }\n\n      class Parser\n        def initialize(on_message)\n          @on_message = on_message\n        end\n      end\n\n      class TextParserWrapperParser < Parser\n        def initialize(conf, on_message)\n          @parser = Plugin.new_parser(conf['format'])\n          @parser.configure(conf)\n          super(on_message)\n        end\n\n        def call(io)\n          io.each_line(&method(:each_line))\n        end\n\n        def each_line(line)\n          line.chomp!\n          @parser.parse(line) { |time, record|\n            @on_message.call(record, time)\n          }\n        end\n      end\n\n      class TSVParser < Parser\n        def initialize(keys, on_message)\n          @keys = keys\n          super(on_message)\n        end\n\n        def call(io)\n          io.each_line(&method(:each_line))\n        end\n\n        def each_line(line)\n          line.chomp!\n          vals = line.split(\"\\t\")\n\n          record = Hash[@keys.zip(vals)]\n\n          @on_message.call(record)\n        end\n      end\n\n      class JSONParser < Parser\n        def call(io)\n          y = Yajl::Parser.new\n          y.on_parse_complete = @on_message\n          y.parse(io)\n        end\n      end\n\n      class MessagePackParser < Parser\n        def call(io)\n          @u = Fluent::MessagePackFactory.msgpack_unpacker(io)\n          begin\n            @u.each(&@on_message)\n          rescue EOFError\n          end\n        end\n      end\n\n      class Formatter\n      end\n\n      class TSVFormatter < Formatter\n        def initialize(in_keys)\n          @in_keys = in_keys\n          super()\n        end\n\n        def call(record, out)\n          last = @in_keys.length-1\n          for i in 0..last\n            key = @in_keys[i]\n            out << record[key].to_s\n            out << \"\\t\" if i != last\n          end\n          out << \"\\n\"\n        end\n      end\n\n      class JSONFormatter < Formatter\n        def call(record, out)\n          out << JSON.generate(record) << \"\\n\"\n        end\n      end\n\n      class MessagePackFormatter < Formatter\n        def call(record, out)\n          record.to_msgpack(out)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/file_util.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Compat\n    module FileUtil\n      # Check file is writable if file exists\n      # Check directory is writable if file does not exist\n      #\n      # @param [String] path File path\n      # @return [Boolean] file is writable or not\n      def writable?(path)\n        return false if File.directory?(path)\n        return File.writable?(path) if File.exist?(path)\n\n        dirname = File.dirname(path)\n        return false if !File.directory?(dirname)\n        File.writable?(dirname)\n      end\n      module_function :writable?\n\n      # Check file is writable in conjunction with mkdir_p(dirname(path))\n      #\n      # @param [String] path File path\n      # @return [Boolean] file writable or not\n      def writable_p?(path)\n        return false if File.directory?(path)\n        return File.writable?(path) if File.exist?(path)\n\n        dirname = File.dirname(path)\n        until File.exist?(dirname)\n          dirname = File.dirname(dirname)\n        end\n\n        return false if !File.directory?(dirname)\n        File.writable?(dirname)\n      end\n      module_function :writable_p?\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/filter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/filter'\nrequire 'fluent/compat/call_super_mixin'\nrequire 'fluent/compat/formatter_utils'\nrequire 'fluent/compat/parser_utils'\n\nmodule Fluent\n  module Compat\n    class Filter < Fluent::Plugin::Filter\n      # TODO: warn when deprecated\n\n      helpers_internal :inject\n\n      def initialize\n        super\n        unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)\n          self.class.prepend Fluent::Compat::CallSuperMixin\n        end\n      end\n\n      def configure(conf)\n        ParserUtils.convert_parser_conf(conf)\n        FormatterUtils.convert_formatter_conf(conf)\n\n        super\n      end\n\n      # These definitions are to get instance methods of superclass of 3rd party plugins\n      # to make it sure to call super\n      def start\n        super\n\n        if instance_variable_defined?(:@formatter) && @inject_config\n          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)\n            if @formatter.respond_to?(:owner) && !@formatter.owner\n              @formatter.owner = self\n              @formatter.singleton_class.prepend FormatterUtils::InjectMixin\n            end\n          end\n        end\n      end\n\n      def before_shutdown\n        super\n      end\n\n      def shutdown\n        super\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/formatter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/formatter'\nrequire 'fluent/compat/handle_tag_and_time_mixin'\nrequire 'fluent/compat/structured_format_mixin'\n\nrequire 'fluent/plugin/formatter_out_file'\nrequire 'fluent/plugin/formatter_stdout'\nrequire 'fluent/plugin/formatter_json'\nrequire 'fluent/plugin/formatter_hash'\nrequire 'fluent/plugin/formatter_msgpack'\nrequire 'fluent/plugin/formatter_ltsv'\nrequire 'fluent/plugin/formatter_csv'\nrequire 'fluent/plugin/formatter_single_value'\n\nmodule Fluent\n  module Compat\n    class Formatter < Fluent::Plugin::Formatter\n      # TODO: warn when deprecated\n    end\n\n    module TextFormatter\n      def self.register_template(type, template)\n        # TODO: warn when deprecated to use Plugin.register_formatter directly\n        if template.is_a?(Class)\n          Fluent::Plugin.register_formatter(type, template)\n        elsif template.respond_to?(:call) && template.arity == 3 # Proc.new { |tag, time, record| }\n          Fluent::Plugin.register_formatter(type, Proc.new { ProcWrappedFormatter.new(template) })\n        elsif template.respond_to?(:call)\n          Fluent::Plugin.register_formatter(type, template)\n        else\n          raise ArgumentError, \"Template for formatter must be a Class or callable object\"\n        end\n      end\n\n      def self.lookup(type)\n        # TODO: warn when deprecated to use Plugin.new_formatter(type, parent: plugin)\n        Fluent::Plugin.new_formatter(type)\n      end\n\n      # Keep backward-compatibility\n      def self.create(conf)\n        # TODO: warn when deprecated\n        format = conf['format']\n        if format.nil?\n          raise ConfigError, \"'format' parameter is required\"\n        end\n\n        formatter = lookup(format)\n        if formatter.respond_to?(:configure)\n          formatter.configure(conf)\n        end\n        formatter\n      end\n\n      HandleTagAndTimeMixin = Fluent::Compat::HandleTagAndTimeMixin\n      StructuredFormatMixin = Fluent::Compat::StructuredFormatMixin\n\n      class ProcWrappedFormatter < Fluent::Plugin::ProcWrappedFormatter\n        # TODO: warn when deprecated\n      end\n\n      class OutFileFormatter < Fluent::Plugin::OutFileFormatter\n        # TODO: warn when deprecated\n      end\n\n      class StdoutFormatter < Fluent::Plugin::StdoutFormatter\n        # TODO: warn when deprecated\n      end\n\n      class JSONFormatter < Fluent::Plugin::JSONFormatter\n        # TODO: warn when deprecated\n      end\n\n      class HashFormatter < Fluent::Plugin::HashFormatter\n        # TODO: warn when deprecated\n      end\n\n      class MessagePackFormatter < Fluent::Plugin::MessagePackFormatter\n        # TODO: warn when deprecated\n      end\n\n      class LabeledTSVFormatter < Fluent::Plugin::LabeledTSVFormatter\n        # TODO: warn when deprecated\n      end\n\n      class CsvFormatter < Fluent::Plugin::CsvFormatter\n        # TODO: warn when deprecated\n\n        # Do not cache because it is hard to consider the thread key correctly.\n        # (We can try, but it would be low priority.)\n        def csv_cacheable?\n          false\n        end\n      end\n\n      class SingleValueFormatter < Fluent::Plugin::SingleValueFormatter\n        # TODO: warn when deprecated\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/formatter_utils.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin_helper/compat_parameters'\n\nmodule Fluent\n  module Compat\n    module FormatterUtils\n      INJECT_PARAMS = Fluent::PluginHelper::CompatParameters::INJECT_PARAMS\n      FORMATTER_PARAMS = Fluent::PluginHelper::CompatParameters::FORMATTER_PARAMS\n\n      module InjectMixin\n        def format(tag, time, record)\n          r = owner.inject_values_to_record(tag, time, record)\n          super(tag, time, r)\n        end\n      end\n\n      def self.convert_formatter_conf(conf)\n        return if conf.elements(name: 'inject').first || conf.elements(name: 'format').first\n\n        inject_params = {}\n        INJECT_PARAMS.each do |older, newer|\n          next unless newer\n          if conf.has_key?(older)\n            inject_params[newer] = conf[older]\n          end\n        end\n\n        if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])\n          inject_params['time_key'] ||= 'time'\n          inject_params['time_type'] ||= 'string'\n        end\n        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])\n          inject_params['time_type'] = 'unixtime'\n        end\n        if conf.has_key?('localtime') || conf.has_key?('utc')\n          if conf.has_key?('localtime') && conf.has_key?('utc')\n            raise Fluent::ConfigError, \"both of utc and localtime are specified, use only one of them\"\n          elsif conf.has_key?('localtime')\n            inject_params['localtime'] = Fluent::Config.bool_value(conf['localtime'])\n          elsif conf.has_key?('utc')\n            inject_params['localtime'] = !(Fluent::Config.bool_value(conf['utc']))\n            # Specifying \"localtime false\" means using UTC in TimeFormatter\n            # And specifying \"utc\" is different from specifying \"timezone +0000\"(it's not always UTC).\n            # There are difference between \"Z\" and \"+0000\" in timezone formatting.\n            # TODO: add kwargs to TimeFormatter to specify \"using localtime\", \"using UTC\" or \"using specified timezone\" in more explicit way\n          end\n        end\n\n        if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])\n          inject_params['tag_key'] ||= 'tag'\n        end\n\n        unless inject_params.empty?\n          conf.elements << Fluent::Config::Element.new('inject', '', inject_params, [])\n        end\n\n        formatter_params = {}\n        FORMATTER_PARAMS.each do |older, newer|\n          next unless newer\n          if conf.has_key?(older)\n            formatter_params[newer] = conf[older]\n          end\n        end\n        unless formatter_params.empty?\n          conf.elements << Fluent::Config::Element.new('format', '', formatter_params, [])\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/handle_tag_and_time_mixin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/time' # TimeFormatter\n\nmodule Fluent\n  module Compat\n    module HandleTagAndTimeMixin\n      def self.included(klass)\n        klass.instance_eval {\n          config_param :include_time_key, :bool, default: false\n          config_param :time_key, :string, default: 'time'\n          config_param :time_format, :string, default: nil\n          config_param :time_as_epoch, :bool, default: false\n          config_param :include_tag_key, :bool, default: false\n          config_param :tag_key, :string, default: 'tag'\n          config_param :localtime, :bool, default: true\n          config_param :timezone, :string, default: nil\n        }\n      end\n\n      def configure(conf)\n        super\n\n        if conf['utc']\n          @localtime = false\n        end\n        @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)\n        if @time_as_epoch && !@include_time_key\n          log.warn \"time_as_epoch will be ignored because include_time_key is false\"\n        end\n      end\n\n      def filter_record(tag, time, record)\n        if @include_tag_key\n          record[@tag_key] = tag\n        end\n        if @include_time_key\n          if @time_as_epoch\n            record[@time_key] = time.to_i\n          else\n            record[@time_key] = @timef.format(time)\n          end\n        end\n      end\n    end\n  end\nend\n\n"
  },
  {
    "path": "lib/fluent/compat/handle_tag_name_mixin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/record_filter_mixin'\n\nmodule Fluent\n  module Compat\n    module HandleTagNameMixin\n      include RecordFilterMixin\n\n      attr_accessor :remove_tag_prefix, :remove_tag_suffix, :add_tag_prefix, :add_tag_suffix\n      def configure(conf)\n        super\n\n        @remove_tag_prefix = if conf.has_key?('remove_tag_prefix')\n                               Regexp.new('^' + Regexp.escape(conf['remove_tag_prefix']))\n                             else\n                               nil\n                             end\n\n        @remove_tag_suffix = if conf.has_key?('remove_tag_suffix')\n                               Regexp.new(Regexp.escape(conf['remove_tag_suffix']) + '$')\n                             else\n                               nil\n                             end\n\n        @add_tag_prefix = conf['add_tag_prefix']\n        @add_tag_suffix = conf['add_tag_suffix']\n      end\n\n      def filter_record(tag, time, record)\n        tag.sub!(@remove_tag_prefix, '') if @remove_tag_prefix\n        tag.sub!(@remove_tag_suffix, '') if @remove_tag_suffix\n        tag.insert(0, @add_tag_prefix) if @add_tag_prefix\n        tag << @add_tag_suffix if @add_tag_suffix\n        super(tag, time, record)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/input.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/input'\nrequire 'fluent/process'\nrequire 'fluent/compat/call_super_mixin'\n\nmodule Fluent\n  module Compat\n    class Input < Fluent::Plugin::Input\n      # TODO: warn when deprecated\n\n      def initialize\n        super\n        unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)\n          self.class.prepend Fluent::Compat::CallSuperMixin\n        end\n      end\n\n      # These definitions are to get instance methods of superclass of 3rd party plugins\n      # to make it sure to call super\n      def start\n        super\n      end\n\n      def before_shutdown\n        super\n      end\n\n      def shutdown\n        super\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/output.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/bare_output'\nrequire 'fluent/compat/call_super_mixin'\nrequire 'fluent/compat/formatter_utils'\nrequire 'fluent/compat/handle_tag_and_time_mixin'\nrequire 'fluent/compat/parser_utils'\nrequire 'fluent/compat/propagate_default'\nrequire 'fluent/compat/record_filter_mixin'\nrequire 'fluent/compat/output_chain'\nrequire 'fluent/timezone'\nrequire 'fluent/mixin'\nrequire 'fluent/process'\nrequire 'fluent/event'\n\nrequire 'fluent/plugin_helper/compat_parameters'\n\nrequire 'time'\n\nmodule Fluent\n  module Compat\n    NULL_OUTPUT_CHAIN = NullOutputChain.instance\n    BufferQueueLimitError = ::Fluent::Plugin::Buffer::BufferOverflowError\n\n    module CompatOutputUtils\n      def self.buffer_section(conf)\n        conf.elements(name: 'buffer').first\n      end\n\n      def self.secondary_section(conf)\n        conf.elements(name: 'secondary').first\n      end\n    end\n\n    module BufferedEventStreamMixin\n      include Enumerable\n\n      def repeatable?\n        true\n      end\n\n      def each(&block)\n        msgpack_each(&block)\n      end\n\n      def to_msgpack_stream\n        read\n      end\n\n      def key\n        metadata.tag\n      end\n    end\n\n    module AddTimeSliceKeyToChunkMixin\n      def time_slice_format=(format)\n        @_time_slice_format = format\n      end\n\n      def timekey=(unit)\n        @_timekey = unit\n      end\n\n      def timezone=(tz)\n        @_timezone = tz\n      end\n\n      def assume_timekey!\n        @_formatter = Fluent::TimeFormatter.new(@_time_slice_format, nil, @_timezone)\n\n        return if self.metadata.timekey\n        if self.respond_to?(:path) && self.path =~ /\\.(\\d+)\\.(?:b|q)(?:[a-z0-9]+)/\n          begin\n            self.metadata.timekey = Time.parse($1, @_time_slice_format).to_i\n          rescue ArgumentError\n            # unknown format / value as timekey\n          end\n        end\n        unless self.metadata.timekey\n          # file creation time is assumed in the time range of that time slice\n          # because the first record should be in that range.\n          time_int = self.created_at.to_i\n          self.metadata.timekey = time_int - (time_int % @_timekey)\n        end\n      end\n\n      def key\n        @_formatter.call(self.metadata.timekey)\n      end\n    end\n\n    module AddKeyToChunkMixin\n      def key\n        self.metadata.variables[:key]\n      end\n    end\n\n    module ChunkSizeCompatMixin\n      def size\n        self.bytesize\n      end\n\n      def size_of_events\n        @size + @adding_size\n      end\n    end\n\n    module BufferedChunkMixin\n      # prepend this module to BufferedOutput (including ObjectBufferedOutput) plugin singleton class\n      def write(chunk)\n        chunk.extend(ChunkSizeCompatMixin)\n        chunk.extend(ChunkMessagePackEventStreamer)\n        chunk.extend(AddKeyToChunkMixin) if chunk.metadata.variables && chunk.metadata.variables.has_key?(:key)\n        super\n      end\n    end\n\n    module TimeSliceChunkMixin\n      # prepend this module to TimeSlicedOutput plugin singleton class\n      def write(chunk)\n        chunk.extend(ChunkSizeCompatMixin)\n        chunk.extend(ChunkMessagePackEventStreamer)\n        chunk.extend(AddTimeSliceKeyToChunkMixin)\n        chunk.time_slice_format = @time_slice_format\n        chunk.timekey = @_timekey\n        chunk.timezone = @timezone\n        chunk.assume_timekey!\n        super\n      end\n    end\n\n    class Output < Fluent::Plugin::Output\n      # TODO: warn when deprecated\n\n      helpers_internal :event_emitter, :inject\n\n      def support_in_v12_style?(feature)\n        case feature\n        when :synchronous    then true\n        when :buffered       then false\n        when :delayed_commit then false\n        when :custom_format  then false\n        end\n      end\n\n      def process(tag, es)\n        emit(tag, es, NULL_OUTPUT_CHAIN)\n      end\n\n      def initialize\n        super\n        unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)\n          self.class.prepend Fluent::Compat::CallSuperMixin\n        end\n      end\n\n      def configure(conf)\n        ParserUtils.convert_parser_conf(conf)\n        FormatterUtils.convert_formatter_conf(conf)\n\n        super\n      end\n\n      def start\n        super\n\n        if instance_variable_defined?(:@formatter) && @inject_config\n          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)\n            if @formatter.respond_to?(:owner) && !@formatter.owner\n              @formatter.owner = self\n              @formatter.singleton_class.prepend FormatterUtils::InjectMixin\n            end\n          end\n        end\n      end\n    end\n\n    class MultiOutput < Fluent::Plugin::BareOutput\n      # TODO: warn when deprecated\n\n      helpers_internal :event_emitter\n\n      def process(tag, es)\n        emit(tag, es, NULL_OUTPUT_CHAIN)\n      end\n    end\n\n    class BufferedOutput < Fluent::Plugin::Output\n      # TODO: warn when deprecated\n\n      helpers_internal :event_emitter, :inject\n\n      def support_in_v12_style?(feature)\n        case feature\n        when :synchronous    then false\n        when :buffered       then true\n        when :delayed_commit then false\n        when :custom_format  then true\n        end\n      end\n\n      desc 'The buffer type (memory, file)'\n      config_param :buffer_type, :string, default: 'memory'\n      desc 'The interval between data flushes.'\n      config_param :flush_interval, :time, default: 60\n      config_param :try_flush_interval, :float, default: 1\n      desc 'If true, the value of `retry_value` is ignored and there is no limit'\n      config_param :disable_retry_limit, :bool, default: false\n      desc 'The limit on the number of retries before buffered data is discarded'\n      config_param :retry_limit, :integer, default: 17\n      desc 'The initial intervals between write retries.'\n      config_param :retry_wait, :time, default: 1.0\n      desc 'The maximum intervals between write retries.'\n      config_param :max_retry_wait, :time, default: nil\n      desc 'The number of threads to flush the buffer.'\n      config_param :num_threads, :integer, default: 1\n      desc 'The interval between data flushes for queued chunk.'\n      config_param :queued_chunk_flush_interval, :time, default: 1\n\n      desc 'The size of each buffer chunk.'\n      config_param :buffer_chunk_limit, :size, default: 8*1024*1024\n      desc 'The length limit of the chunk queue.'\n      config_param :buffer_queue_limit, :integer, default: 256\n      desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'\n      config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception\n\n      config_param :flush_at_shutdown, :bool, default: true\n\n      BUFFER_PARAMS = Fluent::PluginHelper::CompatParameters::BUFFER_PARAMS\n\n      def self.propagate_default_params\n        BUFFER_PARAMS\n      end\n      include PropagateDefault\n\n      def configure(conf)\n        bufconf = CompatOutputUtils.buffer_section(conf)\n        config_style = (bufconf ? :v1 : :v0)\n        if config_style == :v0\n          buf_params = {\n            \"flush_mode\" => \"interval\",\n            \"retry_type\" => \"exponential_backoff\",\n          }\n          BUFFER_PARAMS.each do |older, newer|\n            next unless newer\n            if conf.has_key?(older)\n              if older == 'buffer_queue_full_action' && conf[older] == 'exception'\n                buf_params[newer] = 'throw_exception'\n              else\n                buf_params[newer] = conf[older]\n              end\n            end\n          end\n\n          conf.elements << Fluent::Config::Element.new('buffer', '', buf_params, [])\n        end\n\n        @includes_record_filter = self.class.ancestors.include?(Fluent::Compat::RecordFilterMixin)\n\n        methods_of_plugin = self.class.instance_methods(false)\n        @overrides_emit = methods_of_plugin.include?(:emit)\n        # RecordFilter mixin uses its own #format_stream method implementation\n        @overrides_format_stream = methods_of_plugin.include?(:format_stream) || @includes_record_filter\n\n        ParserUtils.convert_parser_conf(conf)\n        FormatterUtils.convert_formatter_conf(conf)\n\n        super\n\n        if config_style == :v1\n          unless @buffer_config.chunk_keys.empty?\n            raise Fluent::ConfigError, \"this plugin '#{self.class}' cannot handle arguments for <buffer ...> section\"\n          end\n        end\n\n        self.extend BufferedChunkMixin\n\n        if @overrides_emit\n          self.singleton_class.module_eval do\n            attr_accessor :last_emit_via_buffer\n          end\n          output_plugin = self\n          m = Module.new do\n            define_method(:emit) do |key, data, chain|\n              # receivers of this method are buffer instances\n              output_plugin.last_emit_via_buffer = [key, data]\n            end\n          end\n          @buffer.extend m\n        end\n      end\n\n      # original implementation of v0.12 BufferedOutput\n      def emit(tag, es, chain, key=\"\")\n        # this method will not be used except for the case that plugin calls super\n        @emit_count_metrics.inc\n        data = format_stream(tag, es)\n        if @buffer.emit(key, data, chain)\n          submit_flush\n        end\n      end\n\n      def submit_flush\n        # nothing todo: blank method to be called from #emit of 3rd party plugins\n      end\n\n      def format_stream(tag, es)\n        # this method will not be used except for the case that plugin calls super\n        out = ''\n        es.each do |time, record|\n          out << format(tag, time, record)\n        end\n        out\n      end\n\n      # #format MUST be implemented in plugin\n      # #write is also\n\n      # This method overrides Fluent::Plugin::Output#handle_stream_simple\n      # because v0.12 BufferedOutput may overrides #format_stream, but original #handle_stream_simple method doesn't consider about it\n      def handle_stream_simple(tag, es, enqueue: false)\n        if @overrides_emit\n          current_emit_count = @emit_count_metrics.get\n          size = es.size\n          key = data = nil\n          begin\n            emit(tag, es, NULL_OUTPUT_CHAIN)\n            key, data = self.last_emit_via_buffer\n          ensure\n            @emit_count_metrics.set(current_emit_count)\n            self.last_emit_via_buffer = nil\n          end\n          # on-the-fly key assignment can be done, and it's not configurable if Plugin#emit does it dynamically\n          meta = @buffer.metadata(variables: (key && !key.empty? ? {key: key} : nil))\n          write_guard do\n            @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)\n          end\n          @emit_records_metrics.add(es.size)\n          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n          return [meta]\n        end\n\n        if @overrides_format_stream\n          meta = metadata(nil, nil, nil)\n          size = es.size\n          bulk = format_stream(tag, es)\n          write_guard do\n            @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)\n          end\n          @emit_records_metrics.add(es.size)\n          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n          return [meta]\n        end\n\n        meta = metadata(nil, nil, nil)\n        size = es.size\n        data = es.map{|time,record| format(tag, time, record) }\n        write_guard do\n          @buffer.write({meta => data}, enqueue: enqueue)\n        end\n        @emit_records_metrics.add(es.size)\n        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n        [meta]\n      end\n\n      def extract_placeholders(str, metadata)\n        raise \"BUG: compat plugin does not support extract_placeholders: use newer plugin API\"\n      end\n\n      def initialize\n        super\n        unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)\n          self.class.prepend Fluent::Compat::CallSuperMixin\n        end\n      end\n\n      def start\n        super\n\n        if instance_variable_defined?(:@formatter) && @inject_config\n          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)\n            if @formatter.respond_to?(:owner) && !@formatter.owner\n              @formatter.owner = self\n              @formatter.singleton_class.prepend FormatterUtils::InjectMixin\n            end\n          end\n        end\n      end\n\n      def detach_process(&block)\n        log.warn \"detach_process is not supported in this version. ignored.\"\n        block.call\n      end\n\n      def detach_multi_process(&block)\n        log.warn \"detach_process is not supported in this version. ignored.\"\n        block.call\n      end\n    end\n\n    class ObjectBufferedOutput < Fluent::Plugin::Output\n      # TODO: warn when deprecated\n\n      helpers_internal :event_emitter, :inject\n\n      # This plugin cannot inherit BufferedOutput because #configure sets chunk_key 'tag'\n      # to flush chunks per tags, but BufferedOutput#configure doesn't allow setting chunk_key\n      # in v1 style configuration\n\n      def support_in_v12_style?(feature)\n        case feature\n        when :synchronous    then false\n        when :buffered       then true\n        when :delayed_commit then false\n        when :custom_format  then false\n        end\n      end\n\n      desc 'The buffer type (memory, file)'\n      config_param :buffer_type, :string, default: 'memory'\n      desc 'The interval between data flushes.'\n      config_param :flush_interval, :time, default: 60\n      config_param :try_flush_interval, :float, default: 1\n      desc 'If true, the value of `retry_value` is ignored and there is no limit'\n      config_param :disable_retry_limit, :bool, default: false\n      desc 'The limit on the number of retries before buffered data is discarded'\n      config_param :retry_limit, :integer, default: 17\n      desc 'The initial intervals between write retries.'\n      config_param :retry_wait, :time, default: 1.0\n      desc 'The maximum intervals between write retries.'\n      config_param :max_retry_wait, :time, default: nil\n      desc 'The number of threads to flush the buffer.'\n      config_param :num_threads, :integer, default: 1\n      desc 'The interval between data flushes for queued chunk.'\n      config_param :queued_chunk_flush_interval, :time, default: 1\n\n      desc 'The size of each buffer chunk.'\n      config_param :buffer_chunk_limit, :size, default: 8*1024*1024\n      desc 'The length limit of the chunk queue.'\n      config_param :buffer_queue_limit, :integer, default: 256\n      desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'\n      config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception\n\n      config_param :flush_at_shutdown, :bool, default: true\n\n      config_set_default :time_as_integer, true\n\n      BUFFER_PARAMS = Fluent::PluginHelper::CompatParameters::BUFFER_PARAMS\n\n      def self.propagate_default_params\n        BUFFER_PARAMS\n      end\n      include PropagateDefault\n\n      def configure(conf)\n        bufconf = CompatOutputUtils.buffer_section(conf)\n        config_style = (bufconf ? :v1 : :v0)\n        if config_style == :v0\n          buf_params = {\n            \"flush_mode\" => \"interval\",\n            \"retry_type\" => \"exponential_backoff\",\n          }\n          BUFFER_PARAMS.each do |older, newer|\n            next unless newer\n            if conf.has_key?(older)\n              if older == 'buffer_queue_full_action' && conf[older] == 'exception'\n                buf_params[newer] = 'throw_exception'\n              else\n                buf_params[newer] = conf[older]\n              end\n            end\n          end\n\n          conf.elements << Fluent::Config::Element.new('buffer', 'tag', buf_params, [])\n        end\n\n        ParserUtils.convert_parser_conf(conf)\n        FormatterUtils.convert_formatter_conf(conf)\n\n        super\n\n        if config_style == :v1\n          if @buffer_config.chunk_keys == ['tag']\n            raise Fluent::ConfigError, \"this plugin '#{self.class}' allows <buffer tag> only\"\n          end\n        end\n\n        self.extend BufferedChunkMixin\n      end\n\n      def format_stream(tag, es) # for BufferedOutputTestDriver\n        if @compress == :gzip\n          es.to_compressed_msgpack_stream(time_int: @time_as_integer)\n        else\n          es.to_msgpack_stream(time_int: @time_as_integer)\n        end\n      end\n\n      def write(chunk)\n        write_objects(chunk.metadata.tag, chunk)\n      end\n\n      def extract_placeholders(str, metadata)\n        raise \"BUG: compat plugin does not support extract_placeholders: use newer plugin API\"\n      end\n\n      def initialize\n        super\n        unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)\n          self.class.prepend Fluent::Compat::CallSuperMixin\n        end\n      end\n\n      def start\n        super\n\n        if instance_variable_defined?(:@formatter) && @inject_config\n          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)\n            if @formatter.respond_to?(:owner) && !@formatter.owner\n              @formatter.owner = self\n              @formatter.singleton_class.prepend FormatterUtils::InjectMixin\n            end\n          end\n        end\n      end\n\n      def detach_process(&block)\n        log.warn \"detach_process is not supported in this version. ignored.\"\n        block.call\n      end\n\n      def detach_multi_process(&block)\n        log.warn \"detach_process is not supported in this version. ignored.\"\n        block.call\n      end\n    end\n\n    class TimeSlicedOutput < Fluent::Plugin::Output\n      # TODO: warn when deprecated\n\n      helpers_internal :event_emitter, :inject\n\n      def support_in_v12_style?(feature)\n        case feature\n        when :synchronous    then false\n        when :buffered       then true\n        when :delayed_commit then false\n        when :custom_format  then true\n        end\n      end\n\n      desc 'The buffer type (memory, file)'\n      config_param :buffer_type, :string, default: 'file'\n      desc 'The interval between data flushes.'\n      config_param :flush_interval, :time, default: nil\n      config_param :try_flush_interval, :float, default: 1\n      desc 'If true, the value of `retry_value` is ignored and there is no limit'\n      config_param :disable_retry_limit, :bool, default: false\n      desc 'The limit on the number of retries before buffered data is discarded'\n      config_param :retry_limit, :integer, default: 17\n      desc 'The initial intervals between write retries.'\n      config_param :retry_wait, :time, default: 1.0\n      desc 'The maximum intervals between write retries.'\n      config_param :max_retry_wait, :time, default: nil\n      desc 'The number of threads to flush the buffer.'\n      config_param :num_threads, :integer, default: 1\n      desc 'The interval between data flushes for queued chunk.'\n      config_param :queued_chunk_flush_interval, :time, default: 1\n\n      desc 'The time format used as part of the file name.'\n      config_param :time_slice_format, :string, default: '%Y%m%d'\n      desc 'The amount of time Fluentd will wait for old logs to arrive.'\n      config_param :time_slice_wait, :time, default: 10*60\n      desc 'Parse the time value in the specified timezone'\n      config_param :timezone, :string, default: nil\n\n      desc 'The size of each buffer chunk.'\n      config_param :buffer_chunk_limit, :size, default: 256*1024*1024\n      desc 'The length limit of the chunk queue.'\n      config_param :buffer_queue_limit, :integer, default: 256\n      desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'\n      config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception\n\n      config_param :flush_at_shutdown, :bool, default: false\n\n      attr_accessor :localtime\n\n      config_section :buffer do\n        config_set_default :@type, 'file'\n      end\n\n      BUFFER_PARAMS = Fluent::PluginHelper::CompatParameters::BUFFER_PARAMS.merge(Fluent::PluginHelper::CompatParameters::BUFFER_TIME_SLICED_PARAMS)\n\n      def initialize\n        super\n        @localtime = true\n\n        unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)\n          self.class.prepend Fluent::Compat::CallSuperMixin\n        end\n      end\n\n      def self.propagate_default_params\n        BUFFER_PARAMS\n      end\n      include PropagateDefault\n\n      def configure(conf)\n        bufconf = CompatOutputUtils.buffer_section(conf)\n        config_style = (bufconf ? :v1 : :v0)\n        if config_style == :v0\n          buf_params = {\n            \"flush_mode\" => (conf['flush_interval'] ? \"interval\" : \"lazy\"),\n            \"retry_type\" => \"exponential_backoff\",\n          }\n          BUFFER_PARAMS.each do |older, newer|\n            next unless newer\n            if conf.has_key?(older)\n              if older == 'buffer_queue_full_action' && conf[older] == 'exception'\n                buf_params[newer] = 'throw_exception'\n              else\n                buf_params[newer] = conf[older]\n              end\n            end\n          end\n\n          if conf['timezone']\n            Fluent::Timezone.validate!(conf['timezone'])\n          elsif conf['utc']\n            # v0.12 assumes UTC without any configuration\n            # 'localtime=false && no timezone key' means UTC\n            conf['localtime'] = \"false\"\n            conf.delete('utc')\n          elsif conf['localtime']\n            conf['timezone'] = Time.now.strftime('%z')\n            conf['localtime'] = \"true\"\n          else\n            # v0.12 assumes UTC without any configuration\n            # 'localtime=false && no timezone key' means UTC\n            conf['localtime'] = \"false\"\n          end\n\n          @_timekey = case conf['time_slice_format']\n                      when /\\%S/ then 1\n                      when /\\%M/ then 60\n                      when /\\%H/ then 3600\n                      when /\\%d/ then 86400\n                      when nil   then 86400 # default value of TimeSlicedOutput.time_slice_format is '%Y%m%d'\n                      else\n                        raise Fluent::ConfigError, \"time_slice_format only with %Y or %m is too long\"\n                      end\n          buf_params[\"timekey\"] = @_timekey\n\n          conf.elements << Fluent::Config::Element.new('buffer', 'time', buf_params, [])\n        end\n\n        ParserUtils.convert_parser_conf(conf)\n        FormatterUtils.convert_formatter_conf(conf)\n\n        super\n\n        if config_style == :v1\n          if @buffer_config.chunk_keys == ['tag']\n            raise Fluent::ConfigError, \"this plugin '#{self.class}' allows <buffer tag> only\"\n          end\n        end\n\n        self.extend TimeSliceChunkMixin\n      end\n\n      def start\n        super\n\n        if instance_variable_defined?(:@formatter) && @inject_config\n          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)\n            if @formatter.respond_to?(:owner) && !@formatter.owner\n              @formatter.owner = self\n              @formatter.singleton_class.prepend FormatterUtils::InjectMixin\n            end\n          end\n        end\n      end\n\n      def detach_process(&block)\n        log.warn \"detach_process is not supported in this version. ignored.\"\n        block.call\n      end\n\n      def detach_multi_process(&block)\n        log.warn \"detach_process is not supported in this version. ignored.\"\n        block.call\n      end\n\n      # Original TimeSlicedOutput#emit doesn't call #format_stream\n\n      # #format MUST be implemented in plugin\n      # #write is also\n\n      def extract_placeholders(str, metadata)\n        raise \"BUG: compat plugin does not support extract_placeholders: use newer plugin API\"\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/output_chain.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'singleton'\n\nmodule Fluent\n  module Compat\n    # TODO: remove when old plugin API are removed\n    class NullOutputChain\n      include Singleton\n\n      def next\n      end\n    end\n\n    class OutputChain\n      def initialize(array, tag, es, chain=NullOutputChain.instance)\n        @array = array\n        @tag = tag\n        @es = es\n        @offset = 0\n        @chain = chain\n      end\n\n      def next\n        if @array.length <= @offset\n          return @chain.next\n        end\n        @offset += 1\n        @array[@offset-1].emit_events(@tag, @es)\n        self.next\n      end\n    end\n\n    class CopyOutputChain < OutputChain\n      def next\n        if @array.length <= @offset\n          return @chain.next\n        end\n        @offset += 1\n        es = @array.length > @offset ? @es.dup : @es\n        @array[@offset-1].emit_events(@tag, es)\n        self.next\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/parser'\nrequire 'fluent/mixin'\n\nrequire 'fluent/config'\nrequire 'fluent/compat/type_converter'\n\nrequire 'fluent/plugin/parser_regexp'\nrequire 'fluent/plugin/parser_json'\nrequire 'fluent/plugin/parser_tsv'\nrequire 'fluent/plugin/parser_ltsv'\nrequire 'fluent/plugin/parser_csv'\nrequire 'fluent/plugin/parser_none'\nrequire 'fluent/plugin/parser_apache2'\nrequire 'fluent/plugin/parser_syslog'\nrequire 'fluent/plugin/parser_multiline'\n\nmodule Fluent\n  module Compat\n    class Parser < Fluent::Plugin::Parser\n      # TODO: warn when deprecated\n    end\n\n    class TextParser\n      # Keep backward compatibility for existing plugins\n      ParserError = Fluent::Plugin::Parser::ParserError\n      # TODO: will be removed at v1\n      TypeConverter = Fluent::TypeConverter\n\n      def initialize\n        # TODO: warn when deprecated\n        @parser = nil\n        @estimate_current_event = nil\n      end\n\n      attr_reader :parser\n\n      # SET false BEFORE CONFIGURE, to return nil when time not parsed\n      # 'configure()' may raise errors for unexpected configurations\n      attr_accessor :estimate_current_event\n\n      def configure(conf, required=true)\n        format = conf['format']\n\n        @parser = TextParser.lookup(format)\n\n        if @parser.respond_to?(:configure)\n          @parser.configure(conf)\n        end\n        if !@estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')\n          # external code sets parser.estimate_current_event = false\n          @parser.estimate_current_event = @estimate_current_event\n        end\n\n        return true\n      end\n\n      def parse(text, &block)\n        if block\n          @parser.parse(text, &block)\n        else\n          @parser.parse(text) { |time, record|\n            return time, record\n          }\n        end\n      end\n\n      def self.register_template(type, template, time_format=nil)\n        # TODO: warn when deprecated to use Plugin.register_parser directly\n        if template.is_a?(Class) || template.respond_to?(:call)\n          Fluent::Plugin.register_parser(type, template)\n        elsif template.is_a?(Regexp)\n          Fluent::Plugin.register_parser(type, Proc.new { RegexpParser.new(template, {'time_format' => time_format}) })\n        else\n          raise ArgumentError, \"Template for parser must be a Class, callable object or regular expression object\"\n        end\n      end\n\n      def self.lookup(format)\n        # TODO: warn when deprecated to use Plugin.new_parser or RegexpParser.new directly\n        if format.nil?\n          raise ConfigError, \"'format' parameter is required\"\n        end\n\n        if format[0] == ?/ && format[format.length-1] == ?/\n          # regexp\n          begin\n            regexp = Regexp.new(format[1..-2])\n            if regexp.named_captures.empty?\n              raise \"No named captures\"\n            end\n          rescue\n            raise ConfigError, \"Invalid regexp '#{format[1..-2]}': #{$!}\"\n          end\n\n          RegexpParser.new(regexp)\n        else\n          # built-in template\n          begin\n            Fluent::Plugin.new_parser(format)\n          rescue ConfigError # keep same error message\n            raise ConfigError, \"Unknown format template '#{format}'\"\n          end\n        end\n      end\n\n      module TypeConverterCompatParameters\n        def convert_type_converter_parameters!(conf)\n          if conf[\"types\"]\n            delimiter = conf[\"types_delimiter\"] || ','\n            label_delimiter = conf[\"types_label_delimiter\"] || ':'\n            types = {}\n            conf['types'].split(delimiter).each do |pair|\n              key, value = pair.split(label_delimiter, 2)\n              if value.start_with?(\"time#{label_delimiter}\")\n                value = value.split(label_delimiter, 2).join(':')\n              elsif value.start_with?(\"array#{label_delimiter}\")\n                value = value.split(label_delimiter, 2).join(':')\n              end\n              types[key] = value\n            end\n            conf[\"types\"] = JSON.dump(types)\n          end\n        end\n      end\n\n      class TimeParser < Fluent::TimeParser\n        # TODO: warn when deprecated\n      end\n\n      class RegexpParser < Fluent::Plugin::RegexpParser\n        include TypeConverterCompatParameters\n\n        # TODO: warn when deprecated\n        def initialize(regexp, conf = {})\n          super()\n\n          @stored_regexp = regexp\n          @manually_configured = false\n          unless conf.empty?\n            conf_init = if conf.is_a?(Fluent::Config::Element)\n                          conf\n                        else\n                          Fluent::Config::Element.new('parse', '', conf, [])\n                        end\n            self.configure(conf_init)\n            @manually_configured = true\n          end\n        end\n\n        def configure(conf)\n          return if @manually_configured # not to run twice\n\n          conf['expression'] ||= @stored_regexp.source\n          conf['ignorecase'] ||= @stored_regexp.options & Regexp::IGNORECASE != 0\n          conf['multiline'] ||= @stored_regexp.options & Regexp::MULTILINE != 0\n          convert_type_converter_parameters!(conf)\n\n          super\n        end\n\n        def patterns\n          {'format' => @regexp, 'time_format' => @time_format}\n        end\n      end\n\n      class ValuesParser < Parser\n        include Fluent::Compat::TypeConverter\n\n        config_param :keys, :array, default: []\n        config_param :time_key, :string, default: nil\n        config_param :null_value_pattern, :string, default: nil\n        config_param :null_empty_string, :bool, default: false\n\n        def configure(conf)\n          super\n\n          if @time_key && !@keys.include?(@time_key) && @estimate_current_event\n            raise Fluent::ConfigError, \"time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})\"\n          end\n\n          if @time_format && !@time_key\n            raise Fluent::ConfigError, \"time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}\"\n          end\n\n          @time_parser = time_parser_create\n\n          if @null_value_pattern\n            @null_value_pattern = Regexp.new(@null_value_pattern)\n          end\n\n          @mutex = Mutex.new\n        end\n\n        def values_map(values)\n          record = Hash[keys.zip(values.map { |value| convert_value_to_nil(value) })]\n\n          if @time_key\n            value = @keep_time_key ? record[@time_key] : record.delete(@time_key)\n            time = if value.nil?\n                     if @estimate_current_event\n                       Fluent::EventTime.now\n                     else\n                       nil\n                     end\n                   else\n                     @mutex.synchronize { @time_parser.parse(value) }\n                   end\n          elsif @estimate_current_event\n            time = Fluent::EventTime.now\n          else\n            time = nil\n          end\n\n          convert_field_type!(record) if @type_converters\n\n          return time, record\n        end\n\n        private\n\n        def convert_field_type!(record)\n          @type_converters.each_key { |key|\n            if value = record[key]\n              record[key] = convert_type(key, value)\n            end\n          }\n        end\n\n        def convert_value_to_nil(value)\n          if value && @null_empty_string\n            value = (value == '') ? nil : value\n          end\n          if value && @null_value_pattern\n            value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value\n          end\n          value\n        end\n      end\n\n      class JSONParser < Fluent::Plugin::JSONParser\n        include TypeConverterCompatParameters\n        # TODO: warn when deprecated\n        def configure(conf)\n          convert_type_converter_parameters!(conf)\n          super\n        end\n      end\n\n      class TSVParser < Fluent::Plugin::TSVParser\n        include TypeConverterCompatParameters\n        # TODO: warn when deprecated\n        def configure(conf)\n          convert_type_converter_parameters!(conf)\n          super\n        end\n      end\n\n      class LabeledTSVParser < Fluent::Plugin::LabeledTSVParser\n        include TypeConverterCompatParameters\n        # TODO: warn when deprecated\n        def configure(conf)\n          convert_type_converter_parameters!(conf)\n          super\n        end\n      end\n\n      class CSVParser < Fluent::Plugin::CSVParser\n        include TypeConverterCompatParameters\n        # TODO: warn when deprecated\n        def configure(conf)\n          convert_type_converter_parameters!(conf)\n          super\n        end\n      end\n\n      class NoneParser < Fluent::Plugin::NoneParser\n        # TODO: warn when deprecated\n      end\n\n      class ApacheParser < Fluent::Plugin::Apache2Parser\n        # TODO: warn when deprecated\n      end\n\n      class SyslogParser < Fluent::Plugin::SyslogParser\n        # TODO: warn when deprecated\n      end\n\n      class MultilineParser < Fluent::Plugin::MultilineParser\n        # TODO: warn when deprecated\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/parser_utils.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin_helper/compat_parameters'\n\nmodule Fluent\n  module Compat\n    module ParserUtils\n      PARSER_PARAMS = Fluent::PluginHelper::CompatParameters::PARSER_PARAMS\n\n      def self.convert_parser_conf(conf)\n        return if conf.elements(name: 'parse').first\n\n        parser_params = {}\n        PARSER_PARAMS.each do |older, newer|\n          next unless newer\n          if conf.has_key?(older)\n            parser_params[newer] = conf[older]\n          end\n        end\n        unless parser_params.empty?\n          conf.elements << Fluent::Config::Element.new('parse', '', parser_params, [])\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/propagate_default.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/configurable'\n\nmodule Fluent\n  module Compat\n    module PropagateDefault\n      # This mixin is to prepend to 3rd party plugins of v0.12 APIs.\n      # 3rd party plugins may override default values of some parameters, like `buffer_type`.\n      # But default values of such parameters will NOT used, but defaults of <buffer>@type</buffer>\n      # will be used in fact. It should bring troubles.\n      # This mixin defines Class method .config_param and .config_set_default (which should be used by extend)\n      # to propagate changes of default values to subsections.\n      def self.included(mod)\n        mod.extend(ClassMethods)\n      end\n\n      module ClassMethods\n        CONFIGURABLE_CLASS_METHODS = Fluent::Configurable::ClassMethods\n\n        def config_param(name, type = nil, **kwargs, &block)\n          CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind_call(self, name, type, **kwargs, &block)\n          pparams = propagate_default_params\n          if kwargs.has_key?(:default) && pparams[name.to_s]\n            newer = pparams[name.to_s].to_sym\n            overridden_default_value = kwargs[:default]\n\n            CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do\n              config_set_default newer, overridden_default_value\n            end\n          end\n        end\n\n        def config_set_default(name, defval)\n          CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind_call(self, name, defval)\n          pparams = propagate_default_params\n          if pparams[name.to_s]\n            newer = pparams[name.to_s].to_sym\n\n            CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do\n              self.config_set_default newer, defval\n            end\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/record_filter_mixin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Compat\n    module RecordFilterMixin\n      def filter_record(tag, time, record)\n      end\n\n      def format_stream(tag, es)\n        out = ''\n        es.each {|time,record|\n          tag_temp = tag.dup\n          filter_record(tag_temp, time, record)\n          out << format(tag_temp, time, record)\n        }\n        out\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/set_tag_key_mixin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/error'\nrequire 'fluent/config/types'\nrequire 'fluent/compat/record_filter_mixin'\n\nmodule Fluent\n  module Compat\n    module SetTagKeyMixin\n      include RecordFilterMixin\n\n      attr_accessor :include_tag_key, :tag_key\n\n      def configure(conf)\n        @include_tag_key = false\n\n        super\n\n        if s = conf['include_tag_key']\n          include_tag_key = Fluent::Config.bool_value(s)\n          raise Fluent::ConfigError, \"Invalid boolean expression '#{s}' for include_tag_key parameter\" if include_tag_key.nil?\n\n          @include_tag_key = include_tag_key\n        end\n\n        @tag_key = conf['tag_key'] || 'tag' if @include_tag_key\n      end\n\n      def filter_record(tag, time, record)\n        super\n\n        record[@tag_key] = tag if @include_tag_key\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/set_time_key_mixin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/error'\nrequire 'fluent/compat/record_filter_mixin'\nrequire 'fluent/time'\nrequire 'fluent/timezone'\n\nmodule Fluent\n  module Compat\n    module SetTimeKeyMixin\n      include RecordFilterMixin\n\n      attr_accessor :include_time_key, :time_key, :localtime, :timezone\n\n      def configure(conf)\n        @include_time_key = false\n        @localtime = false\n        @timezone = nil\n\n        super\n\n        if s = conf['include_time_key']\n          include_time_key = Fluent::Config.bool_value(s)\n          raise Fluent::ConfigError, \"Invalid boolean expression '#{s}' for include_time_key parameter\" if include_time_key.nil?\n\n          @include_time_key = include_time_key\n        end\n\n        if @include_time_key\n          @time_key     = conf['time_key'] || 'time'\n          @time_format  = conf['time_format']\n\n          if    conf['localtime']\n            @localtime = true\n          elsif conf['utc']\n            @localtime = false\n          end\n\n          if conf['timezone']\n            @timezone = conf['timezone']\n            Fluent::Timezone.validate!(@timezone)\n          end\n\n          @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)\n        end\n      end\n\n      def filter_record(tag, time, record)\n        super\n\n        record[@time_key] = @timef.format(time) if @include_time_key\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/socket_util.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'ipaddr'\n\nrequire 'cool.io'\n\nrequire 'fluent/plugin'\nrequire 'fluent/input'\n\nmodule Fluent\n  module Compat\n    module SocketUtil\n      def create_udp_socket(host)\n        if IPAddr.new(IPSocket.getaddress(host)).ipv4?\n          UDPSocket.new\n        else\n          UDPSocket.new(Socket::AF_INET6)\n        end\n      end\n      module_function :create_udp_socket\n\n      class UdpHandler < Coolio::IO\n        def initialize(io, log, body_size_limit, callback)\n          super(io)\n          @io = io\n          @log = log\n          @body_size_limit = body_size_limit\n          @callback = callback\n        end\n\n        def on_readable\n          msg, addr = @io.recvfrom_nonblock(@body_size_limit)\n          msg.chomp!\n          @callback.call(msg, addr)\n        rescue => e\n          @log.error \"unexpected error\", error: e\n        end\n      end\n\n      class TcpHandler < Coolio::Socket\n        PEERADDR_FAILED = [\"?\", \"?\", \"name resolution failed\", \"?\"]\n\n        def initialize(io, log, delimiter, callback)\n          super(io)\n          @timeout = 0\n          if io.is_a?(TCPSocket)\n            @addr = (io.peeraddr rescue PEERADDR_FAILED)\n\n            opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }\n            io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)\n          end\n          @delimiter = delimiter\n          @callback = callback\n          @log = log\n          @log.trace { \"accepted fluent socket object_id=#{self.object_id}\" }\n          @buffer = \"\".force_encoding('ASCII-8BIT')\n        end\n\n        def on_connect\n        end\n\n        def on_read(data)\n          @buffer << data\n          pos = 0\n\n          while i = @buffer.index(@delimiter, pos)\n            msg = @buffer[pos...i]\n            @callback.call(msg, @addr)\n            pos = i + @delimiter.length\n          end\n          @buffer.slice!(0, pos) if pos > 0\n        rescue => e\n          @log.error \"unexpected error\", error: e\n          close\n        end\n\n        def on_close\n          @log.trace { \"closed fluent socket object_id=#{self.object_id}\" }\n        end\n      end\n\n      class BaseInput < Fluent::Input\n        def initialize\n          super\n          require 'fluent/parser'\n        end\n\n        desc 'Tag of output events.'\n        config_param :tag, :string\n        desc 'The format of the payload.'\n        config_param :format, :string\n        desc 'The port to listen to.'\n        config_param :port, :integer, default: 5150\n        desc 'The bind address to listen to.'\n        config_param :bind, :string, default: '0.0.0.0'\n        desc \"The field name of the client's hostname.\"\n        config_param :source_host_key, :string, default: nil\n        config_param :blocking_timeout, :time, default: 0.5\n\n        def configure(conf)\n          super\n\n          @parser = Plugin.new_parser(@format)\n          @parser.configure(conf)\n        end\n\n        def start\n          super\n\n          @loop = Coolio::Loop.new\n          @handler = listen(method(:on_message))\n          @loop.attach(@handler)\n          @thread = Thread.new(&method(:run))\n        end\n\n        def shutdown\n          @loop.watchers.each { |w| w.detach }\n          @loop.stop if @loop.instance_variable_get(:@running)\n          @handler.close\n          @thread.join\n\n          super\n        end\n\n        def run\n          @loop.run(@blocking_timeout)\n        rescue => e\n          log.error \"unexpected error\", error: e\n          log.error_backtrace\n        end\n\n        private\n\n        def on_message(msg, addr)\n          @parser.parse(msg) { |time, record|\n            unless time && record\n              log.warn { \"pattern not matched: #{msg.inspect}\" }\n              return\n            end\n\n            record[@source_host_key] = addr[3] if @source_host_key\n            router.emit(@tag, time, record)\n          }\n        rescue => e\n          log.error msg.dump, error: e, host: addr[3]\n          log.error_backtrace\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/string_util.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Compat\n    module StringUtil\n      def match_regexp(regexp, string)\n        begin\n          return regexp.match(string)\n        rescue ArgumentError => e\n          raise e unless e.message.index(\"invalid byte sequence in\".freeze).zero?\n          $log.info \"invalid byte sequence is replaced in `#{string}`\"\n          string = string.scrub('?')\n          retry\n        end\n        return true\n      end\n      module_function :match_regexp\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/structured_format_mixin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Compat\n    module StructuredFormatMixin\n      def format(tag, time, record)\n        filter_record(tag, time, record)\n        format_record(record)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/compat/type_converter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Compat\n    module TypeConverter\n      Converters = {\n        'string' => lambda { |v| v.to_s },\n        'integer' => lambda { |v| v.to_i },\n        'float' => lambda { |v| v.to_f },\n        'bool' => lambda { |v|\n          case v.downcase\n          when 'true', 'yes', '1'\n            true\n          else\n            false\n          end\n        },\n        'time' => lambda { |v, time_parser|\n          time_parser.parse(v)\n        },\n        'array' => lambda { |v, delimiter|\n          v.to_s.split(delimiter)\n        }\n      }\n\n      def self.included(klass)\n        klass.instance_eval {\n          config_param :types, :string, default: nil\n          config_param :types_delimiter, :string, default: ','\n          config_param :types_label_delimiter, :string, default: ':'\n        }\n      end\n\n      def configure(conf)\n        super\n\n        @type_converters = nil\n        @type_converters = parse_types_parameter unless @types.nil?\n      end\n\n      private\n\n      def convert_type(name, value)\n        converter = @type_converters[name]\n        converter.nil? ? value : converter.call(value)\n      end\n\n      def parse_types_parameter\n        converters = {}\n\n        @types.split(@types_delimiter).each { |pattern_name|\n          name, type, format = pattern_name.split(@types_label_delimiter, 3)\n          raise ConfigError, \"Type is needed\" if type.nil?\n\n          case type\n          when 'time'\n            require 'fluent/parser'\n            t_parser = Fluent::TextParser::TimeParser.new(format)\n            converters[name] = lambda { |v|\n              Converters[type].call(v, t_parser)\n            }\n          when 'array'\n            delimiter = format || ','\n            converters[name] = lambda { |v|\n              Converters[type].call(v, delimiter)\n            }\n          else\n            converters[name] = Converters[type]\n          end\n        }\n\n        converters\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/basic_parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'stringio'\nrequire 'fluent/config/error'\n\nmodule Fluent\n  module Config\n    class BasicParser\n      def initialize(strscan)\n        @ss = strscan\n      end\n\n      LINE_END = /(?:[ \\t]*(?:\\#.*)?(?:\\z|[\\r\\n]))+/\n      SPACING = /(?:[ \\t\\r\\n]|\\z|\\#.*?(?:\\z|[\\r\\n]))+/\n      ZERO_OR_MORE_SPACING = /(?:[ \\t\\r\\n]|\\z|\\#.*?(?:\\z|[\\r\\n]))*/\n      SPACING_WITHOUT_COMMENT = /(?:[ \\t\\r\\n]|\\z)+/\n      LINE_END_WITHOUT_SPACING_AND_COMMENT = /(?:\\z|[\\r\\n])/\n\n      module ClassMethods\n        def symbol(string)\n          /#{Regexp.escape(string)}/\n        end\n\n        def def_symbol(method_name, string)\n          pattern = symbol(string)\n          define_method(method_name) do\n            skip(pattern) && string\n          end\n        end\n\n        def def_literal(method_name, string)\n          pattern = /#{string}#{LINE_END}/\n          define_method(method_name) do\n            skip(pattern) && string\n          end\n        end\n      end\n\n      extend ClassMethods\n\n      def skip(pattern)\n        @ss.skip(pattern)\n      end\n\n      def scan(pattern)\n        @ss.scan(pattern)\n      end\n\n      def getch\n        @ss.getch\n      end\n\n      def eof?\n        @ss.eos?\n      end\n\n      def prev_match\n        @ss[0]\n      end\n\n      def check(pattern)\n        @ss.check(pattern)\n      end\n\n      def line_end\n        skip(LINE_END)\n      end\n\n      def spacing\n        skip(SPACING)\n      end\n\n      def spacing_without_comment\n        skip(SPACING_WITHOUT_COMMENT)\n      end\n\n      def parse_error!(message)\n        raise ConfigParseError, \"#{message} at #{error_sample}\"\n      end\n\n      def error_sample\n        pos = @ss.pos\n\n        lines = @ss.string.lines.to_a\n        lines.each_with_index { |line, ln|\n          if line.size >= pos\n            msgs = [\"line #{ln + 1},#{pos}\\n\"]\n\n            if ln > 0\n              last_line = lines[ln - 1]\n              msgs << \"%3s: %s\" % [ln, last_line]\n            end\n\n            msgs << \"%3s: %s\" % [ln + 1, line]\n            msgs << \"\\n     #{'-' * pos}^\\n\"\n\n            if next_line = lines[ln + 1]\n              msgs << \"%3s: %s\" % [ln + 2, next_line]\n            end\n\n            return msgs.join\n          end\n          pos -= line.size\n          last_line = line\n        }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/configure_proxy.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Config\n    class ConfigureProxy\n      attr_accessor :name, :final, :param_name, :init, :required, :multi, :alias, :configured_in_section\n      attr_accessor :argument, :params, :defaults, :descriptions, :sections\n      # config_param :desc, :string, default: '....'\n      # config_set_default :buffer_type, :memory\n      #\n      # config_section :default, required: true, multi: false do\n      #   config_argument :arg, :string\n      #   config_param :required, :bool, default: false\n      #   config_param :name, :string\n      #   config_param :power, :integer\n      # end\n      #\n      # config_section :child, param_name: 'children', required: false, multi: true, alias: 'node' do\n      #   config_param :name, :string\n      #   config_param :power, :integer, default: nil\n      #   config_section :item do\n      #     config_param :name\n      #   end\n      # end\n\n      def initialize(name, root: false, param_name: nil, final: nil, init: nil, required: nil, multi: nil, alias: nil, type_lookup:)\n        @name = name.to_sym\n        @final = final\n\n        # For ConfigureProxy of root section, \"@name\" should be a class name of plugins.\n        # Otherwise (like subsections), \"@name\" should be a name of section, like \"buffer\", \"store\".\n        # For subsections, name will be used as parameter names (unless param_name exists), so overriding proxy's name\n        #   should override \"@name\".\n        @root_section = root\n\n        @param_name = param_name&.to_sym\n        @init = init\n        @required = required\n        @multi = multi\n        @alias = binding.local_variable_get(:alias)\n        @type_lookup = type_lookup\n\n        raise \"init and required are exclusive\" if @init && @required\n\n        # specify section name for viewpoint of owner(parent) plugin\n        # for buffer plugins: all params are in <buffer> section of owner\n        # others: <storage>, <format> (formatter/parser), ...\n        @configured_in_section = nil\n\n        @argument = nil # nil: ignore argument\n        @params = {}\n        @defaults = {}\n        @descriptions = {}\n        @sections = {}\n        @current_description = nil\n      end\n\n      def variable_name\n        @param_name || @name\n      end\n\n      def root?\n        @root_section\n      end\n\n      def init?\n        @init.nil? ? false : @init\n      end\n\n      def required?\n        @required.nil? ? false : @required\n      end\n\n      def multi?\n        @multi.nil? ? true : @multi\n      end\n\n      def final?\n        !!@final\n      end\n\n      def merge(other) # self is base class, other is subclass\n        return merge_for_finalized(other) if self.final?\n\n        [:param_name, :required, :multi, :alias, :configured_in_section].each do |prohibited_name|\n          if overwrite?(other, prohibited_name)\n            raise ConfigError, \"BUG: subclass cannot overwrite base class's config_section: #{prohibited_name}\"\n          end\n        end\n\n        options = {}\n        # param_name affects instance variable name, which is just \"internal\" of each plugins.\n        # so it must not be changed. base class's name (or param_name) is always used.\n        options[:param_name] = @param_name\n\n        # subclass cannot overwrite base class's definition\n        options[:init] = @init.nil? ? other.init : self.init\n        options[:required] = @required.nil? ? other.required : self.required\n        options[:multi] = @multi.nil? ? other.multi : self.multi\n        options[:alias] = @alias.nil? ? other.alias : self.alias\n        options[:final] = @final || other.final\n        options[:type_lookup] = @type_lookup\n\n        merged = if self.root?\n                   options[:root] = true\n                   self.class.new(other.name, **options)\n                 else\n                   self.class.new(@name, **options)\n                 end\n\n        # configured_in MUST be kept\n        merged.configured_in_section = self.configured_in_section || other.configured_in_section\n\n        merged.argument = other.argument || self.argument\n        merged.params = self.params.merge(other.params)\n        merged.defaults = self.defaults.merge(other.defaults)\n        merged.sections = {}\n        (self.sections.keys + other.sections.keys).uniq.each do |section_key|\n          self_section = self.sections[section_key]\n          other_section = other.sections[section_key]\n          merged_section = if self_section && other_section\n                             self_section.merge(other_section)\n                           elsif self_section || other_section\n                             self_section || other_section\n                           else\n                             raise \"BUG: both of self and other section are nil\"\n                           end\n          merged.sections[section_key] = merged_section\n        end\n\n        merged\n      end\n\n      def merge_for_finalized(other)\n        # list what subclass can do for finalized section\n        #  * append params/defaults/sections which are missing in superclass\n        #  * change default values of superclass\n        #  * overwrite init to make it enable to instantiate section objects with added default values\n\n        if other.final == false && overwrite?(other, :final)\n          raise ConfigError, \"BUG: subclass cannot overwrite finalized base class's config_section\"\n        end\n\n        [:param_name, :required, :multi, :alias, :configured_in_section].each do |prohibited_name|\n          if overwrite?(other, prohibited_name)\n            raise ConfigError, \"BUG: subclass cannot overwrite base class's config_section: #{prohibited_name}\"\n          end\n        end\n\n        options = {}\n        options[:param_name] = @param_name\n        options[:init] = @init || other.init\n        options[:required] = @required.nil? ? other.required : self.required\n        options[:multi] = @multi.nil? ? other.multi : self.multi\n        options[:alias] = @alias.nil? ? other.alias : self.alias\n        options[:final]  = true\n        options[:type_lookup] = @type_lookup\n\n        merged = if self.root?\n                   options[:root] = true\n                   self.class.new(other.name, **options)\n                 else\n                   self.class.new(@name, **options)\n                 end\n\n        merged.configured_in_section = self.configured_in_section || other.configured_in_section\n\n        merged.argument = self.argument || other.argument\n        merged.params = other.params.merge(self.params)\n        merged.defaults = self.defaults.merge(other.defaults)\n        merged.sections = {}\n        (self.sections.keys + other.sections.keys).uniq.each do |section_key|\n          self_section = self.sections[section_key]\n          other_section = other.sections[section_key]\n          merged_section = if self_section && other_section\n                             other_section.merge(self_section)\n                           elsif self_section || other_section\n                             self_section || other_section\n                           else\n                             raise \"BUG: both of self and other section are nil\"\n                           end\n          merged.sections[section_key] = merged_section\n        end\n\n        merged\n      end\n\n      def overwrite_defaults(other) # other is owner plugin's corresponding proxy\n        self.defaults = self.defaults.merge(other.defaults)\n        self.sections.each_key do |section_key|\n          if other.sections.has_key?(section_key)\n            self.sections[section_key].overwrite_defaults(other.sections[section_key])\n          end\n        end\n      end\n\n      def option_value_type!(name, opts, key, klass=nil, type: nil)\n        if opts.has_key?(key)\n          if klass && !opts[key].is_a?(klass)\n            raise ArgumentError, \"#{name}: #{key} must be a #{klass}, but #{opts[key].class}\"\n          end\n          case type\n          when :boolean\n            unless opts[key].is_a?(TrueClass) || opts[key].is_a?(FalseClass)\n              raise ArgumentError, \"#{name}: #{key} must be true or false, but #{opts[key].class}\"\n            end\n          when nil\n            # ignore\n          else\n            raise \"unknown type: #{type} for option #{key}\"\n          end\n        end\n      end\n\n      def config_parameter_option_validate!(name, type, **kwargs, &block)\n        if type.nil? && !block\n          type = :string\n        end\n        kwargs.each_key do |key|\n          case key\n          when :default, :alias, :secret, :skip_accessor, :deprecated, :obsoleted, :desc\n            # valid for all types\n          when :list\n            raise ArgumentError, \":list is valid only for :enum type, but #{type}: #{name}\" if type != :enum\n          when :value_type\n            raise ArgumentError, \":value_type is valid only for :hash and :array, but #{type}: #{name}\" if type != :hash && type != :array\n          when :symbolize_keys\n            raise ArgumentError, \":symbolize_keys is valid only for :hash, but #{type}: #{name}\" if type != :hash\n          else\n            raise ArgumentError, \"unknown option '#{key}' for configuration parameter: #{name}\"\n          end\n        end\n      end\n\n      def parameter_configuration(name, type = nil, **kwargs, &block)\n        config_parameter_option_validate!(name, type, **kwargs, &block)\n\n        name = name.to_sym\n\n        if block && type\n          raise ArgumentError, \"#{name}: both of block and type cannot be specified\"\n        elsif !block && !type\n          type = :string\n        end\n        opts = {}\n        opts[:type] = type\n        opts.merge!(kwargs)\n\n        begin\n          block ||= @type_lookup.call(type)\n        rescue ConfigError\n          # override error message\n          raise ArgumentError, \"#{name}: unknown config_argument type `#{type}'\"\n        end\n\n        # options for config_param\n        option_value_type!(name, opts, :desc, String)\n        option_value_type!(name, opts, :alias, Symbol)\n        option_value_type!(name, opts, :secret, type: :boolean)\n        option_value_type!(name, opts, :deprecated, String)\n        option_value_type!(name, opts, :obsoleted, String)\n        if type == :enum\n          if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?(Symbol)\n            raise ArgumentError, \"#{name}: enum parameter requires :list of Symbols\"\n          end\n        end\n        option_value_type!(name, opts, :symbolize_keys, type: :boolean)\n        option_value_type!(name, opts, :value_type, Symbol) # hash, array\n        option_value_type!(name, opts, :skip_accessor, type: :boolean)\n\n        if opts.has_key?(:default)\n          config_set_default(name, opts[:default])\n        end\n\n        if opts.has_key?(:desc)\n          config_set_desc(name, opts[:desc])\n        end\n\n        if opts[:deprecated] && opts[:obsoleted]\n          raise ArgumentError, \"#{name}: both of deprecated and obsoleted cannot be specified at once\"\n        end\n\n        [name, block, opts]\n      end\n\n      def configured_in(section_name)\n        if @configured_in_section\n          raise ArgumentError, \"#{self.name}: configured_in called twice\"\n        end\n        @configured_in_section = section_name.to_sym\n      end\n\n      def config_argument(name, type = nil, **kwargs, &block)\n        if @argument\n          raise ArgumentError, \"#{self.name}: config_argument called twice\"\n        end\n        name, block, opts = parameter_configuration(name, type, **kwargs, &block)\n\n        @argument = [name, block, opts]\n        name\n      end\n\n      def config_param(name, type = nil, **kwargs, &block)\n        name, block, opts = parameter_configuration(name, type, **kwargs, &block)\n\n        if @current_description\n          config_set_desc(name, @current_description)\n          @current_description = nil\n        end\n\n        @sections.delete(name)\n        @params[name] = [block, opts]\n        name\n      end\n\n      def config_set_default(name, defval)\n        name = name.to_sym\n\n        if @defaults.has_key?(name)\n          raise ArgumentError, \"#{self.name}: default value specified twice for #{name}\"\n        end\n\n        @defaults[name] = defval\n        nil\n      end\n\n      def config_set_desc(name, description)\n        name = name.to_sym\n\n        if @descriptions.has_key?(name)\n          raise ArgumentError, \"#{self.name}: description specified twice for #{name}\"\n        end\n\n        @descriptions[name] = description\n        nil\n      end\n\n      def desc(description)\n        @current_description = description\n      end\n\n      def config_section(name, **kwargs, &block)\n        unless block_given?\n          raise ArgumentError, \"#{name}: config_section requires block parameter\"\n        end\n        name = name.to_sym\n\n        sub_proxy = ConfigureProxy.new(name, type_lookup: @type_lookup, **kwargs)\n        sub_proxy.instance_exec(&block)\n\n        @params.delete(name)\n        @sections[name] = sub_proxy\n\n        name\n      end\n\n      def dump_config_definition\n        dumped_config = {}\n        if @argument\n          argument_name, _block, options = @argument\n          options[:required] = !@defaults.key?(argument_name)\n          options[:argument] = true\n          dumped_config[argument_name] = options\n        end\n        @params.each do |name, config|\n          dumped_config[name] = config[1]\n          dumped_config[name][:required] = !@defaults.key?(name)\n          dumped_config[name][:default] = @defaults[name] if @defaults.key?(name)\n          dumped_config[name][:desc] = @descriptions[name] if @descriptions.key?(name)\n        end\n        # Overwrite by config_set_default\n        @defaults.each do |name, value|\n          if @params.key?(name) || (@argument && @argument.first == name)\n            dumped_config[name][:default] = value\n          else\n            dumped_config[name] = { default: value }\n          end\n        end\n        # Overwrite by config_set_desc\n        @descriptions.each do |name, value|\n          if @params.key?(name)\n            dumped_config[name][:desc] = value\n          else\n            dumped_config[name] = { desc: value }\n          end\n        end\n        @sections.each do |section_name, sub_proxy|\n          if dumped_config.key?(section_name)\n            dumped_config[section_name].update(sub_proxy.dump_config_definition)\n          else\n            dumped_config[section_name] = sub_proxy.dump_config_definition\n            dumped_config[section_name][:required] = sub_proxy.required?\n            dumped_config[section_name][:multi] = sub_proxy.multi?\n            dumped_config[section_name][:alias] = sub_proxy.alias\n            dumped_config[section_name][:section] = true\n          end\n        end\n        dumped_config\n      end\n\n      private\n\n      def overwrite?(other, attribute_name)\n        value = instance_variable_get(\"@#{attribute_name}\")\n        other_value = other.__send__(attribute_name)\n        !value.nil? && !other_value.nil? && value != other_value\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/dsl.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'json'\n\nrequire 'fluent/config'\nrequire 'fluent/config/element'\n\nmodule Fluent\n  module Config\n    module DSL\n      RESERVED_PARAMETERS = [:type, :id, :log_level, :label] # Need '@' prefix for reserved parameters\n\n      module Parser\n        def self.read(path)\n          path = File.expand_path(path)\n          data = File.read(path)\n          parse(data, path)\n        end\n\n        def self.parse(source, source_path=\"config.rb\")\n          Proxy.new('ROOT', nil).eval(source, source_path).to_config_element\n        end\n      end\n\n      class Proxy\n        def initialize(name, arg, include_basepath = Dir.pwd)\n          @element = Element.new(name, arg, self)\n          @include_basepath = include_basepath\n        end\n\n        def element\n          @element\n        end\n\n        def include_basepath\n          @include_basepath\n        end\n\n        def eval(source, source_path)\n          @element.instance_eval(source, source_path)\n          self\n        end\n\n        def to_config_element\n          @element.instance_eval do\n            Config::Element.new(@name, @arg, @attrs, @elements)\n          end\n        end\n\n        def add_element(name, arg, block)\n          ::Kernel.raise ::ArgumentError, \"#{name} block must be specified\" if block.nil?\n\n          proxy = self.class.new(name.to_s, arg)\n          proxy.element.instance_exec(&block)\n\n          @element.instance_eval do\n            @elements.push(proxy.to_config_element)\n          end\n\n          self\n        end\n      end\n\n      class Element < BasicObject\n        def initialize(name, arg, proxy)\n          @name     = name\n          @arg      = arg || ''\n          @attrs    = {}\n          @elements = []\n          @proxy    = proxy\n        end\n\n        def to_int\n          __id__\n        end\n\n        def method_missing(name, *args, &block)\n          ::Kernel.raise ::ArgumentError, \"Configuration DSL Syntax Error: only one argument allowed\" if args.size > 1\n          value = args.first\n\n          if block\n            proxy = Proxy.new(name.to_s, value)\n            proxy.element.instance_exec(&block)\n            @elements.push(proxy.to_config_element)\n          else\n            param_name = RESERVED_PARAMETERS.include?(name) ? \"@#{name}\" : name.to_s\n            @attrs[param_name] = if value.is_a?(Array) || value.is_a?(Hash)\n                                   JSON.dump(value)\n                                 else\n                                   value.to_s\n                                 end\n          end\n\n          self\n        end\n\n        def include(*args)\n          ::Kernel.raise ::ArgumentError, \"#{name} block requires arguments for include path\" if args.nil? || args.size != 1\n          if /\\.rb$/.match?(args.first)\n            path = File.expand_path(args.first)\n            data = File.read(path)\n            self.instance_eval(data, path)\n          else\n            ss = StringScanner.new('')\n            Config::V1Parser.new(ss, @proxy.include_basepath, '', nil).eval_include(@attrs, @elements, args.first)\n          end\n        end\n\n        def source(&block)\n          @proxy.add_element('source', nil, block)\n        end\n\n        def match(*args, &block)\n          ::Kernel.raise ::ArgumentError, \"#{name} block requires arguments for match pattern\" if args.nil? || args.size != 1\n          @proxy.add_element('match', args.first, block)\n        end\n\n        def self.const_missing(name)\n          return ::Kernel.const_get(name) if ::Kernel.const_defined?(name)\n\n          if name.to_s =~ /^Fluent::Config::DSL::Element::(.*)$/\n            name = \"#{$1}\".to_sym\n            return ::Kernel.const_get(name) if ::Kernel.const_defined?(name)\n          end\n          ::Kernel.eval(\"#{name}\")\n        end\n\n        def ruby(&block)\n          if block\n            @proxy.instance_exec(&block)\n          else\n            ::Kernel\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/element.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/error'\nrequire 'fluent/config/literal_parser'\n\nmodule Fluent\n  module Config\n    class Element < Hash\n      def initialize(name, arg, attrs, elements, unused = nil)\n        @name = name\n        @arg = arg\n        @elements = elements\n        super()\n        attrs.each { |k, v|\n          self[k] = v\n        }\n        @unused = unused || attrs.keys\n        @v1_config = false\n        @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.\n        @unused_in = nil # if this element is not used in plugins, corresponding plugin name and parent element name is set, e.g. [source, plugin class].\n\n        # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.\n        @logger = defined?($log) ? $log : nil\n\n        @target_worker_ids = []\n      end\n\n      attr_accessor :name, :arg, :unused, :v1_config, :corresponding_proxies, :unused_in\n      attr_writer :elements\n      attr_reader :target_worker_ids\n\n      RESERVED_PARAMETERS_COMPAT = {\n        '@type' => 'type',\n        '@id' => 'id',\n        '@log_level' => 'log_level',\n        '@label' => nil,\n      }\n      RESERVED_PARAMETERS = RESERVED_PARAMETERS_COMPAT.keys\n\n      def elements(*names, name: nil, arg: nil)\n        raise ArgumentError, \"name and names are exclusive\" if name && !names.empty?\n        raise ArgumentError, \"arg is available only with name\" if arg && !name\n\n        if name\n          @elements.select{|e| e.name == name && (!arg || e.arg == arg) }\n        elsif !names.empty?\n          @elements.select{|e| names.include?(e.name) }\n        else\n          @elements\n        end\n      end\n\n      def add_element(name, arg = '')\n        e = Element.new(name, arg, {}, [])\n        e.v1_config = @v1_config\n        @elements << e\n        e\n      end\n\n      def inspect\n        attrs = super\n        \"<name:#{@name}, arg:#{@arg}, attrs:#{attrs}, elements:#{@elements.inspect}>\"\n      end\n\n      # Used by PP and Pry\n      def pretty_print(q)\n        q.text(inspect)\n      end\n\n      # This method assumes _o_ is an Element object. Should return false for nil or other object\n      def ==(o)\n        self.name == o.name && self.arg == o.arg &&\n          self.keys.size == o.keys.size &&\n          self.keys.reduce(true){|r, k| r && self[k] == o[k] } &&\n          self.elements.size == o.elements.size &&\n          [self.elements, o.elements].transpose.reduce(true){|r, e| r && e[0] == e[1] }\n      end\n\n      def +(o)\n        e = Element.new(@name.dup, @arg.dup, o.merge(self), @elements + o.elements, (@unused + o.unused).uniq)\n        e.v1_config = @v1_config\n        e\n      end\n\n      # no code in fluentd uses this method\n      def each_element(*names, &block)\n        if names.empty?\n          @elements.each(&block)\n        else\n          @elements.each { |e|\n            if names.include?(e.name)\n              block.yield(e)\n            end\n          }\n        end\n      end\n\n      def has_key?(key)\n        @unused_in = [] # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.\n        @unused.delete(key)\n        super\n      end\n\n      def [](key)\n        @unused_in = [] # ditto\n        @unused.delete(key)\n\n        if RESERVED_PARAMETERS.include?(key) && !has_key?(key) && has_key?(RESERVED_PARAMETERS_COMPAT[key])\n          @logger.warn \"'#{RESERVED_PARAMETERS_COMPAT[key]}' is deprecated parameter name. use '#{key}' instead.\" if @logger\n          return self[RESERVED_PARAMETERS_COMPAT[key]]\n        end\n\n        super\n      end\n\n      def check_not_fetched(&block)\n        each_key { |key|\n          if @unused.include?(key)\n            block.call(key, self)\n          end\n        }\n        @elements.each { |e|\n          e.check_not_fetched(&block)\n        }\n      end\n\n      def to_s(nest = 0)\n        indent = \"  \" * nest\n        nindent = \"  \" * (nest + 1)\n        out = \"\"\n        if @arg.nil? || @arg.empty?\n          out << \"#{indent}<#{@name}>\\n\"\n        else\n          out << \"#{indent}<#{@name} #{@arg}>\\n\"\n        end\n        each_pair { |k, v|\n          out << dump_value(k, v, nindent)\n        }\n        @elements.each { |e|\n          out << e.to_s(nest + 1)\n        }\n        out << \"#{indent}</#{@name}>\\n\"\n        out\n      end\n\n      def to_masked_element\n        new_elems = @elements.map { |e| e.to_masked_element }\n        new_elem = Element.new(@name, @arg, {}, new_elems, @unused)\n        new_elem.v1_config = @v1_config\n        new_elem.corresponding_proxies = @corresponding_proxies\n        each_pair { |k, v|\n          new_elem[k] = secret_param?(k) ? 'xxxxxx' : v\n        }\n        new_elem\n      end\n\n      def secret_param?(key)\n        return false if @corresponding_proxies.empty?\n\n        param_key = key.to_sym\n        @corresponding_proxies.each { |proxy|\n          _block, opts = proxy.params[param_key]\n          if opts && opts.has_key?(:secret)\n            return opts[:secret]\n          end\n        }\n\n        false\n      end\n\n      def param_type(key)\n        return nil if @corresponding_proxies.empty?\n\n        param_key = key.to_sym\n        proxy = @corresponding_proxies.detect do |_proxy|\n          _proxy.params.has_key?(param_key)\n        end\n        return nil unless proxy\n        _block, opts = proxy.params[param_key]\n        opts[:type]\n      end\n\n      def default_value(key)\n        return nil if @corresponding_proxies.empty?\n\n        param_key = key.to_sym\n        proxy = @corresponding_proxies.detect do |_proxy|\n          _proxy.params.has_key?(param_key)\n        end\n        return nil unless proxy\n        proxy.defaults[param_key]\n      end\n\n      def dump_value(k, v, nindent)\n        return \"#{nindent}#{k} xxxxxx\\n\" if secret_param?(k)\n        return \"#{nindent}#{k} #{v}\\n\" unless @v1_config\n\n        # for v1 config\n        if v.nil?\n          \"#{nindent}#{k} \\n\"\n        elsif v == :default\n          \"#{nindent}#{k} #{default_value(k)}\\n\"\n        else\n          case param_type(k)\n          when :string\n            \"#{nindent}#{k} \\\"#{self.class.unescape_parameter(v)}\\\"\\n\"\n          when :enum, :integer, :float, :size, :bool, :time\n            \"#{nindent}#{k} #{v}\\n\"\n          when :hash, :array\n            \"#{nindent}#{k} #{v}\\n\"\n          else\n            # Unknown type\n            \"#{nindent}#{k} #{v}\\n\"\n          end\n        end\n      end\n\n      def self.unescape_parameter(v)\n        result = ''\n        v.each_char { |c| result << LiteralParser.unescape_char(c) }\n        result\n      end\n\n      def set_target_worker_id(worker_id)\n        @target_worker_ids = [worker_id]\n        @elements.each { |e|\n          e.set_target_worker_id(worker_id)\n        }\n      end\n\n      def set_target_worker_ids(worker_ids)\n        @target_worker_ids = worker_ids.uniq\n        @elements.each { |e|\n          e.set_target_worker_ids(worker_ids.uniq)\n        }\n      end\n\n      def for_every_workers?\n        @target_worker_ids.empty?\n      end\n\n      def for_this_worker?\n        @target_worker_ids.include?(Fluent::Engine.worker_id)\n      end\n\n      def for_another_worker?\n        !@target_worker_ids.empty? && !@target_worker_ids.include?(Fluent::Engine.worker_id)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/error.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  class ConfigError < StandardError\n  end\n\n  class ConfigParseError < ConfigError\n  end\n\n  class ObsoletedParameterError < ConfigError\n  end\n\n  class SetNil < Exception\n  end\n\n  class SetDefault < Exception\n  end\n\n  class NotFoundPluginError < ConfigError\n    attr_reader :type, :kind\n\n    def initialize(msg, type: nil, kind: nil)\n      @msg = msg\n      @type = type\n      @kind = kind\n\n      super(msg)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/literal_parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'stringio'\n\nrequire 'json'\nrequire 'yajl'\nrequire 'socket'\nrequire 'ripper'\n\nrequire 'fluent/config/basic_parser'\n\nmodule Fluent\n  module Config\n    class LiteralParser < BasicParser\n      def self.unescape_char(c)\n        case c\n        when '\"'\n          '\\\"'\n        when \"'\"\n          \"\\\\'\"\n        when '\\\\'\n          '\\\\\\\\'\n        when \"\\r\"\n          '\\r'\n        when \"\\n\"\n          '\\n'\n        when \"\\t\"\n          '\\t'\n        when \"\\f\"\n          '\\f'\n        when \"\\b\"\n          '\\b'\n        else\n          c\n        end\n      end\n\n      def initialize(strscan, eval_context)\n        super(strscan)\n        @eval_context = eval_context\n        unless @eval_context.respond_to?(:use_nil)\n          def @eval_context.use_nil\n            raise SetNil\n          end\n        end\n        unless @eval_context.respond_to?(:use_default)\n          def @eval_context.use_default\n            raise SetDefault\n          end\n        end\n      end\n\n      def parse_literal(string_boundary_charset = LINE_END)\n        spacing_without_comment\n\n        value = if skip(/\\[/)\n                  scan_json(true)\n                elsif skip(/\\{/)\n                  scan_json(false)\n                else\n                  scan_string(string_boundary_charset)\n                end\n        value\n      end\n\n      def scan_string(string_boundary_charset = LINE_END)\n        if skip(/\\\"/)\n          return scan_double_quoted_string\n        elsif skip(/\\'/)\n          return scan_single_quoted_string\n        else\n          return scan_nonquoted_string(string_boundary_charset)\n        end\n      end\n\n      def scan_double_quoted_string\n        string = []\n        while true\n          if skip(/\\\"/)\n            if string.include?(nil)\n              return nil\n            elsif string.include?(:default)\n              return :default\n            else\n              return string.join\n            end\n          elsif check(/[^\"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)\n            if s = check(/[^\\\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)\n              string << s\n            end\n            skip(/[^\"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)\n          elsif s = scan(/\\\\./)\n            string << eval_escape_char(s[1,1])\n          elsif skip(/\\#\\{/)\n            string << eval_embedded_code(scan_embedded_code)\n            skip(/\\}/)\n          elsif s = scan(/./)\n            string << s\n          else\n            parse_error! \"unexpected end of file in a double quoted string\"\n          end\n        end\n      end\n\n      def scan_single_quoted_string\n        string = []\n        while true\n          if skip(/\\'/)\n            return string.join\n          elsif s = scan(/\\\\'/)\n            string << \"'\"\n          elsif s = scan(/\\\\\\\\/)\n            string << \"\\\\\"\n          elsif s = scan(/./)\n            string << s\n          else\n            parse_error! \"unexpected end of file in a single quoted string\"\n          end\n        end\n      end\n\n      def scan_nonquoted_string(boundary_charset = LINE_END)\n        charset = /(?!#{boundary_charset})./\n\n        string = []\n        while true\n          if s = scan(/\\#/)\n            string << '#'\n          elsif s = scan(charset)\n            string << s\n          else\n            break\n          end\n        end\n\n        if string.empty?\n          return nil\n        end\n\n        string.join\n      end\n\n      def scan_embedded_code\n        src = '\"#{'+@ss.rest+\"\\n=begin\\n=end\\n}\"\n\n        seek = -1\n        while (seek = src.index('}', seek + 1))\n          unless Ripper.sexp(src[0..seek] + '\"').nil? # eager parsing until valid expression\n            break\n          end\n        end\n\n        unless seek\n          raise Fluent::ConfigParseError, @ss.rest\n        end\n\n        code = src[3, seek-3]\n\n        if @ss.rest.length < code.length\n          @ss.pos += @ss.rest.length\n          parse_error! \"expected end of embedded code but $end\"\n        end\n\n        @ss.pos += code.length\n\n        '\"#{' + code + '}\"'\n      end\n\n      def eval_embedded_code(code)\n        if @eval_context.nil?\n          parse_error! \"embedded code is not allowed in this file\"\n        end\n        # Add hostname and worker_id to code for preventing unused warnings\n        code = <<EOM + code\nhostname = Socket.gethostname\nworker_id = ENV['SERVERENGINE_WORKER_ID'] || ''\nEOM\n        begin\n          @eval_context.instance_eval(code)\n        rescue SetNil\n          nil\n        rescue SetDefault\n          :default\n        end\n      end\n\n      def eval_escape_char(c)\n        case c\n        when '\"'\n          '\"'\n        when \"'\"\n          \"'\"\n        when \"r\"\n          \"\\r\"\n        when \"n\"\n          \"\\n\"\n        when \"t\"\n          \"\\t\"\n        when \"f\"\n          \"\\f\"\n        when \"b\"\n          \"\\b\"\n        when \"0\"\n          \"\\0\"\n        when /[a-zA-Z0-9]/\n          parse_error! \"unexpected back-slash escape character '#{c}'\"\n        else  # symbols\n          c\n        end\n      end\n\n      def scan_json(is_array)\n        result = nil\n        # Yajl does not raise ParseError for incomplete json string, like '[1', '{\"h\"', '{\"h\":' or '{\"h1\":1'\n        # This is the reason to use JSON module.\n\n        buffer = (is_array ? \"[\" : \"{\")\n        line_buffer = \"\"\n\n        until result\n          char = getch\n\n          break if char.nil?\n\n          if char == \"#\"\n            # If this is out of json string literals, this object can be parsed correctly\n            # '{\"foo\":\"bar\", #' -> '{\"foo\":\"bar\"}' (to check)\n            parsed = nil\n            begin\n              parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? \"]\" : \"}\"))\n            rescue JSON::ParserError\n              # This '#' is in json string literals\n            end\n\n            if parsed\n              # ignore chars as comment before newline\n              while (char = getch) != \"\\n\"\n                # ignore comment char\n              end\n              buffer << line_buffer + \"\\n\"\n              line_buffer = \"\"\n            else\n              if @ss.exist?(/^\\{[^}]+\\}/)\n                # if it's interpolated string\n                skip(/\\{/)\n                line_buffer << eval_embedded_code(scan_embedded_code)\n                skip(/\\}/)\n              else\n                # '#' is a char in json string\n                line_buffer << char\n              end\n            end\n\n            next # This char '#' MUST NOT terminate json object.\n          end\n\n          if char == \"\\n\"\n            buffer << line_buffer + \"\\n\"\n            line_buffer = \"\"\n            next\n          end\n\n          line_buffer << char\n          begin\n            result = JSON.parse(buffer + line_buffer)\n          rescue JSON::ParserError\n            # Incomplete json string yet\n          end\n        end\n\n        unless result\n          parse_error! \"got incomplete JSON #{is_array ? 'array' : 'hash'} configuration\"\n        end\n\n        JSON.dump(result)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'uri'\n\nrequire 'fluent/config/error'\nrequire 'fluent/config/element'\n\nmodule Fluent\n  module Config\n    class Parser\n      def self.parse(io, fname, basepath = Dir.pwd)\n        attrs, elems = Parser.new(basepath, io.each_line, fname).parse!(true)\n        Element.new('ROOT', '', attrs, elems)\n      end\n\n      def initialize(basepath, iterator, fname, i = 0)\n        @basepath = basepath\n        @iterator = iterator\n        @i = i\n        @fname = fname\n      end\n\n      def parse!(allow_include, elem_name = nil, attrs = {}, elems = [])\n        while line = @iterator.next\n          line.force_encoding('UTF-8')\n          @i += 1\n          line.lstrip!\n          line.gsub!(/\\s*(?:\\#.*)?$/,'')\n          if line.empty?\n            next\n          elsif m = /^\\<include\\s*(.*)\\s*\\/\\>$/.match(line)\n            value = m[1].strip\n            process_include(attrs, elems, value, allow_include)\n          elsif m = /^\\<([a-zA-Z0-9_]+)\\s*(.+?)?\\>$/.match(line)\n            e_name = m[1]\n            e_arg = m[2] || \"\"\n            e_attrs, e_elems = parse!(false, e_name)\n            elems << Element.new(e_name, e_arg, e_attrs, e_elems)\n          elsif line == \"</#{elem_name}>\"\n            break\n          elsif m = /^([a-zA-Z0-9_]+)\\s*(.*)$/.match(line)\n            key = m[1]\n            value = m[2]\n            if allow_include && key == 'include'\n              process_include(attrs, elems, value)\n            else\n              attrs[key] = value\n            end\n            next\n          else\n            raise ConfigParseError, \"parse error at #{@fname} line #{@i}\"\n          end\n        end\n\n        return attrs, elems\n      rescue StopIteration\n        return attrs, elems\n      end\n\n      def process_include(attrs, elems, uri, allow_include = true)\n        u = URI.parse(uri)\n        if u.scheme == 'file' || u.path == uri  # file path\n          path = u.path\n          if path[0] != ?/\n            pattern = File.expand_path(\"#{@basepath}/#{path}\")\n          else\n            pattern = path\n          end\n\n          Dir.glob(pattern).sort.each { |entry|\n            basepath = File.dirname(entry)\n            fname = File.basename(entry)\n            File.open(entry) { |f|\n              Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)\n            }\n          }\n\n        else\n          basepath = '/'\n          fname = path\n          parser_proc = ->(f) {\n            Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)\n          }\n\n          case u.scheme\n          when 'http', 'https', 'ftp'\n            # URI#open can be able to handle URIs for http, https and ftp.\n            require 'open-uri'\n            u.open(&parser_proc)\n          else\n            # TODO: This case should be handled in the previous if condition. Glob is not applied to some Windows path formats.\n            # 'c:/path/to/file' will be passed as URI, 'uri' and 'u.path' will be:\n            #   - uri is 'c:/path/to/file'\n            #   - u.path is '/path/to/file' and u.scheme is 'c'\n            # Therefore, the condition of the if statement above is not met and it is handled here.\n            File.open(uri, &parser_proc)\n          end\n        end\n\n      rescue SystemCallError => e\n        raise ConfigParseError, \"include error at #{@fname} line #{@i}: #{e.to_s}\"\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/section.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'json'\n\nrequire 'fluent/config/error'\nrequire 'fluent/config/v1_parser'\n\nmodule Fluent\n  module Config\n    class Section < BasicObject\n      def self.name\n        'Fluent::Config::Section'\n      end\n\n      def initialize(params = {}, config_element = nil)\n        @klass = 'Fluent::Config::Section'\n        @params = params\n        @corresponding_config_element = config_element\n      end\n\n      alias :object_id :__id__\n\n      def corresponding_config_element\n        @corresponding_config_element\n      end\n\n      def class\n        Section\n      end\n\n      def to_s\n        inspect\n      end\n\n      def inspect\n        \"<Fluent::Config::Section #{@params.to_json}>\"\n      end\n\n      # Used by PP and Pry\n      def pretty_print(q)\n        q.text(inspect)\n      end\n\n      def nil?\n        false\n      end\n\n      def to_h\n        @params\n      end\n\n      def dup\n        Section.new(@params.dup, @corresponding_config_element.dup)\n      end\n\n      def +(other)\n        Section.new(self.to_h.merge(other.to_h))\n      end\n\n      def instance_of?(mod)\n        @klass == mod.name\n      end\n\n      def kind_of?(mod)\n        @klass == mod.name || BasicObject == mod\n      end\n      alias is_a? kind_of?\n\n      def [](key)\n        @params[key.to_sym]\n      end\n\n      def []=(key, value)\n        @params[key.to_sym] = value\n      end\n\n      def respond_to?(symbol, include_all=false)\n        case symbol\n        when :inspect, :nil?, :to_h, :+, :instance_of?, :kind_of?, :[], :respond_to?, :respond_to_missing?\n          true\n        when :!, :!= , :==, :equal?, :instance_eval, :instance_exec\n          true\n        when :method_missing, :singleton_method_added, :singleton_method_removed, :singleton_method_undefined\n          include_all\n        else\n          false\n        end\n      end\n\n      def respond_to_missing?(symbol, include_private)\n        @params.has_key?(symbol)\n      end\n\n      def method_missing(name, *args)\n        if @params.has_key?(name)\n          @params[name]\n        else\n          ::Kernel.raise ::NoMethodError, \"undefined method `#{name}' for #{self.inspect}\"\n        end\n      end\n    end\n\n    module SectionGenerator\n      def self.generate(proxy, conf, logger, plugin_class, stack = [], strict_config_value = false)\n        return nil if conf.nil?\n\n        section_stack = \"\"\n        unless stack.empty?\n          section_stack = \", in section \" + stack.join(\" > \")\n        end\n\n        section_params = {}\n\n        proxy.defaults.each_pair do |name, defval|\n          varname = name.to_sym\n          section_params[varname] = (defval.dup rescue defval)\n        end\n\n        if proxy.argument\n          unless conf.arg.nil? || conf.arg.empty?\n            key, block, opts = proxy.argument\n            opts = opts.merge(strict: true) if strict_config_value\n\n            if conf.arg == :default\n              unless section_params.has_key?(key)\n                logger.error \"config error in:\\n#{conf}\" if logger\n                raise ConfigError, \"'#{key}' doesn't have default value\"\n              end\n            else\n              begin\n                section_params[key] = self.instance_exec(conf.arg, opts, key, &block)\n              rescue ConfigError => e\n                logger.error \"config error in:\\n#{conf}\" if logger\n                raise e\n              end\n            end\n          end\n          unless section_params.has_key?(proxy.argument.first)\n            logger.error \"config error in:\\n#{conf}\" if logger # logger should exist, but sometimes it's nil (e.g, in tests)\n            raise ConfigError, \"'<#{proxy.name} ARG>' section requires argument\" + section_stack\n          end\n          # argument should NOT be deprecated... (argument always has a value: '')\n        end\n\n        proxy.params.each_pair do |name, defval|\n          varname = name.to_sym\n          block, opts = defval\n          opts = opts.merge(strict: true) if strict_config_value\n\n          if conf.has_key?(name.to_s) || opts[:alias] && conf.has_key?(opts[:alias].to_s)\n            val = if conf.has_key?(name.to_s)\n                    conf[name.to_s]\n                  else\n                    conf[opts[:alias].to_s]\n                  end\n\n            if val == :default\n              # default value is already set if it exists\n              unless section_params.has_key?(varname)\n                logger.error \"config error in:\\n#{conf}\" if logger\n                raise ConfigError, \"'#{varname}' doesn't have default value\"\n              end\n            else\n              begin\n                section_params[varname] = self.instance_exec(val, opts, name, &block)\n              rescue ConfigError => e\n                logger.error \"config error in:\\n#{conf}\" if logger\n                raise e\n              end\n            end\n\n            if section_params[varname].nil?\n              unless proxy.defaults.has_key?(varname) && proxy.defaults[varname].nil?\n                logger.error \"config error in:\\n#{conf}\" if logger\n                raise ConfigError, \"'#{name}' parameter is required but nil is specified\"\n              end\n            end\n\n            # Source of definitions of deprecated/obsoleted:\n            # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features\n            #\n            # Deprecated: These deprecated features can still be used, but should be used with caution\n            #             because they are expected to be removed entirely sometime in the future.\n            # Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.\n            if opts[:deprecated]\n              logger.warn \"'#{name}' parameter is deprecated: #{opts[:deprecated]}\" if logger\n            end\n            if opts[:obsoleted]\n              logger.error \"config error in:\\n#{conf}\" if logger\n              raise ObsoletedParameterError, \"'#{name}' parameter is already removed: #{opts[:obsoleted]}\" + section_stack\n            end\n          end\n          unless section_params.has_key?(varname)\n            logger.error \"config error in:\\n#{conf}\" if logger\n            raise ConfigError, \"'#{name}' parameter is required\" + section_stack\n          end\n        end\n\n        check_unused_section(proxy, conf, plugin_class)\n\n        proxy.sections.each do |name, subproxy|\n          varname = subproxy.variable_name\n          elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }\n          if elements.empty? && subproxy.init?\n            if subproxy.argument && !subproxy.defaults.has_key?(subproxy.argument.first)\n              raise ArgumentError, \"#{name}: init is specified, but default value of argument is missing\"\n            end\n            missing_keys = subproxy.params.keys.select{|param_name| !subproxy.defaults.has_key?(param_name)}\n            if !missing_keys.empty?\n              raise ArgumentError, \"#{name}: init is specified, but there're parameters without default values:#{missing_keys.join(',')}\"\n            end\n            elements << Fluent::Config::Element.new(subproxy.name.to_s, '', {}, [])\n          end\n\n          # set subproxy for secret option\n          elements.each { |element|\n            element.corresponding_proxies << subproxy\n          }\n\n          if subproxy.required? && elements.size < 1\n            logger.error \"config error in:\\n#{conf}\" if logger\n            raise ConfigError, \"'<#{subproxy.name}>' sections are required\" + section_stack\n          end\n          if subproxy.multi?\n            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name], strict_config_value) }\n          else\n            if elements.size > 1\n              logger.error \"config error in:\\n#{conf}\" if logger\n              raise ConfigError, \"'<#{subproxy.name}>' section cannot be written twice or more\" + section_stack\n            end\n            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name], strict_config_value)\n          end\n        end\n\n        Section.new(section_params, conf)\n      end\n\n      def self.check_unused_section(proxy, conf, plugin_class)\n        elems = conf.respond_to?(:elements) ? conf.elements : []\n        elems.each { |e|\n          next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doesn't have proxy section\n          next if e.unused_in&.empty? # the section is used at least once\n\n          if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }\n            e.unused_in = []\n          else\n            parent_name = if conf.arg.empty?\n                            conf.name\n                          else\n                            \"#{conf.name} #{conf.arg}\"\n                          end\n            e.unused_in = [parent_name, plugin_class]\n          end\n        }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/types.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'json'\n\nrequire 'fluent/config/error'\n\nmodule Fluent\n  module Config\n    def self.reformatted_value(type, val, opts = {}, name = nil)\n      REFORMAT_VALUE.call(type, val, opts, name)\n    end\n\n    def self.size_value(str, opts = {}, name = nil)\n      return nil if str.nil?\n\n      case str.to_s\n      when /([0-9]+)k/i\n        $~[1].to_i * 1024\n      when /([0-9]+)m/i\n        $~[1].to_i * (1024 ** 2)\n      when /([0-9]+)g/i\n        $~[1].to_i * (1024 ** 3)\n      when /([0-9]+)t/i\n        $~[1].to_i * (1024 ** 4)\n      else\n        INTEGER_TYPE.call(str, opts, name)\n      end\n    end\n\n    def self.time_value(str, opts = {}, name = nil)\n      return nil if str.nil?\n\n      case str.to_s\n      when /([0-9]+)s/\n        $~[1].to_i\n      when /([0-9]+)m/\n        $~[1].to_i * 60\n      when /([0-9]+)h/\n        $~[1].to_i * 60 * 60\n      when /([0-9]+)d/\n        $~[1].to_i * 24 * 60 * 60\n      else\n        FLOAT_TYPE.call(str, opts, name)\n      end\n    end\n\n    def self.bool_value(str, opts = {}, name = nil)\n      return nil if str.nil?\n\n      case str.to_s\n      when 'true', 'yes'\n        true\n      when 'false', 'no'\n        false\n      when ''\n        true\n      else\n        # Current parser passes comment without actual values, e.g. \"param #foo\".\n        # parser should pass empty string in this case but changing behaviour may break existing environment so keep parser behaviour. Just ignore comment value in boolean handling for now.\n        if str.respond_to?(:start_with?) && str.start_with?('#')\n          true\n        elsif opts[:strict]\n          raise Fluent::ConfigError, \"#{name}: invalid bool value: #{str}\"\n        else\n          nil\n        end\n      end\n    end\n\n    def self.regexp_value(str, opts = {}, name = nil)\n      return nil unless str\n\n      return Regexp.compile(str) unless str.start_with?(\"/\")\n      right_slash_position = str.rindex(\"/\")\n      if right_slash_position < str.size - 3\n        raise Fluent::ConfigError, \"invalid regexp: missing right slash: #{str}\"\n      end\n      options = str[(right_slash_position + 1)..-1]\n      option = 0\n      option |= Regexp::IGNORECASE if options.include?(\"i\")\n      option |= Regexp::MULTILINE if options.include?(\"m\")\n      Regexp.compile(str[1...right_slash_position], option)\n    end\n\n    def self.string_value(val, opts = {}, name = nil)\n      return nil if val.nil?\n\n      v = val.to_s\n      v = v.frozen? ? v.dup : v # config_param can't assume incoming string is mutable\n      v.force_encoding(Encoding::UTF_8)\n    end\n\n    STRING_TYPE = Proc.new { |val, opts = {}, name = nil|\n      Config.string_value(val, opts, name)\n    }\n\n    def self.symbol_value(val, opts = {}, name = nil)\n      return nil if val.nil? || val.empty?\n\n      val.delete_prefix(\":\").to_sym\n    end\n\n    SYMBOL_TYPE = Proc.new { |val, opts = {}, name = nil|\n      Config.symbol_value(val, opts, name)\n    }\n\n    def self.enum_value(val, opts = {}, name = nil)\n      return nil if val.nil?\n\n      s = val.to_sym\n      list = opts[:list]\n      raise \"Plugin BUG: config type 'enum' requires :list of symbols\" unless list.is_a?(Array) && list.all?(Symbol)\n      unless list.include?(s)\n        raise ConfigError, \"valid options are #{list.join(',')} but got #{val}\"\n      end\n      s\n    end\n\n    ENUM_TYPE = Proc.new { |val, opts = {}, name = nil|\n      Config.enum_value(val, opts, name)\n    }\n\n    INTEGER_TYPE = Proc.new { |val, opts = {}, name = nil|\n      if val.nil?\n        nil\n      elsif opts[:strict]\n        begin\n          Integer(val)\n        rescue ArgumentError, TypeError => e\n          raise ConfigError, \"#{name}: #{e.message}\"\n        end\n      else\n        val.to_i\n      end\n    }\n\n    FLOAT_TYPE = Proc.new { |val, opts = {}, name = nil|\n      if val.nil?\n        nil\n      elsif opts[:strict]\n        begin\n          Float(val)\n        rescue ArgumentError, TypeError => e\n          raise ConfigError, \"#{name}: #{e.message}\"\n        end\n      else\n        val.to_f\n      end\n    }\n\n    SIZE_TYPE = Proc.new { |val, opts = {}, name = nil|\n      Config.size_value(val, opts, name)\n    }\n\n    BOOL_TYPE = Proc.new { |val, opts = {}, name = nil|\n      Config.bool_value(val, opts, name)\n    }\n\n    TIME_TYPE = Proc.new { |val, opts = {}, name = nil|\n      Config.time_value(val, opts, name)\n    }\n\n    REGEXP_TYPE = Proc.new { |val, opts = {}, name = nil|\n      Config.regexp_value(val, opts, name)\n    }\n\n    REFORMAT_VALUE = ->(type, value, opts = {}, name = nil) {\n      if value.nil?\n        value\n      else\n        case type\n        when :string  then value.to_s.force_encoding(Encoding::UTF_8)\n        when :integer then INTEGER_TYPE.call(value, opts, name)\n        when :float   then FLOAT_TYPE.call(value, opts, name)\n        when :size then Config.size_value(value, opts, name)\n        when :bool then Config.bool_value(value, opts, name)\n        when :time then Config.time_value(value, opts, name)\n        when :regexp then Config.regexp_value(value, opts, name)\n        when :symbol then Config.symbol_value(value, opts, name)\n        else\n          raise \"unknown type in REFORMAT: #{type}\"\n        end\n      end\n    }\n\n    def self.hash_value(val, opts = {}, name = nil)\n      return nil if val.nil?\n\n      param = if val.is_a?(String)\n                val.start_with?('{') ? JSON.parse(val) : Hash[val.strip.split(/\\s*,\\s*/).map{|v| v.split(':', 2)}]\n              else\n                val\n              end\n      if param.class != Hash\n        raise ConfigError, \"hash required but got #{val.inspect}\"\n      end\n      if opts.empty?\n        param\n      else\n        newparam = {}\n        param.each_pair do |key, value|\n          new_key = opts[:symbolize_keys] ? key.to_sym : key\n          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value, opts, new_key) : value\n        end\n        newparam\n      end\n    end\n\n    HASH_TYPE = Proc.new { |val, opts = {}, name = nil|\n      Config.hash_value(val, opts, name)\n    }\n\n    def self.array_value(val, opts = {}, name = nil)\n      return nil if val.nil?\n\n      param = if val.is_a?(String)\n                val.start_with?('[') ? JSON.parse(val) : val.strip.split(/\\s*,\\s*/)\n              else\n                val\n              end\n      if param.class != Array\n        raise ConfigError, \"array required but got #{val.inspect}\"\n      end\n      if opts[:value_type]\n        param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v, opts, nil) }\n      else\n        param\n      end\n    end\n\n    ARRAY_TYPE = Proc.new { |val, opts = {}, name = nil|\n      Config.array_value(val, opts, name)\n    }\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/v1_parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'strscan'\nrequire 'uri'\n\nrequire 'fluent/config/error'\nrequire 'fluent/config/basic_parser'\nrequire 'fluent/config/literal_parser'\nrequire 'fluent/config/element'\n\nmodule Fluent\n  module Config\n    class V1Parser < LiteralParser\n      ELEMENT_NAME = /[a-zA-Z0-9_]+/\n\n      def self.parse(data, fname, basepath = Dir.pwd, eval_context = nil, on_file_parsed: nil)\n        ss = StringScanner.new(data)\n        ps = V1Parser.new(ss, basepath, fname, eval_context, on_file_parsed: on_file_parsed)\n        ps.parse!\n      end\n\n      def initialize(strscan, include_basepath, fname, eval_context, on_file_parsed: nil)\n        super(strscan, eval_context)\n        @include_basepath = include_basepath\n        @fname = fname\n        @logger = defined?($log) ? $log : nil\n        @on_file_parsed = on_file_parsed\n      end\n\n      def parse!\n        attrs, elems = parse_element(true, nil)\n        root = Element.new('ROOT', '', attrs, elems)\n        root.v1_config = true\n\n        spacing\n        unless eof?\n          parse_error! \"expected EOF\"\n        end\n\n        return root\n      end\n\n      ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']\n      RESERVED_PARAMS = %W(@type @id @label @log_level)\n\n      def parse_element(root_element, elem_name, attrs = {}, elems = [])\n        while true\n          spacing\n          if eof?\n            if root_element\n              break\n            end\n            parse_error! \"expected end tag '</#{elem_name}>' but got end of file\"\n          end\n\n          if skip(/\\<\\//)\n            e_name = scan(ELEMENT_NAME)\n            spacing\n            unless skip(/\\>/)\n              parse_error! \"expected character in tag name\"\n            end\n            unless line_end\n              parse_error! \"expected end of line after end tag\"\n            end\n            if e_name != elem_name\n              parse_error! \"unmatched end tag\"\n            end\n            break\n\n          elsif skip(/\\</)\n            e_name = scan(ELEMENT_NAME)\n            spacing\n            e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\\>)/o)\n            spacing\n            unless skip(/\\>/)\n              parse_error! \"expected '>'\"\n            end\n            unless line_end\n              parse_error! \"expected end of line after tag\"\n            end\n            e_arg ||= ''\n            # call parse_element recursively\n            e_attrs, e_elems = parse_element(false, e_name)\n            new_e = Element.new(e_name, e_arg, e_attrs, e_elems)\n            new_e.v1_config = true\n            elems << new_e\n\n          elsif root_element && skip(/(\\@include|include)#{SPACING}/o)\n            if !prev_match.start_with?('@')\n              @logger.warn \"'include' is deprecated. Use '@include' instead\" if @logger\n            end\n            parse_include(attrs, elems)\n\n          else\n            k = scan_string(SPACING)\n            spacing_without_comment\n            if prev_match.include?(\"\\n\") || eof? # support 'tag_mapped' like \"without value\" configuration\n              attrs[k] = \"\"\n            else\n              if k == '@include'\n                parse_include(attrs, elems)\n              elsif RESERVED_PARAMS.include?(k)\n                v = parse_literal\n                unless line_end\n                  parse_error! \"expected end of line\"\n                end\n                attrs[k] = v\n              else\n                if k.start_with?('@')\n                  if root_element || ELEM_SYMBOLS.include?(elem_name)\n                    parse_error! \"'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}\"\n                  else\n                    # TODO: This is for backward compatibility. It will throw an error in the future.\n                    @logger.warn \"'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}\" if @logger\n                  end\n                end\n\n                v = parse_literal\n                unless line_end\n                  parse_error! \"expected end of line\"\n                end\n                attrs[k] = v\n              end\n            end\n          end\n        end\n\n        @on_file_parsed&.call(File.expand_path(File.join(@include_basepath, @fname))) if root_element\n\n        return attrs, elems\n      end\n\n      def parse_include(attrs, elems)\n        uri = scan_string(LINE_END)\n        eval_include(attrs, elems, uri)\n        line_end\n      end\n\n      def eval_include(attrs, elems, uri)\n        # replace space(s)(' ') with '+' to prevent invalid uri due to space(s).\n        # See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212\n        u = URI.parse(uri.tr(' ', '+'))\n        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.tr(' ', '+') # file path\n          # When the Windows absolute path then u.scheme.length == 1\n          # e.g. C:\n          path = URI.decode_www_form_component(u.path)\n          if path[0] != ?/\n            pattern = File.expand_path(\"#{@include_basepath}/#{path}\")\n          else\n            pattern = path\n          end\n          Dir.glob(pattern).sort.each { |entry|\n            basepath = File.dirname(entry)\n            fname = File.basename(entry)\n            suspicious_backup_extensions = %w(bak old backup orig prev conf tmp temp debug wip)\n            if path.end_with?('*.conf') and\n              suspicious_backup_extensions.any? { |ext| fname.end_with?(\".#{ext}.conf\", \"_#{ext}.conf\") }\n              @logger.warn \"There is a possibility that '@include #{uri}' includes duplicated backed-up config file such as <#{fname}>\" if @logger\n            end\n            data = File.read(entry)\n            data.force_encoding('UTF-8')\n            ss = StringScanner.new(data)\n            V1Parser.new(ss, basepath, fname, @eval_context, on_file_parsed: @on_file_parsed).parse_element(true, nil, attrs, elems)\n          }\n        else\n          require 'open-uri'\n          basepath = '/'\n          fname = path\n          data = u.open { |f| f.read }\n          data.force_encoding('UTF-8')\n          ss = StringScanner.new(data)\n          V1Parser.new(ss, basepath, fname, @eval_context, on_file_parsed: @on_file_parsed).parse_element(true, nil, attrs, elems)\n        end\n      rescue SystemCallError => e\n        cpe = ConfigParseError.new(\"include error #{uri} - #{e}\")\n        cpe.set_backtrace(e.backtrace)\n        raise cpe\n      end\n\n      # override\n      def error_sample\n        \"#{@fname} #{super}\"\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/yaml_parser/fluent_value.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Config\n    module YamlParser\n      module FluentValue\n        JsonValue = Struct.new(:val) do\n          def to_s\n            val.to_json\n          end\n\n          def to_element\n            to_s\n          end\n        end\n\n        StringValue = Struct.new(:val, :context) do\n          def to_s\n            context.instance_eval(\"\\\"#{val}\\\"\")\n          rescue Fluent::SetNil => _\n            ''\n          rescue Fluent::SetDefault => _\n            ':default'\n          end\n\n          def to_element\n            to_s\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/yaml_parser/loader.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'psych'\nrequire 'json'\nrequire 'fluent/config/error'\nrequire 'fluent/config/yaml_parser/fluent_value'\n\n# Based on https://github.com/eagletmt/hako/blob/34cdde06fe8f3aeafd794be830180c3cedfbb4dc/lib/hako/yaml_loader.rb\n\nmodule Fluent\n  module Config\n    module YamlParser\n      class Loader\n        INCLUDE_TAG = 'tag:include'.freeze\n        FLUENT_JSON_TAG = 'tag:fluent/json'.freeze\n        FLUENT_STR_TAG = 'tag:fluent/s'.freeze\n        SHOVEL = '<<'.freeze\n\n        def initialize(context = Kernel.binding, on_file_parsed: nil)\n          @context = context\n          @current_path = nil\n          @on_file_parsed = on_file_parsed\n        end\n\n        # @param [String] path\n        # @return [Hash]\n        def load(path)\n          class_loader = Psych::ClassLoader.new\n          scanner = Psych::ScalarScanner.new(class_loader)\n\n          visitor = Visitor.new(scanner, class_loader)\n\n          visitor._register_domain(INCLUDE_TAG) do |_, val|\n            eval_include(Pathname.new(val), path.parent)\n          end\n\n          visitor._register_domain(FLUENT_JSON_TAG) do |_, val|\n            Fluent::Config::YamlParser::FluentValue::JsonValue.new(val)\n          end\n\n          visitor._register_domain(FLUENT_STR_TAG) do |_, val|\n            Fluent::Config::YamlParser::FluentValue::StringValue.new(val, @context)\n          end\n\n          config = path.open do |f|\n            visitor.accept(Psych.parse(f))\n          end\n\n          @on_file_parsed&.call(File.expand_path(path.to_s))\n\n          config\n        end\n\n        def eval_include(path, parent)\n          if path.relative?\n            pattern = parent.join(path)\n          else\n            pattern = path\n          end\n          result = []\n          Dir.glob(pattern).sort.each do |path|\n            result.concat(load(Pathname.new(path)))\n          end\n          result\n        rescue SystemCallError => e\n          parse_error = ConfigParseError.new(\"include error #{path} - #{e}\")\n          parse_error.set_backtrace(e.backtrace)\n          raise parse_error\n        end\n\n        class Visitor < Psych::Visitors::ToRuby\n          def initialize(scanner, class_loader)\n            super(scanner, class_loader)\n          end\n\n          def _register_domain(name, &block)\n            @domain_types.merge!({ name => [name, block] })\n          end\n\n          def revive_hash(hash, o)\n            super(hash, o).tap do |r|\n              if r[SHOVEL].is_a?(Hash)\n                h2 = {}\n                r.each do |k, v|\n                  if k == SHOVEL\n                    h2.merge!(v)\n                  else\n                    h2[k] = v\n                  end\n                end\n                r.replace(h2)\n              end\n            end\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/yaml_parser/parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/yaml_parser/section_builder'\n\nmodule Fluent\n  module Config\n    module YamlParser\n      class Parser\n        def initialize(config, indent: 2)\n          @base_indent = indent\n          @config = config\n        end\n\n        def build\n          s = @config['system'] && system_config_build(@config['system'])\n          c = @config['config'] && config_build(@config['config'], root: true)\n          RootBuilder.new(s, c)\n        end\n\n        private\n\n        def system_config_build(config)\n          section_build('system', config)\n        end\n\n        def config_build(config, indent: 0, root: false)\n          sb = SectionBodyBuilder.new(indent, root: root)\n          config.each do |c|\n            if (lc = c.delete('label'))\n              sb.add_section(label_build(lc, indent: indent))\n            end\n\n            if (sc = c.delete('source'))\n              sb.add_section(source_build(sc, indent: indent))\n            end\n\n            if (fc = c.delete('filter'))\n              sb.add_section(filter_build(fc, indent: indent))\n            end\n\n            if (mc = c.delete('match'))\n              sb.add_section(match_build(mc, indent: indent))\n            end\n\n            if (wc = c.delete('worker'))\n              sb.add_section(worker_build(wc, indent: indent))\n            end\n\n            included_sections_build(c, sb, indent: indent)\n          end\n\n          sb\n        end\n\n        def label_build(config, indent: 0)\n          config = config.dup\n          name = config.delete('$name')\n          c = config.delete('config')\n          SectionBuilder.new('label', config_build(c, indent: indent + @base_indent), indent, name)\n        end\n\n        def worker_build(config, indent: 0)\n          config = config.dup\n          num = config.delete('$arg')\n          c = config.delete('config')\n          SectionBuilder.new('worker', config_build(c, indent: indent + @base_indent), indent, num)\n        end\n\n        def source_build(config, indent: 0)\n          section_build('source', config, indent: indent)\n        end\n\n        def filter_build(config, indent: 0)\n          config = config.dup\n          tag = config.delete('$tag')\n          if tag.is_a?(Array)\n            section_build('filter', config, indent: indent, arg: tag&.join(','))\n          else\n            section_build('filter', config, indent: indent, arg: tag)\n          end\n        end\n\n        def match_build(config, indent: 0)\n          config = config.dup\n          tag = config.delete('$tag')\n          if tag.is_a?(Array)\n            section_build('match', config, indent: indent, arg: tag&.join(','))\n          else\n            section_build('match', config, indent: indent, arg: tag)\n          end\n        end\n\n        def included_sections_build(config, section_builder, indent: 0)\n          config.each_entry do |e|\n            k = e.keys.first\n            cc = e.delete(k)\n            case k\n            when 'label'\n              section_builder.add_section(label_build(cc, indent: indent))\n            when 'worker'\n              section_builder.add_section(worker_build(cc, indent: indent))\n            when 'source'\n              section_builder.add_section(source_build(cc, indent: indent))\n            when 'filter'\n              section_builder.add_section(filter_build(cc, indent: indent))\n            when 'match'\n              section_builder.add_section(match_build(cc, indent: indent))\n            end\n          end\n        end\n\n        def section_build(name, config, indent: 0, arg: nil)\n          sb = SectionBodyBuilder.new(indent + @base_indent)\n\n          if (v = config.delete('$type'))\n            sb.add_line('@type', v)\n          end\n\n          if (v = config.delete('$label'))\n            sb.add_line('@label', v)\n          end\n\n          if (v = config.delete('$id'))\n            sb.add_line('@id', v)\n          end\n\n          if (v = config.delete('$log_level'))\n            sb.add_line('@log_level', v)\n          end\n\n          config.each do |key, val|\n            if val.is_a?(Array)\n              if section?(val.first)\n                val.each do |v|\n                  sb.add_section(section_build(key, v, indent: indent + @base_indent))\n                end\n              else\n                sb.add_line(key, val)\n              end\n            elsif val.is_a?(Hash)\n              harg = val.delete('$arg')\n              if harg.is_a?(Array)\n                # To prevent to generate invalid configuration for arg.\n                # \"arg\" should be String object and concatenated by \",\"\n                # when two or more objects are specified there.\n                sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg&.join(',')))\n              else\n                sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg))\n              end\n            else\n              sb.add_line(key, val)\n            end\n          end\n\n          SectionBuilder.new(name, sb, indent, arg)\n        end\n\n        def section?(value)\n          value.is_a?(Array) or value.is_a?(Hash)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/yaml_parser/section_builder.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Config\n    module YamlParser\n      SectionBuilder = Struct.new(:name, :body, :indent_size, :arg) do\n        def to_s\n          indent = ' ' * indent_size\n\n          if arg && !arg.to_s.empty?\n            \"#{indent}<#{name} #{arg}>\\n#{body}\\n#{indent}</#{name}>\"\n          else\n            \"#{indent}<#{name}>\\n#{body}\\n#{indent}</#{name}>\"\n          end\n        end\n\n        def to_element\n          elem = body.to_element\n          elem.name = name\n          elem.arg = arg.to_s if arg\n          elem.v1_config = true\n          elem\n        end\n      end\n\n      class RootBuilder\n        def initialize(system, conf)\n          @system = system\n          @conf = conf\n        end\n\n        attr_reader :system, :conf\n\n        def to_element\n          Fluent::Config::Element.new('ROOT', '', {}, [@system, @conf].compact.map(&:to_element).flatten)\n        end\n\n        def to_s\n          s = StringIO.new(+'')\n          s.puts(@system.to_s) if @system\n          s.puts(@conf.to_s) if @conf\n\n          s.string\n        end\n      end\n\n      class SectionBodyBuilder\n        Row = Struct.new(:key, :value, :indent) do\n          def to_s\n            \"#{indent}#{key} #{value}\"\n          end\n        end\n\n        def initialize(indent, root: false)\n          @indent = ' ' * indent\n          @bodies = []\n          @root = root\n        end\n\n        def add_line(k, v)\n          @bodies << Row.new(k, v, @indent)\n        end\n\n        def add_section(section)\n          @bodies << section\n        end\n\n        def to_element\n          if @root\n            return @bodies.map(&:to_element)\n          end\n\n          not_section, section = @bodies.partition { |e| e.is_a?(Row) }\n          r = {}\n          not_section.each do |e|\n            v = e.value\n            r[e.key] = v.respond_to?(:to_element) ? v.to_element : v\n          end\n\n          if @root\n            section.map(&:to_element)\n          else\n            Fluent::Config::Element.new('', '', r, section.map(&:to_element))\n          end\n        end\n\n        def to_s\n          @bodies.map(&:to_s).join(\"\\n\")\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config/yaml_parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/yaml_parser/loader'\nrequire 'fluent/config/yaml_parser/parser'\nrequire 'pathname'\n\nmodule Fluent\n  module Config\n    module YamlParser\n      def self.parse(path, on_file_parsed: nil)\n        context = Kernel.binding\n\n        unless context.respond_to?(:use_nil)\n          context.define_singleton_method(:use_nil) do\n            raise Fluent::SetNil\n          end\n        end\n\n        unless context.respond_to?(:use_default)\n          context.define_singleton_method(:use_default) do\n            raise Fluent::SetDefault\n          end\n        end\n\n        unless context.respond_to?(:hostname)\n          context.define_singleton_method(:hostname) do\n            Socket.gethostname\n          end\n        end\n\n        unless context.respond_to?(:worker_id)\n          context.define_singleton_method(:worker_id) do\n            ENV['SERVERENGINE_WORKER_ID'] || ''\n          end\n        end\n\n        s = Fluent::Config::YamlParser::Loader.new(context, on_file_parsed: on_file_parsed).load(Pathname.new(path))\n        Fluent::Config::YamlParser::Parser.new(s).build.to_element\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/config.rb",
    "content": "#\n# Fluent\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/error'\nrequire 'fluent/config/element'\nrequire 'fluent/configurable'\nrequire 'fluent/config/yaml_parser'\n\nmodule Fluent\n  module Config\n    # @param config_path [String] config file path\n    # @param encoding [String] encoding of config file\n    # @param additional_config [String] config which is added to last of config body\n    # @param use_v1_config [Bool] config is formatted with v1 or not\n    # @return [Fluent::Config]\n    def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil, on_file_parsed: nil)\n      if type == :guess\n        config_file_ext = File.extname(config_path)\n        if config_file_ext == '.yaml' || config_file_ext == '.yml'\n          type = :yaml\n        end\n      end\n\n      if type == :yaml || type == :yml\n        return Fluent::Config::YamlParser.parse(config_path, on_file_parsed: on_file_parsed)\n      end\n\n      config_fname = File.basename(config_path)\n      config_basedir = File.dirname(config_path)\n      config_data = File.open(config_path, \"r:#{encoding}:utf-8\") do |f|\n        s = f.read\n        if additional_config\n          c = additional_config.gsub(\"\\\\n\", \"\\n\")\n          s += \"\\n#{c}\"\n        end\n        s\n      end\n\n      Fluent::Config.parse(config_data, config_fname, config_basedir, use_v1_config, on_file_parsed: on_file_parsed)\n    end\n\n    def self.parse(str, fname, basepath = Dir.pwd, v1_config = nil, syntax: :v1, on_file_parsed: nil)\n      parser = if fname =~ /\\.rb$/ || syntax == :ruby\n                 :ruby\n               elsif v1_config.nil?\n                 case syntax\n                 when :v1 then :v1\n                 when :v0 then :v0\n                 else\n                   raise ArgumentError, \"Unknown Fluentd configuration syntax: '#{syntax}'\"\n                 end\n               elsif v1_config then :v1\n               else :v0\n               end\n      case parser\n      when :v1\n        require 'fluent/config/v1_parser'\n        V1Parser.parse(str, fname, basepath, Kernel.binding, on_file_parsed: on_file_parsed)\n      when :v0\n        # TODO: show deprecated message in v1\n        require 'fluent/config/parser'\n        Parser.parse(str, fname, basepath)\n      when :ruby\n        require 'fluent/config/dsl'\n        $log.warn(\"Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file\") if $log\n        Config::DSL::Parser.parse(str, File.join(basepath, fname))\n      else\n        raise \"[BUG] unknown configuration parser specification:'#{parser}'\"\n      end\n    end\n\n    def self.new(name = '')\n      Element.new(name, '', {}, [])\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/configurable.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/configure_proxy'\nrequire 'fluent/config/section'\nrequire 'fluent/config/error'\nrequire 'fluent/registry'\nrequire 'fluent/plugin'\nrequire 'fluent/config/types'\n\nmodule Fluent\n  module Configurable\n    def self.included(mod)\n      mod.extend(ClassMethods)\n    end\n\n    def initialize\n      super\n      # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default\n      proxy = self.class.merged_configure_proxy\n      proxy.params.each_key do |name|\n        next if name.to_s.start_with?('@')\n        if proxy.defaults.has_key?(name)\n          instance_variable_set(\"@#{name}\".to_sym, proxy.defaults[name])\n        end\n      end\n      proxy.sections.each_key do |name|\n        next if name.to_s.start_with?('@')\n        subproxy = proxy.sections[name]\n        if subproxy.multi?\n          instance_variable_set(\"@#{subproxy.variable_name}\".to_sym, [])\n        else\n          instance_variable_set(\"@#{subproxy.variable_name}\".to_sym, nil)\n        end\n      end\n    end\n\n    def configure_proxy_generate\n      proxy = self.class.merged_configure_proxy\n\n      if self.respond_to?(:owner) && self.owner\n        owner_proxy = owner.class.merged_configure_proxy\n        if proxy.configured_in_section\n          owner_proxy = owner_proxy.sections[proxy.configured_in_section]\n        end\n        proxy.overwrite_defaults(owner_proxy) if owner_proxy\n      end\n\n      proxy\n    end\n\n    def configured_section_create(name, conf = nil)\n      conf ||= Fluent::Config::Element.new(name.to_s, '', {}, [])\n      root_proxy = configure_proxy_generate\n      proxy = if name.nil? # root\n                root_proxy\n              else\n                root_proxy.sections[name]\n              end\n      # take care to raise Fluent::ConfigError if conf mismatched to proxy\n      Fluent::Config::SectionGenerator.generate(proxy, conf, nil, nil)\n    end\n\n    def configure(conf, strict_config_value=false)\n      @config = conf\n\n      logger = if self.respond_to?(:log)\n                 self.log\n               elsif self.respond_to?(:owner) && self.owner.respond_to?(:log)\n                 self.owner.log\n               elsif defined?($log)\n                 $log\n               else\n                 nil\n               end\n      proxy = configure_proxy_generate\n      conf.corresponding_proxies << proxy\n\n      # In the nested section, can't get plugin class through proxies so get plugin class here\n      plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)\n      root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class, [], strict_config_value)\n      @config_root_section = root\n\n      root.instance_eval{ @params.keys }.each do |param_name|\n        next if param_name.to_s.start_with?('@')\n        varname = \"@#{param_name}\".to_sym\n        instance_variable_set(varname, root[param_name])\n      end\n\n      self\n    end\n\n    def config\n      @masked_config ||= @config.to_masked_element\n    end\n\n    CONFIG_TYPE_REGISTRY = Registry.new(:config_type, 'fluent/plugin/type_')\n\n    def self.register_type(type, callable = nil, &block)\n      callable ||= block\n      CONFIG_TYPE_REGISTRY.register(type, callable)\n    end\n\n    def self.lookup_type(type)\n      CONFIG_TYPE_REGISTRY.lookup(type)\n    end\n\n    {\n      string: Config::STRING_TYPE,\n      enum: Config::ENUM_TYPE,\n      integer: Config::INTEGER_TYPE,\n      float: Config::FLOAT_TYPE,\n      size: Config::SIZE_TYPE,\n      bool: Config::BOOL_TYPE,\n      time: Config::TIME_TYPE,\n      hash: Config::HASH_TYPE,\n      array: Config::ARRAY_TYPE,\n      regexp: Config::REGEXP_TYPE,\n    }.each do |name, type|\n      register_type(name, type)\n    end\n\n    module ClassMethods\n      def configure_proxy_map\n        map = {}\n        self.define_singleton_method(:configure_proxy_map){ map }\n        map\n      end\n\n      def configure_proxy(mod_name)\n        map = configure_proxy_map\n        unless map[mod_name]\n          type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }\n          proxy = Fluent::Config::ConfigureProxy.new(mod_name, root: true, required: true, multi: false, type_lookup: type_lookup)\n          map[mod_name] = proxy\n        end\n        map[mod_name]\n      end\n\n      def configured_in(section_name)\n        configure_proxy(self.name).configured_in(section_name)\n      end\n\n      def config_param(name, type = nil, **kwargs, &block)\n        configure_proxy(self.name).config_param(name, type, **kwargs, &block)\n        # reserved names '@foo' are invalid as attr_accessor name\n        attr_accessor(name) unless kwargs[:skip_accessor] || Fluent::Config::Element::RESERVED_PARAMETERS.include?(name.to_s)\n      end\n\n      def config_set_default(name, defval)\n        configure_proxy(self.name).config_set_default(name, defval)\n      end\n\n      def config_set_desc(name, desc)\n        configure_proxy(self.name).config_set_desc(name, desc)\n      end\n\n      def config_section(name, **kwargs, &block)\n        section_already_exists = !!merged_configure_proxy.sections[name]\n        configure_proxy(self.name).config_section(name, **kwargs, &block)\n        variable_name = configure_proxy(self.name).sections[name].variable_name\n        if !section_already_exists && !self.respond_to?(variable_name)\n          attr_accessor variable_name\n        end\n      end\n\n      def desc(description)\n        configure_proxy(self.name).desc(description)\n      end\n\n      def merged_configure_proxy\n        configurables = ancestors.reverse.select{ |a| a.respond_to?(:configure_proxy) }\n\n        # 'a.object_id.to_s' is to support anonymous class\n        #   which created in tests to overwrite original behavior temporally\n        #\n        # p Module.new.name   #=> nil\n        # p Class.new.name    #=> nil\n        # p AnyGreatClass.dup.name #=> nil\n        configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)\n      end\n\n      def dump_config_definition\n        configure_proxy_map[self.to_s].dump_config_definition\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/counter/base_socket.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'cool.io'\nrequire 'fluent/msgpack_factory'\n\nmodule Fluent\n  module Counter\n    class BaseSocket < Coolio::TCPSocket\n      def packed_write(data)\n        write pack(data)\n      end\n\n      def on_read(data)\n        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(data) do |d|\n          on_message d\n        end\n      end\n\n      def on_message(data)\n        raise NotImplementedError\n      end\n\n      private\n\n      def pack(data)\n        Fluent::MessagePackFactory.msgpack_packer.pack(data)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/counter/client.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'cool.io'\nrequire 'fluent/counter/base_socket'\nrequire 'fluent/counter/error'\nrequire 'timeout'\n\nmodule Fluent\n  module Counter\n    class Client\n      DEFAULT_PORT = 24321\n      DEFAULT_ADDR = '127.0.0.1'\n      DEFAULT_TIMEOUT = 5\n      ID_LIMIT_COUNT = 1 << 31\n\n      def initialize(loop = nil, opt = {})\n        @loop = loop || Coolio::Loop.new\n        @port = opt[:port] || DEFAULT_PORT\n        @host = opt[:host] || DEFAULT_ADDR\n        @log = opt[:log] || $log\n        @timeout = opt[:timeout] || DEFAULT_TIMEOUT\n        @conn = Connection.connect(@host, @port, method(:on_message))\n        @responses = {}\n        @id = 0\n        @id_mutex = Mutex.new\n        @loop_mutex = Mutex.new\n      end\n\n      def start\n        @loop.attach(@conn)\n        @log.debug(\"starting counter client: #{@host}:#{@port}\")\n        self\n      rescue => e\n        if @log\n          @log.error e\n        else\n          STDERR.puts e\n        end\n      end\n\n      def stop\n        @conn.close\n        @log.debug(\"calling stop in counter client: #{@host}:#{@port}\")\n      end\n\n      def establish(scope)\n        scope = Timeout.timeout(@timeout) {\n          response = send_request('establish', nil, [scope])\n          Fluent::Counter.raise_error(response.errors.first) if response.errors?\n          data = response.data\n          data.first\n        }\n        @scope = scope\n      rescue Timeout::Error\n        raise \"Can't establish the connection to counter server due to timeout\"\n      end\n\n      # === Example\n      # `init` receives various arguments.\n      #\n      # 1. init(name: 'name')\n      # 2. init({ name: 'name',reset_interval: 20 }, options: {})\n      # 3. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }])\n      # 4. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }], options: {})\n      # 5. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }]) { |res| ... }\n      def init(params, options: {})\n        exist_scope!\n        params = [params] unless params.is_a?(Array)\n        res = send_request('init', @scope, params, options)\n\n        # if `async` is false or missing, block at this method and return a Future::Result object.\n        if block_given?\n          Thread.start do\n            yield res.get\n          end\n        else\n          res\n        end\n      end\n\n      def delete(*params, options: {})\n        exist_scope!\n        res = send_request('delete', @scope, params, options)\n\n        if block_given?\n          Thread.start do\n            yield res.get\n          end\n        else\n          res\n        end\n      end\n\n      # === Example\n      # `inc` receives various arguments.\n      #\n      # 1. inc(name: 'name')\n      # 2. inc({ name: 'name',value: 20 }, options: {})\n      # 3. inc([{ name: 'name1',value: 20 }, { name: 'name2',value: 20 }])\n      # 4. inc([{ name: 'name1',value: 20 }, { name: 'name2',value: 20 }], options: {})\n      def inc(params, options: {})\n        exist_scope!\n        params = [params] unless params.is_a?(Array)\n        res = send_request('inc', @scope, params, options)\n\n        if block_given?\n          Thread.start do\n            yield res.get\n          end\n        else\n          res\n        end\n      end\n\n      def get(*params, options: {})\n        exist_scope!\n        res = send_request('get', @scope, params, options)\n\n        if block_given?\n          Thread.start do\n            yield res.get\n          end\n        else\n          res\n        end\n      end\n\n      def reset(*params, options: {})\n        exist_scope!\n        res = send_request('reset', @scope, params, options)\n\n        if block_given?\n          Thread.start do\n            yield res.get\n          end\n        else\n          res\n        end\n      end\n\n      private\n\n      def exist_scope!\n        raise 'Call `establish` method to get a `scope` before calling this method' unless @scope\n      end\n\n      def on_message(data)\n        if response = @responses.delete(data['id'])\n          response.set(data)\n        else\n          @log.warn(\"Receiving missing id data: #{data}\")\n        end\n      end\n\n      def send_request(method, scope, params, opt = {})\n        id = generate_id\n        res = Future.new(@loop, @loop_mutex)\n        @responses[id] = res # set a response value to this future object at `on_message`\n        request = build_request(method, id, scope, params, opt)\n        @log.debug(request)\n        @conn.send_data request\n        res\n      end\n\n      def build_request(method, id, scope = nil, params = nil, options = nil)\n        r = { id: id, method: method }\n        r[:scope] = scope if scope\n        r[:params] = params if params\n        r[:options] = options if options\n        r\n      end\n\n      def generate_id\n        id = 0\n        @id_mutex.synchronize do\n          id = @id\n          @id += 1\n          @id = 0 if ID_LIMIT_COUNT < @id\n        end\n        id\n      end\n    end\n\n    class Connection < Fluent::Counter::BaseSocket\n      def initialize(io, on_message)\n        super(io)\n        @connection = false\n        @buffer = ''\n        @on_message = on_message\n      end\n\n      def send_data(data)\n        if @connection\n          packed_write data\n        else\n          @buffer += pack(data)\n        end\n      end\n\n      def on_connect\n        @connection = true\n        write @buffer\n        @buffer = ''\n      end\n\n      def on_close\n        @connection = false\n      end\n\n      def on_message(data)\n        @on_message.call(data)\n      end\n    end\n\n    class Future\n      class Result\n        attr_reader :data, :errors\n\n        def initialize(result)\n          @errors = result['errors']\n          @data = result['data']\n        end\n\n        def success?\n          @errors.nil? || @errors.empty?\n        end\n\n        def error?\n          !success?\n        end\n      end\n\n      def initialize(loop, mutex)\n        @set = false\n        @result = nil\n        @mutex = mutex\n        @loop = loop\n      end\n\n      def set(v)\n        @result = Result.new(v)\n        @set = true\n      end\n\n      def errors\n        get.errors\n      end\n\n      def errors?\n        es = errors\n        es && !es.empty?\n      end\n\n      def data\n        get.data\n      end\n\n      def get\n        # Block until `set` method is called and @result is set\n        join if @result.nil?\n        @result\n      end\n\n      def wait\n        res = get\n        if res.error?\n          Fluent::Counter.raise_error(res.errors.first)\n        end\n        res\n      end\n\n      private\n\n      def join\n        until @set\n          @mutex.synchronize do\n            @loop.run_once(0.0001) # return a lock as soon as possible\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/counter/error.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Counter\n    class BaseError < StandardError\n      def to_hash\n        { 'code' => code, 'message' => message }\n      end\n\n      def code\n        raise NotImplementedError\n      end\n    end\n\n    class InvalidParams < BaseError\n      def code\n        'invalid_params'\n      end\n    end\n\n    class UnknownKey < BaseError\n      def code\n        'unknown_key'\n      end\n    end\n\n    class ParseError < BaseError\n      def code\n        'parse_error'\n      end\n    end\n\n    class InvalidRequest < BaseError\n      def code\n        'invalid_request'\n      end\n    end\n\n    class MethodNotFound < BaseError\n      def code\n        'method_not_found'\n      end\n    end\n\n    class InternalServerError < BaseError\n      def code\n        'internal_server_error'\n      end\n    end\n\n    def raise_error(response)\n      msg = response['message']\n      case response['code']\n      when 'invalid_params'\n        raise InvalidParams.new(msg)\n      when 'unknown_key'\n        raise UnknownKey.new(msg)\n      when 'parse_error'\n        raise ParseError.new(msg)\n      when 'invalid_request'\n        raise InvalidRequest.new(msg)\n      when 'method_not_found'\n        raise MethodNotFound.new(msg)\n      when 'internal_server_error'\n        raise InternalServerError.new(msg)\n      else\n        raise \"Unknown code: #{response['code']}\"\n      end\n    end\n    module_function :raise_error\n  end\nend\n"
  },
  {
    "path": "lib/fluent/counter/mutex_hash.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'timeout'\n\nmodule Fluent\n  module Counter\n    class MutexHash\n      def initialize(data_store)\n        @mutex = Mutex.new\n        @data_store = data_store\n        @mutex_hash = {}\n        @thread = nil\n        @cleanup_thread = CleanupThread.new(@data_store, @mutex_hash, @mutex)\n      end\n\n      def start\n        @data_store.start\n        @cleanup_thread.start\n      end\n\n      def stop\n        @data_store.stop\n        @cleanup_thread.stop\n      end\n\n      def synchronize(*keys)\n        return if keys.empty?\n\n        locks = {}\n        loop do\n          @mutex.synchronize do\n            keys.each do |key|\n              mutex = @mutex_hash[key]\n              unless mutex\n                v = Mutex.new\n                @mutex_hash[key] = v\n                mutex = v\n              end\n\n              if mutex.try_lock\n                locks[key] = mutex\n              else\n                locks.each_value(&:unlock)\n                locks = {}          # flush locked keys\n                break\n              end\n            end\n          end\n\n          next if locks.empty?      # failed to lock all keys\n\n          locks.each do |(k, v)|\n            yield @data_store, k\n            v.unlock\n          end\n          break\n        end\n      end\n\n      def synchronize_keys(*keys)\n        return if keys.empty?\n        keys = keys.dup\n\n        while key = keys.shift\n          @mutex.lock\n\n          mutex = @mutex_hash[key]\n          unless mutex\n            v = Mutex.new\n            @mutex_hash[key] = v\n            mutex = v\n          end\n\n          if mutex.try_lock\n            @mutex.unlock\n            yield @data_store, key\n            mutex.unlock\n          else\n            # release global lock\n            @mutex.unlock\n            keys.push(key)          # failed lock, retry this key\n          end\n        end\n      end\n    end\n\n    class CleanupThread\n      CLEANUP_INTERVAL = 60 * 15 # 15 min\n\n      def initialize(store, mutex_hash, mutex)\n        @store = store\n        @mutex_hash = mutex_hash\n        @mutex = mutex\n        @thread = nil\n        @running = false\n      end\n\n      def start\n        @running = true\n        @thread = Thread.new do\n          while @running\n            sleep CLEANUP_INTERVAL\n            run_once\n          end\n        end\n      end\n\n      def stop\n        return unless @running\n        @running = false\n        begin\n          # Avoid waiting CLEANUP_INTERVAL\n          Timeout.timeout(1) do\n            @thread.join\n          end\n        rescue Timeout::Error\n          @thread.kill\n        end\n      end\n\n      private\n\n      def run_once\n        @mutex.synchronize do\n          last_cleanup_at = (Time.now - CLEANUP_INTERVAL).to_i\n          @mutex_hash.each do |(key, mutex)|\n            v = @store.get(key, raw: true)\n            next unless v\n            next if last_cleanup_at < v['last_modified_at'][0] # v['last_modified_at'] = [sec, nsec]\n            next unless mutex.try_lock\n\n            @mutex_hash[key] = nil\n            mutex.unlock\n\n            # Check that a waiting thread is in a lock queue.\n            # Can't get a lock here means this key is used in other places.\n            # So restore a mutex value to a corresponding key.\n            if mutex.try_lock\n              @mutex_hash.delete(key)\n              mutex.unlock\n            else\n              @mutex_hash[key] = mutex\n            end\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/counter/server.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'cool.io'\nrequire 'fluent/counter/base_socket'\nrequire 'fluent/counter/validator'\nrequire 'fluent/counter/store'\nrequire 'fluent/counter/mutex_hash'\n\nmodule Fluent\n  module Counter\n    class Server\n      DEFAULT_ADDR = '127.0.0.1'\n      DEFAULT_PORT = 24321\n\n      def initialize(name, opt = {})\n        raise 'Counter server name is invalid' unless Validator::VALID_NAME.match?(name)\n        @name = name\n        @opt = opt\n        @addr = @opt[:addr] || DEFAULT_ADDR\n        @port = @opt[:port] || DEFAULT_PORT\n        @loop = @opt[:loop] || Coolio::Loop.new\n        @log =  @opt[:log] || $log\n\n        @store = Fluent::Counter::Store.new(opt)\n        @mutex_hash = MutexHash.new(@store)\n\n        @server = Coolio::TCPServer.new(@addr, @port, Handler, method(:on_message))\n        @thread = nil\n        @running = false\n      end\n\n      def start\n        @server.attach(@loop)\n        @thread = Thread.new do\n          @running = true\n          @loop.run(0.5)\n          @running = false\n        end\n        @log.debug(\"starting counter server #{@addr}:#{@port}\")\n        @mutex_hash.start\n        self\n      end\n\n      def stop\n        # This `sleep` for a test to wait for a `@loop` to begin to run\n        sleep 0.1\n        @server.close\n        @loop.stop if @running\n        @mutex_hash.stop\n        @thread.join if @thread\n        @log.debug(\"calling stop in counter server #{@addr}:#{@port}\")\n      end\n\n      def on_message(data)\n        errors = Validator.request(data)\n        unless errors.empty?\n          return { 'id' => data['id'], 'data' => [], 'errors' => errors }\n        end\n\n        result = safe_run do\n          send(data['method'], data['params'], data['scope'], data['options'])\n        end\n        result.merge('id' => data['id'])\n      rescue => e\n        @log.error e.to_s\n      end\n\n      private\n\n      def establish(params, _scope, _options)\n        validator = Fluent::Counter::ArrayValidator.new(:empty, :scope)\n        valid_params, errors = validator.call(params)\n        res = Response.new(errors)\n\n        if scope = valid_params.first\n          new_scope = \"#{@name}\\t#{scope}\"\n          res.push_data new_scope\n          @log.debug(\"Establish new key: #{new_scope}\")\n        end\n\n        res.to_hash\n      end\n\n      def init(params, scope, options)\n        validator = Fluent::Counter::HashValidator.new(:empty, :name, :reset_interval)\n        valid_params, errors = validator.call(params)\n        res = Response.new(errors)\n        key_hash = valid_params.reduce({}) do |acc, vp|\n          acc.merge(Store.gen_key(scope, vp['name']) => vp)\n        end\n\n        do_init = lambda do |store, key|\n          begin\n            param = key_hash[key]\n            v = store.init(key, param, ignore: options['ignore'])\n            @log.debug(\"Create new key: #{param['name']}\")\n            res.push_data v\n          rescue => e\n            res.push_error e\n          end\n        end\n\n        if options['random']\n          @mutex_hash.synchronize_keys(*(key_hash.keys), &do_init)\n        else\n          @mutex_hash.synchronize(*(key_hash.keys), &do_init)\n        end\n\n        res.to_hash\n      end\n\n      def delete(params, scope, options)\n        validator = Fluent::Counter::ArrayValidator.new(:empty, :key)\n        valid_params, errors = validator.call(params)\n        res = Response.new(errors)\n        keys = valid_params.map { |vp| Store.gen_key(scope, vp) }\n\n        do_delete = lambda do |store, key|\n          begin\n            v = store.delete(key)\n            @log.debug(\"delete a key: #{key}\")\n            res.push_data v\n          rescue => e\n            res.push_error e\n          end\n        end\n\n        if options['random']\n          @mutex_hash.synchronize_keys(*keys, &do_delete)\n        else\n          @mutex_hash.synchronize(*keys, &do_delete)\n        end\n\n        res.to_hash\n      end\n\n      def inc(params, scope, options)\n        validate_param = [:empty, :name, :value]\n        validate_param << :reset_interval if options['force']\n        validator = Fluent::Counter::HashValidator.new(*validate_param)\n        valid_params, errors = validator.call(params)\n        res = Response.new(errors)\n        key_hash = valid_params.reduce({}) do |acc, vp|\n          acc.merge(Store.gen_key(scope, vp['name']) => vp)\n        end\n\n        do_inc = lambda do |store, key|\n          begin\n            param = key_hash[key]\n            v = store.inc(key, param, force: options['force'])\n            @log.debug(\"Increment #{key} by #{param['value']}\")\n            res.push_data v\n          rescue => e\n            res.push_error e\n          end\n        end\n\n        if options['random']\n          @mutex_hash.synchronize_keys(*(key_hash.keys), &do_inc)\n        else\n          @mutex_hash.synchronize(*(key_hash.keys), &do_inc)\n        end\n\n        res.to_hash\n      end\n\n      def reset(params, scope, options)\n        validator = Fluent::Counter::ArrayValidator.new(:empty, :key)\n        valid_params, errors = validator.call(params)\n        res = Response.new(errors)\n        keys = valid_params.map { |vp| Store.gen_key(scope, vp) }\n\n        do_reset = lambda do |store, key|\n          begin\n            v = store.reset(key)\n            @log.debug(\"Reset #{key}'s' counter value\")\n            res.push_data v\n          rescue => e\n            res.push_error e\n          end\n        end\n\n        if options['random']\n          @mutex_hash.synchronize_keys(*keys, &do_reset)\n        else\n          @mutex_hash.synchronize(*keys, &do_reset)\n        end\n\n        res.to_hash\n      end\n\n      def get(params, scope, _options)\n        validator = Fluent::Counter::ArrayValidator.new(:empty, :key)\n        valid_params, errors = validator.call(params)\n        res = Response.new(errors)\n\n        keys = valid_params.map { |vp| Store.gen_key(scope, vp) }\n        keys.each do |key|\n          begin\n            v = @store.get(key, raise_error: true)\n            @log.debug(\"Get counter value: #{key}\")\n            res.push_data v\n          rescue => e\n            res.push_error e\n          end\n        end\n        res.to_hash\n      end\n\n      def safe_run\n        yield\n      rescue => e\n        {\n          'errors' => [InternalServerError.new(e).to_hash],\n          'data' => []\n        }\n      end\n\n      class Response\n        def initialize(errors = [], data = [])\n          @errors = errors\n          @data = data\n        end\n\n        def push_error(error)\n          @errors << error\n        end\n\n        def push_data(data)\n          @data << data\n        end\n\n        def to_hash\n          if @errors.empty?\n            { 'data' => @data }\n          else\n            errors = @errors.map do |e|\n              error = e.respond_to?(:to_hash) ? e : InternalServerError.new(e.to_s)\n              error.to_hash\n            end\n            { 'data' => @data, 'errors' => errors }\n          end\n        end\n      end\n    end\n\n    class Handler < Fluent::Counter::BaseSocket\n      def initialize(io, on_message)\n        super(io)\n        @on_message = on_message\n      end\n\n      def on_message(data)\n        res = @on_message.call(data)\n        packed_write res if res\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/counter/store.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config'\nrequire 'fluent/counter/error'\nrequire 'fluent/plugin/storage_local'\nrequire 'fluent/time'\n\nmodule Fluent\n  module Counter\n    class Store\n      def self.gen_key(scope, key)\n        \"#{scope}\\t#{key}\"\n      end\n\n      def initialize(opt = {})\n        @log = opt[:log] || $log\n\n        # Notice: This storage is not be implemented auto save.\n        @storage = Plugin.new_storage('local', parent: DummyParent.new(@log))\n        conf = if opt[:path]\n                 {'persistent' => true, 'path' => opt[:path] }\n               else\n                 {'persistent' => false }\n               end\n        @storage.configure(Fluent::Config::Element.new('storage', {}, conf, []))\n      end\n\n      # This class behaves as a configurable plugin for using in storage (OwnedByMixin).\n      class DummyParent\n        include Configurable\n\n        attr_reader :log\n\n        def initialize(log)\n          @log = log\n        end\n\n        def plugin_id\n          'dummy_parent_store'\n        end\n\n        def plugin_id_configured?\n          false\n        end\n\n        # storage_local calls PluginId#plugin_root_dir\n        def plugin_root_dir\n          nil\n        end\n      end\n\n      def start\n        @storage.load\n      end\n\n      def stop\n        @storage.save\n      end\n\n      def init(key, data, ignore: false)\n        ret = if v = get(key)\n                raise InvalidParams.new(\"#{key} already exists in counter\") unless ignore\n                v\n              else\n                @storage.put(key, build_value(data))\n              end\n\n        build_response(ret)\n      end\n\n      def get(key, raise_error: false, raw: false)\n        ret = if raise_error\n                @storage.get(key) or raise UnknownKey.new(\"`#{key}` doesn't exist in counter\")\n              else\n                @storage.get(key)\n              end\n        if raw\n          ret\n        else\n          ret && build_response(ret)\n        end\n      end\n\n      def key?(key)\n        !!@storage.get(key)\n      end\n\n      def delete(key)\n        ret = @storage.delete(key) or raise UnknownKey.new(\"`#{key}` doesn't exist in counter\")\n        build_response(ret)\n      end\n\n      def inc(key, data, force: false)\n        value = data.delete('value')\n        init(key, data) if !key?(key) && force\n        v = get(key, raise_error: true, raw: true)\n        valid_type!(v, value)\n\n        v['total'] += value\n        v['current'] += value\n        t = EventTime.now\n        v['last_modified_at'] = [t.sec, t.nsec]\n        @storage.put(key, v)\n\n        build_response(v)\n      end\n\n      def reset(key)\n        v = get(key, raise_error: true, raw: true)\n        success = false\n        old_data = v.dup\n        now = EventTime.now\n        last_reset_at = EventTime.new(*v['last_reset_at'])\n\n        #  Does it need reset?\n        if (last_reset_at + v['reset_interval']) <= now\n          success = true\n          v['current'] = initial_value(v['type'])\n          t = [now.sec, now.nsec]\n          v['last_reset_at'] = t\n          v['last_modified_at'] = t\n          @storage.put(key, v)\n        end\n\n        {\n          'elapsed_time' => now - last_reset_at,\n          'success' => success,\n          'counter_data' => build_response(old_data)\n        }\n      end\n\n      private\n\n      def build_response(d)\n        {\n          'name' => d['name'],\n          'total' => d['total'],\n          'current' => d['current'],\n          'type' => d['type'],\n          'reset_interval' => d['reset_interval'],\n          'last_reset_at' => EventTime.new(*d['last_reset_at']),\n        }\n      end\n\n      # value is Hash. value requires these fields.\n      # :name, :total, :current, :type, :reset_interval, :last_reset_at, :last_modified_at\n      def build_value(data)\n        type = data['type'] || 'numeric'\n        now = EventTime.now\n        t = [now.sec, now.nsec]\n\n        v = initial_value(type)\n\n        data.merge(\n          'type' => type,\n          'last_reset_at' => t,\n          'last_modified_at' => t,\n          'current' => v,\n          'total' => v,\n        )\n      end\n\n      def initial_value(type)\n        case type\n        when 'numeric', 'integer' then 0\n        when 'float' then 0.0\n        else raise InvalidParams.new('`type` should be integer, float, or numeric')\n        end\n      end\n\n      def valid_type!(v, value)\n        type = v['type']\n        return unless (type != 'numeric') && (type_str(value) != type)\n        raise InvalidParams.new(\"`type` is #{type}. You should pass #{type} value as a `value`\")\n      end\n\n      def type_str(v)\n        case v\n        when Integer\n          'integer'\n        when Float\n          'float'\n        when Numeric\n          'numeric'\n        else\n          raise InvalidParams.new(\"`type` should be integer, float, or numeric\")\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/counter/validator.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/counter/error'\n\nmodule Fluent\n  module Counter\n    class Validator\n      VALID_NAME = /\\A[a-z][a-zA-Z0-9\\-_]*\\Z/\n      VALID_SCOPE_NAME = /\\A[a-z][\\ta-zA-Z0-9\\-_]*\\Z/\n      VALID_METHODS = %w(establish init delete inc get reset)\n\n      def self.request(data)\n        errors = []\n        raise \"Received data is not Hash: #{data}\" unless data.is_a?(Hash)\n\n        unless data['id']\n          errors << Fluent::Counter::InvalidRequest.new('Request should include `id`')\n        end\n\n        if !data['method']\n          errors << Fluent::Counter::InvalidRequest.new('Request should include `method`')\n        elsif !(VALID_NAME =~ data['method'])\n          errors << Fluent::Counter::InvalidRequest.new('`method` is the invalid format')\n        elsif !VALID_METHODS.include?(data['method'])\n          errors << Fluent::Counter::MethodNotFound.new(\"Unknown method name passed: #{data['method']}\")\n        end\n\n        errors.map(&:to_hash)\n      end\n\n      def initialize(*types)\n        @types = types.map(&:to_s)\n        @empty = @types.delete('empty')\n      end\n\n      def call(data)\n        success = []\n        errors = []\n\n        if @empty && data.empty?\n          errors << Fluent::Counter::InvalidParams.new('One or more `params` are required')\n        else\n          data.each do |d|\n            begin\n              @types.each { |type| dispatch(type, d) }\n              success << d\n            rescue => e\n              errors << e\n            end\n          end\n        end\n\n        [success, errors]\n      end\n\n      private\n\n      def dispatch(type, data)\n        send(\"validate_#{type}!\", data)\n      rescue NoMethodError => e\n        raise Fluent::Counter::InternalServerError.new(e)\n      end\n    end\n\n    class ArrayValidator < Validator\n      def validate_key!(name)\n        unless name.is_a?(String)\n          raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')\n        end\n\n        unless VALID_NAME.match?(name)\n          raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')\n        end\n      end\n\n      def validate_scope!(name)\n        unless name.is_a?(String)\n          raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')\n        end\n\n        unless VALID_SCOPE_NAME.match?(name)\n          raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')\n        end\n      end\n    end\n\n    class HashValidator < Validator\n      def validate_name!(hash)\n        name = hash['name']\n        unless name\n          raise Fluent::Counter::InvalidParams.new('`name` is required')\n        end\n\n        unless name.is_a?(String)\n          raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')\n        end\n\n        unless VALID_NAME.match?(name)\n          raise Fluent::Counter::InvalidParams.new(\"`name` is the invalid format\")\n        end\n      end\n\n      def validate_value!(hash)\n        value = hash['value']\n        unless value\n          raise Fluent::Counter::InvalidParams.new('`value` is required')\n        end\n\n        unless value.is_a?(Numeric)\n          raise Fluent::Counter::InvalidParams.new(\"The type of `value` type should be Numeric\")\n        end\n      end\n\n      def validate_reset_interval!(hash)\n        interval = hash['reset_interval']\n\n        unless interval\n          raise Fluent::Counter::InvalidParams.new('`reset_interval` is required')\n        end\n\n        unless interval.is_a?(Numeric)\n          raise Fluent::Counter::InvalidParams.new('The type of `reset_interval` should be Numeric')\n        end\n\n        if interval < 0\n          raise Fluent::Counter::InvalidParams.new('`reset_interval` should be a positive number')\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/counter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/counter/client'\nrequire 'fluent/counter/server'\n\nmodule Fluent\n  module Counter\n  end\nend\n"
  },
  {
    "path": "lib/fluent/daemon.rb",
    "content": "#!/usr/bin/env ruby\n# -*- coding: utf-8 -*-\n\nhere = File.dirname(__FILE__)\n$LOAD_PATH << File.expand_path(File.join(here, '..'))\n\nrequire 'serverengine'\nrequire 'fluent/supervisor'\n\nserver_module = Fluent.const_get(ARGV[0])\nworker_module = Fluent.const_get(ARGV[1])\nparams = JSON.parse(ARGV[2])\nServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.serverengine_config(params) }\n"
  },
  {
    "path": "lib/fluent/daemonizer.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/error'\n\nmodule Fluent\n  class Daemonizer\n    def self.daemonize(pid_path, args = [], &block)\n      new.daemonize(pid_path, args, &block)\n    end\n\n    def daemonize(pid_path, args = [])\n      pid_fullpath = File.absolute_path(pid_path)\n      check_pidfile(pid_fullpath)\n\n      begin\n        Process.daemon(false, false)\n\n        File.write(pid_fullpath, Process.pid.to_s)\n\n        # install signal and set process name are performed by supervisor\n        install_at_exit_handlers(pid_fullpath)\n\n        yield\n      rescue NotImplementedError\n        daemonize_with_spawn(pid_fullpath, args)\n      end\n    end\n\n    private\n\n    def daemonize_with_spawn(pid_fullpath, args)\n      pid = Process.spawn(*['fluentd'].concat(args))\n\n      File.write(pid_fullpath, pid.to_s)\n\n      pid\n    end\n\n    def check_pidfile(pid_path)\n      if File.exist?(pid_path)\n        if !File.readable?(pid_path) || !File.writable?(pid_path)\n          raise Fluent::ConfigError, \"Cannot access pid file: #{pid_path}\"\n        end\n\n        pid =\n          begin\n            Integer(File.read(pid_path), 10)\n          rescue TypeError, ArgumentError\n            return # ignore\n          end\n\n        begin\n          Process.kill(0, pid)\n          raise Fluent::ConfigError, \"pid(#{pid}) is running\"\n        rescue Errno::EPERM\n          raise Fluent::ConfigError, \"pid(#{pid}) is running\"\n        rescue Errno::ESRCH\n        end\n      else\n        unless File.writable?(File.dirname(pid_path))\n          raise Fluent::ConfigError, \"Cannot access directory for pid file: #{File.dirname(pid_path)}\"\n        end\n      end\n    end\n\n    def install_at_exit_handlers(pidfile)\n      at_exit do\n        if File.exist?(pidfile)\n          File.delete(pidfile)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/engine.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config'\nrequire 'fluent/event'\nrequire 'fluent/event_router'\nrequire 'fluent/msgpack_factory'\nrequire 'fluent/root_agent'\nrequire 'fluent/time'\nrequire 'fluent/system_config'\nrequire 'fluent/plugin'\nrequire 'fluent/fluent_log_event_router'\nrequire 'fluent/static_config_analysis'\n\nmodule Fluent\n  class EngineClass\n    # For compat. remove it in fluentd v2\n    include Fluent::MessagePackFactory::Mixin\n\n    def initialize\n      @root_agent = nil\n      @engine_stopped = false\n      @_worker_id = nil\n\n      @log_event_verbose = false\n      @suppress_config_dump = false\n      @without_source = false\n\n      @fluent_log_event_router = nil\n      @system_config = SystemConfig.new\n\n      @supervisor_mode = false\n\n      @root_agent_mutex = Mutex.new\n    end\n\n    MAINLOOP_SLEEP_INTERVAL = 0.3\n\n    attr_reader :root_agent, :system_config, :supervisor_mode\n\n    def init(system_config, supervisor_mode: false, start_in_parallel: false)\n      @system_config = system_config\n      @supervisor_mode = supervisor_mode\n\n      @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?\n      @without_source = system_config.without_source unless system_config.without_source.nil?\n\n      @log_event_verbose = system_config.log_event_verbose unless system_config.log_event_verbose.nil?\n\n      @root_agent = RootAgent.new(log: log, system_config: @system_config, start_in_parallel: start_in_parallel)\n\n      self\n    end\n\n    def log\n      $log\n    end\n\n    def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)\n      if /\\.rb$/.match?(fname)\n        require 'fluent/config/dsl'\n        Config::DSL::Parser.parse(io, File.join(basepath, fname))\n      else\n        Config.parse(io, fname, basepath, v1_config)\n      end\n    end\n\n    def run_configure(conf, dry_run: false)\n      configure(conf)\n      conf.check_not_fetched do |key, e|\n        parent_name, plugin_name = e.unused_in\n        message = if parent_name && plugin_name\n                    \"section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin\"\n                  elsif parent_name\n                    \"section <#{e.name}> is not used in <#{parent_name}>\"\n                  elsif e.name != 'system' && !(@without_source && e.name == 'source')\n                    \"parameter '#{key}' in #{e.to_s.strip} is not used.\"\n                  else\n                    nil\n                  end\n        next if message.nil?\n\n        if dry_run && @supervisor_mode\n          $log.warn :supervisor, message\n        elsif e.for_every_workers?\n          $log.warn :worker0, message\n        elsif e.for_this_worker?\n          $log.warn message\n        end\n      end\n    end\n\n    def configure(conf)\n      @root_agent.configure(conf)\n\n      @fluent_log_event_router = FluentLogEventRouter.build(@root_agent)\n\n      if @fluent_log_event_router.emittable?\n        $log.enable_event(true)\n      end\n\n      unless @suppress_config_dump\n        $log.info :supervisor, \"using configuration file: #{conf.to_s.rstrip}\"\n      end\n    end\n\n    def add_plugin_dir(dir)\n      $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::Plugin.add_plugin_dir')\n      Plugin.add_plugin_dir(dir)\n    end\n\n    def emit(tag, time, record)\n      raise \"BUG: use router.emit instead of Engine.emit\"\n    end\n\n    def emit_array(tag, array)\n      raise \"BUG: use router.emit_array instead of Engine.emit_array\"\n    end\n\n    def emit_stream(tag, es)\n      raise \"BUG: use router.emit_stream instead of Engine.emit_stream\"\n    end\n\n    def flush!\n      @root_agent_mutex.synchronize do\n        @root_agent.flush!\n      end\n    end\n\n    def cancel_source_only!\n      @root_agent_mutex.synchronize do\n        @root_agent.cancel_source_only!\n      end\n    end\n\n    def now\n      # TODO thread update\n      Fluent::EventTime.now\n    end\n\n    def run\n      begin\n        $log.info \"starting fluentd worker\", pid: Process.pid, ppid: Process.ppid, worker: worker_id\n        @root_agent_mutex.synchronize do\n          start\n        end\n\n        @fluent_log_event_router.start\n\n        $log.info \"fluentd worker is now running\", worker: worker_id\n        sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped\n        $log.info \"fluentd worker is now stopping\", worker: worker_id\n\n      rescue Exception => e\n        $log.error \"unexpected error\", error: e\n        $log.error_backtrace\n        raise\n      end\n\n      @root_agent_mutex.synchronize do\n        stop_phase(@root_agent)\n      end\n    end\n\n    # @param conf [Fluent::Config]\n    # @param supervisor [Bool]\n    # @return nil\n    def reload_config(conf, supervisor: false)\n      @root_agent_mutex.synchronize do\n        # configure first to reduce down time while restarting\n        new_agent = RootAgent.new(log: log, system_config: @system_config)\n        ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)\n\n        ret.all_plugins.each do |plugin|\n          if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?\n            raise Fluent::ConfigError, \"Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})\"\n          end\n        end\n\n        # Assign @root_agent to new root_agent\n        # for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50\n        old_agent, @root_agent = @root_agent, new_agent\n        begin\n          @root_agent.configure(conf)\n        rescue\n          @root_agent = old_agent\n          raise\n        end\n\n        unless @suppress_config_dump\n          $log.info :supervisor, \"using configuration file: #{conf.to_s.rstrip}\"\n        end\n\n        # supervisor doesn't handle actual data. so the following code is unnecessary.\n        if supervisor\n          old_agent.shutdown      # to close thread created in #configure\n          return\n        end\n\n        stop_phase(old_agent)\n\n        $log.info 'restart fluentd worker', worker: worker_id\n        start_phase(new_agent)\n      end\n    end\n\n    def stop\n      @engine_stopped = true\n      nil\n    end\n\n    def push_log_event(tag, time, record)\n      @fluent_log_event_router.emit_event([tag, time, record])\n    end\n\n    def worker_id\n      if @supervisor_mode\n        return -1\n      end\n\n      return @_worker_id if @_worker_id\n      # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests\n      # so it's (almost) a single worker, worker_id=0\n      @_worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i\n      @_worker_id\n    end\n\n    private\n\n    def stop_phase(root_agent)\n      unless @log_event_verbose\n        $log.enable_event(false)\n        @fluent_log_event_router.graceful_stop\n      end\n      $log.info 'shutting down fluentd worker', worker: worker_id\n      root_agent.shutdown\n\n      @fluent_log_event_router.stop\n    end\n\n    def start_phase(root_agent)\n      @fluent_log_event_router = FluentLogEventRouter.build(root_agent)\n      if @fluent_log_event_router.emittable?\n        $log.enable_event(true)\n      end\n\n      @root_agent.start\n    end\n\n    def start\n      @root_agent.start\n    end\n  end\n\n  Engine = EngineClass.new\nend\n"
  },
  {
    "path": "lib/fluent/env.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'securerandom'\n\nrequire 'serverengine/utils'\nrequire 'fluent/oj_options'\n\nmodule Fluent\n  DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'\n  DEFAULT_CONFIG_INCLUDE_DIR = ENV[\"FLUENT_CONF_INCLUDE_DIR\"] || '/etc/fluent/conf.d'\n  DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'\n  DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'\n  DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'\n  DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env\n  DEFAULT_DIR_PERMISSION = 0755\n  DEFAULT_FILE_PERMISSION = 0644\n  INSTANCE_ID = ENV['FLUENT_INSTANCE_ID'] || SecureRandom.uuid\n\n  def self.windows?\n    ServerEngine.windows?\n  end\n\n  def self.linux?\n    RUBY_PLATFORM.include?(\"linux\")\n  end\n\n  def self.macos?\n    RUBY_PLATFORM.include?(\"darwin\")\n  end\nend\n"
  },
  {
    "path": "lib/fluent/error.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  class UnrecoverableError < StandardError\n    def initialize(error_message = nil)\n      @message = error_message || \"an unrecoverable error occurs in Fluentd process\"\n    end\n\n    def to_s\n      @message\n    end\n  end\n\n  class InvalidRootDirectory < UnrecoverableError\n  end\n\n  class InvalidLockDirectory < UnrecoverableError\n  end\n\n  # For internal use\n  class UncatchableError < Exception\n  end\nend\n"
  },
  {
    "path": "lib/fluent/event.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/msgpack_factory'\nrequire 'fluent/plugin/compressable'\n\nmodule Fluent\n  class EventStream\n    include Enumerable\n    include Fluent::Plugin::Compressable\n\n    # dup does deep copy for event stream\n    def dup\n      raise NotImplementedError, \"DO NOT USE THIS CLASS directly.\"\n    end\n\n    def size\n      raise NotImplementedError, \"DO NOT USE THIS CLASS directly.\"\n    end\n    alias :length :size\n\n    def empty?\n      size == 0\n    end\n\n    # for tests\n    def ==(other)\n      other.is_a?(EventStream) && self.to_msgpack_stream == other.to_msgpack_stream\n    end\n\n    def repeatable?\n      false\n    end\n\n    def slice(index, num)\n      raise NotImplementedError, \"DO NOT USE THIS CLASS directly.\"\n    end\n\n    def each(unpacker: nil, &block)\n      raise NotImplementedError, \"DO NOT USE THIS CLASS directly.\"\n    end\n\n    def to_msgpack_stream(time_int: false, packer: nil)\n      return to_msgpack_stream_forced_integer(packer: packer) if time_int\n      out = packer || Fluent::MessagePackFactory.msgpack_packer\n      each {|time,record|\n        out.write([time,record])\n      }\n      out.full_pack\n    end\n\n    def to_compressed_msgpack_stream(time_int: false, packer: nil, type: :gzip)\n      packed = to_msgpack_stream(time_int: time_int, packer: packer)\n      compress(packed, type: type)\n    end\n\n    def to_msgpack_stream_forced_integer(packer: nil)\n      out = packer || Fluent::MessagePackFactory.msgpack_packer\n      each {|time,record|\n        out.write([time.to_i,record])\n      }\n      out.full_pack\n    end\n  end\n\n  class OneEventStream < EventStream\n    def initialize(time, record)\n      @time = time\n      @record = record\n    end\n\n    def dup\n      OneEventStream.new(@time, @record.dup)\n    end\n\n    def empty?\n      false\n    end\n\n    def size\n      1\n    end\n\n    def repeatable?\n      true\n    end\n\n    def slice(index, num)\n      if index > 0 || num == 0\n        ArrayEventStream.new([])\n      else\n        self.dup\n      end\n    end\n\n    def each(unpacker: nil, &block)\n      block.call(@time, @record)\n      nil\n    end\n  end\n\n  # EventStream from entries: Array of [time, record]\n  #\n  # Use this class for many events data with a tag\n  # and its representation is [ [time, record], [time, record], .. ]\n  class ArrayEventStream < EventStream\n    def initialize(entries)\n      @entries = entries\n    end\n\n    def dup\n      entries = @entries.map{ |time, record| [time, record.dup] }\n      ArrayEventStream.new(entries)\n    end\n\n    def size\n      @entries.size\n    end\n\n    def repeatable?\n      true\n    end\n\n    def empty?\n      @entries.empty?\n    end\n\n    def slice(index, num)\n      ArrayEventStream.new(@entries.slice(index, num))\n    end\n\n    def each(unpacker: nil, &block)\n      @entries.each(&block)\n      nil\n    end\n  end\n\n  # EventStream from entries: numbers of pairs of time and record.\n  #\n  # This class can handle many events more efficiently than ArrayEventStream\n  # because this class generate less objects than ArrayEventStream.\n  #\n  # Use this class as below, in loop of data-enumeration:\n  #  1. initialize blank stream:\n  #     streams[tag] ||= MultiEventStream.new\n  #  2. add events\n  #     stream[tag].add(time, record)\n  class MultiEventStream < EventStream\n    def initialize(time_array = [], record_array = [])\n      @time_array = time_array\n      @record_array = record_array\n    end\n\n    def dup\n      MultiEventStream.new(@time_array.dup, @record_array.map(&:dup))\n    end\n\n    def size\n      @time_array.size\n    end\n\n    def add(time, record)\n      @time_array << time\n      @record_array << record\n    end\n\n    def repeatable?\n      true\n    end\n\n    def empty?\n      @time_array.empty?\n    end\n\n    def slice(index, num)\n      MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))\n    end\n\n    def each(unpacker: nil, &block)\n      time_array = @time_array\n      record_array = @record_array\n      for i in 0..time_array.length-1\n        block.call(time_array[i], record_array[i])\n      end\n      nil\n    end\n  end\n\n  class MessagePackEventStream < EventStream\n    # https://github.com/msgpack/msgpack-ruby/issues/119\n\n    # Keep cached_unpacker argument for existing plugins\n    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)\n      @data = data\n      @size = size\n      @unpacked_times = unpacked_times\n      @unpacked_records = unpacked_records\n    end\n\n    def empty?\n      @data.empty?\n    end\n\n    def dup\n      if @unpacked_times\n        self.class.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))\n      else\n        self.class.new(@data.dup, nil, @size)\n      end\n    end\n\n    def size\n      # @size is unbelievable always when @size == 0\n      # If the number of events is really zero, unpacking events takes very short time.\n      ensure_unpacked! if @size == 0\n      @size\n    end\n\n    def repeatable?\n      true\n    end\n\n    def ensure_unpacked!(unpacker: nil)\n      return if @unpacked_times && @unpacked_records\n      @unpacked_times = []\n      @unpacked_records = []\n      (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|\n        @unpacked_times << time\n        @unpacked_records << record\n      end\n      # @size should be updated always right after unpack.\n      # The real size of unpacked objects are correct, rather than given size.\n      @size = @unpacked_times.size\n    end\n\n    # This method returns MultiEventStream, because there are no reason\n    # to survey binary serialized by msgpack.\n    def slice(index, num)\n      ensure_unpacked!\n      MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))\n    end\n\n    def each(unpacker: nil, &block)\n      ensure_unpacked!(unpacker: unpacker)\n      @unpacked_times.each_with_index do |time, i|\n        block.call(time, @unpacked_records[i])\n      end\n      nil\n    end\n\n    def to_msgpack_stream(time_int: false, packer: nil)\n      # time_int is always ignored because @data is always packed binary in this class\n      @data\n    end\n  end\n\n  class CompressedMessagePackEventStream < MessagePackEventStream\n    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil, compress: :gzip)\n      super(data, cached_unpacker, size, unpacked_times: unpacked_times, unpacked_records: unpacked_records)\n      @decompressed_data = nil\n      @compressed_data = data\n      @type = compress\n    end\n\n    def empty?\n      ensure_decompressed!\n      super\n    end\n\n    def ensure_unpacked!(unpacker: nil)\n      ensure_decompressed!\n      super\n    end\n\n    def each(unpacker: nil, &block)\n      ensure_decompressed!\n      super\n    end\n\n    def to_msgpack_stream(time_int: false, packer: nil)\n      ensure_decompressed!\n      super\n    end\n\n    def to_compressed_msgpack_stream(time_int: false, packer: nil)\n      # time_int is always ignored because @data is always packed binary in this class\n      @compressed_data\n    end\n\n    private\n\n    def ensure_decompressed!\n      return if @decompressed_data\n      @data = @decompressed_data = decompress(@data, type: @type)\n    end\n  end\n\n  module ChunkMessagePackEventStreamer\n    # chunk.extend(ChunkMessagePackEventStreamer)\n    #  => chunk.each{|time, record| ... }\n    def each(unpacker: nil, &block)\n      # Note: If need to use `unpacker`, then implement it,\n      # e.g., `unpacker.feed_each(io.read, &block)` (Not tested)\n      raise NotImplementedError, \"'unpacker' argument is not implemented.\" if unpacker\n\n      open do |io|\n        Fluent::MessagePackFactory.msgpack_unpacker(io).each(&block)\n      end\n      nil\n    end\n    alias :msgpack_each :each\n\n    def to_msgpack_stream(time_int: false, packer: nil)\n      # time_int is always ignored because data is already packed and written in chunk\n      read\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/event_router.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/match'\nrequire 'fluent/event'\nrequire 'fluent/filter'\nrequire 'fluent/msgpack_factory'\n\nmodule Fluent\n  #\n  # EventRouter is responsible to route events to a collector.\n  #\n  # It has a list of MatchPattern and Collector pairs:\n  #\n  #  +----------------+     +-----------------+\n  #  |  MatchPattern  |     |    Collector    |\n  #  +----------------+     +-----------------+\n  #  |   access.**  ---------> type forward   |\n  #  |     logs.**  ---------> type copy      |\n  #  |  archive.**  ---------> type s3        |\n  #  +----------------+     +-----------------+\n  #\n  # EventRouter does:\n  #\n  # 1) receive an event at `#emit` methods\n  # 2) match the event's tag with the MatchPatterns\n  # 3) forward the event to the corresponding Collector\n  #\n  # Collector is either of Output, Filter or other EventRouter.\n  #\n  class EventRouter\n    def initialize(default_collector, emit_error_handler)\n      @match_rules = []\n      @match_cache = MatchCache.new\n      @default_collector = default_collector\n      @emit_error_handler = emit_error_handler\n      @metric_callbacks = {}\n      @caller_plugin_id = nil\n    end\n\n    attr_accessor :default_collector\n    attr_accessor :emit_error_handler\n\n    class Rule\n      def initialize(pattern, collector)\n        patterns = pattern.split(/\\s+/).map { |str| MatchPattern.create(str) }\n        @pattern = if patterns.length == 1\n                     patterns[0]\n                   else\n                     OrMatchPattern.new(patterns)\n                   end\n        @pattern_str = pattern\n        @collector = collector\n      end\n\n      def match?(tag)\n        @pattern.match(tag)\n      end\n\n      attr_reader :collector\n      attr_reader :pattern_str\n    end\n\n    def suppress_missing_match!\n      if @default_collector.respond_to?(:suppress_missing_match!)\n        @default_collector.suppress_missing_match!\n      end\n    end\n\n    # called by Agent to add new match pattern and collector\n    def add_rule(pattern, collector)\n      @match_rules << Rule.new(pattern, collector)\n    end\n\n    def add_metric_callbacks(caller_plugin_id, callback)\n      @metric_callbacks[caller_plugin_id] = callback\n    end\n\n    def caller_plugin_id=(caller_plugin_id)\n      @caller_plugin_id = caller_plugin_id\n    end\n\n    def find_callback\n      if @caller_plugin_id\n        @metric_callbacks[@caller_plugin_id]\n      else\n        nil\n      end\n    end\n\n    def emit(tag, time, record)\n      unless record.nil?\n        emit_stream(tag, OneEventStream.new(time, record))\n      end\n    end\n\n    def emit_array(tag, array)\n      emit_stream(tag, ArrayEventStream.new(array))\n    end\n\n    def emit_stream(tag, es)\n      match(tag).emit_events(tag, es)\n      if callback = find_callback\n        callback.call(es)\n      end\n    rescue Pipeline::OutputError => e\n      @emit_error_handler.handle_emits_error(tag, e.processed_es, e.internal_error)\n    rescue => e\n      @emit_error_handler.handle_emits_error(tag, es, e)\n    end\n\n    def emit_error_event(tag, time, record, error)\n      @emit_error_handler.emit_error_event(tag, time, record, error)\n    end\n\n    def match?(tag)\n      !!find(tag)\n    end\n\n    def match(tag)\n      collector = @match_cache.get(tag) {\n        find(tag) || @default_collector\n      }\n      collector\n    end\n\n    class MatchCache\n      MATCH_CACHE_SIZE = 1024\n\n      def initialize\n        super\n        @map = {}\n        @keys = []\n      end\n\n      def get(key)\n        if collector = @map[key]\n          return collector\n        end\n        collector = @map[key] = yield\n        if @keys.size >= MATCH_CACHE_SIZE\n          # expire the oldest key\n          @map.delete @keys.shift\n        end\n        @keys << key\n        collector\n      end\n    end\n\n    private\n\n    class Pipeline\n\n      class OutputError < StandardError\n        attr_reader :internal_error\n        attr_reader :processed_es\n\n        def initialize(internal_error, processed_es)\n          @internal_error = internal_error\n          @processed_es = processed_es\n        end\n      end\n\n      def initialize\n        @filters = []\n        @output = nil\n        @optimizer = FilterOptimizer.new\n      end\n\n      def add_filter(filter)\n        @filters << filter\n        @optimizer.filters = @filters\n      end\n\n      def set_output(output)\n        @output = output\n      end\n\n      def emit_events(tag, es)\n        processed = @optimizer.filter_stream(tag, es)\n\n        begin\n          @output.emit_events(tag, processed)\n        rescue => e\n          raise OutputError.new(e, processed)\n        end\n      end\n\n      class FilterOptimizer\n        def initialize(filters = [])\n          @filters = filters\n          @optimizable = nil\n        end\n\n        def filters=(filters)\n          @filters = filters\n          reset_optimization\n        end\n\n        def filter_stream(tag, es)\n          if optimizable?\n            optimized_filter_stream(tag, es)\n          else\n            @filters.reduce(es) { |acc, filter|\n              filtered_es = filter.filter_stream(tag, acc)\n              filter.measure_metrics(filtered_es)\n              filtered_es\n            }\n          end\n        end\n\n        private\n\n        def optimized_filter_stream(tag, es)\n          new_es = MultiEventStream.new\n          es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|\n            filtered_record = record\n            filtered_time = time\n\n            catch :break_loop do\n              @filters.each do |filter|\n                if filter.has_filter_with_time\n                  begin\n                    filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)\n                    throw :break_loop unless filtered_record && filtered_time\n                    filter.measure_metrics(OneEventStream.new(time, record))\n                  rescue => e\n                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)\n                  end\n                else\n                  begin\n                    filtered_record = filter.filter(tag, filtered_time, filtered_record)\n                    throw :break_loop unless filtered_record\n                    filter.measure_metrics(OneEventStream.new(time, record))\n                  rescue => e\n                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)\n                  end\n                end\n              end\n\n              new_es.add(filtered_time, filtered_record)\n            end\n          end\n          new_es\n        end\n\n        def optimizable?\n          return @optimizable unless @optimizable.nil?\n          fs_filters = filters_having_filter_stream\n          @optimizable = if fs_filters.empty?\n                           true\n                         else\n                           # skip log message when filter is only 1, because its performance is same as non optimized chain.\n                           if @filters.size > 1 && fs_filters.size >= 1\n                             $log.info \"disable filter chain optimization because #{fs_filters.map(&:class)} uses `#filter_stream` method.\"\n                           end\n                           false\n                         end\n        end\n\n        def filters_having_filter_stream\n          @filters_having_filter_stream ||= @filters.select do |filter|\n            filter.class.instance_methods(false).include?(:filter_stream)\n          end\n        end\n\n        def reset_optimization\n          @optimizable = nil\n          @filters_having_filter_stream = nil\n        end\n      end\n    end\n\n    def find(tag)\n      pipeline = nil\n      @match_rules.each do |rule|\n        if rule.match?(tag)\n          if rule.collector.is_a?(Plugin::Filter)\n            pipeline ||= Pipeline.new\n            pipeline.add_filter(rule.collector)\n          else\n            if pipeline\n              pipeline.set_output(rule.collector)\n            else\n              # Use Output directly when filter is not matched\n              pipeline = rule.collector\n            end\n            return pipeline\n          end\n        end\n      end\n\n      if pipeline\n        # filter is matched but no match\n        pipeline.set_output(@default_collector)\n        pipeline\n      else\n        nil\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/ext_monitor_require.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\n# To avoid duplicated requirements, extract this logic as file.\n\nif Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('2.7.0')\n  require 'monitor'\nelse\n  begin\n    # monitor_ext is bundled since ruby 2.7.0\n    require 'ext_monitor'\n  rescue LoadError => _\n    require 'monitor'\n  end\nend\n"
  },
  {
    "path": "lib/fluent/file_wrapper.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nunless Fluent.windows?\n  Fluent::FileWrapper = File\nelse\n  require 'fluent/win32api'\n\n  module Fluent\n    module FileWrapper\n      def self.open(path, mode='r')\n        io = WindowsFile.new(path, mode).io\n        if block_given?\n          v = yield io\n          io.close\n          v\n        else\n          io\n        end\n      end\n\n      def self.stat(path)\n        f = WindowsFile.new(path)\n        s = f.stat\n        f.close\n        s\n      end\n    end\n\n    class WindowsFile\n      include File::Constants\n\n      attr_reader :io\n\n      INVALID_HANDLE_VALUE = -1\n\n      def initialize(path, mode_enc='r')\n        @path = path\n        mode, enc = mode_enc.split(\":\", 2)\n        @io = File.open(path, mode2flags(mode))\n        @io.set_encoding(enc) if enc\n        @file_handle = Win32API._get_osfhandle(@io.to_i)\n        @io.instance_variable_set(:@file_index, self.ino)\n        def @io.ino\n          @file_index\n        end\n      end\n\n      def close\n        @io.close\n        @file_handle = INVALID_HANDLE_VALUE\n      end\n\n      # To keep backward compatibility, we continue to use GetFileInformationByHandle()\n      # to get file id.\n      # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo\n      # and returned value is different with above one, former one is 64 bit while\n      # later one is 128bit.\n      def ino\n        by_handle_file_information = '\\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes\n\n        unless Win32API.GetFileInformationByHandle(@file_handle, by_handle_file_information)\n          return 0\n        end\n\n        by_handle_file_information.unpack(\"I11Q1\")[11] # fileindex\n      end\n\n      def stat\n        raise Errno::ENOENT if delete_pending\n        s = File.stat(@path)\n        s.instance_variable_set :@ino, self.ino\n        def s.ino; @ino; end\n        s\n      end\n\n      private\n\n      def mode2flags(mode)\n        # Always inject File::Constants::SHARE_DELETE\n        # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617\n        # To enable SHARE_DELETE, BINARY is also required.\n        # https://bugs.ruby-lang.org/issues/11218\n        # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345\n        flags = BINARY | SHARE_DELETE\n        case mode.delete(\"b\")\n        when \"r\"\n          flags |= RDONLY\n        when \"r+\"\n          flags |= RDWR\n        when \"w\"\n          flags |= WRONLY | CREAT | TRUNC\n        when \"w+\"\n          flags |= RDWR | CREAT | TRUNC\n        when \"a\"\n          flags |= WRONLY | CREAT | APPEND\n        when \"a+\"\n          flags |= RDWR | CREAT | APPEND\n        else\n          raise Errno::EINVAL.new(\"Unsupported mode by Fluent::FileWrapper: #{mode}\")\n        end\n      end\n\n      # DeletePending is a Windows-specific file state that roughly means\n      # \"this file is queued for deletion, so close any open handlers\"\n      #\n      # This flag can be retrieved via GetFileInformationByHandleEx().\n      #\n      # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex\n      #\n      def delete_pending\n        file_standard_info = 0x01\n        bufsize = 1024\n        buf = '\\0' * bufsize\n\n        unless Win32API.GetFileInformationByHandleEx(@file_handle, file_standard_info, buf, bufsize)\n          return false\n        end\n\n        return buf.unpack(\"QQICC\")[3] != 0\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/filter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/filter'\n\nmodule Fluent\n  Filter = Fluent::Compat::Filter\nend\n"
  },
  {
    "path": "lib/fluent/fluent_log_event_router.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/log'\n\nmodule Fluent\n  # DO NOT write any logic here\n  class NullFluentLogEventRouter\n    def start; end\n\n    def stop; end\n\n    def graceful_stop; end\n\n    def emit_event(_event); end\n\n    def emittable?\n      self.class != NullFluentLogEventRouter\n    end\n  end\n\n  # This class is for handling fluentd's inner log\n  # e.g. <label @FLUNT_LOG> section and <match fluent.**> section\n  class FluentLogEventRouter < NullFluentLogEventRouter\n    # @param root_agent [Fluent::RootAgent]\n    def self.build(root_agent)\n      log_event_router = nil\n\n      begin\n        log_event_agent = root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)\n        log_event_router = log_event_agent.event_router\n\n        # suppress mismatched tags only for <label @FLUENT_LOG> label.\n        # it's not suppressed in default event router for non-log-event events\n        log_event_router.suppress_missing_match!\n\n        unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }\n        unless unmatched_tags.empty?\n          $log.warn \"match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)\", tags: unmatched_tags\n        end\n\n      rescue ArgumentError # ArgumentError \"#{label_name} label not found\"\n        # use default event router if <label @FLUENT_LOG> is missing in configuration\n        root_log_event_router = root_agent.event_router\n        event_tags = Fluent::Log.event_tags\n        if event_tags.any? { |t| root_log_event_router.match?(t) }\n          log_event_router = root_log_event_router\n\n          unmatched_tags = event_tags.select { |t| !log_event_router.match?(t) }\n          if unmatched_tags.empty?\n            $log.warn \"define <match fluent.**> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead\"\n          else\n            matched_sections = (event_tags - unmatched_tags).map { |tag| \"<match #{tag}>\" }.join(', ')\n            $log.warn \"define #{matched_sections} to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead\"\n            $log.warn \"match for some tags of log events are not defined in top level (to be ignored)\", tags: unmatched_tags\n          end\n        end\n      end\n\n      if log_event_router\n        FluentLogEventRouter.new(log_event_router)\n      else\n        $log.debug('No fluent logger for internal event')\n        NullFluentLogEventRouter.new\n      end\n    end\n\n    STOP = :stop\n    GRACEFUL_STOP = :graceful_stop\n\n    # @param event_router [Fluent::EventRouter]\n    def initialize(event_router)\n      @event_router = event_router\n      @thread = nil\n      @graceful_stop = false\n      @event_queue = Queue.new\n    end\n\n    def start\n      @thread = Thread.new do\n        $log.disable_events(Thread.current)\n\n        loop do\n          event = @event_queue.pop\n\n          case event\n          when GRACEFUL_STOP\n            @graceful_stop = true\n          when STOP\n            break\n          else\n            begin\n              tag, time, record = event\n              @event_router.emit(tag, time, record)\n            rescue => e\n              # This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above\n              $log.error \"failed to emit fluentd's log event\", tag: tag, event: record, error: e\n            end\n          end\n\n          if @graceful_stop && @event_queue.empty?\n            break\n          end\n        end\n      end\n\n      @thread.abort_on_exception = true\n    end\n\n    def stop\n      @event_queue.push(STOP)\n      # there is no problem calling Thread#join multiple times.\n      @thread && @thread.join\n    end\n\n    def graceful_stop\n      # to make sure to emit all log events into router, before shutting down\n      @event_queue.push(GRACEFUL_STOP)\n      @thread && @thread.join\n    end\n\n    def emit_event(event)\n      @event_queue.push(event)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/formatter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/formatter'\n\nmodule Fluent\n  Formatter = Fluent::Compat::Formatter\n  TextFormatter = Fluent::Compat::TextFormatter\n  # deprecate_constant is ruby 2.3 feature\nend\n"
  },
  {
    "path": "lib/fluent/input.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/input'\n\nmodule Fluent\n  Input = Fluent::Compat::Input\nend\n"
  },
  {
    "path": "lib/fluent/label.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/agent'\n\nmodule Fluent\n  class Label < Agent\n    def initialize(name, log:)\n      super(log: log)\n\n      @context = name\n      @root_agent = nil\n    end\n\n    attr_accessor :root_agent\n\n    def configure(conf)\n      super\n\n      if conf.elements('match').size == 0\n        raise ConfigError, \"Missing <match> sections in <label #{@context}> section\"\n      end\n    end\n\n    def emit_error_event(tag, time, record, e)\n      @root_agent.emit_error_event(tag, time, record, e)\n    end\n\n    def handle_emits_error(tag, es, e)\n      @root_agent.handle_emits_error(tag, es, e)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/load.rb",
    "content": "require 'socket'\nrequire 'fcntl'\nrequire 'time'\nrequire 'stringio'\nrequire 'fileutils'\nrequire 'json'\nrequire 'yajl'\nrequire 'uri'\nrequire 'msgpack'\nrequire 'strptime'\nbegin\n  require 'sigdump/setup'\nrescue\n  # ignore setup error on Win or similar platform which doesn't support signal\nend\nrequire 'cool.io'\n\nrequire 'fluent/time'\nrequire 'fluent/env'\nrequire 'fluent/version'\nrequire 'fluent/log'\nrequire 'fluent/config'\nrequire 'fluent/engine'\nrequire 'fluent/rpc'\nrequire 'fluent/mixin'\nrequire 'fluent/plugin'\nrequire 'fluent/parser'\nrequire 'fluent/formatter'\nrequire 'fluent/event'\nrequire 'fluent/input'\nrequire 'fluent/output'\nrequire 'fluent/filter'\nrequire 'fluent/match'\nrequire 'fluent/ext_monitor_require'\n"
  },
  {
    "path": "lib/fluent/log/console_adapter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'console'\n\nmodule Fluent\n  class Log\n    # Async gem which is used by http_server helper switched logger mechanism to\n    # Console gem which isn't compatible with Ruby's standard Logger (since\n    # v1.17). This class adapts it to Fluentd's logger mechanism.\n    class ConsoleAdapter < Console::Output::Terminal\n      def self.wrap(logger)\n        _, level = Console::Logger::LEVELS.find { |key, value|\n          if logger.level <= 0\n            key == :debug\n          else\n            value == logger.level - 1\n          end\n        }\n        Console::Logger.new(ConsoleAdapter.new(logger), level: level)\n      end\n\n      def initialize(logger)\n        @logger = logger\n        # When `verbose` is `true`, following items will be added as a prefix or\n        # suffix of the subject:\n        #   * Severity\n        #   * Object ID\n        #   * PID\n        #   * Time\n        # Severity and Time are added by Fluentd::Log too so they are redundant.\n        # PID is the worker's PID so it's also redundant.\n        # Object ID will be too verbose in usual cases.\n        # So set it as `false` here to suppress redundant items.\n        super(StringIO.new, verbose: false)\n      end\n\n      def call(subject = nil, *arguments, name: nil, severity: 'info', **options, &block)\n        if LEVEL_TEXT.include?(severity.to_s)\n          level = severity\n        else\n          @logger.warn(\"Unknown severity: #{severity}\")\n          level = 'warn'\n        end\n\n        @stream.seek(0)\n        @stream.truncate(0)\n        super\n        @logger.send(level, @stream.string.chomp)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/log.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'forwardable'\nrequire 'logger'\n\nmodule Fluent\n  class Log\n    module TTYColor\n      RESET   = \"\\033]R\"\n      CRE     = \"\\033[K\"\n      CLEAR   = \"\\033c\"\n      NORMAL  = \"\\033[0;39m\"\n      RED     = \"\\033[1;31m\"\n      GREEN   = \"\\033[1;32m\"\n      YELLOW  = \"\\033[1;33m\"\n      BLUE    = \"\\033[1;34m\"\n      MAGENTA = \"\\033[1;35m\"\n      CYAN    = \"\\033[1;36m\"\n      WHITE   = \"\\033[1;37m\"\n    end\n\n    LEVEL_TRACE = 0\n    LEVEL_DEBUG = 1\n    LEVEL_INFO  = 2\n    LEVEL_WARN  = 3\n    LEVEL_ERROR = 4\n    LEVEL_FATAL = 5\n\n    LEVEL_TEXT = %w(trace debug info warn error fatal)\n\n    LOG_EVENT_TAG_PREFIX = 'fluent'\n    LOG_EVENT_LABEL = '@FLUENT_LOG'\n    LOG_TYPE_SUPERVISOR = :supervisor # only in supervisor, or a worker with --no-supervisor\n    LOG_TYPE_WORKER0 = :worker0 # only in a worker with worker_id=0 (without showing worker id)\n    LOG_TYPE_DEFAULT = :default # show logs in all supervisor/workers, with worker id in workers (default)\n\n    LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze\n    LOG_ROTATE_AGE = %w(daily weekly monthly)\n\n    IGNORE_SAME_LOG_MAX_CACHE_SIZE = 1000 # If need, make this an option of system config.\n\n    def self.str_to_level(log_level_str)\n      case log_level_str.downcase\n      when \"trace\" then LEVEL_TRACE\n      when \"debug\" then LEVEL_DEBUG\n      when \"info\"  then LEVEL_INFO\n      when \"warn\"  then LEVEL_WARN\n      when \"error\" then LEVEL_ERROR\n      when \"fatal\" then LEVEL_FATAL\n      else raise \"Unknown log level: level = #{log_level_str}\"\n      end\n    end\n\n    def self.event_tags\n      LEVEL_TEXT.map{|t| \"#{LOG_EVENT_TAG_PREFIX}.#{t}\" }\n    end\n\n    # Create a unique path for each process.\n    #\n    # >>> per_process_path(\"C:/tmp/test.log\", :worker, 1)\n    # C:/tmp/test-1.log\n    # >>> per_process_path(\"C:/tmp/test.log\", :supervisor, 0)\n    # C:/tmp/test-supervisor-0.log\n    def self.per_process_path(path, process_type, worker_id)\n      path = Pathname(path)\n      ext = path.extname\n\n      if process_type == :supervisor\n        suffix = \"-#{process_type}-0#{ext}\"  # \"-0\" for backword compatibility.\n      else\n        suffix = \"-#{worker_id}#{ext}\"\n      end\n      return path.sub_ext(suffix).to_s\n    end\n\n    def initialize(logger, opts={})\n      # When ServerEngine changes the logger.level, the Fluentd logger level should also change.\n      # So overwrites logger.level= below.\n      # However, currently Fluentd doesn't use the ServerEngine's reloading feature,\n      # so maybe we don't need this overwriting anymore.\n      orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)\n      me = self\n      # The original ruby logger sets the number as each log level like below.\n      # DEBUG = 0\n      # INFO  = 1\n      # WARN  = 2\n      # ERROR = 3\n      # FATAL = 4\n      # Serverengine use this original log number. In addition to this, serverengine sets -1 as TRACE level.\n      # TRACE = -1\n      #\n      # On the other hand, in fluentd side, it sets the number like below.\n      # TRACE = 0\n      # DEBUG = 1\n      # INFO  = 2\n      # WARN  = 3\n      # ERROR = 4\n      # FATAL = 5\n      #\n      # Then fluentd's level is set as serverengine's level + 1.\n      # So if serverengine's logger level is changed, fluentd's log level will be changed to that + 1.\n      logger.define_singleton_method(:level=) {|level| orig_logger_level_setter.call(level); me.level = self.level + 1 }\n\n      @path = opts[:path]\n      @logger = logger\n      @out = logger.instance_variable_get(:@logdev)\n      @level = logger.level + 1\n      @debug_mode = false\n      @log_event_enabled = false\n      @depth_offset = 1\n      @format = nil\n      @time_format = nil\n      @formatter = nil\n\n      self.format = opts.fetch(:format, :text)\n      self.time_format = opts[:time_format] if opts.key?(:time_format)\n      enable_color out.tty?\n      # TODO: This variable name is unclear so we should change to better name.\n      @threads_exclude_events = []\n\n      # Fluent::Engine requires Fluent::Log, so we must take that object lazily\n      @engine = Fluent.const_get('Engine')\n      @optional_header = nil\n      @optional_attrs = nil\n\n      @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]\n      @forced_stacktrace_level = nil\n      @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]\n      @ignore_same_log_interval = opts[:ignore_same_log_interval]\n\n      @process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone\n      @process_type ||= :standalone # to keep behavior of existing code\n      case @process_type\n      when :supervisor\n        @show_supervisor_log = true\n        @show_worker0_log = false\n      when :worker0\n        @show_supervisor_log = false\n        @show_worker0_log = true\n      when :workers\n        @show_supervisor_log = false\n        @show_worker0_log = false\n      when :standalone\n        @show_supervisor_log = true\n        @show_worker0_log = true\n      else\n        raise \"BUG: unknown process type for logger:#{@process_type}\"\n      end\n      @worker_id = opts[:worker_id]\n      @worker_id_part = \"##{@worker_id} \" # used only for :default log type in workers\n    end\n\n    def dup\n      dl_opts = {}\n      dl_opts[:log_level] = @level - 1\n      logger = ServerEngine::DaemonLogger.new(@out, dl_opts)\n      clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type,\n                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval,\n                             ignore_same_log_interval: @ignore_same_log_interval)\n      clone.format = @format\n      clone.time_format = @time_format\n      clone.log_event_enabled = @log_event_enabled\n      clone.force_stacktrace_level(@forced_stacktrace_level)\n      # optional headers/attrs are not copied, because new PluginLogger should have another one of it\n      clone\n    end\n\n    attr_reader :format\n    attr_reader :time_format\n    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval, :suppress_repeated_stacktrace\n    attr_accessor :out\n    # Strictly speaking, we should also change @logger.level when the setter of @level is called.\n    # Currently, we don't need to do it, since Fluentd::Log doesn't use ServerEngine::DaemonLogger.level.\n    # Since We overwrites logger.level= so that @logger.level is applied to @level,\n    # we need to find a good way to do this, otherwise we will end up in an endless loop.\n    attr_accessor :level\n    attr_accessor :optional_header, :optional_attrs\n\n    def logdev=(logdev)\n      @out = logdev\n      @logger.instance_variable_set(:@logdev, logdev)\n      nil\n    end\n\n    def format=(fmt)\n      return if @format == fmt\n\n      @time_format = '%Y-%m-%d %H:%M:%S %z'\n      @time_formatter = Strftime.new(@time_format) rescue nil\n\n      case fmt\n      when :text\n        @format = :text\n        @formatter = Proc.new { |type, time, level, msg|\n          r = caller_line(type, time, @depth_offset, level)\n          r << msg\n          r\n        }\n      when :json\n        @format = :json\n        @formatter = Proc.new { |type, time, level, msg|\n          r = {\n            'time' => format_time(time),\n            'level' => LEVEL_TEXT[level],\n            'message' => msg\n          }\n          if wid = get_worker_id(type)\n            r['worker_id'] = wid\n          end\n          JSON.generate(r)\n        }\n      end\n\n      nil\n    end\n\n    def time_format=(time_fmt)\n      @time_format = time_fmt\n      @time_formatter = Strftime.new(@time_format) rescue nil\n    end\n\n    def stdout?\n      @out == $stdout\n    end\n\n    def reopen!\n      @out.reopen(@path, \"a\") if @path && @path != \"-\"\n      nil\n    end\n\n    def force_stacktrace_level?\n      not @forced_stacktrace_level.nil?\n    end\n\n    def force_stacktrace_level(level)\n      @forced_stacktrace_level = level\n    end\n\n    def enable_debug(b=true)\n      @debug_mode = b\n      self\n    end\n\n    def enable_event(b=true)\n      @log_event_enabled = b\n      self\n    end\n\n    # If you want to suppress event emitting in specific thread, please use this method.\n    # Events in passed thread are never emitted.\n    def disable_events(thread)\n      # this method is not symmetric with #enable_event.\n      @threads_exclude_events.push(thread) unless @threads_exclude_events.include?(thread)\n    end\n\n    def enable_color?\n      !@color_reset.empty?\n    end\n\n    def enable_color(b=true)\n      if b\n        @color_trace = TTYColor::BLUE\n        @color_debug = TTYColor::WHITE\n        @color_info  = TTYColor::GREEN\n        @color_warn  = TTYColor::YELLOW\n        @color_error = TTYColor::MAGENTA\n        @color_fatal = TTYColor::RED\n        @color_reset = TTYColor::NORMAL\n      else\n        @color_trace = ''\n        @color_debug = ''\n        @color_info  = ''\n        @color_warn  = ''\n        @color_error = ''\n        @color_fatal = ''\n        @color_reset = ''\n      end\n      self\n    end\n\n    def log_type(args)\n      if LOG_TYPES.include?(args.first)\n        args.shift\n      else\n        LOG_TYPE_DEFAULT\n      end\n    end\n\n    # TODO: skip :worker0 logs when Fluentd gracefully restarted\n    def skipped_type?(type)\n      case type\n      when LOG_TYPE_DEFAULT\n        false\n      when LOG_TYPE_WORKER0\n        !@show_worker0_log\n      when LOG_TYPE_SUPERVISOR\n        !@show_supervisor_log\n      else\n        raise \"BUG: unknown log type:#{type}\"\n      end\n    end\n\n    def on_trace\n      return if @level > LEVEL_TRACE\n      yield\n    end\n\n    def trace(*args, &block)\n      return if @level > LEVEL_TRACE\n      type = log_type(args)\n      return if skipped_type?(type)\n      args << block.call if block\n      time, msg = event(:trace, args)\n      return if time.nil?\n      puts [@color_trace, @formatter.call(type, time, LEVEL_TRACE, msg), @color_reset].join\n    rescue\n      # logger should not raise an exception. This rescue prevents unexpected behaviour.\n    end\n    alias TRACE trace\n\n    def trace_backtrace(backtrace=$!.backtrace, type: :default)\n      dump_stacktrace(type, backtrace, LEVEL_TRACE)\n    end\n\n    def on_debug\n      return if @level > LEVEL_DEBUG\n      yield\n    end\n\n    def debug(*args, &block)\n      return if @level > LEVEL_DEBUG\n      type = log_type(args)\n      return if skipped_type?(type)\n      args << block.call if block\n      time, msg = event(:debug, args)\n      return if time.nil?\n      puts [@color_debug, @formatter.call(type, time, LEVEL_DEBUG, msg), @color_reset].join\n    rescue\n    end\n    alias DEBUG debug\n\n    def debug_backtrace(backtrace=$!.backtrace, type: :default)\n      dump_stacktrace(type, backtrace, LEVEL_DEBUG)\n    end\n\n    def on_info\n      return if @level > LEVEL_INFO\n      yield\n    end\n\n    def info(*args, &block)\n      return if @level > LEVEL_INFO\n      type = log_type(args)\n      return if skipped_type?(type)\n      args << block.call if block\n      time, msg = event(:info, args)\n      return if time.nil?\n      puts [@color_info, @formatter.call(type, time, LEVEL_INFO, msg), @color_reset].join\n    rescue\n    end\n    alias INFO info\n\n    def info_backtrace(backtrace=$!.backtrace, type: :default)\n      dump_stacktrace(type, backtrace, LEVEL_INFO)\n    end\n\n    def on_warn\n      return if @level > LEVEL_WARN\n      yield\n    end\n\n    def warn(*args, &block)\n      return if @level > LEVEL_WARN\n      type = log_type(args)\n      return if skipped_type?(type)\n      args << block.call if block\n      time, msg = event(:warn, args)\n      return if time.nil?\n      puts [@color_warn, @formatter.call(type, time, LEVEL_WARN, msg), @color_reset].join\n    rescue\n    end\n    alias WARN warn\n\n    def warn_backtrace(backtrace=$!.backtrace, type: :default)\n      dump_stacktrace(type, backtrace, LEVEL_WARN)\n    end\n\n    def on_error\n      return if @level > LEVEL_ERROR\n      yield\n    end\n\n    def error(*args, &block)\n      return if @level > LEVEL_ERROR\n      type = log_type(args)\n      return if skipped_type?(type)\n      args << block.call if block\n      time, msg = event(:error, args)\n      return if time.nil?\n      puts [@color_error, @formatter.call(type, time, LEVEL_ERROR, msg), @color_reset].join\n    rescue\n    end\n    alias ERROR error\n\n    def error_backtrace(backtrace=$!.backtrace, type: :default)\n      dump_stacktrace(type, backtrace, LEVEL_ERROR)\n    end\n\n    def on_fatal\n      return if @level > LEVEL_FATAL\n      yield\n    end\n\n    def fatal(*args, &block)\n      return if @level > LEVEL_FATAL\n      type = log_type(args)\n      return if skipped_type?(type)\n      args << block.call if block\n      time, msg = event(:fatal, args)\n      return if time.nil?\n      puts [@color_fatal, @formatter.call(type, time, LEVEL_FATAL, msg), @color_reset].join\n    rescue\n    end\n    alias FATAL fatal\n\n    def fatal_backtrace(backtrace=$!.backtrace, type: :default)\n      dump_stacktrace(type, backtrace, LEVEL_FATAL)\n    end\n\n    def puts(msg)\n      @logger << msg + \"\\n\"\n      @out.flush\n      msg\n    rescue\n      # FIXME\n      nil\n    end\n\n    def write(data)\n      @out.write(data)\n    end\n    # We need `#<<` method to use this logger class with other\n    # libraries such as aws-sdk\n    alias << write\n\n    def flush\n      @out.flush\n    end\n\n    def reset\n      @out.reset if @out.respond_to?(:reset)\n    end\n\n    CachedLog = Struct.new(:msg, :time)\n\n    def ignore_repeated_log?(key, time, message)\n      cached_log = Thread.current[key]\n      return false if cached_log.nil?\n      (cached_log.msg == message) && (time - cached_log.time <= @ignore_repeated_log_interval)\n    end\n\n    def ignore_same_log?(time, message)\n      cached_log = Thread.current[:last_same_log]\n      if cached_log.nil?\n        Thread.current[:last_same_log] = {message => time}\n        return false\n      end\n\n      prev_time = cached_log[message]\n      if prev_time\n        if (time - prev_time) <= @ignore_same_log_interval\n          true\n        else\n          cached_log[message] = time\n          false\n        end\n      else\n        if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE\n          cached_log.reject! do |_, cached_time|\n            (time - cached_time) > @ignore_same_log_interval\n          end\n        end\n        # If the size is still over, we have no choice but to clear it.\n        cached_log.clear if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE\n        cached_log[message] = time\n        false\n      end\n    end\n\n    def suppress_stacktrace?(backtrace)\n      cached_log = Thread.current[:last_repeated_stacktrace]\n      return false if cached_log.nil?\n      cached_log.msg == backtrace\n    end\n\n    def dump_stacktrace(type, backtrace, level)\n      return if @level > level\n\n      dump_stacktrace_internal(\n        type,\n        backtrace,\n        force_stacktrace_level? ? @forced_stacktrace_level : level,\n      )\n    end\n\n    def dump_stacktrace_internal(type, backtrace, level)\n      return if @level > level\n\n      time = Time.now\n\n      if @format == :text\n        line = caller_line(type, time, 5, level)\n        if @ignore_repeated_log_interval && ignore_repeated_log?(:last_repeated_stacktrace, time, backtrace)\n          return\n        elsif @suppress_repeated_stacktrace && suppress_stacktrace?(backtrace)\n          puts [\"  \", line, 'suppressed same stacktrace'].join\n          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @ignore_repeated_log_interval\n        else\n          backtrace.each { |msg|\n            puts [\"  \", line, msg].join\n          }\n          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace\n        end\n      else\n        r = {\n          'time' => format_time(time),\n          'level' => LEVEL_TEXT[level],\n        }\n        if wid = get_worker_id(type)\n          r['worker_id'] = wid\n        end\n\n        if @ignore_repeated_log_interval && ignore_repeated_log?(:last_repeated_stacktrace, time, backtrace)\n          return\n        elsif @suppress_repeated_stacktrace && suppress_stacktrace?(backtrace)\n          r['message'] = 'suppressed same stacktrace'\n          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @ignore_repeated_log_interval\n        else\n          r['message'] = backtrace.join(\"\\n\")\n          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace\n        end\n\n        puts JSON.generate(r)\n      end\n\n      nil\n    end\n\n    def get_worker_id(type)\n      if type == :default && (@process_type == :worker0 || @process_type == :workers)\n        @worker_id\n      else\n        nil\n      end\n    end\n\n    def event(level, args)\n      time = Time.now\n      message = @optional_header ? @optional_header.dup : ''\n      map = @optional_attrs ? @optional_attrs.dup : {}\n      args.each {|a|\n        if a.is_a?(Hash)\n          a.each_pair {|k,v|\n            map[k.to_s] = v\n          }\n        else\n          message << a.to_s\n        end\n      }\n\n      map.each_pair {|k,v|\n        if k == \"error\".freeze && v.is_a?(Exception) && !map.has_key?(\"error_class\")\n          message << \" error_class=#{v.class.to_s} error=#{v.to_s.inspect}\"\n        else\n          message << \" #{k}=#{v.inspect}\"\n        end\n      }\n\n      if @ignore_same_log_interval\n        if ignore_same_log?(time, message)\n          return nil, nil\n        end\n      elsif @ignore_repeated_log_interval\n        if ignore_repeated_log?(:last_repeated_log, time, message)\n          return nil, nil\n        else\n          Thread.current[:last_repeated_log] = CachedLog.new(message, time)\n        end\n      end\n\n      if @log_event_enabled && !@threads_exclude_events.include?(Thread.current)\n        record = map.dup\n        record.keys.each {|key|\n          record[key] = record[key].inspect unless record[key].respond_to?(:to_msgpack)\n        }\n        record['message'] = message.dup\n        @engine.push_log_event(\"#{LOG_EVENT_TAG_PREFIX}.#{level}\", Fluent::EventTime.from_time(time), record)\n      end\n\n      return time, message\n    end\n\n    def caller_line(type, time, depth, level)\n      worker_id_part = if type == :default && (@process_type == :worker0 || @process_type == :workers)\n                         @worker_id_part\n                       else\n                         \"\".freeze\n                       end\n      log_msg = \"#{format_time(time)} [#{LEVEL_TEXT[level]}]: #{worker_id_part}\"\n      if @debug_mode\n        line = caller(depth+1)[0]\n        if match = /^(.+?):(\\d+)(?::in `(.*)')?/.match(line)\n          file = match[1].split('/')[-2,2].join('/')\n          line = match[2]\n          method = match[3]\n          return \"#{log_msg}#{file}:#{line}:#{method}: \"\n        end\n      end\n      return log_msg\n    end\n\n    def format_time(time)\n      @time_formatter ? @time_formatter.exec(time) : time.strftime(@time_format)\n    end\n  end\n\n\n  # PluginLogger has own log level separated from global $log object.\n  # This class enables log_level option in each plugin.\n  #\n  # PluginLogger has same functionality as Log but some methods are forwarded to internal logger\n  # for keeping logging action consistency in the process, e.g. color, event, etc.\n  class PluginLogger < Log\n    def initialize(logger)\n      @logger = logger\n      @level = @logger.level\n      @format = nil\n      @depth_offset = 2\n      if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)\n        @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)\n      end\n      if logger.instance_variable_defined?(:@forced_stacktrace_level)\n        @forced_stacktrace_level = logger.instance_variable_get(:@forced_stacktrace_level)\n      end\n      if logger.instance_variable_defined?(:@ignore_repeated_log_interval)\n        @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)\n      end\n      if logger.instance_variable_defined?(:@ignore_same_log_interval)\n        @ignore_same_log_interval = logger.instance_variable_get(:@ignore_same_log_interval)\n      end\n\n      self.format = @logger.format\n      self.time_format = @logger.time_format\n      enable_color @logger.enable_color?\n    end\n\n    def level=(log_level_str)\n      @level = Log.str_to_level(log_level_str)\n    end\n\n    alias orig_format= format=\n    alias orig_time_format= time_format=\n    alias orig_enable_color enable_color\n\n    def format=(fmt)\n      self.orig_format = fmt\n      @logger.format = fmt\n    end\n\n    def time_format=(fmt)\n      self.orig_time_format = fmt\n      @logger.time_format = fmt\n    end\n\n    def enable_color(b = true)\n      orig_enable_color b\n      @logger.enable_color b\n    end\n\n    extend Forwardable\n    def_delegators '@logger', :get_worker_id, :enable_color?, :enable_debug, :enable_event,\n      :disable_events, :log_event_enabled, :log_event_enabled=, :event, :caller_line, :puts, :write,\n      :<<, :flush, :reset, :out, :out=, :optional_header, :optional_header=, :optional_attrs,\n      :optional_attrs=\n  end\n\n\n  module PluginLoggerMixin\n    def self.included(klass)\n      klass.instance_eval {\n        desc 'Allows the user to set different levels of logging for each plugin.'\n        config_param :@log_level, :string, default: nil, alias: :log_level # 'log_level' will be warned as deprecated\n      }\n    end\n\n    def initialize\n      super\n\n      @log = $log # Use $log object directly by default\n    end\n\n    attr_accessor :log\n\n    def configure(conf)\n      super\n\n      if plugin_id_configured? || conf['@log_level']\n        @log = PluginLogger.new($log.dup) unless @log.is_a?(PluginLogger)\n        @log.optional_attrs = {}\n\n        if level = conf['@log_level']\n          @log.level = level\n        end\n\n        if plugin_id_configured?\n          @log.optional_header = \"[#{@id}] \"\n        end\n      end\n    end\n\n    def terminate\n      super\n      @log.reset\n    end\n  end\n\n  # This class delegates some methods which are used in `Fluent::Logger` to a instance variable(`dev`) in `Logger::LogDevice` class\n  # https://github.com/ruby/ruby/blob/7b2d47132ff8ee950b0f978ab772dee868d9f1b0/lib/logger.rb#L661\n  class LogDeviceIO < ::Logger::LogDevice\n    def flush\n      if @dev.respond_to?(:flush)\n        @dev.flush\n      else\n        super\n      end\n    end\n\n    def tty?\n      if @dev.respond_to?(:tty?)\n        @dev.tty?\n      else\n        super\n      end\n    end\n\n    def sync=(v)\n      if @dev.respond_to?(:sync=)\n        @dev.sync = v\n      else\n        super\n      end\n    end\n\n    def reopen(path, mode)\n      if mode != 'a'\n        raise \"Unsupported mode: #{mode}\"\n      end\n      super(path)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/match.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  class MatchPattern\n    def self.create(str)\n      if str == '**'\n        AllMatchPattern.new\n      else\n        GlobMatchPattern.new(str)\n      end\n    end\n  end\n\n  class AllMatchPattern < MatchPattern\n    def match(str)\n      true\n    end\n  end\n\n  class GlobMatchPattern < MatchPattern\n    def initialize(pat)\n      if pat.start_with?('/')\n        if pat.end_with?('/')\n          @regex = Regexp.new(\"\\\\A\"+pat[1..-2]+\"\\\\Z\")\n          return\n        else\n          raise Fluent::ConfigError,  \"invalid match - regex\"\n        end\n      end\n\n      stack = []\n      regex = ['']\n      escape = false\n      dot = false\n\n      i = 0\n      while i < pat.length\n        c = pat[i,1]\n\n        if escape\n          regex.last << Regexp.escape(c)\n          escape = false\n          i += 1\n          next\n\n        elsif pat[i,2] == \"**\"\n          # recursive any\n          if dot\n            regex.last << \"(?![^\\\\.])\"\n            dot = false\n          end\n          if pat[i+2,1] == \".\"\n            regex.last << \"(?:.*\\\\.|\\\\A)\"\n            i += 3\n          else\n            regex.last << \".*\"\n            i += 2\n          end\n          next\n\n        elsif dot\n          regex.last << \"\\\\.\"\n          dot = false\n        end\n\n        if c == \"\\\\\"\n          escape = true\n\n        elsif c == \".\"\n          dot = true\n\n        elsif c == \"*\"\n          # any\n          regex.last << \"[^\\\\.]*\"\n\n          # TODO\n          #elsif c == \"[\"\n          #  # character class\n          #  chars = ''\n          #  while i < pat.length\n          #    c = pat[i,1]\n          #    if c == \"]\"\n          #      break\n          #    else\n          #      chars << c\n          #    end\n          #    i += 1\n          #  end\n          #  regex.last << '['+Regexp.escape(chars).gsub(\"\\\\-\",'-')+']'\n\n        elsif c == \"{\"\n          # or\n          stack.push []\n          regex.push ''\n\n        elsif c == \"}\" && !stack.empty?\n          stack.last << regex.pop\n          regex.last << Regexp.union(*stack.pop.map {|r| Regexp.new(r) }).to_s\n\n        elsif c == \",\" && !stack.empty?\n          stack.last << regex.pop\n          regex.push ''\n\n        elsif /[a-zA-Z0-9_]/.match?(c)\n          regex.last << c\n\n        else\n          regex.last << \"\\\\#{c}\"\n        end\n\n        i += 1\n      end\n\n      until stack.empty?\n        stack.last << regex.pop\n        regex.last << Regexp.union(*stack.pop).to_s\n      end\n\n      @regex = Regexp.new(\"\\\\A\"+regex.last+\"\\\\Z\")\n    end\n\n    def match(str)\n      @regex.match?(str)\n    end\n  end\n\n  class OrMatchPattern < MatchPattern\n    def initialize(patterns)\n      @patterns = patterns\n    end\n\n    def match(str)\n      @patterns.any? {|pattern| pattern.match(str) }\n    end\n  end\n\n  class NoMatchMatch\n    def initialize(log)\n      @log = log\n      @count = 0\n      @warn_not_matched = true\n    end\n\n    def suppress_missing_match!\n      # for <label @FLUENT_LOG>\n      @warn_not_matched = false\n    end\n\n    def emit_events(tag, es)\n      return unless @warn_not_matched\n      # TODO use time instead of num of records\n      c = (@count += 1)\n      if c < 512\n        if Math.log(c) / Math.log(2) % 1.0 == 0\n          @log.warn \"no patterns matched\", tag: tag\n          return\n        end\n      else\n        if c % 512 == 0\n          @log.warn \"no patterns matched\", tag: tag\n          return\n        end\n      end\n      @log.on_trace { @log.trace \"no patterns matched\", tag: tag }\n    end\n\n    def start\n    end\n\n    def shutdown\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/mixin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/record_filter_mixin'\nrequire 'fluent/compat/handle_tag_name_mixin'\nrequire 'fluent/compat/set_time_key_mixin'\nrequire 'fluent/compat/set_tag_key_mixin'\nrequire 'fluent/compat/type_converter'\n\nrequire 'fluent/time' # Fluent::TimeFormatter\n\nmodule Fluent\n  RecordFilterMixin = Fluent::Compat::RecordFilterMixin\n  HandleTagNameMixin = Fluent::Compat::HandleTagNameMixin\n  SetTimeKeyMixin = Fluent::Compat::SetTimeKeyMixin\n  SetTagKeyMixin = Fluent::Compat::SetTagKeyMixin\n  TypeConverter = Fluent::Compat::TypeConverter\nend\n"
  },
  {
    "path": "lib/fluent/msgpack_factory.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'msgpack'\nrequire 'fluent/time'\n\nmodule Fluent\n  module MessagePackFactory\n    @@engine_factory = nil\n\n    module Mixin\n      def msgpack_factory\n        unless @deprecated_log_done\n          deprecated_log('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory')\n        end\n        MessagePackFactory.engine_factory\n      end\n\n      def msgpack_packer(*args)\n        unless @deprecated_log_done\n          deprecated_log('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_packer')\n        end\n        MessagePackFactory.msgpack_packer(*args)\n      end\n\n      def msgpack_unpacker(*args)\n        unless @deprecated_log_done\n          deprecated_log('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_unpacker')\n        end\n        MessagePackFactory.msgpack_unpacker(*args)\n      end\n\n      def deprecated_log(str)\n        if $log\n          $log.warn(str)\n          @deprecated_log_done = true\n        end\n      end\n    end\n\n    def self.engine_factory(enable_time_support: false)\n      @@engine_factory || factory(enable_time_support: enable_time_support)\n    end\n\n    def self.msgpack_packer(*args)\n      engine_factory.packer(*args)\n    end\n\n    def self.msgpack_unpacker(*args)\n      engine_factory.unpacker(*args)\n    end\n\n    def self.factory(enable_time_support: false)\n      factory = MessagePack::Factory.new\n      factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)\n      if enable_time_support\n        factory.register_type(\n          MessagePack::Timestamp::TYPE, Time,\n          packer: MessagePack::Time::Packer,\n          unpacker: MessagePack::Time::Unpacker)\n      end\n      factory\n    end\n\n    def self.packer(*args)\n      factory.packer(*args)\n    end\n\n    def self.unpacker(*args)\n      factory.unpacker(*args)\n    end\n\n    def self.init(enable_time_support: false)\n      factory = MessagePack::Factory.new\n      factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)\n      if enable_time_support\n        factory.register_type(\n          MessagePack::Timestamp::TYPE, Time,\n          packer: MessagePack::Time::Packer,\n          unpacker: MessagePack::Time::Unpacker)\n      end\n      @@engine_factory = factory\n    end\n\n    def self.thread_local_msgpack_packer\n      Thread.current[:local_msgpack_packer] ||= MessagePackFactory.engine_factory.packer\n    end\n\n    def self.thread_local_msgpack_unpacker\n      unpacker = Thread.current[:local_msgpack_unpacker]\n      if unpacker.nil?\n        return Thread.current[:local_msgpack_unpacker] = MessagePackFactory.engine_factory.unpacker\n      end\n      unpacker.reset\n      unpacker\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/oj_options.rb",
    "content": "require 'fluent/config/types'\n\nmodule Fluent\n  class OjOptions\n    OPTIONS = {\n      'bigdecimal_load': :symbol,\n      'mode': :symbol,\n      'use_to_json': :bool\n    }\n\n    ALLOWED_VALUES = {\n      'bigdecimal_load': %i[bigdecimal float auto],\n      'mode': %i[strict null compat json rails custom]\n    }\n\n    DEFAULTS = {\n      'bigdecimal_load': :float,\n      'mode': :compat,\n      'use_to_json': true\n    }\n\n    @@available = false\n\n    def self.available?\n      @@available\n    end\n\n    def self.load_env\n      options = self.get_options\n      begin\n        require 'oj'\n        Oj.default_options = options\n        @@available = true\n      rescue LoadError\n        @@available = false\n      end\n      options\n    end\n\n    private\n\n    def self.get_options\n      options = {}\n      DEFAULTS.each { |key, value| options[key] = value }\n\n      OPTIONS.each do |key, type|\n        env_value = ENV[\"FLUENT_OJ_OPTION_#{key.upcase}\"]\n        next if env_value.nil?\n\n        cast_value = Fluent::Config.reformatted_value(OPTIONS[key], env_value, { strict: true })\n        next if cast_value.nil?\n\n        next if ALLOWED_VALUES[key] && !ALLOWED_VALUES[key].include?(cast_value)\n\n        options[key.to_sym] = cast_value\n      end\n\n      options\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/output.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/output'\nrequire 'fluent/output_chain'\n\nmodule Fluent\n  Output               = Fluent::Compat::Output\n  BufferedOutput       = Fluent::Compat::BufferedOutput\n  ObjectBufferedOutput = Fluent::Compat::ObjectBufferedOutput\n  TimeSlicedOutput     = Fluent::Compat::TimeSlicedOutput\n  MultiOutput          = Fluent::Compat::MultiOutput\n\n  # Some input plugins refer BufferQueueLimitError for throttling\n  BufferQueueLimitError = Fluent::Compat::BufferQueueLimitError\nend\n"
  },
  {
    "path": "lib/fluent/output_chain.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/output_chain'\n\nmodule Fluent\n  OutputChain = Fluent::Compat::OutputChain\n  CopyOutputChain = Fluent::Compat::CopyOutputChain\n  NullOutputChain = Fluent::Compat::NullOutputChain\nend\n"
  },
  {
    "path": "lib/fluent/parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/parser'\n\nmodule Fluent\n  ParserError = Fluent::Compat::Parser::ParserError\n  Parser = Fluent::Compat::Parser\n  TextParser = Fluent::Compat::TextParser\nend\n"
  },
  {
    "path": "lib/fluent/plugin/bare_output.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/base'\n\nrequire 'fluent/log'\nrequire 'fluent/plugin_id'\nrequire 'fluent/plugin_helper'\n\nmodule Fluent\n  module Plugin\n    class BareOutput < Base\n      include PluginHelper::Mixin # for metrics\n\n      # DO NOT USE THIS plugin for normal output plugin. Use Output instead.\n      # This output plugin base class is only for meta-output plugins\n      # which cannot be implemented on MultiOutput.\n      # E.g,: forest, config-expander\n\n      helpers_internal :metrics\n\n      include PluginId\n      include PluginLoggerMixin\n      include PluginHelper::Mixin\n\n      def process(tag, es)\n        raise NotImplementedError, \"BUG: output plugins MUST implement this method\"\n      end\n\n      def initialize\n        super\n        @counter_mutex = Mutex.new\n        # TODO: well organized counters\n        @num_errors_metrics = nil\n        @emit_count_metrics = nil\n        @emit_records_metrics = nil\n        @emit_size_metrics = nil\n      end\n\n      def configure(conf)\n        super\n\n        @num_errors_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"bare_output\", name: \"num_errors\", help_text: \"Number of count num errors\")\n        @emit_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"bare_output\", name: \"emit_count\", help_text: \"Number of count emits\")\n        @emit_records_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"bare_output\", name: \"emit_records\", help_text: \"Number of emit records\")\n        @emit_size_metrics =  metrics_create(namespace: \"fluentd\", subsystem: \"bare_output\", name: \"emit_size\", help_text: \"Total size of emit events\")\n        @enable_size_metrics = !!system_config.enable_size_metrics\n      end\n\n      def statistics\n        stats = {\n          'num_errors' => @num_errors_metrics.get,\n          'emit_records' => @emit_records_metrics.get,\n          'emit_count' => @emit_count_metrics.get,\n          'emit_size' => @emit_size_metrics.get,\n        }\n\n        { 'bare_output' => stats }\n      end\n\n      def emit_sync(tag, es)\n        @emit_count_metrics.inc\n        begin\n          process(tag, es)\n          @emit_records_metrics.add(es.size)\n          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n        rescue\n          @num_errors_metrics.inc\n          raise\n        end\n      end\n      alias :emit_events :emit_sync\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/base.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin'\nrequire 'fluent/configurable'\nrequire 'fluent/system_config'\n\nmodule Fluent\n  module Plugin\n    class Base\n      include Configurable\n      include SystemConfig::Mixin\n\n      State = Struct.new(:configure, :start, :after_start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)\n\n      attr_accessor :under_plugin_development\n\n      def initialize\n        @log = nil\n        super\n        @fluentd_lock_dir = ENV['FLUENTD_LOCK_DIR']\n        @_state = State.new(false, false, false, false, false, false, false, false, false)\n        @_context_router = nil\n        @_fluentd_worker_id = nil\n        @under_plugin_development = false\n      end\n\n      def has_router?\n        false\n      end\n\n      def plugin_root_dir\n        nil # override this in plugin_id.rb\n      end\n\n      def fluentd_worker_id\n        return @_fluentd_worker_id if @_fluentd_worker_id\n        @_fluentd_worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i\n        @_fluentd_worker_id\n      end\n\n      def configure(conf)\n        raise ArgumentError, \"BUG: type of conf must be Fluent::Config::Element, but #{conf.class} is passed.\" unless conf.is_a?(Fluent::Config::Element)\n\n        if conf.for_this_worker? || (Fluent::Engine.supervisor_mode && !conf.for_every_workers?)\n          system_config_override(workers: conf.target_worker_ids.size)\n        end\n\n        super(conf, system_config.strict_config_value)\n        @_state ||= State.new(false, false, false, false, false, false, false, false, false)\n        @_state.configure = true\n        self\n      end\n\n      def multi_workers_ready?\n        true\n      end\n\n      def get_lock_path(name)\n        name = name.gsub(/[^a-zA-Z0-9]/, \"_\")\n        File.join(@fluentd_lock_dir, \"fluentd-#{name}.lock\")\n      end\n\n      def acquire_worker_lock(name)\n        if @fluentd_lock_dir.nil?\n          raise InvalidLockDirectory, \"can't acquire lock because FLUENTD_LOCK_DIR isn't set\"\n        end\n        lock_path = get_lock_path(name)\n        File.open(lock_path, \"w\") do |f|\n          f.flock(File::LOCK_EX)\n          yield\n        end\n        # Update access time to prevent tmpwatch from deleting a lock file.\n        FileUtils.touch(lock_path)\n      end\n\n      def string_safe_encoding(str)\n        unless str.valid_encoding?\n          str = str.scrub('?')\n          log.info \"invalid byte sequence is replaced in `#{str}`\" if self.respond_to?(:log)\n        end\n        yield str\n      end\n\n      def context_router=(router)\n        @_context_router = router\n      end\n\n      def context_router\n        @_context_router\n      end\n\n      def start\n        # By initialization order, plugin logger is created before set log_event_enabled.\n        # It causes '@id' specified plugin, it uses plugin logger instead of global logger, ignores `<label @FLUENT_LOG>` setting.\n        # This is adhoc approach but impact is minimal.\n        if @log.is_a?(Fluent::PluginLogger) && $log.respond_to?(:log_event_enabled) # log_event_enabled check for tests\n          @log.log_event_enabled = $log.log_event_enabled\n        end\n        @_state.start = true\n        self\n      end\n\n      def after_start\n        @_state.after_start = true\n        self\n      end\n\n      def stop\n        @_state.stop = true\n        self\n      end\n\n      def before_shutdown\n        @_state.before_shutdown = true\n        self\n      end\n\n      def shutdown\n        @_state.shutdown = true\n        self\n      end\n\n      def after_shutdown\n        @_state.after_shutdown = true\n        self\n      end\n\n      def close\n        @_state.close = true\n        self\n      end\n\n      def terminate\n        @_state.terminate = true\n        self\n      end\n\n      def configured?\n        @_state.configure\n      end\n\n      def started?\n        @_state.start\n      end\n\n      def after_started?\n        @_state.after_start\n      end\n\n      def stopped?\n        @_state.stop\n      end\n\n      def before_shutdown?\n        @_state.before_shutdown\n      end\n\n      def shutdown?\n        @_state.shutdown\n      end\n\n      def after_shutdown?\n        @_state.after_shutdown\n      end\n\n      def closed?\n        @_state.close\n      end\n\n      def terminated?\n        @_state.terminate\n      end\n\n      def called_in_test?\n        caller_locations.each do |location|\n          # Thread::Backtrace::Location#path returns base filename or absolute path.\n          # #absolute_path returns absolute_path always.\n          # https://bugs.ruby-lang.org/issues/12159\n          if /\\/test_[^\\/]+\\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\\.rb$/\n            return true\n          end\n        end\n        false\n      end\n\n      def inspect\n        # Plugin instances are sometimes too big to dump because it may have too many thins (buffer,storage, ...)\n        # Original commit comment says that:\n        #   To emulate normal inspect behavior `ruby -e'o=Object.new;p o;p (o.__id__<<1).to_s(16)'`.\n        #   https://github.com/ruby/ruby/blob/trunk/gc.c#L788\n        \"#<%s:%014x>\" % [self.class.name, '0x%014x' % (__id__ << 1)]\n      end\n\n      def reloadable_plugin?\n        # Engine can't capture all class variables. so it's forbidden to use class variables in each plugins if enabling reload.\n        self.class.class_variables.empty?\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/buf_file.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fileutils'\n\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin/buffer/file_chunk'\nrequire 'fluent/system_config'\nrequire 'fluent/variable_store'\n\nmodule Fluent\n  module Plugin\n    class FileBuffer < Fluent::Plugin::Buffer\n      Plugin.register_buffer('file', self)\n\n      include SystemConfig::Mixin\n\n      DEFAULT_CHUNK_LIMIT_SIZE = 256 * 1024 * 1024        # 256MB\n      DEFAULT_TOTAL_LIMIT_SIZE =  64 * 1024 * 1024 * 1024 #  64GB, same with v0.12 (TimeSlicedOutput + buf_file)\n\n      desc 'The path where buffer chunks are stored.'\n      config_param :path, :string, default: nil\n      desc 'The suffix of buffer chunks'\n      config_param :path_suffix, :string, default: '.log'\n\n      config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE\n      config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE\n\n      config_param :file_permission, :string, default: nil # '0644' (Fluent::DEFAULT_FILE_PERMISSION)\n      config_param :dir_permission,  :string, default: nil # '0755' (Fluent::DEFAULT_DIR_PERMISSION)\n\n      def initialize\n        super\n        @symlink_path = nil\n        @multi_workers_available = false\n        @additional_resume_path = nil\n        @buffer_path = nil\n        @variable_store = nil\n      end\n\n      def configure(conf)\n        super\n\n        @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)\n\n        multi_workers_configured = owner.system_config.workers > 1\n\n        using_plugin_root_dir = false\n        unless @path\n          if root_dir = owner.plugin_root_dir\n            @path = File.join(root_dir, 'buffer')\n            using_plugin_root_dir = true # plugin_root_dir path contains worker id\n          else\n            raise Fluent::ConfigError, \"buffer path is not configured. specify 'path' in <buffer>\"\n          end\n        end\n\n        type_of_owner = Plugin.lookup_type_from_class(@_owner.class)\n        if @variable_store.has_key?(@path) && !called_in_test?\n          type_using_this_path = @variable_store[@path]\n          raise ConfigError, \"Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}\"\n        end\n\n        @buffer_path = @path\n        @variable_store[@buffer_path] = type_of_owner\n\n        specified_directory_exists = File.exist?(@path) && File.directory?(@path)\n        unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')\n\n        if specified_directory_exists || unexisting_path_for_directory # directory\n          if using_plugin_root_dir || !multi_workers_configured\n            @path = File.join(@path, \"buffer.*#{@path_suffix}\")\n          else\n            @path = File.join(@path, \"worker#{fluentd_worker_id}\", \"buffer.*#{@path_suffix}\")\n            if fluentd_worker_id == 0\n              # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)\n              @additional_resume_path = File.join(File.expand_path(\"../../\", @path), \"buffer.*#{@path_suffix}\")\n            end\n          end\n          @multi_workers_available = true\n        else # specified path is file path\n          if File.basename(@path).include?('.*.')\n            # valid file path\n          elsif File.basename(@path).end_with?('.*')\n            @path = @path + @path_suffix\n          else\n            # existing file will be ignored\n            @path = @path + \".*#{@path_suffix}\"\n          end\n          @multi_workers_available = false\n        end\n\n        if @dir_permission\n          @dir_permission = @dir_permission.to_i(8) if @dir_permission.is_a?(String)\n        else\n          @dir_permission = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION\n        end\n      end\n\n      # This method is called only when multi worker is configured\n      def multi_workers_ready?\n        unless @multi_workers_available\n          log.error \"file buffer with multi workers should be configured to use directory 'path', or system root_dir and plugin id\"\n        end\n        @multi_workers_available\n      end\n\n      def start\n        FileUtils.mkdir_p File.dirname(@path), mode: @dir_permission\n\n        super\n      end\n\n      def stop\n        if @variable_store\n          @variable_store.delete(@buffer_path)\n        end\n\n        super\n      end\n\n      def persistent?\n        true\n      end\n\n      def resume\n        stage = {}\n        queue = []\n        exist_broken_file = false\n\n        patterns = [@path]\n        patterns.unshift @additional_resume_path if @additional_resume_path\n        Dir.glob(escaped_patterns(patterns)) do |path|\n          next unless File.file?(path)\n\n          if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown\n            # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken\n            # since there may be a power failure or similar failure.\n            log.warn { \"restoring buffer file: path = #{path}\" }\n          else\n            log.debug { \"restoring buffer file: path = #{path}\" }\n          end\n\n          m = new_metadata() # this metadata will be overwritten by resuming .meta file content\n                             # so it should not added into @metadata_list for now\n          mode = Fluent::Plugin::Buffer::FileChunk.assume_chunk_state(path)\n          if mode == :unknown\n            log.debug \"unknown state chunk found\", path: path\n            next\n          end\n\n          begin\n            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata\n          rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e\n            exist_broken_file = true\n            handle_broken_files(path, mode, e)\n            next\n          end\n\n          case chunk.state\n          when :staged\n            # unstaged chunk created at Buffer#write_step_by_step is identified as the staged chunk here because FileChunk#assume_chunk_state checks only the file name.\n            # https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L663\n            # This case can happen when fluentd process is killed by signal or other reasons between creating unstaged chunks and changing them to staged mode in Buffer#write\n            # these chunks(unstaged chunks) has shared the same metadata\n            # So perform enqueue step again https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L364\n            if chunk_size_full?(chunk) || stage.key?(chunk.metadata)\n              chunk.metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num\n              queue << chunk.enqueued!\n            else\n              stage[chunk.metadata] = chunk\n            end\n          when :queued\n            queue << chunk\n          end\n        end\n\n        queue.sort_by!{ |chunk| chunk.modified_at }\n\n        # If one of the files is corrupted, other files may also be corrupted and be undetected.\n        # The time periods of each chunk are helpful to check the data.\n        if exist_broken_file\n          log.info \"Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files.\"\n          (stage.values + queue).each { |chunk|\n            log.info \"  #{chunk.path}:\", :created_at => chunk.created_at, :modified_at => chunk.modified_at\n          }\n        end\n\n        return stage, queue\n      end\n\n      def generate_chunk(metadata)\n        # FileChunk generates real path with unique_id\n        perm = @file_permission || system_config.file_permission\n        chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: perm, compress: @compress)\n        log.debug \"Created new chunk\", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata\n\n        return chunk\n      end\n\n      def handle_broken_files(path, mode, e)\n        log.error \"found broken chunk file during resume.\", :path => path, :mode => mode, :err_msg => e.message\n        unique_id = Fluent::Plugin::Buffer::FileChunk.unique_id_from_path(path)\n        backup(unique_id) { |f|\n          File.open(path, 'rb') { |chunk|\n            chunk.set_encoding(Encoding::ASCII_8BIT)\n            chunk.sync = true\n            chunk.binmode\n            IO.copy_stream(chunk, f)\n          }\n        }\n      rescue => error\n        log.error \"backup failed. Delete corresponding files.\", :err_msg => error.message\n      ensure\n        log.warn \"disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away.\" if @disable_chunk_backup\n        File.unlink(path, path + '.meta') rescue nil\n      end\n\n      def evacuate_chunk(chunk)\n        unless chunk.is_a?(Fluent::Plugin::Buffer::FileChunk)\n          raise ArgumentError, \"The chunk must be FileChunk, but it was #{chunk.class}.\"\n        end\n\n        backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id)\n        FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)\n\n        FileUtils.copy([chunk.path, chunk.meta_path], backup_dir)\n        log.warn \"chunk files are evacuated to #{backup_dir}.\", chunk_id: dump_unique_id_hex(chunk.unique_id)\n      rescue => e\n        log.error \"unexpected error while evacuating chunk files.\", error: e\n      end\n\n      private\n\n      def escaped_patterns(patterns)\n        patterns.map { |pattern|\n          # '{' '}' are special character in Dir.glob\n          pattern.gsub(/[\\{\\}]/) { |c| \"\\\\#{c}\" }\n        }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/buf_file_single.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fileutils'\n\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin/buffer/file_single_chunk'\nrequire 'fluent/system_config'\nrequire 'fluent/variable_store'\n\nmodule Fluent\n  module Plugin\n    class FileSingleBuffer < Fluent::Plugin::Buffer\n      Plugin.register_buffer('file_single', self)\n\n      include SystemConfig::Mixin\n\n      DEFAULT_CHUNK_LIMIT_SIZE = 256 * 1024 * 1024        # 256MB\n      DEFAULT_TOTAL_LIMIT_SIZE =  64 * 1024 * 1024 * 1024 #  64GB\n\n      PATH_SUFFIX = \".#{Fluent::Plugin::Buffer::FileSingleChunk::PATH_EXT}\"\n\n      desc 'The path where buffer chunks are stored.'\n      config_param :path, :string, default: nil\n      desc 'Calculate the number of record in chunk during resume'\n      config_param :calc_num_records, :bool, default: true\n      desc 'The format of chunk. This is used to calculate the number of record'\n      config_param :chunk_format, :enum, list: [:msgpack, :text, :auto], default: :auto\n\n      config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE\n      config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE\n\n      desc 'The permission of chunk file. If no specified, <system> setting or 0644 is used'\n      config_param :file_permission, :string, default: nil\n      desc 'The permission of chunk directory. If no specified, <system> setting or 0755 is used'\n      config_param :dir_permission, :string, default: nil\n\n      def initialize\n        super\n\n        @multi_workers_available = false\n        @additional_resume_path = nil\n        @variable_store = nil\n      end\n\n      def configure(conf)\n        super\n\n        @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file_single)\n\n        if @chunk_format == :auto\n          @chunk_format = owner.formatted_to_msgpack_binary? ? :msgpack : :text\n        end\n\n        @key_in_path = nil\n        if owner.chunk_keys.empty?\n          log.debug \"use event tag for buffer key\"\n        else\n          if owner.chunk_key_tag\n            raise Fluent::ConfigError, \"chunk keys must be tag or one field\"\n          elsif owner.chunk_keys.size > 1\n            raise Fluent::ConfigError, \"2 or more chunk keys is not allowed\"\n          else\n            @key_in_path = owner.chunk_keys.first.to_sym\n          end\n        end\n\n        multi_workers_configured = owner.system_config.workers > 1\n        using_plugin_root_dir = false\n        unless @path\n          if root_dir = owner.plugin_root_dir\n            @path = File.join(root_dir, 'buffer')\n            using_plugin_root_dir = true # plugin_root_dir path contains worker id\n          else\n            raise Fluent::ConfigError, \"buffer path is not configured. specify 'path' in <buffer>\"\n          end\n        end\n\n        specified_directory_exists = File.exist?(@path) && File.directory?(@path)\n        unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')\n\n        if specified_directory_exists || unexisting_path_for_directory # directory\n          if using_plugin_root_dir || !multi_workers_configured\n            @path = File.join(@path, \"fsb.*#{PATH_SUFFIX}\")\n          else\n            @path = File.join(@path, \"worker#{fluentd_worker_id}\", \"fsb.*#{PATH_SUFFIX}\")\n            if fluentd_worker_id == 0\n              # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)\n              @additional_resume_path = File.join(File.expand_path(\"../../\", @path), \"fsb.*#{PATH_SUFFIX}\")\n            end\n          end\n          @multi_workers_available = true\n        else # specified path is file path\n          if File.basename(@path).include?('.*.')\n            new_path = File.join(File.dirname(@path), \"fsb.*#{PATH_SUFFIX}\")\n            log.warn \"file_single doesn't allow user specified 'prefix.*.suffix' style path. Use '#{new_path}' for file instead: #{@path}\"\n            @path = new_path\n          elsif File.basename(@path).end_with?('.*')\n            @path = @path + PATH_SUFFIX\n          else\n            # existing file will be ignored\n            @path = @path + \".*#{PATH_SUFFIX}\"\n          end\n          @multi_workers_available = false\n        end\n\n        type_of_owner = Plugin.lookup_type_from_class(@_owner.class)\n        if @variable_store.has_key?(@path) && !called_in_test?\n          type_using_this_path = @variable_store[@path]\n          raise Fluent::ConfigError, \"Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}\"\n        end\n\n        @variable_store[@path] = type_of_owner\n        @dir_permission = if @dir_permission\n                            @dir_permission.to_i(8)\n                          else\n                            system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION\n                          end\n      end\n\n      # This method is called only when multi worker is configured\n      def multi_workers_ready?\n        unless @multi_workers_available\n          log.error \"file_single buffer with multi workers should be configured to use directory 'path', or system root_dir and plugin id\"\n        end\n        @multi_workers_available\n      end\n\n      def start\n        FileUtils.mkdir_p(File.dirname(@path), mode: @dir_permission)\n\n        super\n      end\n\n      def stop\n        if @variable_store\n          @variable_store.delete(@path)\n        end\n\n        super\n      end\n\n      def persistent?\n        true\n      end\n\n      def resume\n        stage = {}\n        queue = []\n        exist_broken_file = false\n\n        patterns = [@path]\n        patterns.unshift @additional_resume_path if @additional_resume_path\n        Dir.glob(escaped_patterns(patterns)) do |path|\n          next unless File.file?(path)\n\n          if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown\n            # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken\n            # since there may be a power failure or similar failure.\n            log.warn { \"restoring buffer file: path = #{path}\" }\n          else\n            log.debug { \"restoring buffer file: path = #{path}\" }\n          end\n\n          m = new_metadata() # this metadata will be updated in FileSingleChunk.new\n          mode = Fluent::Plugin::Buffer::FileSingleChunk.assume_chunk_state(path)\n          if mode == :unknown\n            log.debug \"unknown state chunk found\", path: path\n            next\n          end\n\n          begin\n            chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)\n            chunk.restore_size(@chunk_format) if @calc_num_records\n          rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e\n            exist_broken_file = true\n            handle_broken_files(path, mode, e)\n            next\n          end\n\n          case chunk.state\n          when :staged\n            stage[chunk.metadata] = chunk\n          when :queued\n            queue << chunk\n          end\n        end\n\n        queue.sort_by!(&:modified_at)\n\n        # If one of the files is corrupted, other files may also be corrupted and be undetected.\n        # The time periods of each chunk are helpful to check the data.\n        if exist_broken_file\n          log.info \"Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files.\"\n          (stage.values + queue).each { |chunk|\n            log.info \"  #{chunk.path}:\", :created_at => chunk.created_at, :modified_at => chunk.modified_at\n          }\n        end\n\n        return stage, queue\n      end\n\n      def generate_chunk(metadata)\n        # FileChunk generates real path with unique_id\n        perm = @file_permission || system_config.file_permission\n        chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, perm: perm, compress: @compress)\n\n        log.debug \"Created new chunk\", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata\n\n        chunk\n      end\n\n      def handle_broken_files(path, mode, e)\n        log.error \"found broken chunk file during resume.\", :path => path, :mode => mode, :err_msg => e.message\n        unique_id, _ = Fluent::Plugin::Buffer::FileSingleChunk.unique_id_and_key_from_path(path)\n        backup(unique_id) { |f|\n          File.open(path, 'rb') { |chunk|\n            chunk.set_encoding(Encoding::ASCII_8BIT)\n            chunk.sync = true\n            chunk.binmode\n            IO.copy_stream(chunk, f)\n          }\n        }\n      rescue => error\n        log.error \"backup failed. Delete corresponding files.\", :err_msg => error.message\n      ensure\n        log.warn \"disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away.\" if @disable_chunk_backup\n        File.unlink(path) rescue nil\n      end\n\n      def evacuate_chunk(chunk)\n        unless chunk.is_a?(Fluent::Plugin::Buffer::FileSingleChunk)\n          raise ArgumentError, \"The chunk must be FileSingleChunk, but it was #{chunk.class}.\"\n        end\n\n        backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id)\n        FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)\n\n        FileUtils.copy(chunk.path, backup_dir)\n        log.warn \"chunk files are evacuated to #{backup_dir}.\", chunk_id: dump_unique_id_hex(chunk.unique_id)\n      rescue => e\n        log.error \"unexpected error while evacuating chunk files.\", error: e\n      end\n\n      private\n\n      def escaped_patterns(patterns)\n        patterns.map { |pattern|\n          # '{' '}' are special character in Dir.glob\n          pattern.gsub(/[\\{\\}]/) { |c| \"\\\\#{c}\" }\n        }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/buf_memory.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin/buffer/memory_chunk'\n\nmodule Fluent\n  module Plugin\n    class MemoryBuffer < Fluent::Plugin::Buffer\n      Plugin.register_buffer('memory', self)\n\n      def resume\n        return {}, []\n      end\n\n      def generate_chunk(metadata)\n        Fluent::Plugin::Buffer::MemoryChunk.new(metadata, compress: @compress)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/buffer/chunk.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin/compressable'\nrequire 'fluent/unique_id'\nrequire 'fluent/event'\nrequire 'fluent/ext_monitor_require'\n\nrequire 'tempfile'\nrequire 'zlib'\n\nmodule Fluent\n  module Plugin\n    class Buffer # fluent/plugin/buffer is already loaded\n      class Chunk\n        include MonitorMixin\n        include UniqueId::Mixin\n\n        # Chunks has 2 part:\n        # * metadata: contains metadata which should be restored after resume (if possible)\n        #             v: {key=>value,key=>value,...} (optional)\n        #             t: tag as string (optional)\n        #             k: time slice key (optional)\n        #\n        #             id: unique_id of chunk (*)\n        #             s: size (number of events in chunk) (*)\n        #             c: created_at as unix time (*)\n        #             m: modified_at as unix time (*)\n        #              (*): fields automatically injected by chunk itself\n        # * data: binary data, combined records represented as String, maybe compressed\n\n        # NOTE: keys of metadata are named with a single letter\n        #       to decread bytesize of metadata I/O\n\n        # TODO: CompressedPackedMessage of forward protocol?\n\n        def initialize(metadata, compress: :text)\n          super()\n          @unique_id = generate_unique_id\n          @metadata = metadata\n\n          # state: unstaged/staged/queued/closed\n          @state = :unstaged\n\n          @size = 0\n          @created_at = Fluent::Clock.real_now\n          @modified_at = Fluent::Clock.real_now\n          if compress == :gzip\n            extend GzipDecompressable\n          elsif compress == :zstd\n            extend ZstdDecompressable\n          end\n        end\n\n        attr_reader :unique_id, :metadata, :state\n\n        def raw_create_at\n          @created_at\n        end\n\n        def raw_modified_at\n          @modified_at\n        end\n\n        # for compatibility\n        def created_at\n          @created_at_object ||= Time.at(@created_at)\n        end\n\n        # for compatibility\n        def modified_at\n          @modified_at_object ||= Time.at(@modified_at)\n        end\n\n        # data is array of formatted record string\n        def append(data, **kwargs)\n          raise ArgumentError, \"`compress: #{kwargs[:compress]}` can be used for Compressable module\" if kwargs[:compress] == :gzip || kwargs[:compress] == :zstd\n          begin\n            adding = data.join.force_encoding(Encoding::ASCII_8BIT)\n          rescue\n            # Fallback\n            # Array#join throws an exception if data contains strings with a different encoding.\n            # Although such cases may be rare, it should be considered as a safety precaution.\n            adding = ''.force_encoding(Encoding::ASCII_8BIT)\n            data.each do |d|\n              adding << d.b\n            end\n          end\n          concat(adding, data.size)\n        end\n\n        # for event streams which is packed or zipped (and we want not to unpack/uncompress)\n        def concat(bulk, records)\n          raise NotImplementedError, \"Implement this method in child class\"\n        end\n\n        def commit\n          raise NotImplementedError, \"Implement this method in child class\"\n        end\n\n        def rollback\n          raise NotImplementedError, \"Implement this method in child class\"\n        end\n\n        def bytesize\n          raise NotImplementedError, \"Implement this method in child class\"\n        end\n\n        def size\n          raise NotImplementedError, \"Implement this method in child class\"\n        end\n        alias :length :size\n\n        def empty?\n          size == 0\n        end\n\n        def writable?\n          @state == :staged || @state == :unstaged\n        end\n\n        def unstaged?\n          @state == :unstaged\n        end\n\n        def staged?\n          @state == :staged\n        end\n\n        def queued?\n          @state == :queued\n        end\n\n        def closed?\n          @state == :closed\n        end\n\n        def staged!\n          @state = :staged\n          self\n        end\n\n        def unstaged!\n          @state = :unstaged\n          self\n        end\n\n        def enqueued!\n          @state = :queued\n          self\n        end\n\n        def close\n          @state = :closed\n          self\n        end\n\n        def purge\n          @state = :closed\n          self\n        end\n\n        def read(**kwargs)\n          raise ArgumentError, \"`compressed: #{kwargs[:compressed]}` can be used for Compressable module\" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd\n          raise NotImplementedError, \"Implement this method in child class\"\n        end\n\n        def open(**kwargs, &block)\n          raise ArgumentError, \"`compressed: #{kwargs[:compressed]}` can be used for Compressable module\" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd\n          raise NotImplementedError, \"Implement this method in child class\"\n        end\n\n        def write_to(io, **kwargs)\n          raise ArgumentError, \"`compressed: #{kwargs[:compressed]}` can be used for Compressable module\" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd\n          open do |i|\n            IO.copy_stream(i, io)\n          end\n        end\n\n        module GzipDecompressable\n          include Fluent::Plugin::Compressable\n\n          def append(data, **kwargs)\n            if kwargs[:compress] == :gzip\n              io = StringIO.new\n              Zlib::GzipWriter.wrap(io) do |gz|\n                data.each do |d|\n                  gz.write d\n                end\n              end\n              concat(io.string, data.size)\n            else\n              super\n            end\n          end\n\n          def open(**kwargs, &block)\n            if kwargs[:compressed] == :gzip\n              super\n            else\n              super(**kwargs) do |chunk_io|\n                output_io = if chunk_io.is_a?(StringIO)\n                              StringIO.new\n                            else\n                              Tempfile.new('decompressed-data')\n                            end\n                output_io.binmode if output_io.is_a?(Tempfile)\n                decompress(input_io: chunk_io, output_io: output_io)\n                output_io.seek(0, IO::SEEK_SET)\n                yield output_io\n              end\n            end\n          end\n\n          def read(**kwargs)\n            if kwargs[:compressed] == :gzip\n              super\n            else\n              decompress(super)\n            end\n          end\n\n          def write_to(io, **kwargs)\n            open(compressed: :gzip) do |chunk_io|\n              if kwargs[:compressed] == :gzip\n                IO.copy_stream(chunk_io, io)\n              else\n                decompress(input_io: chunk_io, output_io: io)\n              end\n            end\n          end\n        end\n\n        module ZstdDecompressable\n          include Fluent::Plugin::Compressable\n\n          def append(data, **kwargs)\n            if kwargs[:compress] == :zstd\n              io = StringIO.new\n              stream = Zstd::StreamWriter.new(io)\n              data.each do |d|\n                stream.write(d)\n              end\n              stream.finish\n              concat(io.string, data.size)\n            else\n              super\n            end\n          end\n\n          def open(**kwargs, &block)\n            if kwargs[:compressed] == :zstd\n              super\n            else\n              super(**kwargs) do |chunk_io|\n                output_io = if chunk_io.is_a?(StringIO)\n                              StringIO.new\n                            else\n                              Tempfile.new('decompressed-data')\n                            end\n                output_io.binmode if output_io.is_a?(Tempfile)\n                decompress(input_io: chunk_io, output_io: output_io, type: :zstd)\n                output_io.seek(0, IO::SEEK_SET)\n                yield output_io\n              end\n            end\n          end\n\n          def read(**kwargs)\n            if kwargs[:compressed] == :zstd\n              super\n            else\n              decompress(super,type: :zstd)\n            end\n          end\n\n          def write_to(io, **kwargs)\n            open(compressed: :zstd) do |chunk_io|\n              if kwargs[:compressed] == :zstd\n                IO.copy_stream(chunk_io, io)\n              else\n                decompress(input_io: chunk_io, output_io: io, type: :zstd)\n              end\n            end\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/buffer/file_chunk.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/buffer/chunk'\nrequire 'fluent/unique_id'\nrequire 'fluent/msgpack_factory'\n\nmodule Fluent\n  module Plugin\n    class Buffer\n      class FileChunk < Chunk\n        class FileChunkError < StandardError; end\n\n        BUFFER_HEADER = \"\\xc1\\x00\".force_encoding(Encoding::ASCII_8BIT).freeze\n\n        ### buffer path user specified : /path/to/directory/user_specified_prefix.*.log\n        ### buffer chunk path          : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log\n        ### buffer chunk metadata path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log.meta\n\n        # NOTE: Old style buffer path of time sliced output plugins had a part of key: prefix.20150414.b513b61...suffix\n        #       But this part is not used now for any purpose. (Now metadata is used instead.)\n\n        # state: b/q - 'b'(on stage, compatible with v0.12), 'q'(enqueued)\n        # path_prefix: path prefix string, ended with '.'\n        # path_suffix: path suffix string, like '.log' (or any other user specified)\n\n        attr_reader :path, :meta_path, :permission\n\n        def initialize(metadata, path, mode, perm: nil, compress: :text)\n          super(metadata, compress: compress)\n          perm ||= Fluent::DEFAULT_FILE_PERMISSION\n          @permission = perm.is_a?(String) ? perm.to_i(8) : perm\n          @bytesize = @size = @adding_bytes = @adding_size = 0\n          @meta = nil\n\n          case mode\n          when :create then create_new_chunk(path, @permission)\n          when :staged then load_existing_staged_chunk(path)\n          when :queued then load_existing_enqueued_chunk(path)\n          else\n            raise ArgumentError, \"Invalid file chunk mode: #{mode}\"\n          end\n        end\n\n        def concat(bulk, bulk_size)\n          raise \"BUG: concatenating to unwritable chunk, now '#{self.state}'\" unless self.writable?\n\n          bulk.force_encoding(Encoding::ASCII_8BIT)\n          @chunk.write bulk\n          @adding_bytes += bulk.bytesize\n          @adding_size += bulk_size\n          true\n        end\n\n        def commit\n          write_metadata # this should be at first: of course, this operation may fail\n\n          @commit_position = @chunk.pos\n          @size += @adding_size\n          @bytesize += @adding_bytes\n          @adding_bytes = @adding_size = 0\n          @modified_at = Fluent::Clock.real_now\n          @modified_at_object = nil\n\n          true\n        end\n\n        def rollback\n          if @chunk.pos != @commit_position\n            @chunk.seek(@commit_position, IO::SEEK_SET)\n            @chunk.truncate(@commit_position)\n          end\n          @adding_bytes = @adding_size = 0\n          true\n        end\n\n        def bytesize\n          @bytesize + @adding_bytes\n        end\n\n        def size\n          @size + @adding_size\n        end\n\n        def empty?\n          @bytesize == 0\n        end\n\n        def enqueued!\n          return unless self.staged?\n\n          new_chunk_path = self.class.generate_queued_chunk_path(@path, @unique_id)\n          new_meta_path = new_chunk_path + '.meta'\n\n          write_metadata(update: false) # re-write metadata w/ finalized records\n\n          begin\n            file_rename(@chunk, @path, new_chunk_path, ->(new_io) { @chunk = new_io })\n          rescue => e\n            begin\n              file_rename(@chunk, new_chunk_path, @path, ->(new_io) { @chunk = new_io }) if File.exist?(new_chunk_path)\n            rescue => re\n              # In this point, restore buffer state is hard because previous `file_rename` failed by resource problem.\n              # Retry is one possible approach but it may cause livelock under limited resources or high load environment.\n              # So we ignore such errors for now and log better message instead.\n              # \"Too many open files\" should be fixed by proper buffer configuration and system setting.\n              raise \"can't enqueue buffer file and failed to restore. This may causes inconsistent state: path = #{@path}, error = '#{e}', retry error = '#{re}'\"\n            else\n              raise \"can't enqueue buffer file: path = #{@path}, error = '#{e}'\"\n            end\n          end\n\n          begin\n            file_rename(@meta, @meta_path, new_meta_path, ->(new_io) { @meta = new_io })\n          rescue => e\n            begin\n              file_rename(@chunk, new_chunk_path, @path, ->(new_io) { @chunk = new_io }) if File.exist?(new_chunk_path)\n              file_rename(@meta, new_meta_path, @meta_path, ->(new_io) { @meta = new_io }) if File.exist?(new_meta_path)\n            rescue => re\n              # See above\n              raise \"can't enqueue buffer metadata and failed to restore. This may causes inconsistent state: path = #{@meta_path}, error = '#{e}', retry error = '#{re}'\"\n            else\n              raise \"can't enqueue buffer metadata: path = #{@meta_path}, error = '#{e}'\"\n            end\n          end\n\n          @path = new_chunk_path\n          @meta_path = new_meta_path\n\n          super\n        end\n\n        def close\n          super\n          size = @chunk.size\n          @chunk.close\n          @meta.close if @meta # meta may be missing if chunk is queued at first\n          if size == 0\n            File.unlink(@path, @meta_path)\n          end\n        end\n\n        def purge\n          super\n          @chunk.close\n          @meta.close if @meta\n          @bytesize = @size = @adding_bytes = @adding_size = 0\n          File.unlink(@path, @meta_path)\n        end\n\n        def read(**kwargs)\n          @chunk.seek(0, IO::SEEK_SET)\n          @chunk.read\n        end\n\n        def open(**kwargs, &block)\n          @chunk.seek(0, IO::SEEK_SET)\n          val = yield @chunk\n          @chunk.seek(0, IO::SEEK_END) if self.staged?\n          val\n        end\n\n        def self.assume_chunk_state(path)\n          if /\\.(b|q)([0-9a-f]+)\\.[^\\/]*\\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern\n            $1 == 'b' ? :staged : :queued\n          else\n            # files which matches to glob of buffer file pattern\n            # it includes files which are created by out_file\n            :unknown\n          end\n        end\n\n        def self.generate_stage_chunk_path(path, unique_id)\n          pos = path.index('.*.')\n          raise \"BUG: buffer chunk path on stage MUST have '.*.'\" unless pos\n\n          prefix = path[0...pos]\n          suffix = path[(pos+3)..-1]\n\n          chunk_id = Fluent::UniqueId.hex(unique_id)\n          state = 'b'\n          \"#{prefix}.#{state}#{chunk_id}.#{suffix}\"\n        end\n\n        def self.generate_queued_chunk_path(path, unique_id)\n          chunk_id = Fluent::UniqueId.hex(unique_id)\n          if path.index(\".b#{chunk_id}.\")\n            path.sub(\".b#{chunk_id}.\", \".q#{chunk_id}.\")\n          else # for unexpected cases (ex: users rename files while opened by fluentd)\n            path + \".q#{chunk_id}.chunk\"\n          end\n        end\n\n        # used only for queued v0.12 buffer path or broken files\n        def self.unique_id_from_path(path)\n          if /\\.(b|q)([0-9a-f]+)\\.[^\\/]*\\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern\n            return $2.scan(/../).map{|x| x.to_i(16) }.pack('C*')\n          end\n          nil\n        end\n\n        def restore_metadata(bindata)\n          data = restore_metadata_with_new_format(bindata)\n\n          unless data\n            # old type of restore\n            data = Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}\n          end\n          raise FileChunkError, \"invalid meta data\" if data.nil? || !data.is_a?(Hash)\n          raise FileChunkError, \"invalid unique_id\" unless data[:id]\n          raise FileChunkError, \"invalid created_at\" unless data[:c].to_i > 0\n          raise FileChunkError, \"invalid modified_at\" unless data[:m].to_i > 0\n\n          now = Fluent::Clock.real_now\n\n          @unique_id = data[:id]\n          @size = data[:s] || 0\n          @created_at = data[:c]\n          @modified_at = data[:m]\n\n          @metadata.timekey = data[:timekey]\n          @metadata.tag = data[:tag]\n          @metadata.variables = data[:variables]\n          @metadata.seq = data[:seq] || 0\n        end\n\n        def restore_metadata_partially(chunk)\n          @unique_id = self.class.unique_id_from_path(chunk.path) || @unique_id\n          @size = 0\n          @created_at = chunk.ctime.to_i # birthtime isn't supported on Windows (and Travis?)\n          @modified_at = chunk.mtime.to_i\n\n          @metadata.timekey = nil\n          @metadata.tag = nil\n          @metadata.variables = nil\n          @metadata.seq = 0\n        end\n\n        def write_metadata(update: true)\n          data = @metadata.to_h.merge({\n              id: @unique_id,\n              s: (update ? @size + @adding_size : @size),\n              c: @created_at,\n              m: (update ? Fluent::Clock.real_now : @modified_at),\n          })\n          bin = Fluent::MessagePackFactory.thread_local_msgpack_packer.pack(data).full_pack\n          size = [bin.bytesize].pack('N')\n          @meta.seek(0, IO::SEEK_SET)\n          @meta.write(BUFFER_HEADER + size + bin)\n        end\n\n        def file_rename(file, old_path, new_path, callback=nil)\n          pos = file.pos\n          if Fluent.windows?\n            file.close\n            File.rename(old_path, new_path)\n            file = File.open(new_path, 'rb', @permission)\n          else\n            File.rename(old_path, new_path)\n            file.reopen(new_path, 'rb')\n          end\n          file.set_encoding(Encoding::ASCII_8BIT)\n          file.sync = true\n          file.binmode\n          file.pos = pos\n          callback.call(file) if callback\n        end\n\n        def create_new_chunk(path, perm)\n          @path = self.class.generate_stage_chunk_path(path, @unique_id)\n          @meta_path = @path + '.meta'\n          begin\n            @chunk = File.open(@path, 'wb+', perm)\n            @chunk.set_encoding(Encoding::ASCII_8BIT)\n            @chunk.sync = true\n            @chunk.binmode\n          rescue => e\n            # Here assumes \"Too many open files\" like recoverable error so raising BufferOverflowError.\n            # If other cases are possible, we will change error handling with proper classes.\n            raise BufferOverflowError, \"can't create buffer file for #{path}. Stop creating buffer files: error = #{e}\"\n          end\n          begin\n            @meta = File.open(@meta_path, 'wb', perm)\n            @meta.set_encoding(Encoding::ASCII_8BIT)\n            @meta.sync = true\n            @meta.binmode\n            write_metadata(update: false)\n          rescue => e\n            # This case is easier than enqueued!. Just removing pre-create buffer file\n            @chunk.close rescue nil\n            File.unlink(@path) rescue nil\n\n            if @meta\n              # ensure to unlink when #write_metadata fails\n              @meta.close rescue nil\n              File.unlink(@meta_path) rescue nil\n            end\n\n            # Same as @chunk case. See above\n            raise BufferOverflowError, \"can't create buffer metadata for #{path}. Stop creating buffer files: error = #{e}\"\n          end\n\n          @state = :unstaged\n          @bytesize = 0\n          @commit_position = @chunk.pos # must be 0\n          @adding_bytes = 0\n          @adding_size = 0\n        end\n\n        def load_existing_staged_chunk(path)\n          @path = path\n          @meta_path = @path + '.meta'\n\n          @meta = nil\n          # staging buffer chunk without metadata is classic buffer chunk file\n          # and it should be enqueued immediately\n          if File.exist?(@meta_path)\n            raise FileChunkError, \"staged file chunk is empty\" if File.size(@path).zero?\n\n            @chunk = File.open(@path, 'rb+')\n            @chunk.set_encoding(Encoding::ASCII_8BIT)\n            @chunk.sync = true\n            @chunk.seek(0, IO::SEEK_END)\n            @chunk.binmode\n\n            @meta = File.open(@meta_path, 'rb+')\n            @meta.set_encoding(Encoding::ASCII_8BIT)\n            @meta.sync = true\n            @meta.binmode\n            begin\n              restore_metadata(@meta.read)\n            rescue => e\n              @chunk.close\n              @meta.close\n              raise FileChunkError, \"staged meta file is broken. #{e.message}\"\n            end\n            @meta.seek(0, IO::SEEK_SET)\n\n            @state = :staged\n            @bytesize = @chunk.size\n            @commit_position = @chunk.pos\n            @adding_bytes = 0\n            @adding_size = 0\n          else\n            # classic buffer chunk - read only chunk\n            @chunk = File.open(@path, 'rb')\n            @chunk.set_encoding(Encoding::ASCII_8BIT)\n            @chunk.binmode\n            @chunk.seek(0, IO::SEEK_SET)\n            @state = :queued\n            @bytesize = @chunk.size\n\n            restore_metadata_partially(@chunk)\n\n            @commit_position = @chunk.size\n            @unique_id = self.class.unique_id_from_path(@path) || @unique_id\n          end\n        end\n\n        def load_existing_enqueued_chunk(path)\n          @path = path\n          raise FileChunkError, \"enqueued file chunk is empty\" if File.size(@path).zero?\n\n          @chunk = File.open(@path, 'rb')\n          @chunk.set_encoding(Encoding::ASCII_8BIT)\n          @chunk.binmode\n          @chunk.seek(0, IO::SEEK_SET)\n          @bytesize = @chunk.size\n          @commit_position = @chunk.size\n\n          @meta_path = @path + '.meta'\n          if File.readable?(@meta_path)\n            begin\n              restore_metadata(File.open(@meta_path){|f| f.set_encoding(Encoding::ASCII_8BIT); f.binmode; f.read })\n            rescue => e\n              @chunk.close\n              raise FileChunkError, \"enqueued meta file is broken. #{e.message}\"\n            end\n          else\n            restore_metadata_partially(@chunk)\n          end\n          @state = :queued\n        end\n\n        private\n\n        def restore_metadata_with_new_format(chunk)\n          if chunk.size <= 6 # size of BUFFER_HEADER (2) + size of data size(4)\n            return nil\n          end\n\n          if chunk.slice(0, 2) == BUFFER_HEADER\n            size = chunk.slice(2, 4).unpack1('N')\n            if size\n              return Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil\n            end\n          end\n\n          nil\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/buffer/file_single_chunk.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'uri'\nrequire 'fluent/plugin/buffer/chunk'\nrequire 'fluent/unique_id'\nrequire 'fluent/msgpack_factory'\n\nmodule Fluent\n  module Plugin\n    class Buffer\n      class FileSingleChunk < Chunk\n        class FileChunkError < StandardError; end\n\n        ## buffer path user specified : /path/to/directory\n        ## buffer chunk path          : /path/to/directory/fsb.key.b513b61c9791029c2513b61c9791029c2.buf\n        ## state: b/q - 'b'(on stage), 'q'(enqueued)\n\n        PATH_EXT = 'buf'\n        PATH_SUFFIX = \".#{PATH_EXT}\"\n        PATH_REGEXP = /\\.(b|q)([0-9a-f]+)\\.#{PATH_EXT}*\\Z/n  # //n switch means explicit 'ASCII-8BIT' pattern\n\n        attr_reader :path, :permission\n\n        def initialize(metadata, path, mode, key, perm: Fluent::DEFAULT_FILE_PERMISSION, compress: :text)\n          super(metadata, compress: compress)\n          @key = key\n          perm ||= Fluent::DEFAULT_FILE_PERMISSION\n          @permission = perm.is_a?(String) ? perm.to_i(8) : perm\n          @bytesize = @size = @adding_bytes = @adding_size = 0\n\n          case mode\n          when :create then create_new_chunk(path, metadata, @permission)\n          when :staged then load_existing_staged_chunk(path)\n          when :queued then load_existing_enqueued_chunk(path)\n          else\n            raise ArgumentError, \"Invalid file chunk mode: #{mode}\"\n          end\n        end\n\n        def concat(bulk, bulk_size)\n          raise \"BUG: concatenating to unwritable chunk, now '#{self.state}'\" unless self.writable?\n\n          bulk.force_encoding(Encoding::ASCII_8BIT)\n          @chunk.write(bulk)\n          @adding_bytes += bulk.bytesize\n          @adding_size += bulk_size\n          true\n        end\n\n        def commit\n          @commit_position = @chunk.pos\n          @size += @adding_size\n          @bytesize += @adding_bytes\n          @adding_bytes = @adding_size = 0\n          @modified_at = Fluent::Clock.real_now\n\n          true\n        end\n\n        def rollback\n          if @chunk.pos != @commit_position\n            @chunk.seek(@commit_position, IO::SEEK_SET)\n            @chunk.truncate(@commit_position)\n          end\n          @adding_bytes = @adding_size = 0\n          true\n        end\n\n        def bytesize\n          @bytesize + @adding_bytes\n        end\n\n        def size\n          @size + @adding_size\n        end\n\n        def empty?\n          @bytesize.zero?\n        end\n\n        def enqueued!\n          return unless self.staged?\n\n          new_chunk_path = self.class.generate_queued_chunk_path(@path, @unique_id)\n\n          begin\n            file_rename(@chunk, @path, new_chunk_path, ->(new_io) { @chunk = new_io })\n          rescue => e\n            begin\n              file_rename(@chunk, new_chunk_path, @path, ->(new_io) { @chunk = new_io }) if File.exist?(new_chunk_path)\n            rescue => re\n              # In this point, restore buffer state is hard because previous `file_rename` failed by resource problem.\n              # Retry is one possible approach but it may cause livelock under limited resources or high load environment.\n              # So we ignore such errors for now and log better message instead.\n              # \"Too many open files\" should be fixed by proper buffer configuration and system setting.\n              raise \"can't enqueue buffer file and failed to restore. This may causes inconsistent state: path = #{@path}, error = '#{e}', retry error = '#{re}'\"\n            else\n              raise \"can't enqueue buffer file: path = #{@path}, error = '#{e}'\"\n            end\n          end\n\n          @path = new_chunk_path\n\n          super\n        end\n\n        def close\n          super\n          size = @chunk.size\n          @chunk.close\n          if size == 0\n            File.unlink(@path)\n          end\n        end\n\n        def purge\n          super\n          @chunk.close\n          @bytesize = @size = @adding_bytes = @adding_size = 0\n          File.unlink(@path)\n        end\n\n        def read(**kwargs)\n          @chunk.seek(0, IO::SEEK_SET)\n          @chunk.read\n        end\n\n        def open(**kwargs, &block)\n          @chunk.seek(0, IO::SEEK_SET)\n          val = yield @chunk\n          @chunk.seek(0, IO::SEEK_END) if self.staged?\n          val\n        end\n\n        def self.assume_chunk_state(path)\n          return :unknown unless path.end_with?(PATH_SUFFIX)\n\n          if PATH_REGEXP =~ path\n            $1 == 'b' ? :staged : :queued\n          else\n            # files which matches to glob of buffer file pattern\n            # it includes files which are created by out_file\n            :unknown\n          end\n        end\n\n        def self.unique_id_and_key_from_path(path)\n          base = File.basename(path)\n          res = PATH_REGEXP =~ base\n          return nil unless res\n\n          key = base[4..res - 1] # remove 'fsb.' and '.'\n          hex_id = $2            # remove '.' and '.buf'\n          unique_id = hex_id.scan(/../).map {|x| x.to_i(16) }.pack('C*')\n          [unique_id, key]\n        end\n\n        def self.generate_stage_chunk_path(path, key, unique_id)\n          pos = path.index('.*.')\n          raise \"BUG: buffer chunk path on stage MUST have '.*.'\" unless pos\n\n          prefix = path[0...pos]\n          suffix = path[(pos + 3)..-1]\n\n          chunk_id = Fluent::UniqueId.hex(unique_id)\n          \"#{prefix}.#{key}.b#{chunk_id}.#{suffix}\"\n        end\n\n        def self.generate_queued_chunk_path(path, unique_id)\n          chunk_id = Fluent::UniqueId.hex(unique_id)\n          staged_path = \".b#{chunk_id}.\"\n          if path.index(staged_path)\n            path.sub(staged_path, \".q#{chunk_id}.\")\n          else # for unexpected cases (ex: users rename files while opened by fluentd)\n            path + \".q#{chunk_id}.chunk\"\n          end\n        end\n\n        def restore_metadata\n          if res = self.class.unique_id_and_key_from_path(@path)\n            @unique_id = res.first\n            key = decode_key(res.last)\n            if @key\n              @metadata.variables = {@key => key}\n            else\n              @metadata.tag = key\n            end\n          else\n            raise FileChunkError, \"Invalid chunk found. unique_id and key not exist: #{@path}\"\n          end\n          @size = 0\n\n          stat = File.stat(@path)\n          @created_at = stat.ctime.to_i\n          @modified_at = stat.mtime.to_i\n        end\n\n        def restore_size(chunk_format)\n          count = 0\n          File.open(@path, 'rb') { |f|\n            if chunk_format == :msgpack\n              Fluent::MessagePackFactory.msgpack_unpacker(f).each { |d| count += 1 }\n            else\n              f.each_line { |l| count += 1 }\n            end\n          }\n          @size = count\n        end\n\n        def file_rename(file, old_path, new_path, callback = nil)\n          pos = file.pos\n          if Fluent.windows?\n            file.close\n            File.rename(old_path, new_path)\n            file = File.open(new_path, 'rb', @permission)\n          else\n            File.rename(old_path, new_path)\n            file.reopen(new_path, 'rb')\n          end\n          file.set_encoding(Encoding::ASCII_8BIT)\n          file.sync = true\n          file.binmode\n          file.pos = pos\n          callback.call(file) if callback\n        end\n\n        ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n\n\n        def encode_key(metadata)\n          k = @key ? metadata.variables[@key] : metadata.tag\n          k ||= ''\n          URI::RFC2396_PARSER.escape(k, ESCAPE_REGEXP)\n        end\n\n        def decode_key(key)\n          URI::RFC2396_PARSER.unescape(key)\n        end\n\n        def create_new_chunk(path, metadata, perm)\n          @path = self.class.generate_stage_chunk_path(path, encode_key(metadata), @unique_id)\n          begin\n            @chunk = File.open(@path, 'wb+', perm)\n            @chunk.set_encoding(Encoding::ASCII_8BIT)\n            @chunk.sync = true\n            @chunk.binmode\n          rescue => e\n            # Here assumes \"Too many open files\" like recoverable error so raising BufferOverflowError.\n            # If other cases are possible, we will change error handling with proper classes.\n            raise BufferOverflowError, \"can't create buffer file for #{path}. Stop creating buffer files: error = #{e}\"\n          end\n\n          @state = :unstaged\n          @bytesize = 0\n          @commit_position = @chunk.pos # must be 0\n          @adding_bytes = 0\n          @adding_size = 0\n        end\n\n        def load_existing_staged_chunk(path)\n          @path = path\n          raise FileChunkError, \"staged file chunk is empty\" if File.size(@path).zero?\n\n          @chunk = File.open(@path, 'rb+')\n          @chunk.set_encoding(Encoding::ASCII_8BIT)\n          @chunk.sync = true\n          @chunk.binmode\n          @chunk.seek(0, IO::SEEK_END)\n\n          restore_metadata\n\n          @state = :staged\n          @bytesize = @chunk.size\n          @commit_position = @chunk.pos\n          @adding_bytes = 0\n          @adding_size = 0\n        end\n\n        def load_existing_enqueued_chunk(path)\n          @path = path\n          raise FileChunkError, \"enqueued file chunk is empty\" if File.size(@path).zero?\n\n          @chunk = File.open(@path, 'rb')\n          @chunk.set_encoding(Encoding::ASCII_8BIT)\n          @chunk.binmode\n          @chunk.seek(0, IO::SEEK_SET)\n\n          restore_metadata\n\n          @state = :queued\n          @bytesize = @chunk.size\n          @commit_position = @chunk.size\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/buffer/memory_chunk.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/buffer/chunk'\n\nmodule Fluent\n  module Plugin\n    class Buffer\n      class MemoryChunk < Chunk\n        def initialize(metadata, compress: :text)\n          super\n          @chunk = ''.force_encoding(Encoding::ASCII_8BIT)\n          @chunk_bytes = 0\n          @adding_bytes = 0\n          @adding_size = 0\n        end\n\n        def concat(bulk, bulk_size)\n          raise \"BUG: concatenating to unwritable chunk, now '#{self.state}'\" unless self.writable?\n\n          bulk.force_encoding(Encoding::ASCII_8BIT)\n          @chunk << bulk\n          @adding_bytes += bulk.bytesize\n          @adding_size += bulk_size\n          true\n        end\n\n        def commit\n          @size += @adding_size\n          @chunk_bytes += @adding_bytes\n\n          @adding_bytes = @adding_size = 0\n          @modified_at = Fluent::Clock.real_now\n          @modified_at_object = nil\n          true\n        end\n\n        def rollback\n          @chunk.slice!(@chunk_bytes, @adding_bytes)\n          @adding_bytes = @adding_size = 0\n          true\n        end\n\n        def bytesize\n          @chunk_bytes + @adding_bytes\n        end\n\n        def size\n          @size + @adding_size\n        end\n\n        def empty?\n          @chunk.empty?\n        end\n\n        def purge\n          super\n          @chunk.clear\n          @chunk_bytes = @size = @adding_bytes = @adding_size = 0\n          true\n        end\n\n        def read(**kwargs)\n          @chunk.dup\n        end\n\n        def open(**kwargs, &block)\n          StringIO.open(@chunk, &block)\n        end\n\n        def write_to(io, **kwargs)\n          # re-implementation to optimize not to create StringIO\n          io.write @chunk\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/buffer.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/base'\nrequire 'fluent/plugin/owned_by_mixin'\nrequire 'fluent/plugin_id'\nrequire 'fluent/plugin_helper'\nrequire 'fluent/unique_id'\nrequire 'fluent/ext_monitor_require'\n\nmodule Fluent\n  module Plugin\n    class Buffer < Base\n      include OwnedByMixin\n      include UniqueId::Mixin\n      include PluginId\n      include MonitorMixin\n      include PluginHelper::Mixin # for metrics\n\n      class BufferError < StandardError; end\n      class BufferOverflowError < BufferError; end\n      class BufferChunkOverflowError < BufferError; end # A record size is larger than chunk size limit\n\n      MINIMUM_APPEND_ATTEMPT_RECORDS = 10\n\n      DEFAULT_CHUNK_LIMIT_SIZE =   8 * 1024 * 1024 # 8MB\n      DEFAULT_TOTAL_LIMIT_SIZE = 512 * 1024 * 1024 # 512MB, same with v0.12 (BufferedOutput + buf_memory: 64 x 8MB)\n\n      DEFAULT_CHUNK_FULL_THRESHOLD = 0.95\n\n      configured_in :buffer\n\n      helpers_internal :metrics\n\n      # TODO: system total buffer limit size in bytes by SystemConfig\n\n      config_param :chunk_limit_size, :size, default: DEFAULT_CHUNK_LIMIT_SIZE\n      config_param :total_limit_size, :size, default: DEFAULT_TOTAL_LIMIT_SIZE\n\n      # If user specify this value and (chunk_size * queue_length) is smaller than total_size,\n      # then total_size is automatically configured to that value\n      config_param :queue_limit_length, :integer, default: nil\n\n      # optional new limitations\n      config_param :chunk_limit_records, :integer, default: nil\n\n      # if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued\n      config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD\n\n      desc 'The max number of queued chunks.'\n      config_param :queued_chunks_limit_size, :integer, default: nil\n\n      desc 'Compress buffered data.'\n      config_param :compress, :enum, list: [:text, :gzip, :zstd], default: :text\n\n      desc 'If true, chunks are thrown away when unrecoverable error happens'\n      config_param :disable_chunk_backup, :bool, default: false\n\n      Metadata = Struct.new(:timekey, :tag, :variables, :seq) do\n        def initialize(timekey, tag, variables)\n          super(timekey, tag, variables, 0)\n        end\n\n        def dup_next\n          m = dup\n          m.seq = seq + 1\n          m\n        end\n\n        def empty?\n          timekey.nil? && tag.nil? && variables.nil?\n        end\n\n        def cmp_variables(v1, v2)\n          if v1.nil? && v2.nil?\n            return 0\n          elsif v1.nil? # v2 is non-nil\n            return -1\n          elsif v2.nil? # v1 is non-nil\n            return 1\n          end\n          # both of v1 and v2 are non-nil\n          v1_sorted_keys = v1.keys.sort\n          v2_sorted_keys = v2.keys.sort\n          if v1_sorted_keys != v2_sorted_keys\n            if v1_sorted_keys.size == v2_sorted_keys.size\n              v1_sorted_keys <=> v2_sorted_keys\n            else\n              v1_sorted_keys.size <=> v2_sorted_keys.size\n            end\n          else\n            v1_sorted_keys.each do |k|\n              a = v1[k]\n              b = v2[k]\n              if a && b && a != b\n                return a <=> b\n              elsif a && b || (!a && !b) # same value (including both are nil)\n                next\n              elsif a # b is nil\n                return 1\n              else # a is nil (but b is non-nil)\n                return -1\n              end\n            end\n\n            0\n          end\n        end\n\n        def <=>(o)\n          timekey2 = o.timekey\n          tag2 = o.tag\n          variables2 = o.variables\n          if (!!timekey ^ !!timekey2) || (!!tag ^ !!tag2) || (!!variables ^ !!variables2)\n            # One has value in a field, but another doesn't have value in same field\n            # This case occurs very rarely\n            if timekey == timekey2 # including the case of nil == nil\n              if tag == tag2\n                cmp_variables(variables, variables2)\n              elsif tag.nil?\n                -1\n              elsif tag2.nil?\n                1\n              else\n                tag <=> tag2\n              end\n            elsif timekey.nil?\n              -1\n            elsif timekey2.nil?\n              1\n            else\n              timekey <=> timekey2\n            end\n          else\n            # objects have values in same field pairs (comparison with non-nil and nil doesn't occur here)\n            (timekey <=> timekey2 || 0).nonzero? || # if `a <=> b` is nil, then both are nil\n              (tag <=> tag2 || 0).nonzero? ||\n              cmp_variables(variables, variables2)\n          end\n        end\n\n        # This is an optimization code. Current Struct's implementation is comparing all data.\n        # https://github.com/ruby/ruby/blob/0623e2b7cc621b1733a760b72af246b06c30cf96/struct.c#L1200-L1203\n        # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).\n        # This means that this class is one of the most called object in Fluentd.\n        # See https://github.com/fluent/fluentd/pull/2560\n        def hash\n          timekey.hash\n        end\n      end\n\n      # for metrics\n      attr_reader :stage_size_metrics, :stage_length_metrics, :queue_size_metrics, :queue_length_metrics\n      attr_reader :available_buffer_space_ratios_metrics, :total_queued_size_metrics\n      attr_reader :newest_timekey_metrics, :oldest_timekey_metrics\n      # for tests\n      attr_reader :stage, :queue, :dequeued, :queued_num\n\n      def initialize\n        super\n\n        @chunk_limit_size = nil\n        @total_limit_size = nil\n        @queue_limit_length = nil\n        @chunk_limit_records = nil\n\n        @stage = {}    #=> Hash (metadata -> chunk) : not flushed yet\n        @queue = []    #=> Array (chunks)           : already flushed (not written)\n        @dequeued = {} #=> Hash (unique_id -> chunk): already written (not purged)\n        @queued_num = {} # metadata => int (number of queued chunks)\n        @dequeued_num = {} # metadata => int (number of dequeued chunks)\n\n        @stage_length_metrics = nil\n        @stage_size_metrics = nil\n        @queue_length_metrics = nil\n        @queue_size_metrics = nil\n        @available_buffer_space_ratios_metrics = nil\n        @total_queued_size_metrics = nil\n        @newest_timekey_metrics = nil\n        @oldest_timekey_metrics = nil\n        @timekeys = Hash.new(0)\n        @enable_update_timekeys = false\n        @mutex = Mutex.new\n      end\n\n      # The metrics_create method defines getter methods named stage_byte_size and queue_byte_size.\n      # For compatibility, stage_size, stage_size=, queue_size, and queue_size= are still available.\n      def stage_size\n        @stage_size_metrics.get\n      end\n\n      def stage_size=(value)\n        @stage_size_metrics.set(value)\n      end\n\n      def queue_size\n        @queue_size_metrics.get\n      end\n\n      def queue_size=(value)\n        @queue_size_metrics.set(value)\n      end\n\n      def persistent?\n        false\n      end\n\n      def configure(conf)\n        super\n\n        unless @queue_limit_length.nil?\n          @total_limit_size = @chunk_limit_size * @queue_limit_length\n        end\n        @stage_length_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"buffer\", name: \"stage_length\",\n                                               help_text: 'Length of stage buffers', prefer_gauge: true)\n        @stage_length_metrics.set(0)\n        @stage_size_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"buffer\", name: \"stage_byte_size\",\n                                             help_text: 'Total size of stage buffers', prefer_gauge: true)\n        @stage_size_metrics.set(0) # Ensure zero.\n        @queue_length_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"buffer\", name: \"queue_length\",\n                                               help_text: 'Length of queue buffers', prefer_gauge: true)\n        @queue_length_metrics.set(0)\n        @queue_size_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"buffer\", name: \"queue_byte_size\",\n                                             help_text: 'Total size of queue buffers', prefer_gauge: true)\n        @queue_size_metrics.set(0) # Ensure zero.\n        @available_buffer_space_ratios_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"buffer\", name: \"available_buffer_space_ratios\",\n                                                                help_text: 'Ratio of available space in buffer', prefer_gauge: true)\n        @available_buffer_space_ratios_metrics.set(100) # Default is 100%.\n        @total_queued_size_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"buffer\", name: \"total_queued_size\",\n                                                    help_text: 'Total size of stage and queue buffers', prefer_gauge: true)\n        @total_queued_size_metrics.set(0)\n        @newest_timekey_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"buffer\", name: \"newest_timekey\",\n                                                 help_text: 'Newest timekey in buffer', prefer_gauge: true)\n        @oldest_timekey_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"buffer\", name: \"oldest_timekey\",\n                                                 help_text: 'Oldest timekey in buffer', prefer_gauge: true)\n      end\n\n      def enable_update_timekeys\n        @enable_update_timekeys = true\n      end\n\n      def start\n        super\n\n        @stage, @queue = resume\n        @stage.each_pair do |metadata, chunk|\n          @stage_size_metrics.add(chunk.bytesize)\n        end\n        @queue.each do |chunk|\n          @queued_num[chunk.metadata] ||= 0\n          @queued_num[chunk.metadata] += 1\n          @queue_size_metrics.add(chunk.bytesize)\n        end\n        update_timekeys\n        log.debug \"buffer started\", instance: self.object_id, stage_size: @stage_size_metrics.get, queue_size: @queue_size_metrics.get\n      end\n\n      def close\n        super\n        synchronize do\n          log.debug \"closing buffer\", instance: self.object_id\n          @dequeued.each_pair do |chunk_id, chunk|\n            chunk.close\n          end\n          until @queue.empty?\n            @queue.shift.close\n          end\n          @stage.each_pair do |metadata, chunk|\n            chunk.close\n          end\n        end\n      end\n\n      def terminate\n        super\n        @dequeued = @stage = @queue = @queued_num = nil\n        @stage_length_metrics = @stage_size_metrics = @queue_length_metrics = @queue_size_metrics = nil\n        @available_buffer_space_ratios_metrics = @total_queued_size_metrics = nil\n        @newest_timekey_metrics = @oldest_timekey_metrics = nil\n        @timekeys.clear\n      end\n\n      def storable?\n        @total_limit_size > @stage_size_metrics.get + @queue_size_metrics.get\n      end\n\n      ## TODO: for back pressure feature\n      # def used?(ratio)\n      #   @total_limit_size * ratio > @stage_size_metrics.get + @queue_size_metrics.get\n      # end\n\n      def resume\n        # return {}, []\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def generate_chunk(metadata)\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def new_metadata(timekey: nil, tag: nil, variables: nil)\n        Metadata.new(timekey, tag, variables)\n      end\n\n      # Keep this method for existing code\n      def metadata(timekey: nil, tag: nil, variables: nil)\n        Metadata.new(timekey, tag, variables)\n      end\n\n      def timekeys\n        @timekeys.keys\n      end\n\n      # metadata MUST have consistent object_id for each variation\n      # data MUST be Array of serialized events, or EventStream\n      # metadata_and_data MUST be a hash of { metadata => data }\n      def write(metadata_and_data, format: nil, size: nil, enqueue: false)\n        return if metadata_and_data.size < 1\n        raise BufferOverflowError, \"buffer space has too many data\" unless storable?\n\n        log.on_trace { log.trace \"writing events into buffer\", instance: self.object_id, metadata_size: metadata_and_data.size }\n\n        operated_chunks = []\n        unstaged_chunks = {} # metadata => [chunk, chunk, ...]\n        chunks_to_enqueue = []\n        staged_bytesizes_by_chunk = {}\n        # track internal BufferChunkOverflowError in write_step_by_step\n        buffer_chunk_overflow_errors = []\n\n        begin\n          # sort metadata to get lock of chunks in same order with other threads\n          metadata_and_data.keys.sort.each do |metadata|\n            data = metadata_and_data[metadata]\n            write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize, error|\n              chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads\n              operated_chunks << chunk\n              if chunk.staged?\n                #\n                # https://github.com/fluent/fluentd/issues/2712\n                # write_once is supposed to write to a chunk only once\n                # but this block **may** run multiple times from write_step_by_step and previous write may be rollbacked\n                # So we should be counting the stage_size only for the last successful write\n                #\n                staged_bytesizes_by_chunk[chunk] = adding_bytesize\n              elsif chunk.unstaged?\n                unstaged_chunks[metadata] ||= []\n                unstaged_chunks[metadata] << chunk\n              end\n              if error && !error.empty?\n                buffer_chunk_overflow_errors << error\n              end\n            end\n          end\n\n          return if operated_chunks.empty?\n\n          # Now, this thread acquires many locks of chunks... getting buffer-global lock causes dead lock.\n          # Any operations needs buffer-global lock (including enqueueing) should be done after releasing locks.\n\n          first_chunk = operated_chunks.shift\n          # Following commits for other chunks also can finish successfully if the first commit operation\n          # finishes without any exceptions.\n          # In most cases, #commit just requires very small disk spaces, so major failure reason are\n          # permission errors, disk failures and other permanent(fatal) errors.\n          begin\n            first_chunk.commit\n            if enqueue || first_chunk.unstaged? || chunk_size_full?(first_chunk)\n              chunks_to_enqueue << first_chunk\n            end\n            first_chunk.mon_exit\n          rescue\n            operated_chunks.unshift(first_chunk)\n            raise\n          end\n\n          errors = []\n          # Buffer plugin estimates there's no serious error cause: will commit for all chunks either way\n          operated_chunks.each do |chunk|\n            begin\n              chunk.commit\n              if enqueue || chunk.unstaged? || chunk_size_full?(chunk)\n                chunks_to_enqueue << chunk\n              end\n              chunk.mon_exit\n            rescue => e\n              chunk.rollback\n              chunk.mon_exit\n              errors << e\n            end\n          end\n\n          # All locks about chunks are released.\n\n          #\n          # Now update the stage, stage_size with proper locking\n          # FIX FOR stage_size miscomputation - https://github.com/fluent/fluentd/issues/2712\n          #\n          staged_bytesizes_by_chunk.each do |chunk, bytesize|\n            chunk.synchronize do\n              synchronize { @stage_size_metrics.add(bytesize) }\n              log.on_trace { log.trace { \"chunk #{chunk.path} size_added: #{bytesize} new_size: #{chunk.bytesize}\" } }\n            end\n          end\n\n          chunks_to_enqueue.each do |c|\n            if c.staged? && (enqueue || chunk_size_full?(c))\n              m = c.metadata\n              enqueue_chunk(m)\n              if unstaged_chunks[m] && !unstaged_chunks[m].empty?\n                u = unstaged_chunks[m].pop\n                u.synchronize do\n                  if u.unstaged? && !chunk_size_full?(u)\n                    # `u.metadata.seq` and `m.seq` can be different but Buffer#enqueue_chunk expect them to be the same value\n                    u.metadata.seq = 0\n                    synchronize {\n                      @stage[m] = u.staged!\n                      @stage_size_metrics.add(u.bytesize)\n                    }\n                  end\n                end\n              end\n            elsif c.unstaged?\n              enqueue_unstaged_chunk(c)\n            else\n              # previously staged chunk is already enqueued, closed or purged.\n              # no problem.\n            end\n          end\n\n          operated_chunks.clear if errors.empty?\n\n          if errors.size > 0\n            log.warn \"error occurs in committing chunks: only first one raised\", errors: errors.map(&:class)\n            raise errors.first\n          end\n        ensure\n          operated_chunks.each do |chunk|\n            chunk.rollback rescue nil # nothing possible to do for #rollback failure\n            if chunk.unstaged?\n              chunk.purge rescue nil # to prevent leakage of unstaged chunks\n            end\n            chunk.mon_exit rescue nil # this may raise ThreadError for chunks already committed\n          end\n          unless buffer_chunk_overflow_errors.empty?\n            # Notify delayed BufferChunkOverflowError here\n            raise BufferChunkOverflowError, buffer_chunk_overflow_errors.join(\", \")\n          end\n        end\n      end\n\n      def queue_full?\n        synchronize { @queue.size } >= @queued_chunks_limit_size\n      end\n\n      def queued_records\n        synchronize { @queue.reduce(0){|r, chunk| r + chunk.size } }\n      end\n\n      def queued?(metadata = nil, optimistic: false)\n        if optimistic\n          optimistic_queued?(metadata)\n        else\n          synchronize do\n            optimistic_queued?(metadata)\n          end\n        end\n      end\n\n      def enqueue_chunk(metadata)\n        log.on_trace { log.trace \"enqueueing chunk\", instance: self.object_id, metadata: metadata }\n\n        chunk = synchronize do\n          @stage.delete(metadata)\n        end\n        return nil unless chunk\n\n        chunk.synchronize do\n          synchronize do\n            if chunk.empty?\n              chunk.close\n            else\n              chunk.metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num\n              @queue << chunk\n              @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1\n              chunk.enqueued!\n            end\n            bytesize = chunk.bytesize\n            @stage_size_metrics.sub(bytesize)\n            @queue_size_metrics.add(bytesize)\n          end\n        end\n        nil\n      end\n\n      def enqueue_unstaged_chunk(chunk)\n        log.on_trace { log.trace \"enqueueing unstaged chunk\", instance: self.object_id, metadata: chunk.metadata }\n\n        synchronize do\n          chunk.synchronize do\n            metadata = chunk.metadata\n            metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num\n            @queue << chunk\n            @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1\n            chunk.enqueued!\n          end\n          @queue_size_metrics.add(chunk.bytesize)\n        end\n      end\n\n      def update_timekeys\n        synchronize do\n          chunks = @stage.values\n          chunks.concat(@queue)\n          @timekeys = chunks.each_with_object({}) do |chunk, keys|\n            if chunk.metadata&.timekey\n              t = chunk.metadata.timekey\n              keys[t] = keys.fetch(t, 0) + 1\n            end\n          end\n        end\n      end\n\n      # At flush_at_shutdown, all staged chunks should be enqueued for buffer flush. Set true to force_enqueue for it.\n      def enqueue_all(force_enqueue = false)\n        log.on_trace { log.trace \"enqueueing all chunks in buffer\", instance: self.object_id }\n        update_timekeys if @enable_update_timekeys\n\n        if block_given?\n          synchronize{ @stage.keys }.each do |metadata|\n            return if !force_enqueue && queue_full?\n            # NOTE: The following line might cause data race depending on Ruby implementations except CRuby\n            # cf. https://github.com/fluent/fluentd/pull/1721#discussion_r146170251\n            chunk = @stage[metadata]\n            next unless chunk\n            v = yield metadata, chunk\n            enqueue_chunk(metadata) if v\n          end\n        else\n          synchronize{ @stage.keys }.each do |metadata|\n            return if !force_enqueue && queue_full?\n            enqueue_chunk(metadata)\n          end\n        end\n      end\n\n      def dequeue_chunk\n        return nil if @queue.empty?\n        log.on_trace { log.trace \"dequeueing a chunk\", instance: self.object_id }\n\n        synchronize do\n          chunk = @queue.shift\n\n          # this buffer is dequeued by other thread just before \"synchronize\" in this thread\n          return nil unless chunk\n\n          @dequeued[chunk.unique_id] = chunk\n          @queued_num[chunk.metadata] -= 1 # BUG if nil, 0 or subzero\n          @dequeued_num[chunk.metadata] ||= 0\n          @dequeued_num[chunk.metadata] += 1\n          log.trace \"chunk dequeued\", instance: self.object_id, metadata: chunk.metadata\n          chunk\n        end\n      end\n\n      def takeback_chunk(chunk_id)\n        log.on_trace { log.trace \"taking back a chunk\", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id) }\n\n        synchronize do\n          chunk = @dequeued.delete(chunk_id)\n          return false unless chunk # already purged by other thread\n          @queue.unshift(chunk)\n          log.on_trace { log.trace \"chunk taken back\", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata }\n          @queued_num[chunk.metadata] += 1 # BUG if nil\n          @dequeued_num[chunk.metadata] -= 1\n        end\n        true\n      end\n\n      def purge_chunk(chunk_id)\n        metadata = nil\n        synchronize do\n          chunk = @dequeued.delete(chunk_id)\n          return nil unless chunk # purged by other threads\n\n          metadata = chunk.metadata\n          log.on_trace { log.trace \"purging a chunk\", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata }\n\n          begin\n            bytesize = chunk.bytesize\n            chunk.purge\n            @queue_size_metrics.sub(bytesize)\n          rescue => e\n            log.error \"failed to purge buffer chunk\", chunk_id: dump_unique_id_hex(chunk_id), error_class: e.class, error: e\n            log.error_backtrace\n          end\n\n          @dequeued_num[chunk.metadata] -= 1\n          if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1) && @dequeued_num[metadata].zero?\n            @queued_num.delete(metadata)\n            @dequeued_num.delete(metadata)\n          end\n          log.on_trace { log.trace \"chunk purged\", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata }\n        end\n\n        nil\n      end\n\n      def clear_queue!\n        log.on_trace { log.trace \"clearing queue\", instance: self.object_id }\n\n        synchronize do\n          until @queue.empty?\n            begin\n              q = @queue.shift\n              evacuate_chunk(q)\n              log.trace(\"purging a chunk in queue\"){ {id: dump_unique_id_hex(chunk.unique_id), bytesize: chunk.bytesize, size: chunk.size} }\n              q.purge\n            rescue => e\n              log.error \"unexpected error while clearing buffer queue\", error_class: e.class, error: e\n              log.error_backtrace\n            end\n          end\n          @queue_size_metrics.set(0)\n        end\n      end\n\n      def evacuate_chunk(chunk)\n        # Overwrite this on demand.\n        #\n        # Note: Difference from the `backup` feature.\n        #       The `backup` feature is for unrecoverable errors, mainly for bad chunks.\n        #       On the other hand, this feature is for normal chunks.\n        #       The main motivation for this feature is to enable recovery by evacuating buffer files\n        #       when the retry limit is reached due to external factors such as network issues.\n        #\n        # Note: Difference from the `secondary` feature.\n        #       The `secondary` feature is not suitable for recovery.\n        #       It can be difficult to recover files made by `out_secondary_file` because the metadata\n        #       is lost.\n        #       For file buffers, the easiest way for recovery is to evacuate the chunk files as is.\n        #       Once the issue is recovered, we can put back the chunk files, and restart Fluentd to\n        #       load them.\n        #       This feature enables it.\n      end\n\n      def chunk_size_over?(chunk)\n        chunk.bytesize > @chunk_limit_size || (@chunk_limit_records && chunk.size > @chunk_limit_records)\n      end\n\n      def chunk_size_full?(chunk)\n        chunk.bytesize >= @chunk_limit_size * @chunk_full_threshold || (@chunk_limit_records && chunk.size >= @chunk_limit_records * @chunk_full_threshold)\n      end\n\n      class ShouldRetry < StandardError; end\n\n      # write once into a chunk\n      # 1. append whole data into existing chunk\n      # 2. commit it & return unless chunk_size_over?\n      # 3. enqueue existing chunk & retry whole method if chunk was not empty\n      # 4. go to step_by_step writing\n\n      def write_once(metadata, data, format: nil, size: nil, &block)\n        return if data.empty?\n\n        stored = false\n        adding_bytesize = nil\n\n        chunk = synchronize { @stage[metadata] ||= generate_chunk(metadata).staged! }\n        enqueue_chunk_before_retry = false\n        chunk.synchronize do\n          # retry this method if chunk is already queued (between getting chunk and entering critical section)\n          raise ShouldRetry unless chunk.staged?\n\n          empty_chunk = chunk.empty?\n\n          original_bytesize = chunk.bytesize\n          begin\n            if format\n              serialized = format.call(data)\n              chunk.concat(serialized, size ? size.call : data.size)\n            else\n              chunk.append(data, compress: @compress)\n            end\n            adding_bytesize = chunk.bytesize - original_bytesize\n\n            if chunk_size_over?(chunk)\n              if format && empty_chunk\n                if chunk.bytesize > @chunk_limit_size\n                  log.warn \"chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes\"\n                else\n                  log.warn \"chunk size limit exceeds for an emitted event stream: #{chunk.size}records\"\n                end\n              end\n              chunk.rollback\n\n              if format && !empty_chunk\n                # Event streams should be appended into a chunk at once\n                # as far as possible, to improve performance of formatting.\n                # Event stream may be a MessagePackEventStream. We don't want to split it into\n                # 2 or more chunks (except for a case that the event stream is larger than chunk limit).\n                enqueue_chunk_before_retry = true\n                raise ShouldRetry\n              end\n            else\n              stored = true\n            end\n          rescue\n            chunk.rollback\n            raise\n          end\n\n          if stored\n            block.call(chunk, adding_bytesize)\n          end\n        end\n\n        unless stored\n          # try step-by-step appending if data can't be stored into existing a chunk in non-bulk mode\n          #\n          # 1/10 size of original event stream (splits_count == 10) seems enough small\n          # to try emitting events into existing chunk.\n          # it does not matter to split event stream into very small splits, because chunks have less\n          # overhead to write data many times (even about file buffer chunks).\n          write_step_by_step(metadata, data, format, 10, &block)\n        end\n      rescue ShouldRetry\n        enqueue_chunk(metadata) if enqueue_chunk_before_retry\n        retry\n      end\n\n      # EventStream can be split into many streams\n      # because (es1 + es2).to_msgpack_stream == es1.to_msgpack_stream + es2.to_msgpack_stream\n\n      # 1. split event streams into many (10 -> 100 -> 1000 -> ...) chunks\n      # 2. append splits into the staged chunks as much as possible\n      # 3. create unstaged chunk and append rest splits -> repeat it for all splits\n\n      def write_step_by_step(metadata, data, format, splits_count, &block)\n        splits = []\n        if splits_count > data.size\n          splits_count = data.size\n        end\n        slice_size = if data.size % splits_count == 0\n                       data.size / splits_count\n                     else\n                       data.size / (splits_count - 1)\n                     end\n        slice_origin = 0\n        while slice_origin < data.size\n          splits << data.slice(slice_origin, slice_size)\n          slice_origin += slice_size\n        end\n\n        # This method will append events into the staged chunk at first.\n        # Then, will generate chunks not staged (not queued) to append rest data.\n        staged_chunk_used = false\n        modified_chunks = []\n        modified_metadata = metadata\n        get_next_chunk = ->(){\n          if staged_chunk_used\n            # Staging new chunk here is bad idea:\n            # Recovering whole state including newly staged chunks is much harder than current implementation.\n            modified_metadata = modified_metadata.dup_next\n            generate_chunk(modified_metadata)\n          else\n            synchronize { @stage[modified_metadata] ||= generate_chunk(modified_metadata).staged! }\n          end\n        }\n\n        writing_splits_index = 0\n        enqueue_chunk_before_retry = false\n\n        while writing_splits_index < splits.size\n          chunk = get_next_chunk.call\n          errors = []\n          # The chunk must be locked until being passed to &block.\n          chunk.mon_enter\n          modified_chunks << {chunk: chunk, adding_bytesize: 0, errors: errors}\n\n          raise ShouldRetry unless chunk.writable?\n          staged_chunk_used = true if chunk.staged?\n\n          original_bytesize = committed_bytesize = chunk.bytesize\n          begin\n            while writing_splits_index < splits.size\n              split = splits[writing_splits_index]\n              formatted_split = format ? format.call(split) : nil\n\n              if split.size == 1 # Check BufferChunkOverflowError\n                determined_bytesize = nil\n                if @compress != :text\n                  determined_bytesize = nil\n                elsif formatted_split\n                  determined_bytesize = formatted_split.bytesize\n                elsif split.first.respond_to?(:bytesize)\n                  determined_bytesize = split.first.bytesize\n                end\n\n                if determined_bytesize && determined_bytesize > @chunk_limit_size\n                  # It is a obvious case that BufferChunkOverflowError should be raised here.\n                  # But if it raises here, already processed 'split' or\n                  # the proceeding 'split' will be lost completely.\n                  # So it is a last resort to delay raising such a exception\n                  errors << \"a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})\"\n                  writing_splits_index += 1\n                  next\n                end\n\n                if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size\n                  # The split will (might) cause size over so keep already processed\n                  # 'split' content here (allow performance regression a bit).\n                  chunk.commit\n                  committed_bytesize = chunk.bytesize\n                end\n              end\n\n              if format\n                chunk.concat(formatted_split, split.size)\n              else\n                chunk.append(split, compress: @compress)\n              end\n              adding_bytes = chunk.bytesize - committed_bytesize\n\n              if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?\n                chunk.rollback\n                committed_bytesize = chunk.bytesize\n\n                if split.size == 1 # Check BufferChunkOverflowError again\n                  if adding_bytes > @chunk_limit_size\n                    errors << \"concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})\"\n                    writing_splits_index += 1\n                    next\n                  else\n                    # As already processed content is kept after rollback, then unstaged chunk should be queued.\n                    # After that, re-process current split again.\n                    # New chunk should be allocated, to do it, modify @stage and so on.\n                    synchronize { @stage.delete(modified_metadata) }\n                    staged_chunk_used = false\n                    chunk.unstaged!\n                    break\n                  end\n                end\n\n                if chunk_size_full?(chunk) || split.size == 1\n                  enqueue_chunk_before_retry = true\n                else\n                  splits_count *= 10\n                end\n\n                raise ShouldRetry\n              end\n\n              writing_splits_index += 1\n\n              if chunk_size_full?(chunk)\n                break\n              end\n            end\n          rescue\n            chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it\n            raise\n          end\n\n          modified_chunks.last[:adding_bytesize] = chunk.bytesize - original_bytesize\n        end\n        modified_chunks.each do |data|\n          block.call(data[:chunk], data[:adding_bytesize], data[:errors])\n        end\n      rescue ShouldRetry\n        modified_chunks.each do |data|\n          chunk = data[:chunk]\n          chunk.rollback rescue nil\n          if chunk.unstaged?\n            chunk.purge rescue nil\n          end\n          chunk.mon_exit rescue nil\n        end\n        enqueue_chunk(metadata) if enqueue_chunk_before_retry\n        retry\n      ensure\n        modified_chunks.each do |data|\n          chunk = data[:chunk]\n          chunk.mon_exit\n        end\n      end\n\n      STATS_KEYS = [\n        'stage_length',\n        'stage_byte_size',\n        'queue_length',\n        'queue_byte_size',\n        'available_buffer_space_ratios',\n        'total_queued_size',\n        'oldest_timekey',\n        'newest_timekey'\n      ]\n\n      def statistics\n        stage_size, queue_size = @stage_size_metrics.get, @queue_size_metrics.get\n        buffer_space = 1.0 - ((stage_size + queue_size * 1.0) / @total_limit_size)\n        @stage_length_metrics.set(@stage.size)\n        @queue_length_metrics.set(@queue.size)\n        @available_buffer_space_ratios_metrics.set(buffer_space * 100)\n        @total_queued_size_metrics.set(stage_size + queue_size)\n        stats = {\n          'stage_length' => @stage_length_metrics.get,\n          'stage_byte_size' => stage_size,\n          'queue_length' => @queue_length_metrics.get,\n          'queue_byte_size' => queue_size,\n          'available_buffer_space_ratios' => @available_buffer_space_ratios_metrics.get.round(1),\n          'total_queued_size' => @total_queued_size_metrics.get,\n        }\n\n        tkeys = timekeys\n        if (m = tkeys.min)\n          @oldest_timekey_metrics.set(m)\n          stats['oldest_timekey'] = @oldest_timekey_metrics.get\n        end\n        if (m = tkeys.max)\n          @newest_timekey_metrics.set(m)\n          stats['newest_timekey'] = @newest_timekey_metrics.get\n        end\n\n        { 'buffer' => stats }\n      end\n\n      def backup(chunk_unique_id)\n        unique_id = dump_unique_id_hex(chunk_unique_id)\n\n        if @disable_chunk_backup\n          log.warn \"disable_chunk_backup is true. #{unique_id} chunk is not backed up.\"\n          return\n        end\n\n        backup_file = File.join(backup_base_dir, 'backup', \"worker#{fluentd_worker_id}\", safe_owner_id, \"#{unique_id}.log\")\n        backup_dir = File.dirname(backup_file)\n\n        log.warn \"bad chunk is moved to #{backup_file}\"\n        FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)\n        File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f| yield f }\n      end\n\n      private\n\n      def optimistic_queued?(metadata = nil)\n        if metadata\n          n = @queued_num[metadata]\n          n&.nonzero?\n        else\n          !@queue.empty?\n        end\n      end\n\n      def safe_owner_id\n        owner.plugin_id.gsub(/[ \"\\/\\\\:;|*<>?]/, '_')\n      end\n\n      def backup_base_dir\n        system_config.root_dir || DEFAULT_BACKUP_DIR\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/compressable.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'stringio'\nrequire 'zlib'\nrequire 'zstd-ruby'\n\nmodule Fluent\n  module Plugin\n    module Compressable\n      def compress(data, type: :gzip, **kwargs)\n        output_io = kwargs[:output_io]\n        io = output_io || StringIO.new\n        if type == :gzip\n          writer = Zlib::GzipWriter.new(io)\n        elsif type == :zstd\n          writer = Zstd::StreamWriter.new(io)\n        else\n          raise ArgumentError, \"Unknown compression type: #{type}\"\n        end\n        writer.write(data)\n        writer.finish\n        output_io || io.string\n      end\n\n      # compressed_data is String like `compress(data1) + compress(data2) + ... + compress(dataN)`\n      # https://www.ruby-forum.com/topic/971591#979503\n      def decompress(compressed_data = nil, output_io: nil, input_io: nil, type: :gzip)\n        case\n        when input_io && output_io\n          io_decompress(input_io, output_io, type)\n        when input_io\n          output_io = StringIO.new\n          io = io_decompress(input_io, output_io, type)\n          io.string\n        when compressed_data.nil? || compressed_data.empty?\n          # check compressed_data(String) is 0 length\n          compressed_data\n        when output_io\n          # execute after checking compressed_data is empty or not\n          io = StringIO.new(compressed_data)\n          io_decompress(io, output_io, type)\n        else\n          string_decompress(compressed_data, type)\n        end\n      end\n\n      private\n\n      def string_decompress_gzip(compressed_data)\n        io = StringIO.new(compressed_data)\n        out = ''\n        loop do\n          reader = Zlib::GzipReader.new(io)\n          out << reader.read\n          unused = reader.unused\n          reader.finish\n          unless unused.nil?\n            adjust = unused.length\n            io.pos -= adjust\n          end\n          break if io.eof?\n        end\n        out\n      end\n\n      def string_decompress_zstd(compressed_data)\n        io = StringIO.new(compressed_data)\n        reader = Zstd::StreamReader.new(io)\n        out = ''\n        loop do\n          # Zstd::StreamReader needs to specify the size of the buffer\n          out << reader.read(1024)\n          # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position\n          break if io.eof?\n        end\n        out\n      end\n\n      def string_decompress(compressed_data, type = :gzip)\n        if type == :gzip\n          string_decompress_gzip(compressed_data)\n        elsif type == :zstd\n          string_decompress_zstd(compressed_data)\n        else\n          raise ArgumentError, \"Unknown compression type: #{type}\"\n        end\n      end\n\n      def io_decompress_gzip(input, output)\n        loop do\n          reader = Zlib::GzipReader.new(input)\n          v = reader.read\n          output.write(v)\n          unused = reader.unused\n          reader.finish\n          unless unused.nil?\n            adjust = unused.length\n            input.pos -= adjust\n          end\n          break if input.eof?\n        end\n        output\n      end\n\n      def io_decompress_zstd(input, output)\n        reader = Zstd::StreamReader.new(input)\n        loop do\n          # Zstd::StreamReader needs to specify the size of the buffer\n          v = reader.read(1024)\n          output.write(v)\n          # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position\n          break if input.eof?\n        end\n        output\n      end\n\n      def io_decompress(input, output, type = :gzip)\n        if type == :gzip\n          io_decompress_gzip(input, output)\n        elsif type == :zstd\n          io_decompress_zstd(input, output)\n        else\n          raise ArgumentError, \"Unknown compression type: #{type}\"\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/exec_util.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/exec_util'\n\nmodule Fluent\n  # obsolete\n  ExecUtil = Fluent::Compat::ExecUtil\nend\n"
  },
  {
    "path": "lib/fluent/plugin/file_util.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/file_util'\n\nmodule Fluent\n  # obsolete\n  FileUtil = Fluent::Compat::FileUtil\nend\n"
  },
  {
    "path": "lib/fluent/plugin/filter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/base'\n\nrequire 'fluent/event'\nrequire 'fluent/log'\nrequire 'fluent/plugin_id'\nrequire 'fluent/plugin_helper'\n\nmodule Fluent\n  module Plugin\n    class Filter < Base\n      include PluginId\n      include PluginLoggerMixin\n      include PluginHelper::Mixin\n\n      helpers_internal :event_emitter, :metrics\n\n      attr_reader :has_filter_with_time\n\n      def initialize\n        super\n        @has_filter_with_time = has_filter_with_time?\n        @emit_records_metrics = nil\n        @emit_size_metrics = nil\n        @counter_mutex = Mutex.new\n        @enable_size_metrics = false\n      end\n\n      def configure(conf)\n        super\n\n        @emit_records_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"filter\", name: \"emit_records\", help_text: \"Number of count emit records\")\n        @emit_size_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"filter\", name: \"emit_size\", help_text: \"Total size of emit events\")\n        @enable_size_metrics = !!system_config.enable_size_metrics\n      end\n\n      def statistics\n        stats = {\n          'emit_records' => @emit_records_metrics.get,\n          'emit_size' => @emit_size_metrics.get,\n        }\n\n        { 'filter' => stats }\n      end\n\n      def measure_metrics(es)\n        @emit_records_metrics.add(es.size)\n        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n      end\n\n      def filter(tag, time, record)\n        raise NotImplementedError, \"BUG: filter plugins MUST implement this method\"\n      end\n\n      def filter_with_time(tag, time, record)\n        raise NotImplementedError, \"BUG: filter plugins MUST implement this method\"\n      end\n\n      def filter_stream(tag, es)\n        new_es = MultiEventStream.new\n        if @has_filter_with_time\n          es.each do |time, record|\n            begin\n              filtered_time, filtered_record = filter_with_time(tag, time, record)\n              new_es.add(filtered_time, filtered_record) if filtered_time && filtered_record\n            rescue => e\n              router.emit_error_event(tag, time, record, e)\n            end\n          end\n        else\n          es.each do |time, record|\n            begin\n              filtered_record = filter(tag, time, record)\n              new_es.add(time, filtered_record) if filtered_record\n            rescue => e\n              router.emit_error_event(tag, time, record, e)\n            end\n          end\n        end\n        new_es\n      end\n\n      private\n\n      def has_filter_with_time?\n        implmented_methods = self.class.instance_methods(false)\n        # Plugins that override `filter_stream` don't need check,\n        # because they may not call `filter` or `filter_with_time`\n        # for example fluentd/lib/fluent/plugin/filter_record_transformer.rb\n        return nil if implmented_methods.include?(:filter_stream)\n        case\n        when [:filter, :filter_with_time].all? { |e| implmented_methods.include?(e) }\n          raise \"BUG: Filter plugins MUST implement either `filter` or `filter_with_time`\"\n        when implmented_methods.include?(:filter)\n          false\n        when implmented_methods.include?(:filter_with_time)\n          true\n        else\n          raise NotImplementedError, \"BUG: Filter plugins MUST implement either `filter` or `filter_with_time`\"\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/filter_grep.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/filter'\nrequire 'fluent/config/error'\nrequire 'fluent/plugin/string_util'\n\nmodule Fluent::Plugin\n  class GrepFilter < Filter\n    Fluent::Plugin.register_filter('grep', self)\n\n    def initialize\n      super\n\n      @_regexp_and_conditions = nil\n      @_exclude_and_conditions = nil\n      @_regexp_or_conditions = nil\n      @_exclude_or_conditions = nil\n    end\n\n    # for test\n    attr_reader :_regexp_and_conditions, :_exclude_and_conditions, :_regexp_or_conditions, :_exclude_or_conditions\n\n    helpers :record_accessor\n\n    REGEXP_MAX_NUM = 20\n\n    (1..REGEXP_MAX_NUM).each {|i| config_param :\"regexp#{i}\",  :string, default: nil, deprecated: \"Use <regexp> section\" }\n    (1..REGEXP_MAX_NUM).each {|i| config_param :\"exclude#{i}\", :string, default: nil, deprecated: \"Use <exclude> section\" }\n\n    config_section :regexp, param_name: :regexps, multi: true do\n      desc \"The field name to which the regular expression is applied.\"\n      config_param :key, :string\n      desc \"The regular expression.\"\n      config_param :pattern, :regexp\n    end\n\n    config_section :exclude, param_name: :excludes, multi: true do\n      desc \"The field name to which the regular expression is applied.\"\n      config_param :key, :string\n      desc \"The regular expression.\"\n      config_param :pattern, :regexp\n    end\n\n    config_section :and, param_name: :and_conditions, multi: true do\n      config_section :regexp, param_name: :regexps, multi: true do\n        desc \"The field name to which the regular expression is applied.\"\n        config_param :key, :string\n        desc \"The regular expression.\"\n        config_param :pattern, :regexp\n      end\n      config_section :exclude, param_name: :excludes, multi: true do\n        desc \"The field name to which the regular expression is applied.\"\n        config_param :key, :string\n        desc \"The regular expression.\"\n        config_param :pattern, :regexp\n      end\n    end\n\n    config_section :or, param_name: :or_conditions, multi: true do\n      config_section :regexp, param_name: :regexps, multi: true do\n        desc \"The field name to which the regular expression is applied.\"\n        config_param :key, :string\n        desc \"The regular expression.\"\n        config_param :pattern, :regexp\n      end\n      config_section :exclude, param_name: :excludes, multi: true do\n        desc \"The field name to which the regular expression is applied.\"\n        config_param :key, :string\n        desc \"The regular expression.\"\n        config_param :pattern, :regexp\n      end\n    end\n\n    def configure(conf)\n      super\n\n      regexp_and_conditions = {}\n      regexp_or_conditions = {}\n      exclude_and_conditions = {}\n      exclude_or_conditions = {}\n\n      (1..REGEXP_MAX_NUM).each do |i|\n        next unless conf[\"regexp#{i}\"]\n        key, regexp = conf[\"regexp#{i}\"].split(/ /, 2)\n        raise Fluent::ConfigError, \"regexp#{i} does not contain 2 parameters\" unless regexp\n        raise Fluent::ConfigError, \"regexp#{i} contains a duplicated key, #{key}\" if regexp_and_conditions[key]\n        regexp_and_conditions[key] = Expression.new(record_accessor_create(key), Regexp.compile(regexp))\n      end\n\n      (1..REGEXP_MAX_NUM).each do |i|\n        next unless conf[\"exclude#{i}\"]\n        key, exclude = conf[\"exclude#{i}\"].split(/ /, 2)\n        raise Fluent::ConfigError, \"exclude#{i} does not contain 2 parameters\" unless exclude\n        raise Fluent::ConfigError, \"exclude#{i} contains a duplicated key, #{key}\" if exclude_or_conditions[key]\n        exclude_or_conditions[key] = Expression.new(record_accessor_create(key), Regexp.compile(exclude))\n      end\n\n      if @regexps.size > 1\n        log.info \"Top level multiple <regexp> is interpreted as 'and' condition\"\n      end\n      @regexps.each do |e|\n        raise Fluent::ConfigError, \"Duplicate key: #{e.key}\" if regexp_and_conditions.key?(e.key)\n        regexp_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)\n      end\n\n      if @excludes.size > 1\n        log.info \"Top level multiple <exclude> is interpreted as 'or' condition\"\n      end\n      @excludes.each do |e|\n        raise Fluent::ConfigError, \"Duplicate key: #{e.key}\" if exclude_or_conditions.key?(e.key)\n        exclude_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)\n      end\n\n      @and_conditions.each do |and_condition|\n        if !and_condition.regexps.empty? && !and_condition.excludes.empty?\n          raise Fluent::ConfigError, \"Do not specify both <regexp> and <exclude> in <and>\"\n        end\n        and_condition.regexps.each do |e|\n          raise Fluent::ConfigError, \"Duplicate key in <and>: #{e.key}\" if regexp_and_conditions.key?(e.key)\n          regexp_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)\n        end\n        and_condition.excludes.each do |e|\n          raise Fluent::ConfigError, \"Duplicate key in <and>: #{e.key}\" if exclude_and_conditions.key?(e.key)\n          exclude_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)\n        end\n      end\n\n      @or_conditions.each do |or_condition|\n        if !or_condition.regexps.empty? && !or_condition.excludes.empty?\n          raise Fluent::ConfigError, \"Do not specify both <regexp> and <exclude> in <or>\"\n        end\n        or_condition.regexps.each do |e|\n          raise Fluent::ConfigError, \"Duplicate key in <or>: #{e.key}\" if regexp_or_conditions.key?(e.key)\n          regexp_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)\n        end\n        or_condition.excludes.each do |e|\n          raise Fluent::ConfigError, \"Duplicate key in <or>: #{e.key}\" if exclude_or_conditions.key?(e.key)\n          exclude_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)\n        end\n      end\n\n      @_regexp_and_conditions = regexp_and_conditions.values unless regexp_and_conditions.empty?\n      @_exclude_and_conditions = exclude_and_conditions.values unless exclude_and_conditions.empty?\n      @_regexp_or_conditions = regexp_or_conditions.values unless regexp_or_conditions.empty?\n      @_exclude_or_conditions = exclude_or_conditions.values unless exclude_or_conditions.empty?\n    end\n\n    def filter(tag, time, record)\n      begin\n        if @_regexp_and_conditions && @_regexp_and_conditions.any? { |expression| !expression.match?(record) }\n          return nil\n        end\n        if @_regexp_or_conditions && @_regexp_or_conditions.none? { |expression| expression.match?(record) }\n          return nil\n        end\n        if @_exclude_and_conditions && @_exclude_and_conditions.all? { |expression| expression.match?(record) }\n          return nil\n        end\n        if @_exclude_or_conditions && @_exclude_or_conditions.any? { |expression| expression.match?(record) }\n          return nil\n        end\n      rescue => e\n        log.warn \"failed to grep events\", error: e\n        log.warn_backtrace\n      end\n      record\n    end\n\n    Expression = Struct.new(:key, :pattern) do\n      def match?(record)\n        ::Fluent::StringUtil.match_regexp(pattern, key.call(record).to_s)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/filter_parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/time'\nrequire 'fluent/config/error'\nrequire 'fluent/plugin/filter'\nrequire 'fluent/plugin_helper/parser'\nrequire 'fluent/plugin_helper/compat_parameters'\n\nmodule Fluent::Plugin\n  class ParserFilter < Filter\n    Fluent::Plugin.register_filter('parser', self)\n\n    helpers :parser, :record_accessor, :compat_parameters\n\n    desc 'Specify field name in the record to parse.'\n    config_param :key_name, :string\n    desc 'Keep original key-value pair in parsed result.'\n    config_param :reserve_data, :bool, default: false\n    desc 'Keep original event time in parsed result.'\n    config_param :reserve_time, :bool, default: false\n    desc 'Remove \"key_name\" field from the record when parsing is succeeded'\n    config_param :remove_key_name_field, :bool, default: false\n    desc 'Store parsed values with specified key name prefix.'\n    config_param :inject_key_prefix, :string, default: nil\n    desc 'If true, invalid string is replaced with safe characters and re-parse it.'\n    config_param :replace_invalid_sequence, :bool, default: false\n    desc 'Store parsed values as a hash value in a field.'\n    config_param :hash_value_field, :string, default: nil\n    desc 'Emit invalid record to @ERROR label'\n    config_param :emit_invalid_record_to_error, :bool, default: true\n\n    attr_reader :parser\n\n    def configure(conf)\n      compat_parameters_convert(conf, :parser)\n\n      super\n\n      @accessor = record_accessor_create(@key_name)\n      @parser = parser_create\n    end\n\n    REPLACE_CHAR = '?'.freeze\n\n    def filter_stream(tag, es)\n      new_es = Fluent::MultiEventStream.new\n      es.each do |time, record|\n        begin\n          raw_value = @accessor.call(record)\n          if raw_value.nil?\n            new_es.add(time, handle_parsed(tag, record, time, {})) if @reserve_data\n            raise ArgumentError, \"#{@key_name} does not exist\"\n          else\n            filter_one_record(tag, time, record, raw_value) do |result_time, result_record|\n              new_es.add(result_time, result_record)\n            end\n          end\n        rescue => e\n          router.emit_error_event(tag, time, record, e) if @emit_invalid_record_to_error\n        end\n      end\n      new_es\n    end\n\n    private\n\n    def filter_one_record(tag, time, record, raw_value)\n      begin\n        @parser.parse(raw_value) do |t, values|\n          if values\n            t = if @reserve_time\n                  time\n                else\n                  t.nil? ? time : t\n                end\n            @accessor.delete(record) if @remove_key_name_field\n          else\n            router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new(\"pattern not matched with data '#{raw_value}'\")) if @emit_invalid_record_to_error\n            next unless @reserve_data\n            t = time\n            values = {}\n          end\n          yield(t, handle_parsed(tag, record, t, values))\n        end\n\n      rescue Fluent::Plugin::Parser::ParserError => e\n        raise e\n      rescue ArgumentError => e\n        raise unless @replace_invalid_sequence\n        raise unless e.message.index(\"invalid byte sequence in\") == 0\n\n        raw_value = raw_value.scrub(REPLACE_CHAR)\n        retry\n      rescue => e\n        raise Fluent::Plugin::Parser::ParserError, \"parse failed #{e.message}\"\n      end\n    end\n\n    def handle_parsed(tag, record, t, values)\n      if values && @inject_key_prefix\n        values = Hash[values.map { |k, v| [@inject_key_prefix + k, v] }]\n      end\n      r = @hash_value_field ? {@hash_value_field => values} : values\n      if @reserve_data\n        r = r ? record.merge(r) : record\n      end\n      r\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/filter_record_transformer.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'socket'\nrequire 'json'\nrequire 'ostruct'\n\nrequire 'fluent/plugin/filter'\nrequire 'fluent/config/error'\nrequire 'fluent/event'\nrequire 'fluent/time'\n\nmodule Fluent::Plugin\n  class RecordTransformerFilter < Fluent::Plugin::Filter\n    Fluent::Plugin.register_filter('record_transformer', self)\n\n    helpers :record_accessor\n\n    desc 'A comma-delimited list of keys to delete.'\n    config_param :remove_keys, :array, default: nil\n    desc 'A comma-delimited list of keys to keep.'\n    config_param :keep_keys, :array, default: nil\n    desc 'Create new Hash to transform incoming data'\n    config_param :renew_record, :bool, default: false\n    desc 'Specify field name of the record to overwrite the time of events. Its value must be unix time.'\n    config_param :renew_time_key, :string, default: nil\n    desc 'When set to true, the full Ruby syntax is enabled in the ${...} expression.'\n    config_param :enable_ruby, :bool, default: false\n    desc 'Use original value type.'\n    config_param :auto_typecast, :bool, default: true\n\n    def configure(conf)\n      super\n\n      map = {}\n      # <record></record> directive\n      conf.elements.select { |element| element.name == 'record' }.each do |element|\n        element.each_pair do |k, v|\n          element.has_key?(k) # to suppress unread configuration warning\n          map[k] = parse_value(v)\n        end\n      end\n\n      if @keep_keys\n        raise Fluent::ConfigError, \"`renew_record` must be true to use `keep_keys`\" unless @renew_record\n      end\n\n      @key_deleters = if @remove_keys\n                        @remove_keys.map { |k| record_accessor_create(k) }\n                      end\n\n      placeholder_expander_params = {\n        log: log,\n        auto_typecast: @auto_typecast,\n      }\n      @placeholder_expander =\n        if @enable_ruby\n          # require utilities which would be used in ruby placeholders\n          require 'pathname'\n          require 'uri'\n          require 'cgi/escape'\n          RubyPlaceholderExpander.new(placeholder_expander_params)\n        else\n          PlaceholderExpander.new(placeholder_expander_params)\n        end\n      @map = @placeholder_expander.preprocess_map(map)\n\n      @hostname = Socket.gethostname\n    end\n\n    def filter_stream(tag, es)\n      new_es = Fluent::MultiEventStream.new\n      tag_parts = tag.split('.')\n      tag_prefix = tag_prefix(tag_parts)\n      tag_suffix = tag_suffix(tag_parts)\n      placeholder_values = {\n        'tag'        => tag,\n        'tag_parts'  => tag_parts,\n        'tag_prefix' => tag_prefix,\n        'tag_suffix' => tag_suffix,\n        'hostname'   => @hostname,\n      }\n      es.each do |time, record|\n        begin\n          placeholder_values['time'] = @placeholder_expander.time_value(time)\n          placeholder_values['record'] = record\n\n          new_record = reform(record, placeholder_values)\n          if @renew_time_key && new_record.has_key?(@renew_time_key)\n            time = Fluent::EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))\n          end\n          @key_deleters.each { |deleter| deleter.delete(new_record) } if @key_deleters\n          new_es.add(time, new_record)\n        rescue => e\n          router.emit_error_event(tag, time, record, e)\n          log.debug { \"map:#{@map} record:#{record} placeholder_values:#{placeholder_values}\" }\n        end\n      end\n      new_es\n    end\n\n    private\n\n    def parse_value(value_str)\n      if value_str.start_with?('{', '[')\n        JSON.parse(value_str)\n      else\n        value_str\n      end\n    rescue => e\n      log.warn \"failed to parse #{value_str} as json. Assuming #{value_str} is a string\", error: e\n      value_str # emit as string\n    end\n\n    def reform(record, placeholder_values)\n      placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)\n\n      new_record = @renew_record ? {} : record.dup\n      @keep_keys.each do |k|\n        new_record[k] = record[k] if record.has_key?(k)\n      end if @keep_keys && @renew_record\n      new_record.merge!(expand_placeholders(@map, placeholders))\n\n      new_record\n    end\n\n    def expand_placeholders(value, placeholders)\n      if value.is_a?(String)\n        new_value = @placeholder_expander.expand(value, placeholders)\n      elsif value.is_a?(Hash)\n        new_value = {}\n        value.each_pair do |k, v|\n          new_key = @placeholder_expander.expand(k, placeholders, true)\n          new_value[new_key] = expand_placeholders(v, placeholders)\n        end\n      elsif value.is_a?(Array)\n        new_value = []\n        value.each_with_index do |v, i|\n          new_value[i] = expand_placeholders(v, placeholders)\n        end\n      else\n        new_value = value\n      end\n      new_value\n    end\n\n    def tag_prefix(tag_parts)\n      return [] if tag_parts.empty?\n      tag_prefix = [tag_parts.first]\n      1.upto(tag_parts.size-1).each do |i|\n        tag_prefix[i] = \"#{tag_prefix[i-1]}.#{tag_parts[i]}\"\n      end\n      tag_prefix\n    end\n\n    def tag_suffix(tag_parts)\n      return [] if tag_parts.empty?\n      rev_tag_parts = tag_parts.reverse\n      rev_tag_suffix = [rev_tag_parts.first]\n      1.upto(tag_parts.size-1).each do |i|\n        rev_tag_suffix[i] = \"#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}\"\n      end\n      rev_tag_suffix.reverse!\n    end\n\n    # THIS CLASS MUST BE THREAD-SAFE\n    class PlaceholderExpander\n      attr_reader :placeholders, :log\n\n      def initialize(params)\n        @log = params[:log]\n        @auto_typecast = params[:auto_typecast]\n      end\n\n      def time_value(time)\n        Time.at(time).to_s\n      end\n\n      def preprocess_map(value, force_stringify = false)\n        value\n      end\n\n      def prepare_placeholders(placeholder_values)\n        placeholders = {}\n\n        placeholder_values.each do |key, value|\n          if value.kind_of?(Array) # tag_parts, etc\n            size = value.size\n            value.each_with_index do |v, idx|\n              placeholders.store(\"${#{key}[#{idx}]}\", v)\n              placeholders.store(\"${#{key}[#{idx-size}]}\", v) # support [-1]\n            end\n          elsif value.kind_of?(Hash) # record, etc\n            value.each do |k, v|\n              placeholders.store(%Q[${#{key}[\"#{k}\"]}], v) # record[\"foo\"]\n            end\n          else # string, integer, float, and others?\n            placeholders.store(\"${#{key}}\", value)\n          end\n        end\n\n        placeholders\n      end\n\n      # Expand string with placeholders\n      #\n      # @param [String] str\n      # @param [Boolean] force_stringify the value must be string, used for hash key\n      def expand(str, placeholders, force_stringify = false)\n        if @auto_typecast && !force_stringify\n          single_placeholder_matched = str.match(/\\A(\\${[^}]+}|__[A-Z_]+__)\\z/)\n          if single_placeholder_matched\n            log_if_unknown_placeholder($1, placeholders)\n            return placeholders[single_placeholder_matched[1]]\n          end\n        end\n        str.gsub(/(\\${[^}]+}|__[A-Z_]+__)/) {\n          log_if_unknown_placeholder($1, placeholders)\n          placeholders[$1]\n        }\n      end\n\n      private\n\n      def log_if_unknown_placeholder(placeholder, placeholders)\n        unless placeholders.include?(placeholder)\n          log.warn \"unknown placeholder `#{placeholder}` found\"\n        end\n      end\n    end\n\n    # THIS CLASS MUST BE THREAD-SAFE\n    class RubyPlaceholderExpander\n      attr_reader :log\n\n      def initialize(params)\n        @log = params[:log]\n        @auto_typecast = params[:auto_typecast]\n        @cleanroom_expander = CleanroomExpander.new\n      end\n\n      def time_value(time)\n        Time.at(time)\n      end\n\n      # Preprocess record map to convert into ruby string expansion\n      #\n      # @param [Hash|String|Array] value record map config\n      # @param [Boolean] force_stringify the value must be string, used for hash key\n      def preprocess_map(value, force_stringify = false)\n        new_value = nil\n        if value.is_a?(String)\n          if @auto_typecast && !force_stringify\n            num_placeholders = value.scan('${').size\n            if num_placeholders == 1 && value.start_with?('${') && value.end_with?('}')\n              new_value = value[2..-2] # ${..} => ..\n            end\n          end\n          unless new_value\n            new_value = \"%Q[#{value.gsub('${', '#{')}]\" # xx${..}xx => %Q[xx#{..}xx]\n          end\n        elsif value.is_a?(Hash)\n          new_value = {}\n          value.each_pair do |k, v|\n            new_value[preprocess_map(k, true)] = preprocess_map(v)\n          end\n        elsif value.is_a?(Array)\n          new_value = []\n          value.each_with_index do |v, i|\n            new_value[i] = preprocess_map(v)\n          end\n        else\n          new_value = value\n        end\n        new_value\n      end\n\n      def prepare_placeholders(placeholder_values)\n        placeholder_values\n      end\n\n      # Expand string with placeholders\n      #\n      # @param [String] str\n      def expand(str, placeholders, force_stringify = false)\n        @cleanroom_expander.expand(\n          str,\n          placeholders['tag'],\n          placeholders['time'],\n          placeholders['record'],\n          placeholders['tag_parts'],\n          placeholders['tag_prefix'],\n          placeholders['tag_suffix'],\n          placeholders['hostname'],\n        )\n      rescue => e\n        raise \"failed to expand `#{str}` : error = #{e}\"\n      end\n\n      class CleanroomExpander\n        def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)\n          instance_eval(__str_to_eval__)\n        end\n\n        (Object.instance_methods).each do |m|\n          undef_method m unless /^__|respond_to_missing\\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\\?|new_ostruct_member|^class$/.match?(m.to_s)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/filter_stdout.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/filter'\n\nmodule Fluent::Plugin\n  class StdoutFilter < Filter\n    Fluent::Plugin.register_filter('stdout', self)\n\n    helpers :formatter, :compat_parameters, :inject\n\n    DEFAULT_FORMAT_TYPE = 'stdout'\n\n    config_section :format do\n      config_set_default :@type, DEFAULT_FORMAT_TYPE\n    end\n\n    # for tests\n    attr_reader :formatter\n\n    def configure(conf)\n      compat_parameters_convert(conf, :inject, :formatter)\n      super\n      @formatter = formatter_create\n    end\n\n    def filter_stream(tag, es)\n      es.each { |time, record|\n        begin\n          r = inject_values_to_record(tag, time, record)\n          log.write @formatter.format(tag, time, r)\n        rescue => e\n          router.emit_error_event(tag, time, record, e)\n        end\n      }\n      log.flush\n      es\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/formatter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/env'\nrequire 'fluent/plugin/base'\nrequire 'fluent/plugin/owned_by_mixin'\nrequire 'fluent/time'\n\nmodule Fluent\n  module Plugin\n    class Formatter < Base\n      include OwnedByMixin\n      include TimeMixin::Formatter\n\n      configured_in :format\n\n      PARSER_TYPES = [:text_per_line, :text, :binary]\n      def formatter_type\n        :text_per_line\n      end\n\n      def format(tag, time, record)\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n    end\n\n    class ProcWrappedFormatter < Formatter\n      def initialize(proc)\n        super()\n        @proc = proc\n      end\n\n      def format(tag, time, record)\n        @proc.call(tag, time, record)\n      end\n    end\n\n    module Newline\n      module Mixin\n        include Fluent::Configurable\n\n        DEFAULT_NEWLINE = if Fluent.windows?\n                            :crlf\n                          else\n                            :lf\n                          end\n\n        config_param :newline, :enum, list: [:lf, :crlf], default: DEFAULT_NEWLINE\n\n        def configure(conf)\n          super\n          @newline = case newline\n                     when :lf\n                       \"\\n\".freeze\n                     when :crlf\n                       \"\\r\\n\".freeze\n                     end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/formatter_csv.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin_helper'\nrequire 'fluent/plugin/formatter'\nrequire 'csv'\n\nmodule Fluent\n  module Plugin\n    class CsvFormatter < Formatter\n      Plugin.register_formatter('csv', self)\n\n      include PluginHelper::Mixin\n      helpers :record_accessor\n\n      config_param :delimiter, default: ',' do |val|\n        ['\\t', 'TAB'].include?(val) ? \"\\t\".freeze : val.freeze\n      end\n      config_param :force_quotes, :bool, default: true\n      # \"array\" looks good for type of :fields, but this implementation removes tailing comma\n      # TODO: Is it needed to support tailing comma?\n      config_param :fields, :array, value_type: :string\n      config_param :add_newline, :bool, default: true\n\n      def csv_cacheable?\n        !!owner\n      end\n\n      def csv_thread_key\n        csv_cacheable? ? \"#{owner.plugin_id}_csv_formatter_#{@usage}_csv\" : nil\n      end\n\n      def csv_for_thread\n        if csv_cacheable?\n          Thread.current[csv_thread_key] ||= CSV.new(\"\".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)\n        else\n          CSV.new(\"\".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)\n        end\n      end\n\n      def configure(conf)\n        super\n\n        @fields = fields.select{|f| !f.empty? }\n        raise ConfigError, \"empty value is specified in fields parameter\" if @fields.empty?\n\n        if @fields.any? { |f| record_accessor_nested?(f) }\n          @accessors = @fields.map { |f| record_accessor_create(f) }\n          mformat = method(:format_with_nested_fields)\n          singleton_class.module_eval do\n            define_method(:format, mformat)\n          end\n        end\n\n        @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes, headers: @fields,\n                          row_sep: @add_newline ? :auto : \"\".force_encoding(Encoding::ASCII_8BIT)}\n      end\n\n      def format(tag, time, record)\n        csv = csv_for_thread\n        line = (csv << record).string.dup\n        # Need manual cleanup because CSV writer doesn't provide such method.\n        csv.rewind\n        csv.truncate(0)\n        line\n      end\n\n      def format_with_nested_fields(tag, time, record)\n        csv = csv_for_thread\n        values = @accessors.map { |a| a.call(record) }\n        line = (csv << values).string.dup\n        # Need manual cleanup because CSV writer doesn't provide such method.\n        csv.rewind\n        csv.truncate(0)\n        line\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/formatter_hash.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/formatter'\n\nmodule Fluent\n  module Plugin\n    class HashFormatter < Formatter\n      include Fluent::Plugin::Newline::Mixin\n\n      Plugin.register_formatter('hash', self)\n\n      config_param :add_newline, :bool, default: true\n\n      def format(tag, time, record)\n        line = record.to_s\n        line << @newline if @add_newline\n        line\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/formatter_json.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/formatter'\nrequire 'fluent/oj_options'\n\nmodule Fluent\n  module Plugin\n    class JSONFormatter < Formatter\n      include Fluent::Plugin::Newline::Mixin\n\n      Plugin.register_formatter('json', self)\n\n      config_param :json_parser, :string, default: 'oj'\n      config_param :add_newline, :bool, default: true\n\n      def configure(conf)\n        super\n\n        if @json_parser == 'oj'\n          if Fluent::OjOptions.available?\n            @dump_proc = Oj.method(:dump)\n          else\n            log.info \"Oj isn't installed, fallback to JSON as json parser\"\n            @dump_proc = JSON.method(:generate)\n          end\n        else\n          @dump_proc = JSON.method(:generate)\n        end\n\n        # format json is used on various highload environment, so re-define method to skip if check\n        unless @add_newline\n          define_singleton_method(:format, method(:format_without_nl))\n        end\n      end\n\n      def format(tag, time, record)\n        json_str = @dump_proc.call(record)\n        \"#{json_str}#{@newline}\"\n      ensure\n        json_str&.clear\n      end\n\n      def format_without_nl(tag, time, record)\n        @dump_proc.call(record)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/formatter_ltsv.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/formatter'\n\nmodule Fluent\n  module Plugin\n    class LabeledTSVFormatter < Formatter\n      include Fluent::Plugin::Newline::Mixin\n\n      Plugin.register_formatter('ltsv', self)\n\n      # http://ltsv.org/\n\n      config_param :delimiter, :string, default: \"\\t\".freeze\n      config_param :label_delimiter, :string, default: \":\".freeze\n      config_param :replacement, :string, default: \" \".freeze\n      config_param :add_newline, :bool, default: true\n\n      def format(tag, time, record)\n        formatted = \"\"\n        record.each do |label, value|\n          formatted << @delimiter if formatted.length.nonzero?\n          formatted << \"#{label}#{@label_delimiter}#{value.to_s.gsub(@delimiter, @replacement)}\"\n        end\n        formatted << @newline if @add_newline\n        formatted\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/formatter_msgpack.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/formatter'\n\nmodule Fluent\n  module Plugin\n    class MessagePackFormatter < Formatter\n      Plugin.register_formatter('msgpack', self)\n\n      def formatter_type\n        :binary\n      end\n\n      def format(tag, time, record)\n        record.to_msgpack\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/formatter_out_file.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/formatter'\nrequire 'fluent/time'\nrequire 'json'\n\nmodule Fluent\n  module Plugin\n    class OutFileFormatter < Formatter\n      include Fluent::Plugin::Newline::Mixin\n\n      Plugin.register_formatter('out_file', self)\n\n      config_param :output_time, :bool, default: true\n      config_param :output_tag, :bool, default: true\n      config_param :delimiter, default: \"\\t\" do |val|\n        case val\n        when /SPACE/i then ' '.freeze\n        when /COMMA/i then ','.freeze\n        else \"\\t\".freeze\n        end\n      end\n      config_set_default :time_type, :string\n      config_set_default :time_format, nil # time_format nil => iso8601\n\n      def configure(conf)\n        super\n        @timef = time_formatter_create\n      end\n\n      def format(tag, time, record)\n        json_str = JSON.generate(record)\n        time_str = @output_time ? \"#{@timef.format(time)}#{@delimiter}\" : ''\n        tag_str = @output_tag ? \"#{tag}#{@delimiter}\" : ''\n        \"#{time_str}#{tag_str}#{json_str}#{@newline}\"\n      ensure\n        json_str&.clear\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/formatter_single_value.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/formatter'\n\nmodule Fluent\n  module Plugin\n    class SingleValueFormatter < Formatter\n      include Fluent::Plugin::Newline::Mixin\n\n      Plugin.register_formatter('single_value', self)\n\n      config_param :message_key, :string, default: 'message'.freeze\n      config_param :add_newline, :bool, default: true\n\n      def format(tag, time, record)\n        text = record[@message_key].to_s.dup\n        text << @newline if @add_newline\n        text\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/formatter_stdout.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/formatter'\n\nmodule Fluent\n  module Plugin\n    class StdoutFormatter < Formatter\n      Plugin.register_formatter('stdout', self)\n\n      TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%N %z'\n\n      config_param :output_type, :string, default: 'json'\n\n      def configure(conf)\n        super\n\n        @time_formatter = Strftime.new(@time_format || TIME_FORMAT)\n        @sub_formatter = Plugin.new_formatter(@output_type, parent: self.owner)\n        @sub_formatter.configure(conf)\n      end\n\n      def start\n        super\n        @sub_formatter.start\n      end\n\n      def format(tag, time, record)\n        \"#{@time_formatter.exec(Time.at(time).localtime)} #{tag}: #{@sub_formatter.format(tag, time, record).chomp}\\n\"\n      end\n\n      def stop\n        @sub_formatter.stop\n        super\n      end\n\n      def before_shutdown\n        @sub_formatter.before_shutdown\n        super\n      end\n\n      def shutdown\n        @sub_formatter.shutdown\n        super\n      end\n\n      def after_shutdown\n        @sub_formatter.after_shutdown\n        super\n      end\n\n      def close\n        @sub_formatter.close\n        super\n      end\n\n      def terminate\n        @sub_formatter.terminate\n        super\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/formatter_tsv.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/formatter'\n\nmodule Fluent\n  module Plugin\n    class TSVFormatter < Formatter\n      include Fluent::Plugin::Newline::Mixin\n\n      Plugin.register_formatter('tsv', self)\n\n      desc 'Field names included in each lines'\n      config_param :keys, :array, value_type: :string\n      desc 'The delimiter character (or string) of TSV values'\n      config_param :delimiter, :string, default: \"\\t\".freeze\n      desc 'The parameter to enable writing to new lines'\n      config_param :add_newline, :bool, default: true\n\n      def format(tag, time, record)\n        formatted = @keys.map{|k| record[k].to_s }.join(@delimiter)\n        formatted << @newline if @add_newline\n        formatted\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_debug_agent.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/input'\n\nmodule Fluent::Plugin\n  class DebugAgentInput < Input\n    Fluent::Plugin.register_input('debug_agent', self)\n\n    def initialize\n      require 'drb/drb'\n      require 'fluent/plugin/file_util'\n      super\n    end\n\n    config_param :bind, :string, default: '127.0.0.1'\n    config_param :port, :integer, default: 24230\n    config_param :unix_path, :string, default: nil\n    #config_param :unix_mode  # TODO\n    config_param :object, :string, default: 'Fluent::Engine'\n\n    def configure(conf)\n      super\n      if system_config.workers > 1\n        @port += fluentd_worker_id\n      end\n      if @unix_path\n        unless ::Fluent::FileUtil.writable?(@unix_path)\n          raise Fluent::ConfigError, \"in_debug_agent: `#{@unix_path}` is not writable\"\n        end\n      end\n    end\n\n    def multi_workers_ready?\n      @unix_path.nil?\n    end\n\n    def start\n      super\n\n      if @unix_path\n        require 'drb/unix'\n        uri = \"drbunix:#{@unix_path}\"\n      else\n        uri = \"druby://#{@bind}:#{@port}\"\n      end\n      log.info \"listening dRuby\", uri: uri, object: @object, worker: fluentd_worker_id\n      obj = eval(@object)\n      @server = DRb::DRbServer.new(uri, obj)\n    end\n\n    def shutdown\n      @server.stop_service if @server\n\n      super\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_dummy.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\n# Remove this file in fluentd v2\nrequire_relative 'in_sample'\n"
  },
  {
    "path": "lib/fluent/plugin/in_exec.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/input'\nrequire 'json'\n\nmodule Fluent::Plugin\n  class ExecInput < Fluent::Plugin::Input\n    Fluent::Plugin.register_input('exec', self)\n\n    helpers :compat_parameters, :extract, :parser, :child_process\n\n    desc 'The command (program) to execute.'\n    config_param :command, :string\n    desc 'Specify connect mode to executed process'\n    config_param :connect_mode, :enum, list: [:read, :read_with_stderr], default: :read\n\n    config_section :parse do\n      config_set_default :@type, 'tsv'\n      config_set_default :time_type, :float\n      config_set_default :time_key, nil\n      config_set_default :estimate_current_event, false\n    end\n\n    config_section :extract do\n      config_set_default :time_type, :float\n    end\n\n    desc 'Tag of the output events.'\n    config_param :tag, :string, default: nil\n    desc 'The interval time between periodic program runs.'\n    config_param :run_interval, :time, default: nil\n    desc 'The default block size to read if parser requires partial read.'\n    config_param :read_block_size, :size, default: 10240 # 10k\n    desc 'The encoding to receive the result of the command, especially for non-ascii characters.'\n    config_param :encoding, :string, default: nil\n\n    attr_reader :parser\n\n    def configure(conf)\n      compat_parameters_convert(conf, :extract, :parser)\n      ['parse', 'extract'].each do |subsection_name|\n        if subsection = conf.elements(subsection_name).first\n          if subsection.has_key?('time_format')\n            subsection['time_type'] ||= 'string'\n          end\n        end\n      end\n\n      super\n\n      if !@tag && (!@extract_config || !@extract_config.tag_key)\n        raise Fluent::ConfigError, \"'tag' or 'tag_key' option is required on exec input\"\n      end\n      validate_encoding(@encoding) if @encoding\n      @parser = parser_create\n    end\n\n    def validate_encoding(encoding)\n      Encoding.find(encoding)\n    rescue ArgumentError => e\n      raise Fluent::ConfigError, e.message\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def start\n      super\n\n      options = { mode: [@connect_mode] }\n      options[:external_encoding] = @encoding if @encoding\n\n      if @run_interval\n        child_process_execute(:exec_input, @command, interval: @run_interval, **options, &method(:run))\n      else\n        child_process_execute(:exec_input, @command, immediate: true, **options, &method(:run))\n      end\n    end\n\n    def run(io)\n      case\n      when @parser.implement?(:parse_io)\n        @parser.parse_io(io, &method(:on_record))\n      when @parser.implement?(:parse_partial_data)\n        until io.eof?\n          @parser.parse_partial_data(io.readpartial(@read_block_size), &method(:on_record))\n        end\n      when @parser.parser_type == :text_per_line\n        io.each_line do |line|\n          @parser.parse(line.chomp, &method(:on_record))\n        end\n      else\n        @parser.parse(io.read, &method(:on_record))\n      end\n    end\n\n    def on_record(time, record)\n      tag = extract_tag_from_record(record)\n      tag ||= @tag\n      time ||= extract_time_from_record(record) || Fluent::EventTime.now\n      router.emit(tag, time, record)\n    rescue => e\n      log.error \"exec failed to emit\", tag: tag, record: JSON.generate(record), error: e\n      router.emit_error_event(tag, time, record, e) if tag && time && record\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_forward.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\n\nrequire 'fluent/plugin/input'\nrequire 'fluent/msgpack_factory'\nrequire 'yajl'\nrequire 'digest'\nrequire 'securerandom'\n\nmodule Fluent::Plugin\n  class ForwardInput < Input\n    Fluent::Plugin.register_input('forward', self)\n\n    # See the wiki page below for protocol specification\n    # https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1\n\n    helpers :server\n\n    LISTEN_PORT = 24224\n\n    desc 'The port to listen to.'\n    config_param :port, :integer, default: LISTEN_PORT\n    desc 'The bind address to listen to.'\n    config_param :bind, :string, default: '0.0.0.0'\n\n    config_param :backlog, :integer, default: nil\n    # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src\n    desc 'The timeout time used to set linger option.'\n    config_param :linger_timeout, :integer, default: nil, deprecated: \"use transport directive\"\n    # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.\n    config_param :blocking_timeout, :time, default: 0.5\n    desc 'Try to resolve hostname from IP addresses or not.'\n    config_param :resolve_hostname, :bool, default: nil\n    desc 'Connections will be disconnected right after receiving first message if this value is true.'\n    config_param :deny_keepalive, :bool, default: false\n    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'\n    config_param :send_keepalive_packet, :bool, default: false\n\n    desc 'Log warning if received chunk size is larger than this value.'\n    config_param :chunk_size_warn_limit, :size, default: nil\n    desc 'Received chunk is dropped if it is larger than this value.'\n    config_param :chunk_size_limit, :size, default: nil\n    desc 'Skip an event if incoming event is invalid.'\n    config_param :skip_invalid_event, :bool, default: true\n\n    desc \"The field name of the client's source address.\"\n    config_param :source_address_key, :string, default: nil\n    desc \"The field name of the client's hostname.\"\n    config_param :source_hostname_key, :string, default: nil\n\n    desc \"New tag instead of incoming tag\"\n    config_param :tag, :string, default: nil\n    desc \"Add prefix to incoming tag\"\n    config_param :add_tag_prefix, :string, default: nil\n\n    config_section :security, required: false, multi: false do\n      desc 'The hostname'\n      config_param :self_hostname, :string\n      desc 'Shared key for authentication'\n      config_param :shared_key, :string, secret: true\n      desc 'If true, use user based authentication'\n      config_param :user_auth, :bool, default: false\n      desc 'Allow anonymous source. <client> sections required if disabled.'\n      config_param :allow_anonymous_source, :bool, default: true\n\n      ### User based authentication\n      config_section :user, param_name: :users, required: false, multi: true do\n        desc 'The username for authentication'\n        config_param :username, :string\n        desc 'The password for authentication'\n        config_param :password, :string, secret: true\n      end\n\n      ### Client ip/network authentication & per_host shared key\n      config_section :client, param_name: :clients, required: false, multi: true do\n        desc 'The IP address or host name of the client'\n        config_param :host, :string, default: nil\n        desc 'Network address specification'\n        config_param :network, :string, default: nil\n        desc 'Shared key per client'\n        config_param :shared_key, :string, default: nil, secret: true\n        desc 'Array of username.'\n        config_param :users, :array, default: []\n      end\n    end\n\n    def configure(conf)\n      super\n\n      if @source_hostname_key\n        # TODO: add test\n        if @resolve_hostname.nil?\n          @resolve_hostname = true\n        elsif !@resolve_hostname # user specifies \"false\" in config\n          raise Fluent::ConfigError, \"resolve_hostname must be true with source_hostname_key\"\n        end\n      end\n      @enable_field_injection = @source_address_key || @source_hostname_key\n\n      raise Fluent::ConfigError, \"'tag' parameter must not be empty\" if @tag && @tag.empty?\n      raise Fluent::ConfigError, \"'add_tag_prefix' parameter must not be empty\" if @add_tag_prefix && @add_tag_prefix.empty?\n\n      if @security\n        if @security.user_auth && @security.users.empty?\n          raise Fluent::ConfigError, \"<user> sections required if user_auth enabled\"\n        end\n        if !@security.allow_anonymous_source && @security.clients.empty?\n          raise Fluent::ConfigError, \"<client> sections required if allow_anonymous_source disabled\"\n        end\n\n        @nodes = []\n\n        @security.clients.each do |client|\n          if client.host && client.network\n            raise Fluent::ConfigError, \"both of 'host' and 'network' are specified for client\"\n          end\n          if !client.host && !client.network\n            raise Fluent::ConfigError, \"Either of 'host' and 'network' must be specified for client\"\n          end\n          source = nil\n          if client.host\n            begin\n              source = IPSocket.getaddress(client.host)\n            rescue SocketError\n              raise Fluent::ConfigError, \"host '#{client.host}' cannot be resolved\"\n            end\n          end\n          source_addr = begin\n                          IPAddr.new(source || client.network)\n                        rescue ArgumentError\n                          raise Fluent::ConfigError, \"network '#{client.network}' address format is invalid\"\n                        end\n          @nodes.push({\n              address: source_addr,\n              shared_key: (client.shared_key || @security.shared_key),\n              users: client.users\n            })\n        end\n      end\n\n      if @send_keepalive_packet && @deny_keepalive\n        raise Fluent::ConfigError, \"both 'send_keepalive_packet' and 'deny_keepalive' cannot be set to true\"\n      end\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    HEARTBEAT_UDP_PAYLOAD = \"\\0\"\n\n    def start\n      super\n\n      shared_socket = system_config.workers > 1\n\n      log.info \"listening port\", port: @port, bind: @bind\n      server_create_connection(\n        :in_forward_server, @port,\n        bind: @bind,\n        shared: shared_socket,\n        resolve_name: @resolve_hostname,\n        linger_timeout: @linger_timeout,\n        send_keepalive_packet: @send_keepalive_packet,\n        backlog: @backlog,\n        &method(:handle_connection)\n      )\n\n      server_create(:in_forward_server_udp_heartbeat, @port, shared: shared_socket, proto: :udp, bind: @bind, resolve_name: @resolve_hostname, max_bytes: 128) do |data, sock|\n        log.trace \"heartbeat udp data arrived\", host: sock.remote_host, port: sock.remote_port, data: data\n        begin\n          sock.write HEARTBEAT_UDP_PAYLOAD\n        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR\n          log.trace \"error while heartbeat response\", host: sock.remote_host, error: e\n        end\n      end\n    end\n\n    def handle_connection(conn)\n      send_data = ->(serializer, data){ conn.write serializer.call(data) }\n\n      log.trace \"connected fluent socket\", addr: conn.remote_addr, port: conn.remote_port\n      state = :established\n      nonce = nil\n      user_auth_salt = nil\n\n      if @security\n        # security enabled session MUST use MessagePack as serialization format\n        state = :helo\n        nonce = generate_salt\n        user_auth_salt = generate_salt\n        send_data.call(:to_msgpack.to_proc, generate_helo(nonce, user_auth_salt))\n        state = :pingpong\n      end\n\n      log.trace \"accepted fluent socket\", addr: conn.remote_addr, port: conn.remote_port\n\n      read_messages(conn) do |msg, chunk_size, serializer|\n        case state\n        when :pingpong\n          success, reason_or_salt, shared_key = check_ping(msg, conn.remote_addr, user_auth_salt, nonce)\n          unless success\n            conn.on(:write_complete) { |c| c.close_after_write_complete }\n            send_data.call(serializer, generate_pong(false, reason_or_salt, nonce, shared_key))\n            next\n          end\n          send_data.call(serializer, generate_pong(true, reason_or_salt, nonce, shared_key))\n\n          log.debug \"connection established\", address: conn.remote_addr, port: conn.remote_port\n          state = :established\n        when :established\n          options = on_message(msg, chunk_size, conn)\n          if options && r = response(options)\n            log.trace \"sent response to fluent socket\", address: conn.remote_addr, response: r\n            conn.on(:write_complete) { |c| c.close } if @deny_keepalive\n            send_data.call(serializer, r)\n          else\n            if @deny_keepalive\n              conn.close\n            end\n          end\n        else\n          raise \"BUG: unknown session state: #{state}\"\n        end\n      end\n    end\n\n    def read_messages(conn, &block)\n      feeder = nil\n      serializer = nil\n      bytes = 0\n      conn.data do |data|\n        # only for first call of callback\n        unless feeder\n          first = data[0]\n          if first == '{' || first == '[' # json\n            parser = Yajl::Parser.new\n            parser.on_parse_complete = ->(obj){\n              block.call(obj, bytes, serializer)\n              bytes = 0\n            }\n            serializer = :to_json.to_proc\n            feeder = ->(d){ parser << d }\n          else # msgpack\n            parser = Fluent::MessagePackFactory.msgpack_unpacker\n            serializer = :to_msgpack.to_proc\n            feeder = ->(d){\n              parser.feed_each(d){|obj|\n                block.call(obj, bytes, serializer)\n                bytes = 0\n              }\n            }\n          end\n        end\n\n        bytes += data.bytesize\n        feeder.call(data)\n      end\n    end\n\n    def response(option)\n      if option && option['chunk']\n        return { 'ack' => option['chunk'] }\n      end\n      nil\n    end\n\n    def on_message(msg, chunk_size, conn)\n      if msg.nil?\n        # for future TCP heartbeat_request\n        return\n      end\n\n      # TODO: raise an exception if broken chunk is generated by recoverable situation\n      unless msg.is_a?(Array)\n        log.warn \"incoming chunk is broken:\", host: conn.remote_host, msg: msg\n        return\n      end\n\n      tag = msg[0]\n      entries = msg[1]\n\n      if @chunk_size_limit && (chunk_size > @chunk_size_limit)\n        log.warn \"Input chunk size is larger than 'chunk_size_limit', dropped:\", tag: tag, host: conn.remote_host, limit: @chunk_size_limit, size: chunk_size\n        return\n      elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)\n        log.warn \"Input chunk size is larger than 'chunk_size_warn_limit':\", tag: tag, host: conn.remote_host, limit: @chunk_size_warn_limit, size: chunk_size\n      end\n\n      tag = @tag.dup if @tag\n      tag = \"#{@add_tag_prefix}.#{tag}\" if @add_tag_prefix\n\n      case entries\n      when String\n        # PackedForward\n        option = msg[2] || {}\n        size = option['size'] || 0\n\n        if option['compressed'] && option['compressed'] != 'text'\n          es = Fluent::CompressedMessagePackEventStream.new(entries, nil, size.to_i, compress: option['compressed'].to_sym)\n        else\n          es = Fluent::MessagePackEventStream.new(entries, nil, size.to_i)\n        end\n        es = check_and_skip_invalid_event(tag, es, conn.remote_host) if @skip_invalid_event\n        if @enable_field_injection\n          es = add_source_info(es, conn)\n        end\n        router.emit_stream(tag, es)\n\n      when Array\n        # Forward\n        es = if @skip_invalid_event\n               check_and_skip_invalid_event(tag, entries, conn.remote_host)\n             else\n               es = Fluent::MultiEventStream.new\n               entries.each { |e|\n                 record = e[1]\n                 next if record.nil?\n                 time = e[0]\n                 time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime\n                 es.add(time, record)\n               }\n               es\n             end\n        if @enable_field_injection\n          es = add_source_info(es, conn)\n        end\n        router.emit_stream(tag, es)\n        option = msg[2]\n\n      else\n        # Message\n        time = msg[1]\n        record = msg[2]\n        if @skip_invalid_event && invalid_event?(tag, time, record)\n          log.warn \"got invalid event and drop it:\", host: conn.remote_host, tag: tag, time: time, record: record\n          return msg[3] # retry never succeeded so return ack and drop incoming event.\n        end\n        return if record.nil?\n        time = Fluent::EventTime.now if time.to_i == 0\n        if @enable_field_injection\n          record[@source_address_key] = conn.remote_addr if @source_address_key\n          record[@source_hostname_key] = conn.remote_host if @source_hostname_key\n        end\n        router.emit(tag, time, record)\n        option = msg[3]\n      end\n\n      # return option for response\n      option\n    end\n\n    def invalid_event?(tag, time, record)\n      !((time.is_a?(Integer) || time.is_a?(::Fluent::EventTime)) && record.is_a?(Hash) && tag.is_a?(String))\n    end\n\n    def check_and_skip_invalid_event(tag, es, remote_host)\n      new_es = Fluent::MultiEventStream.new\n      es.each { |time, record|\n        if invalid_event?(tag, time, record)\n          log.warn \"skip invalid event:\", host: remote_host, tag: tag, time: time, record: record\n          next\n        end\n        new_es.add(time, record)\n      }\n      new_es\n    end\n\n    def add_source_info(es, conn)\n      new_es = Fluent::MultiEventStream.new\n      if @source_address_key && @source_hostname_key\n        address = conn.remote_addr\n        hostname = conn.remote_host\n        es.each { |time, record|\n          record[@source_address_key] = address\n          record[@source_hostname_key] = hostname\n          new_es.add(time, record)\n        }\n      elsif @source_address_key\n        address = conn.remote_addr\n        es.each { |time, record|\n          record[@source_address_key] = address\n          new_es.add(time, record)\n        }\n      elsif @source_hostname_key\n        hostname = conn.remote_host\n        es.each { |time, record|\n          record[@source_hostname_key] = hostname\n          new_es.add(time, record)\n        }\n      else\n        raise \"BUG: don't call this method in this case\"\n      end\n      new_es\n    end\n\n    def select_authenticate_users(node, username)\n      if node.nil? || node[:users].empty?\n        @security.users.select{|u| u.username == username}\n      else\n        @security.users.select{|u| node[:users].include?(u.username) && u.username == username}\n      end\n    end\n\n    def generate_salt\n      ::SecureRandom.random_bytes(16)\n    end\n\n    def generate_helo(nonce, user_auth_salt)\n      log.debug \"generating helo\"\n      # ['HELO', options(hash)]\n      ['HELO', {'nonce' => nonce, 'auth' => (@security ? user_auth_salt : ''), 'keepalive' => !@deny_keepalive}]\n    end\n\n    def check_ping(message, remote_addr, user_auth_salt, nonce)\n      log.debug \"checking ping\"\n      # ['PING', self_hostname, shared_key_salt, sha512_hex(shared_key_salt + self_hostname + nonce + shared_key), username || '', sha512_hex(auth_salt + username + password) || '']\n      unless message.size == 6 && message[0] == 'PING'\n        return false, 'invalid ping message'\n      end\n      _ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message\n\n      node = @nodes.find{|n| n[:address].include?(remote_addr) rescue false }\n      if !node && !@security.allow_anonymous_source\n        log.warn \"Anonymous client disallowed\", address: remote_addr, hostname: hostname\n        return false, \"anonymous source host '#{remote_addr}' denied\", nil\n      end\n\n      shared_key = node ? node[:shared_key] : @security.shared_key\n      serverside = Digest::SHA512.new.update(shared_key_salt).update(hostname).update(nonce).update(shared_key).hexdigest\n      if shared_key_hexdigest != serverside\n        log.warn \"Shared key mismatch\", address: remote_addr, hostname: hostname\n        return false, 'shared_key mismatch', nil\n      end\n\n      if @security.user_auth\n        users = select_authenticate_users(node, username)\n        success = false\n        users.each do |user|\n          passhash = Digest::SHA512.new.update(user_auth_salt).update(username).update(user[:password]).hexdigest\n          success ||= (passhash == password_digest)\n        end\n        unless success\n          log.warn \"Authentication failed\", address: remote_addr, hostname: hostname, username: username\n          return false, 'username/password mismatch', nil\n        end\n      end\n\n      return true, shared_key_salt, shared_key\n    end\n\n    def generate_pong(auth_result, reason_or_salt, nonce, shared_key)\n      log.debug \"generating pong\"\n      # ['PONG', bool(authentication result), 'reason if authentication failed', self_hostname, sha512_hex(salt + self_hostname + nonce + sharedkey)]\n      unless auth_result\n        return ['PONG', false, reason_or_salt, '', '']\n      end\n\n      shared_key_digest_hex = Digest::SHA512.new.update(reason_or_salt).update(@security.self_hostname).update(nonce).update(shared_key).hexdigest\n      ['PONG', true, '', @security.self_hostname, shared_key_digest_hex]\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_gc_stat.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/input'\n\nmodule Fluent::Plugin\n  class GCStatInput < Fluent::Plugin::Input\n    Fluent::Plugin.register_input('gc_stat', self)\n\n    helpers :timer\n\n    def initialize\n      super\n      @key_map = nil\n    end\n\n    config_param :emit_interval, :time, default: 60\n    config_param :use_symbol_keys, :bool, default: true\n    config_param :tag, :string\n\n    def configure(conf)\n      super\n\n      unless @use_symbol_keys\n        @key_map = {}\n        GC.stat.each_key { |key|\n          @key_map[key] = key.to_s\n        }\n      end\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def start\n      super\n\n      timer_execute(:in_gc_stat, @emit_interval, &method(:on_timer))\n    end\n\n    def shutdown\n      super\n    end\n\n    def on_timer\n      now = Fluent::EventTime.now\n      record = GC.stat\n      unless @use_symbol_keys\n        new_record = {}\n        record.each_pair { |k, v|\n          new_record[@key_map[k]] = v\n        }\n        record = new_record\n      end\n      router.emit(@tag, now, record)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_http.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/input'\nrequire 'fluent/plugin/parser'\nrequire 'fluent/event'\n\nrequire 'http/parser'\nrequire 'webrick/httputils'\nrequire 'uri'\nrequire 'socket'\nrequire 'json'\n\nmodule Fluent::Plugin\n  class InHttpParser < Parser\n    Fluent::Plugin.register_parser('in_http', self)\n\n    config_set_default :time_key, 'time'\n\n    def configure(conf)\n      super\n\n      # if no time parser related parameters, use in_http's time convert rule\n      @time_parser = if conf.has_key?('time_type') || conf.has_key?('time_format')\n                       time_parser_create\n                     else\n                       nil\n                     end\n    end\n\n    def parse(text)\n      # this plugin is dummy implementation not to raise error\n      yield nil, nil\n    end\n\n    def get_time_parser\n      @time_parser\n    end\n  end\n\n  class HttpInput < Input\n    Fluent::Plugin.register_input('http', self)\n\n    helpers :parser, :compat_parameters, :event_loop, :server\n\n    EMPTY_GIF_IMAGE = \"GIF89a\\u0001\\u0000\\u0001\\u0000\\x80\\xFF\\u0000\\xFF\\xFF\\xFF\\u0000\\u0000\\u0000,\\u0000\\u0000\\u0000\\u0000\\u0001\\u0000\\u0001\\u0000\\u0000\\u0002\\u0002D\\u0001\\u0000;\".force_encoding(\"UTF-8\")\n\n    desc 'The port to listen to.'\n    config_param :port, :integer, default: 9880\n    desc 'The bind address to listen to.'\n    config_param :bind, :string, default: '0.0.0.0'\n    desc 'The size limit of the POSTed element. Default is 32MB.'\n    config_param :body_size_limit, :size, default: 32*1024*1024  # TODO default\n    desc 'The timeout limit for keeping the connection alive.'\n    config_param :keepalive_timeout, :time, default: 10   # TODO default\n    config_param :backlog, :integer, default: nil\n    desc 'Add HTTP_ prefix headers to the record.'\n    config_param :add_http_headers, :bool, default: false\n    desc 'Add REMOTE_ADDR header to the record.'\n    config_param :add_remote_addr, :bool, default: false\n    config_param :blocking_timeout, :time, default: 0.5\n    desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'\n    config_param :cors_allow_origins, :array, default: nil\n    desc 'Tells browsers whether to expose the response to frontend when the credentials mode is \"include\".'\n    config_param :cors_allow_credentials, :bool, default: false\n    desc 'Respond with empty gif image of 1x1 pixel.'\n    config_param :respond_with_empty_img, :bool, default: false\n    desc 'Respond status code with 204.'\n    config_param :use_204_response, :bool, default: false\n    desc 'Dump error log or not'\n    config_param :dump_error_log, :bool, default: true\n    desc 'Add QUERY_ prefix query params to record'\n    config_param :add_query_params, :bool, default: false\n    desc \"Add prefix to incoming tag\"\n    config_param :add_tag_prefix, :string, default: nil\n\n    config_section :parse do\n      config_set_default :@type, 'in_http'\n    end\n\n    EVENT_RECORD_PARAMETER = '_event_record'\n\n    def initialize\n      super\n\n      @km = nil\n      @format_name = nil\n      @parser_time_key = nil\n\n      # default parsers\n      @parser_msgpack = nil\n      @parser_json = nil\n      @default_time_parser = nil\n      @default_keep_time_key = nil\n      @float_time_parser = nil\n\n      # <parse> configured parser\n      @custom_parser = nil\n    end\n\n    def configure(conf)\n      compat_parameters_convert(conf, :parser)\n\n      super\n\n      if @cors_allow_credentials\n        if @cors_allow_origins.nil? || @cors_allow_origins.include?('*')\n          raise Fluent::ConfigError, \"Cannot enable cors_allow_credentials without specific origins\"\n        end\n      end\n\n      raise Fluent::ConfigError, \"'add_tag_prefix' parameter must not be empty\" if @add_tag_prefix && @add_tag_prefix.empty?\n\n      m = if @parser_configs.first['@type'] == 'in_http'\n            @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')\n            @parser_msgpack.time_key = nil\n            @parser_msgpack.estimate_current_event = false\n            @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')\n            @parser_json.time_key = nil\n            @parser_json.estimate_current_event = false\n\n            default_parser = parser_create(usage: '')\n            @format_name = 'default'\n            @parser_time_key = default_parser.time_key\n            @default_time_parser = default_parser.get_time_parser\n            @default_keep_time_key = default_parser.keep_time_key\n            method(:parse_params_default)\n          else\n            @custom_parser = parser_create\n            @format_name = @parser_configs.first['@type']\n            @parser_time_key = @custom_parser.time_key\n            method(:parse_params_with_parser)\n          end\n      self.singleton_class.module_eval do\n        define_method(:parse_params, m)\n      end\n    end\n\n    class KeepaliveManager < Coolio::TimerWatcher\n      def initialize(timeout)\n        super(1, true)\n        @cons = {}\n        @timeout = timeout.to_i\n      end\n\n      def add(sock)\n        @cons[sock] = sock\n      end\n\n      def delete(sock)\n        @cons.delete(sock)\n      end\n\n      def on_timer\n        @cons.each_pair {|sock,val|\n          if sock.step_idle > @timeout\n            sock.close\n          end\n        }\n      end\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def start\n      @_event_loop_run_timeout = @blocking_timeout\n\n      super\n\n      log.debug \"listening http\", bind: @bind, port: @port\n\n      @km = KeepaliveManager.new(@keepalive_timeout)\n      event_loop_attach(@km)\n\n      server_create_connection(:in_http, @port, bind: @bind, backlog: @backlog, &method(:on_server_connect))\n      @float_time_parser = Fluent::NumericTimeParser.new(:float)\n    end\n\n    def close\n      server_wait_until_stop\n      super\n    end\n\n    RES_TEXT_HEADER = {'Content-Type' => 'text/plain'}.freeze\n    RESPONSE_200 = [\"200 OK\".freeze, RES_TEXT_HEADER, \"\".freeze].freeze\n    RESPONSE_204 = [\"204 No Content\".freeze, {}.freeze].freeze\n    RESPONSE_IMG = [\"200 OK\".freeze, {'Content-Type'=>'image/gif; charset=utf-8'}.freeze, EMPTY_GIF_IMAGE].freeze\n    RES_400_STATUS = \"400 Bad Request\".freeze\n    RES_500_STATUS = \"500 Internal Server Error\".freeze\n\n    def on_request(path_info, params)\n      begin\n        path = path_info[1..-1]  # remove /\n        tag = path.split('/').join('.')\n        tag = \"#{@add_tag_prefix}.#{tag}\" if @add_tag_prefix\n\n        mes = Fluent::MultiEventStream.new\n        parse_params(params) do |record_time, record|\n          if record.nil?\n            log.debug { \"incoming event is invalid: path=#{path_info} params=#{params.to_json}\" }\n            next\n          end\n\n          add_params_to_record(record, params)\n\n          time = if param_time = params['time']\n                   param_time = param_time.to_f\n                   param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)\n                 else\n                   record_time.nil? ? convert_time_field(record) : record_time\n                 end\n\n          mes.add(time, record)\n        end\n      rescue => e\n        if @dump_error_log\n          log.error \"failed to process request\", error: e\n        end\n        return [RES_400_STATUS, RES_TEXT_HEADER, \"400 Bad Request\\n#{e}\\n\"]\n      end\n\n      # TODO server error\n      begin\n        router.emit_stream(tag, mes) unless mes.empty?\n      rescue => e\n        if @dump_error_log\n          log.error \"failed to emit data\", error: e\n        end\n        return [RES_500_STATUS, RES_TEXT_HEADER, \"500 Internal Server Error\\n#{e}\\n\"]\n      end\n\n      if @respond_with_empty_img\n        return RESPONSE_IMG\n      else\n        if @use_204_response\n          return RESPONSE_204\n        else\n          return RESPONSE_200\n        end\n      end\n    end\n\n    private\n\n    def on_server_connect(conn)\n      handler = Handler.new(conn, @km, method(:on_request),\n                            @body_size_limit, @format_name, log,\n                            @cors_allow_origins, @cors_allow_credentials,\n                            @add_query_params)\n\n      conn.on(:data) do |data|\n        handler.on_read(data)\n      end\n\n      conn.on(:write_complete) do |_|\n        handler.on_write_complete\n      end\n\n      conn.on(:close) do |_|\n        handler.on_close\n      end\n    end\n\n    def parse_params_default(params)\n      if msgpack = params['msgpack']\n        @parser_msgpack.parse(msgpack) do |_time, record|\n          yield nil, record\n        end\n      elsif js = params['json']\n        @parser_json.parse(js) do |_time, record|\n          yield nil, record\n        end\n      elsif ndjson = params['ndjson']\n        ndjson.split(/\\r?\\n/).each do |js|\n          @parser_json.parse(js) do |_time, record|\n            yield nil, record\n          end\n        end\n      else\n        raise \"'json', 'ndjson' or 'msgpack' parameter is required\"\n      end\n    end\n\n    def parse_params_with_parser(params)\n      if content = params[EVENT_RECORD_PARAMETER]\n        @custom_parser.parse(content) do |time, record|\n          yield time, record\n        end\n      else\n        raise \"'#{EVENT_RECORD_PARAMETER}' parameter is required\"\n      end\n    end\n\n    def add_params_to_record(record, params)\n      if @add_http_headers\n        params.each_pair { |k, v|\n          if k.start_with?(\"HTTP_\".freeze)\n            record[k] = v\n          end\n        }\n      end\n\n      if @add_query_params\n        params.each_pair { |k, v|\n          if k.start_with?(\"QUERY_\".freeze)\n            record[k] = v\n          end\n        }\n      end\n\n      if @add_remote_addr\n        record['REMOTE_ADDR'] = params['REMOTE_ADDR']\n      end\n    end\n\n    def convert_time_field(record)\n      if t = @default_keep_time_key ? record[@parser_time_key] : record.delete(@parser_time_key)\n        if @default_time_parser\n          @default_time_parser.parse(t)\n        else\n          Fluent::EventTime.from_time(Time.at(t))\n        end\n      else\n        Fluent::EventTime.now\n      end\n    end\n\n    class Handler\n      attr_reader :content_type\n\n      def initialize(io, km, callback, body_size_limit, format_name, log,\n                     cors_allow_origins, cors_allow_credentials, add_query_params)\n        @io = io\n        @km = km\n        @callback = callback\n        @body_size_limit = body_size_limit\n        @next_close = false\n        @format_name = format_name\n        @log = log\n        @cors_allow_origins = cors_allow_origins\n        @cors_allow_credentials = cors_allow_credentials\n        @idle = 0\n        @add_query_params = add_query_params\n        @km.add(self)\n\n        @remote_port, @remote_addr = io.remote_port, io.remote_addr\n        @parser = Http::Parser.new(self)\n      end\n\n      def step_idle\n        @idle += 1\n      end\n\n      def on_close\n        @km.delete(self)\n      end\n\n      def on_read(data)\n        @idle = 0\n        @parser << data\n      rescue\n        @log.warn \"unexpected error\", error: $!.to_s\n        @log.warn_backtrace\n        @io.close\n      end\n\n      def on_message_begin\n        @body = ''\n      end\n\n      def on_headers_complete(headers)\n        expect = nil\n        size = nil\n\n        if @parser.http_version == [1, 1]\n          @keep_alive = true\n        else\n          @keep_alive = false\n        end\n        @env = {}\n        @content_type = \"\"\n        @content_encoding = \"\"\n        headers.each_pair {|k,v|\n          @env[\"HTTP_#{k.tr('-','_').upcase}\"] = v\n          case k\n          when /\\AExpect\\z/i\n            expect = v\n          when /\\AContent-Length\\Z/i\n            size = v.to_i\n          when /\\AContent-Type\\Z/i\n            @content_type = v\n          when /\\AContent-Encoding\\Z/i\n            @content_encoding = v\n          when /\\AConnection\\Z/i\n            if /close/i.match?(v)\n              @keep_alive = false\n            elsif /Keep-alive/i.match?(v)\n              @keep_alive = true\n            end\n          when /\\AOrigin\\Z/i\n            @origin  = v\n          when /\\AX-Forwarded-For\\Z/i\n            # For multiple X-Forwarded-For headers. Use first header value.\n            v = v.first if v.is_a?(Array)\n            @remote_addr = v.split(\",\").first\n          when /\\AAccess-Control-Request-Method\\Z/i\n            @access_control_request_method = v\n          when /\\AAccess-Control-Request-Headers\\Z/i\n            @access_control_request_headers = v\n          end\n        }\n        if expect\n          if expect == '100-continue'.freeze\n            if !size || size < @body_size_limit\n              send_response_nobody(\"100 Continue\", {})\n            else\n              send_response_and_close(\"413 Request Entity Too Large\", {}, \"Too large\")\n            end\n          else\n            send_response_and_close(\"417 Expectation Failed\", {}, \"\")\n          end\n        end\n      end\n\n      def on_body(chunk)\n        if @body.bytesize + chunk.bytesize > @body_size_limit\n          unless closing?\n            send_response_and_close(\"413 Request Entity Too Large\", {}, \"Too large\")\n          end\n          return\n        end\n        @body << chunk\n      end\n\n      RES_200_STATUS = \"200 OK\".freeze\n      RES_403_STATUS = \"403 Forbidden\".freeze\n\n      # Azure App Service sends GET requests for health checking purpose.\n      # Respond with `200 OK` to accommodate it.\n      def handle_get_request\n        return send_response_and_close(RES_200_STATUS, {}, \"\")\n      end\n\n      # Web browsers can send an OPTIONS request before performing POST\n      # to check if cross-origin requests are supported.\n      def handle_options_request\n        # Is CORS enabled in the first place?\n        if @cors_allow_origins.nil?\n          return send_response_and_close(RES_403_STATUS, {}, \"\")\n        end\n\n        # in_http does not support HTTP methods except POST\n        if @access_control_request_method != 'POST'\n          return send_response_and_close(RES_403_STATUS, {}, \"\")\n        end\n\n        header = {\n          \"Access-Control-Allow-Methods\" => \"POST\",\n          \"Access-Control-Allow-Headers\" => @access_control_request_headers || \"\",\n        }\n\n        # Check the origin and send back a CORS response\n        if @cors_allow_origins.include?('*')\n          header[\"Access-Control-Allow-Origin\"] = \"*\"\n          send_response_and_close(RES_200_STATUS, header, \"\")\n        elsif include_cors_allow_origin\n          header[\"Access-Control-Allow-Origin\"] = @origin\n          if @cors_allow_credentials\n            header[\"Access-Control-Allow-Credentials\"] = true\n          end\n          send_response_and_close(RES_200_STATUS, header, \"\")\n        else\n          send_response_and_close(RES_403_STATUS, {}, \"\")\n        end\n      end\n\n      def on_message_complete\n        return if closing?\n\n        if @parser.http_method == 'GET'.freeze\n          return handle_get_request()\n        end\n\n        if @parser.http_method == 'OPTIONS'.freeze\n          return handle_options_request()\n        end\n\n        # CORS check\n        # ==========\n        # For every incoming request, we check if we have some CORS\n        # restrictions and allow listed origins through @cors_allow_origins.\n        # If origin is empty, it's likely a server-to-server request and considered safe.\n        unless @cors_allow_origins.nil?\n          unless @cors_allow_origins.include?('*') || include_cors_allow_origin || @origin.nil?\n            send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, \"\")\n            return\n          end\n        end\n\n        # Content Encoding\n        # =================\n        # Decode payload according to the \"Content-Encoding\" header.\n        # For now, we only support 'gzip' and 'deflate'.\n        begin\n          if @content_encoding == 'gzip'.freeze\n            @body = Zlib::GzipReader.new(StringIO.new(@body)).read\n          elsif @content_encoding == 'deflate'.freeze\n            @body = Zlib::Inflate.inflate(@body)\n          end\n        rescue\n          @log.warn 'fails to decode payload', error: $!.to_s\n          send_response_and_close(RES_400_STATUS, {}, \"\")\n          return\n        end\n\n        @env['REMOTE_ADDR'] = @remote_addr if @remote_addr\n\n        uri = URI.parse(@parser.request_url)\n        params = parse_query(uri.query)\n\n        if @format_name != 'default'\n          params[EVENT_RECORD_PARAMETER] = @body\n        elsif /^application\\/x-www-form-urlencoded/.match?(@content_type)\n          params.update parse_query(@body)\n        elsif @content_type =~ /^multipart\\/form-data; boundary=(.+)/\n          boundary = WEBrick::HTTPUtils.dequote($1)\n          params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)\n        elsif /^application\\/json/.match?(@content_type)\n          params['json'] = @body\n        elsif /^application\\/csp-report/.match?(@content_type)\n          params['json'] = @body\n        elsif /^application\\/msgpack/.match?(@content_type)\n          params['msgpack'] = @body\n        elsif /^application\\/x-ndjson/.match?(@content_type)\n          params['ndjson'] = @body\n        end\n        path_info = uri.path\n\n        if (@add_query_params)\n\n          query_params = parse_query(uri.query)\n\n          query_params.each_pair {|k,v|\n            params[\"QUERY_#{k.tr('-','_').upcase}\"] = v\n          }\n        end\n\n        params.merge!(@env)\n\n        @env.clear\n\n        code, header, body = @callback.call(path_info, params)\n        body = body.to_s\n        header = header.dup if header.frozen?\n\n        unless @cors_allow_origins.nil?\n          if @cors_allow_origins.include?('*')\n            header['Access-Control-Allow-Origin'] = '*'\n          elsif include_cors_allow_origin\n            header['Access-Control-Allow-Origin'] = @origin\n            if @cors_allow_credentials\n              header[\"Access-Control-Allow-Credentials\"] = true\n            end\n          end\n        end\n\n        if @keep_alive\n          header['Connection'] = 'Keep-Alive'.freeze\n          send_response(code, header, body)\n        else\n          send_response_and_close(code, header, body)\n        end\n      end\n\n      def close\n        @io.close\n      end\n\n      def on_write_complete\n        @io.close if @next_close\n      end\n\n      def send_response_and_close(code, header, body)\n        send_response(code, header, body)\n        @next_close = true\n      end\n\n      def closing?\n        @next_close\n      end\n\n      def send_response(code, header, body)\n        header['Content-Length'] ||= body.bytesize\n        header['Content-Type'] ||= 'text/plain'.freeze\n\n        data = %[HTTP/1.1 #{code}\\r\\n]\n        header.each_pair {|k,v|\n          data << \"#{k}: #{v}\\r\\n\"\n        }\n        data << \"\\r\\n\".freeze\n        @io.write(data)\n\n        @io.write(body)\n      end\n\n      def send_response_nobody(code, header)\n        data = %[HTTP/1.1 #{code}\\r\\n]\n        header.each_pair {|k,v|\n          data << \"#{k}: #{v}\\r\\n\"\n        }\n        data << \"\\r\\n\".freeze\n        @io.write(data)\n      end\n\n      def include_cors_allow_origin\n        if @origin.nil?\n          return false\n        end\n\n        if @cors_allow_origins.include?(@origin)\n          return true\n        end\n        filtered_cors_allow_origins = @cors_allow_origins.select {|origin| origin != \"\"}\n        r = filtered_cors_allow_origins.find do |origin|\n          (start_str, end_str) = origin.split(\"*\", 2)\n          @origin.start_with?(start_str) && @origin.end_with?(end_str)\n        end\n\n        !r.nil?\n      end\n\n      def parse_query(query)\n        query.nil? ? {} : Hash[URI.decode_www_form(query, Encoding::ASCII_8BIT)]\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_monitor_agent.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'json'\n\nrequire 'fluent/config/types'\nrequire 'fluent/plugin/input'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/multi_output'\nrequire 'fluent/plugin/filter'\n\nmodule Fluent::Plugin\n  class MonitorAgentInput < Input\n    Fluent::Plugin.register_input('monitor_agent', self)\n\n    helpers :timer, :thread, :http_server\n\n    desc 'The address to bind to.'\n    config_param :bind, :string, default: '0.0.0.0'\n    desc 'The port to listen to.'\n    config_param :port, :integer, default: 24220\n    desc 'The tag with which internal metrics are emitted.'\n    config_param :tag, :string, default: nil\n    desc 'Determine the rate to emit internal metrics as events.'\n    config_param :emit_interval, :time, default: 60\n    desc 'Determine whether to include the config information.'\n    config_param :include_config, :bool, default: true\n    desc 'Determine whether to include the retry information.'\n    config_param :include_retry, :bool, default: true\n\n    class APIHandler\n      def initialize(agent)\n        @agent = agent\n      end\n\n      def plugins_ltsv(req)\n        list = build_object(build_option(req))\n\n        render_ltsv(list)\n      end\n\n      def plugins_json(req)\n        opts = build_option(req)\n        obj = build_object(opts)\n\n        render_json({ 'plugins' => obj }, pretty_json: opts[:pretty_json])\n      end\n\n      def config_ltsv(_req)\n        obj = {\n          'pid' => Process.pid,\n          'ppid' => Process.ppid,\n          'version' => Fluent::VERSION,\n        }.merge(@agent.fluentd_opts)\n\n        render_ltsv([obj])\n      end\n\n      def config_json(req)\n        obj = {\n          'pid' => Process.pid,\n          'ppid' => Process.ppid,\n          'version' => Fluent::VERSION,\n        }.merge(@agent.fluentd_opts)\n        opts = build_option(req)\n\n        render_json(obj, pretty_json: opts[:pretty_json])\n      end\n\n      private\n\n      def render_error_json(code:, msg:, pretty_json: nil, **additional_params)\n        resp = additional_params.merge('message' => msg)\n        render_json(resp, code: code, pretty_json: pretty_json)\n      end\n\n      def render_json(obj, code: 200, pretty_json: nil)\n        body =\n          if pretty_json\n            JSON.pretty_generate(obj)\n          else\n            obj.to_json\n          end\n\n        [code, { 'Content-Type' => 'application/json' }, body]\n      end\n\n      def render_ltsv(obj, code: 200)\n        normalized = JSON.parse(obj.to_json)\n        text = ''\n        normalized.each do |hash|\n          row = []\n          hash.each do |k, v|\n            if v.is_a?(Array)\n              row << \"#{k}:#{v.join(',')}\"\n            elsif v.is_a?(Hash)\n              next\n            else\n              row << \"#{k}:#{v}\"\n            end\n          end\n\n          text << row.join(\"\\t\") << \"\\n\"\n        end\n\n        [code, { 'Content-Type' => 'text/plain' }, text]\n      end\n\n      def build_object(opts)\n        qs = opts[:query]\n        if tag = qs['tag'.freeze].first\n          # ?tag= to search an output plugin by match pattern\n          if obj = @agent.plugin_info_by_tag(tag, opts)\n            list = [obj]\n          else\n            list = []\n          end\n        elsif plugin_id = (qs['@id'.freeze].first || qs['id'.freeze].first)\n          # ?@id= to search a plugin by 'id <plugin_id>' config param\n          if obj = @agent.plugin_info_by_id(plugin_id, opts)\n            list = [obj]\n          else\n            list = []\n          end\n        elsif plugin_type = (qs['@type'.freeze].first || qs['type'.freeze].first)\n          # ?@type= to search plugins by 'type <type>' config param\n          list = @agent.plugins_info_by_type(plugin_type, opts)\n        else\n          # otherwise show all plugins\n          list = @agent.plugins_info_all(opts)\n        end\n\n        list\n      end\n\n      def build_option(req)\n        qs = Hash.new { |_, _| [] }\n        # parse ?=query string\n        qs.merge!(req.query || {})\n\n        # if ?debug=1 is set, set :with_debug_info for get_monitor_info\n        # and :pretty_json for render_json_error\n        opts = { query: qs }\n        if qs['debug'.freeze].first\n          opts[:with_debug_info] = true\n          opts[:pretty_json] = true\n        end\n\n        if ivars = qs['with_ivars'.freeze].first\n          opts[:ivars] = ivars.split(',')\n        end\n\n        if with_config = qs['with_config'.freeze].first\n          opts[:with_config] = Fluent::Config.bool_value(with_config)\n        else\n          opts[:with_config] = @agent.include_config\n        end\n\n        if with_retry = qs['with_retry'.freeze].first\n          opts[:with_retry] = Fluent::Config.bool_value(with_retry)\n        else\n          opts[:with_retry] = @agent.include_retry\n        end\n\n        opts\n      end\n    end\n\n    def initialize\n      super\n\n      @first_warn = false\n    end\n\n    def configure(conf)\n      super\n      @port += fluentd_worker_id\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    class NotFoundJson\n      BODY = { 'message' => 'Not found' }.to_json\n      def self.call(_req)\n        [404, { 'Content-Type' => 'application/json' }, BODY]\n      end\n    end\n\n    def start\n      super\n\n      log.debug { \"listening monitoring http server on http://#{@bind}:#{@port}/api/plugins for worker#{fluentd_worker_id}\" }\n      api_handler = APIHandler.new(self)\n      http_server_create_http_server(:in_monitor_http_server_helper, addr: @bind, port: @port, logger: log, default_app: NotFoundJson) do |serv|\n        serv.get('/api/plugins') { |req| api_handler.plugins_ltsv(req) }\n        serv.get('/api/plugins.json') { |req| api_handler.plugins_json(req) }\n        serv.get('/api/config') { |req| api_handler.config_ltsv(req) }\n        serv.get('/api/config.json') { |req| api_handler.config_json(req) }\n      end\n\n      if @tag\n        log.debug { \"tag parameter is specified. Emit plugins info to '#{@tag}'\" }\n\n        opts = {with_config: false, with_retry: false}\n        timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {\n          es = Fluent::MultiEventStream.new\n          now = Fluent::EventTime.now\n          plugins_info_all(opts).each { |record|\n            es.add(now, record)\n          }\n          router.emit_stream(@tag, es)\n        }\n      end\n    end\n\n    # They are deprecated but remain for compatibility\n    MONITOR_INFO = {\n      'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },\n      'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },\n      'buffer_timekeys' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.timekeys },\n      'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size + @buffer.queue_size },\n      'retry_count' => ->(){ respond_to?(:num_errors) ? num_errors : nil },\n    }\n\n    def all_plugins\n      array = []\n\n      # get all input plugins\n      array.concat Fluent::Engine.root_agent.inputs\n\n      # get all output plugins\n      array.concat Fluent::Engine.root_agent.outputs\n\n      # get all filter plugins\n      array.concat Fluent::Engine.root_agent.filters\n\n      Fluent::Engine.root_agent.labels.each { |name, l|\n        # TODO: Add label name to outputs / filters for identifying plugins\n        array.concat l.outputs\n        array.concat l.filters\n      }\n\n      array\n    end\n\n    # try to match the tag and get the info from the matched output plugin\n    # TODO: Support output in label\n    def plugin_info_by_tag(tag, opts={})\n      matches = Fluent::Engine.root_agent.event_router.instance_variable_get(:@match_rules)\n      matches.each { |rule|\n        if rule.match?(tag)\n          if rule.collector.is_a?(Fluent::Plugin::Output) || rule.collector.is_a?(Fluent::Output)\n            return get_monitor_info(rule.collector, opts)\n          end\n        end\n      }\n      nil\n    end\n\n    # search a plugin by plugin_id\n    def plugin_info_by_id(plugin_id, opts={})\n      found = all_plugins.find {|pe|\n        pe.respond_to?(:plugin_id) && pe.plugin_id.to_s == plugin_id\n      }\n      if found\n        get_monitor_info(found, opts)\n      else\n        nil\n      end\n    end\n\n    # This method returns an array because\n    # multiple plugins could have the same type\n    def plugins_info_by_type(type, opts={})\n      array = all_plugins.select {|pe|\n        (pe.config['@type'] == type) rescue nil\n      }\n      array.map {|pe|\n        get_monitor_info(pe, opts)\n      }\n    end\n\n    def plugins_info_all(opts={})\n      all_plugins.map {|pe|\n        get_monitor_info(pe, opts)\n      }\n    end\n\n    IGNORE_ATTRIBUTES = %i(@config_root_section @config @masked_config)\n\n    # get monitor info from the plugin `pe` and return a hash object\n    def get_monitor_info(pe, opts={})\n      obj = {}\n\n      # Common plugin information\n      obj['plugin_id'] = pe.plugin_id\n      obj['plugin_category'] = plugin_category(pe)\n      obj['type'] = pe.config['@type']\n      obj['config'] = pe.config if opts[:with_config]\n\n      # run MONITOR_INFO in plugins' instance context and store the info to obj\n      MONITOR_INFO.each_pair {|key,code|\n        begin\n          catch(:skip) do\n            obj[key] = pe.instance_exec(&code)\n          end\n        rescue NoMethodError => e\n          unless @first_warn\n            log.error \"NoMethodError in monitoring plugins\", key: key, plugin: pe.class, error: e\n            log.error_backtrace\n            @first_warn = true\n          end\n        rescue => e\n          log.warn \"unexpected error in monitoring plugins\", key: key, plugin: pe.class, error: e\n        end\n      }\n\n      if pe.respond_to?(:statistics)\n        obj.merge!(pe.statistics.dig('output') || {})\n        obj.merge!(pe.statistics.dig('filter') || {})\n        obj.merge!(pe.statistics.dig('input') || {})\n      end\n\n      obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)\n\n      # include all instance variables if :with_debug_info is set\n      if opts[:with_debug_info]\n        iv = {}\n        pe.instance_eval do\n          instance_variables.each {|sym|\n            next if IGNORE_ATTRIBUTES.include?(sym)\n            key = sym.to_s[1..-1]  # removes first '@'\n            iv[key] = instance_variable_get(sym)\n          }\n        end\n        obj['instance_variables'] = iv\n      elsif ivars = opts[:ivars]\n        iv = {}\n        ivars.each {|name|\n          iname = \"@#{name}\"\n          iv[name] = pe.instance_variable_get(iname) if pe.instance_variable_defined?(iname)\n        }\n        obj['instance_variables'] = iv\n      end\n\n      obj\n    end\n\n    RETRY_INFO = {\n        'start' => '@start',\n        'steps' => '@steps',\n        'next_time' => '@next_time',\n    }\n\n    def get_retry_info(pe_retry)\n      retry_variables = {}\n\n      if pe_retry\n        RETRY_INFO.each_pair { |key, param|\n          retry_variables[key] = pe_retry.instance_variable_get(param)\n        }\n      end\n\n      retry_variables\n    end\n\n    def plugin_category(pe)\n      case pe\n      when Fluent::Plugin::Input\n        'input'.freeze\n      when Fluent::Plugin::Output, Fluent::Plugin::MultiOutput, Fluent::Plugin::BareOutput\n        'output'.freeze\n      when Fluent::Plugin::Filter\n        'filter'.freeze\n      else\n        'unknown'.freeze\n      end\n    end\n\n    def fluentd_opts\n      @fluentd_opts ||= get_fluentd_opts\n    end\n\n    def get_fluentd_opts\n      opts = {}\n      ObjectSpace.each_object(Fluent::Supervisor) { |obj|\n        opts.merge!(obj.options)\n        break\n      }\n      opts\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_object_space.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'json'\n\nrequire 'fluent/plugin/input'\n\nmodule Fluent::Plugin\n  class ObjectSpaceInput < Fluent::Plugin::Input\n    Fluent::Plugin.register_input('object_space', self)\n\n    helpers :timer\n\n    def initialize\n      super\n    end\n\n    config_param :emit_interval, :time, default: 60\n    config_param :tag, :string\n    config_param :top, :integer, default: 15\n\n    def multi_workers_ready?\n      true\n    end\n\n    def start\n      super\n\n      timer_execute(:object_space_input, @emit_interval, &method(:on_timer))\n    end\n\n    class Counter\n      def initialize(klass, init_count)\n        @klass = klass\n        @count = init_count\n      end\n\n      def incr!\n        @count += 1\n      end\n\n      def name\n        @klass.name\n      end\n\n      attr_reader :count\n    end\n\n    def on_timer\n      now = Fluent::EventTime.now\n\n      array = []\n      map = {}\n\n      ObjectSpace.each_object {|obj|\n        klass = obj.class rescue Object\n        if c = map[klass]\n          c.incr!\n        else\n          c = Counter.new(klass, 1)\n          array << c\n          map[klass] = c\n        end\n      }\n\n      array.sort_by! {|c| -c.count }\n\n      record = {}\n      array.each_with_index {|c,i|\n        break if i >= @top\n        record[c.name] = c.count\n      }\n\n      router.emit(@tag, now, record)\n    rescue => e\n      log.error \"object space failed to emit\", error: e, tag: @tag, record: JSON.generate(record)\n      log.error_backtrace\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_sample.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'json'\n\nrequire 'fluent/plugin/input'\nrequire 'fluent/config/error'\n\nmodule Fluent::Plugin\n  class SampleInput < Input\n    Fluent::Plugin.register_input('sample', self)\n    Fluent::Plugin.register_input('dummy', self)\n\n    helpers :thread, :storage\n\n    BIN_NUM = 10\n    DEFAULT_STORAGE_TYPE = 'local'\n\n    desc \"The value is the tag assigned to the generated events.\"\n    config_param :tag, :string\n    desc \"The number of events in event stream of each emits.\"\n    config_param :size, :integer, default: 1\n    desc \"It configures how many events to generate per second.\"\n    config_param :rate, :integer, default: 1\n    desc \"If specified, each generated event has an auto-incremented key field.\"\n    config_param :auto_increment_key, :string, default: nil\n    desc \"The boolean to suspend-and-resume incremental value after restart\"\n    config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'\n    desc \"Reuse the sample data to reduce the load when sending large amounts of data. You can enable it if filter does not do destructive change\"\n    config_param :reuse_record, :bool, default: false\n    desc \"The sample data to be generated. An array of JSON hashes or a single JSON hash.\"\n    config_param :sample, alias: :dummy, default: [{\"message\" => \"sample\"}] do |val|\n      begin\n        parsed = JSON.parse(val)\n      rescue JSON::ParserError => ex\n        # Fluent::ConfigParseError, \"got incomplete JSON\" will be raised\n        # at literal_parser.rb with --use-v1-config, but I had to\n        # take care at here for the case of --use-v0-config.\n        raise Fluent::ConfigError, \"#{ex.class}: #{ex.message}\"\n      end\n      sample = parsed.is_a?(Array) ? parsed : [parsed]\n      sample.each_with_index do |e, i|\n        raise Fluent::ConfigError, \"#{i}th element of sample, #{e}, is not a hash\" unless e.is_a?(Hash)\n      end\n      sample\n    end\n\n    def initialize\n      super\n      @storage = nil\n    end\n\n    def configure(conf)\n      super\n      @sample_index = 0\n      config = conf.elements.find{|e| e.name == 'storage' }\n      @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def start\n      super\n\n      @storage.put(:increment_value, 0) unless @storage.get(:increment_value)\n      # keep 'dummy' to avoid breaking changes for existing environment. Change it in fluentd v2\n      @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)\n\n      if @auto_increment_key && !@storage.get(:auto_increment_value)\n        @storage.put(:auto_increment_value, -1)\n      end\n\n      thread_create(:sample_input, &method(:run))\n    end\n\n    def run\n      batch_num    = (@rate / BIN_NUM).to_i\n      residual_num = (@rate % BIN_NUM)\n      while thread_current_running?\n        current_time = Time.now.to_i\n        BIN_NUM.times do\n          break unless (thread_current_running? && Time.now.to_i <= current_time)\n          wait(0.1) { emit(batch_num) }\n        end\n        emit(residual_num) if thread_current_running?\n        # wait for next second\n        while thread_current_running? && Time.now.to_i <= current_time\n          sleep 0.01\n        end\n      end\n    end\n\n    def emit(num)\n      begin\n        if @size > 1\n          num.times do\n            router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })\n          end\n        else\n          num.times { router.emit(@tag, Fluent::EventTime.now, generate) }\n        end\n      rescue => _\n        # ignore all errors not to stop emits by emit errors\n      end\n    end\n\n    def next_sample\n      d = @reuse_record ? @sample[@sample_index] : @sample[@sample_index].dup\n      @sample_index += 1\n      return d if d\n\n      @sample_index = 0\n      next_sample\n    end\n\n    def generate\n      d = next_sample\n      if @auto_increment_key\n        d = d.dup if @reuse_record\n        d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }\n      end\n      d\n    end\n\n    def wait(time)\n      start_time = Time.now\n      yield\n      sleep_time = time - (Time.now - start_time)\n      sleep sleep_time if sleep_time > 0\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_syslog.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/input'\nrequire 'fluent/config/error'\nrequire 'fluent/plugin/parser'\n\nrequire 'json'\n\nmodule Fluent::Plugin\n  class SyslogInput < Input\n    Fluent::Plugin.register_input('syslog', self)\n\n    helpers :parser, :compat_parameters, :server\n\n    DEFAULT_PARSER = 'syslog'\n    SYSLOG_REGEXP = /^\\<([0-9]+)\\>(.*)/\n\n    FACILITY_MAP = {\n      0   => 'kern',\n      1   => 'user',\n      2   => 'mail',\n      3   => 'daemon',\n      4   => 'auth',\n      5   => 'syslog',\n      6   => 'lpr',\n      7   => 'news',\n      8   => 'uucp',\n      9   => 'cron',\n      10  => 'authpriv',\n      11  => 'ftp',\n      12  => 'ntp',\n      13  => 'audit',\n      14  => 'alert',\n      15  => 'at',\n      16  => 'local0',\n      17  => 'local1',\n      18  => 'local2',\n      19  => 'local3',\n      20  => 'local4',\n      21  => 'local5',\n      22  => 'local6',\n      23  => 'local7'\n    }\n\n    SEVERITY_MAP = {\n      0  => 'emerg',\n      1  => 'alert',\n      2  => 'crit',\n      3  => 'err',\n      4  => 'warn',\n      5  => 'notice',\n      6  => 'info',\n      7  => 'debug'\n    }\n\n    desc 'The port to listen to.'\n    config_param :port, :integer, default: 5140\n    desc 'The bind address to listen to.'\n    config_param :bind, :string, default: '0.0.0.0'\n    desc 'The prefix of the tag. The tag itself is generated by the tag prefix, facility level, and priority.'\n    config_param :tag, :string\n    desc 'The transport protocol used to receive logs.(udp, tcp)'\n    config_param :protocol_type, :enum, list: [:tcp, :udp], default: nil, deprecated: \"use transport directive\"\n    desc 'The message frame type.(traditional, octet_count)'\n    config_param :frame_type, :enum, list: [:traditional, :octet_count], default: :traditional\n\n    desc 'If true, add source host to event record.'\n    config_param :include_source_host, :bool, default: false, deprecated: 'use \"source_hostname_key\" or \"source_address_key\" instead.'\n    desc 'Specify key of source host when include_source_host is true.'\n    config_param :source_host_key, :string, default: 'source_host'.freeze\n    desc 'Enable the option to emit unmatched lines.'\n    config_param :emit_unmatched_lines, :bool, default: false\n\n    desc 'The field name of hostname of sender.'\n    config_param :source_hostname_key, :string, default: nil\n    desc 'Try to resolve hostname from IP addresses or not.'\n    config_param :resolve_hostname, :bool, default: nil\n    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'\n    config_param :send_keepalive_packet, :bool, default: false\n    desc 'The field name of source address of sender.'\n    config_param :source_address_key, :string, default: nil\n    desc 'The field name of the severity.'\n    config_param :severity_key, :string, default: nil, alias: :priority_key\n    desc 'The field name of the facility.'\n    config_param :facility_key, :string, default: nil\n\n    desc \"The max bytes of message\"\n    config_param :message_length_limit, :size, default: 2048\n\n    config_param :blocking_timeout, :time, default: 0.5\n\n    desc 'The delimiter value \"\\n\"'\n    config_param :delimiter, :string, default: \"\\n\" # syslog family add \"\\n\" to each message\n\n    config_section :parse do\n      config_set_default :@type, DEFAULT_PARSER\n      config_param :with_priority, :bool, default: true\n    end\n\n    # overwrite server plugin to change default to :udp\n    config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do\n      config_argument :protocol, :enum, list: [:tcp, :udp, :tls], default: :udp\n    end\n\n    def configure(conf)\n      compat_parameters_convert(conf, :parser)\n\n      super\n\n      if conf.has_key?('priority_key')\n        log.warn \"priority_key is deprecated. Use severity_key instead\"\n      end\n\n      @use_default = false\n\n      @parser = parser_create\n      @parser_parse_priority = @parser.respond_to?(:with_priority) && @parser.with_priority\n\n      if @include_source_host\n        if @source_address_key\n          raise Fluent::ConfigError, \"specify either source_address_key or include_source_host\"\n        end\n        @source_address_key = @source_host_key\n      end\n      if @source_hostname_key\n        if @resolve_hostname.nil?\n          @resolve_hostname = true\n        elsif !@resolve_hostname # user specifies \"false\" in config\n          raise Fluent::ConfigError, \"resolve_hostname must be true with source_hostname_key\"\n        end\n      end\n\n      @_event_loop_run_timeout = @blocking_timeout\n\n      protocol = @protocol_type || @transport_config.protocol\n      if @send_keepalive_packet && protocol == :udp\n        raise Fluent::ConfigError, \"send_keepalive_packet is available for tcp/tls\"\n      end\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def zero_downtime_restart_ready?\n      true\n    end\n\n    def start\n      super\n\n      log.info \"listening syslog socket on #{@bind}:#{@port} with #{@protocol_type || @transport_config.protocol}\"\n      case @protocol_type || @transport_config.protocol\n      when :udp then start_udp_server\n      when :tcp then start_tcp_server\n      when :tls then start_tcp_server(tls: true)\n      else\n        raise \"BUG: invalid transport value: #{@protocol_type || @transport_config.protocol}\"\n      end\n    end\n\n    def start_udp_server\n      server_create_udp(:in_syslog_udp_server, @port, bind: @bind, max_bytes: @message_length_limit, resolve_name: @resolve_hostname) do |data, sock|\n        message_handler(data.chomp, sock)\n      end\n    end\n\n    def start_tcp_server(tls: false)\n      octet_count_frame = @frame_type == :octet_count\n\n      delimiter = octet_count_frame ? \" \" : @delimiter\n      delimiter_size = delimiter.size\n      server_create_connection(\n        tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port,\n        bind: @bind,\n        resolve_name: @resolve_hostname,\n        send_keepalive_packet: @send_keepalive_packet\n      ) do |conn|\n        conn.data do |data|\n          buffer = conn.buffer\n          buffer << data\n          pos = 0\n          if octet_count_frame\n            while idx = buffer.index(delimiter, pos)\n              num = Integer(buffer[pos..idx])\n              msg = buffer[idx + delimiter_size, num]\n              if msg.size != num\n                break\n              end\n\n              pos = idx + delimiter_size + num\n              message_handler(msg, conn)\n            end\n          else\n            while idx = buffer.index(delimiter, pos)\n              msg = buffer[pos...idx]\n              pos = idx + delimiter_size\n              message_handler(msg, conn)\n            end\n          end\n          buffer.slice!(0, pos) if pos > 0\n        end\n      end\n    end\n\n    private\n\n    def emit_unmatched(data, sock)\n      record = {\"unmatched_line\" => data}\n      record[@source_address_key] = sock.remote_addr if @source_address_key\n      record[@source_hostname_key] = sock.remote_host if @source_hostname_key\n      emit(\"#{@tag}.unmatched\", Fluent::EventTime.now, record)\n    end\n\n    def message_handler(data, sock)\n      pri = nil\n      text = data\n      unless @parser_parse_priority\n        m = SYSLOG_REGEXP.match(data)\n        unless m\n          if @emit_unmatched_lines\n            emit_unmatched(data, sock)\n          end\n          log.warn \"invalid syslog message: #{data.dump}\"\n          return\n        end\n        pri = m[1].to_i\n        text = m[2]\n      end\n\n      @parser.parse(text) do |time, record|\n        unless time && record\n          if @emit_unmatched_lines\n            emit_unmatched(data, sock)\n          end\n          log.warn \"failed to parse message\", data: data\n          return\n        end\n\n        pri ||= record.delete('pri')\n        facility = FACILITY_MAP[pri >> 3]\n        severity = SEVERITY_MAP[pri & 0b111]\n\n        record[@severity_key] = severity if @severity_key\n        record[@facility_key] = facility if @facility_key\n        record[@source_address_key] = sock.remote_addr if @source_address_key\n        record[@source_hostname_key] = sock.remote_host if @source_hostname_key\n\n        tag = \"#{@tag}.#{facility}.#{severity}\"\n        emit(tag, time, record)\n      end\n    rescue => e\n      if @emit_unmatched_lines\n        emit_unmatched(data, sock)\n      end\n      log.error \"invalid input\", data: data, error: e\n      log.error_backtrace\n    end\n\n    def emit(tag, time, record)\n      router.emit(tag, time, record)\n    rescue => e\n      log.error \"syslog failed to emit\", error: e, tag: tag, record: JSON.generate(record)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_tail/group_watch.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/input'\n\nmodule Fluent::Plugin\n  class TailInput < Fluent::Plugin::Input\n    module GroupWatchParams\n      include Fluent::Configurable\n\n      DEFAULT_KEY = /.*/\n      DEFAULT_LIMIT = -1\n      REGEXP_JOIN = \"_\"\n\n      config_section :group, param_name: :group, required: false, multi: false do\n        desc 'Regex for extracting group\\'s metadata'\n        config_param :pattern,\n                     :regexp,\n                     default: /^\\/var\\/log\\/containers\\/(?<podname>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\/[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container>.+)-(?<docker_id>[a-z0-9]{64})\\.log$/\n\n        desc 'Period of time in which the group_line_limit is applied'\n        config_param :rate_period, :time, default: 5\n\n        config_section :rule, param_name: :rule, required: true, multi: true do\n          desc 'Key-value pairs for grouping'\n          config_param :match, :hash, value_type: :regexp, default: { namespace: [DEFAULT_KEY], podname: [DEFAULT_KEY] }\n          desc 'Maximum number of log lines allowed per group over a period of rate_period'\n          config_param :limit, :integer, default: DEFAULT_LIMIT\n        end\n      end\n    end\n\n    module GroupWatch\n      def self.included(mod)\n        mod.include GroupWatchParams\n      end\n\n      attr_reader :group_watchers, :default_group_key\n\n      def initialize\n        super\n        @group_watchers = {}\n        @group_keys = nil\n        @default_group_key = nil\n      end\n\n      def configure(conf)\n        super\n\n        unless @group.nil?\n          ## Ensuring correct time period syntax\n          @group.rule.each { |rule|\n            raise \"Metadata Group Limit >= DEFAULT_LIMIT\" unless rule.limit >= GroupWatchParams::DEFAULT_LIMIT\n          }\n\n          @group_keys = Regexp.compile(@group.pattern).named_captures.keys\n          @default_group_key = ([GroupWatchParams::DEFAULT_KEY] * @group_keys.length).join(GroupWatchParams::REGEXP_JOIN)\n\n          ## Ensures that \"specific\" rules (with larger number of `rule.match` keys)\n          ## have a higher priority against \"generic\" rules (with less number of `rule.match` keys).\n          ## This will be helpful when a file satisfies more than one rule.\n          @group.rule.sort_by! { |rule| -rule.match.length() }\n          construct_groupwatchers\n          @group_watchers[@default_group_key] ||= GroupWatcher.new(@group.rate_period, GroupWatchParams::DEFAULT_LIMIT)\n        end\n      end\n\n      def add_path_to_group_watcher(path)\n        return nil if @group.nil?\n        group_watcher = find_group_from_metadata(path)\n        group_watcher.add(path) unless group_watcher.include?(path)\n        group_watcher\n      end\n\n      def remove_path_from_group_watcher(path)\n        return if @group.nil?\n        group_watcher = find_group_from_metadata(path)\n        group_watcher.delete(path)\n      end\n\n      def construct_group_key(named_captures)\n        match_rule = []\n        @group_keys.each { |key|\n          match_rule.append(named_captures.fetch(key, GroupWatchParams::DEFAULT_KEY))\n        }\n        match_rule = match_rule.join(GroupWatchParams::REGEXP_JOIN)\n\n        match_rule\n      end\n\n      def construct_groupwatchers\n        @group.rule.each { |rule|\n          match_rule = construct_group_key(rule.match)\n          @group_watchers[match_rule] ||= GroupWatcher.new(@group.rate_period, rule.limit)\n        }\n      end\n\n      def find_group(metadata)\n        metadata_key = construct_group_key(metadata)\n        gw_key = @group_watchers.keys.find { |regexp| metadata_key.match?(regexp) && regexp != @default_group_key }\n        gw_key ||= @default_group_key\n\n        @group_watchers[gw_key]\n      end\n\n      def find_group_from_metadata(path)\n        begin\n          metadata = @group.pattern.match(path).named_captures\n          group_watcher = find_group(metadata)\n        rescue\n          log.warn \"Cannot find group from metadata, Adding file in the default group\"\n          group_watcher = @group_watchers[@default_group_key]\n        end\n\n        group_watcher\n      end\n    end\n\n    class GroupWatcher\n      attr_accessor :current_paths, :limit, :number_lines_read, :start_reading_time, :rate_period\n\n      FileCounter = Struct.new(\n        :number_lines_read,\n        :start_reading_time,\n      )\n\n      def initialize(rate_period = 60, limit = -1)\n        @current_paths = {}\n        @rate_period = rate_period\n        @limit = limit\n      end\n\n      def add(path)\n        @current_paths[path] = FileCounter.new(0, nil)\n      end\n\n      def include?(path)\n        @current_paths.key?(path)\n      end\n\n      def size\n        @current_paths.size\n      end\n\n      def delete(path)\n        @current_paths.delete(path)\n      end\n\n      def update_reading_time(path)\n        @current_paths[path].start_reading_time ||= Fluent::Clock.now\n      end\n\n      def update_lines_read(path, value)\n        @current_paths[path].number_lines_read += value\n      end\n\n      def reset_counter(path)\n        @current_paths[path].start_reading_time = nil\n        @current_paths[path].number_lines_read = 0\n      end\n\n      def time_spent_reading(path)\n        Fluent::Clock.now - @current_paths[path].start_reading_time\n      end\n\n      def limit_time_period_reached?(path)\n        time_spent_reading(path) < @rate_period\n      end\n\n      def limit_lines_reached?(path)\n        return true unless include?(path)\n        return true if @limit == 0\n\n        return false if @limit < 0\n        return false if @current_paths[path].number_lines_read < @limit / size\n\n        # update_reading_time(path)\n        if limit_time_period_reached?(path) # Exceeds limit\n          true\n        else # Does not exceed limit\n          reset_counter(path)\n          false\n        end\n      end\n\n      def to_s\n        super + \" current_paths: #{@current_paths} rate_period: #{@rate_period} limit: #{@limit}\"\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_tail/position_file.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/input'\n\nmodule Fluent::Plugin\n  class TailInput < Fluent::Plugin::Input\n    class PositionFile\n      UNWATCHED_POSITION = 0xffffffffffffffff\n      POSITION_FILE_ENTRY_REGEX = /^([^\\t]+)\\t([0-9a-fA-F]+)\\t([0-9a-fA-F]+)/.freeze\n\n      def self.load(file, follow_inodes, existing_targets, logger:)\n        pf = new(file, follow_inodes, logger: logger)\n        pf.load(existing_targets)\n        pf\n      end\n\n      def initialize(file, follow_inodes, logger: nil)\n        @file = file\n        @logger = logger\n        @file_mutex = Mutex.new\n        @map = {}\n        @follow_inodes = follow_inodes\n      end\n\n      def [](target_info)\n        if m = @map[@follow_inodes ? target_info.ino : target_info.path]\n          return m\n        end\n\n        @file_mutex.synchronize {\n          @file.seek(0, IO::SEEK_END)\n          seek = @file.pos + target_info.path.bytesize + 1\n          @file.write \"#{target_info.path}\\t0000000000000000\\t0000000000000000\\n\"\n          if @follow_inodes\n            @map[target_info.ino] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)\n          else\n            @map[target_info.path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)\n          end\n        }\n      end\n\n      def unwatch_removed_targets(existing_targets)\n        @map.reject { |key, entry|\n          existing_targets.key?(key)\n        }.each_key { |key|\n          unwatch_key(key)\n        }\n      end\n\n      def unwatch(target_info)\n        unwatch_key(@follow_inodes ? target_info.ino : target_info.path)\n      end\n\n      def load(existing_targets = nil)\n        compact(existing_targets)\n\n        map = {}\n        @file_mutex.synchronize do\n          @file.pos = 0\n\n          @file.each_line do |line|\n            m = POSITION_FILE_ENTRY_REGEX.match(line)\n            next if m.nil?\n\n            path = m[1]\n            pos = m[2].to_i(16)\n            ino = m[3].to_i(16)\n            seek = @file.pos - line.bytesize + path.bytesize + 1\n            if @follow_inodes\n              map[ino] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)\n            else\n              map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)\n            end\n          end\n        end\n\n        @map = map\n      end\n\n      # This method is similar to #compact but it tries to get less lock to avoid a lock contention\n      def try_compact\n        last_modified = nil\n        size = nil\n\n        @file_mutex.synchronize do\n          size = @file.size\n          last_modified = @file.mtime\n        end\n\n        entries = fetch_compacted_entries\n        @logger&.debug \"Compacted entries: \", entries.keys\n\n        @file_mutex.synchronize do\n          if last_modified == @file.mtime && size == @file.size\n            @file.pos = 0\n            @file.truncate(0)\n            @file.write(entries.values.map(&:to_entry_fmt).join)\n\n            # entry contains path/ino key and value.\n            entries.each do |key, val|\n              if (m = @map[key])\n                m.seek = val.seek\n              end\n            end\n          else\n            # skip\n          end\n        end\n      end\n\n      private\n\n      def unwatch_key(key)\n        if (entry = @map.delete(key))\n          entry.update_pos(UNWATCHED_POSITION)\n        end\n      end\n\n      def compact(existing_targets = nil)\n        @file_mutex.synchronize do\n          entries = fetch_compacted_entries\n          @logger&.debug \"Compacted entries: \", entries.keys\n\n          if existing_targets\n            entries = remove_deleted_files_entries(entries, existing_targets)\n            @logger&.debug \"Remove missing entries.\",\n                           existing_targets: existing_targets.keys,\n                           entries_after_removing: entries.keys\n          end\n\n          @file.pos = 0\n          @file.truncate(0)\n          @file.write(entries.values.map(&:to_entry_fmt).join)\n        end\n      end\n\n      def fetch_compacted_entries\n        entries = {}\n\n        @file.pos = 0\n        file_pos = 0\n        @file.each_line do |line|\n          m = POSITION_FILE_ENTRY_REGEX.match(line)\n          if m.nil?\n            @logger.warn \"Unparsable line in pos_file: #{line}\" if @logger\n            next\n          end\n\n          path = m[1]\n          pos = m[2].to_i(16)\n          ino = m[3].to_i(16)\n          if pos == UNWATCHED_POSITION\n            @logger.debug \"Remove unwatched line from pos_file: #{line}\" if @logger\n          else\n            if @follow_inodes\n              @logger&.warn(\"#{path} (inode: #{ino}) already exists. use latest one: deleted #{entries[ino]}\") if entries.include?(ino)\n              entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)\n            else\n              @logger&.warn(\"#{path} already exists. use latest one: deleted #{entries[path]}\") if entries.include?(path)\n              entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)\n            end\n            file_pos += line.size\n          end\n        end\n\n        entries\n      end\n\n      def remove_deleted_files_entries(existent_entries, existing_targets)\n        existent_entries.select { |path_or_ino|\n          existing_targets.key?(path_or_ino)\n        }\n      end\n    end\n\n    Entry = Struct.new(:path, :pos, :ino, :seek) do\n      POSITION_FILE_ENTRY_FORMAT = \"%s\\t%016x\\t%016x\\n\".freeze\n\n      def to_entry_fmt\n        POSITION_FILE_ENTRY_FORMAT % [path, pos, ino]\n      end\n    end\n\n    # pos               inode\n    # ffffffffffffffff\\tffffffffffffffff\\n\n    class FilePositionEntry\n      POS_SIZE = 16\n      INO_OFFSET = 17\n      INO_SIZE = 16\n      LN_OFFSET = 33\n      SIZE = 34\n\n      def initialize(file, file_mutex, seek, pos, inode)\n        @file = file\n        @file_mutex = file_mutex\n        @seek = seek\n        @pos = pos\n        @inode = inode\n      end\n\n      attr_writer :seek\n\n      def update(ino, pos)\n        @file_mutex.synchronize {\n          @file.pos = @seek\n          @file.write \"%016x\\t%016x\" % [pos, ino]\n        }\n        @pos = pos\n        @inode = ino\n      end\n\n      def update_pos(pos)\n        @file_mutex.synchronize {\n          @file.pos = @seek\n          @file.write \"%016x\" % pos\n        }\n        @pos = pos\n      end\n\n      def read_inode\n        @inode\n      end\n\n      def read_pos\n        @pos\n      end\n    end\n\n    class MemoryPositionEntry\n      def initialize\n        @pos = 0\n        @inode = 0\n      end\n\n      def update(ino, pos)\n        @inode = ino\n        @pos = pos\n      end\n\n      def update_pos(pos)\n        @pos = pos\n      end\n\n      def read_pos\n        @pos\n      end\n\n      def read_inode\n        @inode\n      end\n    end\n\n    TargetInfo = Struct.new(:path, :ino)\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_tail.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'cool.io'\n\nrequire 'fluent/plugin/input'\nrequire 'fluent/config/error'\nrequire 'fluent/event'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin/parser_multiline'\nrequire 'fluent/variable_store'\nrequire 'fluent/capability'\nrequire 'fluent/plugin/in_tail/position_file'\nrequire 'fluent/plugin/in_tail/group_watch'\nrequire 'fluent/file_wrapper'\n\nmodule Fluent::Plugin\n  class TailInput < Fluent::Plugin::Input\n    include GroupWatch\n\n    Fluent::Plugin.register_input('tail', self)\n\n    helpers :timer, :event_loop, :parser, :compat_parameters\n\n    RESERVED_CHARS = ['/', '*', '%'].freeze\n    MetricsInfo = Struct.new(:opened, :closed, :rotated, :throttled, :tracked)\n\n    class WatcherSetupError < StandardError\n      def initialize(msg)\n        @message = msg\n      end\n\n      def to_s\n        @message\n      end\n    end\n\n    def initialize\n      super\n      @paths = []\n      @tails = {}\n      @tails_rotate_wait = {}\n      @pf_file = nil\n      @pf = nil\n      @ignore_list = []\n      @shutdown_start_time = nil\n      @metrics = nil\n      @startup = true\n      @capability = Fluent::Capability.new(:current_process)\n    end\n\n    desc 'The paths to read. Multiple paths can be specified, separated by comma.'\n    config_param :path, :string\n    desc 'path delimiter used for splitting path config'\n    config_param :path_delimiter, :string, default: ','\n    desc 'Choose using glob patterns. Adding capabilities to handle [] and ?, and {}.'\n    config_param :glob_policy, :enum, list: [:backward_compatible, :extended, :always], default: :backward_compatible\n    desc 'The tag of the event.'\n    config_param :tag, :string\n    desc 'The paths to exclude the files from watcher list.'\n    config_param :exclude_path, :array, default: []\n    desc 'Specify interval to keep reference to old file when rotate a file.'\n    config_param :rotate_wait, :time, default: 5\n    desc 'Fluentd will record the position it last read into this file.'\n    config_param :pos_file, :string, default: nil\n    desc 'The cleanup interval of pos file'\n    config_param :pos_file_compaction_interval, :time, default: nil\n    desc 'Start to read the logs from the head of file, not bottom.'\n    config_param :read_from_head, :bool, default: false\n    # When the program deletes log file and re-creates log file with same filename after passed refresh_interval,\n    # in_tail may raise a pos_file related error. This is a known issue but there is no such program on production.\n    # If we find such program / application, we will fix the problem.\n    desc 'The interval of refreshing the list of watch file.'\n    config_param :refresh_interval, :time, default: 60\n    desc 'The number of reading lines at each IO.'\n    config_param :read_lines_limit, :integer, default: 1000\n    desc 'The number of reading bytes per second'\n    config_param :read_bytes_limit_per_second, :size, default: -1\n    desc 'The interval of flushing the buffer for multiline format'\n    config_param :multiline_flush_interval, :time, default: nil\n    desc 'Enable the option to emit unmatched lines.'\n    config_param :emit_unmatched_lines, :bool, default: false\n    desc 'Enable the additional watch timer.'\n    config_param :enable_watch_timer, :bool, default: true\n    desc 'Enable the stat watcher based on inotify.'\n    config_param :enable_stat_watcher, :bool, default: true\n    desc 'The encoding of the input.'\n    config_param :encoding, :string, default: nil\n    desc \"The original encoding of the input. If set, in_tail tries to encode string from this to 'encoding'. Must be set with 'encoding'. \"\n    config_param :from_encoding, :string, default: nil\n    desc 'Add the log path being tailed to records. Specify the field name to be used.'\n    config_param :path_key, :string, default: nil\n    desc 'Open and close the file on every update instead of leaving it open until it gets rotated.'\n    config_param :open_on_every_update, :bool, default: false\n    desc 'Limit the watching files that the modification time is within the specified time range (when use \\'*\\' in path).'\n    config_param :limit_recently_modified, :time, default: nil\n    desc 'Enable the option to skip the refresh of watching list on startup.'\n    config_param :skip_refresh_on_startup, :bool, default: false\n    desc 'Ignore repeated permission error logs'\n    config_param :ignore_repeated_permission_error, :bool, default: false\n    desc 'Format path with the specified timezone'\n    config_param :path_timezone, :string, default: nil\n    desc 'Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files'\n    config_param :follow_inodes, :bool, default: false\n    desc 'Maximum length of line. The longer line is just skipped.'\n    config_param :max_line_size, :size, default: nil\n\n    config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do\n      config_argument :usage, :string, default: 'in_tail_parser'\n    end\n\n    attr_reader :paths\n\n    def configure(conf)\n      @variable_store = Fluent::VariableStore.fetch_or_build(:in_tail)\n      compat_parameters_convert(conf, :parser)\n      parser_config = conf.elements('parse').first\n      unless parser_config\n        raise Fluent::ConfigError, \"<parse> section is required.\"\n      end\n\n      (1..Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM).each do |n|\n        parser_config[\"format#{n}\"] = conf[\"format#{n}\"] if conf[\"format#{n}\"]\n      end\n\n      parser_config['unmatched_lines'] = conf['emit_unmatched_lines']\n\n      super\n\n      if !@enable_watch_timer && !@enable_stat_watcher\n        raise Fluent::ConfigError, \"either of enable_watch_timer or enable_stat_watcher must be true\"\n      end\n\n      if @glob_policy == :always && @path_delimiter == ','\n        raise Fluent::ConfigError, \"cannot use glob_policy as always with the default path_delimiter: `,\\\"\"\n      end\n\n      if @glob_policy == :extended && /\\{.*,.*\\}/.match?(@path) && extended_glob_pattern(@path)\n        raise Fluent::ConfigError, \"cannot include curly braces with glob patterns in `#{@path}\\\". Use glob_policy always instead.\"\n      end\n\n      if RESERVED_CHARS.include?(@path_delimiter)\n        rc = RESERVED_CHARS.join(', ')\n        raise Fluent::ConfigError, \"#{rc} are reserved words: #{@path_delimiter}\"\n      end\n\n      @paths = @path.split(@path_delimiter).map(&:strip).uniq\n      if @paths.empty?\n        raise Fluent::ConfigError, \"tail: 'path' parameter is required on tail input\"\n      end\n      if @path_timezone\n        Fluent::Timezone.validate!(@path_timezone)\n        @path_formatters = @paths.map{|path| [path, Fluent::Timezone.formatter(@path_timezone, path)]}.to_h\n        @exclude_path_formatters = @exclude_path.map{|path| [path, Fluent::Timezone.formatter(@path_timezone, path)]}.to_h\n      end\n      check_dir_permission unless Fluent.windows?\n\n      # TODO: Use plugin_root_dir and storage plugin to store positions if available\n      if @pos_file\n        if @variable_store.key?(@pos_file) && !called_in_test?\n          plugin_id_using_this_path = @variable_store[@pos_file]\n          raise Fluent::ConfigError, \"Other 'in_tail' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}\"\n        end\n        @variable_store[@pos_file] = self.plugin_id\n      else\n        if @follow_inodes\n          raise Fluent::ConfigError, \"Can't follow inodes without pos_file configuration parameter\"\n        end\n        log.warn \"'pos_file PATH' parameter is not set to a 'tail' source.\"\n        log.warn \"this parameter is highly recommended to save the position to resume tailing.\"\n      end\n\n      configure_tag\n      configure_encoding\n\n      @multiline_mode = parser_config[\"@type\"].include?(\"multiline\")\n      @receive_handler = if @multiline_mode\n                           method(:parse_multilines)\n                         else\n                           method(:parse_singleline)\n                         end\n      @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION\n      @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION\n      # parser is already created by parser helper\n      @parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)\n      if @read_bytes_limit_per_second > 0\n        if !@enable_watch_timer\n          raise Fluent::ConfigError, \"Need to enable watch timer when using log throttling feature\"\n        end\n        min_bytes = TailWatcher::IOHandler::BYTES_TO_READ\n        if @read_bytes_limit_per_second < min_bytes\n          log.warn \"Should specify greater equal than #{min_bytes}. Use #{min_bytes} for read_bytes_limit_per_second\"\n          @read_bytes_limit_per_second = min_bytes\n        end\n      end\n\n      opened_file_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"input\", name: \"files_opened_total\", help_text: \"Total number of opened files\")\n      closed_file_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"input\", name: \"files_closed_total\", help_text: \"Total number of closed files\")\n      rotated_file_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"input\", name: \"files_rotated_total\", help_text: \"Total number of rotated files\")\n      throttling_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"input\", name: \"files_throttled_total\", help_text: \"Total number of times throttling occurs per file when throttling enabled\")\n      # The metrics for currently tracking files. Since the value may decrease, it cannot be represented using the counter type, so 'prefer_gauge: true' is used instead.\n      tracked_file_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"input\", name: \"files_tracked_count\", help_text: \"Number of tracked files\", prefer_gauge: true)\n\n      @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics, throttling_metrics, tracked_file_metrics)\n    end\n\n    def check_dir_permission\n      expand_paths_raw.select { |path|\n        not File.exist?(path)\n      }.each { |path|\n        inaccessible_dir = Pathname.new(File.expand_path(path))\n          .ascend\n          .reverse_each\n          .find { |p| p.directory? && !p.executable? }\n        if inaccessible_dir\n          log.warn \"Skip #{path} because '#{inaccessible_dir}' lacks execute permission.\"\n        end\n      }\n    end\n\n    def configure_tag\n      if @tag.index('*')\n        @tag_prefix, @tag_suffix = @tag.split('*')\n        @tag_prefix ||= ''\n        @tag_suffix ||= ''\n      else\n        @tag_prefix = nil\n        @tag_suffix = nil\n      end\n    end\n\n    def configure_encoding\n      unless @encoding\n        if @from_encoding\n          raise Fluent::ConfigError, \"tail: 'from_encoding' parameter must be specified with 'encoding' parameter.\"\n        end\n      end\n\n      @encoding = parse_encoding_param(@encoding) if @encoding\n      @from_encoding = parse_encoding_param(@from_encoding) if @from_encoding\n      if @encoding && (@encoding == @from_encoding)\n        log.warn \"'encoding' and 'from_encoding' are same encoding. No effect\"\n      end\n    end\n\n    def parse_encoding_param(encoding_name)\n      begin\n        Encoding.find(encoding_name) if encoding_name\n      rescue ArgumentError => e\n        raise Fluent::ConfigError, e.message\n      end\n    end\n\n    def start\n      super\n\n      if @pos_file\n        pos_file_dir = File.dirname(@pos_file)\n        FileUtils.mkdir_p(pos_file_dir, mode: @dir_perm) unless Dir.exist?(pos_file_dir)\n        @pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)\n        @pf_file.sync = true\n        @pf = PositionFile.load(@pf_file, @follow_inodes, expand_paths, logger: log)\n\n        if @pos_file_compaction_interval\n          timer_execute(:in_tail_refresh_compact_pos_file, @pos_file_compaction_interval) do\n            log.info('Clean up the pos file')\n            @pf.try_compact\n          end\n        end\n      end\n\n      refresh_watchers unless @skip_refresh_on_startup\n      timer_execute(:in_tail_refresh_watchers, @refresh_interval, &method(:refresh_watchers))\n    end\n\n    def stop\n      if @variable_store\n        @variable_store.delete(@pos_file)\n      end\n\n      super\n    end\n\n    def shutdown\n      @shutdown_start_time = Fluent::Clock.now\n      # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.\n      stop_watchers(existence_path, immediate: true, remove_watcher: false)\n      @tails_rotate_wait.keys.each do |tw|\n        detach_watcher(tw, @tails_rotate_wait[tw][:ino], false)\n      end\n      @pf_file.close if @pf_file\n\n      super\n    end\n\n    def close\n      super\n      # close file handles after all threads stopped (in #close of thread plugin helper)\n      # It may be because we need to wait IOHandler.ready_to_shutdown()\n      close_watcher_handles\n    end\n\n    def have_read_capability?\n      @capability.have_capability?(:effective, :dac_read_search) ||\n        @capability.have_capability?(:effective, :dac_override)\n    end\n\n    def extended_glob_pattern(path)\n      path.include?('*') || path.include?('?') || /\\[.*\\]/.match?(path)\n    end\n\n    # Curly braces is not supported with default path_delimiter\n    # because the default delimiter of path is \",\".\n    # This should be collided for wildcard pattern for curly braces and\n    # be handled as an error on #configure.\n    def use_glob?(path)\n      if @glob_policy == :always\n        # For future extensions, we decided to use `always' term to handle\n        # regular expressions as much as possible.\n        # This is because not using `true' as a returning value\n        # when choosing :always here.\n        extended_glob_pattern(path) || /\\{.*,.*\\}/.match?(path)\n      elsif @glob_policy == :extended\n        extended_glob_pattern(path)\n      elsif @glob_policy == :backward_compatible\n        path.include?('*')\n      end\n    end\n\n    def expand_paths_raw\n      date = Fluent::EventTime.now\n      paths = []\n      @paths.each { |path|\n        path = if @path_timezone\n                 @path_formatters[path].call(date)\n               else\n                 date.to_time.strftime(path)\n               end\n        if use_glob?(path)\n          paths += Dir.glob(path).select { |p|\n            begin\n              is_file = !File.directory?(p)\n              if (File.readable?(p) || have_read_capability?) && is_file\n                if @limit_recently_modified && File.mtime(p) < (date.to_time - @limit_recently_modified)\n                  false\n                else\n                  true\n                end\n              else\n                if is_file\n                  unless @ignore_list.include?(p)\n                    log.warn \"#{p} unreadable. It is excluded and would be examined next time.\"\n                    @ignore_list << p if @ignore_repeated_permission_error\n                  end\n                end\n                false\n              end\n            rescue Errno::ENOENT, Errno::EACCES\n              log.debug { \"#{p} is missing after refresh file list\" }\n              false\n            end\n          }\n        else\n          # When file is not created yet, Dir.glob returns an empty array. So just add when path is static.\n          paths << path\n        end\n      }\n      excluded = @exclude_path.map { |path|\n        path = if @path_timezone\n                 @exclude_path_formatters[path].call(date)\n               else\n                 date.to_time.strftime(path)\n               end\n        use_glob?(path) ? Dir.glob(path) : path\n      }.flatten.uniq\n      paths - excluded\n    end\n\n    def expand_paths\n      # filter out non existing files, so in case pattern is without '*' we don't do unnecessary work\n      hash = {}\n      expand_paths_raw.select { |path|\n        File.exist?(path)\n      }.each { |path|\n        # Even we just checked for existence, there is a race condition here as\n        # of which stat() might fail with ENOENT. See #3224.\n        begin\n          target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)\n          if @follow_inodes\n            hash[target_info.ino] = target_info\n          else\n            hash[target_info.path] = target_info\n          end\n        rescue Errno::ENOENT, Errno::EACCES  => e\n          log.warn \"expand_paths: stat() for #{path} failed with #{e.class.name}. Skip file.\"\n        end\n      }\n      hash\n    end\n\n    def existence_path\n      hash = {}\n      @tails.each {|path, tw|\n        if @follow_inodes\n          hash[tw.ino] = TargetInfo.new(tw.path, tw.ino)\n        else\n          hash[tw.path] = TargetInfo.new(tw.path, tw.ino)\n        end\n      }\n      hash\n    end\n\n    # in_tail with '*' path doesn't check rotation file equality at refresh phase.\n    # So you should not use '*' path when your logs will be rotated by another tool.\n    # It will cause log duplication after updated watch files.\n    # In such case, you should separate log directory and specify two paths in path parameter.\n    # e.g. path /path/to/dir/*,/path/to/rotated_logs/target_file\n    def refresh_watchers\n      target_paths_hash = expand_paths\n      existence_paths_hash = existence_path\n\n      log.debug {\n        target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(\",\")\n        existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(\",\")\n        \"tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}\"\n      }\n\n      if !@follow_inodes\n        need_unwatch_in_stop_watchers = true\n      else\n        # When using @follow_inodes, need this to unwatch the rotated old inode when it disappears.\n        # After `update_watcher` detaches an old TailWatcher, the inode is lost from the `@tails`.\n        # So that inode can't be contained in `removed_hash`, and can't be unwatched by `stop_watchers`.\n        #\n        # This logic may work for `@follow_inodes false` too.\n        # Just limiting the case to suppress the impact to existing logics.\n        @pf&.unwatch_removed_targets(target_paths_hash)\n        need_unwatch_in_stop_watchers = false\n      end\n\n      removed_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}\n      added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}\n\n      # If an existing TailWatcher already follows a target path with the different inode,\n      # it means that the TailWatcher following the rotated file still exists. In this case,\n      # `refresh_watcher` can't start the new TailWatcher for the new current file. So, we\n      # should output a warning log in order to prevent silent collection stops.\n      # (Such as https://github.com/fluent/fluentd/pull/4327)\n      # (Usually, such a TailWatcher should be removed from `@tails` in `update_watcher`.)\n      # (The similar warning may work for `@follow_inodes true` too. Just limiting the case\n      # to suppress the impact to existing logics.)\n      unless @follow_inodes\n        target_paths_hash.each do |path, target|\n          next unless @tails.key?(path)\n          # We can't use `existence_paths_hash[path].ino` because it is from `TailWatcher.ino`,\n          # which is very unstable parameter. (It can be `nil` or old).\n          # So, we need to use `TailWatcher.pe.read_inode`.\n          existing_watcher_inode = @tails[path].pe.read_inode\n          if existing_watcher_inode != target.ino\n            log.warn \"Could not follow a file (inode: #{target.ino}) because an existing watcher for that filepath follows a different inode: #{existing_watcher_inode} (e.g. keeps watching a already rotated file). If you keep getting this message, please restart Fluentd.\",\n              filepath: target.path\n          end\n        end\n      end\n\n      stop_watchers(removed_hash, unwatched: need_unwatch_in_stop_watchers) unless removed_hash.empty?\n      start_watchers(added_hash) unless added_hash.empty?\n      @metrics.tracked.set(@tails.size)\n      @startup = false if @startup\n    end\n\n    def setup_watcher(target_info, pe)\n      line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil\n      read_from_head = !@startup || @read_from_head\n      tw = TailWatcher.new(target_info, pe, log, read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler), @metrics)\n\n      if @enable_watch_timer\n        tt = TimerTrigger.new(1, log) { tw.on_notify }\n        tw.register_watcher(tt)\n      end\n\n      if @enable_stat_watcher\n        tt = StatWatcher.new(target_info.path, log) { tw.on_notify }\n        tw.register_watcher(tt)\n      end\n\n      tw.watchers.each do |watcher|\n        event_loop_attach(watcher)\n      end\n\n      tw.group_watcher = add_path_to_group_watcher(target_info.path)\n\n      tw\n    rescue => e\n      if tw\n        tw.watchers.each do |watcher|\n          event_loop_detach(watcher)\n        end\n\n        tw.detach(@shutdown_start_time)\n        tw.close\n      end\n      raise e\n    end\n\n    def construct_watcher(target_info)\n      path = target_info.path\n\n      # The file might be rotated or removed after collecting paths, so check inode again here.\n      begin\n        target_info.ino = Fluent::FileWrapper.stat(path).ino\n      rescue Errno::ENOENT, Errno::EACCES\n        log.warn \"stat() for #{path} failed. Continuing without tailing it.\"\n        return\n      end\n\n      pe = nil\n      if @pf\n        pe = @pf[target_info]\n        pe.update(target_info.ino, 0) if @read_from_head && pe.read_inode.zero?\n      end\n\n      begin\n        tw = setup_watcher(target_info, pe)\n      rescue WatcherSetupError => e\n        log.warn \"Skip #{path} because unexpected setup error happens: #{e}\"\n        return\n      end\n\n      @tails[path] = tw\n      tw.on_notify\n    end\n\n    def start_watchers(targets_info)\n      targets_info.each_value {|target_info|\n        construct_watcher(target_info)\n        break if before_shutdown?\n      }\n    end\n\n    def stop_watchers(targets_info, immediate: false, unwatched: false, remove_watcher: true)\n      targets_info.each_value { |target_info|\n        remove_path_from_group_watcher(target_info.path)\n\n        if remove_watcher\n          tw = @tails.delete(target_info.path)\n        else\n          tw = @tails[target_info.path]\n        end\n        if tw\n          tw.unwatched = unwatched\n          if immediate\n            detach_watcher(tw, target_info.ino, false)\n          else\n            detach_watcher_after_rotate_wait(tw, target_info.ino)\n          end\n        end\n      }\n    end\n\n    def close_watcher_handles\n      @tails.keys.each do |path|\n        tw = @tails.delete(path)\n        if tw\n          tw.close\n        end\n      end\n      @tails_rotate_wait.keys.each do |tw|\n        tw.close\n      end\n    end\n\n    # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.\n    def update_watcher(tail_watcher, pe, new_inode)\n      # TODO we should use another callback for this.\n      # To suppress impact to existing logics, limit the case to `@follow_inodes`.\n      # We may not need `@follow_inodes` condition.\n      if @follow_inodes && new_inode.nil?\n        # nil inode means the file disappeared, so we only need to stop it.\n        @tails.delete(tail_watcher.path)\n        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632\n        # Because of this problem, log duplication can occur during `rotate_wait`.\n        # Need to set `rotate_wait 0` for a workaround.\n        # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.\n        # In that case, `refresh_watcher` will add the new TailWatcher to tail the same target,\n        # and it causes the log duplication.\n        # (Other `detach_watcher_after_rotate_wait` may have the same problem.\n        #  We need the mechanism not to add duplicated TailWatcher with detaching TailWatcher.)\n        detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)\n        return\n      end\n\n      path = tail_watcher.path\n\n      log.info(\"detected rotation of #{path}; waiting #{@rotate_wait} seconds\")\n\n      if @pf\n        pe_inode = pe.read_inode\n        target_info_from_position_entry = TargetInfo.new(path, pe_inode)\n        unless pe_inode == @pf[target_info_from_position_entry].read_inode\n          log.warn \"Skip update_watcher because watcher has been already updated by other inotify event\",\n                   path: path, inode: pe.read_inode, inode_in_pos_file: @pf[target_info_from_position_entry].read_inode\n          return\n        end\n      end\n\n      new_target_info = TargetInfo.new(path, new_inode)\n\n      if @follow_inodes\n        new_position_entry = @pf[new_target_info]\n        # If `refresh_watcher` find the new file before, this will not be zero.\n        # In this case, only we have to do is detaching the current tail_watcher.\n        if new_position_entry.read_inode == 0\n          @tails[path] = setup_watcher(new_target_info, new_position_entry)\n          @tails[path].on_notify\n        end\n      else\n        @tails[path] = setup_watcher(new_target_info, pe)\n        @tails[path].on_notify\n      end\n\n      detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)\n    end\n\n    def detach_watcher(tw, ino, close_io = true)\n      if @follow_inodes && tw.ino != ino\n        log.warn(\"detach_watcher could be detaching an unexpected tail_watcher with a different ino.\",\n                  path: tw.path, actual_ino_in_tw: tw.ino, expect_ino_to_close: ino)\n      end\n      tw.watchers.each do |watcher|\n        event_loop_detach(watcher)\n      end\n      tw.detach(@shutdown_start_time)\n\n      tw.close if close_io\n\n      if @pf && tw.unwatched && (@follow_inode || !@tails[tw.path])\n        target_info = TargetInfo.new(tw.path, ino)\n        @pf.unwatch(target_info)\n      end\n    end\n\n    def throttling_is_enabled?(tw)\n      return true if @read_bytes_limit_per_second > 0\n      return true if tw.group_watcher && tw.group_watcher.limit >= 0\n      false\n    end\n\n    def detach_watcher_after_rotate_wait(tw, ino)\n      # Call event_loop_attach/event_loop_detach is high-cost for short-live object.\n      # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.\n      if @open_on_every_update\n        # Detach now because it's already closed, waiting it doesn't make sense.\n        detach_watcher(tw, ino)\n      end\n\n      return if @tails_rotate_wait[tw]\n\n      if throttling_is_enabled?(tw)\n        # When the throttling feature is enabled, it might not reach EOF yet.\n        # Should ensure to read all contents before closing it, with keeping throttling.\n        start_time_to_wait = Fluent::Clock.now\n        timer = timer_execute(:in_tail_close_watcher, 1, repeat: true) do\n          elapsed = Fluent::Clock.now - start_time_to_wait\n          if tw.eof? && elapsed >= @rotate_wait\n            timer.detach\n            @tails_rotate_wait.delete(tw)\n            detach_watcher(tw, ino)\n          end\n        end\n        @tails_rotate_wait[tw] = { ino: ino, timer: timer }\n      else\n        # when the throttling feature isn't enabled, just wait @rotate_wait\n        timer = timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do\n          @tails_rotate_wait.delete(tw)\n          detach_watcher(tw, ino)\n        end\n        @tails_rotate_wait[tw] = { ino: ino, timer: timer }\n      end\n    end\n\n    def flush_buffer(tw, buf)\n      buf.chomp!\n      @parser.parse(buf) { |time, record|\n        if time && record\n          tag = if @tag_prefix || @tag_suffix\n                  @tag_prefix + tw.tag + @tag_suffix\n                else\n                  @tag\n                end\n          record[@path_key] ||= tw.path unless @path_key.nil?\n          router.emit(tag, time, record)\n        else\n          if @emit_unmatched_lines\n            record = { 'unmatched_line' => buf }\n            record[@path_key] ||= tail_watcher.path unless @path_key.nil?\n            tag = if @tag_prefix || @tag_suffix\n                    @tag_prefix + tw.tag + @tag_suffix\n                  else\n                    @tag\n                  end\n            router.emit(tag, Fluent::EventTime.now, record)\n          end\n          log.warn \"got incomplete line at shutdown from #{tw.path}: #{buf.inspect}\"\n        end\n      }\n    end\n\n    # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError\n    def receive_lines(lines, tail_watcher)\n      es = @receive_handler.call(lines, tail_watcher)\n      unless es.empty?\n        tag = if @tag_prefix || @tag_suffix\n                @tag_prefix + tail_watcher.tag + @tag_suffix\n              else\n                @tag\n              end\n        begin\n          router.emit_stream(tag, es)\n        rescue Fluent::Plugin::Buffer::BufferOverflowError\n          return false\n        rescue\n          # ignore non BufferQueueLimitError errors because in_tail can't recover. Engine shows logs and backtraces.\n          return true\n        end\n      end\n\n      return true\n    end\n\n    def convert_line_to_event(line, es, tail_watcher)\n      begin\n        line.chomp!  # remove \\n\n        @parser.parse(line) { |time, record|\n          if time && record\n            record[@path_key] ||= tail_watcher.path unless @path_key.nil?\n            es.add(time, record)\n          else\n            if @emit_unmatched_lines\n              record = {'unmatched_line' => line}\n              record[@path_key] ||= tail_watcher.path unless @path_key.nil?\n              es.add(Fluent::EventTime.now, record)\n            end\n            log.warn { \"pattern not matched: #{line.inspect}\" }\n          end\n        }\n      rescue => e\n        log.warn 'invalid line found', file: tail_watcher.path, line: line, error: e.to_s\n        log.debug_backtrace(e.backtrace)\n      end\n    end\n\n    def parse_singleline(lines, tail_watcher)\n      es = Fluent::MultiEventStream.new\n      lines.each { |line|\n        convert_line_to_event(line, es, tail_watcher)\n      }\n      es\n    end\n\n    # No need to check if line_buffer_timer_flusher is nil, since line_buffer_timer_flusher should exist\n    def parse_multilines(lines, tail_watcher)\n      lb = tail_watcher.line_buffer_timer_flusher.line_buffer\n      es = Fluent::MultiEventStream.new\n      if @parser.has_firstline?\n        tail_watcher.line_buffer_timer_flusher.reset_timer\n        lines.each { |line|\n          if @parser.firstline?(line)\n            if lb\n              convert_line_to_event(lb, es, tail_watcher)\n            end\n            lb = line\n          else\n            if lb.nil?\n              if @emit_unmatched_lines\n                convert_line_to_event(line, es, tail_watcher)\n              end\n              log.warn \"got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}\"\n            else\n              lb << line\n            end\n          end\n        }\n      else\n        lb ||= ''\n        lines.each do |line|\n          lb << line\n          @parser.parse(lb) { |time, record|\n            if time && record\n              convert_line_to_event(lb, es, tail_watcher)\n              lb = ''\n            end\n          }\n        end\n      end\n      tail_watcher.line_buffer_timer_flusher.line_buffer = lb\n      es\n    end\n\n    def statistics\n      stats = super\n\n      stats = {\n        'input' => stats[\"input\"].merge({\n          'opened_file_count' => @metrics.opened.get,\n          'closed_file_count' => @metrics.closed.get,\n          'rotated_file_count' => @metrics.rotated.get,\n          'throttled_log_count' => @metrics.throttled.get,\n          'tracked_file_count' => @metrics.tracked.get,\n        })\n      }\n      stats\n    end\n\n    private\n\n    def io_handler(watcher, path)\n      opts = {\n        path: path,\n        log: log,\n        read_lines_limit: @read_lines_limit,\n        read_bytes_limit_per_second: @read_bytes_limit_per_second,\n        open_on_every_update: @open_on_every_update,\n        metrics: @metrics,\n        max_line_size: @max_line_size,\n      }\n      unless @encoding.nil?\n        if @from_encoding.nil?\n          opts[:encoding] = @encoding\n        else\n          opts[:encoding] = @from_encoding\n          opts[:encoding_to_convert] = @encoding\n        end\n      end\n\n      TailWatcher::IOHandler.new(\n        watcher,\n        **opts,\n        &method(:receive_lines)\n      )\n    end\n\n    class StatWatcher < Coolio::StatWatcher\n      def initialize(path, log, &callback)\n        @callback = callback\n        @log = log\n        super(path)\n      end\n\n      def on_change(prev, cur)\n        @callback.call\n      rescue\n        @log.error $!.to_s\n        @log.error_backtrace\n      end\n    end\n\n    class TimerTrigger < Coolio::TimerWatcher\n      def initialize(interval, log, &callback)\n        @log = log\n        @callback = callback\n        super(interval, true)\n      end\n\n      def on_timer\n        @callback.call\n      rescue => e\n        @log.error e.to_s\n        @log.error_backtrace\n      end\n    end\n\n    class TailWatcher\n      def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build, metrics)\n        @path = target_info.path\n        @ino = target_info.ino\n        @pe = pe || MemoryPositionEntry.new\n        @read_from_head = read_from_head\n        @follow_inodes = follow_inodes\n        @update_watcher = update_watcher\n        @log = log\n        @rotate_handler = RotateHandler.new(log, &method(:on_rotate))\n        @line_buffer_timer_flusher = line_buffer_timer_flusher\n        @io_handler = nil\n        @io_handler_build = io_handler_build\n        @metrics = metrics\n        @watchers = []\n      end\n\n      attr_reader :path, :ino\n      attr_reader :pe\n      attr_reader :line_buffer_timer_flusher\n      attr_accessor :unwatched  # This is used for removing position entry from PositionFile\n      attr_reader :watchers\n      attr_accessor :group_watcher\n\n      def tag\n        @parsed_tag ||= @path.tr('/', '.').squeeze('.').gsub(/^\\./, '')\n      end\n\n      def register_watcher(watcher)\n        @watchers << watcher\n      end\n\n      def detach(shutdown_start_time = nil)\n        if @io_handler\n          @io_handler.ready_to_shutdown(shutdown_start_time)\n          @io_handler.on_notify\n        end\n        @line_buffer_timer_flusher&.close(self)\n      end\n\n      def close\n        if @io_handler\n          @io_handler.close\n          @io_handler = nil\n        end\n      end\n\n      def eof?\n        @io_handler.nil? || @io_handler.eof?\n      end\n\n      def on_notify\n        begin\n          stat = Fluent::FileWrapper.stat(@path)\n        rescue Errno::ENOENT, Errno::EACCES\n          # moved or deleted\n          stat = nil\n        end\n\n        @rotate_handler.on_notify(stat) if @rotate_handler\n        @line_buffer_timer_flusher.on_notify(self) if @line_buffer_timer_flusher\n        @io_handler.on_notify if @io_handler\n      end\n\n      def on_rotate(stat)\n        if @io_handler.nil?\n          if stat\n            # first time\n            fsize = stat.size\n            inode = stat.ino\n\n            last_inode = @pe.read_inode\n            if inode == last_inode\n              # rotated file has the same inode number with the last file.\n              # assuming following situation:\n              #   a) file was once renamed and backed, or\n              #   b) symlink or hardlink to the same file is recreated\n              # in either case of a and b, seek to the saved position\n              #   c) file was once renamed, truncated and then backed\n              # in this case, consider it truncated\n              @pe.update(inode, 0) if fsize < @pe.read_pos\n            elsif last_inode != 0\n              # this is FilePositionEntry and fluentd once started.\n              # read data from the head of the rotated file.\n              # logs never duplicate because this file is a rotated new file.\n              @pe.update(inode, 0)\n            else\n              # this is MemoryPositionEntry or this is the first time fluentd started.\n              # seek to the end of the any files.\n              # logs may duplicate without this seek because it's not sure the file is\n              # existent file or rotated new file.\n              pos = @read_from_head ? 0 : fsize\n              @pe.update(inode, pos)\n            end\n            @io_handler = io_handler\n          else\n            @io_handler = NullIOHandler.new\n          end\n        else\n          watcher_needs_update = false\n\n          if stat\n            inode = stat.ino\n            if inode == @pe.read_inode # truncated\n              @pe.update_pos(0)\n              @io_handler.close\n            elsif !@io_handler.opened? # There is no previous file. Reuse TailWatcher\n              @pe.update(inode, 0)\n            else # file is rotated and new file found\n              watcher_needs_update = true\n              # Handle the old log file before renewing TailWatcher [fluentd#1055]\n              @io_handler.on_notify\n            end\n          else # file is rotated and new file not found\n            # Clear RotateHandler to avoid duplicated file watch in same path.\n            @rotate_handler = nil\n            watcher_needs_update = true\n          end\n\n          if watcher_needs_update\n            if @follow_inodes\n              # If stat is nil (file not present), NEED to stop and discard this watcher.\n              #   When the file is disappeared but is resurrected soon, then `#refresh_watcher`\n              #   can't recognize this TailWatcher needs to be stopped.\n              #   This can happens when the file is rotated.\n              #   If a notify comes before the new file for the path is created during rotation,\n              #   then it appears as if the file was resurrected once it disappeared.\n              # Don't want to swap state because we need latest read offset in pos file even after rotate_wait\n              @update_watcher.call(self, @pe, stat&.ino)\n            else\n              # Permit to handle if stat is nil (file not present).\n              # If a file is mv-ed and a new file is created during\n              # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`\n              # and `#stop_watchers()` for the path because `target_paths_hash`\n              # always contains the path.\n              @update_watcher.call(self, swap_state(@pe), stat&.ino)\n            end\n          else\n            @log.info \"detected rotation of #{@path}\"\n            @io_handler = io_handler\n          end\n          @metrics.rotated.inc\n        end\n      end\n\n      def io_handler\n        @io_handler_build.call(self, @path)\n      end\n\n      def swap_state(pe)\n        # Use MemoryPositionEntry for rotated file temporary\n        mpe = MemoryPositionEntry.new\n        mpe.update(pe.read_inode, pe.read_pos)\n        @pe = mpe\n        pe # This pe will be updated in on_rotate after TailWatcher is initialized\n      end\n\n      class FIFO\n        def initialize(encoding, log, max_line_size=nil, encoding_to_convert=nil)\n          @buffer = ''.force_encoding(encoding)\n          @eol = \"\\n\".encode(encoding).freeze\n          @encoding_to_convert = encoding_to_convert\n          @max_line_size = max_line_size\n          @skip_current_line = false\n          @skipping_current_line_bytesize = 0\n          @log = log\n        end\n\n        attr_reader :buffer, :max_line_size\n\n        def <<(chunk)\n          @buffer << chunk\n        end\n\n        def convert(s)\n          if @encoding_to_convert\n            s.encode!(@encoding_to_convert)\n          else\n            s\n          end\n        rescue\n          s.encode!(@encoding_to_convert, :invalid => :replace, :undef => :replace)\n        end\n\n        def read_lines(lines)\n          idx = @buffer.index(@eol)\n          has_skipped_line = false\n\n          until idx.nil?\n            # Using freeze and slice is faster than slice!\n            # See https://github.com/fluent/fluentd/pull/2527\n            @buffer.freeze\n            slice_position = idx + 1\n            rbuf = @buffer.slice(0, slice_position)\n            @buffer = @buffer.slice(slice_position, @buffer.size - slice_position)\n            idx = @buffer.index(@eol)\n\n            is_long_line = @max_line_size && (\n              @skip_current_line || rbuf.bytesize > @max_line_size\n            )\n\n            if is_long_line\n              @log.warn \"received line length is longer than #{@max_line_size}\"\n              if @skip_current_line\n                @log.debug(\"The continuing line is finished. Finally discarded data: \") { convert(rbuf).chomp }\n              else\n                @log.debug(\"skipped line: \") { convert(rbuf).chomp }\n              end\n              has_skipped_line = true\n              @skip_current_line = false\n              @skipping_current_line_bytesize = 0\n              next\n            end\n\n            lines << convert(rbuf)\n          end\n\n          is_long_current_line = @max_line_size && (\n            @skip_current_line || @buffer.bytesize > @max_line_size\n          )\n\n          if is_long_current_line\n            @log.debug(\n              \"The continuing current line length is longer than #{@max_line_size}.\" +\n              \" The received data will be discarded until this line is finished.\" +\n              \" Discarded data: \"\n            ) { convert(@buffer).chomp }\n            @skip_current_line = true\n            @skipping_current_line_bytesize += @buffer.bytesize\n            @buffer.clear\n          end\n\n          return has_skipped_line\n        end\n\n        def reading_bytesize\n          return @skipping_current_line_bytesize if @skip_current_line\n          @buffer.bytesize\n        end\n      end\n\n      class IOHandler\n        BYTES_TO_READ = 64 * 1024\n        SHUTDOWN_TIMEOUT = 5\n\n        attr_accessor :shutdown_timeout\n\n        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, max_line_size: nil, log:, open_on_every_update:, encoding: Encoding::ASCII_8BIT, encoding_to_convert: nil, metrics:, &receive_lines)\n          @watcher = watcher\n          @path = path\n          @read_lines_limit = read_lines_limit\n          @read_bytes_limit_per_second = read_bytes_limit_per_second\n          @receive_lines = receive_lines\n          @open_on_every_update = open_on_every_update\n          @encoding = encoding\n          @fifo = FIFO.new(encoding, log, max_line_size, encoding_to_convert)\n          @lines = []\n          @io = nil\n          @notify_mutex = Mutex.new\n          @log = log\n          @start_reading_time = nil\n          @number_bytes_read = 0\n          @shutdown_start_time = nil\n          @shutdown_timeout = SHUTDOWN_TIMEOUT\n          @shutdown_mutex = Mutex.new\n          @eof = false\n          @metrics = metrics\n\n          @log.info \"following tail of #{@path}\"\n        end\n\n        def group_watcher\n          @watcher.group_watcher\n        end\n\n        def on_notify\n          @notify_mutex.synchronize { handle_notify }\n        end\n\n        def ready_to_shutdown(shutdown_start_time = nil)\n          @shutdown_mutex.synchronize {\n            @shutdown_start_time =\n              shutdown_start_time || Fluent::Clock.now\n          }\n        end\n\n        def close\n          if @io && !@io.closed?\n            @io.close\n            @io = nil\n            @metrics.closed.inc\n          end\n        end\n\n        def opened?\n          !!@io\n        end\n\n        def eof?\n          @eof\n        end\n\n        private\n\n        def limit_bytes_per_second_reached?\n          return false if @read_bytes_limit_per_second < 0 # not enabled by conf\n          return false if @number_bytes_read < @read_bytes_limit_per_second\n\n          @start_reading_time ||= Fluent::Clock.now\n          time_spent_reading = Fluent::Clock.now - @start_reading_time\n          @log.debug(\"time_spent_reading: #{time_spent_reading} #{ @watcher.path}\")\n\n          if time_spent_reading < 1\n            true\n          else\n            @start_reading_time = nil\n            @number_bytes_read = 0\n            false\n          end\n        end\n\n        def should_shutdown_now?\n          # Ensure to read all remaining lines, but abort immediately if it\n          # seems to take too long time.\n          @shutdown_mutex.synchronize {\n            return false if @shutdown_start_time.nil?\n            return Fluent::Clock.now - @shutdown_start_time > @shutdown_timeout\n          }\n        end\n\n        def handle_notify\n          if limit_bytes_per_second_reached? || group_watcher&.limit_lines_reached?(@path)\n            @metrics.throttled.inc\n            return\n          end\n\n          with_io do |io|\n            iobuf = ''.force_encoding(@encoding)\n            begin\n              read_more = false\n              has_skipped_line = false\n\n              if !io.nil? && @lines.empty?\n                begin\n                  while true\n                    @start_reading_time ||= Fluent::Clock.now\n                    group_watcher&.update_reading_time(@path)\n\n                    data = io.readpartial(BYTES_TO_READ, iobuf)\n                    @eof = false\n                    @number_bytes_read += data.bytesize\n                    @fifo << data\n\n                    n_lines_before_read = @lines.size\n                    has_skipped_line = @fifo.read_lines(@lines) || has_skipped_line\n                    group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)\n\n                    group_watcher_limit = group_watcher&.limit_lines_reached?(@path)\n                    @log.debug \"Reading Limit exceeded #{@path} #{group_watcher.number_lines_read}\" if group_watcher_limit\n\n                    if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?\n                      # Just get out from tailing loop.\n                      @metrics.throttled.inc if group_watcher_limit || limit_bytes_per_second_reached?\n                      read_more = false\n                      break\n                    end\n\n                    if @lines.size >= @read_lines_limit\n                      # not to use too much memory in case the file is very large\n                      read_more = true\n                      break\n                    end\n                  end\n                rescue EOFError\n                  @eof = true\n                ensure\n                  iobuf.clear\n                end\n              end\n\n              if @lines.empty?\n                @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize) if has_skipped_line\n              else\n                if @receive_lines.call(@lines, @watcher)\n                  @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize)\n                  @lines.clear\n                else\n                  read_more = false\n                end\n              end\n            end while read_more\n          end\n        end\n\n        def open\n          io = Fluent::FileWrapper.open(@path)\n          io.seek(@watcher.pe.read_pos + @fifo.reading_bytesize)\n          @metrics.opened.inc\n          io\n        rescue RangeError\n          io.close if io\n          raise WatcherSetupError, \"seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.reading_bytesize.to_s(16)}\"\n        rescue Errno::EACCES => e\n          @log.warn \"#{e}\"\n          nil\n        rescue Errno::ENOENT\n          nil\n        end\n\n        def with_io\n          if @open_on_every_update\n            io = open\n            begin\n              yield io\n            ensure\n              io.close unless io.nil?\n            end\n          else\n            @io ||= open\n            yield @io\n            @eof = true if @io.nil?\n          end\n        rescue WatcherSetupError => e\n          close\n          @eof = true\n          raise e\n        rescue\n          @log.error $!.to_s\n          @log.error_backtrace\n          close\n          @eof = true\n        end\n      end\n\n      class NullIOHandler\n        def initialize\n        end\n\n        def io\n        end\n\n        def on_notify\n        end\n\n        def close\n        end\n\n        def opened?\n          false\n        end\n\n        def eof?\n          true\n        end\n      end\n\n      class RotateHandler\n        def initialize(log, &on_rotate)\n          @log = log\n          @inode = nil\n          @fsize = -1  # first\n          @on_rotate = on_rotate\n        end\n\n        def on_notify(stat)\n          if stat.nil?\n            inode = nil\n            fsize = 0\n          else\n            inode = stat.ino\n            fsize = stat.size\n          end\n\n          if @inode != inode || fsize < @fsize\n            @on_rotate.call(stat)\n          end\n          @inode = inode\n          @fsize = fsize\n        rescue\n          @log.error $!.to_s\n          @log.error_backtrace\n        end\n      end\n\n      class LineBufferTimerFlusher\n        attr_accessor :line_buffer\n\n        def initialize(log, flush_interval, &flush_method)\n          @log = log\n          @flush_interval = flush_interval\n          @flush_method = flush_method\n          @start = nil\n          @line_buffer = nil\n        end\n\n        def on_notify(tw)\n          unless @start && @flush_method\n            return\n          end\n\n          if Time.now - @start >= @flush_interval\n            @flush_method.call(tw, @line_buffer) if @line_buffer\n            @line_buffer = nil\n            @start = nil\n          end\n        end\n\n        def close(tw)\n          return unless @line_buffer\n\n          @flush_method.call(tw, @line_buffer)\n          @line_buffer = nil\n        end\n\n        def reset_timer\n          return unless @flush_interval\n\n          @start = Time.now\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_tcp.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/input'\n\nmodule Fluent::Plugin\n  class TcpInput < Input\n    Fluent::Plugin.register_input('tcp', self)\n\n    helpers :server, :parser, :extract, :compat_parameters\n\n    desc 'Tag of output events.'\n    config_param :tag, :string\n    desc 'The port to listen to.'\n    config_param :port, :integer, default: 5170\n    desc 'The bind address to listen to.'\n    config_param :bind, :string, default: '0.0.0.0'\n\n    desc \"The field name of the client's hostname.\"\n    config_param :source_host_key, :string, default: nil, deprecated: \"use source_hostname_key instead.\"\n    desc \"The field name of the client's hostname.\"\n    config_param :source_hostname_key, :string, default: nil\n    desc \"The field name of the client's address.\"\n    config_param :source_address_key, :string, default: nil\n\n    # Setting default to nil for backward compatibility\n    desc \"The max bytes of message.\"\n    config_param :message_length_limit, :size, default: nil\n\n    config_param :blocking_timeout, :time, default: 0.5\n\n    desc 'The payload is read up to this character.'\n    config_param :delimiter, :string, default: \"\\n\" # syslog family add \"\\n\" to each message and this seems only way to split messages in tcp stream\n    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'\n    config_param :send_keepalive_packet, :bool, default: false\n\n    # in_forward like host/network restriction\n    config_section :security, required: false, multi: false do\n      config_section :client, param_name: :clients, required: true, multi: true do\n        desc 'The IP address or host name of the client'\n        config_param :host, :string, default: nil\n        desc 'Network address specification'\n        config_param :network, :string, default: nil\n      end\n    end\n\n    def configure(conf)\n      compat_parameters_convert(conf, :parser)\n      parser_config = conf.elements('parse').first\n      unless parser_config\n        raise Fluent::ConfigError, \"<parse> section is required.\"\n      end\n      super\n      @_event_loop_blocking_timeout = @blocking_timeout\n      @source_hostname_key ||= @source_host_key if @source_host_key\n\n      @nodes = nil\n      if @security\n        @nodes = []\n        @security.clients.each do |client|\n          if client.host && client.network\n            raise Fluent::ConfigError, \"both of 'host' and 'network' are specified for client\"\n          end\n          if !client.host && !client.network\n            raise Fluent::ConfigError, \"Either of 'host' and 'network' must be specified for client\"\n          end\n          source = nil\n          if client.host\n            begin\n              source = IPSocket.getaddress(client.host)\n            rescue SocketError\n              raise Fluent::ConfigError, \"host '#{client.host}' cannot be resolved\"\n            end\n          end\n          source_addr = begin\n                          IPAddr.new(source || client.network)\n                        rescue ArgumentError\n                          raise Fluent::ConfigError, \"network '#{client.network}' address format is invalid\"\n                        end\n          @nodes.push(source_addr)\n        end\n      end\n\n      @parser = parser_create(conf: parser_config)\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def zero_downtime_restart_ready?\n      true\n    end\n\n    def start\n      super\n\n      log.info \"listening tcp socket\", bind: @bind, port: @port\n      del_size = @delimiter.length\n      discard_till_next_delimiter = false\n      if @_extract_enabled && @_extract_tag_key\n        server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|\n          unless check_client(conn)\n            conn.close\n            next\n          end\n\n          conn.buffer << data\n          buf = conn.buffer\n          pos = 0\n          while i = buf.index(@delimiter, pos)\n            msg = buf[pos...i]\n            pos = i + del_size\n\n            if discard_till_next_delimiter\n              discard_till_next_delimiter = false\n              next\n            end\n\n            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize\n              log.info \"The received data is larger than 'message_length_limit', dropped:\", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]\n              next\n            end\n\n            @parser.parse(msg) do |time, record|\n              unless time && record\n                log.on_warn { log.warn \"pattern not matched\", message: msg }\n                next\n              end\n\n              tag = extract_tag_from_record(record)\n              tag ||= @tag\n              time ||= extract_time_from_record(record) || Fluent::EventTime.now\n              record[@source_address_key] = conn.remote_addr if @source_address_key\n              record[@source_hostname_key] = conn.remote_host if @source_hostname_key\n              router.emit(tag, time, record)\n            end\n          end\n          buf.slice!(0, pos) if pos > 0\n          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.\n          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.\n          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize\n            log.info \"The buffer size exceeds 'message_length_limit', cleared:\", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]\n            buf.clear\n            # We should discard the subsequent data until the next delimiter comes.\n            discard_till_next_delimiter = true\n            next\n          end\n        end\n      else\n        server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|\n          unless check_client(conn)\n            conn.close\n            next\n          end\n\n          conn.buffer << data\n          buf = conn.buffer\n          pos = 0\n          es = Fluent::MultiEventStream.new\n          while i = buf.index(@delimiter, pos)\n            msg = buf[pos...i]\n            pos = i + del_size\n\n            if discard_till_next_delimiter\n              discard_till_next_delimiter = false\n              next\n            end\n\n            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize\n              log.info \"The received data is larger than 'message_length_limit', dropped:\", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]\n              next\n            end\n\n            @parser.parse(msg) do |time, record|\n              unless time && record\n                log.on_warn { log.warn \"pattern not matched\", message: msg }\n                next\n              end\n\n              time ||= extract_time_from_record(record) || Fluent::EventTime.now\n              record[@source_address_key] = conn.remote_addr if @source_address_key\n              record[@source_hostname_key] = conn.remote_host if @source_hostname_key\n              es.add(time, record)\n            end\n          end\n          router.emit_stream(@tag, es)\n          buf.slice!(0, pos) if pos > 0\n          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.\n          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.\n          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize\n            log.info \"The buffer size exceeds 'message_length_limit', cleared:\", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]\n            buf.clear\n            # We should discard the subsequent data until the next delimiter comes.\n            discard_till_next_delimiter = true\n            next\n          end\n        end\n      end\n    end\n\n    private\n\n    def check_client(conn)\n      if @nodes\n        remote_addr = conn.remote_addr\n        node = @nodes.find { |n| n.include?(remote_addr) rescue false }\n        unless node\n          log.warn \"anonymous client '#{remote_addr}' denied\"\n          return false\n        end\n      end\n\n      true\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_udp.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/input'\n\nmodule Fluent::Plugin\n  class UdpInput < Input\n    Fluent::Plugin.register_input('udp', self)\n\n    helpers :server, :parser, :extract, :compat_parameters\n\n    desc 'Tag of output events.'\n    config_param :tag, :string\n    desc 'The port to listen to.'\n    config_param :port, :integer, default: 5160\n    desc 'The bind address to listen to.'\n    config_param :bind, :string, default: '0.0.0.0'\n\n    desc \"The field name of the client's hostname.\"\n    config_param :source_host_key, :string, default: nil, deprecated: \"use source_hostname_key instead.\"\n    desc \"The field name of the client's hostname.\"\n    config_param :source_hostname_key, :string, default: nil\n    desc \"The field name of the client's address.\"\n    config_param :source_address_key, :string, default: nil\n\n    desc \"Deprecated parameter. Use message_length_limit instead\"\n    config_param :body_size_limit, :size, default: nil, deprecated: \"use message_length_limit instead.\"\n    desc \"The max bytes of message\"\n    config_param :message_length_limit, :size, default: 4096\n    desc \"Remove newline from the end of incoming payload\"\n    config_param :remove_newline, :bool, default: true\n    desc \"The max size of socket receive buffer. SO_RCVBUF\"\n    config_param :receive_buffer_size, :size, default: nil, deprecated: \"use receive_buffer_size in transport section instead.\"\n\n    config_param :blocking_timeout, :time, default: 0.5\n\n    # overwrite server plugin to change default to :udp and remove tcp/tls protocol from list\n    config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do\n      config_argument :protocol, :enum, list: [:udp], default: :udp\n    end\n\n    def configure(conf)\n      compat_parameters_convert(conf, :parser)\n      parser_config = conf.elements('parse').first\n      unless parser_config\n        raise Fluent::ConfigError, \"<parse> section is required.\"\n      end\n      super\n      @_event_loop_blocking_timeout = @blocking_timeout\n      @source_hostname_key ||= @source_host_key if @source_host_key\n      @message_length_limit = @body_size_limit if @body_size_limit\n\n      @parser = parser_create(conf: parser_config)\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def zero_downtime_restart_ready?\n      true\n    end\n\n    def start\n      super\n\n      log.info \"listening udp socket\", bind: @bind, port: @port\n      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, resolve_name: !!@source_hostname_key, max_bytes: @message_length_limit, receive_buffer_size: @receive_buffer_size) do |data, sock|\n        data.chomp! if @remove_newline\n        begin\n          @parser.parse(data) do |time, record|\n            unless time && record\n              log.on_warn { log.warn \"pattern not matched\", data: data }\n              next\n            end\n\n            tag = extract_tag_from_record(record)\n            tag ||= @tag\n            time ||= extract_time_from_record(record) || Fluent::EventTime.now\n            record[@source_address_key] = sock.remote_addr if @source_address_key\n            record[@source_hostname_key] = sock.remote_host if @source_hostname_key\n            router.emit(tag, time, record)\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/in_unix.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/env'\nrequire 'fluent/plugin/input'\nrequire 'fluent/msgpack_factory'\n\nrequire 'cool.io'\nrequire 'yajl'\nrequire 'fileutils'\nrequire 'socket'\n\nmodule Fluent::Plugin\n  # TODO: This plugin will be 3rd party plugin\n  class UnixInput < Input\n    Fluent::Plugin.register_input('unix', self)\n\n    helpers :event_loop\n\n    def initialize\n      super\n\n      @lsock = nil\n    end\n\n    desc 'The path to your Unix Domain Socket.'\n    config_param :path, :string, default: Fluent::DEFAULT_SOCKET_PATH\n    desc 'The backlog of Unix Domain Socket.'\n    config_param :backlog, :integer, default: nil\n    desc \"New tag instead of incoming tag\"\n    config_param :tag, :string, default: nil\n\n    def configure(conf)\n      super\n    end\n\n    def start\n      super\n\n      @lsock = listen\n      event_loop_attach(@lsock)\n    end\n\n    def shutdown\n      if @lsock\n        event_loop_detach(@lsock)\n        @lsock.close\n      end\n\n      super\n    end\n\n    def listen\n      if File.exist?(@path)\n        log.warn \"Found existing '#{@path}'. Remove this file for in_unix plugin\"\n        File.unlink(@path)\n      end\n      FileUtils.mkdir_p(File.dirname(@path))\n\n      log.info \"listening fluent socket on #{@path}\"\n      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))\n      s.listen(@backlog) unless @backlog.nil?\n      s\n    end\n\n    # message Entry {\n    #   1: long time\n    #   2: object record\n    # }\n    #\n    # message Forward {\n    #   1: string tag\n    #   2: list<Entry> entries\n    # }\n    #\n    # message PackedForward {\n    #   1: string tag\n    #   2: raw entries  # msgpack stream of Entry\n    # }\n    #\n    # message Message {\n    #   1: string tag\n    #   2: long? time\n    #   3: object record\n    # }\n    def on_message(msg)\n      unless msg.is_a?(Array)\n        log.warn \"incoming data is broken:\", msg: msg\n        return\n      end\n\n      tag = @tag || (msg[0].to_s)\n      entries = msg[1]\n\n      case entries\n      when String\n        # PackedForward\n        es = Fluent::MessagePackEventStream.new(entries)\n        router.emit_stream(tag, es)\n\n      when Array\n        # Forward\n        es = Fluent::MultiEventStream.new\n        entries.each {|e|\n          record = e[1]\n          next if record.nil?\n          time = convert_time(e[0])\n          es.add(time, record)\n        }\n        router.emit_stream(tag, es)\n\n      else\n        # Message\n        record = msg[2]\n        return if record.nil?\n\n        time = convert_time(msg[1])\n        router.emit(tag, time, record)\n      end\n    end\n\n    def convert_time(time)\n      case time\n      when nil, 0\n        Fluent::EventTime.now\n      when Fluent::EventTime\n        time\n      else\n        Fluent::EventTime.from_time(Time.at(time))\n      end\n    end\n\n    class Handler < Coolio::Socket\n      def initialize(io, log, on_message)\n        super(io)\n\n        @on_message = on_message\n        @log = log\n      end\n\n      def on_connect\n      end\n\n      def on_read(data)\n        first = data[0]\n        if first == '{'.freeze || first == '['.freeze\n          m = method(:on_read_json)\n          @parser = Yajl::Parser.new\n          @parser.on_parse_complete = @on_message\n        else\n          m = method(:on_read_msgpack)\n          @parser = Fluent::MessagePackFactory.msgpack_unpacker\n        end\n\n        singleton_class.module_eval do\n          define_method(:on_read, m)\n        end\n        m.call(data)\n      end\n\n      def on_read_json(data)\n        @parser << data\n      rescue => e\n        @log.error \"unexpected error in json payload\", error: e.to_s\n        @log.error_backtrace\n        close\n      end\n\n      def on_read_msgpack(data)\n        @parser.feed_each(data, &@on_message)\n      rescue => e\n        @log.error \"unexpected error in msgpack payload\", error: e.to_s\n        @log.error_backtrace\n        close\n      end\n\n      def on_close\n        @log.trace { \"closed fluent socket object_id=#{self.object_id}\" }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/input.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/base'\n\nrequire 'fluent/log'\nrequire 'fluent/plugin_id'\nrequire 'fluent/plugin_helper'\n\nmodule Fluent\n  module Plugin\n    class Input < Base\n      include PluginId\n      include PluginLoggerMixin\n      include PluginHelper::Mixin\n\n      helpers_internal :event_emitter, :metrics\n\n      def initialize\n        super\n        @emit_records_metrics = nil\n        @emit_size_metrics = nil\n        @counter_mutex = Mutex.new\n        @enable_size_metrics = false\n      end\n\n      def configure(conf)\n        super\n\n        @emit_records_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"input\", name: \"emit_records\", help_text: \"Number of count emit records\")\n        @emit_size_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"input\", name: \"emit_size\", help_text: \"Total size of emit events\")\n        @enable_size_metrics = !!system_config.enable_size_metrics\n      end\n\n      def statistics\n        stats = {\n          'emit_records' => @emit_records_metrics.get,\n          'emit_size' => @emit_size_metrics.get,\n        }\n\n        { 'input' => stats }\n      end\n\n      def metric_callback(es)\n        @emit_records_metrics.add(es.size)\n        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n      end\n\n      def multi_workers_ready?\n        false\n      end\n\n      def zero_downtime_restart_ready?\n        false\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/metrics.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'socket'\n\nrequire 'fluent/plugin/base'\n\nrequire 'fluent/log'\nrequire 'fluent/unique_id'\nrequire 'fluent/plugin_id'\n\nmodule Fluent\n  module Plugin\n    class Metrics < Base\n      include PluginId\n      include PluginLoggerMixin\n      include UniqueId::Mixin\n\n      DEFAULT_TYPE = 'local'\n\n      configured_in :metrics\n\n      config_param :default_labels, :hash, default: {agent: \"Fluentd\", hostname: \"#{Socket.gethostname}\"}\n      config_param :labels, :hash, default: {}\n\n      attr_reader :use_gauge_metric\n      attr_reader :has_methods_for_gauge, :has_methods_for_counter\n\n      def initialize\n        super\n\n        @has_methods_for_counter = false\n        @has_methods_for_gauge = false\n        @use_gauge_metric = false\n      end\n\n      def configure(conf)\n        super\n\n        if use_gauge_metric\n          @has_methods_for_gauge = has_methods_for_gauge?\n        else\n          @has_methods_for_counter = has_methods_for_counter?\n        end\n      end\n\n      # Some metrics should be counted by gauge.\n      # ref: https://prometheus.io/docs/concepts/metric_types/#gauge\n      def use_gauge_metric=(use_gauge_metric=false)\n        @use_gauge_metric = use_gauge_metric\n      end\n\n      def create(namespace:, subsystem:,name:,help_text:,labels: {})\n        # This API is for cmetrics type.\n      end\n\n      def get\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def inc\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def dec\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def add(value)\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def sub(value)\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def set(value)\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      private\n\n      def has_methods_for_counter?\n        implemented_methods = self.class.instance_methods(false)\n\n        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&\n           [:set].all?{|e| self.class.method_defined?(e)}\n          true\n        else\n          raise \"BUG: metrics plugin on counter mode MUST implement `get`, `inc`, `add` methods. And aliased `set` methods should be aliased from another method\"\n        end\n      end\n\n      def has_methods_for_gauge?\n        implemented_methods = self.class.instance_methods(false)\n\n        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&\n           [:set, :dec, :sub].all?{|e| self.class.method_defined?(e)}\n          true\n        else\n          raise \"BUG: metrics plugin on gauge mode MUST implement `get`, `inc`, and `add` methods. And `dec`, `sub`, and `set` methods should be aliased from other methods\"\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/metrics_local.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/metrics'\n\nmodule Fluent\n  module Plugin\n    class LocalMetrics < Metrics\n      Fluent::Plugin.register_metrics('local', self)\n\n      def initialize\n        super\n        @store = 0\n        @monitor = Monitor.new\n      end\n\n      def configure(conf)\n        super\n\n        if use_gauge_metric\n          class << self\n            alias_method :dec, :dec_gauge\n            alias_method :set, :set_gauge\n            alias_method :sub, :sub_gauge\n          end\n        else\n          class << self\n            alias_method :set, :set_counter\n          end\n        end\n      end\n\n      def multi_workers_ready?\n        true\n      end\n\n      def get\n        @monitor.synchronize do\n          @store\n        end\n      end\n\n      def inc\n        @monitor.synchronize do\n          @store += 1\n        end\n      end\n\n      def dec_gauge\n        @monitor.synchronize do\n          @store -= 1\n        end\n      end\n\n      def add(value)\n        @monitor.synchronize do\n          @store += value\n        end\n      end\n\n      def sub_gauge(value)\n        @monitor.synchronize do\n          @store -= value\n        end\n      end\n\n      def set_counter(value)\n        return if @store > value\n\n        @monitor.synchronize do\n          @store = value\n        end\n      end\n\n      def set_gauge(value)\n        @monitor.synchronize do\n          @store = value\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/multi_output.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/base'\nrequire 'fluent/log'\nrequire 'fluent/plugin_id'\nrequire 'fluent/plugin_helper'\n\nmodule Fluent\n  module Plugin\n    class MultiOutput < Base\n      include PluginId\n      include PluginLoggerMixin\n      include PluginHelper::Mixin # for event_emitter\n\n      helpers :event_emitter # to get router from agent, which will be supplied to child plugins\n      helpers_internal :metrics\n\n      config_section :store, param_name: :stores, multi: true, required: true do\n        config_argument :arg, :string, default: ''\n        config_param :@type, :string, default: nil\n      end\n\n      attr_reader :outputs, :outputs_statically_created\n\n      def process(tag, es)\n        raise NotImplementedError, \"BUG: output plugins MUST implement this method\"\n      end\n\n      def initialize\n        super\n        @outputs = []\n        @outputs_statically_created = false\n\n        @counter_mutex = Mutex.new\n        # TODO: well organized counters\n        @num_errors_metrics = nil\n        @emit_count_metrics = nil\n        @emit_records_metrics = nil\n        @emit_size_metrics = nil\n        # @write_count = 0\n        # @rollback_count = 0\n        @enable_size_metrics = false\n      end\n\n      def statistics\n        stats = {\n          'num_errors' => @num_errors_metrics.get,\n          'emit_records' => @emit_records_metrics.get,\n          'emit_count' => @emit_count_metrics.get,\n          'emit_size' => @emit_size_metrics.get,\n        }\n\n        { 'multi_output' => stats }\n      end\n\n      def multi_output?\n        true\n      end\n\n      def configure(conf)\n        super\n\n        @num_errors_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"multi_output\", name: \"num_errors\", help_text: \"Number of count num errors\")\n        @emit_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"multi_output\", name: \"emit_count\", help_text: \"Number of count emits\")\n        @emit_records_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"multi_output\", name: \"emit_records\", help_text: \"Number of emit records\")\n        @emit_size_metrics =  metrics_create(namespace: \"fluentd\", subsystem: \"multi_output\", name: \"emit_size\", help_text: \"Total size of emit events\")\n        @enable_size_metrics = !!system_config.enable_size_metrics\n\n        @stores.each do |store|\n          store_conf = store.corresponding_config_element\n          type = store_conf['@type']\n          unless type\n            raise Fluent::ConfigError, \"Missing '@type' parameter in <store> section\"\n          end\n\n          log.debug \"adding store\", type: type\n\n          output = Fluent::Plugin.new_output(type)\n          output.context_router = self.context_router\n          output.configure(store_conf)\n          @outputs << output\n        end\n      end\n\n      def static_outputs\n        @outputs_statically_created = true\n        @outputs\n      end\n\n      # Child plugin's lifecycles are controlled by agent automatically.\n      # It calls `outputs` to traverse plugins, and invoke start/stop/*shutdown/close/terminate on these directly.\n      # * `start` of this plugin will be called after child plugins\n      # * `stop`, `*shutdown`, `close` and `terminate` of this plugin will be called before child plugins\n\n      # But when MultiOutput plugins are created dynamically (by forest plugin or others), agent cannot find\n      # sub-plugins. So child plugins' lifecycles MUST be controlled by MultiOutput plugin itself.\n      # TODO: this hack will be removed at v2.\n      def call_lifecycle_method(method_name, checker_name)\n        return if @outputs_statically_created\n        @outputs.each do |o|\n          begin\n            log.debug \"calling #{method_name} on output plugin dynamically created\", type: Fluent::Plugin.lookup_type_from_class(o.class), plugin_id: o.plugin_id\n            o.__send__(method_name) unless o.__send__(checker_name)\n          rescue Exception => e\n            log.warn \"unexpected error while calling #{method_name} on output plugin dynamically created\", plugin: o.class, plugin_id: o.plugin_id, error: e\n            log.warn_backtrace\n          end\n        end\n      end\n\n      def start\n        super\n        call_lifecycle_method(:start, :started?)\n      end\n\n      def after_start\n        super\n        call_lifecycle_method(:after_start, :after_started?)\n      end\n\n      def stop\n        super\n        call_lifecycle_method(:stop, :stopped?)\n      end\n\n      def before_shutdown\n        super\n        call_lifecycle_method(:before_shutdown, :before_shutdown?)\n      end\n\n      def shutdown\n        super\n        call_lifecycle_method(:shutdown, :shutdown?)\n      end\n\n      def after_shutdown\n        super\n        call_lifecycle_method(:after_shutdown, :after_shutdown?)\n      end\n\n      def close\n        super\n        call_lifecycle_method(:close, :closed?)\n      end\n\n      def terminate\n        super\n        call_lifecycle_method(:terminate, :terminated?)\n      end\n\n      def emit_sync(tag, es)\n        @emit_count_metrics.inc\n        begin\n          process(tag, es)\n          @emit_records_metrics.add(es.size)\n          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n        rescue\n          @num_errors_metrics.inc\n          raise\n        end\n      end\n      alias :emit_events :emit_sync\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_buffer.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\n\nmodule Fluent::Plugin\n  class BufferOutput < Output\n    Fluent::Plugin.register_output(\"buffer\", self)\n    helpers :event_emitter\n\n    config_section :buffer do\n      config_set_default :@type, \"file\"\n      config_set_default :chunk_keys, [\"tag\"]\n      config_set_default :flush_mode, :interval\n      config_set_default :flush_interval, 10\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def write(chunk)\n      return if chunk.empty?\n      router.emit_stream(chunk.metadata.tag, Fluent::MessagePackEventStream.new(chunk.read))\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_copy.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/multi_output'\nrequire 'fluent/config/error'\nrequire 'fluent/event'\n\nmodule Fluent::Plugin\n  class CopyOutput < MultiOutput\n    Fluent::Plugin.register_output('copy', self)\n\n    desc 'If true, pass different record to each `store` plugin.'\n    config_param :deep_copy, :bool, default: false, deprecated: \"use 'copy_mode' parameter instead\"\n    desc 'Pass different record to each `store` plugin by specified method'\n    config_param :copy_mode, :enum, list: [:no_copy, :shallow, :deep, :marshal], default: :no_copy\n\n    attr_reader :ignore_errors, :ignore_if_prev_successes\n\n    def initialize\n      super\n      @ignore_errors = []\n      @ignore_if_prev_successes = []\n    end\n\n    def configure(conf)\n      super\n\n      @copy_proc = gen_copy_proc\n      @stores.each_with_index { |store, i|\n        if i == 0 && store.arg.include?('ignore_if_prev_success')\n          raise Fluent::ConfigError, \"ignore_if_prev_success must specify 2nd or later <store> directives\"\n        end\n        @ignore_errors << (store.arg.include?('ignore_error'))\n        @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))\n      }\n      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)\n        log.warn \"ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?\"\n      end\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def process(tag, es)\n      unless es.repeatable?\n        m = Fluent::MultiEventStream.new\n        es.each {|time,record|\n          m.add(time, record)\n        }\n        es = m\n      end\n      success = Array.new(outputs.size)\n      outputs.each_with_index do |output, i|\n        begin\n          if i > 0 && success[i - 1] && @ignore_if_prev_successes[i]\n            log.debug \"ignore copy because prev_success in #{output.plugin_id}\", index: i\n          else\n            output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)\n            success[i] = true\n          end\n        rescue => e\n          if @ignore_errors[i]\n            log.error \"ignore emit error in #{output.plugin_id}\", error: e\n          else\n            raise e\n          end\n        end\n      end\n    end\n\n    private\n\n    def gen_copy_proc\n      @copy_mode = :shallow if @deep_copy\n\n      case @copy_mode\n      when :no_copy\n        nil\n      when :shallow\n        Proc.new { |es| es.dup }\n      when :deep\n        Proc.new { |es|\n          packer = Fluent::MessagePackFactory.msgpack_packer\n          times = []\n          records = []\n          es.each { |time, record|\n            times << time\n            packer.pack(record)\n          }\n          Fluent::MessagePackFactory.msgpack_unpacker.feed_each(packer.full_pack) { |record|\n            records << record\n          }\n          Fluent::MultiEventStream.new(times, records)\n        }\n      when :marshal\n        Proc.new { |es|\n          new_es = Fluent::MultiEventStream.new\n          es.each { |time, record|\n            new_es.add(time, Marshal.load(Marshal.dump(record)))\n          }\n          new_es\n        }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_exec.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'tempfile'\n\nrequire 'fluent/plugin/output'\nrequire 'fluent/config/error'\n\nmodule Fluent::Plugin\n  class ExecOutput < Output\n    Fluent::Plugin.register_output('exec', self)\n\n    helpers :inject, :formatter, :compat_parameters, :child_process\n\n    desc 'The command (program) to execute. The exec plugin passes the path of a TSV file as the last argument'\n    config_param :command, :string\n\n    config_param :command_timeout, :time, default: 270 # 4min 30sec\n\n    config_section :format do\n      config_set_default :@type, 'tsv'\n    end\n\n    config_section :inject do\n      config_set_default :time_type, :string\n      config_set_default :localtime, false\n    end\n\n    config_section :buffer do\n      config_set_default :delayed_commit_timeout, 300 # 5 min\n    end\n\n    attr_reader :formatter # for tests\n\n    def configure(conf)\n      compat_parameters_convert(conf, :inject, :formatter, :buffer, default_chunk_key: 'time')\n      super\n      @formatter = formatter_create\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    NEWLINE = \"\\n\"\n\n    def format(tag, time, record)\n      record = inject_values_to_record(tag, time, record)\n      if @formatter.formatter_type == :text_per_line\n        @formatter.format(tag, time, record).chomp + NEWLINE\n      else\n        @formatter.format(tag, time, record)\n      end\n    end\n\n    def try_write(chunk)\n      tmpfile = nil\n      prog = if chunk.respond_to?(:path)\n               \"#{@command} #{chunk.path}\"\n             else\n               tmpfile = Tempfile.new(\"fluent-plugin-out-exec-\")\n               tmpfile.binmode\n               chunk.write_to(tmpfile)\n               tmpfile.close\n               \"#{@command} #{tmpfile.path}\"\n             end\n      chunk_id = chunk.unique_id\n      callback = ->(status){\n        begin\n          if tmpfile\n            tmpfile.delete rescue nil\n          end\n          if status && status.success?\n            commit_write(chunk_id)\n          elsif status\n            # #rollback_write will be done automatically if it isn't called at here.\n            # But it's after command_timeout, and this timeout should be longer than users expectation.\n            # So here, this plugin calls it explicitly.\n            rollback_write(chunk_id)\n            log.warn \"command exits with error code\", prog: prog, status: status.exitstatus, signal: status.termsig\n          else\n            rollback_write(chunk_id)\n            log.warn \"command unexpectedly exits without exit status\", prog: prog\n          end\n        rescue => e\n          log.error \"unexpected error in child process callback\", error: e\n        end\n      }\n      child_process_execute(:out_exec_process, prog, stderr: :connect, immediate: true, parallel: true, mode: [], wait_timeout: @command_timeout, on_exit_callback: callback)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_exec_filter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\nrequire 'fluent/plugin/output'\nrequire 'fluent/env'\nrequire 'fluent/config/error'\n\nrequire 'json'\n\nmodule Fluent::Plugin\n  class ExecFilterOutput < Output\n    Fluent::Plugin.register_output('exec_filter', self)\n\n    helpers :compat_parameters, :inject, :formatter, :parser, :extract, :child_process, :event_emitter\n\n    desc 'The command (program) to execute.'\n    config_param :command, :string\n\n    config_param :remove_prefix, :string, default: nil, deprecated: \"use @label instead for event routing\"\n    config_param :add_prefix, :string, default: nil, deprecated: \"use @label instead for event routing\"\n\n    config_section :inject do\n      config_set_default :time_type, :unixtime\n    end\n\n    config_section :format do\n      config_set_default :@type, 'tsv'\n      config_set_default :localtime, true\n    end\n\n    config_section :parse do\n      config_set_default :@type, 'tsv'\n      config_set_default :time_key, nil\n      config_set_default :time_format, nil\n      config_set_default :localtime, true\n      config_set_default :estimate_current_event, false\n    end\n\n    config_section :extract do\n      config_set_default :time_type, :float\n    end\n\n    config_section :buffer do\n      config_set_default :flush_mode, :interval\n      config_set_default :flush_interval, 1\n    end\n\n    config_param :tag, :string, default: nil\n\n    config_param :tag_key, :string, default: nil, deprecated: \"use 'tag_key' in <inject>/<extract> instead\"\n    config_param :time_key, :string, default: nil, deprecated: \"use 'time_key' in <inject>/<extract> instead\"\n    config_param :time_format, :string, default: nil, deprecated: \"use 'time_format' in <inject>/<extract> instead\"\n\n    desc 'The default block size to read if parser requires partial read.'\n    config_param :read_block_size, :size, default: 10240 # 10k\n\n    desc 'The number of spawned process for command.'\n    config_param :num_children, :integer, default: 1\n\n    desc 'Respawn command when command exit. [\"none\", \"inf\" or positive integer for times to respawn (default: none)]'\n    # nil, 'none' or 0: no respawn, 'inf' or -1: infinite times, positive integer: try to respawn specified times only\n    config_param :child_respawn, :string, default: nil\n\n    # 0: output logs for all of messages to emit\n    config_param :suppress_error_log_interval, :time, default: 0\n\n    attr_reader :formatter, :parser # for tests\n\n    KEYS_FOR_IN_AND_OUT = {\n      'tag_key' => ['in_tag_key', 'out_tag_key'],\n      'time_key' => ['in_time_key', 'out_time_key'],\n      'time_format' => ['in_time_format', 'out_time_format'],\n    }\n    COMPAT_INJECT_PARAMS = {\n      'in_tag_key' => 'tag_key',\n      'in_time_key' => 'time_key',\n      'in_time_format' => 'time_format',\n    }\n    COMPAT_FORMAT_PARAMS = {\n      'in_format' => '@type',\n      'in_keys' => 'keys',\n    }\n    COMPAT_PARSE_PARAMS = {\n      'out_format' => '@type',\n      'out_keys' => 'keys',\n      'out_stream_buffer_size' => 'stream_buffer_size',\n    }\n    COMPAT_EXTRACT_PARAMS = {\n      'out_tag_key' => 'tag_key',\n      'out_time_key' => 'time_key',\n      'out_time_format' => 'time_format',\n    }\n\n    def exec_filter_compat_parameters_copy_to_subsection!(conf, subsection_name, params)\n      return unless conf.elements(subsection_name).empty?\n      return unless params.keys.any?{|k| conf.has_key?(k) }\n      hash = {}\n      params.each_pair do |compat, current|\n        hash[current] = conf[compat] if conf.has_key?(compat)\n      end\n      conf.elements << Fluent::Config::Element.new(subsection_name, '', hash, [])\n    end\n\n    def exec_filter_compat_parameters_convert!(conf)\n      KEYS_FOR_IN_AND_OUT.each_pair do |inout, keys|\n        if conf.has_key?(inout)\n          keys.each do |k|\n            conf[k] = conf[inout]\n          end\n        end\n      end\n      exec_filter_compat_parameters_copy_to_subsection!(conf, 'inject', COMPAT_INJECT_PARAMS)\n      exec_filter_compat_parameters_copy_to_subsection!(conf, 'format', COMPAT_FORMAT_PARAMS)\n      exec_filter_compat_parameters_copy_to_subsection!(conf, 'parse', COMPAT_PARSE_PARAMS)\n      exec_filter_compat_parameters_copy_to_subsection!(conf, 'extract', COMPAT_EXTRACT_PARAMS)\n    end\n\n    def configure(conf)\n      exec_filter_compat_parameters_convert!(conf)\n      compat_parameters_convert(conf, :buffer)\n\n      if inject_section = conf.elements('inject').first\n        if inject_section.has_key?('time_format')\n          inject_section['time_type'] ||= 'string'\n        end\n      end\n      if extract_section = conf.elements('extract').first\n        if extract_section.has_key?('time_format')\n          extract_section['time_type'] ||= 'string'\n        end\n      end\n\n      super\n\n      if !@tag && (!@extract_config || !@extract_config.tag_key)\n        raise Fluent::ConfigError, \"'tag' or '<extract> tag_key </extract>' option is required on exec_filter output\"\n      end\n\n      @formatter = formatter_create\n      @parser = parser_create\n\n      if @remove_prefix\n        @removed_prefix_string = @remove_prefix + '.'\n        @removed_length = @removed_prefix_string.length\n      end\n      if @add_prefix\n        @added_prefix_string = @add_prefix + '.'\n      end\n\n      @respawns = if @child_respawn.nil? || (@child_respawn == 'none') || (@child_respawn == '0')\n                    0\n                  elsif (@child_respawn == 'inf') || (@child_respawn == '-1')\n                    -1\n                  elsif /^\\d+$/.match?(@child_respawn)\n                    @child_respawn.to_i\n                  else\n                    raise ConfigError, \"child_respawn option argument invalid: none(or 0), inf(or -1) or positive number\"\n                  end\n\n      @suppress_error_log_interval ||= 0\n      @next_log_time = Time.now.to_i\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    ExecutedProcess = Struct.new(:mutex, :pid, :respawns, :readio, :writeio)\n\n    def start\n      super\n\n      @children_mutex = Mutex.new\n      @children = []\n      @rr = 0\n\n      exit_callback = ->(status){\n        c = @children.find{|child| child.pid == status.pid }\n        if c\n          unless self.stopped?\n            log.warn \"child process exits with error code\", code: status.to_i, status: status.exitstatus, signal: status.termsig\n          end\n          c.mutex.synchronize do\n            c.writeio&.close rescue nil\n            c.readio&.close rescue nil\n            c.pid = c.readio = c.writeio = nil\n          end\n        end\n      }\n      child_process_callback = ->(index, readio, writeio){\n        pid = child_process_id\n        c = @children[index]\n        writeio.sync = true\n        c.mutex.synchronize do\n          c.pid = pid\n          c.respawns = @respawns\n          c.readio = readio\n          c.writeio = writeio\n        end\n\n        run(readio)\n      }\n      execute_child_process = ->(index){\n        child_process_execute(\"out_exec_filter_child#{index}\".to_sym, @command, on_exit_callback: exit_callback) do |readio, writeio|\n          child_process_callback.call(index, readio, writeio)\n        end\n      }\n\n      @children_mutex.synchronize do\n        @num_children.times do |i|\n          @children << ExecutedProcess.new(Mutex.new, nil, 0, nil, nil)\n          execute_child_process.call(i)\n        end\n      end\n\n      if @respawns != 0\n        thread_create(:out_exec_filter_respawn_monitor) do\n          while thread_current_running?\n            @children.each_with_index do |c, i|\n              if c.mutex && c.mutex.synchronize{ c.pid.nil? && c.respawns != 0 }\n                respawns = c.mutex.synchronize do\n                  c.respawns -= 1 if c.respawns > 0\n                  c.respawns\n                end\n                log.info \"respawning child process\", num: i, respawn_counter: respawns\n                execute_child_process.call(i)\n              end\n            end\n            sleep 0.2\n          end\n        end\n      end\n    end\n\n    def terminate\n      @children = []\n      super\n    end\n\n    def tag_remove_prefix(tag)\n      if @remove_prefix\n        if ((tag[0, @removed_length] == @removed_prefix_string) && (tag.length > @removed_length)) || (tag == @removed_prefix_string)\n          tag = tag[@removed_length..-1] || ''\n        end\n      end\n      tag\n    end\n\n    NEWLINE = \"\\n\"\n\n    def format(tag, time, record)\n      tag = tag_remove_prefix(tag)\n      record = inject_values_to_record(tag, time, record)\n      if @formatter.formatter_type == :text_per_line\n        @formatter.format(tag, time, record).chomp + NEWLINE\n      else\n        @formatter.format(tag, time, record)\n      end\n    end\n\n    def write(chunk)\n      try_times = 0\n      while true\n        r = @rr = (@rr + 1) % @children.length\n        if @children[r].pid && writeio = @children[r].writeio\n          chunk.write_to(writeio)\n          break\n        end\n        try_times += 1\n        raise \"no healthy child processes exist\" if try_times >= @children.length\n      end\n    end\n\n    def run(io)\n      io.set_encoding(Encoding::ASCII_8BIT)\n      case\n      when @parser.implement?(:parse_io)\n        @parser.parse_io(io, &method(:on_record))\n      when @parser.implement?(:parse_partial_data)\n        until io.eof?\n          @parser.parse_partial_data(io.readpartial(@read_block_size), &method(:on_record))\n        end\n      when @parser.parser_type == :text_per_line\n        io.each_line do |line|\n          @parser.parse(line.chomp, &method(:on_record))\n        end\n      else\n        @parser.parse(io.read, &method(:on_record))\n      end\n    end\n\n    def on_record(time, record)\n      tag = extract_tag_from_record(record)\n      tag = @added_prefix_string + tag if tag && @add_prefix\n      tag ||= @tag\n      time ||= extract_time_from_record(record) || Fluent::EventTime.now\n      router.emit(tag, time, record)\n    rescue => e\n      if @suppress_error_log_interval == 0 || Time.now.to_i > @next_log_time\n        log.error \"exec_filter failed to emit\", record: JSON.generate(record), error: e\n        log.error_backtrace e.backtrace\n        @next_log_time = Time.now.to_i + @suppress_error_log_interval\n      end\n      router.emit_error_event(tag, time, record, e) if tag && time && record\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_file.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fileutils'\nrequire 'zlib'\nrequire 'time'\nrequire 'pathname'\n\nrequire 'fluent/plugin/output'\nrequire 'fluent/config/error'\n# TODO remove ...\nrequire 'fluent/plugin/file_util'\n\nmodule Fluent::Plugin\n  class FileOutput < Output\n    Fluent::Plugin.register_output('file', self)\n\n    helpers :formatter, :inject, :compat_parameters\n\n    SUPPORTED_COMPRESS = [:text, :gz, :gzip, :zstd]\n    SUPPORTED_COMPRESS_MAP = {\n      text: nil,\n      gz: :gzip,\n      gzip: :gzip,\n      zstd: :zstd,\n    }\n\n    DEFAULT_TIMEKEY = 60 * 60 * 24\n\n    desc \"The Path of the file.\"\n    config_param :path, :string\n\n    desc \"Specify to add file suffix for bare file path or not.\"\n    config_param :add_path_suffix, :bool, default: true\n    desc \"The file suffix added to bare file path.\"\n    config_param :path_suffix, :string, default: '.log'\n    desc \"The flushed chunk is appended to existence file or not.\"\n    config_param :append, :bool, default: false\n    desc \"Compress flushed file.\"\n    config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text\n    desc \"Execute compression again even when buffer chunk is already compressed.\"\n    config_param :recompress, :bool, default: false\n    desc \"Create symlink to temporary buffered file when buffer_type is file (disabled on Windows).\"\n    config_param :symlink_path, :string, default: nil\n    desc \"Use relative path for symlink target (default: false)\"\n    config_param :symlink_path_use_relative, :bool, default: false\n\n    config_section :format do\n      config_set_default :@type, 'out_file'\n    end\n\n    config_section :buffer do\n      config_set_default :@type, 'file'\n      config_set_default :chunk_keys, ['time']\n      config_set_default :timekey, DEFAULT_TIMEKEY\n    end\n\n    attr_reader :dir_perm\n    attr_accessor :last_written_path # for tests\n\n    module SymlinkBufferMixin\n      def metadata(timekey: nil, tag: nil, variables: nil)\n        metadata = super\n\n        @latest_metadata ||= new_metadata(timekey: 0)\n        if metadata.timekey && (metadata.timekey >= @latest_metadata.timekey)\n          @latest_metadata = metadata\n        end\n\n        metadata\n      end\n\n      def output_plugin_for_symlink=(output_plugin)\n        @_output_plugin_for_symlink = output_plugin\n      end\n\n      def symlink_path=(path)\n        @_symlink_path = path\n      end\n\n      def generate_chunk(metadata)\n        chunk = super\n        # \"symlink\" feature is to link from symlink_path to the latest file chunk. Records with latest\n        # timekey will be appended into that file chunk. On the other side, resumed file chunks might NOT\n        # have timekey, especially in the cases that resumed file chunks are generated by Fluentd v0.12.\n        # These chunks will be enqueued immediately, and will be flushed soon.\n        if chunk.metadata == @latest_metadata\n          sym_path = @_output_plugin_for_symlink.extract_placeholders(@_symlink_path, chunk)\n          FileUtils.mkdir_p(File.dirname(sym_path), mode: @_output_plugin_for_symlink.dir_perm)\n          if @_output_plugin_for_symlink.symlink_path_use_relative\n            relative_path = Pathname.new(chunk.path).relative_path_from(Pathname.new(File.dirname(sym_path)))\n            FileUtils.ln_sf(relative_path, sym_path)\n          else\n            FileUtils.ln_sf(chunk.path, sym_path)\n          end\n        end\n        chunk\n      end\n    end\n\n    def configure(conf)\n      compat_parameters_convert(conf, :formatter, :buffer, :inject, default_chunk_key: \"time\")\n\n      configured_time_slice_format = conf['time_slice_format']\n\n      if conf.elements(name: 'buffer').empty?\n        # no <buffer> section, default time chunk key and timekey (1d) will be used.\n        log.warn \"default timekey interval (1d) will be used because of missing <buffer> section. To change the output frequency, please modify the timekey value\"\n\n        conf.add_element('buffer', 'time')\n      else\n        unless conf.elements(name: 'buffer').first.has_key?('timekey')\n          if conf.elements(name: 'buffer').first.arg != \"[]\"\n            # with <buffer> section (except <buffer []>), and no timekey\n            log.warn \"default timekey interval (1d) will be used. To change the output frequency, please modify the timekey value\"\n          end\n        end\n      end\n      buffer_conf = conf.elements(name: 'buffer').first\n      # Fluent::PluginId#configure is not called yet, so we can't use #plugin_root_dir here.\n      if !buffer_conf.has_key?('path') && !(conf['@id'] && system_config.root_dir)\n        # v0.14 file buffer handles path as directory if '*' is missing\n        # 'dummy_path' is not to raise configuration error for 'path' in file buffer plugin,\n        # but raise it in this plugin.\n        buffer_conf['path'] = conf['path'] || '/tmp/dummy_path'\n      end\n\n      if conf.has_key?('utc') || conf.has_key?('localtime')\n        param_name = conf.has_key?('utc') ? 'utc' : 'localtime'\n        log.warn \"'#{param_name}' is deprecated for output plugin. This parameter is used for formatter plugin in compatibility layer. If you want to use same feature, use timekey_use_utc parameter in <buffer> directive instead\"\n      end\n\n      super\n\n      @compress_method = SUPPORTED_COMPRESS_MAP[@compress]\n\n      if @path.include?('*') && !@buffer_config.timekey\n        raise Fluent::ConfigError, \"path including '*' must be used with buffer chunk key 'time'\"\n      end\n\n      path_suffix = @add_path_suffix ? @path_suffix : ''\n      path_timekey = if @chunk_key_time\n                       @as_secondary ? @primary_instance.buffer_config.timekey : @buffer_config.timekey\n                     else\n                       nil\n                     end\n      @path_template = generate_path_template(@path, path_timekey, @append, @compress_method, path_suffix: path_suffix, time_slice_format: configured_time_slice_format)\n\n      if @as_secondary\n        # When this plugin is configured as secondary & primary plugin has tag key, but this plugin may not have it.\n        # Increment placeholder can make another output file per chunk tag/keys even if original path doesn't include it.\n        placeholder_validators(:path, @path_template).select{|v| v.type == :time }.each do |v|\n          v.validate!\n        end\n      else\n        placeholder_validate!(:path, @path_template)\n\n        max_tag_index = get_placeholders_tag(@path_template).max || 1\n        max_tag_index = 1 if max_tag_index < 1\n        dummy_tag = (['a'] * max_tag_index).join('.')\n        dummy_record_keys = get_placeholders_keys(@path_template) || ['message']\n        dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]\n\n        test_chunk1 = chunk_for_test(dummy_tag, Fluent::EventTime.now, dummy_record)\n        test_path = extract_placeholders(@path_template, test_chunk1)\n        unless ::Fluent::FileUtil.writable_p?(test_path)\n          raise Fluent::ConfigError, \"out_file: `#{test_path}` is not writable\"\n        end\n      end\n\n      @formatter = formatter_create\n\n      if @symlink_path && @buffer.respond_to?(:path)\n        if @as_secondary\n          raise Fluent::ConfigError, \"symlink_path option is unavailable in <secondary>: consider to use secondary_file plugin\"\n        end\n        if Fluent.windows?\n          log.warn \"symlink_path is unavailable on Windows platform. disabled.\"\n          @symlink_path = nil\n        else\n          placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|\n            begin\n              v.validate!\n            rescue Fluent::ConfigError => e\n              log.warn \"#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink.\"\n            end\n          end\n\n          @buffer.extend SymlinkBufferMixin\n          @buffer.symlink_path = @symlink_path\n          @buffer.output_plugin_for_symlink = self\n        end\n\n        if @compress != :text && @buffer.compress != :text && @buffer.compress != @compress_method\n          raise Fluent::ConfigError, \"You cannot specify different compression formats for Buffer (Buffer: #{@buffer.compress}, Self: #{@compress})\"\n        end\n      end\n\n      @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION\n      @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION\n      @need_lock = system_config.workers > 1\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def format(tag, time, record)\n      r = inject_values_to_record(tag, time, record)\n      @formatter.format(tag, time, r)\n    end\n\n    def write(chunk)\n      path = extract_placeholders(@path_template, chunk)\n      FileUtils.mkdir_p File.dirname(path), mode: @dir_perm\n\n      writer = case\n                when @compress_method.nil?\n                  method(:write_without_compression)\n                when @compress_method != :text\n                  if @buffer.compress == :text || @recompress\n                    method(:write_with_compression).curry.call(@compress_method)\n                  else\n                    method(:write_from_compressed_chunk).curry.call(@compress_method)\n                  end\n                else\n                  raise \"BUG: unknown compression method #{@compress_method}\"\n                end\n\n      if @append\n        if @need_lock\n          acquire_worker_lock(path) do\n            writer.call(path, chunk)\n          end\n        else\n          writer.call(path, chunk)\n        end\n      else\n        find_filepath_available(path, with_lock: @need_lock) do |actual_path|\n          writer.call(actual_path, chunk)\n          path = actual_path\n        end\n      end\n\n      @last_written_path = path\n    end\n\n    def write_without_compression(path, chunk)\n      File.open(path, \"ab\", @file_perm) do |f|\n        chunk.write_to(f)\n      end\n    end\n\n    def write_with_compression(type, path, chunk)\n      File.open(path, \"ab\", @file_perm) do |f|\n        gz = nil\n        if type == :gzip\n          gz = Zlib::GzipWriter.new(f)\n        elsif type == :zstd\n          gz = Zstd::StreamWriter.new(f)\n        end\n        chunk.write_to(gz, compressed: :text)\n        gz.close\n      end\n    end\n\n    def write_from_compressed_chunk(type, path, chunk)\n      File.open(path, \"ab\", @file_perm) do |f|\n        chunk.write_to(f, compressed: type)\n      end\n    end\n\n    def timekey_to_timeformat(timekey)\n      case timekey\n      when nil          then ''\n      when 0...60       then '%Y%m%d%H%M%S' # 60 exclusive\n      when 60...3600    then '%Y%m%d%H%M'\n      when 3600...86400 then '%Y%m%d%H'\n      else                   '%Y%m%d'\n      end\n    end\n\n    def compression_suffix(compress)\n      case compress\n      when :gzip then '.gz'\n      when :zstd then '.zstd'\n      when nil then ''\n      else\n        raise ArgumentError, \"unknown compression type #{compress}\"\n      end\n    end\n\n    # /path/to/dir/file.*      -> /path/to/dir/file.%Y%m%d\n    # /path/to/dir/file.*.data -> /path/to/dir/file.%Y%m%d.data\n    # /path/to/dir/file        -> /path/to/dir/file.%Y%m%d.log\n    #   %Y%m%d -> %Y%m%d_** (non append)\n    # + .gz (gzipped)\n    ## TODO: remove time_slice_format when end of support of compat_parameters\n    def generate_path_template(original, timekey, append, compress, path_suffix: '', time_slice_format: nil)\n      comp_suffix = compression_suffix(compress)\n      index_placeholder = append ? '' : '_**'\n      if original.index('*')\n        raise \"BUG: configuration error must be raised for path including '*' without timekey\" unless timekey\n        time_placeholders_part = time_slice_format || timekey_to_timeformat(timekey)\n        original.gsub('*', time_placeholders_part + index_placeholder) + comp_suffix\n      else\n        if timekey\n          if time_slice_format\n            \"#{original}.#{time_slice_format}#{index_placeholder}#{path_suffix}#{comp_suffix}\"\n          else\n            time_placeholders = timekey_to_timeformat(timekey)\n            if time_placeholders.scan(/../).any?{|ph| original.include?(ph) }\n              raise Fluent::ConfigError, \"insufficient timestamp placeholders in path\" if time_placeholders.scan(/../).any?{|ph| !original.include?(ph) }\n              \"#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}\"\n            else\n              \"#{original}.#{time_placeholders}#{index_placeholder}#{path_suffix}#{comp_suffix}\"\n            end\n          end\n        else\n          \"#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}\"\n        end\n      end\n    end\n\n    def find_filepath_available(path_with_placeholder, with_lock: false) # for non-append\n      raise \"BUG: index placeholder not found in path: #{path_with_placeholder}\" unless path_with_placeholder.index('_**')\n      i = 0\n      dir_path = locked = nil\n      while true\n        path = path_with_placeholder.sub('_**', \"_#{i}\")\n        i += 1\n        next if File.exist?(path)\n\n        if with_lock\n          dir_path = path + '.lock'\n          locked = Dir.mkdir(dir_path) rescue false\n          next unless locked\n          # ensure that other worker doesn't create a file (and release lock)\n          # between previous File.exist? and Dir.mkdir\n          next if File.exist?(path)\n        end\n\n        break\n      end\n      yield path\n    ensure\n      if dir_path && locked && Dir.exist?(dir_path)\n        Dir.rmdir(dir_path) rescue nil\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_forward/ack_handler.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin_helper/socket'\nrequire 'fluent/engine'\nrequire 'fluent/clock'\n\nmodule Fluent::Plugin\n  class ForwardOutput < Output\n    class AckHandler\n      module Result\n        SUCCESS = :success\n        FAILED = :failed\n        CHUNKID_UNMATCHED = :chunkid_unmatched\n      end\n\n      def initialize(timeout:, log:, read_length:)\n        @mutex = Mutex.new\n        @ack_waitings = []\n        @timeout = timeout\n        @log = log\n        @read_length = read_length\n        @unpacker = Fluent::MessagePackFactory.msgpack_unpacker\n      end\n\n      def collect_response(select_interval)\n        now = Fluent::Clock.now\n        sockets = []\n        results = []\n        begin\n          new_list = []\n          @mutex.synchronize do\n            @ack_waitings.each do |info|\n              if info.expired?(now)\n                # There are 2 types of cases when no response has been received from socket:\n                # (1) the node does not support sending responses\n                # (2) the node does support sending response but responses have not arrived for some reasons.\n                @log.warn 'no response from node. regard it as unavailable.', host: info.node.host, port: info.node.port\n                results << [info, Result::FAILED]\n              else\n                sockets << info.sock\n                new_list << info\n              end\n            end\n            @ack_waitings = new_list\n          end\n\n          begin\n            readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)\n          rescue IOError\n            @log.info \"connection closed while waiting for readable sockets\"\n            readable_sockets = nil\n          end\n\n          if readable_sockets\n            readable_sockets.each do |sock|\n              results << read_ack_from_sock(sock)\n            end\n          end\n\n          results.each do |info, ret|\n            if info.nil?\n              yield nil, nil, nil, ret\n            else\n              yield info.chunk_id, info.node, info.sock, ret\n            end\n          end\n        rescue => e\n          @log.error 'unexpected error while receiving ack', error: e\n          @log.error_backtrace\n        end\n      end\n\n      ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time) do\n        def expired?(now)\n          expired_time < now\n        end\n      end\n\n      Ack = Struct.new(:chunk_id, :node, :handler) do\n        def enqueue(sock)\n          handler.enqueue(node, sock, chunk_id)\n        end\n      end\n\n      def create_ack(chunk_id, node)\n        Ack.new(chunk_id, node, self)\n      end\n\n      def enqueue(node, sock, cid)\n        info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout)\n        @mutex.synchronize do\n          @ack_waitings << info\n        end\n      end\n\n      private\n\n      def read_ack_from_sock(sock)\n        begin\n          raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)\n        rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial\n          raw_data = ''\n        rescue IOError\n          @log.info \"socket closed while receiving ack response\"\n          return nil, Result::FAILED\n        end\n\n        info = find(sock)\n\n        if info.nil?\n          # The info can be deleted by another thread during `sock.recv()` and `find()`.\n          # This is OK since another thread has completed to process the ack, so we can skip this.\n          # Note: exclusion mechanism about `collect_response()` may need to be considered.\n          @log.debug \"could not find the ack info. this ack may be processed by another thread.\"\n          return nil, Result::FAILED\n        elsif raw_data.empty?\n          # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.\n          # If this happens we assume the data wasn't delivered and retry it.\n          @log.warn 'destination node closed the connection. regard it as unavailable.', host: info.node.host, port: info.node.port\n          # info.node.disable!\n          return info, Result::FAILED\n        else\n          @unpacker.feed(raw_data)\n          res = @unpacker.read\n          @log.trace 'getting response from destination', host: info.node.host, port: info.node.port, chunk_id: dump_unique_id_hex(info.chunk_id), response: res\n          if res['ack'] != info.chunk_id_base64\n            # Some errors may have occurred when ack and chunk id is different, so send the chunk again.\n            @log.warn 'ack in response and chunk id in sent data are different', chunk_id: dump_unique_id_hex(info.chunk_id), ack: res['ack']\n            return info, Result::CHUNKID_UNMATCHED\n          else\n            @log.trace 'got a correct ack response', chunk_id: dump_unique_id_hex(info.chunk_id)\n          end\n\n          return info, Result::SUCCESS\n        end\n      rescue => e\n        @log.error 'unexpected error while receiving ack message', error: e\n        @log.error_backtrace\n        [nil, Result::FAILED]\n      ensure\n        delete(info)\n      end\n\n      def dump_unique_id_hex(unique_id)\n        Fluent::UniqueId.hex(unique_id)\n      end\n\n      def find(sock)\n        @mutex.synchronize do\n          @ack_waitings.find { |info| info.sock == sock }\n        end\n      end\n\n      def delete(info)\n        @mutex.synchronize do\n          @ack_waitings.delete(info)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_forward/connection_manager.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\n\nmodule Fluent::Plugin\n  class ForwardOutput < Output\n    class ConnectionManager\n      RequestInfo = Struct.new(:state, :shared_key_nonce, :auth)\n\n      # @param log [Logger]\n      # @param secure [Boolean]\n      # @param connection_factory [Proc]\n      # @param socket_cache [Fluent::ForwardOutput::SocketCache]\n      def initialize(log:, secure:, connection_factory:, socket_cache:)\n        @log = log\n        @secure = secure\n        @connection_factory = connection_factory\n        @socket_cache = socket_cache\n      end\n\n      def stop\n        @socket_cache && @socket_cache.clear\n      end\n\n      # @param ack [Fluent::Plugin::ForwardOutput::AckHandler::Ack|nil]\n      def connect(host:, port:, hostname:, ack: nil, &block)\n        if @socket_cache\n          return connect_keepalive(host: host, port: port, hostname: hostname, ack: ack, &block)\n        end\n\n        @log.debug('connect new socket')\n        socket = @connection_factory.call(host, port, hostname)\n        request_info = RequestInfo.new(@secure ? :helo : :established)\n\n        unless block_given?\n          return [socket, request_info]\n        end\n\n        begin\n          yield(socket, request_info)\n        ensure\n          if ack\n            ack.enqueue(socket)\n          else\n            socket.close_write rescue nil\n            socket.close rescue nil\n          end\n        end\n      end\n\n      def purge_obsolete_socks\n        unless @socket_cache\n          raise \"Do not call this method without keepalive option\"\n        end\n        @socket_cache.purge_obsolete_socks\n      end\n\n      def close(sock)\n        if @socket_cache\n          @socket_cache.checkin(sock)\n        else\n          sock.close_write rescue nil\n          sock.close rescue nil\n        end\n      end\n\n      private\n\n      def connect_keepalive(host:, port:, hostname:, ack: nil)\n        request_info = RequestInfo.new(:established)\n        socket = @socket_cache.checkout_or([host, port, hostname]) do\n          s = @connection_factory.call(host, port, hostname)\n          request_info = RequestInfo.new(@secure ? :helo : :established) # overwrite if new connection\n          s\n        end\n\n        unless block_given?\n          return [socket, request_info]\n        end\n\n        ret = nil\n        begin\n          ret = yield(socket, request_info)\n        rescue\n          @socket_cache.revoke(socket)\n          raise\n        else\n          if ack\n            ack.enqueue(socket)\n          else\n            @socket_cache.checkin(socket)\n          end\n        end\n\n        ret\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_forward/error.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\n\nmodule Fluent::Plugin\n  class ForwardOutput < Output\n    class Error < StandardError; end\n    class NoNodesAvailable < Error; end\n    class ConnectionClosedError < Error; end\n    class HandshakeError < Error; end\n    class HeloError < HandshakeError; end\n    class PingpongError < HandshakeError; end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_forward/failure_detector.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\n\nmodule Fluent::Plugin\n  class ForwardOutput < Output\n    class FailureDetector\n      PHI_FACTOR = 1.0 / Math.log(10.0)\n      SAMPLE_SIZE = 1000\n\n      def initialize(heartbeat_interval, hard_timeout, init_last)\n        @heartbeat_interval = heartbeat_interval\n        @last = init_last\n        @hard_timeout = hard_timeout\n\n        # microsec\n        @init_gap = (heartbeat_interval * 1e6).to_i\n        @window = [@init_gap]\n      end\n\n      def hard_timeout?(now)\n        now - @last > @hard_timeout\n      end\n\n      def add(now)\n        if @window.empty?\n          @window << @init_gap\n          @last = now\n        else\n          gap = now - @last\n          @window << (gap * 1e6).to_i\n          @window.shift if @window.length > SAMPLE_SIZE\n          @last = now\n        end\n      end\n\n      def phi(now)\n        size = @window.size\n        return 0.0 if size == 0\n\n        # Calculate weighted moving average\n        mean_usec = 0\n        fact = 0\n        @window.each_with_index {|gap,i|\n          mean_usec += gap * (1+i)\n          fact += (1+i)\n        }\n        mean_usec = mean_usec / fact\n\n        # Normalize arrive intervals into 1sec\n        mean = (mean_usec.to_f / 1e6) - @heartbeat_interval + 1\n\n        # Calculate phi of the phi accrual failure detector\n        t = now - @last - @heartbeat_interval + 1\n        phi = PHI_FACTOR * t / mean\n\n        return phi\n      end\n\n      def sample_size\n        @window.size\n      end\n\n      def clear\n        @window.clear\n        @last = 0\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_forward/handshake_protocol.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/out_forward/error'\nrequire 'digest'\n\nmodule Fluent::Plugin\n  class ForwardOutput < Output\n    class HandshakeProtocol\n      def initialize(log:, hostname:, shared_key:, password:, username:)\n        @log = log\n        @hostname = hostname\n        @shared_key = shared_key\n        @password = password\n        @username = username\n        @shared_key_salt = generate_salt\n      end\n\n      def invoke(sock, ri, data)\n        @log.trace __callee__\n\n        case ri.state\n        when :helo\n          unless check_helo(ri, data)\n            raise HeloError, 'received invalid helo message'\n          end\n\n          sock.write(generate_ping(ri).to_msgpack)\n          ri.state = :pingpong\n        when :pingpong\n          succeeded, reason = check_pong(ri, data)\n          unless succeeded\n            raise PingpongError, reason\n          end\n\n          ri.state = :established\n        else\n          raise \"BUG: unknown session state: #{ri.state}\"\n        end\n      end\n\n      private\n\n      def check_pong(ri, message)\n        @log.debug('checking pong')\n        # ['PONG', bool(authentication result), 'reason if authentication failed',\n        #  self_hostname, sha512\\_hex(salt + self_hostname + nonce + sharedkey)]\n        unless message.size == 5 && message[0] == 'PONG'\n          return false, 'invalid format for PONG message'\n        end\n        _pong, auth_result, reason, hostname, shared_key_hexdigest = message\n\n        unless auth_result\n          return false, 'authentication failed: ' + reason\n        end\n\n        if hostname == @hostname\n          return false, 'same hostname between input and output: invalid configuration'\n        end\n\n        clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(ri.shared_key_nonce).update(@shared_key).hexdigest\n        unless shared_key_hexdigest == clientside\n          return false, 'shared key mismatch'\n        end\n\n        [true, nil]\n      end\n\n      def check_helo(ri, message)\n        @log.debug('checking helo')\n        # ['HELO', options(hash)]\n        unless message.size == 2 && message[0] == 'HELO'\n          return false\n        end\n\n        opts = message[1] || {}\n        # make shared_key_check failed (instead of error) if protocol version mismatch exist\n        ri.shared_key_nonce = opts['nonce'] || ''\n        ri.auth = opts['auth'] || ''\n        true\n      end\n\n      def generate_ping(ri)\n        @log.debug('generating ping')\n        # ['PING', self_hostname, sharedkey\\_salt, sha512\\_hex(sharedkey\\_salt + self_hostname + nonce + shared_key),\n        #  username || '', sha512\\_hex(auth\\_salt + username + password) || '']\n        shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt)\n          .update(@hostname)\n          .update(ri.shared_key_nonce)\n          .update(@shared_key)\n          .hexdigest\n        ping = ['PING', @hostname, @shared_key_salt, shared_key_hexdigest]\n        if !ri.auth.empty?\n          if @username.nil? || @password.nil?\n            raise PingpongError, \"username and password are required\"\n          end\n\n          password_hexdigest = Digest::SHA512.new.update(ri.auth).update(@username).update(@password).hexdigest\n          ping.push(@username, password_hexdigest)\n        else\n          ping.push('', '')\n        end\n        ping\n      end\n\n      def generate_salt\n        SecureRandom.hex(16)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_forward/load_balancer.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/out_forward/error'\n\nmodule Fluent::Plugin\n  class ForwardOutput < Output\n    class LoadBalancer\n      def initialize(log)\n        @log = log\n        @weight_array = []\n        @rand_seed = Random.new.seed\n        @rr = 0\n        @mutex = Mutex.new\n      end\n\n      def select_healthy_node\n        error = nil\n\n        # Don't care about the change of @weight_array's size while looping since\n        # it's only used for determining the number of loops and it is not so important.\n        wlen = @weight_array.size\n        wlen.times do\n          node = @mutex.synchronize do\n            r = @rr % @weight_array.size\n            @rr = (r + 1) % @weight_array.size\n            @weight_array[r]\n          end\n          next unless node.available?\n\n          begin\n            ret = yield node\n            return ret, node\n          rescue\n            # for load balancing during detecting crashed servers\n            error = $!  # use the latest error\n          end\n        end\n\n        raise error if error\n        raise NoNodesAvailable, \"no nodes are available\"\n      end\n\n      def rebuild_weight_array(nodes)\n        standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|\n          n.standby?\n        }\n\n        lost_weight = 0\n        regular_nodes.each {|n|\n          unless n.available?\n            lost_weight += n.weight\n          end\n        }\n        @log.debug(\"rebuilding weight array\", lost_weight: lost_weight)\n\n        if lost_weight > 0\n          standby_nodes.each {|n|\n            if n.available?\n              regular_nodes << n\n              @log.warn \"using standby node #{n.host}:#{n.port}\", weight: n.weight\n              lost_weight -= n.weight\n              break if lost_weight <= 0\n            end\n          }\n        end\n\n        weight_array = []\n        if regular_nodes.empty?\n          @log.warn('No nodes are available')\n          @mutex.synchronize do\n            @weight_array = weight_array\n          end\n          return @weight_array\n        end\n\n        gcd = regular_nodes.map {|n| n.weight }.inject(0) {|r,w| r.gcd(w) }\n        regular_nodes.each {|n|\n          (n.weight / gcd).times {\n            weight_array << n\n          }\n        }\n\n        # for load balancing during detecting crashed servers\n        coe = (regular_nodes.size * 6) / weight_array.size\n        weight_array *= coe if coe > 1\n\n        r = Random.new(@rand_seed)\n        weight_array.sort_by! { r.rand }\n\n        @mutex.synchronize do\n          @weight_array = weight_array\n        end\n      end\n\n      alias select_service select_healthy_node\n      alias rebalance rebuild_weight_array\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_forward/socket_cache.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\n\nmodule Fluent::Plugin\n  class ForwardOutput < Output\n    class SocketCache\n      TimedSocket = Struct.new(:timeout, :key, :sock)\n\n      def initialize(timeout, log)\n        @log = log\n        @timeout = timeout\n        @available_sockets = Hash.new { |obj, k| obj[k] = [] }\n        @inflight_sockets = {}\n        @inactive_sockets = []\n        @mutex = Mutex.new\n      end\n\n      def checkout_or(key)\n        @mutex.synchronize do\n          tsock = pick_socket(key)\n\n          if tsock\n            tsock.sock\n          else\n            sock = yield\n            new_tsock = TimedSocket.new(timeout, key, sock)\n            @log.debug(\"connect new socket #{new_tsock}\")\n\n            @inflight_sockets[sock] = new_tsock\n            new_tsock.sock\n          end\n        end\n      end\n\n      def checkin(sock)\n        @mutex.synchronize do\n          if (s = @inflight_sockets.delete(sock))\n            s.timeout = timeout\n            @available_sockets[s.key] << s\n          else\n            @log.debug(\"there is no socket #{sock}\")\n          end\n        end\n      end\n\n      def revoke(sock)\n        @mutex.synchronize do\n          if (s = @inflight_sockets.delete(sock))\n            @inactive_sockets << s\n          else\n            @log.debug(\"there is no socket #{sock}\")\n          end\n        end\n      end\n\n      def purge_obsolete_socks\n        sockets = []\n\n        @mutex.synchronize do\n          # don't touch @inflight_sockets\n\n          @available_sockets.each do |_, socks|\n            socks.each do |sock|\n              if expired_socket?(sock)\n                sockets << sock\n                socks.delete(sock)\n              end\n            end\n          end\n\n          # reuse same object (@available_sockets)\n          @available_sockets.reject! { |_, v| v.empty? }\n\n          sockets += @inactive_sockets\n          @inactive_sockets.clear\n        end\n\n        sockets.each do |s|\n          s.sock.close rescue nil\n        end\n      end\n\n      def clear\n        sockets = []\n        @mutex.synchronize do\n          sockets += @available_sockets.values.flat_map { |v| v }\n          sockets += @inflight_sockets.values\n          sockets += @inactive_sockets\n\n          @available_sockets.clear\n          @inflight_sockets.clear\n          @inactive_sockets.clear\n        end\n\n        sockets.each do |s|\n          s.sock.close rescue nil\n        end\n      end\n\n      private\n\n      # this method is not thread safe\n      def pick_socket(key)\n        if @available_sockets[key].empty?\n          return nil\n        end\n\n        t = Time.now\n        if (s = @available_sockets[key].find { |sock| !expired_socket?(sock, time: t) })\n          @inflight_sockets[s.sock] = @available_sockets[key].delete(s)\n          s.timeout = timeout\n          s\n        else\n          nil\n        end\n      end\n\n      def timeout\n        @timeout && Time.now + @timeout\n      end\n\n      def expired_socket?(sock, time: Time.now)\n        sock.timeout ? sock.timeout < time : false\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_forward.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/output'\nrequire 'fluent/config/error'\nrequire 'fluent/clock'\nrequire 'fluent/tls'\nrequire 'base64'\nrequire 'forwardable'\n\nrequire 'fluent/compat/socket_util'\nrequire 'fluent/plugin/out_forward/handshake_protocol'\nrequire 'fluent/plugin/out_forward/load_balancer'\nrequire 'fluent/plugin/out_forward/socket_cache'\nrequire 'fluent/plugin/out_forward/failure_detector'\nrequire 'fluent/plugin/out_forward/error'\nrequire 'fluent/plugin/out_forward/connection_manager'\nrequire 'fluent/plugin/out_forward/ack_handler'\n\nmodule Fluent::Plugin\n  class ForwardOutput < Output\n    Fluent::Plugin.register_output('forward', self)\n\n    helpers :socket, :server, :timer, :thread, :compat_parameters, :service_discovery\n\n    LISTEN_PORT = 24224\n\n    desc 'The transport protocol.'\n    config_param :transport, :enum, list: [:tcp, :tls], default: :tcp\n    # TODO: TLS session cache/tickets\n\n    desc 'The timeout time when sending event logs.'\n    config_param :send_timeout, :time, default: 60\n    desc 'The timeout time for socket connect'\n    config_param :connect_timeout, :time, default: nil\n    # TODO: add linger_timeout, recv_timeout\n\n    desc 'The protocol to use for heartbeats (default is the same with \"transport\").'\n    config_param :heartbeat_type, :enum, list: [:transport, :tcp, :udp, :none], default: :transport\n    desc 'The interval of the heartbeat packer.'\n    config_param :heartbeat_interval, :time, default: 1\n    desc 'The wait time before accepting a server fault recovery.'\n    config_param :recover_wait, :time, default: 10\n    desc 'The hard timeout used to detect server failure.'\n    config_param :hard_timeout, :time, default: 60\n    desc 'The threshold parameter used to detect server faults.'\n    config_param :phi_threshold, :integer, default: 16\n    desc 'Use the \"Phi accrual failure detector\" to detect server failure.'\n    config_param :phi_failure_detector, :bool, default: true\n\n    desc 'Change the protocol to at-least-once.'\n    config_param :require_ack_response, :bool, default: false  # require in_forward to respond with ack\n\n    ## The reason of default value of :ack_response_timeout:\n    # Linux default tcp_syn_retries is 5 (in many environment)\n    # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)\n    desc 'This option is used when require_ack_response is true.'\n    config_param :ack_response_timeout, :time, default: 190\n\n    desc 'The interval while reading data from server'\n    config_param :read_interval_msec, :integer, default: 50 # 50ms\n    desc 'Reading data size from server'\n    config_param :read_length, :size, default: 512 # 512bytes\n\n    desc 'Set TTL to expire DNS cache in seconds.'\n    config_param :expire_dns_cache, :time, default: nil  # 0 means disable cache\n    desc 'Enable client-side DNS round robin.'\n    config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this\n\n    desc 'Ignore DNS resolution and errors at startup time.'\n    config_param :ignore_network_errors_at_startup, :bool, default: false\n\n    desc 'Verify that a connection can be made with one of out_forward nodes at the time of startup.'\n    config_param :verify_connection_at_startup, :bool, default: false\n\n    desc 'Compress buffered data.'\n    config_param :compress, :enum, list: [:text, :gzip, :zstd], default: :text\n\n    desc 'The default version of TLS transport.'\n    config_param :tls_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION\n    desc 'The cipher configuration of TLS transport.'\n    config_param :tls_ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT\n    desc 'Skip all verification of certificates or not.'\n    config_param :tls_insecure_mode, :bool, default: false\n    desc 'Allow self signed certificates or not.'\n    config_param :tls_allow_self_signed_cert, :bool, default: false\n    desc 'Verify hostname of servers and certificates or not in TLS transport.'\n    config_param :tls_verify_hostname, :bool, default: true\n    desc 'The additional CA certificate path for TLS.'\n    config_param :tls_ca_cert_path, :array, value_type: :string, default: nil\n    desc 'The additional certificate path for TLS.'\n    config_param :tls_cert_path, :array, value_type: :string, default: nil\n    desc 'The client certificate path for TLS.'\n    config_param :tls_client_cert_path, :string, default: nil\n    desc 'The client private key path for TLS.'\n    config_param :tls_client_private_key_path, :string, default: nil\n    desc 'The client private key passphrase for TLS.'\n    config_param :tls_client_private_key_passphrase, :string, default: nil, secret: true\n    desc 'The certificate thumbprint for searching from Windows system certstore.'\n    config_param :tls_cert_thumbprint, :string, default: nil, secret: true\n    desc 'The certificate logical store name on Windows system certstore.'\n    config_param :tls_cert_logical_store_name, :string, default: nil\n    desc 'Enable to use certificate enterprise store on Windows system certstore.'\n    config_param :tls_cert_use_enterprise_store, :bool, default: true\n    desc \"Enable keepalive connection.\"\n    config_param :keepalive, :bool, default: false\n    desc \"Expired time of keepalive. Default value is nil, which means to keep connection as long as possible\"\n    config_param :keepalive_timeout, :time, default: nil\n    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'\n    config_param :send_keepalive_packet, :bool, default: false\n\n    config_section :security, required: false, multi: false do\n      desc 'The hostname'\n      config_param :self_hostname, :string\n      desc 'Shared key for authentication'\n      config_param :shared_key, :string, secret: true\n    end\n\n    config_section :server, param_name: :servers do\n      desc \"The IP address or host name of the server.\"\n      config_param :host, :string\n      desc \"The name of the server. Used for logging and certificate verification in TLS transport (when host is address).\"\n      config_param :name, :string, default: nil\n      desc \"The port number of the host.\"\n      config_param :port, :integer, default: LISTEN_PORT\n      desc \"The shared key per server.\"\n      config_param :shared_key, :string, default: nil, secret: true\n      desc \"The username for authentication.\"\n      config_param :username, :string, default: ''\n      desc \"The password for authentication.\"\n      config_param :password, :string, default: '', secret: true\n      desc \"Marks a node as the standby node for an Active-Standby model between Fluentd nodes.\"\n      config_param :standby, :bool, default: false\n      desc \"The load balancing weight.\"\n      config_param :weight, :integer, default: 60\n    end\n\n    attr_reader :nodes\n\n    config_param :port, :integer, default: LISTEN_PORT, obsoleted: \"User <server> section instead.\"\n    config_param :host, :string, default: nil, obsoleted: \"Use <server> section instead.\"\n\n    config_section :buffer do\n      config_set_default :chunk_keys, [\"tag\"]\n    end\n\n    attr_reader :read_interval, :recover_sample_size\n\n    def initialize\n      super\n\n      @nodes = [] #=> [Node]\n      @loop = nil\n      @thread = nil\n\n      @usock = nil\n      @keep_alive_watcher_interval = 5 # TODO\n      @suspend_flush = false\n      @healthy_nodes_count_metrics = nil\n      @registered_nodes_count_metrics = nil\n    end\n\n    def configure(conf)\n      compat_parameters_convert(conf, :buffer, default_chunk_key: 'tag')\n\n      super\n\n      unless @chunk_key_tag\n        raise Fluent::ConfigError, \"buffer chunk key must include 'tag' for forward output\"\n      end\n\n      @read_interval = @read_interval_msec / 1000.0\n      @recover_sample_size = @recover_wait / @heartbeat_interval\n\n      if @heartbeat_type == :tcp\n        log.warn \"'heartbeat_type tcp' is deprecated. use 'transport' instead.\"\n        @heartbeat_type = :transport\n      end\n\n      if @dns_round_robin && @heartbeat_type == :udp\n        raise Fluent::ConfigError, \"forward output heartbeat type must be 'transport' or 'none' to use dns_round_robin option\"\n      end\n\n      if @transport == :tls\n        # socket helper adds CA cert or signed certificate to same cert store internally so unify it in this place.\n        if @tls_cert_path && !@tls_cert_path.empty?\n          @tls_ca_cert_path = @tls_cert_path\n        end\n        if @tls_ca_cert_path && !@tls_ca_cert_path.empty?\n          @tls_ca_cert_path.each do |path|\n            raise Fluent::ConfigError, \"specified cert path does not exist:#{path}\" unless File.exist?(path)\n            raise Fluent::ConfigError, \"specified cert path is not readable:#{path}\" unless File.readable?(path)\n          end\n        end\n\n        if @tls_insecure_mode\n          log.warn \"TLS transport is configured in insecure way\"\n          @tls_verify_hostname = false\n          @tls_allow_self_signed_cert = true\n        end\n\n        if Fluent.windows?\n          if (@tls_cert_path || @tls_ca_cert_path) && @tls_cert_logical_store_name\n            raise Fluent::ConfigError, \"specified both cert path and tls_cert_logical_store_name is not permitted\"\n          end\n        else\n          raise Fluent::ConfigError, \"This parameter is for only Windows\" if @tls_cert_logical_store_name\n          raise Fluent::ConfigError, \"This parameter is for only Windows\" if @tls_cert_thumbprint\n        end\n      end\n\n      @ack_handler = @require_ack_response ? AckHandler.new(timeout: @ack_response_timeout, log: @log, read_length: @read_length) : nil\n      socket_cache = @keepalive ? SocketCache.new(@keepalive_timeout, @log) : nil\n      @connection_manager = ConnectionManager.new(\n        log: @log,\n        secure: !!@security,\n        connection_factory: method(:create_transfer_socket),\n        socket_cache: socket_cache,\n      )\n\n      service_discovery_configure(\n        :out_forward_service_discovery_watcher,\n        static_default_service_directive: 'server',\n        load_balancer: LoadBalancer.new(log),\n        custom_build_method: method(:build_node),\n      )\n\n      service_discovery_services.each do |server|\n        # it's only for test\n        @nodes << server\n        unless @heartbeat_type == :none\n          begin\n            server.validate_host_resolution!\n          rescue => e\n            raise unless @ignore_network_errors_at_startup\n            log.warn \"failed to resolve node name when configured\", server: (server.name || server.host), error: e\n            server.disable!\n          end\n        end\n      end\n\n      unless @as_secondary\n        if @buffer.compress == :text\n          @buffer.compress = @compress\n        else\n          if @compress == :text\n            log.info \"buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>\"\n          elsif @compress != @buffer.compress\n            raise Fluent::ConfigError, \"You cannot specify different compression formats for Buffer (Buffer: #{@buffer.compress}, Self: #{@compress})\"\n          end\n        end\n      end\n\n      if service_discovery_services.empty?\n        raise Fluent::ConfigError, \"forward output plugin requires at least one node is required. Add <server> or <service_discovery>\"\n      end\n\n      if !@keepalive && @keepalive_timeout\n        log.warn('The value of keepalive_timeout is ignored. if you want to use keepalive, please add `keepalive true` to your conf.')\n      end\n\n      if @send_keepalive_packet && !@keepalive\n        raise Fluent::ConfigError, \"'send_keepalive_packet' is enabled but 'keepalive' is not. Enable 'keepalive' to use TCP keepalive.\"\n      end\n\n      raise Fluent::ConfigError, \"ack_response_timeout must be a positive integer\" if @ack_response_timeout < 1\n\n      if @compress == :zstd\n        log.warn \"zstd compression feature is an experimental new feature supported since v1.19.0.\" +\n                 \" Please make sure that the destination server also supports this feature before using it.\" +\n                 \" in_forward plugin for Fluentd supports it since v1.19.0.\"\n      end\n\n      @healthy_nodes_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"healthy_nodes_count\", help_text: \"Number of count healthy nodes\", prefer_gauge: true)\n      @registered_nodes_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"registered_nodes_count\", help_text: \"Number of count registered nodes\", prefer_gauge: true)\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def prefer_delayed_commit\n      @require_ack_response\n    end\n\n    def overwrite_delayed_commit_timeout\n      # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout\n      # But it should be overwritten by ack_response_timeout to rollback chunks after timeout\n      if @delayed_commit_timeout != @ack_response_timeout\n        log.info \"delayed_commit_timeout is overwritten by ack_response_timeout\"\n        @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s\n      end\n    end\n\n    def start\n      super\n\n      unless @heartbeat_type == :none\n        if @heartbeat_type == :udp\n          @usock = socket_create_udp(service_discovery_services.first.host, service_discovery_services.first.port, nonblock: true)\n          server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length, &method(:on_udp_heartbeat_response_recv))\n        end\n        timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_heartbeat_timer))\n      end\n\n      if @require_ack_response\n        overwrite_delayed_commit_timeout\n        thread_create(:out_forward_receiving_ack, &method(:ack_reader))\n      end\n\n      if @verify_connection_at_startup\n        service_discovery_services.each do |node|\n          begin\n            node.verify_connection\n          rescue StandardError => e\n            log.fatal \"forward's connection setting error: #{e.message}\"\n            raise Fluent::UnrecoverableError, e.message\n          end\n        end\n      end\n\n      if @keepalive\n        timer_execute(:out_forward_keep_alived_socket_watcher, @keep_alive_watcher_interval, &method(:on_purge_obsolete_socks))\n      end\n    end\n\n    def close\n      if @usock\n        # close socket and ignore errors: this socket will not be used anyway.\n        @usock.close rescue nil\n      end\n\n      super\n    end\n\n    def stop\n      super\n\n      if @keepalive\n        @connection_manager.stop\n      end\n    end\n\n    def before_shutdown\n      super\n      @suspend_flush = true\n    end\n\n    def after_shutdown\n      last_ack if @require_ack_response\n      super\n    end\n\n    def try_flush\n      return if @require_ack_response && @suspend_flush\n      super\n    end\n\n    def last_ack\n      overwrite_delayed_commit_timeout\n      ack_check(ack_select_interval)\n    end\n\n    def write(chunk)\n      return if chunk.empty?\n      tag = chunk.metadata.tag\n\n      service_discovery_select_service { |node| node.send_data(tag, chunk) }\n    end\n\n    def try_write(chunk)\n      log.trace \"writing a chunk to destination\", chunk_id: dump_unique_id_hex(chunk.unique_id)\n      if chunk.empty?\n        commit_write(chunk.unique_id)\n        return\n      end\n      tag = chunk.metadata.tag\n      service_discovery_select_service { |node| node.send_data(tag, chunk) }\n      last_ack if @require_ack_response && @suspend_flush\n    end\n\n    def create_transfer_socket(host, port, hostname, &block)\n      case @transport\n      when :tls\n        socket_create_tls(\n          host, port,\n          version: @tls_version,\n          ciphers: @tls_ciphers,\n          insecure: @tls_insecure_mode,\n          verify_fqdn: @tls_verify_hostname,\n          fqdn: hostname,\n          allow_self_signed_cert: @tls_allow_self_signed_cert,\n          cert_paths: @tls_ca_cert_path,\n          cert_path: @tls_client_cert_path,\n          private_key_path: @tls_client_private_key_path,\n          private_key_passphrase: @tls_client_private_key_passphrase,\n          cert_thumbprint: @tls_cert_thumbprint,\n          cert_logical_store_name: @tls_cert_logical_store_name,\n          cert_use_enterprise_store: @tls_cert_use_enterprise_store,\n\n          # Enabling SO_LINGER causes tcp port exhaustion on Windows.\n          # This is because dynamic ports are only 16384 (from 49152 to 65535) and\n          # expiring SO_LINGER enabled ports should wait 4 minutes\n          # where set by TcpTimeDelay. Its default value is 4 minutes.\n          # So, we should disable SO_LINGER on Windows to prevent flood of waiting ports.\n          linger_timeout: Fluent.windows? ? nil : @send_timeout,\n          send_timeout: @send_timeout,\n          recv_timeout: @ack_response_timeout,\n          connect_timeout: @connect_timeout,\n          send_keepalive_packet: @send_keepalive_packet,\n          &block\n        )\n      when :tcp\n        socket_create_tcp(\n          host, port,\n          linger_timeout: @send_timeout,\n          send_timeout: @send_timeout,\n          recv_timeout: @ack_response_timeout,\n          connect_timeout: @connect_timeout,\n          send_keepalive_packet: @send_keepalive_packet,\n          &block\n        )\n      else\n        raise \"BUG: unknown transport protocol #{@transport}\"\n      end\n    end\n\n    def statistics\n      stats = super\n      services = service_discovery_services\n      @healthy_nodes_count_metrics.set(0)\n      @registered_nodes_count_metrics.set(services.size)\n      services.each do |s|\n        if s.available?\n          @healthy_nodes_count_metrics.inc\n        end\n      end\n\n      stats = {\n        'output' => stats[\"output\"].merge({\n          'healthy_nodes_count' => @healthy_nodes_count_metrics.get,\n          'registered_nodes_count' => @registered_nodes_count_metrics.get,\n        })\n      }\n      stats\n    end\n\n    # MessagePack FixArray length is 3\n    FORWARD_HEADER = [0x93].pack('C').freeze\n    def forward_header\n      FORWARD_HEADER\n    end\n\n    private\n\n    def build_node(server)\n      name = server.name || \"#{server.host}:#{server.port}\"\n      log.info \"adding forwarding server '#{name}'\", host: server.host, port: server.port, weight: server.weight, plugin_id: plugin_id\n\n      failure = FailureDetector.new(@heartbeat_interval, @hard_timeout, Time.now.to_i.to_f)\n      if @heartbeat_type == :none\n        NoneHeartbeatNode.new(self, server, failure: failure, connection_manager: @connection_manager, ack_handler: @ack_handler)\n      else\n        Node.new(self, server, failure: failure, connection_manager: @connection_manager, ack_handler: @ack_handler)\n      end\n    end\n\n    def on_heartbeat_timer\n      need_rebuild = false\n      service_discovery_services.each do |n|\n        begin\n          log.trace \"sending heartbeat\", host: n.host, port: n.port, heartbeat_type: @heartbeat_type\n          n.usock = @usock if @usock\n          need_rebuild = n.send_heartbeat || need_rebuild\n        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED, Errno::ETIMEDOUT => e\n          log.debug \"failed to send heartbeat packet\", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e\n        rescue => e\n          log.debug \"unexpected error happen during heartbeat\", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e\n        end\n\n        need_rebuild = n.tick || need_rebuild\n      end\n\n      if need_rebuild\n        service_discovery_rebalance\n      end\n    end\n\n    def on_udp_heartbeat_response_recv(data, sock)\n      sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)\n      if node = service_discovery_services.find { |n| n.sockaddr == sockaddr }\n        # log.trace \"heartbeat arrived\", name: node.name, host: node.host, port: node.port\n        if node.heartbeat\n          service_discovery_rebalance\n        end\n      else\n        log.warn(\"Unknown heartbeat response received from #{sock.remote_host}:#{sock.remote_port}. It may service out\")\n      end\n    end\n\n    def on_purge_obsolete_socks\n      @connection_manager.purge_obsolete_socks\n    end\n\n    def ack_select_interval\n      if @delayed_commit_timeout > 3\n        1\n      else\n        @delayed_commit_timeout / 3.0\n      end\n    end\n\n    def ack_reader\n      select_interval = ack_select_interval\n\n      while thread_current_running?\n        ack_check(select_interval)\n      end\n    end\n\n    def ack_check(select_interval)\n      @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|\n        @connection_manager.close(sock)\n\n        case result\n        when AckHandler::Result::SUCCESS\n          commit_write(chunk_id)\n        when AckHandler::Result::FAILED\n          node&.disable!\n          rollback_write(chunk_id, update_retry: false) if chunk_id\n        when AckHandler::Result::CHUNKID_UNMATCHED\n          rollback_write(chunk_id, update_retry: false)\n        else\n          log.warn(\"BUG: invalid status #{result} #{chunk_id}\")\n\n          if chunk_id\n            rollback_write(chunk_id, update_retry: false)\n          end\n        end\n      end\n    end\n\n    class Node\n      extend Forwardable\n      def_delegators :@server, :discovery_id, :host, :port, :name, :weight, :standby\n\n      # @param connection_manager [Fluent::Plugin::ForwardOutput::ConnectionManager]\n      # @param ack_handler [Fluent::Plugin::ForwardOutput::AckHandler]\n      def initialize(sender, server, failure:, connection_manager:, ack_handler:)\n        @sender = sender\n        @log = sender.log\n        @compress = sender.compress\n        @server = server\n\n        @name = server.name\n        @host = server.host\n        @port = server.port\n        @weight = server.weight\n        @standby = server.standby\n        @failure = failure\n        @available = true\n\n        # @hostname is used for certificate verification & TLS SNI\n        host_is_hostname = !(IPAddr.new(@host) rescue false)\n        @hostname = case\n                    when host_is_hostname then @host\n                    when @name then @name\n                    else nil\n                    end\n\n        @usock = nil\n\n        @handshake = HandshakeProtocol.new(\n          log: @log,\n          hostname: sender.security&.self_hostname,\n          shared_key: server.shared_key || sender.security&.shared_key || '',\n          password: server.password || '',\n          username: server.username || '',\n        )\n\n        @resolved_host = nil\n        @resolved_time = 0\n        @resolved_once = false\n\n        @connection_manager = connection_manager\n        @ack_handler = ack_handler\n      end\n\n      attr_accessor :usock\n\n      attr_reader :state\n      attr_reader :sockaddr  # used by on_udp_heartbeat_response_recv\n      attr_reader :failure # for test\n\n      def validate_host_resolution!\n        resolved_host\n      end\n\n      def available?\n        @available\n      end\n\n      def disable!\n        @available = false\n      end\n\n      def standby?\n        @standby\n      end\n\n      def verify_connection\n        connect do |sock, ri|\n          ensure_established_connection(sock, ri)\n        end\n      end\n\n      def establish_connection(sock, ri)\n        start_time = Fluent::Clock.now\n        timeout = @sender.hard_timeout\n\n        while ri.state != :established\n          # Check for timeout to prevent infinite loop\n          if Fluent::Clock.now - start_time > timeout\n            @log.warn \"handshake timeout after #{timeout}s\", host: @host, port: @port\n            disable!\n            break\n          end\n\n          begin\n            # TODO: On Ruby 2.2 or earlier, read_nonblock doesn't work expectedly.\n            # We need rewrite around here using new socket/server plugin helper.\n            buf = sock.read_nonblock(@sender.read_length)\n            if buf.empty?\n              sleep @sender.read_interval\n              next\n            end\n            Fluent::MessagePackFactory.msgpack_unpacker.feed_each(buf) do |data|\n              if @handshake.invoke(sock, ri, data) == :established\n                @log.debug \"connection established\", host: @host, port: @port\n              end\n            end\n          rescue IO::WaitReadable\n            # If the exception is Errno::EWOULDBLOCK or Errno::EAGAIN, it is extended by IO::WaitReadable.\n            # So IO::WaitReadable can be used to rescue the exceptions for retrying read_nonblock.\n            # https//docs.ruby-lang.org/en/2.3.0/IO.html#method-i-read_nonblock\n            sleep @sender.read_interval unless ri.state == :established\n          rescue SystemCallError => e\n            @log.warn \"disconnected by error\", host: @host, port: @port, error: e\n            disable!\n            break\n          rescue EOFError\n            @log.warn \"disconnected\", host: @host, port: @port\n            disable!\n            break\n          rescue HeloError => e\n            @log.warn \"received invalid helo message from #{@name}\"\n            disable!\n            break\n          rescue PingpongError => e\n            @log.warn \"connection refused to #{@name || @host}: #{e.message}\"\n            disable!\n            break\n          end\n        end\n      end\n\n      def send_data_actual(sock, tag, chunk)\n        option = { 'size' => chunk.size, 'compressed' => @compress }\n        option['chunk'] = Base64.encode64(chunk.unique_id) if @ack_handler\n\n        # https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#packedforward-mode\n        # out_forward always uses str32 type for entries.\n        # str16 can store only 64kbytes, and it should be much smaller than buffer chunk size.\n\n        tag = tag.dup.force_encoding(Encoding::UTF_8)\n\n        sock.write @sender.forward_header                    # array, size=3\n        sock.write tag.to_msgpack                            # 1. tag: String (str)\n        chunk.open(compressed: @compress) do |chunk_io|\n          entries = [0xdb, chunk_io.size].pack('CN')\n          sock.write entries.force_encoding(Encoding::UTF_8) # 2. entries: String (str32)\n          IO.copy_stream(chunk_io, sock)                     #    writeRawBody(packed_es)\n        end\n        sock.write option.to_msgpack                         # 3. option: Hash(map)\n\n        # TODO: use bin32 for non-utf8 content(entries) when old msgpack-ruby (0.5.x or earlier) not supported\n      end\n\n      def send_data(tag, chunk)\n        ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self)\n        connect(nil, ack: ack) do |sock, ri|\n          ensure_established_connection(sock, ri)\n          send_data_actual(sock, tag, chunk)\n        end\n\n        heartbeat(false)\n        nil\n      end\n\n      # FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack\n      #\n      # @return [Boolean] return true if it needs to rebuild nodes\n      def send_heartbeat\n        begin\n          dest_addr = resolved_host\n          @resolved_once = true\n        rescue ::SocketError => e\n          if !@resolved_once && @sender.ignore_network_errors_at_startup\n            @log.warn \"failed to resolve node name in heartbeating\", server: @name || @host, error: e\n            return false\n          end\n          raise\n        end\n\n        case @sender.heartbeat_type\n        when :transport\n          connect(dest_addr) do |sock, ri|\n            ensure_established_connection(sock, ri)\n\n            ## don't send any data to not cause a compatibility problem\n            # sock.write FORWARD_TCP_HEARTBEAT_DATA\n\n            # successful tcp connection establishment is considered as valid heartbeat.\n            # When heartbeat is succeeded after detached, return true. It rebuilds weight array.\n            heartbeat(true)\n          end\n        when :udp\n          @usock.send \"\\0\", 0, Socket.pack_sockaddr_in(@port, dest_addr)\n          # response is going to receive at on_udp_heartbeat_response_recv\n          false\n        when :none # :none doesn't use this class\n          raise \"BUG: heartbeat_type none must not use Node\"\n        else\n          raise \"BUG: unknown heartbeat_type '#{@sender.heartbeat_type}'\"\n        end\n      end\n\n      def resolved_host\n        case @sender.expire_dns_cache\n        when 0\n          # cache is disabled\n          resolve_dns!\n\n        when nil\n          # persistent cache\n          @resolved_host ||= resolve_dns!\n\n        else\n          now = Fluent::EventTime.now\n          rh = @resolved_host\n          if !rh || now - @resolved_time >= @sender.expire_dns_cache\n            rh = @resolved_host = resolve_dns!\n            @resolved_time = now\n          end\n          rh\n        end\n      end\n\n      def resolve_dns!\n        addrinfo_list = Socket.getaddrinfo(@host, @port, nil, Socket::SOCK_STREAM)\n        addrinfo = @sender.dns_round_robin ? addrinfo_list.sample : addrinfo_list.first\n        @sockaddr = Socket.pack_sockaddr_in(addrinfo[1], addrinfo[3]) # used by on_udp_heartbeat_response_recv\n        addrinfo[3]\n      end\n      private :resolve_dns!\n\n      def tick\n        now = Time.now.to_f\n        unless available?\n          if @failure.hard_timeout?(now)\n            @failure.clear\n          end\n          return nil\n        end\n\n        if @failure.hard_timeout?(now)\n          @log.warn \"detached forwarding server '#{@name}'\", host: @host, port: @port, hard_timeout: true\n          disable!\n          @resolved_host = nil  # expire cached host\n          @failure.clear\n          return true\n        end\n\n        if @sender.phi_failure_detector\n          phi = @failure.phi(now)\n          if phi > @sender.phi_threshold\n            @log.warn \"detached forwarding server '#{@name}'\", host: @host, port: @port, phi: phi, phi_threshold: @sender.phi_threshold\n            disable!\n            @resolved_host = nil  # expire cached host\n            @failure.clear\n            return true\n          end\n        end\n        false\n      end\n\n      def heartbeat(detect=true)\n        now = Time.now.to_f\n        @failure.add(now)\n        if detect && !available? && @failure.sample_size > @sender.recover_sample_size\n          @available = true\n          @log.warn \"recovered forwarding server '#{@name}'\", host: @host, port: @port\n          true\n        else\n          nil\n        end\n      end\n\n      private\n\n      def ensure_established_connection(sock, request_info)\n        if request_info.state != :established\n          establish_connection(sock, request_info)\n\n          if request_info.state != :established\n            raise ConnectionClosedError, \"failed to establish connection with node #{@name}\"\n          end\n        end\n      end\n\n      def connect(host = nil, ack: false, &block)\n        @connection_manager.connect(host: host || resolved_host, port: port, hostname: @hostname, ack: ack, &block)\n      end\n    end\n\n    # Override Node to disable heartbeat\n    class NoneHeartbeatNode < Node\n      def available?\n        true\n      end\n\n      def tick\n        false\n      end\n\n      def heartbeat(detect=true)\n        true\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_http.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'net/http'\nrequire 'uri'\nrequire 'openssl'\nrequire 'fluent/tls'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin_helper/socket'\n\n# patch Net::HTTP to support extra_chain_cert which was added in Ruby feature #9758.\n# see: https://github.com/ruby/ruby/commit/31af0dafba6d3769d2a39617c0dddedb97883712\nunless Net::HTTP::SSL_IVNAMES.include?(:@extra_chain_cert)\n  class Net::HTTP\n    SSL_IVNAMES << :@extra_chain_cert\n    SSL_ATTRIBUTES << :extra_chain_cert\n    attr_accessor :extra_chain_cert\n  end\nend\n\nmodule Fluent::Plugin\n  class HTTPOutput < Output\n    Fluent::Plugin.register_output('http', self)\n\n    class RetryableResponse < StandardError; end\n\n    ConnectionCache = Struct.new(:uri, :conn)\n\n    helpers :formatter\n\n    desc 'The endpoint for HTTP request, e.g. http://example.com/api'\n    config_param :endpoint, :string\n    desc 'The method for HTTP request'\n    config_param :http_method, :enum, list: [:put, :post], default: :post\n    desc 'The proxy for HTTP request'\n    config_param :proxy, :string, default: ENV['HTTP_PROXY'] || ENV['http_proxy']\n    desc 'Content-Type for HTTP request'\n    config_param :content_type, :string, default: nil\n    desc 'JSON array data format for HTTP request body'\n    config_param :json_array, :bool, default: false\n    desc 'Additional headers for HTTP request'\n    config_param :headers, :hash, default: nil\n    desc 'Additional placeholder based headers for HTTP request'\n    config_param :headers_from_placeholders, :hash, default: nil\n    desc 'Compress HTTP request body'\n    config_param :compress, :enum, list: [:text, :gzip], default: :text\n\n    desc 'The connection open timeout in seconds'\n    config_param :open_timeout, :integer, default: nil\n    desc 'The read timeout in seconds'\n    config_param :read_timeout, :integer, default: nil\n    desc 'The TLS timeout in seconds'\n    config_param :ssl_timeout, :integer, default: nil\n    desc 'Try to reuse connections'\n    config_param :reuse_connections, :bool, default: false\n\n    desc 'The CA certificate path for TLS'\n    config_param :tls_ca_cert_path, :string, default: nil\n    desc 'The client certificate path for TLS'\n    config_param :tls_client_cert_path, :string, default: nil\n    desc 'The client private key path for TLS'\n    config_param :tls_private_key_path, :string, default: nil\n    desc 'The client private key passphrase for TLS'\n    config_param :tls_private_key_passphrase, :string, default: nil, secret: true\n    desc 'The verify mode of TLS'\n    config_param :tls_verify_mode, :enum, list: [:none, :peer], default: :peer\n    desc 'The default version of TLS'\n    config_param :tls_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION\n    desc 'The cipher configuration of TLS'\n    config_param :tls_ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT\n\n    desc 'Raise UnrecoverableError when the response is non success, 4xx/5xx'\n    config_param :error_response_as_unrecoverable, :bool, default: true\n    desc 'The list of retryable response code'\n    config_param :retryable_response_codes, :array, value_type: :integer, default: nil\n\n    config_section :format do\n      config_set_default :@type, 'json'\n    end\n\n    config_section :auth, required: false, multi: false do\n      desc 'The method for HTTP authentication'\n      config_param :method, :enum, list: [:basic, :aws_sigv4], default: :basic\n      desc 'The username for basic authentication'\n      config_param :username, :string, default: nil\n      desc 'The password for basic authentication'\n      config_param :password, :string, default: nil, secret: true\n      desc 'The AWS service to authenticate against'\n      config_param :aws_service, :string, default: nil\n      desc 'The AWS region to use when authenticating'\n      config_param :aws_region, :string, default: nil\n      desc 'The AWS role ARN to assume when authenticating'\n      config_param :aws_role_arn, :string, default: nil\n    end\n\n    def connection_cache_id_thread_key\n      \"#{plugin_id}_connection_cache_id\"\n    end\n\n    def connection_cache_id_for_thread\n      Thread.current[connection_cache_id_thread_key]\n    end\n\n    def connection_cache_id_for_thread=(id)\n      Thread.current[connection_cache_id_thread_key] = id\n    end\n\n    def initialize\n      super\n\n      @uri = nil\n      @proxy_uri = nil\n      @formatter = nil\n\n      @connection_cache = []\n      @connection_cache_id_mutex = Mutex.new\n      @connection_cache_next_id = 0\n    end\n\n    def close\n      super\n\n      @connection_cache.each {|entry| entry.conn.finish if entry.conn&.started? }\n    end\n\n    def configure(conf)\n      super\n\n      @connection_cache = Array.new(actual_flush_thread_count, ConnectionCache.new(\"\", nil)) if @reuse_connections\n\n      if @retryable_response_codes.nil?\n        log.warn('Status code 503 is going to be removed from default `retryable_response_codes` from fluentd v2. Please add it by yourself if you wish')\n        @retryable_response_codes = [503]\n      end\n\n      @http_opt = setup_http_option\n      @proxy_uri = URI.parse(@proxy) if @proxy\n      @formatter = formatter_create\n      @content_type = setup_content_type unless @content_type\n\n      if @json_array\n        if @formatter_configs.first[:@type] != \"json\"\n          raise Fluent::ConfigError, \"json_array option could be used with json formatter only\"\n        end\n        define_singleton_method(:format, method(:format_json_array))\n      end\n\n      if @auth and @auth.method == :aws_sigv4\n        begin\n          require 'aws-sigv4'\n          require 'aws-sdk-core'\n        rescue LoadError\n          raise Fluent::ConfigError, \"The aws-sdk-core and aws-sigv4 gems are required for aws_sigv4 auth. Run: gem install aws-sdk-core -v '~> 3.191'\"\n        end\n\n        raise Fluent::ConfigError, \"aws_service is required for aws_sigv4 auth\" unless @auth.aws_service != nil\n        raise Fluent::ConfigError, \"aws_region is required for aws_sigv4 auth\" unless @auth.aws_region != nil\n\n        if @auth.aws_role_arn == nil\n          aws_credentials = Aws::CredentialProviderChain.new.resolve\n        else\n          aws_credentials = Aws::AssumeRoleCredentials.new(\n            client: Aws::STS::Client.new(\n              region: @auth.aws_region\n            ),\n            role_arn: @auth.aws_role_arn,\n            role_session_name: \"fluentd\"\n          )\n        end\n\n        @aws_signer = Aws::Sigv4::Signer.new(\n          service: @auth.aws_service,\n          region: @auth.aws_region,\n          credentials_provider: aws_credentials\n        )\n      end\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def formatted_to_msgpack_binary?\n      @formatter_configs.first[:@type] == 'msgpack'\n    end\n\n    def format(tag, time, record)\n      @formatter.format(tag, time, record)\n    end\n\n    def format_json_array(tag, time, record)\n      @formatter.format(tag, time, record) << \",\"\n    end\n\n    def write(chunk)\n      uri = parse_endpoint(chunk)\n      req = create_request(chunk, uri)\n\n      log.debug { \"#{@http_method.capitalize} data to #{uri.to_s} with chunk(#{dump_unique_id_hex(chunk.unique_id)})\" }\n\n      send_request(uri, req)\n    end\n\n    private\n\n    def setup_content_type\n      case @formatter_configs.first[:@type]\n      when 'json'\n        @json_array ? 'application/json' : 'application/x-ndjson'\n      when 'csv'\n        'text/csv'\n      when 'tsv', 'ltsv'\n        'text/tab-separated-values'\n      when 'msgpack'\n        'application/x-msgpack'\n      when 'out_file', 'single_value', 'stdout', 'hash'\n        'text/plain'\n      else\n        raise Fluent::ConfigError, \"can't determine Content-Type from formatter type. Set content_type parameter explicitly\"\n      end\n    end\n\n    def setup_http_option\n      use_ssl = @endpoint.start_with?('https')\n      opt = {\n        open_timeout: @open_timeout,\n        read_timeout: @read_timeout,\n        ssl_timeout: @ssl_timeout,\n        use_ssl: use_ssl\n      }\n\n      if use_ssl\n        if @tls_ca_cert_path\n          raise Fluent::ConfigError, \"tls_ca_cert_path is wrong: #{@tls_ca_cert_path}\" unless File.file?(@tls_ca_cert_path)\n          opt[:ca_file] = @tls_ca_cert_path\n        end\n        if @tls_client_cert_path\n          raise Fluent::ConfigError, \"tls_client_cert_path is wrong: #{@tls_client_cert_path}\" unless File.file?(@tls_client_cert_path)\n\n          bundle = File.read(@tls_client_cert_path)\n          bundle_certs = bundle.scan(/-----BEGIN CERTIFICATE-----(?:.|\\n)+?-----END CERTIFICATE-----/)\n          opt[:cert] = OpenSSL::X509::Certificate.new(bundle_certs[0])\n\n          intermediate_certs = bundle_certs[1..-1]\n          if intermediate_certs\n            opt[:extra_chain_cert] = intermediate_certs.map { |cert| OpenSSL::X509::Certificate.new(cert) }\n          end\n        end\n        if @tls_private_key_path\n          raise Fluent::ConfigError, \"tls_private_key_path is wrong: #{@tls_private_key_path}\" unless File.file?(@tls_private_key_path)\n          opt[:key] = OpenSSL::PKey.read(File.read(@tls_private_key_path), @tls_private_key_passphrase)\n        end\n        opt[:verify_mode] = case @tls_verify_mode\n                            when :none\n                              OpenSSL::SSL::VERIFY_NONE\n                            when :peer\n                              OpenSSL::SSL::VERIFY_PEER\n                            end\n        opt[:ciphers] = @tls_ciphers\n        opt = Fluent::TLS.set_version_to_options(opt, @tls_version, nil, nil)\n      end\n\n      opt\n    end\n\n    def parse_endpoint(chunk)\n      endpoint = extract_placeholders(@endpoint, chunk)\n      URI.parse(endpoint)\n    end\n\n    def set_headers(req, uri, chunk)\n      if @headers\n        @headers.each do |k, v|\n          req[k] = v\n        end\n      end\n      if @headers_from_placeholders\n        @headers_from_placeholders.each do |k, v|\n          req[k] = extract_placeholders(v, chunk)\n        end\n      end\n      if @compress == :gzip\n        req['Content-Encoding'] = \"gzip\"\n      end\n      req['Content-Type'] = @content_type\n    end\n\n    def set_auth(req, uri)\n      return unless @auth\n\n      if @auth.method == :basic\n        req.basic_auth(@auth.username, @auth.password)\n      elsif @auth.method == :aws_sigv4\n        signature = @aws_signer.sign_request(\n          http_method: req.method,\n          url: uri.request_uri,\n          headers: {\n            'Content-Type' => @content_type,\n            'Host' => uri.host\n          },\n          body: req.body\n        )\n        req.add_field('x-amz-date', signature.headers['x-amz-date'])\n        req.add_field('x-amz-security-token', signature.headers['x-amz-security-token'])\n        req.add_field('x-amz-content-sha256', signature.headers['x-amz-content-sha256'])\n        req.add_field('authorization', signature.headers['authorization'])\n      end\n    end\n\n    def create_request(chunk, uri)\n      req = case @http_method\n            when :post\n              Net::HTTP::Post.new(uri.request_uri)\n            when :put\n              Net::HTTP::Put.new(uri.request_uri)\n            end\n      set_headers(req, uri, chunk)\n\n      req.body = @json_array ? \"[#{chunk.read.chop}]\" : chunk.read\n\n      if @compress == :gzip\n        gz = Zlib::GzipWriter.new(StringIO.new)\n        gz << req.body\n        req.body = gz.close.string\n      end\n\n      # At least one authentication method requires the body and other headers, so the order of this call matters\n      set_auth(req, uri)\n      req\n    end\n\n    def make_request_cached(uri, req)\n      id = self.connection_cache_id_for_thread\n      if id.nil?\n        @connection_cache_id_mutex.synchronize {\n          id = @connection_cache_next_id\n          @connection_cache_next_id += 1\n        }\n        self.connection_cache_id_for_thread = id\n      end\n      uri_str = uri.to_s\n      if @connection_cache[id].uri != uri_str\n        @connection_cache[id].conn.finish if @connection_cache[id].conn&.started?\n        http =  if @proxy_uri\n                  Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt)\n                else\n                  Net::HTTP.start(uri.host, uri.port, @http_opt)\n                end\n        @connection_cache[id] = ConnectionCache.new(uri_str, http)\n      end\n      @connection_cache[id].conn.request(req)\n    end\n\n    def make_request(uri, req, &block)\n      if @proxy_uri\n        Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt, &block)\n      else\n        Net::HTTP.start(uri.host, uri.port, @http_opt, &block)\n      end\n    end\n\n    def send_request(uri, req)\n      res = if @reuse_connections\n              make_request_cached(uri, req)\n            else\n              make_request(uri, req) { |http| http.request(req) }\n            end\n\n      if res.is_a?(Net::HTTPSuccess)\n        log.debug { \"#{res.code} #{res.message.rstrip}#{res.body.lstrip}\" }\n      else\n        msg = \"#{res.code} #{res.message.rstrip} #{res.body.lstrip}\"\n\n        if @retryable_response_codes.include?(res.code.to_i)\n          raise RetryableResponse, msg\n        end\n\n        if @error_response_as_unrecoverable\n          raise Fluent::UnrecoverableError, msg\n        else\n          log.error \"got error response from '#{@http_method.capitalize} #{uri.to_s}' : #{msg}\"\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_null.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\n\nmodule Fluent::Plugin\n  class NullOutput < Output\n    # This plugin is for tests of non-buffered/buffered plugins\n    Fluent::Plugin.register_output('null', self)\n\n    desc \"The parameter for testing to simulate output plugin which never succeed to flush.\"\n    config_param :never_flush, :bool, default: false\n\n    config_section :buffer do\n      config_set_default :chunk_keys, ['tag']\n      config_set_default :flush_at_shutdown, true\n      config_set_default :chunk_limit_size, 10 * 1024\n    end\n\n    def prefer_buffered_processing\n      false\n    end\n\n    def prefer_delayed_commit\n      @delayed\n    end\n\n    attr_accessor :feed_proc, :delayed\n\n    def initialize\n      super\n      @delayed = false\n      @feed_proc = nil\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def process(tag, es)\n      raise \"failed to flush\" if @never_flush\n      # Do nothing\n    end\n\n    def write(chunk)\n      raise \"failed to flush\" if @never_flush\n      if @feed_proc\n        @feed_proc.call(chunk)\n      end\n    end\n\n    def try_write(chunk)\n      raise \"failed to flush\" if @never_flush\n      if @feed_proc\n        @feed_proc.call(chunk)\n      end\n      # not to commit chunks for testing\n      # commit_write(chunk.unique_id)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_relabel.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\n\nmodule Fluent::Plugin\n  class RelabelOutput < Output\n    Fluent::Plugin.register_output('relabel', self)\n    helpers :event_emitter\n\n    def multi_workers_ready?\n      true\n    end\n\n    def process(tag, es)\n      router.emit_stream(tag, es)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_roundrobin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/multi_output'\nrequire 'fluent/config/error'\n\nmodule Fluent::Plugin\n  class RoundRobinOutput < MultiOutput\n    Fluent::Plugin.register_output('roundrobin', self)\n\n    config_section :store do\n      config_param :weight, :integer, default: 1\n    end\n\n    def initialize\n      super\n      @weights = []\n    end\n\n    attr_reader :weights\n\n    def configure(conf)\n      super\n\n      @stores.each do |store|\n        @weights << store.weight\n      end\n      @rr = -1  # starts from @output[0]\n      @rand_seed = Random.new.seed\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def start\n      super\n      rebuild_weight_array\n    end\n\n    def process(tag, es)\n      next_output.emit_events(tag, es)\n    end\n\n    private\n\n    def next_output\n      @rr = 0 if (@rr += 1) >= @weight_array.size\n      @weight_array[@rr]\n    end\n\n    def rebuild_weight_array\n      gcd = @weights.inject(0) {|r,w| r.gcd(w) }\n\n      weight_array = []\n      @outputs.zip(@weights).each {|output,weight|\n        (weight / gcd).times {\n          weight_array << output\n        }\n      }\n\n      # don't randomize order if all weight is 1 (=default)\n      if @weights.any? {|w| w > 1 }\n        r = Random.new(@rand_seed)\n        weight_array.sort_by! { r.rand }\n      end\n\n      @weight_array = weight_array\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_secondary_file.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire \"fileutils\"\nrequire \"fluent/plugin/file_util\"\nrequire \"fluent/plugin/output\"\nrequire \"fluent/config/error\"\n\nmodule Fluent::Plugin\n  class SecondaryFileOutput < Output\n    Fluent::Plugin.register_output(\"secondary_file\", self)\n\n    PLACEHOLDER_REGEX = /\\${(tag(\\[\\d+\\])?|[\\w.@-]+)}/\n\n    desc \"The directory path of the output file.\"\n    config_param :directory, :string\n    desc \"The basename of the output file.\"\n    config_param :basename, :string, default: \"dump.bin\"\n    desc \"The flushed chunk is appended to existence file or not.\"\n    config_param :append, :bool, default: false\n    config_param :compress, :enum, list: [:text, :gzip], default: :text\n\n    def configure(conf)\n      super\n\n      unless @as_secondary\n        raise Fluent::ConfigError, \"This plugin can only be used in the <secondary> section\"\n      end\n\n      if @basename.include?(\"/\")\n        raise Fluent::ConfigError, \"basename should not include `/`\"\n      end\n\n      @path_without_suffix = File.join(@directory, @basename)\n      validate_compatible_with_primary_buffer!(@path_without_suffix)\n\n      @suffix = case @compress\n                when :text\n                  \"\"\n                when :gzip\n                  \".gz\"\n                end\n\n      test_path = @path_without_suffix\n      unless Fluent::FileUtil.writable_p?(test_path)\n        raise Fluent::ConfigError, \"out_secondary_file: `#{@directory}` should be writable\"\n      end\n\n      @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION\n      @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def write(chunk)\n      path_without_suffix = extract_placeholders(@path_without_suffix, chunk)\n      generate_path(path_without_suffix) do |path|\n        FileUtils.mkdir_p File.dirname(path), mode: @dir_perm\n\n        case @compress\n        when :text\n          File.open(path, \"ab\", @file_perm) {|f|\n            f.flock(File::LOCK_EX)\n            chunk.write_to(f)\n          }\n        when :gzip\n          File.open(path, \"ab\", @file_perm) {|f|\n            f.flock(File::LOCK_EX)\n            gz = Zlib::GzipWriter.new(f)\n            chunk.write_to(gz)\n            gz.close\n          }\n        end\n      end\n    end\n\n    private\n\n    def validate_compatible_with_primary_buffer!(path_without_suffix)\n      placeholders = path_without_suffix.scan(PLACEHOLDER_REGEX).flat_map(&:first) # to trim suffix [\\d+]\n\n      if !@chunk_key_time && has_time_format?(path_without_suffix)\n        raise Fluent::ConfigError, \"out_secondary_file: basename or directory has an incompatible placeholder, remove time formats, like `%Y%m%d`, from basename or directory\"\n      end\n\n      if !@chunk_key_tag && (ph = placeholders.find { |placeholder| placeholder.match?(/tag(\\[\\d+\\])?/) })\n        raise Fluent::ConfigError, \"out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove tag placeholder, like `${tag}`, from basename or directory\"\n      end\n\n      vars = placeholders.reject { |placeholder| placeholder.match?(/tag(\\[\\d+\\])?/) || (placeholder == 'chunk_id') }\n\n      if ph = vars.find { |v| !@chunk_keys.include?(v) }\n        raise Fluent::ConfigError, \"out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove variable placeholder, like `${varname}`, from basename or directory\"\n      end\n    end\n\n    def has_time_format?(str)\n      str != Time.now.strftime(str)\n    end\n\n    def generate_path(path_without_suffix)\n      if @append\n        path = \"#{path_without_suffix}#{@suffix}\"\n        synchronize_path(path) do\n          yield path\n        end\n        return path\n      end\n\n      begin\n        i = 0\n        loop do\n          path = \"#{path_without_suffix}.#{i}#{@suffix}\"\n          break unless File.exist?(path)\n          i += 1\n        end\n        synchronize_path(path) do\n          # If multiple processes or threads select the same path and another\n          # one entered this locking block first, the file should already\n          # exist and this one should retry to find new path.\n          raise FileAlreadyExist if File.exist?(path)\n          yield path\n        end\n      rescue FileAlreadyExist\n        retry\n      end\n      path\n    end\n\n    class FileAlreadyExist < StandardError\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_stdout.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\n\nmodule Fluent::Plugin\n  class StdoutOutput < Output\n    Fluent::Plugin.register_output('stdout', self)\n\n    helpers :inject, :formatter, :compat_parameters\n\n    DEFAULT_LINE_FORMAT_TYPE = 'stdout'\n    DEFAULT_FORMAT_TYPE = 'json'\n\n    desc \"If Fluentd logger outputs logs to a file (with -o option), this plugin outputs events to the file as well.\"\n    config_param :use_logger, :bool, default: true\n\n    config_section :buffer do\n      config_set_default :chunk_keys, ['tag']\n      config_set_default :flush_at_shutdown, true\n      config_set_default :chunk_limit_size, 10 * 1024\n    end\n\n    config_section :format do\n      config_set_default :@type, DEFAULT_LINE_FORMAT_TYPE\n      config_set_default :output_type, DEFAULT_FORMAT_TYPE\n    end\n\n    def prefer_buffered_processing\n      false\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def dest_io\n      @use_logger ? $log : $stdout\n    end\n\n    attr_accessor :formatter\n\n    def configure(conf)\n      compat_parameters_convert(conf, :inject, :formatter)\n\n      super\n\n      @formatter = formatter_create\n    end\n\n    def process(tag, es)\n      es = inject_values_to_event_stream(tag, es)\n      es.each {|time,record|\n        dest_io.write(format(tag, time, record))\n      }\n      dest_io.flush\n    end\n\n    def format(tag, time, record)\n      record = inject_values_to_record(tag, time, record)\n      @formatter.format(tag, time, record).chomp + \"\\n\"\n    end\n\n    def write(chunk)\n      chunk.write_to(dest_io)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/out_stream.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'socket'\nrequire 'fileutils'\n\nrequire 'fluent/output'\nrequire 'fluent/event'\n\nmodule Fluent\n  # obsolete\n  class StreamOutput < BufferedOutput\n    config_param :send_timeout, :time, default: 60\n\n    helpers :compat_parameters\n\n    def configure(conf)\n      compat_parameters_convert(conf, :buffer)\n      super\n    end\n\n    def format_stream(tag, es)\n      # use PackedForward\n      [tag, es.to_msgpack_stream].to_msgpack\n    end\n\n    def write(chunk)\n      sock = connect\n      begin\n        opt = [1, @send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }\n        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)\n\n        opt = [@send_timeout.to_i, 0].pack('L!L!')  # struct timeval\n        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)\n\n        chunk.write_to(sock)\n      ensure\n        sock.close\n      end\n    end\n\n    def flush_secondary(secondary)\n      unless secondary.is_a?(StreamOutput)\n        secondary = ReformatWriter.new(secondary)\n      end\n      @buffer.pop(secondary)\n    end\n\n    class ReformatWriter\n      def initialize(secondary)\n        @secondary = secondary\n      end\n\n      def write(chunk)\n        chain = NullOutputChain.instance\n        chunk.open {|io|\n          # TODO use MessagePackIoEventStream\n          u = Fluent::MessagePackFactory.msgpack_unpacker(io)\n          begin\n            u.each {|(tag,entries)|\n              es = MultiEventStream.new\n              entries.each {|o|\n                es.add(o[0], o[1])\n              }\n              @secondary.emit(tag, es, chain)\n            }\n          rescue EOFError\n          end\n        }\n      end\n    end\n  end\n\n  # obsolete\n  class TcpOutput < StreamOutput\n    Plugin.register_output('tcp', self)\n\n    LISTEN_PORT = 24224\n\n    def initialize\n      super\n      log.warn \"'tcp' output is obsoleted and will be removed. Use 'forward' instead.\"\n      log.warn \"see 'forward' section in https://docs.fluentd.org/ for the high-availability configuration.\"\n    end\n\n    config_param :port, :integer, default: LISTEN_PORT\n    config_param :host, :string\n\n    def configure(conf)\n      super\n    end\n\n    def connect\n      TCPSocket.new(@host, @port)\n    end\n  end\n\n  # obsolete\n  class UnixOutput < StreamOutput\n    Plugin.register_output('unix', self)\n\n    def initialize\n      super\n      log.warn \"'unix' output is obsoleted and will be removed.\"\n    end\n\n    config_param :path, :string\n\n    def configure(conf)\n      super\n    end\n\n    def connect\n      UNIXSocket.new(@path)\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/output.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/env'\nrequire 'fluent/error'\nrequire 'fluent/plugin/base'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin_helper/record_accessor'\nrequire 'fluent/msgpack_factory'\nrequire 'fluent/log'\nrequire 'fluent/plugin_id'\nrequire 'fluent/plugin_helper'\nrequire 'fluent/timezone'\nrequire 'fluent/unique_id'\nrequire 'fluent/clock'\nrequire 'fluent/ext_monitor_require'\n\nrequire 'time'\n\nmodule Fluent\n  module Plugin\n    class Output < Base\n      include PluginId\n      include PluginLoggerMixin\n      include PluginHelper::Mixin\n      include UniqueId::Mixin\n\n      helpers_internal :thread, :retry_state, :metrics\n\n      CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/\n      CHUNK_KEY_PLACEHOLDER_PATTERN = /\\$\\{([-_.@$a-zA-Z0-9]+)\\}/\n      CHUNK_TAG_PLACEHOLDER_PATTERN = /\\$\\{(tag(?:\\[-?\\d+\\])?)\\}/\n      CHUNK_ID_PLACEHOLDER_PATTERN = /\\$\\{chunk_id\\}/\n\n      CHUNKING_FIELD_WARN_NUM = 4\n\n      config_param :time_as_integer, :bool, default: false\n      desc 'The threshold to show slow flush logs'\n      config_param :slow_flush_log_threshold, :float, default: 20.0\n\n      # `<buffer>` and `<secondary>` sections are available only when '#format' and '#write' are implemented\n      config_section :buffer, param_name: :buffer_config, init: true, required: false, multi: false, final: true do\n        config_argument :chunk_keys, :array, value_type: :string, default: []\n        config_param :@type, :string, default: 'memory', alias: :type\n\n        config_param :timekey, :time, default: nil # range size to be used: `time.to_i / @timekey`\n        config_param :timekey_wait, :time, default: 600\n        # These are for #extract_placeholders\n        config_param :timekey_use_utc, :bool, default: false # default is localtime\n        config_param :timekey_zone, :string, default: Time.now.strftime('%z') # e.g., \"-0700\" or \"Asia/Tokyo\"\n\n        desc 'If true, plugin will try to flush buffer just before shutdown.'\n        config_param :flush_at_shutdown, :bool, default: nil # change default by buffer_plugin.persistent?\n\n        desc 'How to enqueue chunks to be flushed. \"interval\" flushes per flush_interval, \"immediate\" flushes just after event arrival.'\n        config_param :flush_mode, :enum, list: [:default, :lazy, :interval, :immediate], default: :default\n        config_param :flush_interval, :time, default: 60, desc: 'The interval between buffer chunk flushes.'\n\n        config_param :flush_thread_count, :integer, default: 1, desc: 'The number of threads to flush the buffer.'\n\n        config_param :flush_thread_interval, :float, default: 1.0, desc: 'Seconds to sleep between checks for buffer flushes in flush threads.'\n        config_param :flush_thread_burst_interval, :float, default: 1.0, desc: 'Seconds to sleep between flushes when many buffer chunks are queued.'\n\n        config_param :delayed_commit_timeout, :time, default: 60, desc: 'Seconds of timeout for buffer chunks to be committed by plugins later.'\n\n        config_param :overflow_action, :enum, list: [:throw_exception, :block, :drop_oldest_chunk], default: :throw_exception, desc: 'The action when the size of buffer exceeds the limit.'\n\n        config_param :retry_forever, :bool, default: false, desc: 'If true, plugin will ignore retry_timeout and retry_max_times options and retry flushing forever.'\n        config_param :retry_timeout, :time, default: 72 * 60 * 60, desc: 'The maximum seconds to retry to flush while failing, until plugin discards buffer chunks.'\n        # 72hours == 17 times with exponential backoff (not to change default behavior)\n        config_param :retry_max_times, :integer, default: nil, desc: 'The maximum number of times to retry to flush while failing.'\n\n        config_param :retry_secondary_threshold, :float, default: 0.8, desc: 'ratio of retry_timeout to switch to use secondary while failing.'\n        # exponential backoff sequence will be initialized at the time of this threshold\n\n        desc 'How to wait next retry to flush buffer.'\n        config_param :retry_type, :enum, list: [:exponential_backoff, :periodic], default: :exponential_backoff\n        ### Periodic -> fixed :retry_wait\n        ### Exponential backoff: k is number of retry times\n        # c: constant factor, @retry_wait\n        # b: base factor, @retry_exponential_backoff_base\n        # k: times\n        # total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1\n        config_param :retry_wait, :time, default: 1, desc: 'Seconds to wait before next retry to flush, or constant factor of exponential backoff.'\n        config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponential backoff for retries.'\n        config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'\n\n        config_param :retry_randomize, :bool, default: true, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'\n      end\n\n      config_section :secondary, param_name: :secondary_config, required: false, multi: false, final: true do\n        config_param :@type, :string, default: nil, alias: :type\n        config_section :buffer, required: false, multi: false do\n          # dummy to detect invalid specification for here\n        end\n        config_section :secondary, required: false, multi: false do\n          # dummy to detect invalid specification for here\n        end\n      end\n\n      def process(tag, es)\n        raise NotImplementedError, \"BUG: output plugins MUST implement this method\"\n      end\n\n      def write(chunk)\n        raise NotImplementedError, \"BUG: output plugins MUST implement this method\"\n      end\n\n      def try_write(chunk)\n        raise NotImplementedError, \"BUG: output plugins MUST implement this method\"\n      end\n\n      def format(tag, time, record)\n        # standard msgpack_event_stream chunk will be used if this method is not implemented in plugin subclass\n        raise NotImplementedError, \"BUG: output plugins MUST implement this method\"\n      end\n\n      def formatted_to_msgpack_binary?\n        # To indicate custom format method (#format) returns msgpack binary or not.\n        # If #format returns msgpack binary, override this method to return true.\n        false\n      end\n\n      # Compatibility for existing plugins\n      def formatted_to_msgpack_binary\n        formatted_to_msgpack_binary?\n      end\n\n      def prefer_buffered_processing\n        # override this method to return false only when all of these are true:\n        #  * plugin has both implementation for buffered and non-buffered methods\n        #  * plugin is expected to work as non-buffered plugin if no `<buffer>` sections specified\n        true\n      end\n\n      def prefer_delayed_commit\n        # override this method to decide which is used of `write` or `try_write` if both are implemented\n        true\n      end\n\n      def multi_workers_ready?\n        false\n      end\n\n      # Internal states\n      FlushThreadState = Struct.new(:thread, :next_clock, :mutex, :cond_var)\n      DequeuedChunkInfo = Struct.new(:chunk_id, :time, :timeout) do\n        def expired?\n          time + timeout < Time.now\n        end\n      end\n\n      attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone\n\n      # for tests\n      attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag\n      attr_accessor :output_enqueue_thread_waiting, :dequeued_chunks, :dequeued_chunks_mutex\n      # output_enqueue_thread_waiting: for test of output.rb itself\n      attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true\n\n      def initialize\n        super\n        @counter_mutex = Mutex.new\n        @flush_thread_mutex = Mutex.new\n        @buffering = false\n        @delayed_commit = false\n        @as_secondary = false\n        @primary_instance = nil\n\n        # TODO: well organized counters\n        @num_errors_metrics = nil\n        @emit_count_metrics = nil\n        @emit_records_metrics = nil\n        @emit_size_metrics = nil\n        @write_count_metrics = nil\n        @write_secondary_count_metrics = nil\n        @rollback_count_metrics = nil\n        @flush_time_count_metrics = nil\n        @slow_flush_count_metrics = nil\n        @drop_oldest_chunk_count_metrics = nil\n        @enable_size_metrics = false\n\n        # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start\n        if implement?(:synchronous)\n          if implement?(:buffered) || implement?(:delayed_commit)\n            @buffering = nil # do #configure or #start to determine this for full-featured plugins\n          else\n            @buffering = false\n          end\n        else\n          @buffering = true\n        end\n        @custom_format = implement?(:custom_format)\n        @enable_msgpack_streamer = false # decided later\n\n        @buffer = nil\n        @secondary = nil\n        @retry = nil\n        @dequeued_chunks = nil\n        @dequeued_chunks_mutex = nil\n        @output_enqueue_thread = nil\n        @output_flush_threads = nil\n        @output_flush_thread_current_position = 0\n\n        @simple_chunking = nil\n        @chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil\n        @flush_mode = nil\n        @timekey_zone = nil\n\n        @retry_for_error_chunk = false\n      end\n\n      def acts_as_secondary(primary)\n        @as_secondary = true\n        @primary_instance = primary\n        @chunk_keys = @primary_instance.chunk_keys || []\n        @chunk_key_tag = @primary_instance.chunk_key_tag || false\n        if @primary_instance.chunk_key_time\n          @chunk_key_time = @primary_instance.chunk_key_time\n          @timekey_zone = @primary_instance.timekey_zone\n          @output_time_formatter_cache = {}\n        end\n        self.context_router = primary.context_router\n\n        singleton_class.module_eval do\n          define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) }\n          define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry) }\n        end\n      end\n\n      def configure(conf)\n        unless implement?(:synchronous) || implement?(:buffered) || implement?(:delayed_commit)\n          raise \"BUG: output plugin must implement some methods. see developer documents.\"\n        end\n\n        has_buffer_section = (conf.elements(name: 'buffer').size > 0)\n        has_flush_interval = conf.has_key?('flush_interval')\n\n        super\n\n        @num_errors_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"num_errors\", help_text: \"Number of count num errors\")\n        @emit_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"emit_count\", help_text: \"Number of count emits\")\n        @emit_records_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"emit_records\", help_text: \"Number of emit records\")\n        @emit_size_metrics =  metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"emit_size\", help_text: \"Total size of emit events\")\n        @write_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"write_count\", help_text: \"Number of writing events\")\n        @write_secondary_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"write_secondary_count\", help_text: \"Number of writing events in secondary\")\n        @rollback_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"rollback_count\", help_text: \"Number of rollbacking operations\")\n        @flush_time_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"flush_time_count\", help_text: \"Count of flush time\")\n        @slow_flush_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"slow_flush_count\", help_text: \"Count of slow flush occurred time(s)\")\n        @drop_oldest_chunk_count_metrics = metrics_create(namespace: \"fluentd\", subsystem: \"output\", name: \"drop_oldest_chunk_count\", help_text: \"Number of count that old chunk were discarded with drop_oldest_chunk\")\n\n        if has_buffer_section\n          unless implement?(:buffered) || implement?(:delayed_commit)\n            raise Fluent::ConfigError, \"<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering\"\n          end\n          @buffering = true\n        else # no buffer sections\n          if implement?(:synchronous)\n            if !implement?(:buffered) && !implement?(:delayed_commit)\n              if @as_secondary\n                raise Fluent::ConfigError, \"secondary plugin '#{self.class}' must support buffering, but doesn't.\"\n              end\n              @buffering = false\n            else\n              if @as_secondary\n                # secondary plugin always works as buffered plugin without buffer instance\n                @buffering = true\n              else\n                # @buffering.nil? shows that enabling buffering or not will be decided in lazy way in #start\n                @buffering = nil\n              end\n            end\n          else # buffered or delayed_commit is supported by `unless` of first line in this method\n            @buffering = true\n          end\n        end\n        # Enable to update record size metrics or not\n        @enable_size_metrics = !!system_config.enable_size_metrics\n\n        if @as_secondary\n          if !@buffering && !@buffering.nil?\n            raise Fluent::ConfigError, \"secondary plugin '#{self.class}' must support buffering, but doesn't\"\n          end\n        end\n\n        if (@buffering || @buffering.nil?) && !@as_secondary\n          # When @buffering.nil?, @buffer_config was initialized with default value for all parameters.\n          # If so, this configuration MUST success.\n          @chunk_keys = @buffer_config.chunk_keys.dup\n          @chunk_key_time = !!@chunk_keys.delete('time')\n          @chunk_key_tag = !!@chunk_keys.delete('tag')\n          if @chunk_keys.any? { |key|\n              begin\n                k = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(key)\n                if k.is_a?(String)\n                  k !~ CHUNK_KEY_PATTERN\n                else\n                  if key.start_with?('$[')\n                    raise Fluent::ConfigError, \"in chunk_keys: bracket notation is not allowed\"\n                  else\n                    false\n                  end\n                end\n              rescue => e\n                raise Fluent::ConfigError, \"in chunk_keys: #{e.message}\"\n              end\n            }\n            raise Fluent::ConfigError, \"chunk_keys specification includes invalid char\"\n          else\n            @chunk_key_accessors = Hash[@chunk_keys.map { |key| [key.to_sym, Fluent::PluginHelper::RecordAccessor::Accessor.new(key)] }]\n          end\n\n          if @chunk_key_time\n            raise Fluent::ConfigError, \"<buffer ...> argument includes 'time', but timekey is not configured\" unless @buffer_config.timekey\n            Fluent::Timezone.validate!(@buffer_config.timekey_zone)\n            @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone\n            @timekey = @buffer_config.timekey\n            if @timekey <= 0\n              raise Fluent::ConfigError, \"timekey should be greater than 0. current timekey: #{@timekey}\"\n            end\n            @timekey_use_utc = @buffer_config.timekey_use_utc\n            @offset = Fluent::Timezone.utc_offset(@timekey_zone)\n            @calculate_offset = @offset.respond_to?(:call) ? @offset : nil\n            @output_time_formatter_cache = {}\n          end\n\n          if (@chunk_key_tag ? 1 : 0) + @chunk_keys.size >= CHUNKING_FIELD_WARN_NUM\n            log.warn \"many chunk keys specified, and it may cause too many chunks on your system.\"\n          end\n\n          # no chunk keys or only tags (chunking can be done without iterating event stream)\n          @simple_chunking = !@chunk_key_time && @chunk_keys.empty?\n\n          @flush_mode = @buffer_config.flush_mode\n          if @flush_mode == :default\n            if has_flush_interval\n              log.info \"'flush_interval' is configured at out side of <buffer>. 'flush_mode' is set to 'interval' to keep existing behaviour\"\n              @flush_mode = :interval\n            else\n              @flush_mode = (@chunk_key_time ? :lazy : :interval)\n            end\n          end\n\n          buffer_type = @buffer_config[:@type]\n          buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])\n          @buffer = Plugin.new_buffer(buffer_type, parent: self)\n          @buffer.configure(buffer_conf)\n          keep_buffer_config_compat\n          @buffer.enable_update_timekeys if @chunk_key_time\n\n          @flush_at_shutdown = @buffer_config.flush_at_shutdown\n          if @flush_at_shutdown.nil?\n            @flush_at_shutdown = if @buffer.persistent?\n                                   false\n                                 else\n                                   true # flush_at_shutdown is true in default for on-memory buffer\n                                 end\n          elsif !@flush_at_shutdown && !@buffer.persistent?\n            buf_type = Plugin.lookup_type_from_class(@buffer.class)\n            log.warn \"'flush_at_shutdown' is false, and buffer plugin '#{buf_type}' is not persistent buffer.\"\n            log.warn \"your configuration will lose buffered data at shutdown. please confirm your configuration again.\"\n          end\n\n          if (@flush_mode != :interval) && buffer_conf.has_key?('flush_interval')\n            if buffer_conf.has_key?('flush_mode')\n              raise Fluent::ConfigError, \"'flush_interval' can't be specified when 'flush_mode' is not 'interval' explicitly: '#{@flush_mode}'\"\n            else\n              log.warn \"'flush_interval' is ignored because default 'flush_mode' is not 'interval': '#{@flush_mode}'\"\n            end\n          end\n\n          if @buffer.queued_chunks_limit_size.nil?\n            @buffer.queued_chunks_limit_size = @buffer_config.flush_thread_count\n          end\n        end\n\n        if @secondary_config\n          raise Fluent::ConfigError, \"Invalid <secondary> section for non-buffered plugin\" unless @buffering\n          raise Fluent::ConfigError, \"<secondary> section cannot have <buffer> section\" if @secondary_config.buffer\n          raise Fluent::ConfigError, \"<secondary> section cannot have <secondary> section\" if @secondary_config.secondary\n          if @buffer_config.retry_forever\n            log.warn \"<secondary> with 'retry_forever', only unrecoverable errors are moved to secondary\"\n          end\n\n          secondary_type = @secondary_config[:@type]\n          unless secondary_type\n            secondary_type = conf['@type'] # primary plugin type\n          end\n          secondary_conf = conf.elements(name: 'secondary').first\n          @secondary = Plugin.new_output(secondary_type)\n          unless @secondary.respond_to?(:acts_as_secondary)\n            raise Fluent::ConfigError, \"Failed to setup secondary plugin in '#{conf['@type']}'. '#{secondary_type}' plugin in not allowed due to non buffered output\"\n          end\n          @secondary.acts_as_secondary(self)\n          @secondary.configure(secondary_conf)\n          if (@secondary.class.to_s != \"Fluent::Plugin::SecondaryFileOutput\") &&\n             (self.class != @secondary.class) &&\n             (@custom_format || @secondary.implement?(:custom_format))\n            log.warn \"Use different plugin for secondary. Check the plugin works with primary like secondary_file\", primary: self.class.to_s, secondary: @secondary.class.to_s\n          end\n        else\n          @secondary = nil\n        end\n\n        self\n      end\n\n      def keep_buffer_config_compat\n        # Need this to call `@buffer_config.disable_chunk_backup` just as before,\n        # since some plugins may use this option in this way.\n        @buffer_config[:disable_chunk_backup] = @buffer.disable_chunk_backup\n      end\n\n      def start\n        super\n\n        if @buffering.nil?\n          @buffering = prefer_buffered_processing\n          if !@buffering && @buffer\n            @buffer.terminate # it's not started, so terminate will be enough\n            # At here, this plugin works as non-buffered plugin.\n            # Un-assign @buffer not to show buffering metrics (e.g., in_monitor_agent)\n            @buffer = nil\n          end\n        end\n\n        if @buffering\n          m = method(:emit_buffered)\n          singleton_class.module_eval do\n            define_method(:emit_events, m)\n          end\n\n          @custom_format = implement?(:custom_format)\n          @enable_msgpack_streamer = @custom_format ? formatted_to_msgpack_binary : true\n          @delayed_commit = if implement?(:buffered) && implement?(:delayed_commit)\n                              prefer_delayed_commit\n                            else\n                              implement?(:delayed_commit)\n                            end\n          @delayed_commit_timeout = @buffer_config.delayed_commit_timeout\n        else # !@buffering\n          m = method(:emit_sync)\n          singleton_class.module_eval do\n            define_method(:emit_events, m)\n          end\n        end\n\n        if @buffering && !@as_secondary\n          @retry = nil\n          @retry_mutex = Mutex.new\n\n          @buffer.start\n\n          @output_enqueue_thread = nil\n          @output_enqueue_thread_running = true\n\n          @output_flush_threads = []\n          @output_flush_threads_mutex = Mutex.new\n          @output_flush_threads_running = true\n\n          # mainly for test: detect enqueue works as code below:\n          #   @output.interrupt_flushes\n          #   # emits\n          #   @output.enqueue_thread_wait\n          @output_flush_interrupted = false\n          @output_enqueue_thread_mutex = Mutex.new\n          @output_enqueue_thread_waiting = false\n\n          @dequeued_chunks = []\n          @dequeued_chunks_mutex = Mutex.new\n\n          @output_flush_thread_current_position = 0\n          @buffer_config.flush_thread_count.times do |i|\n            thread_title = \"flush_thread_#{i}\".to_sym\n            thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)\n            thread = thread_create(thread_title) do\n              flush_thread_run(thread_state)\n            end\n            thread_state.thread = thread\n            @output_flush_threads_mutex.synchronize do\n              @output_flush_threads << thread_state\n            end\n          end\n\n          if !@under_plugin_development && (@flush_mode == :interval || @chunk_key_time)\n            @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))\n          end\n        end\n        @secondary.start if @secondary\n      end\n\n      def after_start\n        super\n        @secondary.after_start if @secondary\n      end\n\n      def stop\n        @secondary.stop if @secondary\n        @buffer.stop if @buffering && @buffer\n\n        super\n      end\n\n      def before_shutdown\n        @secondary.before_shutdown if @secondary\n\n        if @buffering && @buffer\n          if @flush_at_shutdown\n            force_flush\n          end\n          @buffer.before_shutdown\n          # Need to ensure to stop enqueueing ... after #shutdown, we cannot write any data\n          @output_enqueue_thread_running = false\n          if @output_enqueue_thread && @output_enqueue_thread.alive?\n            @output_enqueue_thread.wakeup\n            @output_enqueue_thread.join\n          end\n        end\n\n        super\n      end\n\n      def shutdown\n        @secondary.shutdown if @secondary\n        @buffer.shutdown if @buffering && @buffer\n\n        super\n      end\n\n      def after_shutdown\n        try_rollback_all if @buffering && !@as_secondary # rollback regardless with @delayed_commit, because secondary may do it\n        @secondary.after_shutdown if @secondary\n\n        if @buffering && @buffer\n          @buffer.after_shutdown\n\n          @output_flush_threads_running = false\n          if @output_flush_threads && !@output_flush_threads.empty?\n            @output_flush_threads.each do |state|\n              # to wakeup thread and make it to stop by itself\n              state.mutex.synchronize {\n                if state.thread&.status\n                  state.next_clock = 0\n                  state.cond_var.signal\n                end\n              }\n              Thread.pass\n              state.thread.join\n            end\n          end\n        end\n\n        super\n      end\n\n      def close\n        @buffer.close if @buffering && @buffer\n        @secondary.close if @secondary\n\n        super\n      end\n\n      def terminate\n        @buffer.terminate if @buffering && @buffer\n        @secondary.terminate if @secondary\n\n        super\n      end\n\n      def actual_flush_thread_count\n        return 0 unless @buffering\n        return @buffer_config.flush_thread_count unless @as_secondary\n        @primary_instance.buffer_config.flush_thread_count\n      end\n\n      # Ensures `path` (filename or filepath) processable\n      # only by the current thread in the current process.\n      # For multiple workers, the lock is shared if `path` is the same value.\n      # For multiple threads, the lock is shared by all threads in the same process.\n      def synchronize_path(path)\n        synchronize_path_in_workers(path) do\n          synchronize_in_threads do\n            yield\n          end\n        end\n      end\n\n      def synchronize_path_in_workers(path)\n        need_worker_lock = system_config.workers > 1\n        if need_worker_lock\n          acquire_worker_lock(path) { yield }\n        else\n          yield\n        end\n      end\n\n      def synchronize_in_threads\n        need_thread_lock = actual_flush_thread_count > 1\n        if need_thread_lock\n          @flush_thread_mutex.synchronize { yield }\n        else\n          yield\n        end\n      end\n\n      def support_in_v12_style?(feature)\n        # for plugins written in v0.12 styles\n        case feature\n        when :synchronous    then false\n        when :buffered       then false\n        when :delayed_commit then false\n        when :custom_format  then false\n        else\n          raise ArgumentError, \"unknown feature: #{feature}\"\n        end\n      end\n\n      def implement?(feature)\n        methods_of_plugin = self.class.instance_methods(false)\n        case feature\n        when :synchronous    then methods_of_plugin.include?(:process) || support_in_v12_style?(:synchronous)\n        when :buffered       then methods_of_plugin.include?(:write) || support_in_v12_style?(:buffered)\n        when :delayed_commit then methods_of_plugin.include?(:try_write)\n        when :custom_format  then methods_of_plugin.include?(:format) || support_in_v12_style?(:custom_format)\n        else\n          raise ArgumentError, \"Unknown feature for output plugin: #{feature}\"\n        end\n      end\n\n      def placeholder_validate!(name, str)\n        placeholder_validators(name, str).each do |v|\n          v.validate!\n        end\n      end\n\n      def placeholder_validators(name, str, time_key = (@chunk_key_time && @buffer_config.timekey), tag_key = @chunk_key_tag, chunk_keys = @chunk_keys)\n        validators = []\n\n        sec, title, example = get_placeholders_time(str)\n        if sec || time_key\n          validators << PlaceholderValidator.new(name, str, :time, {sec: sec, title: title, example: example, timekey: time_key})\n        end\n\n        parts = get_placeholders_tag(str)\n        if tag_key || !parts.empty?\n          validators << PlaceholderValidator.new(name, str, :tag, {parts: parts, tagkey: tag_key})\n        end\n\n        keys = get_placeholders_keys(str)\n        if chunk_keys && !chunk_keys.empty? || !keys.empty?\n          validators << PlaceholderValidator.new(name, str, :keys, {keys: keys, chunkkeys: chunk_keys})\n        end\n\n        validators\n      end\n\n      class PlaceholderValidator\n        attr_reader :name, :string, :type, :argument\n\n        def initialize(name, str, type, arg)\n          @name = name\n          @string = str\n          @type = type\n          raise ArgumentError, \"invalid type:#{type}\" if @type != :time && @type != :tag && @type != :keys\n          @argument = arg\n        end\n\n        def time?\n          @type == :time\n        end\n\n        def tag?\n          @type == :tag\n        end\n\n        def keys?\n          @type == :keys\n        end\n\n        def validate!\n          case @type\n          when :time then validate_time!\n          when :tag  then validate_tag!\n          when :keys then validate_keys!\n          end\n        end\n\n        def validate_time!\n          sec = @argument[:sec]\n          title = @argument[:title]\n          example = @argument[:example]\n          timekey = @argument[:timekey]\n          if !sec && timekey\n            raise Fluent::ConfigError, \"Parameter '#{name}: #{string}' doesn't have timestamp placeholders for timekey #{timekey.to_i}\"\n          end\n          if sec && !timekey\n            raise Fluent::ConfigError, \"Parameter '#{name}: #{string}' has timestamp placeholders, but chunk key 'time' is not configured\"\n          end\n          if sec && timekey && timekey < sec\n            raise Fluent::ConfigError, \"Parameter '#{name}: #{string}' doesn't have timestamp placeholder for #{title}('#{example}') for timekey #{timekey.to_i}\"\n          end\n        end\n\n        def validate_tag!\n          parts = @argument[:parts]\n          tagkey = @argument[:tagkey]\n          if tagkey && parts.empty?\n            raise Fluent::ConfigError, \"Parameter '#{name}: #{string}' doesn't have tag placeholder\"\n          end\n          if !tagkey && !parts.empty?\n            raise Fluent::ConfigError, \"Parameter '#{name}: #{string}' has tag placeholders, but chunk key 'tag' is not configured\"\n          end\n        end\n\n        def validate_keys!\n          keys = @argument[:keys]\n          chunk_keys = @argument[:chunkkeys]\n          if (chunk_keys - keys).size > 0\n            not_specified = (chunk_keys - keys).sort\n            raise Fluent::ConfigError, \"Parameter '#{name}: #{string}' doesn't have enough placeholders for keys #{not_specified.join(',')}\"\n          end\n          if (keys - chunk_keys).size > 0\n            not_satisfied = (keys - chunk_keys).sort\n            raise Fluent::ConfigError, \"Parameter '#{name}: #{string}' has placeholders, but chunk keys doesn't have keys #{not_satisfied.join(',')}\"\n          end\n        end\n      end\n\n      TIME_KEY_PLACEHOLDER_THRESHOLDS = [\n        [1, :second, '%S'],\n        [60, :minute, '%M'],\n        [3600, :hour, '%H'],\n        [86400, :day, '%d'],\n      ]\n      TIMESTAMP_CHECK_BASE_TIME = Time.parse(\"2016-01-01 00:00:00 UTC\")\n      # it's not validated to use timekey larger than 1 day\n      def get_placeholders_time(str)\n        base_str = TIMESTAMP_CHECK_BASE_TIME.strftime(str)\n        TIME_KEY_PLACEHOLDER_THRESHOLDS.each do |triple|\n          sec = triple.first\n          return triple if (TIMESTAMP_CHECK_BASE_TIME + sec).strftime(str) != base_str\n        end\n        nil\n      end\n\n      # -1 means whole tag\n      def get_placeholders_tag(str)\n        # [[\"tag\"],[\"tag[0]\"]]\n        parts = []\n        str.scan(CHUNK_TAG_PLACEHOLDER_PATTERN).map(&:first).each do |ph|\n          if ph == \"tag\"\n            parts << -1\n          elsif ph =~ /^tag\\[(-?\\d+)\\]$/\n            parts << $1.to_i\n          end\n        end\n        parts.sort\n      end\n\n      def get_placeholders_keys(str)\n        str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map(&:first).reject{|s| (s == \"tag\") || (s == 'chunk_id') }.sort\n      end\n\n      # TODO: optimize this code\n      def extract_placeholders(str, chunk)\n        metadata = if chunk.is_a?(Fluent::Plugin::Buffer::Chunk)\n                     chunk_passed = true\n                     chunk.metadata\n                   else\n                     chunk_passed = false\n                     # For existing plugins. Old plugin passes Chunk.metadata instead of Chunk\n                     chunk\n                   end\n        if metadata.empty?\n          str.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {\n            if chunk_passed\n              dump_unique_id_hex(chunk.unique_id)\n            else\n              log.warn \"${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument\"\n            end\n          }\n        else\n          rvalue = str.dup\n          # strftime formatting\n          if @chunk_key_time # this section MUST be earlier than rest to use raw 'str'\n            @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)\n            rvalue = @output_time_formatter_cache[str].call(metadata.timekey)\n          end\n          # ${tag}, ${tag[0]}, ${tag[1]}, ... , ${tag[-2]}, ${tag[-1]}\n          if @chunk_key_tag\n            if str.include?('${tag}')\n              rvalue = rvalue.gsub('${tag}', metadata.tag)\n            end\n            if CHUNK_TAG_PLACEHOLDER_PATTERN.match?(str)\n              hash = {}\n              tag_parts = metadata.tag.split('.')\n              tag_parts.each_with_index do |part, i|\n                hash[\"${tag[#{i}]}\"] = part\n                hash[\"${tag[#{i-tag_parts.size}]}\"] = part\n              end\n              rvalue = rvalue.gsub(CHUNK_TAG_PLACEHOLDER_PATTERN, hash)\n            end\n            if rvalue =~ CHUNK_TAG_PLACEHOLDER_PATTERN\n              log.warn \"tag placeholder '#{$1}' not replaced. tag:#{metadata.tag}, template:#{str}\"\n            end\n          end\n\n          # First we replace ${chunk_id} with chunk.unique_id (hexlified).\n          rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {\n            if chunk_passed\n              dump_unique_id_hex(chunk.unique_id)\n            else\n              log.warn \"${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument\"\n            end\n          }\n\n          # Then, replace other ${chunk_key}s.\n          if !@chunk_keys.empty? && metadata.variables\n            hash = {'${tag}' => '${tag}'} # not to erase this wrongly\n            @chunk_keys.each do |key|\n              hash[\"${#{key}}\"] = metadata.variables[key.to_sym]\n            end\n\n            rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN) do |matched|\n              hash.fetch(matched) do\n                log.warn \"chunk key placeholder '#{matched[2..-2]}' not replaced. template:#{str}\"\n                ''\n              end\n            end\n          end\n\n          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN\n            log.warn \"chunk key placeholder '#{$1}' not replaced. template:#{str}\"\n          end\n\n          rvalue\n        end\n      end\n\n      def emit_events(tag, es)\n        # actually this method will be overwritten by #configure\n        if @buffering\n          emit_buffered(tag, es)\n        else\n          emit_sync(tag, es)\n        end\n      end\n\n      def emit_sync(tag, es)\n        @emit_count_metrics.inc\n        begin\n          process(tag, es)\n          @emit_records_metrics.add(es.size)\n          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n        rescue\n          @num_errors_metrics.inc\n          raise\n        end\n      end\n\n      def emit_buffered(tag, es)\n        @emit_count_metrics.inc\n        begin\n          execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))\n          if !@retry && @buffer.queued?(nil, optimistic: true)\n            submit_flush_once\n          end\n        rescue\n          # TODO: separate number of errors into emit errors and write/flush errors\n          @num_errors_metrics.inc\n          raise\n        end\n      end\n\n      # TODO: optimize this code\n      def metadata(tag, time, record)\n        # this arguments are ordered in output plugin's rule\n        # Metadata 's argument order is different from this one (timekey, tag, variables)\n\n        raise ArgumentError, \"tag must be a String: #{tag.class}\" unless tag.nil? || tag.is_a?(String)\n        raise ArgumentError, \"time must be a Fluent::EventTime (or Integer): #{time.class}\" unless time.nil? || time.is_a?(Fluent::EventTime) || time.is_a?(Integer)\n        raise ArgumentError, \"record must be a Hash: #{record.class}\" unless record.nil? || record.is_a?(Hash)\n\n        if @chunk_keys.nil? && @chunk_key_time.nil? && @chunk_key_tag.nil?\n          # for tests\n          return Struct.new(:timekey, :tag, :variables).new\n        end\n\n        # timekey is int from epoch, and `timekey - timekey % 60` is assumed to mach with 0s of each minutes.\n        # it's wrong if timezone is configured as one which supports leap second, but it's very rare and\n        # we can ignore it (especially in production systems).\n        if @chunk_keys.empty?\n          if !@chunk_key_time && !@chunk_key_tag\n            @buffer.metadata()\n          elsif @chunk_key_time && @chunk_key_tag\n            timekey = calculate_timekey(time)\n            @buffer.metadata(timekey: timekey, tag: tag)\n          elsif @chunk_key_time\n            timekey = calculate_timekey(time)\n            @buffer.metadata(timekey: timekey)\n          else\n            @buffer.metadata(tag: tag)\n          end\n        else\n          timekey = if @chunk_key_time\n                      calculate_timekey(time)\n                    else\n                      nil\n                    end\n          pairs = Hash[@chunk_key_accessors.map { |k, a| [k, a.call(record)] }]\n          @buffer.metadata(timekey: timekey, tag: (@chunk_key_tag ? tag : nil), variables: pairs)\n        end\n      end\n\n      def calculate_timekey(time)\n        time_int = time.to_i\n        if @timekey_use_utc\n          (time_int - (time_int % @timekey)).to_i\n        else\n          offset = @calculate_offset ? @calculate_offset.call(time) : @offset\n          (time_int - ((time_int + offset)% @timekey)).to_i\n        end\n      end\n\n      def chunk_for_test(tag, time, record)\n        require 'fluent/plugin/buffer/memory_chunk'\n\n        m = metadata(tag, time, record)\n        Fluent::Plugin::Buffer::MemoryChunk.new(m)\n      end\n\n      def execute_chunking(tag, es, enqueue: false)\n        if @simple_chunking\n          handle_stream_simple(tag, es, enqueue: enqueue)\n        elsif @custom_format\n          handle_stream_with_custom_format(tag, es, enqueue: enqueue)\n        else\n          handle_stream_with_standard_format(tag, es, enqueue: enqueue)\n        end\n      end\n\n      def write_guard(&block)\n        begin\n          block.call\n        rescue Fluent::Plugin::Buffer::BufferOverflowError\n          log.warn \"failed to write data into buffer by buffer overflow\", action: @buffer_config.overflow_action\n          case @buffer_config.overflow_action\n          when :throw_exception\n            raise\n          when :block\n            log.debug \"buffer.write is now blocking\"\n            until @buffer.storable?\n              if self.stopped?\n                log.error \"breaking block behavior to shutdown Fluentd\"\n                # to break infinite loop to exit Fluentd process\n                raise\n              end\n              log.trace \"sleeping until buffer can store more data\"\n              sleep 1\n            end\n            log.debug \"retrying buffer.write after blocked operation\"\n            retry\n          when :drop_oldest_chunk\n            begin\n              oldest = @buffer.dequeue_chunk\n              if oldest\n                log.warn \"dropping oldest chunk to make space after buffer overflow\", chunk_id: dump_unique_id_hex(oldest.unique_id)\n                @buffer.purge_chunk(oldest.unique_id)\n                @drop_oldest_chunk_count_metrics.inc\n              else\n                log.error \"no queued chunks to be dropped for drop_oldest_chunk\"\n              end\n            rescue\n              # ignore any errors\n            end\n            raise unless @buffer.storable?\n            retry\n          else\n            raise \"BUG: unknown overflow_action '#{@buffer_config.overflow_action}'\"\n          end\n        end\n      end\n\n      FORMAT_MSGPACK_STREAM = ->(e){ e.to_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }\n      FORMAT_COMPRESSED_MSGPACK_STREAM_GZIP = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }\n      FORMAT_COMPRESSED_MSGPACK_STREAM_ZSTD = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer, type: :zstd) }\n      FORMAT_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }\n      FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_GZIP = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }\n      FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_ZSTD = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer, type: :zstd) }\n\n      def generate_format_proc\n        if @buffer && @buffer.compress == :gzip\n          @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_GZIP : FORMAT_COMPRESSED_MSGPACK_STREAM_GZIP\n        elsif @buffer && @buffer.compress == :zstd\n          @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_ZSTD : FORMAT_COMPRESSED_MSGPACK_STREAM_ZSTD\n        else\n          @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM\n        end\n      end\n\n      # metadata_and_data is a Hash of:\n      #  (standard format) metadata => event stream\n      #  (custom format)   metadata => array of formatted event\n      # For standard format, formatting should be done for whole event stream, but\n      #   \"whole event stream\" may be a split of \"es\" here when it's bigger than chunk_limit_size.\n      #   `@buffer.write` will do this splitting.\n      # For custom format, formatting will be done here. Custom formatting always requires\n      #   iteration of event stream, and it should be done just once even if total event stream size\n      #   is bigger than chunk_limit_size because of performance.\n      def handle_stream_with_custom_format(tag, es, enqueue: false)\n        meta_and_data = Hash.new { |h, k| h[k] = [] }\n        records = 0\n        es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|\n          meta = metadata(tag, time, record)\n          res = format(tag, time, record)\n          if res\n            meta_and_data[meta] << res\n            records += 1\n          end\n        end\n        meta_and_data.default_proc = nil\n        write_guard do\n          @buffer.write(meta_and_data, enqueue: enqueue)\n        end\n        @emit_records_metrics.add(es.size)\n        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n        true\n      end\n\n      def handle_stream_with_standard_format(tag, es, enqueue: false)\n        format_proc = generate_format_proc\n        meta_and_data = Hash.new { |h, k| h[k] = MultiEventStream.new }\n        records = 0\n        es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|\n          meta = metadata(tag, time, record)\n          meta_and_data[meta].add(time, record)\n          records += 1\n        end\n        meta_and_data.default_proc = nil\n        write_guard do\n          @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)\n        end\n        @emit_records_metrics.add(es.size)\n        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n        true\n      end\n\n      def handle_stream_simple(tag, es, enqueue: false)\n        format_proc = nil\n        meta = metadata((@chunk_key_tag ? tag : nil), nil, nil)\n        records = es.size\n        if @custom_format\n          records = 0\n          data = []\n          es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|\n            res = format(tag, time, record)\n            if res\n              data << res\n              records += 1\n            end\n          end\n        else\n          format_proc = generate_format_proc\n          data = es\n        end\n        write_guard do\n          @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)\n        end\n        @emit_records_metrics.add(es.size)\n        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics\n        true\n      end\n\n      def commit_write(chunk_id, delayed: @delayed_commit, secondary: false)\n        log.on_trace { log.trace \"committing write operation to a chunk\", chunk: dump_unique_id_hex(chunk_id), delayed: delayed }\n\n        if delayed\n          @dequeued_chunks_mutex.synchronize do\n            @dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id }\n          end\n        end\n        @buffer.purge_chunk(chunk_id)\n\n        @retry_mutex.synchronize do\n          if @retry # success to flush chunks in retries\n            if secondary\n              log.warn \"retry succeeded by secondary.\", chunk_id: dump_unique_id_hex(chunk_id)\n            else\n              log.warn \"retry succeeded.\", chunk_id: dump_unique_id_hex(chunk_id)\n            end\n            @retry = nil\n          end\n        end\n      end\n\n      # update_retry parameter is for preventing busy loop by async write\n      # We will remove this parameter by re-design retry_state management between threads.\n      def rollback_write(chunk_id, update_retry: true)\n        # This API is to rollback chunks explicitly from plugins.\n        # 3rd party plugins can depend it on automatic rollback of #try_rollback_write\n        @dequeued_chunks_mutex.synchronize do\n          @dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id }\n        end\n        # returns true if chunk was rollbacked as expected\n        #         false if chunk was already flushed and couldn't be rollbacked unexpectedly\n        # in many cases, false can be just ignored\n        if @buffer.takeback_chunk(chunk_id)\n          @rollback_count_metrics.inc\n          if update_retry\n            primary = @as_secondary ? @primary_instance : self\n            primary.update_retry_state(chunk_id, @as_secondary)\n          end\n          true\n        else\n          false\n        end\n      end\n\n      def try_rollback_write\n        @dequeued_chunks_mutex.synchronize do\n          while @dequeued_chunks.first&.expired?\n            info = @dequeued_chunks.shift\n            if @buffer.takeback_chunk(info.chunk_id)\n              @rollback_count_metrics.inc\n              log.warn \"failed to flush the buffer chunk, timeout to commit.\", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time\n              primary = @as_secondary ? @primary_instance : self\n              primary.update_retry_state(info.chunk_id, @as_secondary)\n            end\n          end\n        end\n      end\n\n      def try_rollback_all\n        return unless @dequeued_chunks\n        @dequeued_chunks_mutex.synchronize do\n          until @dequeued_chunks.empty?\n            info = @dequeued_chunks.shift\n            if @buffer.takeback_chunk(info.chunk_id)\n              @rollback_count_metrics.inc\n              log.info \"delayed commit for buffer chunks was cancelled in shutdown\", chunk_id: dump_unique_id_hex(info.chunk_id)\n              primary = @as_secondary ? @primary_instance : self\n              primary.update_retry_state(info.chunk_id, @as_secondary)\n            end\n          end\n        end\n      end\n\n      def next_flush_time\n        if @buffer.queued?\n          @retry_mutex.synchronize do\n            @retry ? @retry.next_time : Time.now + @buffer_config.flush_thread_burst_interval\n          end\n        else\n          Time.now + @buffer_config.flush_thread_interval\n        end\n      end\n\n      UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError, MessagePack::UnpackError, EncodingError]\n\n      def try_flush\n        chunk = @buffer.dequeue_chunk\n        return unless chunk\n\n        log.on_trace { log.trace \"trying flush for a chunk\", chunk: dump_unique_id_hex(chunk.unique_id) }\n\n        output = self\n        using_secondary = false\n        if @retry_mutex.synchronize{ @retry && @retry.secondary? }\n          output = @secondary\n          using_secondary = true\n        end\n\n        if @enable_msgpack_streamer\n          chunk.extend ChunkMessagePackEventStreamer\n        end\n\n        begin\n          chunk_write_start = Fluent::Clock.now\n\n          if output.delayed_commit\n            log.trace \"executing delayed write and commit\", chunk: dump_unique_id_hex(chunk.unique_id)\n            @write_count_metrics.inc\n            @write_secondary_count_metrics.inc if using_secondary\n            @dequeued_chunks_mutex.synchronize do\n              # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)\n              @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)\n            end\n\n            output.try_write(chunk)\n            check_slow_flush(chunk_write_start)\n          else # output plugin without delayed purge\n            chunk_id = chunk.unique_id\n            dump_chunk_id = dump_unique_id_hex(chunk_id)\n            log.trace \"adding write count\", instance: self.object_id\n            @write_count_metrics.inc\n            @write_secondary_count_metrics.inc if using_secondary\n            log.trace \"executing sync write\", chunk: dump_chunk_id\n\n            output.write(chunk)\n            check_slow_flush(chunk_write_start)\n\n            log.trace \"write operation done, committing\", chunk: dump_chunk_id\n            commit_write(chunk_id, delayed: false, secondary: using_secondary)\n            log.trace \"done to commit a chunk\", chunk: dump_chunk_id\n          end\n        rescue *UNRECOVERABLE_ERRORS => e\n          if @secondary\n            if using_secondary\n              log.warn \"got unrecoverable error in secondary.\", error: e\n              log.warn_backtrace\n              backup_chunk(chunk, using_secondary, output.delayed_commit)\n            else\n              if (self.class == @secondary.class)\n                log.warn \"got unrecoverable error in primary and secondary type is same as primary. Skip secondary\", error: e\n                log.warn_backtrace\n                backup_chunk(chunk, using_secondary, output.delayed_commit)\n              else\n                # Call secondary output directly without retry update.\n                # In this case, delayed commit causes inconsistent state in dequeued chunks so async output in secondary is not allowed for now.\n                if @secondary.delayed_commit\n                  log.warn \"got unrecoverable error in primary and secondary is async output. Skip secondary for backup\", error: e\n                  log.warn_backtrace\n                  backup_chunk(chunk, using_secondary, output.delayed_commit)\n                else\n                  log.warn \"got unrecoverable error in primary. Skip retry and flush chunk to secondary\", error: e\n                  log.warn_backtrace\n                  begin\n                    @secondary.write(chunk)\n                    commit_write(chunk_id, delayed: output.delayed_commit, secondary: true)\n                  rescue => e\n                    log.warn \"got an error in secondary for unrecoverable error\", error: e\n                    log.warn_backtrace\n                    backup_chunk(chunk, using_secondary, output.delayed_commit)\n                  end\n                end\n              end\n            end\n          else\n            log.warn \"got unrecoverable error in primary and no secondary\", error: e\n            log.warn_backtrace\n            backup_chunk(chunk, using_secondary, output.delayed_commit)\n          end\n        rescue => e\n          log.debug \"taking back chunk for errors.\", chunk: dump_unique_id_hex(chunk.unique_id)\n          if output.delayed_commit\n            @dequeued_chunks_mutex.synchronize do\n              @dequeued_chunks.delete_if{|d| d.chunk_id == chunk.unique_id }\n            end\n          end\n\n          if @buffer.takeback_chunk(chunk.unique_id)\n            @rollback_count_metrics.inc\n          end\n\n          update_retry_state(chunk.unique_id, using_secondary, e)\n\n          raise if @under_plugin_development && !@retry_for_error_chunk\n        end\n      end\n\n      def backup_chunk(chunk, using_secondary, delayed_commit)\n        if @buffer.disable_chunk_backup\n          log.warn \"disable_chunk_backup is true. #{dump_unique_id_hex(chunk.unique_id)} chunk is thrown away\"\n        else\n          @buffer.backup(chunk.unique_id) { |f|\n            chunk.write_to(f)\n          }\n        end\n        commit_write(chunk.unique_id, secondary: using_secondary, delayed: delayed_commit)\n      end\n\n      def check_slow_flush(start)\n        elapsed_time = Fluent::Clock.now - start\n        elapsed_millsec = (elapsed_time * 1000).to_i\n        @flush_time_count_metrics.add(elapsed_millsec)\n        if elapsed_time > @slow_flush_log_threshold\n          @slow_flush_count_metrics.inc\n          log.warn \"buffer flush took longer time than slow_flush_log_threshold:\",\n                   elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id\n        end\n      end\n\n      def update_retry_state(chunk_id, using_secondary, error = nil)\n        @retry_mutex.synchronize do\n          @num_errors_metrics.inc\n          chunk_id_hex = dump_unique_id_hex(chunk_id)\n\n          unless @retry\n            @retry = retry_state(@buffer_config.retry_randomize)\n\n            if @retry.limit?\n              handle_limit_reached(error)\n            elsif error\n              log_retry_error(error, chunk_id_hex, using_secondary)\n            end\n\n            return\n          end\n\n          # @retry exists\n\n          # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when\n          # @retry.step is called almost as many times as the number of flush threads in a short time.\n          if Time.now >= @retry.next_time\n            @retry.step\n          else\n            @retry.recalc_next_time # to prevent all flush threads from retrying at the same time\n          end\n\n          if @retry.limit?\n            handle_limit_reached(error)\n          elsif error\n            log_retry_error(error, chunk_id_hex, using_secondary)\n          end\n        end\n      end\n\n      def log_retry_error(error, chunk_id_hex, using_secondary)\n        return unless error\n        if using_secondary\n          msg = \"failed to flush the buffer with secondary output.\"\n        else\n          msg = \"failed to flush the buffer.\"\n        end\n        log.warn(msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error)\n        log.warn_backtrace(error.backtrace)\n      end\n\n      def handle_limit_reached(error)\n        if error\n          records = @buffer.queued_records\n          msg = \"Hit limit for retries. dropping all chunks in the buffer queue.\"\n          log.error msg, retry_times: @retry.steps, records: records, error: error\n          log.error_backtrace error.backtrace\n        end\n        @buffer.clear_queue!\n        log.debug \"buffer queue cleared\"\n        @retry = nil\n      end\n\n      def retry_state(randomize)\n        if @secondary\n          retry_state_create(\n            :output_retries, @buffer_config.retry_type, @buffer_config.retry_wait, @buffer_config.retry_timeout,\n            forever: @buffer_config.retry_forever, max_steps: @buffer_config.retry_max_times, backoff_base: @buffer_config.retry_exponential_backoff_base,\n            max_interval: @buffer_config.retry_max_interval,\n            secondary: true, secondary_threshold: @buffer_config.retry_secondary_threshold,\n            randomize: randomize\n          )\n        else\n          retry_state_create(\n            :output_retries, @buffer_config.retry_type, @buffer_config.retry_wait, @buffer_config.retry_timeout,\n            forever: @buffer_config.retry_forever, max_steps: @buffer_config.retry_max_times, backoff_base: @buffer_config.retry_exponential_backoff_base,\n            max_interval: @buffer_config.retry_max_interval,\n            randomize: randomize\n          )\n        end\n      end\n\n      def submit_flush_once\n        return unless @buffer_config.flush_thread_count > 0\n        # Without locks: it is rough but enough to select \"next\" writer selection\n        @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count\n        state = @output_flush_threads[@output_flush_thread_current_position]\n        state.mutex.synchronize {\n          if state.thread&.status # \"run\"/\"sleep\"/\"aborting\" or false(successfully stop) or nil(killed by exception)\n            state.next_clock = 0\n            state.cond_var.signal\n          else\n            log.warn \"thread is already dead\"\n          end\n        }\n        Thread.pass\n      end\n\n      def force_flush\n        if @buffering\n          @buffer.enqueue_all(true)\n          submit_flush_all\n        end\n      end\n\n      def submit_flush_all\n        return unless @buffer_config.flush_thread_count > 0\n        while !@retry && @buffer.queued?\n          submit_flush_once\n          sleep @buffer_config.flush_thread_burst_interval\n        end\n      end\n\n      # only for tests of output plugin\n      def interrupt_flushes\n        @output_flush_interrupted = true\n      end\n\n      # only for tests of output plugin\n      def enqueue_thread_wait\n        @output_enqueue_thread_mutex.synchronize do\n          @output_flush_interrupted = false\n          @output_enqueue_thread_waiting = true\n        end\n        require 'timeout'\n        Timeout.timeout(10) do\n          Thread.pass while @output_enqueue_thread_waiting\n        end\n      end\n\n      # only for tests of output plugin\n      def flush_thread_wakeup\n        @output_flush_threads.each do |state|\n          state.mutex.synchronize {\n            if state.thread&.status\n              state.next_clock = 0\n              state.cond_var.signal\n            end\n          }\n          Thread.pass\n        end\n      end\n\n      def enqueue_thread_run\n        value_for_interval = nil\n        if @flush_mode == :interval\n          value_for_interval = @buffer_config.flush_interval\n        end\n        if @chunk_key_time\n          if !value_for_interval || @buffer_config.timekey < value_for_interval\n            value_for_interval = [@buffer_config.timekey, @buffer_config.timekey_wait].min\n          end\n        end\n        unless value_for_interval\n          raise \"BUG: both of flush_interval and timekey are disabled\"\n        end\n        interval = value_for_interval / 11.0\n        if interval < @buffer_config.flush_thread_interval\n          interval = @buffer_config.flush_thread_interval\n        end\n\n        while !self.after_started? && !self.stopped?\n          sleep 0.5\n        end\n        log.debug \"enqueue_thread actually running\"\n\n        begin\n          while @output_enqueue_thread_running\n            now_int = Time.now.to_i\n            if @output_flush_interrupted\n              sleep interval\n              next\n            end\n\n            @output_enqueue_thread_mutex.lock\n            begin\n              if @flush_mode == :interval\n                flush_interval = @buffer_config.flush_interval.to_i\n                # This block should be done by integer values.\n                # If both of flush_interval & flush_thread_interval are 1s, expected actual flush timing is 1.5s.\n                # If we use integered values for this comparison, expected actual flush timing is 1.0s.\n                @buffer.enqueue_all{ |metadata, chunk| chunk.raw_create_at + flush_interval <= now_int }\n              end\n\n              if @chunk_key_time\n                timekey_unit = @buffer_config.timekey\n                timekey_wait = @buffer_config.timekey_wait\n                current_timekey = now_int - now_int % timekey_unit\n                @buffer.enqueue_all{ |metadata, chunk| metadata.timekey < current_timekey && metadata.timekey + timekey_unit + timekey_wait <= now_int }\n              end\n            rescue => e\n              raise if @under_plugin_development\n              log.error \"unexpected error while checking flushed chunks. ignored.\", error: e\n              log.error_backtrace\n            ensure\n              @output_enqueue_thread_waiting = false\n              @output_enqueue_thread_mutex.unlock\n            end\n            sleep interval\n          end\n        rescue => e\n          # normal errors are rescued by inner begin-rescue clause.\n          log.error \"error on enqueue thread\", error: e\n          log.error_backtrace\n          raise\n        end\n      end\n\n      def flush_thread_run(state)\n        flush_thread_interval = @buffer_config.flush_thread_interval\n\n        state.next_clock = Fluent::Clock.now + flush_thread_interval\n\n        while !self.after_started? && !self.stopped?\n          sleep 0.5\n        end\n        log.debug \"flush_thread actually running\"\n\n        state.mutex.lock\n        begin\n          # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase\n          while @output_flush_threads_running\n            current_clock = Fluent::Clock.now\n            next_retry_time = nil\n\n            @retry_mutex.synchronize do\n              next_retry_time = @retry ? @retry.next_time : nil\n            end\n\n            if state.next_clock > current_clock\n              interval = state.next_clock - current_clock\n            elsif next_retry_time && next_retry_time > Time.now\n              interval = next_retry_time.to_f - Time.now.to_f\n            else\n              state.mutex.unlock\n              begin\n                try_flush\n                # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)\n                interval = next_flush_time.to_f - Time.now.to_f\n                # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected\n                #       because @retry still exists (#commit_write is not called yet in #try_flush)\n                #       @retry should be cleared if delayed commit is enabled? Or any other solution?\n                state.next_clock = Fluent::Clock.now + interval\n              ensure\n                state.mutex.lock\n              end\n            end\n\n            if @dequeued_chunks_mutex.synchronize{ !@dequeued_chunks.empty? && @dequeued_chunks.first.expired? }\n              unless @output_flush_interrupted\n                state.mutex.unlock\n                begin\n                  try_rollback_write\n                ensure\n                  state.mutex.lock\n                end\n              end\n            end\n\n            state.cond_var.wait(state.mutex, interval) if interval > 0\n          end\n        rescue => e\n          # normal errors are rescued by output plugins in #try_flush\n          # so this rescue section is for critical & unrecoverable errors\n          log.error \"error on output thread\", error: e\n          log.error_backtrace\n          raise\n        ensure\n          state.mutex.unlock\n        end\n      end\n\n      BUFFER_STATS_KEYS = {}\n      Fluent::Plugin::Buffer::STATS_KEYS.each { |key|\n        BUFFER_STATS_KEYS[key] = \"buffer_#{key}\"\n      }\n\n      def statistics\n        stats = {\n          'emit_records' => @emit_records_metrics.get,\n          'emit_size' => @emit_size_metrics.get,\n          # Respect original name\n          # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284\n          'retry_count' => @num_errors_metrics.get,\n          'emit_count' => @emit_count_metrics.get,\n          'write_count' => @write_count_metrics.get,\n          'write_secondary_count' => @write_secondary_count_metrics.get,\n          'rollback_count' => @rollback_count_metrics.get,\n          'slow_flush_count' => @slow_flush_count_metrics.get,\n          'flush_time_count' => @flush_time_count_metrics.get,\n          'drop_oldest_chunk_count' => @drop_oldest_chunk_count_metrics.get,\n        }\n\n        if @buffer && @buffer.respond_to?(:statistics)\n          (@buffer.statistics['buffer'] || {}).each do |k, v|\n            stats[BUFFER_STATS_KEYS[k]] = v\n          end\n        end\n\n        { 'output' => stats }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/owned_by_mixin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module Plugin\n    module OwnedByMixin\n      def owner=(plugin)\n        @_owner = plugin\n\n        @_plugin_id = plugin.plugin_id\n\n        @log = plugin.log\n      end\n\n      def owner\n        if instance_variable_defined?(:@_owner)\n          @_owner\n        end\n      end\n\n      def log\n        if instance_variable_defined?(:@log)\n          @log\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/base'\nrequire 'fluent/plugin/owned_by_mixin'\n\nrequire 'fluent/error'\nrequire 'fluent/mixin' # for TypeConverter\nrequire 'fluent/time'\nrequire 'fluent/plugin/string_util'\n\nrequire 'serverengine/blocking_flag'\n\nmodule Fluent\n  module Plugin\n    class Parser < Base\n      class TimeoutChecker\n        # This implementation now uses mutex because parser is typically used in input.\n        # If this has a performance issue under high concurrent, use concurrent-ruby's map instead.\n        def initialize(timeout)\n          @map = {}\n          @flag = ServerEngine::BlockingFlag.new\n          @mutex = Mutex.new\n          @timeout = timeout\n          @timeout_checker = nil\n        end\n\n        def start\n          @thread = ::Thread.new {\n            until @flag.wait_for_set(0.5)\n              now = Time.now\n              @mutex.synchronize {\n                @map.keys.each { |th|\n                  time = @map[th]\n                  if now - time > @timeout\n                    @map.delete(th)\n                    th.raise UncatchableError, \"parsing timed out\"\n                  end\n                }\n              }\n            end\n          }\n        end\n\n        def stop\n          @flag.set!\n          @thread.join\n        end\n\n        def execute\n          th = Thread.current\n          @mutex.synchronize { @map[th] = Time.now }\n          yield\n        ensure\n          # Need clean up here because if next event is delayed, incorrect exception will be raised in normal flow.\n          @mutex.synchronize { @map.delete(th) }\n        end\n      end\n\n      include OwnedByMixin\n      include TimeMixin::Parser\n\n      class ParserError < StandardError; end\n\n      configured_in :parse\n\n      ### types can be specified as string-based hash style\n      # field1:type, field2:type, field3:type:option, field4:type:option\n      ### or, JSON format\n      # {\"field1\":\"type\", \"field2\":\"type\", \"field3\":\"type:option\", \"field4\":\"type:option\"}\n      config_param :types, :hash, value_type: :string, default: nil\n\n      # available options are:\n      # array: (1st) delimiter\n      # time : type[, format, timezone] -> type should be a valid \"time_type\"(string/unixtime/float)\n      #      : format[, timezone]\n\n      config_param :time_key, :string, default: nil\n      config_param :null_value_pattern, :regexp, default: nil\n      config_param :null_empty_string, :bool, default: false\n      config_param :estimate_current_event, :bool, default: true\n      config_param :keep_time_key, :bool, default: false\n      config_param :timeout, :time, default: nil\n\n      AVAILABLE_PARSER_VALUE_TYPES = ['string', 'integer', 'float', 'bool', 'time', 'array']\n\n      # for tests\n      attr_reader :type_converters\n\n      PARSER_TYPES = [:text_per_line, :text, :binary]\n      def parser_type\n        :text_per_line\n      end\n\n      def initialize\n        super\n\n        @timeout_checker = nil\n      end\n\n      def configure(conf)\n        super\n\n        @time_parser = time_parser_create\n        @type_converters = build_type_converters(@types)\n        @execute_convert_values = @type_converters || @null_value_pattern || @null_empty_string\n        @timeout_checker = if @timeout\n                             class << self\n                               alias_method :parse_orig, :parse\n                               alias_method :parse, :parse_with_timeout\n                             end\n                             TimeoutChecker.new(@timeout)\n                           else\n                             nil\n                           end\n      end\n\n      def start\n        super\n\n        @timeout_checker.start if @timeout_checker\n      end\n\n      def stop\n        super\n\n        @timeout_checker.stop if @timeout_checker\n      end\n\n      def parse(text, &block)\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def parse_with_timeout(text, &block)\n        @timeout_checker.execute {\n          parse_orig(text, &block)\n        }\n      rescue UncatchableError\n        log.warn \"parsing timed out with #{self.class}: text = #{text}\"\n        # Return nil instead of raising error. in_tail or other plugin can emit broken line.\n        yield nil, nil\n      end\n\n      def call(*a, &b)\n        # Keep backward compatibility for existing plugins\n        # TODO: warn when deprecated\n        parse(*a, &b)\n      end\n\n      def implement?(feature)\n        methods_of_plugin = self.class.instance_methods(false)\n        case feature\n        when :parse_io then methods_of_plugin.include?(:parse_io)\n        when :parse_partial_data then methods_of_plugin.include?(:parse_partial_data)\n        else\n          raise ArgumentError, \"Unknown feature for parser plugin: #{feature}\"\n        end\n      end\n\n      def parse_io(io, &block)\n        raise NotImplementedError, \"Optional API #parse_io is not implemented\"\n      end\n\n      def parse_partial_data(data, &block)\n        raise NotImplementedError, \"Optional API #parse_partial_data is not implemented\"\n      end\n\n      def parse_time(record)\n        if @time_key && record.respond_to?(:has_key?) && record.has_key?(@time_key)\n          src = if @keep_time_key\n                  record[@time_key]\n                else\n                  record.delete(@time_key)\n                end\n          @time_parser.parse(src)\n        elsif @estimate_current_event\n          Fluent::EventTime.now\n        else\n          nil\n        end\n      rescue Fluent::TimeParser::TimeParseError => e\n        raise ParserError, e.message\n      end\n\n      # def parse(text, &block)\n      #   time, record = convert_values(time, record)\n      #   yield time, record\n      # end\n      def convert_values(time, record)\n        return time, record unless @execute_convert_values\n\n        record.each_key do |key|\n          value = record[key]\n          next unless value # nil/null value is always left as-is.\n\n          if value.is_a?(String) && string_like_null(value)\n            record[key] = nil\n            next\n          end\n\n          if @type_converters && @type_converters.has_key?(key)\n            record[key] = @type_converters[key].call(value)\n          end\n        end\n\n        return time, record\n      end\n\n      def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_pattern)\n        null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match?(s) }\n      end\n\n      TRUTHY_VALUES = ['true', 'yes', '1']\n\n      def build_type_converters(types)\n        return nil unless types\n\n        converters = {}\n\n        types.each_pair do |field_name, type_definition|\n          type, option = type_definition.split(\":\", 2)\n          unless AVAILABLE_PARSER_VALUE_TYPES.include?(type)\n            raise Fluent::ConfigError, \"unknown value conversion for key:'#{field_name}', type:'#{type}'\"\n          end\n\n          conv = case type\n                 when 'string' then ->(v){ v.to_s }\n                 when 'integer' then ->(v){ v.to_i rescue v.to_s.to_i }\n                 when 'float' then ->(v){ v.to_f rescue v.to_s.to_f }\n                 when 'bool' then ->(v){ TRUTHY_VALUES.include?(v.to_s.downcase) }\n                 when 'time'\n                   # comma-separated: time:[timezone:]time_format\n                   # time_format is unixtime/float/string-time-format\n                   timep = if option\n                             time_type = 'string' # estimate\n                             timezone, time_format = option.split(':', 2)\n                             unless Fluent::Timezone.validate(timezone)\n                               timezone, time_format = nil, option\n                             end\n                             if Fluent::TimeMixin::TIME_TYPES.include?(time_format)\n                               time_type, time_format = time_format, nil # unixtime/float\n                             end\n                             time_parser_create(type: time_type.to_sym, format: time_format, timezone: timezone)\n                           else\n                             time_parser_create(type: :string, format: nil, timezone: nil)\n                           end\n                   ->(v){ timep.parse(v) rescue nil }\n                 when 'array'\n                   delimiter = option ? option.to_s : ','\n                   ->(v){ string_safe_encoding(v.to_s){|s| s.split(delimiter) } }\n                 else\n                   raise \"BUG: unknown type even after check: #{type}\"\n                 end\n          converters[field_name] = conv\n        end\n\n        converters\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_apache.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser_regexp'\n\nmodule Fluent\n  module Plugin\n    class ApacheParser < RegexpParser\n      Plugin.register_parser(\"apache\", self)\n\n      config_set_default :expression, /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \"(?<method>\\S+)(?: +(?<path>[^ ]*) +\\S*)?\" (?<code>[^ ]*) (?<size>[^ ]*)(?: \"(?<referer>[^\\\"]*)\" \"(?<agent>[^\\\"]*)\")?$/\n      config_set_default :time_format, \"%d/%b/%Y:%H:%M:%S %z\"\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_apache2.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser'\n\nmodule Fluent\n  module Plugin\n    class Apache2Parser < Parser\n      Plugin.register_parser('apache2', self)\n\n      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \"(?<method>\\S+)(?: +(?<path>(?:[^\\\"]|\\\\\")*?)(?: +\\S*)?)?\" (?<code>[^ ]*) (?<size>[^ ]*)(?: \"(?<referer>(?:[^\\\"]|\\\\\")*)\" \"(?<agent>(?:[^\\\"]|\\\\\")*)\")?$/\n      TIME_FORMAT = \"%d/%b/%Y:%H:%M:%S %z\"\n\n      def initialize\n        super\n        @mutex = Mutex.new\n      end\n\n      def configure(conf)\n        super\n        @time_parser = time_parser_create(format: TIME_FORMAT)\n      end\n\n      def patterns\n        {'format' => REGEXP, 'time_format' => TIME_FORMAT}\n      end\n\n      def parse(text)\n        m = REGEXP.match(text)\n        unless m\n          yield nil, nil\n          return\n        end\n\n        host = m['host']\n        host = (host == '-') ? nil : host\n\n        user = m['user']\n        user = (user == '-') ? nil : user\n\n        time = m['time']\n        time = @mutex.synchronize { @time_parser.parse(time) }\n\n        method = m['method']\n        path = m['path']\n\n        code = m['code'].to_i\n        code = nil if code == 0\n\n        size = m['size']\n        size = (size == '-') ? nil : size.to_i\n\n        referer = m['referer']\n        referer = (referer == '-') ? nil : referer\n\n        agent = m['agent']\n        agent = (agent == '-') ? nil : agent\n\n        record = {\n          \"host\" => host,\n          \"user\" => user,\n          \"method\" => method,\n          \"path\" => path,\n          \"code\" => code,\n          \"size\" => size,\n          \"referer\" => referer,\n          \"agent\" => agent,\n        }\n        record[\"time\"] = m['time'] if @keep_time_key\n\n        yield time, record\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_apache_error.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser_regexp'\n\nmodule Fluent\n  module Plugin\n    class ApacheErrorParser < RegexpParser\n      Plugin.register_parser(\"apache_error\", self)\n      config_set_default :expression, /^\\[[^ ]* (?<time>[^\\]]*)\\] \\[(?<level>[^\\]]*)\\](?: \\[pid (?<pid>[^\\]]*)\\])?( \\[client (?<client>[^\\]]*)\\])? (?<message>.*)$/\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_csv.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser'\n\nrequire 'csv'\n\nmodule Fluent\n  module Plugin\n    class CSVParser < Parser\n      Plugin.register_parser('csv', self)\n\n      desc 'Names of fields included in each lines'\n      config_param :keys, :array, value_type: :string\n      desc 'The delimiter character (or string) of CSV values'\n      config_param :delimiter, :string, default: ','\n      desc 'The parser type used to parse CSV line'\n      config_param :parser_engine, :enum, list: [:normal, :fast], default: :normal, alias: :parser_type\n\n      def configure(conf)\n        super\n\n\n        if @parser_engine == :fast\n          @quote_char = '\"'\n          @escape_pattern = Regexp.compile(@quote_char * 2)\n\n          m = method(:parse_fast)\n          self.singleton_class.module_eval do\n            define_method(:parse, m)\n          end\n        end\n      end\n\n      def parse(text, &block)\n        values = CSV.parse_line(text, col_sep: @delimiter)\n        r = Hash[@keys.zip(values)]\n        time, record = convert_values(parse_time(r), r)\n        yield time, record\n      end\n\n      def parse_fast(text, &block)\n        r = parse_fast_internal(text)\n        time, record = convert_values(parse_time(r), r)\n        yield time, record\n      end\n\n      # CSV.parse_line is too slow due to initialize lots of object and\n      # CSV module doesn't provide the efficient method for parsing single line.\n      # This method avoids the overhead of CSV.parse_line for typical patterns\n      def parse_fast_internal(text)\n        record = {}\n        text.chomp!\n\n        return record if text.empty?\n\n        # use while because while is now faster than each_with_index\n        columns = text.split(@delimiter, -1)\n        num_columns = columns.size\n        i = 0\n        j = 0\n        while j < num_columns\n          column = columns[j]\n\n          case column.count(@quote_char)\n          when 0\n            if column.empty?\n              column = nil\n            end\n          when 1\n            if column.start_with?(@quote_char)\n              to_merge = [column]\n              j += 1\n              while j < num_columns\n                merged_col = columns[j]\n                to_merge << merged_col\n                break if merged_col.end_with?(@quote_char)\n                j += 1\n              end\n              column = to_merge.join(@delimiter)[1..-2]\n            end\n          when 2\n            if column.start_with?(@quote_char) && column.end_with?(@quote_char)\n              column = column[1..-2]\n            end\n          else\n            if column.start_with?(@quote_char) && column.end_with?(@quote_char)\n              column = column[1..-2]\n            end\n            column.gsub!(@escape_pattern, @quote_char)\n          end\n\n          record[@keys[i]] = column\n          j += 1\n          i += 1\n        end\n        record\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_json.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser'\nrequire 'fluent/time'\nrequire 'fluent/oj_options'\n\nrequire 'yajl'\nrequire 'json'\n\nmodule Fluent\n  module Plugin\n    class JSONParser < Parser\n      Plugin.register_parser('json', self)\n\n      config_set_default :time_key, 'time'\n      desc 'Set JSON parser'\n      config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj\n\n      # The Yajl library defines a default buffer size of 8KiB when parsing\n      # from IO streams, so maintain this for backwards-compatibility.\n      # https://www.rubydoc.info/github/brianmario/yajl-ruby/Yajl%2FParser:parse\n      desc 'Set the buffer size that Yajl will use when parsing streaming input'\n      config_param :stream_buffer_size, :integer, default: 8192\n\n      config_set_default :time_type, :float\n\n      def configure(conf)\n        if conf.has_key?('time_format')\n          conf['time_type'] ||= 'string'\n        end\n\n        super\n        @load_proc, @error_class = configure_json_parser(@json_parser)\n      end\n\n      def configure_json_parser(name)\n        case name\n        when :oj\n          return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?\n\n          log&.info \"Oj is not installed, and failing back to JSON for json parser\"\n          configure_json_parser(:json)\n        when :json then [JSON.method(:parse), JSON::ParserError]\n        when :yajl then [Yajl.method(:load), Yajl::ParseError]\n        else\n          raise \"BUG: unknown json parser specified: #{name}\"\n        end\n      end\n\n      def parse(text)\n        parsed_json = @load_proc.call(text)\n\n        if parsed_json.is_a?(Hash)\n          time, record = parse_one_record(parsed_json)\n          yield time, record\n        elsif parsed_json.is_a?(Array)\n          parsed_json.each do |record|\n            unless record.is_a?(Hash)\n              yield nil, nil\n              next\n            end\n            time, parsed_record = parse_one_record(record)\n            yield time, parsed_record\n          end\n        else\n          yield nil, nil\n        end\n\n      rescue @error_class, EncodingError # EncodingError is for oj 3.x or later\n        yield nil, nil\n      end\n\n      def parse_one_record(record)\n        time = parse_time(record)\n        convert_values(time, record)\n      end\n\n      def parser_type\n        :text\n      end\n\n      def parse_io(io, &block)\n        y = Yajl::Parser.new\n        y.on_parse_complete = ->(record){\n          block.call(parse_time(record), record)\n        }\n        y.parse(io, @stream_buffer_size)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_ltsv.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser'\n\nmodule Fluent\n  module Plugin\n    class LabeledTSVParser < Parser\n      Plugin.register_parser('ltsv', self)\n\n      desc 'The delimiter character (or string) of TSV values'\n      config_param :delimiter, :string, default: \"\\t\"\n      desc 'The delimiter pattern of TSV values'\n      config_param :delimiter_pattern, :regexp, default: nil\n      desc 'The delimiter character between field name and value'\n      config_param :label_delimiter, :string, default: \":\"\n\n      config_set_default :time_key, 'time'\n\n      def configure(conf)\n        super\n        @delimiter = @delimiter_pattern || @delimiter\n      end\n\n      def parse(text)\n        r = {}\n        text.split(@delimiter).each do |pair|\n          if pair.include? @label_delimiter\n            key, value = pair.split(@label_delimiter, 2)\n            r[key] = value\n          end\n        end\n        time, record = convert_values(parse_time(r), r)\n        yield time, record\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_msgpack.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser'\nrequire 'fluent/msgpack_factory'\n\nmodule Fluent\n  module Plugin\n    class MessagePackParser < Parser\n      Plugin.register_parser('msgpack', self)\n\n      def configure(conf)\n        super\n        @unpacker = Fluent::MessagePackFactory.engine_factory.unpacker\n      end\n\n      def parser_type\n        :binary\n      end\n\n      def parse(data, &block)\n        @unpacker.feed_each(data) do |obj|\n          parse_unpacked_data(obj, &block)\n        end\n      end\n      alias parse_partial_data parse\n\n      def parse_io(io, &block)\n        u = Fluent::MessagePackFactory.engine_factory.unpacker(io)\n        u.each do |obj|\n          parse_unpacked_data(obj, &block)\n        end\n      end\n\n      def parse_unpacked_data(data)\n        if data.is_a?(Hash)\n          time, record = convert_values(parse_time(data), data)\n          yield time, record\n          return\n        end\n\n        unless data.is_a?(Array)\n          yield nil, nil\n          return\n        end\n\n        data.each do |record|\n          unless record.is_a?(Hash)\n            yield nil, nil\n            next\n          end\n          time, converted_record = convert_values(parse_time(record), record)\n          yield time, converted_record\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_multiline.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser'\nrequire 'fluent/plugin/parser_regexp'\n\nmodule Fluent\n  module Plugin\n    class MultilineParser < Parser\n      Plugin.register_parser('multiline', self)\n\n      desc 'Specify regexp pattern for start line of multiple lines'\n      config_param :format_firstline, :string, default: nil\n      desc 'Enable an option returning line as unmatched_line'\n      config_param :unmatched_lines, :string, default: nil\n\n      FORMAT_MAX_NUM = 20\n\n      class MultilineRegexpParser < Fluent::Plugin::RegexpParser\n        def parse(text)\n          m = @expression.match(text)\n          unless m\n            yield nil, nil\n            return m\n          end\n\n          r = {}\n          m.names.each do |name|\n            if (value = m[name])\n              r[name] = value\n            end\n          end\n\n          time, record = convert_values(parse_time(r), r)\n\n          yield(time, record)\n          m\n        end\n      end\n\n      def configure(conf)\n        super\n\n        formats = parse_formats(conf).compact.map { |f| f[1..-2] }.join\n        begin\n          regexp = Regexp.new(formats, Regexp::MULTILINE)\n          if regexp.named_captures.empty?\n            raise \"No named captures\"\n          end\n          regexp_conf = Fluent::Config::Element.new(\"\", \"\", { \"expression\" => \"/#{formats}/m\" }, [])\n          @parser = Fluent::Plugin::MultilineParser::MultilineRegexpParser.new\n          @parser.configure(conf + regexp_conf)\n        rescue => e\n          raise Fluent::ConfigError, \"Invalid regexp '#{formats}': #{e}\"\n        end\n\n        if @format_firstline\n          check_format_regexp(@format_firstline, 'format_firstline')\n          @firstline_regex = Regexp.new(@format_firstline[1..-2])\n        end\n      end\n\n      def parse(text, &block)\n        loop do\n          m =\n            if @unmatched_lines\n              @parser.call(text) do |time, record|\n                if time && record\n                  yield(time, record)\n                else\n                  yield(Fluent::EventTime.now, { 'unmatched_line' => text })\n                end\n              end\n            else\n              @parser.call(text, &block)\n            end\n\n          return if m.nil?\n\n          text = m.post_match\n          if text.start_with?(\"\\n\")\n            text = text[1..-1]\n          end\n\n          return if text.empty?\n        end\n      end\n\n      def has_firstline?\n        !!@format_firstline\n      end\n\n      def firstline?(text)\n        @firstline_regex.match?(text)\n      end\n\n      private\n\n      def parse_formats(conf)\n        check_format_range(conf)\n\n        prev_format = nil\n        (1..FORMAT_MAX_NUM).map { |i|\n          format = conf[\"format#{i}\"]\n          if (i > 1) && prev_format.nil? && !format.nil?\n            raise Fluent::ConfigError, \"Jump of format index found. format#{i - 1} is missing.\"\n          end\n          prev_format = format\n          next if format.nil?\n\n          check_format_regexp(format, \"format#{i}\")\n          format\n        }\n      end\n\n      def check_format_range(conf)\n        invalid_formats = conf.keys.select { |k|\n          m = k.match(/^format(\\d+)$/)\n          m ? !((1..FORMAT_MAX_NUM).include?(m[1].to_i)) : false\n        }\n        unless invalid_formats.empty?\n          raise Fluent::ConfigError, \"Invalid formatN found. N should be 1 - #{FORMAT_MAX_NUM}: \" + invalid_formats.join(\",\")\n        end\n      end\n\n      def check_format_regexp(format, key)\n        if format[0] == '/' && format[-1] == '/'\n          begin\n            Regexp.new(format[1..-2], Regexp::MULTILINE)\n          rescue => e\n            raise Fluent::ConfigError, \"Invalid regexp in #{key}: #{e}\"\n          end\n        else\n          raise Fluent::ConfigError, \"format should be Regexp, need //, in #{key}: '#{format}'\"\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_nginx.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser_regexp'\n\nmodule Fluent\n  module Plugin\n    class NginxParser < RegexpParser\n      Plugin.register_parser(\"nginx\", self)\n\n      config_set_default :expression, /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \"(?<method>\\S+)(?: +(?<path>[^\\\"]*?)(?: +\\S*)?)?\" (?<code>[^ ]*) (?<size>[^ ]*)(?: \"(?<referer>[^\\\"]*)\" \"(?<agent>[^\\\"]*)\"(?:\\s+\\\"?(?<http_x_forwarded_for>[^\\\"]*)\\\"?)?)?$/\n      config_set_default :time_format, \"%d/%b/%Y:%H:%M:%S %z\"\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_none.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser'\n\nrequire 'fluent/time'\n\nmodule Fluent\n  module Plugin\n    class NoneParser < Parser\n      Plugin.register_parser('none', self)\n\n      desc 'Field name to contain logs'\n      config_param :message_key, :string, default: 'message'\n\n      def parse(text)\n        record = {@message_key => text}\n        time = @estimate_current_event ? Fluent::EventTime.now : nil\n        yield time, record\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_regexp.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser'\n\nmodule Fluent\n  module Plugin\n    class RegexpParser < Parser\n      Plugin.register_parser(\"regexp\", self)\n\n      desc 'Regular expression for matching logs'\n      config_param :expression, :regexp\n      desc 'Ignore case in matching'\n      config_param :ignorecase, :bool, default: false, deprecated: \"Use /pattern/i instead, this option is no longer effective\"\n      desc 'Build regular expression as a multline mode'\n      config_param :multiline, :bool, default: false, deprecated: \"Use /pattern/m instead, this option is no longer effective\"\n\n      config_set_default :time_key, 'time'\n\n      def configure(conf)\n        super\n        # For compat layer\n        if @ignorecase || @multiline\n          options = 0\n          options |= Regexp::IGNORECASE if @ignorecase\n          options |= Regexp::MULTILINE if @multiline\n          @expression = Regexp.compile(@expression.source, options)\n        end\n        @regexp = @expression # For backward compatibility\n\n        if @expression.named_captures.empty?\n          raise Fluent::ConfigError, \"No named captures in 'expression' parameter. The regexp must have at least one named capture\"\n        end\n      end\n\n      def parse(text)\n        m = @expression.match(text)\n        unless m\n          yield nil, nil\n          return\n        end\n\n        r = {}\n        m.names.each do |name|\n          if value = m[name]\n            r[name] = value\n          end\n        end\n\n        time, record = convert_values(parse_time(r), r)\n        yield time, record\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_syslog.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser'\n\nrequire 'fluent/time'\n\nmodule Fluent\n  module Plugin\n    class SyslogParser < Parser\n      Plugin.register_parser('syslog', self)\n\n      # TODO: Remove them since these regexps are no longer needed. but keep them for compatibility for now\n      # From existence TextParser pattern\n      REGEXP = /^(?<time>[^ ]*\\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\\[]*)(?:\\[(?<pid>[0-9]+)\\])?(?:[^\\:]*\\:)? *(?<message>.*)$/\n      # From in_syslog default pattern\n      REGEXP_WITH_PRI = /^\\<(?<pri>[0-9]+)\\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\\[]*)(?:\\[(?<pid>[0-9]+)\\])?(?:[^\\:]*\\:)? *(?<message>.*)$/\n      REGEXP_RFC5424 = <<~'EOS'.chomp\n        (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\\-|(?:\\[.*?(?<!\\\\)\\])+))(?: (?<message>.+))?\n      EOS\n      REGEXP_RFC5424_NO_PRI = Regexp.new(<<~'EOS'.chomp % REGEXP_RFC5424, Regexp::MULTILINE)\n        \\A%s\\z\n      EOS\n      REGEXP_RFC5424_WITH_PRI = Regexp.new(<<~'EOS'.chomp % REGEXP_RFC5424, Regexp::MULTILINE)\n        \\A<(?<pri>[0-9]{1,3})\\>[1-9]\\d{0,2} %s\\z\n      EOS\n\n      REGEXP_DETECT_RFC5424 = /^\\<[0-9]{1,3}\\>[1-9]\\d{0,2}/\n\n      RFC3164_WITHOUT_TIME_AND_PRI_REGEXP = /(?<host>[^ ]*) (?<ident>[^ :\\[]*)(?:\\[(?<pid>[0-9]+)\\])?(?:[^\\:]*\\:)? *(?<message>.*)$/\n      RFC3164_CAPTURES = RFC3164_WITHOUT_TIME_AND_PRI_REGEXP.names.freeze\n      RFC3164_PRI_REGEXP = /^<(?<pri>[0-9]{1,3})>/\n\n      RFC5424_WITHOUT_TIME_AND_PRI_REGEXP = /(?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\\-|(?:\\[.*?(?<!\\\\)\\])+))(?: (?<message>.+))?\\z/m\n      RFC5424_CAPTURES = RFC5424_WITHOUT_TIME_AND_PRI_REGEXP.names.freeze\n      RFC5424_PRI_REGEXP = /^<(?<pri>\\d{1,3})>\\d\\d{0,2}\\s/\n\n      config_set_default :time_format, \"%b %d %H:%M:%S\"\n      desc 'If the incoming logs have priority prefix, e.g. <9>, set true'\n      config_param :with_priority, :bool, default: false\n      desc 'Specify protocol format'\n      config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164\n      desc 'Specify time format for event time for rfc5424 protocol'\n      config_param :rfc5424_time_format, :string, default: \"%Y-%m-%dT%H:%M:%S.%L%z\"\n      desc 'The parser type used to parse syslog message'\n      config_param :parser_engine, :enum, list: [:regexp, :string], default: :regexp, alias: :parser_type\n      desc 'support colonless ident in string parser'\n      config_param :support_colonless_ident, :bool, default: true\n\n      def initialize\n        super\n        @mutex = Mutex.new\n        @regexp = nil\n        @regexp3164 = nil\n        @regexp5424 = nil\n        @regexp_parser = nil\n        @time_parser_rfc3164 = nil\n        @time_parser_rfc5424 = nil\n        @space_count_rfc3164 = nil\n        @space_count_rfc5424 = nil\n        @skip_space_count_rfc3164 = false\n        @skip_space_count_rfc5424 = false\n        @time_parser_rfc5424_without_subseconds = nil\n      end\n\n      def configure(conf)\n        super\n\n        @regexp_parser = @parser_engine == :regexp\n        @regexp = case @message_format\n                  when :rfc3164\n                    if @regexp_parser\n                      class << self\n                        alias_method :parse, :parse_rfc3164_regex\n                      end\n                    else\n                      class << self\n                        alias_method :parse, :parse_rfc3164\n                      end\n                    end\n                    setup_time_parser_3164(@time_format)\n                    RFC3164_WITHOUT_TIME_AND_PRI_REGEXP\n                  when :rfc5424\n                    if @regexp_parser\n                      class << self\n                        alias_method :parse, :parse_rfc5424_regex\n                      end\n                    else\n                      class << self\n                        alias_method :parse, :parse_rfc5424\n                      end\n                    end\n                    @time_format = @rfc5424_time_format unless conf.has_key?('time_format')\n                    setup_time_parser_5424(@time_format)\n                    RFC5424_WITHOUT_TIME_AND_PRI_REGEXP\n                  when :auto\n                    class << self\n                      alias_method :parse, :parse_auto\n                    end\n                    setup_time_parser_3164(@time_format)\n                    setup_time_parser_5424(@rfc5424_time_format)\n                    nil\n                  end\n\n        if @regexp_parser\n          @regexp3164 = RFC3164_WITHOUT_TIME_AND_PRI_REGEXP\n          @regexp5424 = RFC5424_WITHOUT_TIME_AND_PRI_REGEXP\n        end\n      end\n\n      def setup_time_parser_3164(time_fmt)\n        @time_parser_rfc3164 = time_parser_create(format: time_fmt)\n        if ['%b %d %H:%M:%S', '%b %d %H:%M:%S.%N'].include?(time_fmt)\n          @skip_space_count_rfc3164 = true\n        end\n        @space_count_rfc3164 = time_fmt.squeeze(' ').count(' ') + 1\n      end\n\n      def setup_time_parser_5424(time_fmt)\n        @time_parser_rfc5424 = time_parser_create(format: time_fmt)\n        @time_parser_rfc5424_without_subseconds = time_parser_create(format: \"%Y-%m-%dT%H:%M:%S%z\")\n        @skip_space_count_rfc5424 = time_fmt.count(' ').zero?\n        @space_count_rfc5424 = time_fmt.squeeze(' ').count(' ') + 1\n      end\n\n      # this method is for tests\n      def patterns\n        {'format' => @regexp, 'time_format' => @time_format}\n      end\n\n      def parse(text)\n        # This is overwritten in configure\n      end\n\n      def parse_auto(text, &block)\n        if REGEXP_DETECT_RFC5424.match?(text)\n          if @regexp_parser\n            parse_rfc5424_regex(text, &block)\n          else\n            parse_rfc5424(text, &block)\n          end\n        else\n          if @regexp_parser\n            parse_rfc3164_regex(text, &block)\n          else\n            parse_rfc3164(text, &block)\n          end\n        end\n      end\n\n      SPLIT_CHAR = ' '.freeze\n\n      def parse_rfc3164_regex(text, &block)\n        idx = 0\n        record = {}\n\n        if @with_priority\n          if RFC3164_PRI_REGEXP.match?(text)\n            v = text.index('>')\n            record['pri'] = text[1..v].to_i # trim `<` and ``>\n            idx = v + 1\n          else\n            yield(nil, nil)\n            return\n          end\n        end\n\n        i = idx - 1\n        sq = false\n        @space_count_rfc3164.times do\n          while text[i + 1] == SPLIT_CHAR\n            sq = true\n            i += 1\n          end\n\n          i = text.index(SPLIT_CHAR, i + 1)\n        end\n\n        time_str = sq ? text.slice(idx, i - idx).squeeze(SPLIT_CHAR) : text.slice(idx, i - idx)\n        time = @mutex.synchronize { @time_parser_rfc3164.parse(time_str) }\n        if @keep_time_key\n          record['time'] = time_str\n        end\n\n        parse_plain(@regexp3164, time, text, i + 1, record, RFC3164_CAPTURES, &block)\n      end\n\n      def parse_rfc5424_regex(text, &block)\n        idx = 0\n        record = {}\n\n        if @with_priority\n          if (m = RFC5424_PRI_REGEXP.match(text))\n            record['pri'] = m['pri'].to_i\n            idx = m.end(0)\n          else\n            yield(nil, nil)\n            return\n          end\n        end\n\n        i = idx - 1\n        sq = false\n        @space_count_rfc5424.times {\n          while text[i + 1] == SPLIT_CHAR\n            sq = true\n            i += 1\n          end\n\n          i = text.index(SPLIT_CHAR, i + 1)\n        }\n\n        time_str = sq ? text.slice(idx, i - idx).squeeze(SPLIT_CHAR) : text.slice(idx, i - idx)\n        time = @mutex.synchronize do\n          begin\n            @time_parser_rfc5424.parse(time_str)\n          rescue Fluent::TimeParser::TimeParseError => e\n            log.trace(e)\n            @time_parser_rfc5424_without_subseconds.parse(time_str)\n          end\n        end\n\n        if @keep_time_key\n          record['time'] = time_str\n        end\n        parse_plain(@regexp5424, time, text, i + 1, record, RFC5424_CAPTURES, &block)\n      end\n\n      # @param time [EventTime]\n      # @param idx [Integer] note: this argument is needed to avoid string creation\n      # @param record [Hash]\n      # @param capture_list [Array] for performance\n      def parse_plain(re, time, text, idx, record, capture_list, &block)\n        m = re.match(text, idx)\n        if m.nil?\n          yield nil, nil\n          return\n        end\n\n        capture_list.each { |name|\n          if value = (m[name] rescue nil)\n            case name\n            when \"message\"\n              value.chomp!\n              record[name] = value\n            else\n              record[name] = value\n            end\n          end\n        }\n\n        if @estimate_current_event\n          time ||= Fluent::EventTime.now\n        end\n\n        yield time, record\n      end\n\n      def parse_rfc3164(text, &block)\n        pri = nil\n        cursor = 0\n        if @with_priority\n          if text.start_with?('<'.freeze)\n            i = text.index('>'.freeze, 1)\n            if i < 2\n              yield nil, nil\n              return\n            end\n            pri = text.slice(1, i - 1).to_i\n            cursor = i + 1\n          else\n            yield nil, nil\n            return\n          end\n        end\n\n        if @skip_space_count_rfc3164\n          # header part\n          time_size = 15 # skip Mmm dd hh:mm:ss\n          time_end = text[cursor + time_size]\n          if time_end == SPLIT_CHAR\n            time_str = text.slice(cursor, time_size)\n            cursor += 16 # time + ' '\n          elsif time_end == '.'.freeze\n            # support subsecond time\n            i = text.index(SPLIT_CHAR, time_size)\n            time_str = text.slice(cursor, i - cursor)\n            cursor = i + 1\n          else\n            yield nil, nil\n            return\n          end\n        else\n          i = cursor - 1\n          sq = false\n          @space_count_rfc3164.times do\n            while text[i + 1] == SPLIT_CHAR\n              sq = true\n              i += 1\n            end\n            i = text.index(SPLIT_CHAR, i + 1)\n          end\n\n          time_str = sq ? text.slice(idx, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)\n          cursor = i + 1\n        end\n\n        i = text.index(SPLIT_CHAR, cursor)\n        if i.nil?\n          yield nil, nil\n          return\n        end\n        host_size = i - cursor\n        host = text.slice(cursor, host_size)\n        cursor += host_size + 1\n\n        record = {'host' => host}\n        record['pri'] = pri if pri\n\n        i = text.index(SPLIT_CHAR, cursor)\n\n        # message part\n        msg = if i.nil?  # for 'only non-space content case'\n                text.slice(cursor, text.bytesize)\n              else\n                if text[i - 1] == ':'.freeze\n                  if text[i - 2] == ']'.freeze\n                    left_braket_pos = text.index('['.freeze, cursor)\n                    record['ident'] = text.slice(cursor, left_braket_pos - cursor)\n                    record['pid'] = text.slice(left_braket_pos + 1, i - left_braket_pos - 3) # remove '[' / ']:'\n                  else\n                    record['ident'] = text.slice(cursor, i - cursor - 1)\n                  end\n                  text.slice(i + 1, text.bytesize)\n                else\n                  if @support_colonless_ident\n                    if text[i - 1] == ']'.freeze\n                      left_braket_pos = text.index('['.freeze, cursor)\n                      record['ident'] = text.slice(cursor, left_braket_pos - cursor)\n                      record['pid'] = text.slice(left_braket_pos + 1, i - left_braket_pos - 2) # remove '[' / ']'\n                    else\n                      record['ident'] = text.slice(cursor, i - cursor)\n                    end\n                    text.slice(i + 1, text.bytesize)\n                  else\n                    text.slice(cursor, text.bytesize)\n                  end\n                end\n              end\n        msg.chomp!\n        record['message'] = msg\n\n        time = @time_parser_rfc3164.parse(time_str)\n        record['time'] = time_str if @keep_time_key\n\n        yield time, record\n      end\n\n      NILVALUE = '-'.freeze\n\n      def parse_rfc5424(text, &block)\n        pri = nil\n        cursor = 0\n        if @with_priority\n          if text.start_with?('<'.freeze)\n            i = text.index('>'.freeze, 1)\n            if i < 2\n              yield nil, nil\n              return\n            end\n            pri = text.slice(1, i - 1).to_i\n            i = text.index(SPLIT_CHAR, i)\n            cursor = i + 1\n          else\n            yield nil, nil\n            return\n          end\n        end\n\n        # timestamp part\n        if @skip_space_count_rfc5424\n          i = text.index(SPLIT_CHAR, cursor)\n          time_str = text.slice(cursor, i - cursor)\n          cursor = i + 1\n        else\n          i = cursor - 1\n          sq = false\n          @space_count_rfc5424.times do\n            while text[i + 1] == SPLIT_CHAR\n              sq = true\n              i += 1\n            end\n            i = text.index(SPLIT_CHAR, i + 1)\n          end\n\n          time_str = sq ? text.slice(idx, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)\n          cursor = i + 1\n        end\n\n        # Repeat same code for the performance\n\n        # host part\n        i = text.index(SPLIT_CHAR, cursor)\n        unless i\n          yield nil, nil\n          return\n        end\n        slice_size = i - cursor\n        host = text.slice(cursor, slice_size)\n        cursor += slice_size + 1\n\n        # ident part\n        i = text.index(SPLIT_CHAR, cursor)\n        unless i\n          yield nil, nil\n          return\n        end\n        slice_size = i - cursor\n        ident = text.slice(cursor, slice_size)\n        cursor += slice_size + 1\n\n        # pid part\n        i = text.index(SPLIT_CHAR, cursor)\n        unless i\n          yield nil, nil\n          return\n        end\n        slice_size = i - cursor\n        pid = text.slice(cursor, slice_size)\n        cursor += slice_size + 1\n\n        # msgid part\n        i = text.index(SPLIT_CHAR, cursor)\n        unless i\n          yield nil, nil\n          return\n        end\n        slice_size = i - cursor\n        msgid = text.slice(cursor, slice_size)\n        cursor += slice_size + 1\n\n        record = {'host' => host, 'ident' => ident, 'pid' => pid, 'msgid' => msgid}\n        record['pri'] = pri if pri\n\n        # extradata part\n        ed_start = text[cursor]\n        if ed_start == NILVALUE\n          record['extradata'] = NILVALUE\n          cursor += 1\n        else\n          start = cursor\n          i = text.index('] '.freeze, cursor)\n          extradata = if i\n                        diff = i + 1 - start # calculate ']' position\n                        cursor += diff\n                        text.slice(start, diff)\n                      else  # No message part case\n                        cursor = text.bytesize\n                        text.slice(start, cursor)\n                      end\n          extradata.tr!(\"\\\\\".freeze, ''.freeze)\n          record['extradata'] = extradata\n        end\n\n        # message part\n        if cursor != text.bytesize\n          msg = text.slice(cursor + 1, text.bytesize)\n          msg.chomp!\n          record['message'] = msg\n        end\n\n        time = begin\n                 @time_parser_rfc5424.parse(time_str)\n               rescue Fluent::TimeParser::TimeParseError\n                 @time_parser_rfc5424_without_subseconds.parse(time_str)\n               end\n        record['time'] = time_str if @keep_time_key\n\n        yield time, record\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/parser_tsv.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/parser'\n\nmodule Fluent\n  module Plugin\n    class TSVParser < Parser\n      Plugin.register_parser('tsv', self)\n\n      desc 'Names of fields included in each lines'\n      config_param :keys, :array, value_type: :string\n      desc 'The delimiter character (or string) of TSV values'\n      config_param :delimiter, :string, default: \"\\t\"\n\n      def configure(conf)\n        super\n        @key_num = @keys.length\n      end\n\n      def parse(text)\n        values = text.split(@delimiter, @key_num)\n        r = Hash[@keys.zip(values)]\n        time, record = convert_values(parse_time(r), r)\n        yield time, record\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/sd_file.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'cool.io'\n\nrequire 'fluent/plugin_helper'\nrequire 'fluent/plugin/service_discovery'\n\nmodule Fluent\n  module Plugin\n    class FileServiceDiscovery < ServiceDiscovery\n      include PluginHelper::Mixin\n\n      Plugin.register_sd('file', self)\n\n      DEFAULT_FILE_TYPE = :yaml\n      DEFAULT_WEIGHT = 60\n      DEFAULT_SD_FILE_PATH = ENV['DEFAULT_SD_FILE_PATH'] || '/etc/fluent/sd.yaml'\n\n      helpers :event_loop\n\n      config_param :path, :string, default: DEFAULT_SD_FILE_PATH\n      config_param :conf_encoding, :string, default: 'utf-8'\n\n      def initialize\n        super\n\n        @file_type = nil\n      end\n\n      def configure(conf)\n        super\n\n        unless File.exist?(@path)\n          raise Fluent::ConfigError, \"sd_file: path=#{@path} not found\"\n        end\n\n        @file_type = File.basename(@path).split('.', 2).last.to_sym\n        unless %i[yaml yml json].include?(@file_type)\n          @file_type = DEFAULT_FILE_TYPE\n        end\n\n        @services = fetch_server_info\n      end\n\n      def start(queue)\n        watcher = StatWatcher.new(@path, @log) do |_prev, _cur|\n          refresh_file(queue)\n        end\n        event_loop_attach(watcher)\n\n        super()\n      end\n\n      private\n\n      def parser\n        @parser ||=\n          case @file_type\n          when :yaml, :yml\n            require 'yaml'\n            -> (v) { YAML.safe_load(v).map }\n          when :json\n            require 'json'\n            -> (v) { JSON.parse(v) }\n          end\n      end\n\n      def refresh_file(queue)\n        s =\n          begin\n            fetch_server_info\n          rescue => e\n            @log.error(\"sd_file: #{e}\")\n            return\n          end\n\n        if s.nil?\n          # if any error occurs, skip this turn\n          return\n        end\n\n        diff = []\n        join = s - @services\n        # Need service_in first to guarantee that server exist at least one all time.\n        join.each do |j|\n          diff << ServiceDiscovery.service_in_msg(j)\n        end\n\n        drain = @services - s\n        drain.each do |d|\n          diff << ServiceDiscovery.service_out_msg(d)\n        end\n\n        @services = s\n\n        diff.each do |a|\n          queue.push(a)\n        end\n      end\n\n      def fetch_server_info\n        config_data =\n          begin\n            File.open(@path, \"r:#{@conf_encoding}:utf-8\", &:read)\n          rescue => e\n            raise Fluent::ConfigError, \"sd_file: path=#{@path} couldn't open #{e}\"\n          end\n\n        parser.call(config_data).map do |s|\n          Service.new(\n            :file,\n            s.fetch('host'),\n            s.fetch('port'),\n            s['name'],\n            s.fetch('weight', DEFAULT_WEIGHT),\n            s['standby'],\n            s['username'],\n            s['password'],\n            s['shared_key'],\n          )\n        end\n      rescue KeyError => e\n        raise Fluent::ConfigError, \"#{e}. Service must have `host` and `port`\"\n      end\n\n      class StatWatcher < Coolio::StatWatcher\n        def initialize(path, log, &callback)\n          @path = path\n          @log = log\n          @callback = callback\n          super(@path)\n        end\n\n        def on_change(prev_stat, cur_stat)\n          @callback.call(prev_stat, cur_stat)\n        rescue => e\n          @log.error(e)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/sd_srv.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'resolv'\n\nrequire 'fluent/plugin_helper'\nrequire 'fluent/plugin/service_discovery'\n\nmodule Fluent\n  module Plugin\n    class SrvServiceDiscovery < ServiceDiscovery\n      include PluginHelper::Mixin\n\n      Plugin.register_sd('srv', self)\n\n      helpers :timer\n\n      desc 'Service without underscore in RFC2782'\n      config_param :service, :string\n      desc 'Proto without underscore in RFC2782'\n      config_param :proto, :string, default: 'tcp'\n      desc 'Name in RFC2782'\n      config_param :hostname, :string\n      desc 'hostname of DNS server to request the SRV record'\n      config_param :dns_server_host, :string, default: nil\n      desc 'interval of requesting to DNS server'\n      config_param :interval, :integer, default: 60\n      desc \"resolve hostname to IP addr of SRV's Target\"\n      config_param :dns_lookup, :bool, default: true\n      desc 'The shared key per server'\n      config_param :shared_key, :string, default: nil, secret: true\n      desc 'The username for authentication'\n      config_param :username, :string, default: ''\n      desc 'The password for authentication'\n      config_param :password, :string, default: '', secret: true\n\n      def initialize\n        super\n        @target = nil\n      end\n\n      def configure(conf)\n        super\n\n        @target = \"_#{@service}._#{@proto}.#{@hostname}\"\n        @dns_resolve =\n          if @dns_server_host.nil?\n            Resolv::DNS.new\n          elsif @dns_server_host.include?(':') # e.g. 127.0.0.1:8600\n            host, port = @dns_server_host.split(':', 2)\n            Resolv::DNS.new(nameserver_port: [[host, port.to_i]])\n          else\n            Resolv::DNS.new(nameserver: @dns_server_host)\n          end\n\n        @services = fetch_srv_record\n      end\n\n      def start(queue)\n        timer_execute(:\"sd_srv_record_#{@target}\", @interval) do\n          refresh_srv_records(queue)\n        end\n\n        super()\n      end\n\n      private\n\n      def refresh_srv_records(queue)\n        s = begin\n              fetch_srv_record\n            rescue => e\n              @log.error(\"sd_srv: #{e}\")\n              return\n            end\n\n        if s.nil? || s.empty?\n          return\n        end\n\n        diff = []\n        join = s - @services\n        # Need service_in first to guarantee that server exist at least one all time.\n        join.each do |j|\n          diff << ServiceDiscovery.service_in_msg(j)\n        end\n\n        drain = @services - s\n        drain.each do |d|\n          diff << ServiceDiscovery.service_out_msg(d)\n        end\n\n        @services = s\n\n        diff.each do |a|\n          queue.push(a)\n        end\n      end\n\n      def fetch_srv_record\n        adders = @dns_resolve.getresources(@target, Resolv::DNS::Resource::IN::SRV)\n\n        services = []\n\n        adders.each do |addr|\n          host = @dns_lookup ? dns_lookup!(addr.target) : addr.target\n          services << [\n            addr.priority,\n            Service.new(:srv, host.to_s, addr.port.to_i, addr.target.to_s, addr.weight, false, @username, @password, @shared_key)\n          ]\n        end\n\n        services.sort_by(&:first).flat_map { |s| s[1] }\n      end\n\n      def dns_lookup!(host)\n        # may need to cache the result\n        @dns_resolve.getaddress(host) # get first result for now\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/sd_static.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/service_discovery'\n\nmodule Fluent\n  module Plugin\n    class StaticServiceDiscovery < ServiceDiscovery\n      Plugin.register_sd('static', self)\n\n      LISTEN_PORT = 24224\n\n      config_section :service, param_name: :service_configs do\n        desc 'The IP address or host name of the server.'\n        config_param :host, :string\n        desc 'The name of the server. Used for logging and certificate verification in TLS transport (when host is address).'\n        config_param :name, :string, default: nil\n        desc 'The port number of the host.'\n        config_param :port, :integer, default: LISTEN_PORT\n        desc 'The shared key per server.'\n        config_param :shared_key, :string, default: nil, secret: true\n        desc 'The username for authentication.'\n        config_param :username, :string, default: ''\n        desc 'The password for authentication.'\n        config_param :password, :string, default: '', secret: true\n        desc 'Marks a node as the standby node for an Active-Standby model between Fluentd nodes.'\n        config_param :standby, :bool, default: false\n        desc 'The load balancing weight.'\n        config_param :weight, :integer, default: 60\n      end\n\n      def configure(conf)\n        super\n\n        @services = @service_configs.map do |s|\n          ServiceDiscovery::Service.new(:static, s.host, s.port, s.name, s.weight, s.standby, s.username, s.password, s.shared_key)\n        end\n      end\n\n      def start(queue = nil)\n        super()\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/service_discovery.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/base'\n\nrequire 'fluent/log'\nrequire 'fluent/unique_id'\nrequire 'fluent/plugin_id'\n\nmodule Fluent\n  module Plugin\n    class ServiceDiscovery < Base\n      include PluginId\n      include PluginLoggerMixin\n      include UniqueId::Mixin\n\n      configured_in :service_discovery\n\n      attr_reader :services\n\n      Service = Struct.new(:plugin_name, :host, :port, :name, :weight, :standby, :username, :password, :shared_key) do\n        def discovery_id\n          @discovery_id ||= Base64.encode64(to_h.to_s)\n        end\n      end\n\n      SERVICE_IN = :service_in\n      SERVICE_OUT = :service_out\n      DiscoveryMessage = Struct.new(:type, :service)\n\n      class << self\n        def service_in_msg(service)\n          DiscoveryMessage.new(SERVICE_IN, service)\n        end\n\n        def service_out_msg(service)\n          DiscoveryMessage.new(SERVICE_OUT, service)\n        end\n      end\n\n      def initialize\n        @services = []\n\n        super\n      end\n\n      def start(queue = nil)\n        super()\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/socket_util.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/socket_util'\n\nmodule Fluent\n  # obsolete\n  SocketUtil = Fluent::Compat::SocketUtil\nend\n"
  },
  {
    "path": "lib/fluent/plugin/storage.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/base'\nrequire 'fluent/plugin/owned_by_mixin'\n\nmodule Fluent\n  module Plugin\n    class Storage < Base\n      include OwnedByMixin\n\n      DEFAULT_TYPE = 'local'\n\n      configured_in :storage\n\n      config_param :persistent,        :bool, default: false # load/save with all operations\n      config_param :autosave,          :bool, default: true\n      config_param :autosave_interval, :time, default: 10\n      config_param :save_at_shutdown,  :bool, default: true\n\n      def self.validate_key(key)\n        raise ArgumentError, \"key must be a string (or symbol for to_s)\" unless key.is_a?(String) || key.is_a?(Symbol)\n        key.to_s\n      end\n\n      attr_accessor :log\n\n      def persistent_always?\n        false\n      end\n\n      def synchronized?\n        false\n      end\n\n      def implementation\n        self\n      end\n\n      def load\n        # load storage data from any data source, or initialize storage internally\n      end\n\n      def save\n        # save internal data store into data source (to be loaded)\n      end\n\n      def get(key)\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def fetch(key, defval)\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def put(key, value)\n        # return value\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def delete(key)\n        # return deleted value\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n\n      def update(key, &block) # transactional get-and-update\n        raise NotImplementedError, \"Implement this method in child class\"\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/storage_local.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/env'\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/storage'\n\nrequire 'fileutils'\nrequire 'json'\n\nmodule Fluent\n  module Plugin\n    class LocalStorage < Storage\n      Fluent::Plugin.register_storage('local', self)\n\n      config_param :path, :string, default: nil\n      config_param :mode, default: Fluent::DEFAULT_FILE_PERMISSION do |v|\n        v.to_i(8)\n      end\n      config_param :dir_mode, default: Fluent::DEFAULT_DIR_PERMISSION do |v|\n        v.to_i(8)\n      end\n      config_param :pretty_print, :bool, default: false\n\n      attr_reader :store # for test\n\n      def initialize\n        super\n        @store = {}\n        @multi_workers_available = nil\n      end\n\n      def configure(conf)\n        super\n\n        @on_memory = false\n        if @path\n          if File.exist?(@path) && File.file?(@path)\n            @multi_workers_available = false\n          elsif File.exist?(@path) && File.directory?(@path)\n            @path = File.join(@path, \"worker#{fluentd_worker_id}\", \"storage.json\")\n            @multi_workers_available = true\n          else # path file/directory doesn't exist\n            if @path.end_with?('.json') # file\n              @multi_workers_available = false\n            else # directory\n              @path = File.join(@path, \"worker#{fluentd_worker_id}\", \"storage.json\")\n              @multi_workers_available = true\n            end\n          end\n        elsif root_dir = owner.plugin_root_dir\n          basename = (conf.arg && !conf.arg.empty?) ? \"storage.#{conf.arg}.json\" : \"storage.json\"\n          @path = File.join(root_dir, basename)\n          @multi_workers_available = true\n        else\n          if @persistent\n            raise Fluent::ConfigError, \"Plugin @id or path for <storage> required when 'persistent' is true\"\n          else\n            if @autosave\n              log.warn \"both of Plugin @id and path for <storage> are not specified. Using on-memory store.\"\n            else\n              log.info \"both of Plugin @id and path for <storage> are not specified. Using on-memory store.\"\n            end\n            @on_memory = true\n            @multi_workers_available = true\n          end\n        end\n\n        if !@on_memory\n          dir = File.dirname(@path)\n          FileUtils.mkdir_p(dir, mode: @dir_mode) unless Dir.exist?(dir)\n          if File.exist?(@path)\n            raise Fluent::ConfigError, \"Plugin storage path '#{@path}' is not readable/writable\" unless File.readable?(@path) && File.writable?(@path)\n            begin\n              data = File.open(@path, 'r:utf-8') { |io| io.read }\n              if data.empty?\n                log.warn \"detect empty plugin storage file during startup. Ignored: #{@path}\"\n                return\n              end\n              data = JSON.parse(data)\n              raise Fluent::ConfigError, \"Invalid contents (not object) in plugin storage file: '#{@path}'\" unless data.is_a?(Hash)\n            rescue => e\n              log.error \"failed to read data from plugin storage file\", path: @path, error: e\n              raise Fluent::ConfigError, \"Unexpected error: failed to read data from plugin storage file: '#{@path}'\"\n            end\n          else\n            raise Fluent::ConfigError, \"Directory is not writable for plugin storage file '#{@path}'\" unless File.stat(dir).writable?\n          end\n        end\n      end\n\n      def multi_workers_ready?\n        unless @multi_workers_available\n          log.error \"local plugin storage with multi workers should be configured to use directory 'path', or system root_dir and plugin id\"\n        end\n        @multi_workers_available\n      end\n\n      def load\n        return if @on_memory\n        return unless File.exist?(@path)\n        begin\n          json_string = File.open(@path, 'r:utf-8'){ |io| io.read }\n          json = JSON.parse(json_string)\n          unless json.is_a?(Hash)\n            log.error \"broken content for plugin storage (Hash required: ignored)\", type: json.class\n            log.debug \"broken content\", content: json_string\n            return\n          end\n          @store = json\n        rescue => e\n          log.error \"failed to load data for plugin storage from file\", path: @path, error: e\n        end\n      end\n\n      def save\n        return if @on_memory\n        tmp_path = @path + '.tmp.' + Fluent::UniqueId.hex(Fluent::UniqueId.generate)\n        begin\n          if @pretty_print\n            json_string = JSON.pretty_generate(@store)\n          else\n            json_string = JSON.generate(@store)\n          end\n          File.open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }\n          File.rename(tmp_path, @path)\n        rescue => e\n          log.error \"failed to save data for plugin storage to file\", path: @path, tmp: tmp_path, error: e\n        end\n      end\n\n      def get(key)\n        @store[key.to_s]\n      end\n\n      def fetch(key, defval)\n        @store.fetch(key.to_s, defval)\n      end\n\n      def put(key, value)\n        @store[key.to_s] = value\n      end\n\n      def delete(key)\n        @store.delete(key.to_s)\n      end\n\n      def update(key, &block)\n        @store[key.to_s] = block.call(@store[key.to_s])\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin/string_util.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/string_util'\n\nmodule Fluent\n  # obsolete\n  StringUtil = Fluent::Compat::StringUtil\nend\n"
  },
  {
    "path": "lib/fluent/plugin.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/registry'\nrequire 'fluent/config/error'\n\nmodule Fluent\n  module Plugin\n    SEARCH_PATHS = []\n\n    # plugins for fluentd:         fluent/plugin/type_NAME.rb\n    # plugins for fluentd plugins: fluent/plugin/type/NAME.rb\n    #   ex: storage, buffer chunk, ...\n\n    # first class plugins (instantiated by Engine)\n    INPUT_REGISTRY     = Registry.new(:input,     'fluent/plugin/in_',         dir_search_prefix: 'in_')\n    OUTPUT_REGISTRY    = Registry.new(:output,    'fluent/plugin/out_',        dir_search_prefix: 'out_')\n    FILTER_REGISTRY    = Registry.new(:filter,    'fluent/plugin/filter_',     dir_search_prefix: 'filter_')\n\n    # feature plugin: second class plugins (instantiated by Plugins or Helpers)\n    BUFFER_REGISTRY    = Registry.new(:buffer,    'fluent/plugin/buf_',        dir_search_prefix: 'buf_')\n    PARSER_REGISTRY    = Registry.new(:parser,    'fluent/plugin/parser_',     dir_search_prefix: 'parser_')\n    FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')\n    STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')\n    SD_REGISTRY        = Registry.new(:sd,        'fluent/plugin/sd_',         dir_search_prefix: 'sd_')\n    METRICS_REGISTRY   = Registry.new(:metrics,   'fluent/plugin/metrics_',    dir_search_prefix: 'metrics_')\n\n    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY, METRICS_REGISTRY]\n\n    def self.register_input(type, klass)\n      register_impl('input', INPUT_REGISTRY, type, klass)\n    end\n\n    def self.register_output(type, klass)\n      register_impl('output', OUTPUT_REGISTRY, type, klass)\n    end\n\n    def self.register_filter(type, klass)\n      register_impl('filter', FILTER_REGISTRY, type, klass)\n    end\n\n    def self.register_buffer(type, klass)\n      register_impl('buffer', BUFFER_REGISTRY, type, klass)\n    end\n\n    def self.register_sd(type, klass)\n      register_impl('sd', SD_REGISTRY, type, klass)\n    end\n\n    def self.register_metrics(type, klass)\n      register_impl('metrics', METRICS_REGISTRY, type, klass)\n    end\n\n    def self.register_parser(type, klass_or_proc)\n      if klass_or_proc.is_a?(Regexp)\n        # This usage is not recommended for new API\n        require 'fluent/parser'\n        register_impl('parser', PARSER_REGISTRY, type, Proc.new { Fluent::TextParser::RegexpParser.new(klass_or_proc) })\n      else\n        register_impl('parser', PARSER_REGISTRY, type, klass_or_proc)\n      end\n    end\n\n    def self.register_formatter(type, klass_or_proc)\n      if klass_or_proc.respond_to?(:call) && klass_or_proc.arity == 3 # Proc.new { |tag, time, record| }\n        # This usage is not recommended for new API\n        require 'fluent/formatter'\n        register_impl('formatter', FORMATTER_REGISTRY, type, Proc.new { Fluent::TextFormatter::ProcWrappedFormatter.new(klass_or_proc) })\n      else\n        register_impl('formatter', FORMATTER_REGISTRY, type, klass_or_proc)\n      end\n    end\n\n    def self.register_storage(type, klass)\n      register_impl('storage', STORAGE_REGISTRY, type, klass)\n    end\n\n    def self.lookup_type_from_class(klass_or_its_name)\n      klass = if klass_or_its_name.is_a? Class\n                klass_or_its_name\n              elsif klass_or_its_name.is_a? String\n                eval(klass_or_its_name) # const_get can't handle qualified klass name (ex: A::B)\n              else\n                raise ArgumentError, \"invalid argument type #{klass_or_its_name.class}: #{klass_or_its_name}\"\n              end\n      REGISTRIES.reduce(nil){|a, r| a || r.reverse_lookup(klass) }\n    end\n\n    def self.add_plugin_dir(dir)\n      REGISTRIES.each do |r|\n        r.paths.push(dir)\n      end\n      nil\n    end\n\n    def self.new_input(type)\n      new_impl('input', INPUT_REGISTRY, type)\n    end\n\n    def self.new_output(type)\n      new_impl('output', OUTPUT_REGISTRY, type)\n    end\n\n    def self.new_filter(type)\n      new_impl('filter', FILTER_REGISTRY, type)\n    end\n\n    def self.new_buffer(type, parent: nil)\n      new_impl('buffer', BUFFER_REGISTRY, type, parent)\n    end\n\n    def self.new_sd(type, parent: nil)\n      new_impl('sd', SD_REGISTRY, type, parent)\n    end\n\n    def self.new_metrics(type, parent: nil)\n      new_impl('metrics', METRICS_REGISTRY, type, parent)\n    end\n\n    class << self\n      # This should be defined for fluent-plugin-config-formatter type arguments.\n      alias_method :new_service_discovery, :new_sd\n    end\n\n    def self.new_parser(type, parent: nil)\n      if type[0] == '/' && type[-1] == '/'\n        # This usage is not recommended for new API... create RegexpParser directly\n        require 'fluent/parser'\n        impl = Fluent::TextParser.lookup(type)\n        impl.extend FeatureAvailabilityChecker\n        impl\n      else\n        new_impl('parser', PARSER_REGISTRY, type, parent)\n      end\n    end\n\n    def self.new_formatter(type, parent: nil)\n      new_impl('formatter', FORMATTER_REGISTRY, type, parent)\n    end\n\n    def self.new_storage(type, parent: nil)\n      new_impl('storage', STORAGE_REGISTRY, type, parent)\n    end\n\n    def self.register_impl(kind, registry, type, value)\n      if !value.is_a?(Class) && !value.respond_to?(:call)\n        raise Fluent::ConfigError, \"Invalid implementation as #{kind} plugin: '#{type}'. It must be a Class, or callable.\"\n      end\n      registry.register(type, value)\n      $log.trace \"registered #{kind} plugin '#{type}'\" if defined?($log)\n      nil\n    end\n\n    def self.new_impl(kind, registry, type, parent=nil)\n      # \"'type' not found\" is handled by registry\n      obj = registry.lookup(type)\n      impl = case\n             when obj.is_a?(Class)\n               obj.new\n             when obj.respond_to?(:call) && obj.arity == 0\n               obj.call\n             else\n               raise Fluent::ConfigError, \"#{kind} plugin '#{type}' is not a Class nor callable (without arguments).\"\n             end\n      if parent && impl.respond_to?(:owner=)\n        impl.owner = parent\n      end\n      impl.extend FeatureAvailabilityChecker\n      impl\n    end\n\n    module FeatureAvailabilityChecker\n      def configure(conf)\n        super\n\n        # extend plugin instance by this module\n        # to run this check after all #configure methods of plugins and plugin helpers\n        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)\n                    owner.system_config\n                  elsif self.respond_to?(:system_config)\n                    self.system_config\n                  else\n                    nil\n                  end\n\n        if sysconf && sysconf.workers > 1 && !self.multi_workers_ready?\n          type = Fluent::Plugin.lookup_type_from_class(self.class)\n          raise Fluent::ConfigError, \"Plugin '#{type}' does not support multi workers configuration (#{self.class})\"\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/cert_option.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'openssl'\nrequire 'socket'\n\nrequire 'fluent/tls'\n\n# this module is only for Socket/Server/HttpServer plugin helpers\nmodule Fluent\n  module PluginHelper\n    module CertOption\n      def cert_option_create_context(version, insecure, ciphers, conf)\n        cert, key, extra = cert_option_server_validate!(conf)\n\n        ctx = OpenSSL::SSL::SSLContext.new\n        # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS\n        # https://bugs.ruby-lang.org/issues/9424\n        ctx.set_params({}) unless insecure\n\n        if conf.client_cert_auth\n          ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT\n        else\n          ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE\n        end\n\n        if conf.ensure_fips\n          unless OpenSSL.fips_mode\n            raise Fluent::ConfigError, \"Cannot enable FIPS compliant mode. OpenSSL FIPS configuration is disabled\"\n          end\n        end\n\n        ctx.ca_file = conf.ca_path\n        ctx.cert = cert\n        ctx.key = key\n        if extra && !extra.empty?\n          ctx.extra_chain_cert = extra\n        end\n        if conf.cert_verifier\n          sandbox = Class.new\n          ctx.verify_callback = if File.exist?(conf.cert_verifier)\n                                  verifier = File.read(conf.cert_verifier)\n                                  sandbox.instance_eval(verifier, File.basename(conf.cert_verifier))\n                                else\n                                  sandbox.instance_eval(conf.cert_verifier)\n                                end\n        end\n\n        Fluent::TLS.set_version_to_context(ctx, version, conf.min_version, conf.max_version)\n        ctx.ciphers = ciphers unless insecure\n\n        ctx\n      end\n\n      def cert_option_server_validate!(conf)\n        case\n        when conf.cert_path\n          raise Fluent::ConfigError, \"private_key_path is required when cert_path is specified\" unless conf.private_key_path\n          log.warn \"For security reason, setting private_key_passphrase is recommended when cert_path is specified\" unless conf.private_key_passphrase\n          cert_option_load(conf.cert_path, conf.private_key_path, conf.private_key_passphrase)\n\n        when conf.ca_cert_path\n          raise Fluent::ConfigError, \"ca_private_key_path is required when ca_cert_path is specified\" unless conf.ca_private_key_path\n          log.warn \"For security reason, setting ca_private_key_passphrase is recommended when ca_cert_path is specified\" unless conf.ca_private_key_passphrase\n          generate_opts = cert_option_cert_generation_opts_from_conf(conf)\n          cert_option_generate_server_pair_by_ca(\n            conf.ca_cert_path,\n            conf.ca_private_key_path,\n            conf.ca_private_key_passphrase,\n            generate_opts\n          )\n\n        when conf.insecure\n          log.warn \"insecure TLS communication server is configured (using 'insecure' mode)\"\n          generate_opts = cert_option_cert_generation_opts_from_conf(conf)\n          cert_option_generate_server_pair_self_signed(generate_opts)\n\n        else\n          raise Fluent::ConfigError, \"no valid cert options configured. specify either 'cert_path', 'ca_cert_path' or 'insecure'\"\n        end\n      end\n\n      def cert_option_load(cert_path, private_key_path, private_key_passphrase)\n        key = OpenSSL::PKey::read(File.read(private_key_path), private_key_passphrase)\n        certs = cert_option_certificates_from_file(cert_path)\n        cert = certs.shift\n        return cert, key, certs\n      end\n\n      def cert_option_cert_generation_opts_from_conf(conf)\n        {\n          private_key_length: conf.generate_private_key_length,\n          country: conf.generate_cert_country,\n          state: conf.generate_cert_state,\n          locality: conf.generate_cert_locality,\n          common_name: conf.generate_cert_common_name || ::Socket.gethostname,\n          expiration: conf.generate_cert_expiration,\n          digest: conf.generate_cert_digest,\n        }\n      end\n\n      def cert_option_generate_pair(opts, issuer = nil)\n        key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])\n\n        subject = OpenSSL::X509::Name.new\n        subject.add_entry('C', opts[:country])\n        subject.add_entry('ST', opts[:state])\n        subject.add_entry('L', opts[:locality])\n        subject.add_entry('CN', opts[:common_name])\n\n        issuer ||= subject\n\n        cert = OpenSSL::X509::Certificate.new\n        cert.not_before = Time.at(0)\n        cert.not_after = Time.now + opts[:expiration]\n        cert.public_key = key\n        cert.version = 2\n        cert.serial = rand(2**(8*10))\n        cert.issuer = issuer\n        cert.subject  = subject\n\n        return cert, key\n      end\n\n      def cert_option_add_extensions(cert, extensions)\n        ef = OpenSSL::X509::ExtensionFactory.new\n        extensions.each do |ext|\n          oid, value = ext\n          cert.add_extension ef.create_extension(oid, value)\n        end\n      end\n\n      def cert_option_generate_ca_pair_self_signed(generate_opts)\n        cert, key = cert_option_generate_pair(generate_opts)\n\n        cert_option_add_extensions(cert, [\n          ['basicConstraints', 'CA:TRUE']\n        ])\n\n        cert.sign(key, generate_opts[:digest].to_s)\n        return cert, key\n      end\n\n      def cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, generate_opts)\n        ca_key = OpenSSL::PKey::read(File.read(ca_key_path), ca_key_passphrase)\n        ca_cert = OpenSSL::X509::Certificate.new(File.read(ca_cert_path))\n        cert, key = cert_option_generate_pair(generate_opts, ca_cert.subject)\n        raise \"BUG: certificate digest algorithm not set\" unless generate_opts[:digest]\n\n        cert_option_add_extensions(cert, [\n          ['basicConstraints', 'CA:FALSE'],\n          ['nsCertType', 'server'],\n          ['keyUsage', 'digitalSignature,keyEncipherment'],\n          ['extendedKeyUsage', 'serverAuth']\n        ])\n\n        cert.sign(ca_key, generate_opts[:digest].to_s)\n        return cert, key, nil\n      end\n\n      def cert_option_generate_server_pair_self_signed(generate_opts)\n        cert, key = cert_option_generate_pair(generate_opts)\n        raise \"BUG: certificate digest algorithm not set\" unless generate_opts[:digest]\n\n        cert_option_add_extensions(cert, [\n          ['basicConstraints', 'CA:FALSE'],\n          ['nsCertType', 'server']\n        ])\n\n        cert.sign(key, generate_opts[:digest].to_s)\n        return cert, key, nil\n      end\n\n      def cert_option_certificates_from_file(path)\n        data = File.read(path)\n        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\\r?\\n(?:[^-]*\\r?\\n)+-+END CERTIFICATE-+\\r?\\n?', Regexp::MULTILINE)\n        list = []\n        data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }\n        if list.length == 0\n          raise Fluent::ConfigError, \"cert_path does not contain a valid certificate\"\n        end\n        list\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/child_process.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin_helper/thread'\nrequire 'fluent/plugin_helper/timer'\nrequire 'fluent/clock'\n\nrequire 'open3'\nrequire 'timeout'\n\nmodule Fluent\n  module PluginHelper\n    module ChildProcess\n      include Fluent::PluginHelper::Thread\n      include Fluent::PluginHelper::Timer\n\n      CHILD_PROCESS_LOOP_CHECK_INTERVAL = 0.2 # sec\n      CHILD_PROCESS_DEFAULT_EXIT_TIMEOUT = 10 # sec\n      CHILD_PROCESS_DEFAULT_KILL_TIMEOUT = 60 # sec\n\n      MODE_PARAMS = [:read, :write, :stderr, :read_with_stderr]\n      STDERR_OPTIONS = [:discard, :connect]\n\n      # stop     : mark callback thread as stopped\n      # shutdown : close write IO to child processes (STDIN of child processes), send TERM (KILL for Windows) to all child processes\n      # close    : send KILL to all child processes\n      # terminate: [-]\n\n      attr_reader :_child_process_processes # for tests\n\n      def child_process_running?\n        # checker for code in callback of child_process_execute\n        ::Thread.current[:_fluentd_plugin_helper_child_process_running] || false\n      end\n\n      def child_process_id\n        ::Thread.current[:_fluentd_plugin_helper_child_process_pid]\n      end\n\n      def child_process_exist?(pid)\n        pinfo = @_child_process_processes[pid]\n        return false unless pinfo\n\n        return false if pinfo.exit_status\n\n        true\n      end\n\n      # on_exit_callback = ->(status){ ... }\n      # status is an instance of Process::Status\n      # On Windows, exitstatus=0 and termsig=nil even when child process was killed.\n      def child_process_execute(\n          title, command,\n          arguments: nil, subprocess_name: nil, interval: nil, immediate: false, parallel: false,\n          mode: [:read, :write], stderr: :discard, env: {}, unsetenv: false, chdir: nil,\n          internal_encoding: 'utf-8', external_encoding: 'ascii-8bit', scrub: true, replace_string: nil,\n          wait_timeout: nil, on_exit_callback: nil,\n          &block\n      )\n        raise ArgumentError, \"BUG: title must be a symbol\" unless title.is_a? Symbol\n        raise ArgumentError, \"BUG: arguments required if subprocess name is replaced\" if subprocess_name && !arguments\n\n        mode ||= []\n        mode = [] unless block\n        raise ArgumentError, \"BUG: invalid mode specification\" unless mode.all?{|m| MODE_PARAMS.include?(m) }\n        raise ArgumentError, \"BUG: read_with_stderr is exclusive with :read and :stderr\" if mode.include?(:read_with_stderr) && (mode.include?(:read) || mode.include?(:stderr))\n        raise ArgumentError, \"BUG: invalid stderr handling specification\" unless STDERR_OPTIONS.include?(stderr)\n\n        raise ArgumentError, \"BUG: number of block arguments are different from size of mode\" if block && block.arity != mode.size\n\n        running = false\n        callback = ->(*args) {\n          running = true\n          begin\n            block && block.call(*args)\n          ensure\n            running = false\n          end\n        }\n\n        retval = nil\n        execute_child_process = ->(){\n          child_process_execute_once(\n            title, command, arguments,\n            subprocess_name, mode, stderr, env, unsetenv, chdir,\n            internal_encoding, external_encoding, scrub, replace_string,\n            wait_timeout, on_exit_callback,\n            &callback\n          )\n        }\n\n        if immediate || !interval\n          retval = execute_child_process.call\n        end\n\n        if interval\n          timer_execute(:child_process_execute, interval, repeat: true) do\n            if !parallel && running\n              log.warn \"previous child process is still running. skipped.\", title: title, command: command, arguments: arguments, interval: interval, parallel: parallel\n            else\n              execute_child_process.call\n            end\n          end\n        end\n\n        retval # nil if interval\n      end\n\n      def initialize\n        super\n        # plugins MAY configure this parameter\n        @_child_process_exit_timeout = CHILD_PROCESS_DEFAULT_EXIT_TIMEOUT\n        @_child_process_kill_timeout = CHILD_PROCESS_DEFAULT_KILL_TIMEOUT\n        @_child_process_mutex = Mutex.new\n        @_child_process_processes = {} # pid => ProcessInfo\n      end\n\n      def stop\n        @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|\n          process_info = @_child_process_processes[pid]\n          if process_info\n            process_info.thread[:_fluentd_plugin_helper_child_process_running] = false\n          end\n        end\n\n        super\n      end\n\n      def shutdown\n        @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|\n          process_info = @_child_process_processes[pid]\n          next if !process_info\n          process_info.writeio&.close rescue nil\n        end\n\n        super\n\n        @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|\n          process_info = @_child_process_processes[pid]\n          next if !process_info\n          child_process_kill(process_info)\n        end\n\n        exit_wait_timeout = Fluent::Clock.now + @_child_process_exit_timeout\n        while Fluent::Clock.now < exit_wait_timeout\n          process_exists = false\n          @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|\n            unless @_child_process_processes[pid].exit_status\n              process_exists = true\n              break\n            end\n          end\n          if process_exists\n            sleep CHILD_PROCESS_LOOP_CHECK_INTERVAL\n          else\n            break\n          end\n        end\n      end\n\n      def close\n        while true\n          pids = @_child_process_mutex.synchronize{ @_child_process_processes.keys }\n          break if pids.size < 1\n\n          living_process_exist = false\n          pids.each do |pid|\n            process_info = @_child_process_processes[pid]\n            next if !process_info || process_info.exit_status\n\n            living_process_exist = true\n\n            process_info.killed_at ||= Fluent::Clock.now # for irregular case (e.g., created after shutdown)\n            timeout_at = process_info.killed_at + @_child_process_kill_timeout\n            now = Fluent::Clock.now\n            next if now < timeout_at\n\n            child_process_kill(process_info, force: true)\n          end\n\n          break if living_process_exist\n\n          sleep CHILD_PROCESS_LOOP_CHECK_INTERVAL\n        end\n\n        super\n      end\n\n      def terminate\n        @_child_process_processes = {}\n\n        super\n      end\n\n      def child_process_kill(pinfo, force: false)\n        return if !pinfo\n        pinfo.killed_at = Fluent::Clock.now unless force\n\n        pid = pinfo.pid\n        begin\n          if !pinfo.exit_status && child_process_exist?(pid)\n            signal = (Fluent.windows? || force) ? :KILL : :TERM\n            Process.kill(signal, pinfo.pid)\n          end\n        rescue Errno::ECHILD, Errno::ESRCH\n          # ignore\n        end\n      end\n\n      ProcessInfo = Struct.new(\n        :title,\n        :thread, :pid,\n        :readio, :readio_in_use, :writeio, :writeio_in_use, :stderrio, :stderrio_in_use,\n        :wait_thread, :alive, :killed_at, :exit_status,\n        :on_exit_callback, :on_exit_callback_mutex,\n      )\n\n      def child_process_execute_once(\n          title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir,\n          internal_encoding, external_encoding, scrub, replace_string, wait_timeout, on_exit_callback, &block\n      )\n        spawn_args = if arguments || subprocess_name\n                       [ env, (subprocess_name ? [command, subprocess_name] : command), *(arguments || []) ]\n                     else\n                       [ env, command ]\n                     end\n        spawn_opts = {\n          unsetenv_others: unsetenv,\n        }\n        if chdir\n          spawn_opts[:chdir] = chdir\n        end\n\n        encoding_options = {}\n        if scrub\n          encoding_options[:invalid] = encoding_options[:undef] = :replace\n          if replace_string\n            encoding_options[:replace] = replace_string\n          end\n        end\n\n        log.debug \"Executing command\", title: title, spawn: spawn_args, mode: mode, stderr: stderr\n\n        readio = writeio = stderrio = wait_thread = nil\n        readio_in_use = writeio_in_use = stderrio_in_use = false\n\n        if !mode.include?(:stderr) && !mode.include?(:read_with_stderr)\n          spawn_opts[:err] = IO::NULL if stderr == :discard\n          if !mode.include?(:read) && !mode.include?(:read_with_stderr)\n            spawn_opts[:out] = IO::NULL\n          end\n          writeio, readio, wait_thread = *Open3.popen2(*spawn_args, spawn_opts)\n        elsif mode.include?(:read_with_stderr)\n          writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)\n        else\n          writeio, readio, stderrio, wait_thread = *Open3.popen3(*spawn_args, spawn_opts)\n        end\n\n        if mode.include?(:write)\n          writeio.set_encoding(external_encoding, internal_encoding, **encoding_options)\n          writeio_in_use = true\n        else\n          writeio.reopen(IO::NULL) if writeio\n        end\n        if mode.include?(:read) || mode.include?(:read_with_stderr)\n          readio.set_encoding(external_encoding, internal_encoding, **encoding_options)\n          readio_in_use = true\n        end\n        if mode.include?(:stderr)\n          stderrio.set_encoding(external_encoding, internal_encoding, **encoding_options)\n          stderrio_in_use = true\n        else\n          stderrio.reopen(IO::NULL) if stderrio && stderr == :discard\n        end\n\n        pid = wait_thread.pid # wait_thread => Process::Waiter\n\n        io_objects = []\n        mode.each do |m|\n          io_obj = case m\n                   when :read then readio\n                   when :write then writeio\n                   when :read_with_stderr then readio\n                   when :stderr then stderrio\n                   else\n                     raise \"BUG: invalid mode must be checked before here: '#{m}'\"\n                   end\n          io_objects << io_obj\n        end\n\n        m = Mutex.new\n        m.lock\n        thread = thread_create :child_process_callback do\n          m.lock # run after plugin thread get pid, thread instance and i/o\n          m.unlock\n          begin\n            @_child_process_processes[pid].alive = true\n            block.call(*io_objects) if block_given?\n            writeio.close if writeio\n          rescue EOFError => e\n            log.debug \"Process exit and I/O closed\", title: title, pid: pid, command: command, arguments: arguments\n          rescue IOError => e\n            if e.message == 'stream closed' || e.message == 'closed stream' # \"closed stream\" is of ruby 2.1\n              log.debug \"Process I/O stream closed\", title: title, pid: pid, command: command, arguments: arguments\n            else\n              log.error \"Unexpected I/O error for child process\", title: title, pid: pid, command: command, arguments: arguments, error: e\n            end\n          rescue Errno::EPIPE => e\n            log.debug \"Broken pipe, child process unexpectedly exits\", title: title, pid: pid, command: command, arguments: arguments\n          rescue => e\n            log.warn \"Unexpected error while processing I/O for child process\", title: title, pid: pid, command: command, error: e\n          end\n\n          if wait_timeout\n            if wait_thread.join(wait_timeout) # Thread#join returns nil when limit expires\n              # wait_thread successfully exits\n              @_child_process_processes[pid].exit_status = wait_thread.value\n            else\n              log.warn \"child process timed out\", title: title, pid: pid, command: command, arguments: arguments\n              child_process_kill(@_child_process_processes[pid], force: true)\n              @_child_process_processes[pid].exit_status = wait_thread.value\n            end\n          else\n            @_child_process_processes[pid].exit_status = wait_thread.value # with join\n          end\n          process_info = @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }\n\n          cb = process_info.on_exit_callback_mutex.synchronize do\n            cback = process_info.on_exit_callback\n            process_info.on_exit_callback = nil\n            cback\n          end\n          if cb\n            cb.call(process_info.exit_status) rescue nil\n          end\n          process_info.readio&.close rescue nil\n          process_info.writeio&.close rescue nil\n          process_info.stderrio&.close rescue nil\n        end\n        thread[:_fluentd_plugin_helper_child_process_running] = true\n        thread[:_fluentd_plugin_helper_child_process_pid] = pid\n        pinfo = ProcessInfo.new(\n          title, thread, pid,\n          readio, readio_in_use, writeio, writeio_in_use, stderrio, stderrio_in_use,\n          wait_thread, false, nil, nil, on_exit_callback, Mutex.new\n        )\n\n        @_child_process_mutex.synchronize do\n          @_child_process_processes[pid] = pinfo\n        end\n        m.unlock\n        pid\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/compat_parameters.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/types'\nrequire 'fluent/config/element'\n\nmodule Fluent\n  module PluginHelper\n    module CompatParameters\n      # This plugin helper is to bring old-fashioned buffer/other\n      # configuration parameters to v0.14 plugin API configurations.\n      # This helper is mainly to convert plugins from v0.12 API\n      # to v0.14 API safely, without breaking user deployment.\n\n      BUFFER_PARAMS = {\n        \"buffer_type\" => \"@type\",\n        \"buffer_path\" => \"path\",\n        \"num_threads\"                 => \"flush_thread_count\",\n        \"flush_interval\"              => \"flush_interval\",\n        \"try_flush_interval\"          => \"flush_thread_interval\",\n        \"queued_chunk_flush_interval\" => \"flush_thread_burst_interval\",\n        \"disable_retry_limit\" => \"retry_forever\",\n        \"retry_limit\"         => \"retry_max_times\",\n        \"max_retry_wait\"      => \"retry_max_interval\",\n        \"buffer_chunk_limit\"  => \"chunk_limit_size\",\n        \"buffer_queue_limit\"  => \"queue_limit_length\",\n        \"buffer_queue_full_action\" => \"overflow_action\",\n        \"flush_at_shutdown\" => \"flush_at_shutdown\",\n      }\n\n      BUFFER_TIME_SLICED_PARAMS = {\n        \"time_slice_format\" => nil,\n        \"time_slice_wait\" => \"timekey_wait\",\n        \"timezone\" => \"timekey_zone\",\n      }\n\n      PARSER_PARAMS = {\n        \"format\" => nil,\n        \"types\" => nil,\n        \"types_delimiter\" => nil,\n        \"types_label_delimiter\" => nil,\n        \"keys\" => \"keys\", # CSVParser, TSVParser (old ValuesParser)\n        \"time_key\"    => \"time_key\",\n        \"time_format\" => \"time_format\",\n        \"localtime\" => nil,\n        \"utc\" => nil,\n        \"delimiter\"   => \"delimiter\",\n        \"keep_time_key\" => \"keep_time_key\",\n        \"null_empty_string\" => \"null_empty_string\",\n        \"null_value_pattern\" => \"null_value_pattern\",\n        \"json_parser\"      => \"json_parser\", # JSONParser\n        \"label_delimiter\"  => \"label_delimiter\", # LabeledTSVParser\n        \"format_firstline\" => \"format_firstline\", # MultilineParser\n        \"message_key\"      => \"message_key\", # NoneParser\n        \"with_priority\"       => \"with_priority\", # SyslogParser\n        \"message_format\"      => \"message_format\", # SyslogParser\n        \"rfc5424_time_format\" => \"rfc5424_time_format\", # SyslogParser\n        # There has been no parsers which can handle timezone in v0.12\n      }\n\n      INJECT_PARAMS = {\n        \"include_time_key\" => nil,\n        \"time_key\"    => \"time_key\",\n        \"time_format\" => \"time_format\",\n        \"timezone\"    => \"timezone\",\n        \"include_tag_key\" => nil,\n        \"tag_key\" => \"tag_key\",\n        \"localtime\" => nil,\n        \"utc\" => nil,\n      }\n\n      EXTRACT_PARAMS = {\n        \"time_key\"    => \"time_key\",\n        \"time_format\" => \"time_format\",\n        \"timezone\"    => \"timezone\",\n        \"tag_key\" => \"tag_key\",\n        \"localtime\" => nil,\n        \"utc\" => nil,\n      }\n\n      FORMATTER_PARAMS = {\n        \"format\" => \"@type\",\n        \"delimiter\" => \"delimiter\",\n        \"force_quotes\" => \"force_quotes\", # CsvFormatter\n        \"keys\" => \"keys\", # TSVFormatter\n        \"fields\" => \"fields\", # CsvFormatter\n        \"json_parser\" => \"json_parser\", # JSONFormatter\n        \"label_delimiter\" => \"label_delimiter\", # LabeledTSVFormatter\n        \"output_time\" => \"output_time\", # OutFileFormatter\n        \"output_tag\"  => \"output_tag\",  # OutFileFormatter\n        \"localtime\"   => \"localtime\",   # OutFileFormatter\n        \"utc\"         => \"utc\",         # OutFileFormatter\n        \"timezone\"    => \"timezone\",    # OutFileFormatter\n        \"message_key\" => \"message_key\", # SingleValueFormatter\n        \"add_newline\" => \"add_newline\", # SingleValueFormatter\n        \"output_type\" => \"output_type\", # StdoutFormatter\n      }\n\n      def compat_parameters_convert(conf, *types, **kwargs)\n        types.each do |type|\n          case type\n          when :buffer\n            compat_parameters_buffer(conf, **kwargs)\n          when :inject\n            compat_parameters_inject(conf)\n          when :extract\n            compat_parameters_extract(conf)\n          when :parser\n            compat_parameters_parser(conf)\n          when :formatter\n            compat_parameters_formatter(conf)\n          else\n            raise \"BUG: unknown compat_parameters type: #{type}\"\n          end\n        end\n\n        conf\n      end\n\n      def compat_parameters_buffer(conf, default_chunk_key: '')\n        # return immediately if <buffer> section exists, or any buffer-related parameters don't exist\n        return unless conf.elements('buffer').empty?\n        return if (BUFFER_PARAMS.keys + BUFFER_TIME_SLICED_PARAMS.keys).all?{|k| !conf.has_key?(k) }\n\n        # TODO: warn obsolete parameters if these are deprecated\n        buffer_params = BUFFER_PARAMS.merge(BUFFER_TIME_SLICED_PARAMS)\n        hash = compat_parameters_copy_to_subsection_attributes(conf, buffer_params) do |compat_key, value|\n          if compat_key == 'buffer_queue_full_action' && value == 'exception'\n            'throw_exception'\n          else\n            value\n          end\n        end\n\n        chunk_key = default_chunk_key\n\n        if conf.has_key?('time_slice_format')\n          chunk_key = 'time'\n          hash['timekey'] = case conf['time_slice_format']\n                            when /\\%S/ then 1\n                            when /\\%M/ then 60\n                            when /\\%H/ then 3600\n                            when /\\%d/ then 86400\n                            else\n                              raise Fluent::ConfigError, \"time_slice_format only with %Y or %m is too long\"\n                            end\n          if conf.has_key?('localtime') || conf.has_key?('utc')\n            if conf.has_key?('localtime') && conf.has_key?('utc')\n              raise Fluent::ConfigError, \"both of utc and localtime are specified, use only one of them\"\n            elsif conf.has_key?('localtime')\n              hash['timekey_use_utc'] = !(Fluent::Config.bool_value(conf['localtime']))\n            elsif conf.has_key?('utc')\n              hash['timekey_use_utc'] = Fluent::Config.bool_value(conf['utc'])\n            end\n          end\n        else\n          if chunk_key == 'time'\n            hash['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'\n          end\n        end\n\n        e = Fluent::Config::Element.new('buffer', chunk_key, hash, [])\n        conf.elements << e\n\n        conf\n      end\n\n      def compat_parameters_inject(conf)\n        return unless conf.elements('inject').empty?\n        return if INJECT_PARAMS.keys.all?{|k| !conf.has_key?(k) }\n\n        # TODO: warn obsolete parameters if these are deprecated\n        hash = compat_parameters_copy_to_subsection_attributes(conf, INJECT_PARAMS)\n\n        if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])\n          hash['time_key'] ||= 'time'\n          hash['time_type'] ||= 'string'\n        end\n        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])\n          hash['time_key'] ||= 'time'\n          hash['time_type'] = 'unixtime'\n        end\n        if conf.has_key?('localtime') || conf.has_key?('utc')\n          utc = to_bool(conf['utc'])\n          localtime = to_bool(conf['localtime'])\n          if conf.has_key?('localtime') && conf.has_key?('utc') && !(localtime ^ utc)\n            raise Fluent::ConfigError, \"both of utc and localtime are specified, use only one of them\"\n          elsif conf.has_key?('localtime')\n            hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])\n          elsif conf.has_key?('utc')\n            hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))\n            # Specifying \"localtime false\" means using UTC in TimeFormatter\n            # And specifying \"utc\" is different from specifying \"timezone +0000\"(it's not always UTC).\n            # There are difference between \"Z\" and \"+0000\" in timezone formatting.\n            # TODO: add kwargs to TimeFormatter to specify \"using localtime\", \"using UTC\" or \"using specified timezone\" in more explicit way\n          end\n        end\n\n        if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])\n          hash['tag_key'] ||= 'tag'\n        end\n\n        e = Fluent::Config::Element.new('inject', '', hash, [])\n        conf.elements << e\n\n        conf\n      end\n\n      def to_bool(v)\n        if  v.is_a?(FalseClass) || v == 'false' || v.nil?\n          false\n        else\n          true\n        end\n      end\n\n      def compat_parameters_extract(conf)\n        return unless conf.elements('extract').empty?\n        return if EXTRACT_PARAMS.keys.all?{|k| !conf.has_key?(k) } && !conf.has_key?('format')\n\n        # TODO: warn obsolete parameters if these are deprecated\n        hash = compat_parameters_copy_to_subsection_attributes(conf, EXTRACT_PARAMS)\n\n        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])\n          hash['time_key'] ||= 'time'\n          hash['time_type'] = 'unixtime'\n        elsif conf.has_key?('format') && conf[\"format\"].start_with?(\"/\") && conf[\"format\"].end_with?(\"/\") # old-style regexp parser\n          hash['time_key'] ||= 'time'\n          hash['time_type'] ||= 'string'\n        end\n        if conf.has_key?('localtime') || conf.has_key?('utc')\n          if conf.has_key?('localtime') && conf.has_key?('utc')\n            raise Fluent::ConfigError, \"both of utc and localtime are specified, use only one of them\"\n          elsif conf.has_key?('localtime')\n            hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])\n          elsif conf.has_key?('utc')\n            hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))\n            # Specifying \"localtime false\" means using UTC in TimeFormatter\n            # And specifying \"utc\" is different from specifying \"timezone +0000\"(it's not always UTC).\n            # There are difference between \"Z\" and \"+0000\" in timezone formatting.\n            # TODO: add kwargs to TimeFormatter to specify \"using localtime\", \"using UTC\" or \"using specified timezone\" in more explicit way\n          end\n        end\n\n        e = Fluent::Config::Element.new('extract', '', hash, [])\n        conf.elements << e\n\n        conf\n      end\n\n      def compat_parameters_parser(conf)\n        return unless conf.elements('parse').empty?\n        return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }\n\n        # TODO: warn obsolete parameters if these are deprecated\n        hash = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)\n\n        if conf[\"format\"]\n          if conf[\"format\"].start_with?(\"/\") && conf[\"format\"].end_with?(\"/\")\n            hash[\"@type\"] = \"regexp\"\n            hash[\"expression\"] = conf[\"format\"][1..-2]\n          else\n            hash[\"@type\"] = conf[\"format\"]\n          end\n        end\n\n        if conf[\"types\"]\n          delimiter = conf[\"types_delimiter\"] || ','\n          label_delimiter = conf[\"types_label_delimiter\"] || ':'\n          types = {}\n          conf['types'].split(delimiter).each do |pair|\n            key, value = pair.split(label_delimiter, 2)\n            types[key] = value\n          end\n          hash[\"types\"] = JSON.dump(types)\n        end\n        if conf.has_key?('localtime') || conf.has_key?('utc')\n          if conf.has_key?('localtime') && conf.has_key?('utc')\n            raise Fluent::ConfigError, \"both of utc and localtime are specified, use only one of them\"\n          elsif conf.has_key?('localtime')\n            hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])\n          elsif conf.has_key?('utc')\n            hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))\n            # Specifying \"localtime false\" means using UTC in TimeFormatter\n            # And specifying \"utc\" is different from specifying \"timezone +0000\"(it's not always UTC).\n            # There are difference between \"Z\" and \"+0000\" in timezone formatting.\n            # TODO: add kwargs to TimeFormatter to specify \"using localtime\", \"using UTC\" or \"using specified timezone\" in more explicit way\n          end\n        end\n\n        e = Fluent::Config::Element.new('parse', '', hash, [])\n        conf.elements << e\n\n        conf\n      end\n\n      def compat_parameters_formatter(conf)\n        return unless conf.elements('format').empty?\n        return if FORMATTER_PARAMS.keys.all?{|k| !conf.has_key?(k) }\n\n        # TODO: warn obsolete parameters if these are deprecated\n        hash = compat_parameters_copy_to_subsection_attributes(conf, FORMATTER_PARAMS)\n\n        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])\n          hash['time_type'] = 'unixtime'\n        end\n\n        e = Fluent::Config::Element.new('format', '', hash, [])\n        conf.elements << e\n\n        conf\n      end\n\n      def compat_parameters_copy_to_subsection_attributes(conf, params, &block)\n        hash = {}\n        params.each do |compat, current|\n          next unless current\n          if conf.has_key?(compat)\n            if block_given?\n              hash[current] = block.call(compat, conf[compat])\n            else\n              hash[current] = conf[compat]\n            end\n          end\n        end\n        hash\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/counter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin'\nrequire 'fluent/counter/client'\n\nmodule Fluent\n  module PluginHelper\n    module Counter\n      def counter_client_create(scope:, loop: Coolio::Loop.new)\n        client_conf = system_config.counter_client\n        raise Fluent::ConfigError, '<counter_client> is required in <system>' unless client_conf\n        counter_client = Fluent::Counter::Client.new(loop, port: client_conf.port, host: client_conf.host, log: log, timeout: client_conf.timeout)\n        counter_client.start\n        counter_client.establish(scope)\n        @_counter_client = counter_client\n        counter_client\n      end\n\n      attr_reader :_counter_client\n\n      def initialize\n        super\n        @_counter_client = nil\n      end\n\n      def stop\n        super\n        @_counter_client.stop\n      end\n\n      def terminate\n        @_counter_client = nil\n        super\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/event_emitter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/time'\n\nmodule Fluent\n  module PluginHelper\n    module EventEmitter\n      # stop     : [-]\n      # shutdown : disable @router\n      # close    : [-]\n      # terminate: [-]\n\n      def router\n        @_event_emitter_used_actually = true\n\n        return Engine.root_agent.source_only_router if @_event_emitter_force_source_only_router\n\n        if @_event_emitter_lazy_init\n          @router = @primary_instance.router\n        end\n        if @router.respond_to?(:caller_plugin_id=)\n          @router.caller_plugin_id = self.plugin_id\n        end\n        @router\n      end\n\n      def router=(r)\n        # not recommended now...\n        @router = r\n      end\n\n      def has_router?\n        true\n      end\n\n      def event_emitter_used_actually?\n        @_event_emitter_used_actually\n      end\n\n      def event_emitter_apply_source_only\n        @_event_emitter_force_source_only_router = true\n      end\n\n      def event_emitter_cancel_source_only\n        @_event_emitter_force_source_only_router = false\n      end\n\n      def event_emitter_router(label_name)\n        if label_name\n          if label_name == \"@ROOT\"\n            Engine.root_agent.event_router\n          else\n            Engine.root_agent.find_label(label_name).event_router\n          end\n        elsif self.respond_to?(:as_secondary) && self.as_secondary\n          if @primary_instance.has_router?\n            @_event_emitter_lazy_init = true\n            nil # primary plugin's event router is not initialized yet, here.\n          else\n            @primary_instance.context_router\n          end\n        else\n          # `Engine.root_agent.event_router` is for testing\n          self.context_router || Engine.root_agent.event_router\n        end\n      end\n\n      def initialize\n        super\n        @_event_emitter_used_actually = false\n        @_event_emitter_lazy_init = false\n        @_event_emitter_force_source_only_router = false\n        @router = nil\n      end\n\n      def configure(conf)\n        require 'fluent/engine'\n        super\n        @router = event_emitter_router(conf['@label'])\n      end\n\n      def after_shutdown\n        @router = nil\n        super\n      end\n\n      def close # unset router many times to reduce test cost\n        @router = nil\n        super\n      end\n\n      def terminate\n        @router = nil\n        super\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/event_loop.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'cool.io'\nrequire 'fluent/plugin_helper/thread'\nrequire 'fluent/clock'\n\nmodule Fluent\n  module PluginHelper\n    module EventLoop\n      # Currently this plugin helper is only for other helpers, not plugins.\n      # there's no way to create customized watchers to attach event loops.\n      include Fluent::PluginHelper::Thread\n\n      # stop     : [-]\n      # shutdown : detach all event watchers on event loop\n      # close    : stop event loop\n      # terminate: initialize internal state\n\n      EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5\n      EVENT_LOOP_SHUTDOWN_TIMEOUT = 5\n\n      attr_reader :_event_loop # for tests\n\n      def event_loop_attach(watcher)\n        @_event_loop_mutex.synchronize do\n          @_event_loop.attach(watcher)\n          @_event_loop_attached_watchers << watcher\n          watcher\n        end\n      end\n\n      def event_loop_detach(watcher)\n        if watcher.attached?\n          watcher.detach\n        end\n        @_event_loop_mutex.synchronize do\n          @_event_loop_attached_watchers.delete(watcher)\n        end\n      end\n\n      def event_loop_wait_until_start\n        sleep(0.1) until event_loop_running?\n      end\n\n      def event_loop_wait_until_stop\n        timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT\n        sleep(0.1) while event_loop_running? && Fluent::Clock.now < timeout_at\n        if @_event_loop_running\n          puts \"terminating event_loop forcedly\"\n          caller.each{|bt| puts \"\\t#{bt}\" }\n          @_event_loop.stop rescue nil\n          @_event_loop_running = true\n        end\n      end\n\n      def event_loop_running?\n        @_event_loop_running\n      end\n\n      def initialize\n        super\n        @_event_loop = Coolio::Loop.new\n        @_event_loop_running = false\n        @_event_loop_mutex = Mutex.new\n        # plugin MAY configure loop run timeout in #configure\n        @_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT\n        @_event_loop_attached_watchers = []\n      end\n\n      def start\n        super\n\n        # event loop does not run here, so mutex lock is not required\n        thread_create :event_loop do\n          begin\n            default_watcher = DefaultWatcher.new\n            event_loop_attach(default_watcher)\n            @_event_loop_running = true\n            @_event_loop.run(@_event_loop_run_timeout) # this method blocks\n          ensure\n            @_event_loop_running = false\n          end\n        end\n      end\n\n      def shutdown\n        @_event_loop_mutex.synchronize do\n          @_event_loop_attached_watchers.reverse_each do |w|\n            if w.attached?\n              begin\n                w.detach\n              rescue => e\n                log.warn \"unexpected error while detaching event loop watcher\", error: e\n              end\n            end\n          end\n        end\n\n        super\n      end\n\n      def after_shutdown\n        timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT\n        @_event_loop_mutex.synchronize do\n          @_event_loop.watchers.reverse_each do |w|\n            begin\n              w.detach\n            rescue => e\n              log.warn \"unexpected error while detaching event loop watcher\", error: e\n            end\n          end\n        end\n        while @_event_loop_running\n          if Fluent::Clock.now >= timeout_at\n            log.warn \"event loop does NOT exit until hard timeout.\"\n            raise \"event loop does NOT exit until hard timeout.\" if @under_plugin_development\n            break\n          end\n          sleep 0.1\n        end\n\n        super\n      end\n\n      def close\n        if @_event_loop_running\n          begin\n            @_event_loop.stop # we cannot check loop is running or not\n          rescue RuntimeError => e\n            raise unless e.message == 'loop not running'\n          end\n        end\n\n        super\n      end\n\n      def terminate\n        @_event_loop = nil\n        @_event_loop_running = false\n        @_event_loop_mutex = nil\n        @_event_loop_run_timeout = nil\n\n        super\n      end\n\n      # watcher to block to run event loop until shutdown\n      class DefaultWatcher < Coolio::TimerWatcher\n        def initialize\n          super(1, true) # interval: 1, repeat: true\n        end\n        # do nothing\n        def on_timer; end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/extract.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/event'\nrequire 'fluent/time'\nrequire 'fluent/configurable'\n\nmodule Fluent\n  module PluginHelper\n    module Extract\n      def extract_tag_from_record(record)\n        return nil unless @_extract_enabled\n\n        if @_extract_tag_key && record.has_key?(@_extract_tag_key)\n          v = @_extract_keep_tag_key ? record[@_extract_tag_key] : record.delete(@_extract_tag_key)\n          return v.to_s\n        end\n\n        nil\n      end\n\n      def extract_time_from_record(record)\n        return nil unless @_extract_enabled\n\n        if @_extract_time_key && record.has_key?(@_extract_time_key)\n          v = @_extract_keep_time_key ? record[@_extract_time_key] : record.delete(@_extract_time_key)\n          return @_extract_time_parser.call(v)\n        end\n\n        nil\n      end\n\n      module ExtractParams\n        include Fluent::Configurable\n        config_section :extract, required: false, multi: false, param_name: :extract_config do\n          config_param :tag_key, :string, default: nil\n          config_param :keep_tag_key, :bool, default: false\n          config_param :time_key, :string, default: nil\n          config_param :keep_time_key, :bool, default: false\n\n          # To avoid defining :time_type twice\n          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float\n\n          Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|\n            config_param(name, type, **opts)\n          end\n        end\n      end\n\n      def self.included(mod)\n        mod.include ExtractParams\n      end\n\n      def initialize\n        super\n        @_extract_enabled = false\n        @_extract_tag_key = nil\n        @_extract_keep_tag_key = nil\n        @_extract_time_key = nil\n        @_extract_keep_time_key = nil\n        @_extract_time_parser = nil\n      end\n\n      def configure(conf)\n        super\n\n        if @extract_config\n          @_extract_tag_key = @extract_config.tag_key\n          @_extract_keep_tag_key = @extract_config.keep_tag_key\n          @_extract_time_key = @extract_config.time_key\n          if @_extract_time_key\n            @_extract_keep_time_key = @extract_config.keep_time_key\n            @_extract_time_parser = case @extract_config.time_type\n                                    when :float then Fluent::NumericTimeParser.new(:float)\n                                    when :unixtime then Fluent::NumericTimeParser.new(:unixtime)\n                                    else\n                                      localtime = @extract_config.localtime && !@extract_config.utc\n                                      Fluent::TimeParser.new(@extract_config.time_format, localtime, @extract_config.timezone)\n                                    end\n          else\n            if @extract_config.time_format\n              log.warn \"'time_format' specified without 'time_key', will be ignored\"\n            end\n          end\n\n          @_extract_enabled = @_extract_tag_key || @_extract_time_key\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/formatter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/formatter'\nrequire 'fluent/config/element'\nrequire 'fluent/configurable'\n\nmodule Fluent\n  module PluginHelper\n    module Formatter\n      def formatter_create(usage: '', type: nil, conf: nil, default_type: nil)\n        formatter = @_formatters[usage]\n        return formatter if formatter && !type && !conf\n\n        type = if type\n                 type\n               elsif conf && conf.respond_to?(:[])\n                 raise Fluent::ConfigError, \"@type is required in <format>\" unless conf['@type']\n                 conf['@type']\n               elsif default_type\n                 default_type\n               else\n                 raise ArgumentError, \"BUG: both type and conf are not specified\"\n               end\n        formatter = Fluent::Plugin.new_formatter(type, parent: self)\n        config = case conf\n                 when Fluent::Config::Element\n                   conf\n                 when Hash\n                   # in code, programmer may use symbols as keys, but Element needs strings\n                   conf = Hash[conf.map{|k,v| [k.to_s, v]}]\n                   Fluent::Config::Element.new('format', usage, conf, [])\n                 when nil\n                   Fluent::Config::Element.new('format', usage, {}, [])\n                 else\n                   raise ArgumentError, \"BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'\"\n                 end\n        formatter.configure(config)\n        if @_formatters_started\n          formatter.start\n        end\n\n        @_formatters[usage] = formatter\n        formatter\n      end\n\n      module FormatterParams\n        include Fluent::Configurable\n        # minimum section definition to instantiate formatter plugin instances\n        config_section :format, required: false, multi: true, init: true, param_name: :formatter_configs do\n          config_argument :usage, :string, default: ''\n          config_param    :@type, :string # config_set_default required for :@type\n        end\n      end\n\n      def self.included(mod)\n        mod.include FormatterParams\n      end\n\n      attr_reader :_formatters # for tests\n\n      def initialize\n        super\n        @_formatters_started = false\n        @_formatters = {} # usage => formatter\n      end\n\n      def configure(conf)\n        super\n\n        @formatter_configs.each do |section|\n          if @_formatters[section.usage]\n            raise Fluent::ConfigError, \"duplicated formatter configured: #{section.usage}\"\n          end\n          formatter = Plugin.new_formatter(section[:@type], parent: self)\n          formatter.configure(section.corresponding_config_element)\n          @_formatters[section.usage] = formatter\n        end\n      end\n\n      def start\n        super\n        @_formatters_started = true\n        @_formatters.each_pair do |usage, formatter|\n          formatter.start\n        end\n      end\n\n      def formatter_operate(method_name, &block)\n        @_formatters.each_pair do |usage, formatter|\n          begin\n            formatter.__send__(method_name)\n            block.call(formatter) if block_given?\n          rescue => e\n            log.error \"unexpected error while #{method_name}\", usage: usage, formatter: formatter, error: e\n          end\n        end\n      end\n\n      def stop\n        super\n        formatter_operate(:stop)\n      end\n\n      def before_shutdown\n        formatter_operate(:before_shutdown)\n        super\n      end\n\n      def shutdown\n        formatter_operate(:shutdown)\n        super\n      end\n\n      def after_shutdown\n        formatter_operate(:after_shutdown)\n        super\n      end\n\n      def close\n        formatter_operate(:close)\n        super\n      end\n\n      def terminate\n        formatter_operate(:terminate)\n        @_formatters_started = false\n        @_formatters = {}\n        super\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/http_server/app.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'async/http/protocol'\nrequire 'fluent/plugin_helper/http_server/methods'\nrequire 'fluent/plugin_helper/http_server/request'\n\nmodule Fluent\n  module PluginHelper\n    module HttpServer\n      class App\n        def initialize(router, logger)\n          @logger = logger\n          @router = router\n        end\n\n        # Required method by async-http\n        def call(request)\n          method = request.method\n          resp =\n            case method\n            when HttpServer::Methods::GET\n              get(request)\n            when HttpServer::Methods::HEAD\n              head(request)\n            when HttpServer::Methods::POST\n              post(request)\n            when HttpServer::Methods::PATCH\n              patch(request)\n            when HttpServer::Methods::PUT\n              put(request)\n            when HttpServer::Methods::DELETE\n              delete(request)\n            when HttpServer::Methods::OPTIONS\n              options(request)\n            when HttpServer::Methods::CONNECT\n              connect(request)\n            when HttpServer::Methods::TRACE\n              trace(request)\n            else\n              raise \"Unknown method #{method}\"\n            end\n          Protocol::HTTP::Response[*resp]\n        rescue => e\n          @logger.error(e)\n          Protocol::HTTP::Response[500, { 'Content-Type' => 'text/plain' }, 'Internal Server Error']\n        end\n\n        HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|\n          define_method(name) do |request|\n            req = Request.new(request)\n\n            path = req.path\n            canonical_path =\n              if path.size >= 2 && !path.end_with?('/')\n                \"#{path}/\"\n              else\n                path\n              end\n            @router.route!(name, canonical_path, req)\n          ensure\n            request.body&.close\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/http_server/methods.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module PluginHelper\n    module HttpServer\n      module Methods\n        GET = 'GET'.freeze\n        HEAD = 'HEAD'.freeze\n        POST = 'POST'.freeze\n        PUT = 'PUT'.freeze\n        PATCH = 'PATCH'.freeze\n        DELETE = 'DELETE'.freeze\n        OPTIONS = 'OPTIONS'.freeze\n        CONNECT = 'CONNECT'.freeze\n        TRACE = 'TRACE'.freeze\n\n        ALL = [GET, HEAD, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS, TRACE].freeze\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/http_server/request.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'uri'\nrequire 'async/http/protocol'\nrequire 'fluent/plugin_helper/http_server/methods'\n\nmodule Fluent\n  module PluginHelper\n    module HttpServer\n      class Request\n        attr_reader :path, :query_string\n\n        def initialize(request)\n          @request = request\n          path = request.path\n          @path, @query_string = path.split('?', 2)\n        end\n\n        def headers\n          @request.headers\n        end\n\n        def query\n          if @query_string\n            hash = Hash.new { |h, k| h[k] = [] }\n            # For compatibility with CGI.parse\n            URI.decode_www_form(@query_string).each_with_object(hash) do |(key, value), h|\n              h[key] << value\n            end\n          end\n        end\n\n        def body\n          @request.body&.read\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/http_server/router.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'async/http/protocol'\n\nmodule Fluent\n  module PluginHelper\n    module HttpServer\n      class Router\n        class NotFoundApp\n          def self.call(req)\n            [404, { 'Content-Type' => 'text/plain' }, \"404 Not Found\\n\"]\n          end\n        end\n\n        def initialize(default_app = nil)\n          @router = { get: {}, head: {}, post: {}, put: {}, patch: {}, delete: {}, connect: {}, options: {}, trace: {} }\n          @default_app = default_app || NotFoundApp\n        end\n\n        # @param method [Symbol]\n        # @param path [String]\n        # @param app [Object]\n        def mount(method, path, app)\n          if @router[method].include?(path)\n            raise \"#{path} is already mounted\"\n          end\n\n          @router[method][path] = app\n        end\n\n        # @param method [Symbol]\n        # @param path [String]\n        # @param request [Fluent::PluginHelper::HttpServer::Request]\n        def route!(method, path, request)\n          @router.fetch(method).fetch(path, @default_app).call(request)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/http_server/server.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'async'\nrequire 'async/http'\nrequire 'async/http/endpoint'\n\nrequire 'fluent/plugin_helper/http_server/app'\nrequire 'fluent/plugin_helper/http_server/router'\nrequire 'fluent/plugin_helper/http_server/methods'\nrequire 'fluent/log/console_adapter'\n\nmodule Fluent\n  module PluginHelper\n    module HttpServer\n      class Server\n        # @param logger [Logger]\n        # @param default_app [Object] This method must have #call.\n        # @param tls_context [OpenSSL::SSL::SSLContext]\n        def initialize(addr:, port:, logger:, default_app: nil, tls_context: nil)\n          # Normalize the address: strip brackets if present\n          # Brackets are only for URI formatting, not for socket binding\n          @addr = if addr.start_with?('[') && addr.end_with?(']')\n                    addr[1..-2]  # Remove surrounding brackets\n                  else\n                    addr\n                  end\n          @port = port\n          @logger = logger\n\n          # TODO: support http2\n          scheme = tls_context ? 'https' : 'http'\n          # Handle IPv6 addresses properly in URI construction per RFC 3986\n          # Add brackets to IPv6 addresses for URI compliance (ip-literal per RFC 3986)\n          ip_literal = if @addr.include?(\":\")\n                         \"[#{@addr}]\"  # IPv6 address - add brackets\n                       else\n                         @addr         # IPv4 or hostname - use directly\n                       end\n          @uri = URI(\"#{scheme}://#{ip_literal}:#{@port}\").to_s\n          @router = Router.new(default_app)\n          @server_task = nil\n          Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)\n\n          opts = if tls_context\n                   { ssl_context: tls_context }\n                 else\n                   {}\n                 end\n          @server = Async::HTTP::Server.new(App.new(@router, @logger), Async::HTTP::Endpoint.parse(@uri, **opts))\n\n          if block_given?\n            yield(self)\n          end\n        end\n\n        def start(notify = nil)\n          Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)\n          @logger.debug(\"Start async HTTP server listening #{@uri}\")\n\n          Async do |task|\n            Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)\n            @server_task = task.async do\n              Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)\n              @server.run\n            end\n            if notify\n              notify.push(:ready)\n            end\n\n            @server_task_queue = ::Thread::Queue.new\n            @server_task_queue.pop\n            @server_task&.stop\n          end\n\n          @logger.debug('Finished HTTP server')\n        end\n\n        def stop\n          @logger.debug('closing HTTP server')\n          @server_task_queue.push(:stop)\n        end\n\n        HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|\n          define_method(name) do |path, app = nil, &block|\n            unless path.end_with?('/')\n              path += '/'\n            end\n\n            if (block && app) || (!block && !app)\n              raise 'You must specify either app or block in the same time'\n            end\n\n            @router.mount(name, path, app || block)\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/http_server/ssl_context_builder.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin_helper/cert_option'\n\nmodule Fluent\n  module PluginHelper\n    module HttpServer\n      # In order not to expose CertOption's methods unnecessary\n      class SSLContextBuilder\n        include Fluent::PluginHelper::CertOption\n\n        def initialize(log)\n          @log = log\n        end\n\n        # @param config [Fluent::Config::Section] @transport_config\n        def build(config)\n          cert_option_create_context(config.version, config.insecure, config.ciphers, config)\n        end\n\n        private\n\n        attr_reader :log\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/http_server.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin_helper/thread'\nrequire 'fluent/plugin_helper/server' # For Server::ServerTransportParams\nrequire 'fluent/plugin_helper/http_server/server'\nrequire 'fluent/plugin_helper/http_server/ssl_context_builder'\n\nmodule Fluent\n  module PluginHelper\n    module HttpServer\n      include Fluent::PluginHelper::Thread\n      include Fluent::Configurable\n\n      # stop     : stop http server and mark callback thread as stopped\n      # shutdown : [-]\n      # close    : correct stopped threads\n      # terminate: kill thread\n\n      def self.included(mod)\n        mod.include Fluent::PluginHelper::Server::ServerTransportParams\n      end\n\n      def initialize(*)\n        super\n        @_http_server = nil\n      end\n\n      def create_http_server(title, addr:, port:, logger:, default_app: nil, proto: nil, tls_opts: nil, &block)\n        logger.warn('this method is deprecated. Use #http_server_create_http_server instead')\n        http_server_create_http_server(title, addr: addr, port: port, logger: logger, default_app: default_app, proto: proto, tls_opts: tls_opts, &block)\n      end\n\n      # @param title [Symbol] the thread name. this value should be unique.\n      # @param addr [String] Listen address\n      # @param port [String] Listen port\n      # @param logger [Logger] logger used in this server\n      # @param default_app [Object] This method must have #call.\n      # @param proto [Symbol] :tls or :tcp\n      # @param tls_opts [Hash] options for TLS.\n      def http_server_create_http_server(title, addr:, port:, logger:, default_app: nil, proto: nil, tls_opts: nil, &block)\n        unless block_given?\n          raise ArgumentError, 'BUG: callback not specified'\n        end\n\n        if proto == :tls || (@transport_config && @transport_config.protocol == :tls)\n          http_server_create_https_server(title, addr: addr, port: port, logger: logger, default_app: default_app, tls_opts: tls_opts, &block)\n        else\n          @_http_server = HttpServer::Server.new(addr: addr, port: port, logger: logger, default_app: default_app) do |serv|\n            yield(serv)\n          end\n\n          _block_until_http_server_start do |notify|\n            thread_create(title) do\n              @_http_server.start(notify)\n            end\n          end\n        end\n      end\n\n      # @param title [Symbol] the thread name. this value should be unique.\n      # @param addr [String] Listen address\n      # @param port [String] Listen port\n      # @param logger [Logger] logger used in this server\n      # @param default_app [Object] This method must have #call.\n      # @param tls_opts [Hash] options for TLS.\n      def http_server_create_https_server(title, addr:, port:, logger:, default_app: nil, tls_opts: nil)\n        topt =\n          if tls_opts\n            _http_server_overwrite_config(@transport_config, tls_opts)\n          else\n            @transport_config\n          end\n        ctx = Fluent::PluginHelper::HttpServer::SSLContextBuilder.new($log).build(topt)\n\n        @_http_server = HttpServer::Server.new(addr: addr, port: port, logger: logger, default_app: default_app, tls_context: ctx) do |serv|\n          yield(serv)\n        end\n\n        _block_until_http_server_start do |notify|\n          thread_create(title) do\n            @_http_server.start(notify)\n          end\n        end\n      end\n\n      def stop\n        if @_http_server\n          @_http_server.stop\n        end\n\n        super\n      end\n\n      private\n\n      def _http_server_overwrite_config(config, opts)\n        conf = config.dup\n        Fluent::PluginHelper::Server::SERVER_TRANSPORT_PARAMS.map(&:to_s).each do |param|\n          if opts.key?(param)\n            conf[param] = opts[param]\n          end\n        end\n        conf\n      end\n\n      # To block until server is ready to listen\n      def _block_until_http_server_start\n        que = Queue.new\n        yield(que)\n        que.pop\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/inject.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/event'\nrequire 'fluent/time'\nrequire 'fluent/configurable'\nrequire 'socket'\n\nmodule Fluent\n  module PluginHelper\n    module Inject\n      def inject_values_to_record(tag, time, record)\n        return record unless @_inject_enabled\n\n        r = record.dup\n        if @_inject_hostname_key\n          r[@_inject_hostname_key] = @_inject_hostname\n        end\n        if @_inject_worker_id_key\n          r[@_inject_worker_id_key] = @_inject_worker_id\n        end\n        if @_inject_tag_key\n          r[@_inject_tag_key] = tag\n        end\n        if @_inject_time_key\n          r[@_inject_time_key] = @_inject_time_formatter.call(time)\n        end\n\n        r\n      end\n\n      def inject_values_to_event_stream(tag, es)\n        return es unless @_inject_enabled\n\n        new_es = Fluent::MultiEventStream.new\n        es.each do |time, record|\n          r = record.dup\n          if @_inject_hostname_key\n            r[@_inject_hostname_key] = @_inject_hostname\n          end\n          if @_inject_worker_id_key\n            r[@_inject_worker_id_key] = @_inject_worker_id\n          end\n          if @_inject_tag_key\n            r[@_inject_tag_key] = tag\n          end\n          if @_inject_time_key\n            r[@_inject_time_key] = @_inject_time_formatter.call(time)\n          end\n          new_es.add(time, r)\n        end\n\n        new_es\n      end\n\n      module InjectParams\n        include Fluent::Configurable\n        config_section :inject, required: false, multi: false, param_name: :inject_config do\n          config_param :hostname_key, :string, default: nil\n          config_param :hostname, :string, default: nil\n          config_param :worker_id_key, :string, default: nil\n          config_param :tag_key, :string, default: nil\n          config_param :time_key, :string, default: nil\n\n          # To avoid defining :time_type twice\n          config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :unixtime_micros, :unixtime_nanos, :string], default: :float\n\n          Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|\n            config_param(name, type, **opts)\n          end\n        end\n      end\n\n      def self.included(mod)\n        mod.include InjectParams\n      end\n\n      def initialize\n        super\n        @_inject_enabled = false\n        @_inject_hostname_key = nil\n        @_inject_hostname = nil\n        @_inject_worker_id_key = nil\n        @_inject_worker_id = nil\n        @_inject_tag_key = nil\n        @_inject_time_key = nil\n        @_inject_time_formatter = nil\n      end\n\n      def configure(conf)\n        super\n\n        if @inject_config\n          @_inject_hostname_key = @inject_config.hostname_key\n          if @_inject_hostname_key\n            if self.respond_to?(:buffer_config)\n              # Output plugin cannot use \"hostname\"(specified by @hostname_key),\n              # injected by this plugin helper, in chunk keys.\n              # This plugin helper works in `#format` (in many cases), but modified record\n              # don't have any side effect in chunking of output plugin.\n              if self.buffer_config.chunk_keys.include?(@_inject_hostname_key)\n                log.error \"Use filters to inject hostname to use it in buffer chunking.\"\n                raise Fluent::ConfigError, \"the key specified by 'hostname_key' in <inject> cannot be used in buffering chunk key.\"\n              end\n            end\n\n            @_inject_hostname =  @inject_config.hostname\n            unless @_inject_hostname\n              @_inject_hostname = ::Socket.gethostname\n              log.info \"using hostname for specified field\", host_key: @_inject_hostname_key, host_name: @_inject_hostname\n            end\n          end\n          @_inject_worker_id_key = @inject_config.worker_id_key\n          if @_inject_worker_id_key\n            @_inject_worker_id = fluentd_worker_id # get id here, because #with_worker_config method may be used only for #configure in tests\n          end\n          @_inject_tag_key = @inject_config.tag_key\n          @_inject_time_key = @inject_config.time_key\n          if @_inject_time_key\n            @_inject_time_formatter = case @inject_config.time_type\n                                      when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value\n                                      when :unixtime_millis then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000 + time.nsec / 1_000_000 : (time * 1_000).floor }\n                                      when :unixtime_micros then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000 + time.nsec / 1_000 : (time * 1_000_000).floor }\n                                      when :unixtime_nanos then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000_000 + time.nsec : (time * 1_000_000_000).floor }\n                                      when :unixtime then ->(time){ time.to_i }\n                                      else\n                                        localtime = @inject_config.localtime && !@inject_config.utc\n                                        Fluent::TimeFormatter.new(@inject_config.time_format, localtime, @inject_config.timezone)\n                                      end\n          else\n            if @inject_config.time_format\n              log.warn \"'time_format' specified without 'time_key', will be ignored\"\n            end\n          end\n\n          @_inject_enabled = @_inject_hostname_key || @_inject_worker_id_key || @_inject_tag_key || @_inject_time_key\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/metrics.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'forwardable'\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/metrics'\nrequire 'fluent/plugin_helper/timer'\nrequire 'fluent/config/element'\nrequire 'fluent/configurable'\nrequire 'fluent/system_config'\n\nmodule Fluent\n  module PluginHelper\n    module Metrics\n      include Fluent::SystemConfig::Mixin\n\n      attr_reader :_metrics # For tests.\n\n      def initialize\n        super\n        @_metrics_started = false\n        @_metrics = {} # usage => metrics_state\n      end\n\n      def configure(conf)\n        super\n\n        @plugin_type_or_id = if self.plugin_id_configured?\n                               self.plugin_id\n                             else\n                               if type = (conf[\"@type\"] || conf[\"type\"])\n                                 \"#{type}.#{self.plugin_id}\"\n                               else\n                                 \"#{self.class.to_s.split(\"::\").last.downcase}.#{self.plugin_id}\"\n                               end\n                             end\n      end\n\n      def metrics_create(namespace: \"fluentd\", subsystem: \"metrics\", name:, help_text:, labels: {}, prefer_gauge: false)\n        metrics = if system_config.metrics\n                    Fluent::Plugin.new_metrics(system_config.metrics[:@type], parent: self)\n                  else\n                    Fluent::Plugin.new_metrics(Fluent::Plugin::Metrics::DEFAULT_TYPE, parent: self)\n                  end\n        config = if system_config.metrics\n                   system_config.metrics.corresponding_config_element\n                 else\n                   Fluent::Config::Element.new('metrics', '', {'@type' => Fluent::Plugin::Metrics::DEFAULT_TYPE}, [])\n                 end\n        metrics.use_gauge_metric = prefer_gauge\n        metrics.configure(config)\n        # For multi workers environment, cmetrics should be distinguish with static labels.\n        if Fluent::Engine.system_config.workers > 1\n          labels[:worker_id] = fluentd_worker_id.to_s\n        end\n        labels[:plugin] = @plugin_type_or_id\n        metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)\n\n        @_metrics[\"#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}\"] = metrics\n\n        # define the getter method for the calling instance.\n        singleton_class.module_eval do\n          unless method_defined?(name)\n            define_method(name) { metrics.get }\n          end\n        end\n\n        metrics\n      end\n\n      def metrics_operate(method_name, &block)\n        @_metrics.each_pair do |key, m|\n          begin\n            block.call(s) if block_given?\n            m.__send__(method_name)\n          rescue => e\n            log.error \"unexpected error while #{method_name}\", key: key, metrics: m, error: e\n          end\n        end\n      end\n\n      def start\n        super\n\n        metrics_operate(:start)\n        @_metrics_started = true\n      end\n\n      def stop\n        super\n        # timer stops automatically in super\n        metrics_operate(:stop)\n      end\n\n      def before_shutdown\n        metrics_operate(:before_shutdown)\n        super\n      end\n\n      def shutdown\n        metrics_operate(:shutdown)\n        super\n      end\n\n      def after_shutdown\n        metrics_operate(:after_shutdown)\n        super\n      end\n\n      def close\n        metrics_operate(:close)\n        super\n      end\n\n      def terminate\n        metrics_operate(:terminate)\n        @_metrics = {}\n        super\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/parser'\nrequire 'fluent/config/element'\nrequire 'fluent/configurable'\n\nmodule Fluent\n  module PluginHelper\n    module Parser\n      def parser_create(usage: '', type: nil, conf: nil, default_type: nil)\n        parser = @_parsers[usage]\n        return parser if parser && !type && !conf\n\n        type = if type\n                 type\n               elsif conf && conf.respond_to?(:[])\n                 raise Fluent::ConfigError, \"@type is required in <parse>\" unless conf['@type']\n                 conf['@type']\n               elsif default_type\n                 default_type\n               else\n                 raise ArgumentError, \"BUG: both type and conf are not specified\"\n               end\n        parser = Fluent::Plugin.new_parser(type, parent: self)\n        config = case conf\n                 when Fluent::Config::Element\n                   conf\n                 when Hash\n                   # in code, programmer may use symbols as keys, but Element needs strings\n                   conf = Hash[conf.map{|k,v| [k.to_s, v]}]\n                   Fluent::Config::Element.new('parse', usage, conf, [])\n                 when nil\n                   Fluent::Config::Element.new('parse', usage, {}, [])\n                 else\n                   raise ArgumentError, \"BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'\"\n                 end\n        parser.configure(config)\n        if @_parsers_started\n          parser.start\n        end\n\n        @_parsers[usage] = parser\n        parser\n      end\n\n      module ParserParams\n        include Fluent::Configurable\n        # minimum section definition to instantiate parser plugin instances\n        config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do\n          config_argument :usage, :string, default: ''\n          config_param    :@type, :string # config_set_default required for :@type\n        end\n      end\n\n      def self.included(mod)\n        mod.include ParserParams\n      end\n\n      attr_reader :_parsers # for tests\n\n      def initialize\n        super\n        @_parsers_started = false\n        @_parsers = {} # usage => parser\n      end\n\n      def configure(conf)\n        super\n\n        @parser_configs.each do |section|\n          if @_parsers[section.usage]\n            raise Fluent::ConfigError, \"duplicated parsers configured: #{section.usage}\"\n          end\n          parser = Plugin.new_parser(section[:@type], parent: self)\n          parser.configure(section.corresponding_config_element)\n          @_parsers[section.usage] = parser\n        end\n      end\n\n      def start\n        super\n        @_parsers_started = true\n        @_parsers.each_pair do |usage, parser|\n          parser.start\n        end\n      end\n\n      def parser_operate(method_name, &block)\n        @_parsers.each_pair do |usage, parser|\n          begin\n            parser.__send__(method_name)\n            block.call(parser) if block_given?\n          rescue => e\n            log.error \"unexpected error while #{method_name}\", usage: usage, parser: parser, error: e\n          end\n        end\n      end\n\n      def stop\n        super\n        parser_operate(:stop)\n      end\n\n      def before_shutdown\n        parser_operate(:before_shutdown)\n        super\n      end\n\n      def shutdown\n        parser_operate(:shutdown)\n        super\n      end\n\n      def after_shutdown\n        parser_operate(:after_shutdown)\n        super\n      end\n\n      def close\n        parser_operate(:close)\n        super\n      end\n\n      def terminate\n        parser_operate(:terminate)\n        @_parsers_started = false\n        @_parsers = {}\n        super\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/record_accessor.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/error'\n\nmodule Fluent\n  module PluginHelper\n    module RecordAccessor\n      def record_accessor_create(param)\n        Accessor.new(param)\n      end\n\n      def record_accessor_nested?(param)\n        Accessor.parse_parameter(param).is_a?(Array)\n      end\n\n      class Accessor\n        attr_reader :keys\n\n        def initialize(param)\n          @keys = Accessor.parse_parameter(param)\n\n          if @keys.is_a?(Array)\n            @last_key = @keys.last\n            @dig_keys = @keys[0..-2]\n            if @dig_keys.empty?\n              @keys = @keys.first\n            else\n              mcall = method(:call_dig)\n              mdelete = method(:delete_nest)\n              mset = method(:set_nest)\n              singleton_class.module_eval do\n                define_method(:call, mcall)\n                define_method(:delete, mdelete)\n                define_method(:set, mset)\n              end\n            end\n          end\n        end\n\n        def call(r)\n          r[@keys]\n        end\n\n        # To optimize the performance, use class_eval with pre-expanding @keys\n        # See https://gist.github.com/repeatedly/ab553ed260cd080bd01ec71da9427312\n        def call_dig(r)\n          r.dig(*@keys)\n        end\n\n        def delete(r)\n          r.delete(@keys)\n        end\n\n        def delete_nest(r)\n          if target = r.dig(*@dig_keys)\n            if target.is_a?(Array)\n              target.delete_at(@last_key)\n            else\n              target.delete(@last_key)\n            end\n          end\n        rescue\n          nil\n        end\n\n        def set(r, v)\n          r[@keys] = v\n        end\n\n        # set_nest doesn't create intermediate object. If key doesn't exist, no effect.\n        # See also: https://bugs.ruby-lang.org/issues/11747\n        def set_nest(r, v)\n          r.dig(*@dig_keys)&.[]=(@last_key, v)\n        rescue\n          nil\n        end\n\n        def self.parse_parameter(param)\n          if param.start_with?('$.')\n            parse_dot_notation(param)\n          elsif param.start_with?('$[')\n            parse_bracket_notation(param)\n          else\n            param\n          end\n        end\n\n        def self.parse_dot_notation(param)\n          result = []\n          keys = param[2..-1].split('.')\n          keys.each { |key|\n            if key.include?('[')\n              result.concat(parse_dot_array_op(key, param))\n            else\n              result << key\n            end\n          }\n\n          raise Fluent::ConfigError, \"empty keys in dot notation\" if result.empty?\n          validate_dot_keys(result)\n\n          result\n        end\n\n        def self.validate_dot_keys(keys)\n          keys.each { |key|\n            next unless key.is_a?(String)\n            if /\\s+/.match?(key)\n              raise Fluent::ConfigError, \"whitespace character is not allowed in dot notation. Use bracket notation: #{key}\"\n            end\n          }\n        end\n\n        def self.parse_dot_array_op(key, param)\n          start = key.index('[')\n          result = if start.zero?\n                     []\n                   else\n                     [key[0..start - 1]]\n                   end\n          key = key[start + 1..-1]\n          in_bracket = true\n\n          until key.empty?\n            if in_bracket\n              if i = key.index(']')\n                index_value = key[0..i - 1]\n                raise Fluent::ConfigError, \"missing array index in '[]'. Invalid syntax: #{param}\" if index_value == ']'\n                result << Integer(index_value)\n                key = key[i + 1..-1]\n                in_bracket = false\n              else\n                raise Fluent::ConfigError, \"'[' found but ']' not found. Invalid syntax: #{param}\"\n              end\n            else\n              if i = key.index('[')\n                key = key[i + 1..-1]\n                in_bracket = true\n              else\n                raise Fluent::ConfigError, \"found more characters after ']'. Invalid syntax: #{param}\"\n              end\n            end\n          end\n\n          result\n        end\n\n        def self.parse_bracket_notation(param)\n          orig_param = param\n          result = []\n          param = param[1..-1]\n          in_bracket = false\n\n          until param.empty?\n            if in_bracket\n              if param[0] == \"'\" || param[0] == '\"'\n                if i = param.index(\"']\") || param.index('\"]')\n                  raise Fluent::ConfigError, \"Mismatched quotes. Invalid syntax: #{orig_param}\" unless param[0] == param[i]\n                  result << param[1..i - 1]\n                  param = param[i + 2..-1]\n                  in_bracket = false\n                else\n                  raise Fluent::ConfigError, \"Incomplete bracket. Invalid syntax: #{orig_param}\"\n                end\n              else\n                if i = param.index(']')\n                  index_value = param[0..i - 1]\n                  raise Fluent::ConfigError, \"missing array index in '[]'. Invalid syntax: #{param}\" if index_value == ']'\n                  result << Integer(index_value)\n                  param = param[i + 1..-1]\n                  in_bracket = false\n                else\n                  raise Fluent::ConfigError, \"'[' found but ']' not found. Invalid syntax: #{orig_param}\"\n                end\n              end\n            else\n              if i = param.index('[')\n                param = param[i + 1..-1]\n                in_bracket = true\n              else\n                raise Fluent::ConfigError, \"found more characters after ']'. Invalid syntax: #{orig_param}\"\n              end\n            end\n          end\n\n          raise Fluent::ConfigError, \"empty keys in bracket notation\" if result.empty?\n\n          result\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/retry_state.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module PluginHelper\n    module RetryState\n      def retry_state_create(\n          title, retry_type, wait, timeout,\n          forever: false, max_steps: nil, backoff_base: 2, max_interval: nil, randomize: true, randomize_width: 0.125,\n          secondary: false, secondary_threshold: 0.8\n      )\n        case retry_type\n        when :exponential_backoff\n          ExponentialBackOffRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)\n        when :periodic\n          PeriodicRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)\n        else\n          raise \"BUG: unknown retry_type specified: '#{retry_type}'\"\n        end\n      end\n\n      class RetryStateMachine\n        attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_steps\n\n        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)\n          @title = title\n\n          @start = current_time\n          @steps = 0\n          @next_time = nil # should be initialized for first retry by child class\n\n          @timeout = timeout\n          @timeout_at = @start + timeout\n          @has_reached_timeout = false\n          @has_timed_out = false\n          @current = :primary\n\n          if randomize_width < 0 || randomize_width > 0.5\n            raise \"BUG: randomize_width MUST be between 0 and 0.5\"\n          end\n\n          @randomize = randomize\n          @randomize_width = randomize_width\n\n          @forever = forever\n          @max_steps = max_steps\n\n          @secondary = secondary\n          @secondary_threshold = secondary_threshold\n          if @secondary\n            raise \"BUG: secondary_transition_threshold MUST be between 0 and 1\" if @secondary_threshold <= 0 || @secondary_threshold >= 1\n            max_retry_timeout = timeout\n            if max_steps\n              timeout_by_max_steps = calc_max_retry_timeout(max_steps)\n              max_retry_timeout = timeout_by_max_steps if timeout_by_max_steps < max_retry_timeout\n            end\n            @secondary_transition_at = @start + max_retry_timeout * @secondary_threshold\n            @secondary_transition_steps = nil\n          end\n        end\n\n        def current_time\n          Time.now\n        end\n\n        def randomize(interval)\n          return interval unless @randomize\n\n          interval + (interval * @randomize_width * (2 * rand - 1.0))\n        end\n\n        def calc_next_time\n          if @forever || !@secondary # primary\n            naive = naive_next_time(@steps)\n            if @forever\n              naive\n            elsif naive >= @timeout_at\n              @timeout_at\n            else\n              naive\n            end\n          elsif @current == :primary && @secondary\n            naive = naive_next_time(@steps)\n            if naive >= @secondary_transition_at\n              @secondary_transition_at\n            else\n              naive\n            end\n          elsif @current == :secondary\n            naive = naive_next_time(@steps - @secondary_transition_steps)\n            if naive >= @timeout_at\n              @timeout_at\n            else\n              naive\n            end\n          else\n            raise \"BUG: it's out of design\"\n          end\n        end\n\n        def naive_next_time(retry_times)\n          raise NotImplementedError\n        end\n\n        def secondary?\n          !@forever && @secondary && (@current == :secondary || current_time >= @secondary_transition_at)\n        end\n\n        def step\n          @steps += 1\n          if !@forever && @secondary && @current != :secondary && current_time >= @secondary_transition_at\n            @current = :secondary\n            @secondary_transition_steps = @steps\n          end\n\n          @next_time = calc_next_time\n\n          if @has_reached_timeout\n            @has_timed_out = @next_time >= @timeout_at\n          else\n            @has_reached_timeout = @next_time >= @timeout_at\n          end\n\n          nil\n        end\n\n        def recalc_next_time\n          @next_time = calc_next_time\n        end\n\n        def limit?\n          if @forever\n            false\n          else\n            @has_timed_out || !!(@max_steps && @steps >= @max_steps)\n          end\n        end\n      end\n\n      class ExponentialBackOffRetry < RetryStateMachine\n        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)\n          @constant_factor = wait\n          @backoff_base = backoff_base\n          @max_interval = max_interval\n\n          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)\n\n          @next_time = @start + @constant_factor\n        end\n\n        def naive_next_time(retry_next_times)\n          intr = calc_interval(retry_next_times)\n          current_time + randomize(intr)\n        end\n\n        def calc_max_retry_timeout(max_steps)\n          result = 0\n          max_steps.times { |i|\n            result += calc_interval(i)\n          }\n          result\n        end\n\n        def calc_interval(num)\n          interval = raw_interval(num)\n          if @max_interval && interval > @max_interval\n            @max_interval\n          else\n            if interval.finite?\n              interval\n            else\n              # Calculate previous finite value to avoid inf related errors. If this re-computing is heavy, use cache.\n              until interval.finite?\n                num -= 1\n                interval = raw_interval(num)\n              end\n              interval\n            end\n          end\n        end\n\n        def raw_interval(num)\n          @constant_factor.to_f * (@backoff_base ** (num))\n        end\n      end\n\n      class PeriodicRetry < RetryStateMachine\n        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)\n          @retry_wait = wait\n\n          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)\n\n          @next_time = @start + @retry_wait\n        end\n\n        def naive_next_time(retry_next_times)\n          current_time + randomize(@retry_wait)\n        end\n\n        def calc_max_retry_timeout(max_steps)\n          @retry_wait * max_steps\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/server.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin_helper/event_loop'\n\nrequire 'serverengine'\nrequire 'cool.io'\nrequire 'socket'\nrequire 'ipaddr'\nrequire 'fcntl'\nrequire 'openssl'\n\nrequire_relative 'socket_option'\nrequire_relative 'cert_option'\n\nmodule Fluent\n  module PluginHelper\n    module Server\n      include Fluent::PluginHelper::EventLoop\n      include Fluent::PluginHelper::SocketOption\n      include Fluent::PluginHelper::CertOption\n\n      # This plugin helper doesn't support these things for now:\n      # * TCP/TLS keepalive\n      # * TLS session cache/tickets\n      # * unix domain sockets\n\n      # stop     : [-]\n      # shutdown : detach server event handler from event loop (event_loop)\n      # close    : close listening sockets\n      # terminate: remote all server instances\n\n      attr_reader :_servers # for tests\n\n      def server_wait_until_start\n        # event_loop_wait_until_start works well for this\n      end\n\n      def server_wait_until_stop\n        sleep 0.1 while @_servers.any?{|si| si.server.attached? }\n        @_servers.each{|si| si.server.close rescue nil }\n      end\n\n      PROTOCOLS = [:tcp, :udp, :tls, :unix]\n      CONNECTION_PROTOCOLS = [:tcp, :tls, :unix]\n\n      # server_create_connection(:title, @port) do |conn|\n      #   # on connection\n      #   source_addr = conn.remote_host\n      #   source_port = conn.remote_port\n      #   conn.data do |data|\n      #     # on data\n      #     conn.write resp # ...\n      #     conn.close\n      #   end\n      # end\n      def server_create_connection(title, port, proto: nil, bind: '0.0.0.0', shared: true, backlog: nil, tls_options: nil, **socket_options, &block)\n        proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp\n\n        raise ArgumentError, \"BUG: title must be a symbol\" unless title && title.is_a?(Symbol)\n        raise ArgumentError, \"BUG: port must be an integer\" unless port && port.is_a?(Integer)\n        raise ArgumentError, \"BUG: invalid protocol name\" unless PROTOCOLS.include?(proto)\n        raise ArgumentError, \"BUG: cannot create connection for UDP\" unless CONNECTION_PROTOCOLS.include?(proto)\n\n        raise ArgumentError, \"BUG: tls_options is available only for tls\" if tls_options && proto != :tls\n\n        raise ArgumentError, \"BUG: block not specified which handles connection\" unless block_given?\n        raise ArgumentError, \"BUG: block must have just one argument\" unless block.arity == 1\n\n        if proto == :tcp || proto == :tls\n          socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0\n        end\n\n        socket_options[:receive_buffer_size] ||= @transport_config&.receive_buffer_size\n\n        socket_option_validate!(proto, **socket_options)\n        socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }\n\n        case proto\n        when :tcp\n          server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)\n        when :tls\n          transport_config = if tls_options\n                               server_create_transport_section_object(tls_options)\n                             elsif @transport_config && @transport_config.protocol == :tls\n                               @transport_config\n                             else\n                               raise ArgumentError, \"BUG: TLS transport specified, but certification options are not specified\"\n                             end\n          server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter, &block)\n        when :unix\n          raise \"not implemented yet\"\n        else\n          raise \"unknown protocol #{proto}\"\n        end\n\n        server_attach(title, proto, port, bind, shared, server)\n      end\n\n      # server_create(:title, @port) do |data|\n      #   # ...\n      # end\n      # server_create(:title, @port) do |data, conn|\n      #   # ...\n      # end\n      # server_create(:title, @port, proto: :udp, max_bytes: 2048) do |data, sock|\n      #   sock.remote_host\n      #   sock.remote_port\n      #   # ...\n      # end\n      def server_create(title, port, proto: nil, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, tls_options: nil, max_bytes: nil, flags: 0, **socket_options, &callback)\n        proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp\n\n        raise ArgumentError, \"BUG: title must be a symbol\" unless title && title.is_a?(Symbol)\n        raise ArgumentError, \"BUG: port must be an integer\" unless port && port.is_a?(Integer)\n        raise ArgumentError, \"BUG: invalid protocol name\" unless PROTOCOLS.include?(proto)\n\n        raise ArgumentError, \"BUG: socket option is available only for udp\" if socket && proto != :udp\n        raise ArgumentError, \"BUG: tls_options is available only for tls\" if tls_options && proto != :tls\n\n        raise ArgumentError, \"BUG: block not specified which handles received data\" unless block_given?\n        raise ArgumentError, \"BUG: block must have 1 or 2 arguments\" unless callback.arity == 1 || callback.arity == 2\n\n        if proto == :tcp || proto == :tls\n          socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0\n        end\n\n        socket_options[:receive_buffer_size] ||= @transport_config&.receive_buffer_size\n\n        unless socket\n          socket_option_validate!(proto, **socket_options)\n          socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }\n        end\n\n        if proto != :tcp && proto != :tls && proto != :unix # options to listen/accept connections\n          raise ArgumentError, \"BUG: backlog is available for tcp/tls\" if backlog\n        end\n        if proto != :udp # UDP options\n          raise ArgumentError, \"BUG: max_bytes is available only for udp\" if max_bytes\n          raise ArgumentError, \"BUG: flags is available only for udp\" if flags != 0\n        end\n\n        case proto\n        when :tcp\n          server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter) do |conn|\n            conn.data(&callback)\n          end\n        when :tls\n          transport_config = if tls_options\n                               server_create_transport_section_object(tls_options)\n                             elsif @transport_config && @transport_config.protocol == :tls\n                               @transport_config\n                             else\n                               raise ArgumentError, \"BUG: TLS transport specified, but certification options are not specified\"\n                             end\n          server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter) do |conn|\n            conn.data(&callback)\n          end\n        when :udp\n          raise ArgumentError, \"BUG: max_bytes must be specified for UDP\" unless max_bytes\n          if socket\n            sock = socket\n            close_socket = false\n          else\n            sock = server_create_udp_socket(shared, bind, port)\n            socket_option_setter.call(sock)\n            close_socket = true\n          end\n          server = EventHandler::UDPServer.new(sock, max_bytes, flags, close_socket, @log, @under_plugin_development, &callback)\n        when :unix\n          raise \"not implemented yet\"\n        else\n          raise \"BUG: unknown protocol #{proto}\"\n        end\n\n        server_attach(title, proto, port, bind, shared, server)\n      end\n\n      def server_create_tcp(title, port, **kwargs, &callback)\n        server_create(title, port, proto: :tcp, **kwargs, &callback)\n      end\n\n      def server_create_udp(title, port, **kwargs, &callback)\n        server_create(title, port, proto: :udp, **kwargs, &callback)\n      end\n\n      def server_create_tls(title, port, **kwargs, &callback)\n        server_create(title, port, proto: :tls, **kwargs, &callback)\n      end\n\n      def server_create_unix(title, port, **kwargs, &callback)\n        server_create(title, port, proto: :unix, **kwargs, &callback)\n      end\n\n      ServerInfo = Struct.new(:title, :proto, :port, :bind, :shared, :server)\n\n      def server_attach(title, proto, port, bind, shared, server)\n        @_servers << ServerInfo.new(title, proto, port, bind, shared, server)\n        event_loop_attach(server)\n      end\n\n      def server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)\n        sock = server_create_tcp_socket(shared, bind, port)\n        socket_option_setter.call(sock)\n        close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }\n        server = Coolio::TCPServer.new(sock, nil, EventHandler::TCPServer, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|\n          unless conn.closing\n            @_server_mutex.synchronize do\n              @_server_connections << conn\n            end\n          end\n        end\n        server.listen(backlog) if backlog\n        server\n      end\n\n      def server_create_for_tls_connection(shared, bind, port, conf, backlog, socket_option_setter, &block)\n        context = cert_option_create_context(conf.version, conf.insecure, conf.ciphers, conf)\n        sock = server_create_tcp_socket(shared, bind, port)\n        socket_option_setter.call(sock)\n        close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }\n        server = Coolio::TCPServer.new(sock, nil, EventHandler::TLSServer, context, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|\n          unless conn.closing\n            @_server_mutex.synchronize do\n              @_server_connections << conn\n            end\n          end\n        end\n        server.listen(backlog) if backlog\n        server\n      end\n\n      SERVER_TRANSPORT_PARAMS = [\n        :protocol, :version, :min_version, :max_version, :ciphers, :insecure,\n        :ca_path, :cert_path, :private_key_path, :private_key_passphrase, :client_cert_auth,\n        :ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,\n        :cert_verifier, :generate_private_key_length,\n        :generate_cert_country, :generate_cert_state, :generate_cert_state,\n        :generate_cert_locality, :generate_cert_common_name,\n        :generate_cert_expiration, :generate_cert_digest,\n        :ensure_fips,\n      ]\n\n      def server_create_transport_section_object(opts)\n        transport_section = configured_section_create(:transport)\n        SERVER_TRANSPORT_PARAMS.each do |param|\n          if opts.has_key?(param)\n            transport_section[param] = opts[param]\n          end\n        end\n        transport_section\n      end\n\n      module ServerTransportParams\n        include Fluent::Configurable\n        config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do\n          config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp\n\n          ### Socket Params ###\n\n          desc \"The max size of socket receive buffer. SO_RCVBUF\"\n          config_param :receive_buffer_size, :size, default: nil\n\n          # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.\n          # Set positive value if needing to send FIN on closing on non-Windows.\n          # (On Windows, Fluentd can send FIN with zero `linger_timeout` since Fluentd doesn't set 0 to SO_LINGER on Windows.\n          # See `socket_option.rb`.)\n          # NOTE:\n            # Socket-options can be specified from each plugin as needed, so most of them is not defined here for now.\n            # This is because there is no positive reason to do so.\n            # `linger_timeout` option in particular needs to be defined here\n            # although it can be specified from each plugin as well.\n            # This is because this helper fixes the default value to `0` for its own reason\n            # and it has a critical effect on the behavior.\n          desc 'The timeout time used to set linger option.'\n          config_param :linger_timeout, :integer, default: 0\n\n          ### TLS Params ###\n\n          config_param :version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION\n          config_param :min_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil\n          config_param :max_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil\n          config_param :ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT\n          config_param :insecure, :bool, default: false\n          config_param :ensure_fips, :bool, default: false\n\n          # Cert signed by public CA\n          config_param :ca_path, :string, default: nil\n          config_param :cert_path, :string, default: nil\n          config_param :private_key_path, :string, default: nil\n          config_param :private_key_passphrase, :string, default: nil, secret: true\n          config_param :client_cert_auth, :bool, default: false\n\n          # Cert generated and signed by private CA Certificate\n          config_param :ca_cert_path, :string, default: nil\n          config_param :ca_private_key_path, :string, default: nil\n          config_param :ca_private_key_passphrase, :string, default: nil, secret: true\n\n          config_param :cert_verifier, :string, default: nil\n\n          # Options for generating certs by private CA certs or self-signed\n          config_param :generate_private_key_length, :integer, default: 2048\n          config_param :generate_cert_country, :string, default: 'US'\n          config_param :generate_cert_state, :string, default: 'CA'\n          config_param :generate_cert_locality, :string, default: 'Mountain View'\n          config_param :generate_cert_common_name, :string, default: nil\n          config_param :generate_cert_expiration, :time, default: 10 * 365 * 86400 # 10years later\n          config_param :generate_cert_digest, :enum, list: [:sha1, :sha256, :sha384, :sha512], default: :sha256\n        end\n      end\n\n      def self.included(mod)\n        mod.include ServerTransportParams\n      end\n\n      def initialize\n        super\n        @_servers = []\n        @_server_connections = []\n        @_server_mutex = Mutex.new\n      end\n\n      def configure(conf)\n        super\n\n        if @transport_config\n          if @transport_config.protocol == :tls\n            cert_option_server_validate!(@transport_config)\n          end\n        end\n      end\n\n      def stop\n        @_server_mutex.synchronize do\n          @_servers.each do |si|\n            si.server.detach if si.server.attached?\n            # to refuse more connections: (connected sockets are still alive here)\n            si.server.close rescue nil\n          end\n        end\n\n        super\n      end\n\n      def shutdown\n        # When it invokes conn.cose, it reduces elements in @_server_connections by close_callback,\n        # and it reduces the number of loops. This prevents the connection closing.\n        # So, it requires invoking #dup to avoid the problem.\n        @_server_connections.dup.each do |conn|\n          conn.close rescue nil\n        end\n\n        super\n      end\n\n      def terminate\n        @_servers = []\n        super\n      end\n\n      def server_socket_manager_client\n        socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']\n        if Fluent.windows?\n          socket_manager_path = socket_manager_path.to_i\n        end\n        ServerEngine::SocketManager::Client.new(socket_manager_path)\n      end\n\n      def server_create_tcp_socket(shared, bind, port)\n        sock = if shared\n                 server_socket_manager_client.listen_tcp(bind, port)\n               else\n                 # TCPServer.new doesn't set IPV6_V6ONLY flag, so use Addrinfo class instead.\n                 # backlog will be set by the caller, we don't need to set backlog here\n                 tsock = Addrinfo.tcp(bind, port).listen\n                 tsock.autoclose = false\n                 TCPServer.for_fd(tsock.fileno)\n               end\n        # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)\n        sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock\n        sock\n      end\n\n      def server_create_udp_socket(shared, bind, port)\n        sock = if shared\n                 server_socket_manager_client.listen_udp(bind, port)\n               else\n                 # UDPSocket.new doesn't set IPV6_V6ONLY flag, so use Addrinfo class instead.\n                 usock = Addrinfo.udp(bind, port).bind\n                 usock.autoclose = false\n                 UDPSocket.for_fd(usock.fileno)\n               end\n        # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)\n        sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock\n        sock\n      end\n\n      # Use string \"?\" for port, not integer or nil. \"?\" is clear than -1 or nil in the log.\n      PEERADDR_FAILED = [\"?\", \"?\", \"name resolution failed\", \"?\"]\n\n      class CallbackSocket\n        def initialize(server_type, sock, enabled_events = [], close_socket: true)\n          @server_type = server_type\n          @sock = sock\n          @peeraddr = nil\n          @enabled_events = enabled_events\n          @close_socket = close_socket\n        end\n\n        def remote_addr\n          @peeraddr[3]\n        end\n\n        def remote_host\n          @peeraddr[2]\n        end\n\n        def remote_port\n          @peeraddr[1]\n        end\n\n        def send(data, flags = 0)\n          @sock.send(data, flags)\n        end\n\n        def write(data)\n          raise \"not implemented here\"\n        end\n\n        def close_after_write_complete\n          @sock.close_after_write_complete = true\n        end\n\n        def close\n          @sock.close if @close_socket\n        end\n\n        def data(&callback)\n          on(:data, &callback)\n        end\n\n        def on(event, &callback)\n          raise \"BUG: this event is disabled for #{@server_type}: #{event}\" unless @enabled_events.include?(event)\n          case event\n          when :data\n            @sock.data(&callback)\n          when :write_complete\n            cb = ->(){ callback.call(self) }\n            @sock.on_write_complete(&cb)\n          when :close\n            cb = ->(){ callback.call(self) }\n            @sock.on_close(&cb)\n          else\n            raise \"BUG: unknown event: #{event}\"\n          end\n        end\n      end\n\n      class TCPCallbackSocket < CallbackSocket\n        ENABLED_EVENTS = [:data, :write_complete, :close]\n\n        attr_accessor :buffer\n\n        def initialize(sock)\n          super(\"tcp\", sock, ENABLED_EVENTS)\n          @peeraddr = (@sock.peeraddr rescue PEERADDR_FAILED)\n          @buffer = ''\n        end\n\n        def write(data)\n          @sock.write(data)\n        end\n      end\n\n      class TLSCallbackSocket < CallbackSocket\n        ENABLED_EVENTS = [:data, :write_complete, :close]\n\n        attr_accessor :buffer\n\n        def initialize(sock)\n          super(\"tls\", sock, ENABLED_EVENTS)\n          @peeraddr = (@sock.to_io.peeraddr rescue PEERADDR_FAILED)\n          @buffer = ''\n        end\n\n        def write(data)\n          @sock.write(data)\n        end\n      end\n\n      class UDPCallbackSocket < CallbackSocket\n        ENABLED_EVENTS = []\n\n        def initialize(sock, peeraddr, **kwargs)\n          super(\"udp\", sock, ENABLED_EVENTS, **kwargs)\n          @peeraddr = peeraddr\n        end\n\n        def remote_addr\n          @peeraddr[3]\n        end\n\n        def remote_host\n          @peeraddr[2]\n        end\n\n        def remote_port\n          @peeraddr[1]\n        end\n\n        def write(data)\n          @sock.send(data, 0, @peeraddr[3], @peeraddr[1])\n        end\n      end\n\n      module EventHandler\n        class UDPServer < Coolio::IO\n          attr_writer :close_after_write_complete # dummy for consistent method call in callbacks\n\n          def initialize(sock, max_bytes, flags, close_socket, log, under_plugin_development, &callback)\n            raise ArgumentError, \"socket must be a UDPSocket: sock = #{sock}\" unless sock.is_a?(UDPSocket)\n\n            super(sock)\n\n            @sock = sock\n            @max_bytes = max_bytes\n            @flags = flags\n            @close_socket = close_socket\n            @log = log\n            @under_plugin_development = under_plugin_development\n            @callback = callback\n\n            on_readable_impl = case @callback.arity\n                               when 1 then :on_readable_without_sock\n                               when 2 then :on_readable_with_sock\n                               else\n                                 raise \"BUG: callback block must have 1 or 2 arguments\"\n                               end\n            self.define_singleton_method(:on_readable, method(on_readable_impl))\n          end\n\n          def on_readable_without_sock\n            begin\n              data = @sock.recv(@max_bytes, @flags)\n            rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF\n              return\n            rescue Errno::EMSGSIZE\n              # Windows ONLY: This happens when the data size is larger than `@max_bytes`.\n              @log.info \"A received data was ignored since it was too large.\"\n              return\n            end\n            @callback.call(data)\n          rescue => e\n            @log.error \"unexpected error in processing UDP data\", error: e\n            @log.error_backtrace\n            raise if @under_plugin_development\n          end\n\n          def on_readable_with_sock\n            begin\n              data, addr = @sock.recvfrom(@max_bytes)\n            rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF\n              return\n            rescue Errno::EMSGSIZE\n              # Windows ONLY: This happens when the data size is larger than `@max_bytes`.\n              @log.info \"A received data was ignored since it was too large.\"\n              return\n            end\n            @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))\n          rescue => e\n            @log.error \"unexpected error in processing UDP data\", error: e\n            @log.error_backtrace\n            raise if @under_plugin_development\n          end\n        end\n\n        class TCPServer < Coolio::TCPSocket\n          attr_reader :closing\n          attr_writer :close_after_write_complete\n\n          def initialize(sock, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)\n            raise ArgumentError, \"socket must be a TCPSocket: sock=#{sock}\" unless sock.is_a?(TCPSocket)\n\n            socket_option_setter.call(sock)\n\n            @_handler_socket = sock\n            super(sock)\n\n            @log = log\n            @under_plugin_development = under_plugin_development\n\n            @connect_callback = connect_callback\n            @data_callback = nil\n            @close_callback = close_callback\n\n            @callback_connection = nil\n            @close_after_write_complete = false\n            @closing = false\n\n            @mutex = Mutex.new # to serialize #write and #close\n          end\n\n          def to_io\n            @_handler_socket\n          end\n\n          def data(&callback)\n            raise \"data callback can be registered just once, but registered twice\" if self.singleton_methods.include?(:on_read)\n            @data_callback = callback\n            on_read_impl = case callback.arity\n                           when 1 then :on_read_without_connection\n                           when 2 then :on_read_with_connection\n                           else\n                             raise \"BUG: callback block must have 1 or 2 arguments\"\n                           end\n            self.define_singleton_method(:on_read, method(on_read_impl))\n          end\n\n          def write(data)\n            @mutex.synchronize do\n              super\n            end\n          end\n\n          def on_writable\n            super\n            close if @close_after_write_complete\n          end\n\n          def on_connect\n            @callback_connection = TCPCallbackSocket.new(self)\n            @connect_callback.call(@callback_connection)\n            unless @data_callback\n              raise \"connection callback must call #data to set data callback\"\n            end\n          end\n\n          def on_read_without_connection(data)\n            @data_callback.call(data)\n          rescue => e\n            @log.error \"unexpected error on reading data\", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e\n            @log.error_backtrace\n            close rescue nil\n            raise if @under_plugin_development\n          end\n\n          def on_read_with_connection(data)\n            @data_callback.call(data, @callback_connection)\n          rescue => e\n            @log.error \"unexpected error on reading data\", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e\n            @log.error_backtrace\n            close rescue nil\n            raise if @under_plugin_development\n          end\n\n          def close\n            @mutex.synchronize do\n              return if @closing\n              @closing = true\n              @close_callback.call(self)\n              super\n            end\n          end\n        end\n\n        class TLSServer < Coolio::Socket\n          attr_reader :closing\n          attr_writer :close_after_write_complete\n\n          # It can't use Coolio::TCPSocket, because Coolio::TCPSocket checks that underlying socket (1st argument of super) is TCPSocket.\n          def initialize(sock, context, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)\n            raise ArgumentError, \"socket must be a TCPSocket: sock=#{sock}\" unless sock.is_a?(TCPSocket)\n\n            socket_option_setter.call(sock)\n            @_handler_socket = OpenSSL::SSL::SSLSocket.new(sock, context)\n            @_handler_socket.sync_close = true\n            @_handler_write_buffer = ''.force_encoding('ascii-8bit')\n            @_handler_accepted = false\n            super(@_handler_socket)\n\n            @log = log\n            @under_plugin_development = under_plugin_development\n\n            @connect_callback = connect_callback\n            @data_callback = nil\n            @close_callback = close_callback\n\n            @callback_connection = nil\n            @close_after_write_complete = false\n            @closing = false\n\n            @mutex = Mutex.new # to serialize #write and #close\n          end\n\n          def to_io\n            @_handler_socket.to_io\n          end\n\n          def data(&callback)\n            raise \"data callback can be registered just once, but registered twice\" if self.singleton_methods.include?(:on_read)\n            @data_callback = callback\n            on_read_impl = case callback.arity\n                           when 1 then :on_read_without_connection\n                           when 2 then :on_read_with_connection\n                           else\n                             raise \"BUG: callback block must have 1 or 2 arguments\"\n                           end\n            self.define_singleton_method(:on_read, method(on_read_impl))\n          end\n\n          def write(data)\n            @mutex.synchronize do\n              @_handler_write_buffer << data\n              schedule_write\n              data.bytesize\n            end\n          end\n\n          def try_tls_accept\n            return true if @_handler_accepted\n\n            begin\n              result = @_handler_socket.accept_nonblock(exception: false) # this method call actually try to do handshake via TLS\n              if result == :wait_readable || result == :wait_writable\n                # retry accept_nonblock: there aren't enough data in underlying socket buffer\n              else\n                @_handler_accepted = true\n\n                @callback_connection = TLSCallbackSocket.new(self)\n                @connect_callback.call(@callback_connection)\n                unless @data_callback\n                  raise \"connection callback must call #data to set data callback\"\n                end\n\n                return true\n              end\n            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e\n              peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)\n              @log.trace \"unexpected error before accepting TLS connection\",\n                         addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e\n              close rescue nil\n            rescue OpenSSL::SSL::SSLError => e\n              peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)\n              # Use same log level as on_readable\n              @log.warn \"unexpected error before accepting TLS connection by OpenSSL\",\n                        addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e\n              close rescue nil\n            end\n\n            false\n          end\n\n          def on_connect\n            try_tls_accept\n          end\n\n          def on_readable\n            if try_tls_accept\n              super\n            end\n          rescue IO::WaitReadable, IO::WaitWritable\n            # ignore and return with doing nothing\n          rescue OpenSSL::SSL::SSLError => e\n            @log.warn \"close socket due to unexpected ssl error: #{e}\"\n            close rescue nil\n          end\n\n          def on_writable\n            begin\n              @mutex.synchronize do\n                # Consider write_nonblock with {exception: false} when IO::WaitWritable error happens frequently.\n                written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)\n                @_handler_write_buffer.slice!(0, written_bytes)\n              end\n\n              # No need to call `super` in a synchronized context because TLSServer doesn't use the inner buffer(::IO::Buffer) of Coolio::IO.\n              # Instead of using Coolio::IO's inner buffer, TLSServer has own buffer(`@_handler_write_buffer`). See also TLSServer#write.\n              # Actually, the only reason calling `super` here is call Coolio::IO#disable_write_watcher.\n              # If `super` is called in a synchronized context, it could cause a mutex recursive locking since Coolio::IO#on_write_complete\n              # eventually calls TLSServer#close which try to get a lock.\n              super\n\n              close if @close_after_write_complete\n            rescue IO::WaitWritable, IO::WaitReadable\n              return\n            rescue Errno::EINTR\n              return\n            rescue SystemCallError, IOError, SocketError\n              # SystemCallError catches Errno::EPIPE & Errno::ECONNRESET amongst others.\n              close rescue nil\n              return\n            rescue OpenSSL::SSL::SSLError => e\n              @log.debug \"unexpected SSLError while writing data into socket connected via TLS\", error: e\n            end\n          end\n\n          def on_read_without_connection(data)\n            @data_callback.call(data)\n          rescue => e\n            @log.error \"unexpected error on reading data\", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e\n            @log.error_backtrace\n            close rescue nil\n            raise if @under_plugin_development\n          end\n\n          def on_read_with_connection(data)\n            @data_callback.call(data, @callback_connection)\n          rescue => e\n            @log.error \"unexpected error on reading data\", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e\n            @log.error_backtrace\n            close rescue nil\n            raise if @under_plugin_development\n          end\n\n          def close\n            @mutex.synchronize do\n              return if @closing\n              @closing = true\n              @close_callback.call(self)\n              super\n            end\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/service_discovery/manager.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/service_discovery'\nrequire 'fluent/plugin_helper/service_discovery/round_robin_balancer'\n\nmodule Fluent\n  module PluginHelper\n    module ServiceDiscovery\n      class Manager\n        def initialize(log:, load_balancer: nil, custom_build_method: nil)\n          @log = log\n          @load_balancer = load_balancer || RoundRobinBalancer.new\n          @custom_build_method = custom_build_method\n\n          @discoveries = []\n          @services = {}\n          @queue = Queue.new\n          @static_config = true\n        end\n\n        def configure(configs, parent: nil)\n          configs.each do |config|\n            type, conf = if config.has_key?(:conf) # for compatibility with initial API\n                           [config[:type], config[:conf]]\n                         else\n                           [config['@type'], config]\n                         end\n\n            sd = Fluent::Plugin.new_sd(type, parent: parent)\n            sd.configure(conf)\n\n            sd.services.each do |s|\n              @services[s.discovery_id] = build_service(s)\n            end\n            @discoveries << sd\n\n            if @static_config && type.to_sym != :static\n              @static_config = false\n            end\n          end\n\n          rebalance\n        end\n\n        def static_config?\n          @static_config\n        end\n\n        def start\n          @discoveries.each do |d|\n            d.start(@queue)\n          end\n        end\n\n        %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|\n          define_method(mth) do\n            @discoveries.each do |d|\n              d.__send__(mth)\n            end\n          end\n        end\n\n        def run_once\n          # Don't care race in this loop intentionally\n          s = @queue.size\n\n          if s == 0\n            return\n          end\n\n          s.times do\n            msg = @queue.pop\n\n            unless msg.is_a?(Fluent::Plugin::ServiceDiscovery::DiscoveryMessage)\n              @log.warn(\"BUG: #{msg}\")\n              next\n            end\n\n            begin\n              handle_message(msg)\n            rescue => e\n              @log.error(e)\n            end\n          end\n\n          rebalance\n        end\n\n        def rebalance\n          @load_balancer.rebalance(services)\n        end\n\n        def select_service(&block)\n          @load_balancer.select_service(&block)\n        end\n\n        def services\n          @services.values\n        end\n\n        private\n\n        def handle_message(msg)\n          service = msg.service\n\n          case msg.type\n          when Fluent::Plugin::ServiceDiscovery::SERVICE_IN\n            if (n = build_service(service))\n              @log.info(\"Service in: name=#{service.name} #{service.host}:#{service.port}\")\n              @services[service.discovery_id] = n\n            else\n              raise \"failed to build service in name=#{service.name} #{service.host}:#{service.port}\"\n            end\n          when Fluent::Plugin::ServiceDiscovery::SERVICE_OUT\n            s = @services.delete(service.discovery_id)\n            if s\n              @log.info(\"Service out: name=#{service.name} #{service.host}:#{service.port}\")\n            else\n              @log.warn(\"Not found service: name=#{service.name} #{service.host}:#{service.port}\")\n            end\n          else\n            @log.error(\"BUG: unknow message type: #{msg.type}\")\n          end\n        end\n\n        def build_service(n)\n          @custom_build_method ? @custom_build_method.call(n) : n\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module PluginHelper\n    module ServiceDiscovery\n      class RoundRobinBalancer\n        def initialize\n          @services = []\n          @mutex = Mutex.new\n        end\n\n        def rebalance(services)\n          @mutex.synchronize do\n            @services = services\n          end\n        end\n\n        def select_service\n          s = @mutex.synchronize do\n            s = @services.shift\n            @services.push(s)\n            s\n          end\n          yield(s)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/service_discovery.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin_helper/timer'\nrequire 'fluent/plugin_helper/service_discovery/manager'\n\nmodule Fluent\n  module PluginHelper\n    module ServiceDiscovery\n      include Fluent::PluginHelper::Timer\n\n      # For the compatibility with older versions without `param_name: :service_discovery_configs`\n      attr_reader :service_discovery\n\n      def self.included(mod)\n        mod.include ServiceDiscoveryParams\n      end\n\n      def configure(conf)\n        super\n        # For the compatibility with older versions without `param_name: :service_discovery_configs`\n        @service_discovery = @service_discovery_configs\n      end\n\n      def start\n        unless @discovery_manager\n          log.warn('There is no discovery_manager. skip start them')\n          super\n          return\n        end\n\n        @discovery_manager.start\n        unless @discovery_manager.static_config?\n          timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_interval) do\n            @discovery_manager.run_once\n          end\n        end\n\n        super\n      end\n\n      %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|\n        define_method(mth) do\n          @discovery_manager&.__send__(mth)\n          super()\n        end\n      end\n\n      private\n\n      # @param title [Symbol] the thread name. this value should be unique.\n      # @param static_default_service_directive [String] the directive name of each service when \"static\" service discovery is enabled in default\n      # @param load_balancer [Object] object which has two methods #rebalance and #select_service\n      # @param custom_build_method [Proc]\n      def service_discovery_configure(title, static_default_service_directive: nil, load_balancer: nil, custom_build_method: nil, interval: 3)\n        configs = @service_discovery_configs.map(&:corresponding_config_element)\n        if static_default_service_directive\n          configs.prepend Fluent::Config::Element.new(\n            'service_discovery',\n            '',\n            {'@type' => 'static'},\n            @config.elements(name: static_default_service_directive.to_s).map{|e| Fluent::Config::Element.new('service', e.arg, e.dup, e.elements, e.unused) }\n          )\n        end\n        service_discovery_create_manager(title, configurations: configs, load_balancer: load_balancer, custom_build_method: custom_build_method, interval: interval)\n      end\n\n      def service_discovery_select_service(&block)\n        @discovery_manager.select_service(&block)\n      end\n\n      def service_discovery_services\n        @discovery_manager.services\n      end\n\n      def service_discovery_rebalance\n        @discovery_manager.rebalance\n      end\n\n      # @param title [Symbol] the thread name. this value should be unique.\n      # @param configurations [Hash] hash which must has discovery_service type and its configuration like `{ type: :static, conf: <Fluent::Config::Element> }`\n      # @param load_balancer [Object] object which has two methods #rebalance and #select_service\n      # @param custom_build_method [Proc]\n      def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)\n        @_plugin_helper_service_discovery_title = title\n        @_plugin_helper_service_discovery_interval = interval\n\n        @discovery_manager = Fluent::PluginHelper::ServiceDiscovery::Manager.new(\n          log: log,\n          load_balancer: load_balancer,\n          custom_build_method: custom_build_method,\n        )\n\n        @discovery_manager.configure(configurations, parent: self)\n\n        @discovery_manager\n      end\n\n      def discovery_manager\n        @discovery_manager\n      end\n\n      module ServiceDiscoveryParams\n        include Fluent::Configurable\n\n        config_section :service_discovery, multi: true, param_name: :service_discovery_configs do\n          config_param :@type, :string\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/socket.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'socket'\nrequire 'ipaddr'\nrequire 'openssl'\nif Fluent.windows?\n  require 'certstore'\nend\n\nrequire 'fluent/tls'\nrequire_relative 'socket_option'\n\nmodule Fluent\n  module PluginHelper\n    module Socket\n      # stop     : [-]\n      # shutdown : [-]\n      # close    : [-]\n      # terminate: [-]\n\n      include Fluent::PluginHelper::SocketOption\n\n      attr_reader :_sockets # for tests\n\n      # TODO: implement connection pool for specified host\n\n      def socket_create(proto, host, port, **kwargs, &block)\n        case proto\n        when :tcp\n          socket_create_tcp(host, port, **kwargs, &block)\n        when :udp\n          socket_create_udp(host, port, **kwargs, &block)\n        when :tls\n          socket_create_tls(host, port, **kwargs, &block)\n        when :unix\n          raise \"not implemented yet\"\n        else\n          raise ArgumentError, \"invalid protocol: #{proto}\"\n        end\n      end\n\n      def socket_create_tcp(host, port, resolve_name: false, connect_timeout: nil, **kwargs, &block)\n        sock = if connect_timeout\n                 s = ::Socket.tcp(host, port, connect_timeout: connect_timeout)\n                 s.autoclose = false # avoid GC triggered close\n                 WrappedSocket::TCP.for_fd(s.fileno)\n               else\n                 WrappedSocket::TCP.new(host, port)\n               end\n        socket_option_set(sock, resolve_name: resolve_name, **kwargs)\n        if block\n          begin\n            block.call(sock)\n          ensure\n            sock.close_write rescue nil\n            sock.close rescue nil\n          end\n        else\n          sock\n        end\n      end\n\n      def socket_create_udp(host, port, resolve_name: false, connect: false, **kwargs, &block)\n        family = IPAddr.new(IPSocket.getaddress(host)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6\n        sock = WrappedSocket::UDP.new(family)\n        socket_option_set(sock, resolve_name: resolve_name, **kwargs)\n        sock.connect(host, port) if connect\n        if block\n          begin\n            block.call(sock)\n          ensure\n            sock.close rescue nil\n          end\n        else\n          sock\n        end\n      end\n\n      def socket_create_tls(\n          host, port,\n          version: Fluent::TLS::DEFAULT_VERSION, min_version: nil, max_version: nil, ciphers: Fluent::TLS::CIPHERS_DEFAULT, insecure: false, verify_fqdn: true, fqdn: nil,\n          enable_system_cert_store: true, allow_self_signed_cert: false, cert_paths: nil,\n          cert_path: nil, private_key_path: nil, private_key_passphrase: nil,\n          cert_thumbprint: nil, cert_logical_store_name: nil, cert_use_enterprise_store: true,\n          connect_timeout: nil,\n          **kwargs, &block)\n\n        host_is_ipaddress = IPAddr.new(host) rescue false\n        fqdn ||= host unless host_is_ipaddress\n\n        context = OpenSSL::SSL::SSLContext.new\n\n        if insecure\n          log.trace \"setting TLS verify_mode NONE\"\n          context.verify_mode = OpenSSL::SSL::VERIFY_NONE\n        else\n          cert_store = OpenSSL::X509::Store.new\n          if allow_self_signed_cert && OpenSSL::X509.const_defined?('V_FLAG_CHECK_SS_SIGNATURE')\n            cert_store.flags = OpenSSL::X509::V_FLAG_CHECK_SS_SIGNATURE\n          end\n          begin\n            if enable_system_cert_store\n              if Fluent.windows? && cert_logical_store_name\n                log.trace \"loading Windows system certificate store\"\n                loader = Certstore::OpenSSL::Loader.new(log, cert_store, cert_logical_store_name,\n                                                        enterprise: cert_use_enterprise_store)\n                loader.load_cert_store\n                cert_store = loader.cert_store\n                context.cert = loader.get_certificate(cert_thumbprint) if cert_thumbprint\n              end\n              log.trace \"loading system default certificate store\"\n              cert_store.set_default_paths\n            end\n          rescue OpenSSL::X509::StoreError\n            log.warn \"failed to load system default certificate store\", error: e\n          end\n          if cert_paths\n            if cert_paths.respond_to?(:each)\n              cert_paths.each do |cert_path|\n                log.trace \"adding CA cert\", path: cert_path\n                cert_store.add_file(cert_path)\n              end\n            else\n              cert_path = cert_paths\n              log.trace \"adding CA cert\", path: cert_path\n              cert_store.add_file(cert_path)\n            end\n          end\n\n          log.trace \"setting TLS context\", mode: \"peer\", ciphers: ciphers\n          context.set_params({})\n          context.ciphers = ciphers\n          context.verify_mode = OpenSSL::SSL::VERIFY_PEER\n          context.cert_store = cert_store\n          context.verify_hostname = verify_fqdn && fqdn\n          context.key = OpenSSL::PKey::read(File.read(private_key_path), private_key_passphrase) if private_key_path\n\n          if cert_path\n            certs = socket_certificates_from_file(cert_path)\n            context.cert = certs.shift\n            unless certs.empty?\n              context.extra_chain_cert = certs\n            end\n          end\n        end\n        Fluent::TLS.set_version_to_context(context, version, min_version, max_version)\n\n        tcpsock = socket_create_tcp(host, port, connect_timeout: connect_timeout, **kwargs)\n        sock = WrappedSocket::TLS.new(tcpsock, context)\n        sock.sync_close = true\n        sock.hostname = fqdn if verify_fqdn && fqdn && sock.respond_to?(:hostname=)\n\n        log.trace \"entering TLS handshake\"\n        if connect_timeout\n          begin\n            Timeout.timeout(connect_timeout) { sock.connect }\n          rescue Timeout::Error\n            log.warn \"timeout while connecting tls session\", host: host\n            sock.close rescue nil\n            raise\n          end\n        else\n          sock.connect\n        end\n\n        begin\n          if verify_fqdn\n            log.trace \"checking peer's certificate\", subject: sock.peer_cert.subject\n            sock.post_connection_check(fqdn)\n            verify = sock.verify_result\n            if verify != OpenSSL::X509::V_OK\n              err_name = Socket.tls_verify_result_name(verify)\n              log.warn \"BUG: failed to verify certification while connecting (but not raised, why?)\", host: host, fqdn: fqdn, error: err_name\n              raise RuntimeError, \"BUG: failed to verify certification and to handle it correctly while connecting host #{host} as #{fqdn}\"\n            end\n          end\n        rescue OpenSSL::SSL::SSLError => e\n          log.warn \"failed to verify certification while connecting tls session\", host: host, fqdn: fqdn, error: e\n          raise\n        end\n\n        if block\n          begin\n            block.call(sock)\n          ensure\n            sock.close rescue nil\n          end\n        else\n          sock\n        end\n      end\n\n      def socket_certificates_from_file(path)\n        data = File.read(path)\n        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\\r?\\n(?:[^-]*\\r?\\n)+-+END CERTIFICATE-+\\r?\\n?', Regexp::MULTILINE)\n        list = []\n        data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }\n        if list.length == 0\n          raise Fluent::ConfigError, \"cert_path does not contain a valid certificate\"\n        end\n        list\n      end\n\n      def self.tls_verify_result_name(code)\n        case code\n        when OpenSSL::X509::V_OK then 'V_OK'\n        when OpenSSL::X509::V_ERR_AKID_SKID_MISMATCH then 'V_ERR_AKID_SKID_MISMATCH'\n        when OpenSSL::X509::V_ERR_APPLICATION_VERIFICATION then 'V_ERR_APPLICATION_VERIFICATION'\n        when OpenSSL::X509::V_ERR_CERT_CHAIN_TOO_LONG then 'V_ERR_CERT_CHAIN_TOO_LONG'\n        when OpenSSL::X509::V_ERR_CERT_HAS_EXPIRED then 'V_ERR_CERT_HAS_EXPIRED'\n        when OpenSSL::X509::V_ERR_CERT_NOT_YET_VALID then 'V_ERR_CERT_NOT_YET_VALID'\n        when OpenSSL::X509::V_ERR_CERT_REJECTED then 'V_ERR_CERT_REJECTED'\n        when OpenSSL::X509::V_ERR_CERT_REVOKED then 'V_ERR_CERT_REVOKED'\n        when OpenSSL::X509::V_ERR_CERT_SIGNATURE_FAILURE then 'V_ERR_CERT_SIGNATURE_FAILURE'\n        when OpenSSL::X509::V_ERR_CERT_UNTRUSTED then 'V_ERR_CERT_UNTRUSTED'\n        when OpenSSL::X509::V_ERR_CRL_HAS_EXPIRED then 'V_ERR_CRL_HAS_EXPIRED'\n        when OpenSSL::X509::V_ERR_CRL_NOT_YET_VALID then 'V_ERR_CRL_NOT_YET_VALID'\n        when OpenSSL::X509::V_ERR_CRL_SIGNATURE_FAILURE then 'V_ERR_CRL_SIGNATURE_FAILURE'\n        when OpenSSL::X509::V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT then 'V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT'\n        when OpenSSL::X509::V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD then 'V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD'\n        when OpenSSL::X509::V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD then 'V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD'\n        when OpenSSL::X509::V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD then 'V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD'\n        when OpenSSL::X509::V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD then 'V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD'\n        when OpenSSL::X509::V_ERR_INVALID_CA then 'V_ERR_INVALID_CA'\n        when OpenSSL::X509::V_ERR_INVALID_PURPOSE then 'V_ERR_INVALID_PURPOSE'\n        when OpenSSL::X509::V_ERR_KEYUSAGE_NO_CERTSIGN then 'V_ERR_KEYUSAGE_NO_CERTSIGN'\n        when OpenSSL::X509::V_ERR_OUT_OF_MEM then 'V_ERR_OUT_OF_MEM'\n        when OpenSSL::X509::V_ERR_PATH_LENGTH_EXCEEDED then 'V_ERR_PATH_LENGTH_EXCEEDED'\n        when OpenSSL::X509::V_ERR_SELF_SIGNED_CERT_IN_CHAIN then 'V_ERR_SELF_SIGNED_CERT_IN_CHAIN'\n        when OpenSSL::X509::V_ERR_SUBJECT_ISSUER_MISMATCH then 'V_ERR_SUBJECT_ISSUER_MISMATCH'\n        when OpenSSL::X509::V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY then 'V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY'\n        when OpenSSL::X509::V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE then 'V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY'\n        when OpenSSL::X509::V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE then 'V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE'\n        when OpenSSL::X509::V_ERR_UNABLE_TO_GET_CRL then 'V_ERR_UNABLE_TO_GET_CRL'\n        when OpenSSL::X509::V_ERR_UNABLE_TO_GET_ISSUER_CERT then 'V_ERR_UNABLE_TO_GET_ISSUER_CERT'\n        when OpenSSL::X509::V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY then 'V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY'\n        when OpenSSL::X509::V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE then 'V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE'\n        end\n      end\n\n      # socket_create_socks ?\n\n      module WrappedSocket\n        class TCP < ::TCPSocket\n          def remote_addr; peeraddr[3]; end\n          def remote_host; peeraddr[2]; end\n          def remote_port; peeraddr[1]; end\n        end\n        class UDP < ::UDPSocket\n          def remote_addr; peeraddr[3]; end\n          def remote_host; peeraddr[2]; end\n          def remote_port; peeraddr[1]; end\n        end\n        class TLS < OpenSSL::SSL::SSLSocket\n          def remote_addr; peeraddr[3]; end\n          def remote_host; peeraddr[2]; end\n          def remote_port; peeraddr[1]; end\n        end\n      end\n\n      def initialize\n        super\n        # @_sockets = [] # for keepalived sockets / connection pool\n      end\n\n      # def close\n      #   @_sockets.each do |sock|\n      #     sock.close\n      #   end\n      #   super\n      # end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/socket_option.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'socket'\nrequire 'fcntl'\n\n# this module is only for Socket/Server plugin helpers\nmodule Fluent\n  module PluginHelper\n    module SocketOption\n      # ref: https://docs.microsoft.com/en-us/windows/win32/api/winsock/ns-winsock-linger\n      FORMAT_STRUCT_LINGER_WINDOWS  = 'S!S!' # { u_short l_onoff; u_short l_linger; }\n      FORMAT_STRUCT_LINGER  = 'I!I!' # { int l_onoff; int l_linger; }\n      FORMAT_STRUCT_TIMEVAL = 'L!L!' # { time_t tv_sec; suseconds_t tv_usec; }\n\n      def socket_option_validate!(protocol, resolve_name: nil, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, receive_buffer_size: nil, send_keepalive_packet: nil)\n        unless resolve_name.nil?\n          if protocol != :tcp && protocol != :udp && protocol != :tls\n            raise ArgumentError, \"BUG: resolve_name in available for tcp/udp/tls\"\n          end\n        end\n        if linger_timeout\n          if protocol != :tcp && protocol != :tls\n            raise ArgumentError, \"BUG: linger_timeout is available for tcp/tls\"\n          end\n        end\n        if send_keepalive_packet\n          if protocol != :tcp && protocol != :tls\n            raise ArgumentError, \"BUG: send_keepalive_packet is available for tcp/tls\"\n          end\n        end\n      end\n\n      def socket_option_set(sock, resolve_name: nil, nonblock: false, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, receive_buffer_size: nil, send_keepalive_packet: nil)\n        unless resolve_name.nil?\n          sock.do_not_reverse_lookup = !resolve_name\n        end\n        if nonblock\n          sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)\n        end\n        if Fluent.windows?\n          # To prevent closing socket forcibly on Windows,\n          # this options shouldn't be set up when linger_timeout equals to 0 (including nil).\n          # This unintended behavior always occurs on Windows when linger_timeout.to_i == 0.\n          # This unintended behavior causes \"Errno::ECONNRESET: An existing connection was forcibly\n          # closed by the remote host.\" on Windows.\n          if linger_timeout.to_i > 0\n            if linger_timeout >= 2**16\n              log.warn \"maximum linger_timeout is 65535(2^16 - 1). Set to 65535 forcibly.\"\n              linger_timeout = 2**16 - 1\n            end\n            optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER_WINDOWS)\n            socket_option_set_one(sock, :SO_LINGER, optval)\n          end\n        else\n          if linger_timeout\n            optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)\n            socket_option_set_one(sock, :SO_LINGER, optval)\n          end\n        end\n        if recv_timeout\n          optval = [recv_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)\n          socket_option_set_one(sock, :SO_RCVTIMEO, optval)\n        end\n        if send_timeout\n          optval = [send_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)\n          socket_option_set_one(sock, :SO_SNDTIMEO, optval)\n        end\n        if receive_buffer_size\n          socket_option_set_one(sock, :SO_RCVBUF, receive_buffer_size.to_i)\n        end\n        if send_keepalive_packet\n          socket_option_set_one(sock, :SO_KEEPALIVE, true)\n        end\n        sock\n      end\n\n      def socket_option_set_one(sock, option, value)\n        sock.setsockopt(::Socket::SOL_SOCKET, option, value)\n      rescue => e\n        log.warn \"failed to set socket option\", sock: sock.class, option: option, value: value, error: e\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/storage.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'forwardable'\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/storage'\nrequire 'fluent/plugin_helper/timer'\nrequire 'fluent/config/element'\nrequire 'fluent/configurable'\n\nmodule Fluent\n  module PluginHelper\n    module Storage\n      include Fluent::PluginHelper::Timer\n\n      StorageState = Struct.new(:storage, :running)\n\n      def storage_create(usage: '', type: nil, conf: nil, default_type: nil)\n        if conf && conf.respond_to?(:arg) && !conf.arg.empty?\n          usage = conf.arg\n        end\n        if !usage.empty? && usage !~ /^[a-zA-Z][-_.a-zA-Z0-9]*$/\n          raise Fluent::ConfigError, \"Argument in <storage ARG> uses invalid characters: '#{usage}'\"\n        end\n\n        s = @_storages[usage]\n        if s&.running\n          return s.storage\n        elsif s\n          # storage is already created, but not loaded / started\n        else # !s\n          type = if type\n                   type\n                 elsif conf && conf.respond_to?(:[])\n                   raise Fluent::ConfigError, \"@type is required in <storage>\" unless conf['@type']\n                   conf['@type']\n                 elsif default_type\n                   default_type\n                 else\n                   raise ArgumentError, \"BUG: both type and conf are not specified\"\n                 end\n          storage = Plugin.new_storage(type, parent: self)\n          config = case conf\n                   when Fluent::Config::Element\n                     conf\n                   when Hash\n                     # in code, programmer may use symbols as keys, but Element needs strings\n                     conf = Hash[conf.map{|k,v| [k.to_s, v]}]\n                     Fluent::Config::Element.new('storage', usage, conf, [])\n                   when nil\n                     Fluent::Config::Element.new('storage', usage, {'@type' => type}, [])\n                   else\n                     raise ArgumentError, \"BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'\"\n                   end\n          storage.configure(config)\n          if @_storages_started\n            storage.start\n          end\n          s = @_storages[usage] = StorageState.new(wrap_instance(storage), false)\n        end\n\n        s.storage\n      end\n\n      module StorageParams\n        include Fluent::Configurable\n        # minimum section definition to instantiate storage plugin instances\n        config_section :storage, required: false, multi: true, param_name: :storage_configs, init: true do\n          config_argument :usage, :string, default: ''\n          config_param    :@type, :string, default: Fluent::Plugin::Storage::DEFAULT_TYPE\n        end\n      end\n\n      def self.included(mod)\n        mod.include StorageParams\n      end\n\n      attr_reader :_storages # for tests\n\n      def initialize\n        super\n        @_storages_started = false\n        @_storages = {} # usage => storage_state\n      end\n\n      def configure(conf)\n        super\n\n        @storage_configs.each do |section|\n          if !section.usage.empty? && section.usage !~ /^[a-zA-Z][-_.a-zA-Z0-9]*$/\n            raise Fluent::ConfigError, \"Argument in <storage ARG> uses invalid characters: '#{section.usage}'\"\n          end\n          if @_storages[section.usage]\n            raise Fluent::ConfigError, \"duplicated storages configured: #{section.usage}\"\n          end\n          storage = Plugin.new_storage(section[:@type], parent: self)\n          storage.configure(section.corresponding_config_element)\n          @_storages[section.usage] = StorageState.new(wrap_instance(storage), false)\n        end\n      end\n\n      def start\n        super\n\n        @_storages_started = true\n        @_storages.each_pair do |usage, s|\n          s.storage.start\n          s.storage.load\n\n          if s.storage.autosave && !s.storage.persistent\n            timer_execute(:storage_autosave, s.storage.autosave_interval, repeat: true) do\n              begin\n                s.storage.save\n              rescue => e\n                log.error \"plugin storage failed to save its data\", usage: usage, type: type, error: e\n              end\n            end\n          end\n          s.running = true\n        end\n      end\n\n      def storage_operate(method_name, &block)\n        @_storages.each_pair do |usage, s|\n          begin\n            block.call(s) if block_given?\n            s.storage.__send__(method_name)\n          rescue => e\n            log.error \"unexpected error while #{method_name}\", usage: usage, storage: s.storage, error: e\n          end\n        end\n      end\n\n      def stop\n        super\n        # timer stops automatically in super\n        storage_operate(:stop)\n      end\n\n      def before_shutdown\n        storage_operate(:before_shutdown)\n        super\n      end\n\n      def shutdown\n        storage_operate(:shutdown) do |s|\n          s.storage.save if s.storage.save_at_shutdown\n        end\n        super\n      end\n\n      def after_shutdown\n        storage_operate(:after_shutdown)\n        super\n      end\n\n      def close\n        storage_operate(:close){|s| s.running = false }\n        super\n      end\n\n      def terminate\n        storage_operate(:terminate)\n        @_storages = {}\n        super\n      end\n\n      def wrap_instance(storage)\n        if storage.persistent && storage.persistent_always?\n          storage\n        elsif storage.persistent\n          PersistentWrapper.new(storage)\n        elsif !storage.synchronized?\n          SynchronizeWrapper.new(storage)\n        else\n          storage\n        end\n      end\n\n      class PersistentWrapper\n        # PersistentWrapper always provides synchronized operations\n        extend Forwardable\n\n        def initialize(storage)\n          @storage = storage\n          @monitor = Monitor.new\n        end\n\n        def_delegators :@storage, :autosave_interval, :save_at_shutdown\n        def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate\n        def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?\n\n        def method_missing(name, *args)\n          @monitor.synchronize{ @storage.__send__(name, *args) }\n        end\n\n        def persistent_always?\n          true\n        end\n\n        def persistent\n          true\n        end\n\n        def autosave\n          false\n        end\n\n        def synchronized?\n          true\n        end\n\n        def implementation\n          @storage\n        end\n\n        def load\n          @monitor.synchronize do\n            @storage.load\n          end\n        end\n\n        def save\n          @monitor.synchronize do\n            @storage.save\n          end\n        end\n\n        def get(key)\n          @monitor.synchronize do\n            @storage.load\n            @storage.get(key)\n          end\n        end\n\n        def fetch(key, defval)\n          @monitor.synchronize do\n            @storage.load\n            @storage.fetch(key, defval)\n          end\n        end\n\n        def put(key, value)\n          @monitor.synchronize do\n            @storage.load\n            @storage.put(key, value)\n            @storage.save\n            value\n          end\n        end\n\n        def delete(key)\n          @monitor.synchronize do\n            @storage.load\n            val = @storage.delete(key)\n            @storage.save\n            val\n          end\n        end\n\n        def update(key, &block)\n          @monitor.synchronize do\n            @storage.load\n            v = block.call(@storage.get(key))\n            @storage.put(key, v)\n            @storage.save\n            v\n          end\n        end\n      end\n\n      class SynchronizeWrapper\n        extend Forwardable\n\n        def initialize(storage)\n          @storage = storage\n          @monitor = Monitor.new\n        end\n\n        def_delegators :@storage, :persistent, :autosave, :autosave_interval, :save_at_shutdown\n        def_delegators :@storage, :persistent_always?\n        def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate\n        def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?\n\n        def method_missing(name, *args)\n          @monitor.synchronize{ @storage.__send__(name, *args) }\n        end\n\n        def synchronized?\n          true\n        end\n\n        def implementation\n          @storage\n        end\n\n        def load\n          @monitor.synchronize do\n            @storage.load\n          end\n        end\n\n        def save\n          @monitor.synchronize do\n            @storage.save\n          end\n        end\n\n        def get(key)\n          @monitor.synchronize{ @storage.get(key) }\n        end\n\n        def fetch(key, defval)\n          @monitor.synchronize{ @storage.fetch(key, defval) }\n        end\n\n        def put(key, value)\n          @monitor.synchronize{ @storage.put(key, value) }\n        end\n\n        def delete(key)\n          @monitor.synchronize{ @storage.delete(key) }\n        end\n\n        def update(key, &block)\n          @monitor.synchronize do\n            v = block.call(@storage.get(key))\n            @storage.put(key, v)\n            v\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/thread.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/clock'\n\nmodule Fluent\n  module PluginHelper\n    module Thread\n      THREAD_DEFAULT_WAIT_SECONDS = 1\n      THREAD_SHUTDOWN_HARD_TIMEOUT_IN_TESTS = 100 # second\n\n      # stop     : mark callback thread as stopped\n      # shutdown : [-]\n      # close    : correct stopped threads\n      # terminate: kill all threads\n\n      attr_reader :_threads # for test driver\n\n      def thread_current_running?\n        # checker for code in callback of thread_create\n        ::Thread.current[:_fluentd_plugin_helper_thread_running] || false\n      end\n\n      def thread_wait_until_start\n        until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && t[:_fluentd_plugin_helper_thread_started] } }\n          sleep 0.1\n        end\n      end\n\n      def thread_wait_until_stop\n        timeout_at = Fluent::Clock.now + THREAD_SHUTDOWN_HARD_TIMEOUT_IN_TESTS\n        until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && !t[:_fluentd_plugin_helper_thread_running] } }\n          break if Fluent::Clock.now > timeout_at\n          sleep 0.1\n        end\n        @_threads_mutex.synchronize{ @_threads.values }.each do |t|\n          if t.alive?\n            puts \"going to kill the thread still running: #{t[:_fluentd_plugin_helper_thread_title]}\"\n            t.kill rescue nil\n            t[:_fluentd_plugin_helper_thread_running] = false\n          end\n        end\n      end\n\n      # Ruby 2.2.3 or earlier (and all 2.1.x) cause bug about Threading (\"Stack consistency error\")\n      #  by passing splatted argument to `yield`\n      # https://bugs.ruby-lang.org/issues/11027\n      # We can enable to pass arguments after expire of Ruby 2.1 (& older 2.2.x)\n      # def thread_create(title, *args)\n      #   Thread.new(*args) do |*t_args|\n      #     yield *t_args\n      def thread_create(title)\n        raise ArgumentError, \"BUG: title must be a symbol\" unless title.is_a? Symbol\n        raise ArgumentError, \"BUG: callback not specified\" unless block_given?\n        m = Mutex.new\n        m.lock\n        thread = ::Thread.new do\n          m.lock # run thread after that thread is successfully set into @_threads\n          m.unlock\n          thread_exit = false\n          ::Thread.current[:_fluentd_plugin_helper_thread_title] = title\n          ::Thread.current[:_fluentd_plugin_helper_thread_started] = true\n          ::Thread.current[:_fluentd_plugin_helper_thread_running] = true\n          begin\n            yield\n            thread_exit = true\n          rescue Exception => e\n            log.warn \"thread exited by unexpected error\", plugin: self.class, title: title, error: e\n            thread_exit = true\n            raise\n          ensure\n            @_threads_mutex.synchronize do\n              @_threads.delete(::Thread.current.object_id)\n            end\n            ::Thread.current[:_fluentd_plugin_helper_thread_running] = false\n            if ::Thread.current.alive? && !thread_exit\n              log.warn \"thread doesn't exit correctly (killed or other reason)\", plugin: self.class, title: title, thread: ::Thread.current, error: $!\n            end\n          end\n        end\n        thread.abort_on_exception = true\n        thread.name = title.to_s if thread.respond_to?(:name)\n        @_threads_mutex.synchronize do\n          @_threads[thread.object_id] = thread\n        end\n        m.unlock\n        thread\n      end\n\n      def thread_exist?(title)\n        @_threads.values.count{|thread| title == thread[:_fluentd_plugin_helper_thread_title] } > 0\n      end\n\n      def thread_started?(title)\n        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }\n        t && t[:_fluentd_plugin_helper_thread_started]\n      end\n\n      def thread_running?(title)\n        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }\n        t && t[:_fluentd_plugin_helper_thread_running]\n      end\n\n      def initialize\n        super\n        @_threads_mutex = Mutex.new\n        @_threads = {}\n        @_thread_wait_seconds = THREAD_DEFAULT_WAIT_SECONDS\n      end\n\n      def stop\n        super\n        wakeup_threads = []\n        @_threads_mutex.synchronize do\n          @_threads.each_value do |thread|\n            thread[:_fluentd_plugin_helper_thread_running] = false\n            wakeup_threads << thread if thread.alive? && thread.status == \"sleep\"\n          end\n        end\n        wakeup_threads.each do |thread|\n          if thread.alive?\n            thread.wakeup\n          end\n        end\n      end\n\n      def after_shutdown\n        super\n        wakeup_threads = []\n        @_threads_mutex.synchronize do\n          @_threads.each_value do |thread|\n            wakeup_threads << thread if thread.alive? && thread.status == \"sleep\"\n          end\n        end\n        wakeup_threads.each do |thread|\n          thread.wakeup if thread.alive?\n        end\n      end\n\n      def close\n        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|\n          thread = @_threads[obj_id]\n          if !thread || thread.join(@_thread_wait_seconds)\n            @_threads_mutex.synchronize{ @_threads.delete(obj_id) }\n          end\n        end\n\n        super\n      end\n\n      def terminate\n        super\n        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|\n          thread = @_threads[obj_id]\n          log.warn \"killing existing thread\", thread: thread\n          thread.kill if thread\n        end\n        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|\n          thread = @_threads[obj_id]\n          thread.join\n          @_threads_mutex.synchronize{ @_threads.delete(obj_id) }\n        end\n        @_thread_wait_seconds = nil\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper/timer.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin_helper/event_loop'\nrequire 'set'\n\nmodule Fluent\n  module PluginHelper\n    module Timer\n      include Fluent::PluginHelper::EventLoop\n\n      # stop     : turn checker into false (callbacks not called anymore)\n      # shutdown : [-]\n      # close    : [-]\n      # terminate: [-]\n\n      attr_reader :_timers # for tests\n\n      # interval: integer/float, repeat: true/false\n      def timer_execute(title, interval, repeat: true, &block)\n        raise ArgumentError, \"BUG: title must be a symbol\" unless title.is_a? Symbol\n        raise ArgumentError, \"BUG: block not specified for callback\" unless block_given?\n        checker = ->(){ @_timer_running }\n        detacher = ->(watcher){ event_loop_detach(watcher) }\n        timer = TimerWatcher.new(title, interval, repeat, log, checker, detacher, &block)\n        @_timers << title\n        event_loop_attach(timer)\n        timer\n      end\n\n      def timer_running?\n        defined?(@_timer_running) && @_timer_running\n      end\n\n      def initialize\n        super\n        @_timers ||= Set.new\n      end\n\n      def start\n        super\n        @_timer_running = true\n      end\n\n      def stop\n        super\n        @_timer_running = false\n      end\n\n      def terminate\n        super\n        @_timers = nil\n      end\n\n      class TimerWatcher < Coolio::TimerWatcher\n        def initialize(title, interval, repeat, log, checker, detacher, &callback)\n          @title = title\n          @callback = callback\n          @repeat = repeat\n          @log = log\n          @checker = checker\n          @detacher = detacher\n          super(interval, repeat)\n        end\n\n        def on_timer\n          @callback.call if @checker.call\n        rescue => e\n          @log.error \"Unexpected error raised. Stopping the timer.\", title: @title, error: e\n          @log.error_backtrace\n          detach\n          @log.error \"Timer detached.\", title: @title\n        ensure\n          @detacher.call(self) unless @repeat\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_helper.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin_helper/event_emitter'\nrequire 'fluent/plugin_helper/thread'\nrequire 'fluent/plugin_helper/event_loop'\nrequire 'fluent/plugin_helper/timer'\nrequire 'fluent/plugin_helper/child_process'\nrequire 'fluent/plugin_helper/storage'\nrequire 'fluent/plugin_helper/parser'\nrequire 'fluent/plugin_helper/formatter'\nrequire 'fluent/plugin_helper/http_server'\nrequire 'fluent/plugin_helper/inject'\nrequire 'fluent/plugin_helper/extract'\nrequire 'fluent/plugin_helper/socket'\nrequire 'fluent/plugin_helper/server'\nrequire 'fluent/plugin_helper/counter'\nrequire 'fluent/plugin_helper/retry_state'\nrequire 'fluent/plugin_helper/record_accessor'\nrequire 'fluent/plugin_helper/compat_parameters'\nrequire 'fluent/plugin_helper/service_discovery'\nrequire 'fluent/plugin_helper/metrics'\n\nmodule Fluent\n  module PluginHelper\n    module Mixin\n      def self.included(mod)\n        mod.extend(Fluent::PluginHelper)\n      end\n    end\n\n    def self.extended(mod)\n      def mod.inherited(subclass)\n        subclass.module_eval do\n          @_plugin_helpers_list = []\n        end\n      end\n    end\n\n    def helpers_internal(*snake_case_symbols)\n      helper_modules = []\n      snake_case_symbols.each do |name|\n        begin\n          helper_modules << Fluent::PluginHelper.const_get(name.to_s.split('_').map(&:capitalize).join)\n        rescue NameError\n          raise \"Unknown plugin helper:#{name}\"\n        end\n      end\n      include(*helper_modules)\n    end\n\n    def helpers(*snake_case_symbols)\n      @_plugin_helpers_list ||= []\n      @_plugin_helpers_list.concat(snake_case_symbols)\n      helpers_internal(*snake_case_symbols)\n    end\n\n    def plugin_helpers\n      @_plugin_helpers_list || []\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/plugin_id.rb",
    "content": "#\n# Fluent\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'set'\nrequire 'fluent/env'\nrequire 'fluent/variable_store'\n\nmodule Fluent\n  module PluginId\n\n    def initialize\n      super\n\n      @_plugin_id_variable_store = nil\n      @_plugin_root_dir = nil\n      @id = nil\n    end\n\n    def configure(conf)\n      @_plugin_id_variable_store = Fluent::VariableStore.fetch_or_build(:plugin_id, default_value: Set.new)\n      @id = conf['@id']\n      @_id_configured = !!@id # plugin id is explicitly configured by users (or not)\n      if @id\n        @id = @id.to_s\n        if @_plugin_id_variable_store.include?(@id) && !plugin_id_for_test?\n          raise Fluent::ConfigError, \"Duplicated plugin id `#{@id}`. Check whole configuration and fix it.\"\n        end\n        @_plugin_id_variable_store.add(@id)\n      end\n\n      super\n    end\n\n    def plugin_id_for_test?\n      caller_locations.each do |location|\n        # Thread::Backtrace::Location#path returns base filename or absolute path.\n        # #absolute_path returns absolute_path always.\n        # https://bugs.ruby-lang.org/issues/12159\n        if /\\/test_[^\\/]+\\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\\.rb$/\n          return true\n        end\n      end\n      false\n    end\n\n    def plugin_id_configured?\n      if instance_variable_defined?(:@_id_configured)\n        @_id_configured\n      end\n    end\n\n    def plugin_id\n      if instance_variable_defined?(:@id)\n        @id || \"object:#{object_id.to_s(16)}\"\n      else\n        \"object:#{object_id.to_s(16)}\"\n      end\n    end\n\n    def plugin_root_dir\n      return @_plugin_root_dir if @_plugin_root_dir\n      return nil unless system_config.root_dir\n      return nil unless plugin_id_configured?\n\n      # Fluent::Plugin::Base#fluentd_worker_id\n      dir = File.join(system_config.root_dir, \"worker#{fluentd_worker_id}\", plugin_id)\n      FileUtils.mkdir_p(dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(dir)\n      @_plugin_root_dir = dir.freeze\n      dir\n    end\n\n    def stop\n      if @_plugin_id_variable_store\n        @_plugin_id_variable_store.delete(@id)\n      end\n\n      super\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/process.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/compat/detach_process_mixin'\n\nmodule Fluent\n  DetachProcessMixin = Fluent::Compat::DetachProcessMixin\n  DetachMultiProcessMixin = Fluent::Compat::DetachMultiProcessMixin\nend\n"
  },
  {
    "path": "lib/fluent/registry.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/error'\n\nmodule Fluent\n  class Registry\n    DEFAULT_PLUGIN_PATH = File.expand_path('../plugin', __FILE__)\n    FLUENT_LIB_PATH = File.dirname(File.dirname(DEFAULT_PLUGIN_PATH))\n\n    def initialize(kind, search_prefix, dir_search_prefix: nil)\n      @kind = kind\n      @search_prefix = search_prefix\n      @dir_search_prefix = dir_search_prefix\n      @map = {}\n      @paths = []\n    end\n\n    attr_reader :kind, :paths, :map, :dir_search_prefix\n\n    def register(type, value)\n      type = type.to_sym\n      @map[type] = value\n    end\n\n    def lookup(type)\n      type = type.to_sym\n      if value = @map[type]\n        return value\n      end\n      search(type)\n      if value = @map[type]\n        return value\n      end\n      raise NotFoundPluginError.new(\"Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins\",\n                                    kind: @kind, type: type)\n    end\n\n    def reverse_lookup(value)\n      @map.each do |k, v|\n        return k if v == value\n      end\n      nil\n    end\n\n    def search(type)\n      # search from additional plugin directories\n      if @dir_search_prefix\n        path = \"#{@dir_search_prefix}#{type}\"\n        files = @paths.filter_map { |lp|\n          lpath = File.expand_path(File.join(lp, \"#{path}.rb\"))\n          File.exist?(lpath) ? lpath : nil\n        }\n        unless files.empty?\n          # prefer newer version\n          require files.max\n          return\n        end\n      end\n\n      path = \"#{@search_prefix}#{type}\"\n\n      # prefer LOAD_PATH than gems\n      files = $LOAD_PATH.filter_map { |lp|\n        if lp == FLUENT_LIB_PATH\n          nil\n        else\n          lpath = File.expand_path(File.join(lp, \"#{path}.rb\"))\n          File.exist?(lpath) ? lpath : nil\n        end\n      }\n      unless files.empty?\n        # prefer newer version\n        require files.max\n        return\n      end\n\n      # Find from gems and prefer newer version\n      specs = Gem::Specification.find_all { |spec|\n        if spec.name == 'fluentd'.freeze\n          false\n        else\n          spec.contains_requirable_file? path\n        end\n      }.sort_by { |spec| spec.version }\n      if spec = specs.last\n        spec.require_paths.each { |lib|\n          file = \"#{spec.full_gem_path}/#{lib}/#{path}\"\n          if File.exist?(\"#{file}.rb\")\n            require file\n            return\n          end\n        }\n      end\n\n      # Lastly, load built-in plugin\n      lpath = File.expand_path(File.join(FLUENT_LIB_PATH, \"#{@search_prefix}#{type}.rb\"))\n      if File.exist?(lpath)\n        require lpath\n        return\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/root_agent.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'delegate'\n\nrequire 'fluent/config/error'\nrequire 'fluent/agent'\nrequire 'fluent/label'\nrequire 'fluent/plugin'\nrequire 'fluent/system_config'\nrequire 'fluent/time'\nrequire 'fluent/source_only_buffer_agent'\n\nmodule Fluent\n  #\n  # Fluentd forms a tree structure to manage plugins:\n  #\n  #                      RootAgent\n  #                          |\n  #             +------------+-------------+-------------+\n  #             |            |             |             |\n  #          <label>      <source>      <filter>      <match>\n  #             |\n  #        +----+----+\n  #        |         |\n  #     <filter>   <match>\n  #\n  # Relation:\n  # * RootAgent has many <label>, <source>, <filter> and <match>\n  # * <label>   has many <match> and <filter>\n  #\n  # Next step: `fluentd/agent.rb`\n  # Next step: 'fluentd/label.rb'\n  #\n  class RootAgent < Agent\n    ERROR_LABEL = \"@ERROR\".freeze # @ERROR is built-in error label\n\n    class SourceOnlyMode\n      DISABLED = 0\n      NORMAL = 1\n      ONLY_ZERO_DOWNTIME_RESTART_READY = 2\n\n      def initialize(with_source_only, start_in_parallel)\n        if start_in_parallel\n          @mode = ONLY_ZERO_DOWNTIME_RESTART_READY\n        elsif with_source_only\n          @mode = NORMAL\n        else\n          @mode = DISABLED\n        end\n      end\n\n      def enabled?\n        @mode != DISABLED\n      end\n\n      def only_zero_downtime_restart_ready?\n        @mode == ONLY_ZERO_DOWNTIME_RESTART_READY\n      end\n\n      def disable!\n        @mode = DISABLED\n      end\n    end\n\n    def initialize(log:, system_config: SystemConfig.new, start_in_parallel: false)\n      super(log: log)\n\n      @labels = {}\n      @inputs = []\n      @suppress_emit_error_log_interval = 0\n      @next_emit_error_log_time = nil\n      @without_source = system_config.without_source || false\n      @source_only_mode = SourceOnlyMode.new(system_config.with_source_only, start_in_parallel)\n      @source_only_buffer_agent = nil\n      @enable_input_metrics = system_config.enable_input_metrics\n\n      suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?\n    end\n\n    attr_reader :inputs\n    attr_reader :labels\n\n    def source_only_router\n      raise \"[BUG] 'RootAgent#source_only_router' should not be called when 'with_source_only' is false\" unless @source_only_mode.enabled?\n      @source_only_buffer_agent.event_router\n    end\n\n    def configure(conf)\n      used_worker_ids = []\n      available_worker_ids = (0..Fluent::Engine.system_config.workers - 1).to_a\n      # initialize <worker> elements\n      supported_directives = ['source', 'match', 'filter', 'label']\n      conf.elements(name: 'worker').each do |e|\n        target_worker_id_str = e.arg\n        if target_worker_id_str.empty?\n          raise Fluent::ConfigError, \"Missing worker id on <worker> directive\"\n        end\n\n        target_worker_ids = target_worker_id_str.split(\"-\")\n        if target_worker_ids.size == 2\n          first_worker_id = target_worker_ids.first.to_i\n          last_worker_id = target_worker_ids.last.to_i\n          if first_worker_id > last_worker_id\n            raise Fluent::ConfigError, \"greater first_worker_id<#{first_worker_id}> than last_worker_id<#{last_worker_id}> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>\"\n          end\n          target_worker_ids = []\n          first_worker_id.step(last_worker_id, 1) do |worker_id|\n            target_worker_id = worker_id.to_i\n            target_worker_ids << target_worker_id\n\n            if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)\n              raise Fluent::ConfigError, \"worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}\"\n            end\n            available_worker_ids.delete(target_worker_id) if available_worker_ids.include?(target_worker_id)\n            if used_worker_ids.include?(target_worker_id)\n              raise Fluent::ConfigError, \"specified worker_id<#{worker_id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}\"\n            end\n            used_worker_ids << target_worker_id\n\n            e.elements.each do |elem|\n              unless supported_directives.include?(elem.name)\n                raise Fluent::ConfigError, \"<worker> section cannot have <#{elem.name}> directive\"\n              end\n            end\n\n            unless target_worker_ids.empty?\n              e.set_target_worker_ids(target_worker_ids.uniq)\n            end\n          end\n        else\n          target_worker_id = target_worker_id_str.to_i\n          if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)\n            raise Fluent::ConfigError, \"worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}\"\n          end\n\n          e.elements.each do |elem|\n            unless supported_directives.include?(elem.name)\n              raise Fluent::ConfigError, \"<worker> section cannot have <#{elem.name}> directive\"\n            end\n            elem.set_target_worker_id(target_worker_id)\n          end\n        end\n        conf += e\n      end\n      conf.elements.delete_if{|e| e.name == 'worker'}\n\n      error_label_config = nil\n\n      # initialize <label> elements before configuring all plugins to avoid 'label not found' in input, filter and output.\n      label_configs = {}\n      conf.elements(name: 'label').each { |e|\n        if !Fluent::Engine.supervisor_mode && e.for_another_worker?\n          next\n        end\n        name = e.arg\n        raise ConfigError, \"Missing symbol argument on <label> directive\" if name.empty?\n        raise ConfigError, \"@ROOT for <label> is not permitted, reserved for getting root router\" if name == '@ROOT'\n\n        if name == ERROR_LABEL\n          error_label_config = e\n        else\n          add_label(name)\n          label_configs[name] = e\n        end\n      }\n      # Call 'configure' here to avoid 'label not found'\n      label_configs.each { |name, e| @labels[name].configure(e) }\n      setup_error_label(error_label_config) if error_label_config\n\n      super\n\n      setup_source_only_buffer_agent if @source_only_mode.enabled?\n\n      # initialize <source> elements\n      if @without_source\n        log.info :worker0, \"'--without-source' is applied. Ignore <source> sections\"\n      else\n        conf.elements(name: 'source').each { |e|\n          if !Fluent::Engine.supervisor_mode && e.for_another_worker?\n            next\n          end\n          type = e['@type']\n          raise ConfigError, \"Missing '@type' parameter on <source> directive\" unless type\n          add_source(type, e)\n        }\n      end\n    end\n\n    def setup_error_label(e)\n      error_label = add_label(ERROR_LABEL)\n      error_label.configure(e)\n      @error_collector = error_label.event_router\n    end\n\n    def setup_source_only_buffer_agent(flush: false)\n      @source_only_buffer_agent = SourceOnlyBufferAgent.new(log: log, system_config: Fluent::Engine.system_config)\n      @source_only_buffer_agent.configure(flush: flush)\n    end\n\n    def cleanup_source_only_buffer_agent\n      @source_only_buffer_agent&.cleanup\n    end\n\n    def lifecycle(desc: false, kind_callback: nil, kind_or_agent_list: nil)\n      only_zero_downtime_restart_ready = false\n\n      unless kind_or_agent_list\n        if @source_only_mode.enabled?\n          kind_or_agent_list = [:input, @source_only_buffer_agent]\n          only_zero_downtime_restart_ready = @source_only_mode.only_zero_downtime_restart_ready?\n        elsif @source_only_buffer_agent\n          # source_only_buffer_agent can re-reroute events, so the priority is equal to output_with_router.\n          kind_or_agent_list = [:input, :output_with_router, @source_only_buffer_agent, @labels.values, :filter, :output].flatten\n        else\n          kind_or_agent_list = [:input, :output_with_router, @labels.values, :filter, :output].flatten\n        end\n\n        kind_or_agent_list.reverse! if desc\n      end\n\n      kind_or_agent_list.each do |kind|\n        if kind.respond_to?(:lifecycle)\n          agent = kind\n          agent.lifecycle(desc: desc) do |plugin, display_kind|\n            yield plugin, display_kind\n          end\n        else\n          list = if desc\n                   lifecycle_control_list[kind].reverse\n                 else\n                   lifecycle_control_list[kind]\n                 end\n          display_kind = (kind == :output_with_router ? :output : kind)\n          list.each do |instance|\n            if only_zero_downtime_restart_ready\n              next unless instance.respond_to?(:zero_downtime_restart_ready?) and instance.zero_downtime_restart_ready?\n            end\n            yield instance, display_kind\n          end\n        end\n        if kind_callback\n          kind_callback.call\n        end\n      end\n    end\n\n    def start(kind_or_agent_list: nil)\n      lifecycle(desc: true, kind_or_agent_list: kind_or_agent_list) do |i| # instance\n        i.start unless i.started?\n        # Input#start sometimes emits lots of events with in_tail/`read_from_head true` case\n        # and it causes deadlock for small buffer/queue output. To avoid such problem,\n        # buffer related output threads should be run before `Input#start`.\n        # This is why after_start should be called immediately after start call.\n        # This depends on `desc: true` because calling plugin order of `desc: true` is\n        # Output, Filter, Label, Output with Router, then Input.\n        i.after_start unless i.after_started?\n      end\n    end\n\n    def flush!\n      log.info \"flushing all buffer forcedly\"\n      flushing_threads = []\n      lifecycle(desc: true) do |instance|\n        if instance.respond_to?(:force_flush)\n          t = Thread.new do\n            Thread.current.abort_on_exception = true\n            begin\n              instance.force_flush\n            rescue => e\n              log.warn \"unexpected error while flushing buffer\", plugin: instance.class, plugin_id: instance.plugin_id, error: e\n              log.warn_backtrace\n            end\n          end\n          flushing_threads << t\n        end\n      end\n      flushing_threads.each{|t| t.join }\n    end\n\n    def cancel_source_only!\n      unless @source_only_mode.enabled?\n        log.info \"do nothing for canceling with-source-only because the current mode is not with-source-only.\"\n        return\n      end\n\n      log.info \"cancel with-source-only mode and start the other plugins\"\n      all_plugins = [:input, :output_with_router, @labels.values, :filter, :output].flatten.reverse\n      start(kind_or_agent_list: all_plugins)\n\n      lifecycle_control_list[:input].each(&:event_emitter_cancel_source_only)\n\n      # Want to make sure that the source_only_router finishes all process before\n      # shutting down the agent.\n      # Strictly speaking, it would be necessary to have exclusive lock between\n      # EventRouter and the shutting down process of this agent.\n      # However, adding lock to EventRouter would worsen its performance, and\n      # the entire shutting down process does not care about it either.\n      # So, sleep here just in case.\n      sleep 1\n\n      shutdown(kind_or_agent_list: [@source_only_buffer_agent])\n      @source_only_buffer_agent = nil\n\n      # This agent can stop after flushing its all buffer, but it is not implemented for now.\n      log.info \"starts the loading agent for with-source-only\"\n      setup_source_only_buffer_agent(flush: true)\n      start(kind_or_agent_list: [@source_only_buffer_agent])\n\n      @source_only_mode.disable!\n    end\n\n    def shutdown(kind_or_agent_list: nil)\n      # Fluentd's shutdown sequence is stop, before_shutdown, shutdown, after_shutdown, close, terminate for plugins\n      # These method callers does `rescue Exception` to call methods of shutdown sequence as far as possible\n      # if plugin methods does something like infinite recursive call, `exit`, unregistering signal handlers or others.\n      # Plugins should be separated and be in sandbox to protect data in each plugins/buffers.\n\n      lifecycle_safe_sequence = ->(method, checker) {\n        lifecycle(kind_or_agent_list: kind_or_agent_list) do |instance, kind|\n          begin\n            log.debug \"calling #{method} on #{kind} plugin\", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id\n            instance.__send__(method) unless instance.__send__(checker)\n          rescue Exception => e\n            log.warn \"unexpected error while calling #{method} on #{kind} plugin\", plugin: instance.class, plugin_id: instance.plugin_id, error: e\n            log.warn_backtrace\n          end\n        end\n      }\n\n      lifecycle_unsafe_sequence = ->(method, checker) {\n        operation = case method\n                    when :shutdown then \"shutting down\"\n                    when :close    then \"closing\"\n                    else\n                      raise \"BUG: unknown method name '#{method}'\"\n                    end\n        operation_threads = []\n        callback = ->(){\n          operation_threads.each{|t| t.join }\n          operation_threads.clear\n        }\n        lifecycle(kind_callback: callback, kind_or_agent_list: kind_or_agent_list) do |instance, kind|\n          t = Thread.new do\n            Thread.current.abort_on_exception = true\n            begin\n              if method == :shutdown\n                # To avoid Input#shutdown and Output#before_shutdown mismatch problem, combine before_shutdown and shutdown call in one sequence.\n                # The problem is in_tail flushes buffered multiline in shutdown but output's flush_at_shutdown is invoked in before_shutdown\n                operation = \"preparing shutdown\" # for logging\n                log.debug \"#{operation} #{kind} plugin\", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id\n                begin\n                  instance.__send__(:before_shutdown) unless instance.__send__(:before_shutdown?)\n                rescue Exception => e\n                  log.warn \"unexpected error while #{operation} on #{kind} plugin\", plugin: instance.class, plugin_id: instance.plugin_id, error: e\n                  log.warn_backtrace\n                end\n                operation = \"shutting down\"\n                log.info \"#{operation} #{kind} plugin\", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id\n                instance.__send__(:shutdown) unless instance.__send__(:shutdown?)\n              else\n                log.debug \"#{operation} #{kind} plugin\", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id\n                instance.__send__(method) unless instance.__send__(checker)\n              end\n            rescue Exception => e\n              log.warn \"unexpected error while #{operation} on #{kind} plugin\", plugin: instance.class, plugin_id: instance.plugin_id, error: e\n              log.warn_backtrace\n            end\n          end\n          operation_threads << t\n        end\n      }\n\n      lifecycle_safe_sequence.call(:stop, :stopped?)\n\n      # before_shutdown does force_flush for output plugins: it should block, so it's unsafe operation\n      lifecycle_unsafe_sequence.call(:shutdown, :shutdown?)\n\n      lifecycle_safe_sequence.call(:after_shutdown, :after_shutdown?)\n\n      lifecycle_unsafe_sequence.call(:close, :closed?)\n\n      lifecycle_safe_sequence.call(:terminate, :terminated?)\n\n      cleanup_source_only_buffer_agent unless kind_or_agent_list\n    end\n\n    def suppress_interval(interval_time)\n      @suppress_emit_error_log_interval = interval_time\n      @next_emit_error_log_time = Time.now.to_i\n    end\n\n    def add_source(type, conf)\n      log_type = conf.for_this_worker? ? :default : :worker0\n      log.info log_type, \"adding source\", type: type\n\n      input = Plugin.new_input(type)\n      # <source> emits events to the top-level event router (RootAgent#event_router).\n      # Input#configure overwrites event_router to a label's event_router if it has `@label` parameter.\n      # See also 'fluentd/plugin/input.rb'\n      input.context_router = @event_router\n      input.configure(conf)\n      input.event_emitter_apply_source_only if @source_only_mode.enabled?\n      if @enable_input_metrics\n        @event_router.add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })\n      end\n      @inputs << input\n\n      input\n    end\n\n    def add_label(name)\n      label = Label.new(name, log: log)\n      raise ConfigError, \"Section <label #{name}> appears twice\" if @labels[name]\n      label.root_agent = self\n      @labels[name] = label\n    end\n\n    def find_label(label_name)\n      if label = @labels[label_name]\n        label\n      else\n        raise ArgumentError, \"#{label_name} label not found\"\n      end\n    end\n\n    def emit_error_event(tag, time, record, error)\n      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time}\n      if @error_collector\n        # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification\n        log.warn \"send an error event to @ERROR:\", error_info\n        @error_collector.emit(tag, time, record)\n      else\n        error_info[:record] = record\n        log.warn \"dump an error event:\", error_info\n      end\n    end\n\n    def handle_emits_error(tag, es, error)\n      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}\n      if @error_collector\n        log.warn \"send an error event stream to @ERROR:\", error_info\n        @error_collector.emit_stream(tag, es)\n      else\n        now = Time.now.to_i\n        if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time\n          log.warn \"emit transaction failed:\", error_info\n          log.warn_backtrace\n          @next_emit_error_log_time = now + @suppress_emit_error_log_interval\n        end\n        raise error\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/rpc.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'webrick'\n\nmodule Fluent\n  module RPC\n    class Server\n      def initialize(endpoint, log)\n        m = endpoint.match(/^\\[?(?<host>[0-9a-zA-Z:\\-\\.]+)\\]?:(?<port>[0-9]+)$/)\n        raise Fluent::ConfigError, \"Invalid rpc_endpoint: #{endpoint}\" unless m\n        @bind = m[:host]\n        @port = m[:port]\n        @log = log\n\n        @server = WEBrick::HTTPServer.new(\n          BindAddress: @bind,\n          Port: @port,\n          Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),\n          AccessLog: [],\n        )\n      end\n\n      def mount(path, servlet, *args)\n        @server.mount(path, servlet, *args)\n        @log.debug \"register #{path} RPC servlet\"\n      end\n\n      def mount_proc(path, &block)\n        @server.mount_proc(path) { |req, res|\n          begin\n            code, header, response = block.call(req, res)\n          rescue => e\n            @log.warn \"failed to handle RPC request\", path: path, error: e.to_s\n            @log.warn_backtrace e.backtrace\n\n            code = 500\n            body = {\n              'message '=> 'Internal Server Error',\n              'error' => \"#{e}\",\n              'backtrace'=> e.backtrace,\n            }\n          end\n\n          code = 200 if code.nil?\n          header = {'Content-Type' => 'application/json'} if header.nil?\n          body = if response.nil?\n                   '{\"ok\":true}'\n                 else\n                   response.body['ok'] = code == 200\n                   response.body.to_json\n                 end\n\n          res.status = code\n          header.each_pair { |k, v|\n            res[k] = v\n          }\n          res.body = body\n        }\n        @log.debug \"register #{path} RPC handler\"\n      end\n\n      def start\n        @log.debug \"listening RPC http server on http://#{@bind}:#{@port}/\"\n        @thread = Thread.new {\n          @server.start\n        }\n      end\n\n      def shutdown\n        if @server\n          @server.shutdown\n          @server = nil\n        end\n        if @thread\n          @thread.join\n          @thread = nil\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/source_only_buffer_agent.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/agent'\nrequire 'fluent/system_config'\n\nmodule Fluent\n  class SourceOnlyBufferAgent < Agent\n    # Use INSTANCE_ID to use the same base dir as the other workers.\n    # This will make recovery easier.\n    BUFFER_DIR_NAME = Fluent::INSTANCE_ID\n\n    def initialize(log:, system_config:)\n      super(log: log)\n\n      @default_buffer_path = File.join(system_config.root_dir || DEFAULT_BACKUP_DIR, 'source-only-buffer', BUFFER_DIR_NAME)\n      @optional_buffer_config = system_config.source_only_buffer.to_h.transform_keys(&:to_s)\n      @base_buffer_dir = nil\n      @actual_buffer_dir = nil\n    end\n\n    def configure(flush: false)\n      buffer_config = @optional_buffer_config.compact\n      buffer_config['flush_at_shutdown'] = flush ? 'true' : 'false'\n      buffer_config['flush_thread_count'] = 0 unless flush\n      buffer_config['path'] ||= @default_buffer_path\n\n      super(\n        Config::Element.new('SOURCE_ONLY_BUFFER', '', {}, [\n          Config::Element.new('match', '**', {'@type' => 'buffer', '@label' => '@ROOT'}, [\n            Config::Element.new('buffer', '', buffer_config, [])\n          ])\n        ])\n      )\n\n      @base_buffer_dir = buffer_config['path']\n      # It can be \"#{@base_buffer_dir}/worker#{fluentd_worker_id}/\" when using multiple workers\n      @actual_buffer_dir = File.dirname(outputs[0].buffer.path)\n\n      unless flush\n        log.info \"with-source-only: the emitted data will be stored in the buffer files under\" +\n                 \" #{@base_buffer_dir}. You can send SIGWINCH to the supervisor process to cancel\" +\n                 \" with-source-only mode and process data.\"\n      end\n    end\n\n    def cleanup\n      unless (Dir.empty?(@actual_buffer_dir) rescue true)\n        log.warn \"some buffer files remain in #{@base_buffer_dir}.\" +\n                 \" Please consider recovering or saving the buffer files in the directory.\" +\n                 \" To recover them, you can set the buffer path manually to system config and\" +\n                 \" retry, i.e., restart Fluentd with with-source-only mode and send SIGWINCH again.\" +\n                 \" Config Example:\\n#{config_example_to_recover(@base_buffer_dir)}\"\n        return\n      end\n\n      begin\n        FileUtils.remove_dir(@base_buffer_dir)\n      rescue Errno::ENOENT\n        # This worker doesn't need to do anything. Another worker may remove the dir first.\n      rescue => e\n        log.warn \"failed to remove the buffer directory: #{@base_buffer_dir}\", error: e\n      end\n    end\n\n    def emit_error_event(tag, time, record, error)\n      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}\n      log.warn \"SourceOnlyBufferAgent: dump an error event:\", error_info\n    end\n\n    def handle_emits_error(tag, es, error)\n      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}\n      log.warn \"SourceOnlyBufferAgent: emit transaction failed:\", error_info\n      log.warn_backtrace\n    end\n\n    private\n\n    def config_example_to_recover(path)\n      <<~EOC\n        <system>\n          <source_only_buffer>\n            path #{path}\n          </source_only_buffer>\n        </system>\n      EOC\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/static_config_analysis.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config'\nrequire 'fluent/plugin'\n\nmodule Fluent\n  # Static Analysis means analysing all plugins and Fluent::Element without invoking Plugin#configure\n  class StaticConfigAnalysis\n    module Elem\n      Input = Struct.new(:plugin, :config)\n      Output = Struct.new(:plugin, :config)\n      Filter = Struct.new(:plugin, :config)\n      Label = Struct.new(:name, :config, :nodes)\n      Worker = Struct.new(:ids, :config, :nodes)\n    end\n\n    Result = Struct.new(:tree, :outputs, :inputs, :filters, :labels) do\n      def all_plugins\n        (outputs + inputs + filters).map(&:plugin)\n      end\n    end\n\n    # @param workers [Integer] Number of workers\n    # @return [Fluent::StaticConfigAnalysis::Result]\n    def self.call(conf, workers: 1)\n      new(workers).call(conf)\n    end\n\n    def initialize(workers)\n      @workers = workers\n\n      reset\n    end\n\n    def call(config)\n      reset\n\n      tree = [\n        static_worker_analyse(config),\n        static_label_analyse(config),\n        static_filter_and_output_analyse(config),\n        static_input_analyse(config),\n      ].flatten\n\n      Result.new(tree, @outputs, @inputs, @filters, @labels.values)\n    end\n\n    private\n\n    def reset\n      @outputs = []\n      @inputs = []\n      @filters = []\n      @labels = {}\n    end\n\n    def static_worker_analyse(conf)\n      available_worker_ids = [*0...@workers]\n\n      ret = []\n      supported_directives = %w[source match filter label]\n      conf.elements(name: 'worker').each do |config|\n        ids = parse_worker_id(config)\n        ids.each do |id|\n          if available_worker_ids.include?(id)\n            available_worker_ids.delete(id)\n          else\n            raise Fluent::ConfigError, \"specified worker_id<#{id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}\"\n          end\n        end\n\n        config.elements.each do |elem|\n          unless supported_directives.include?(elem.name)\n            raise Fluent::ConfigError, \"<worker> section cannot have <#{elem.name}> directive\"\n          end\n        end\n\n        nodes = [\n          static_label_analyse(config),\n          static_filter_and_output_analyse(config),\n          static_input_analyse(config),\n        ].flatten\n        ret << Elem::Worker.new(ids, config, nodes)\n      end\n\n      ret\n    end\n\n    def parse_worker_id(conf)\n      worker_id_str = conf.arg\n\n      if worker_id_str.empty?\n        raise Fluent::ConfigError, 'Missing worker id on <worker> directive'\n      end\n\n      l, r =\n         begin\n           worker_id_str.split('-', 2).map { |v| Integer(v) }\n         rescue TypeError, ArgumentError\n           raise Fluent::ConfigError, \"worker id should be integer: #{worker_id_str}\"\n         end\n\n      if l < 0 || l >= @workers\n        raise Fluent::ConfigError, \"worker id #{l} specified by <worker> directive is not allowed. Available worker id is between 0 and #{@workers-1}\"\n      end\n\n      # e.g. specified one worker id like `<worker 0>`\n      if r.nil?\n        return [l]\n      end\n\n      if r < 0 || r >= @workers\n        raise Fluent::ConfigError, \"worker id #{r} specified by <worker> directive is not allowed. Available worker id is between 0 and #{@workers-1}\"\n      end\n\n      if l > r\n        raise Fluent::ConfigError, \"greater first_worker_id<#{l}> than last_worker_id<#{r}> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>\"\n      end\n\n      [l, r]\n    end\n\n    def static_label_analyse(conf)\n      ret = []\n      conf.elements(name: 'label').each do |e|\n        name = e.arg\n        if name.empty?\n          raise ConfigError, 'Missing symbol argument on <label> directive'\n        end\n\n        if @labels[name]\n          raise ConfigError, \"Section <label #{name}> appears twice\"\n        end\n\n        l = Elem::Label.new(name, e, static_filter_and_output_analyse(e))\n        ret << l\n        @labels[name] = l\n      end\n\n      ret\n    end\n\n    def static_filter_and_output_analyse(conf)\n      ret = []\n      conf.elements('filter', 'match').each do |e|\n        type = e['@type']\n        if type.nil? || type.empty?\n          raise Fluent::ConfigError, \"Missing '@type' parameter on <#{e.name}> directive\"\n        end\n\n        if e.name == 'filter'\n          f = Elem::Filter.new(Fluent::Plugin.new_filter(type), e)\n          ret << f\n          @filters << f\n        else\n          o = Elem::Output.new(Fluent::Plugin.new_output(type), e)\n          ret << o\n          @outputs << o\n        end\n      end\n\n      ret\n    end\n\n    def static_input_analyse(conf)\n      ret = []\n      conf.elements(name: 'source').each do |e|\n        type = e['@type']\n        if type.nil? || type.empty?\n          raise Fluent::ConfigError, \"Missing '@type' parameter on <#{e.name}> directive\"\n        end\n\n        i = Elem::Input.new(Fluent::Plugin.new_input(type), e)\n        @inputs << i\n        ret << i\n      end\n\n      ret\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/supervisor.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fileutils'\nrequire 'open3'\nrequire 'pathname'\nrequire 'find'\nrequire 'set'\n\nrequire 'fluent/config'\nrequire 'fluent/counter'\nrequire 'fluent/env'\nrequire 'fluent/engine'\nrequire 'fluent/error'\nrequire 'fluent/log'\nrequire 'fluent/plugin'\nrequire 'fluent/rpc'\nrequire 'fluent/system_config'\nrequire 'fluent/msgpack_factory'\nrequire 'fluent/variable_store'\nrequire 'serverengine'\n\nif Fluent.windows?\n  require 'win32/ipc'\n  require 'win32/event'\nend\n\nmodule Fluent\n  module ServerModule\n    def before_run\n      @fluentd_conf = config[:fluentd_conf]\n      @rpc_endpoint = nil\n      @rpc_server = nil\n      @counter = nil\n      @socket_manager_server = nil\n      @starting_new_supervisor_with_zero_downtime = false\n      @new_supervisor_pid = nil\n      start_in_parallel = ENV.key?(\"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\")\n      @zero_downtime_restart_mutex = Mutex.new\n\n      @fluentd_lock_dir = Dir.mktmpdir(\"fluentd-lock-\")\n      ENV['FLUENTD_LOCK_DIR'] = @fluentd_lock_dir\n\n      if config[:rpc_endpoint] and not start_in_parallel\n        @rpc_endpoint = config[:rpc_endpoint]\n        @enable_get_dump = config[:enable_get_dump]\n        run_rpc_server\n      end\n\n      if Fluent.windows?\n        install_windows_event_handler\n      else\n        install_supervisor_signal_handlers\n      end\n\n      if counter = config[:counter_server] and not start_in_parallel\n        run_counter_server(counter)\n      end\n\n      if config[:disable_shared_socket]\n        $log.info \"shared socket for multiple workers is disabled\"\n      elsif start_in_parallel\n        begin\n          raise \"[BUG] SERVERENGINE_SOCKETMANAGER_PATH env var must exist when starting in parallel\" unless ENV.key?('SERVERENGINE_SOCKETMANAGER_PATH')\n          @socket_manager_server = ServerEngine::SocketManager::Server.share_sockets_with_another_server(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])\n          $log.info \"zero-downtime-restart: took over the shared sockets\", path: ENV['SERVERENGINE_SOCKETMANAGER_PATH']\n        rescue => e\n          $log.error \"zero-downtime-restart: cancel sequence because failed to take over the shared sockets\", error: e\n          raise\n        end\n      else\n        @socket_manager_server = ServerEngine::SocketManager::Server.open\n        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_server.path.to_s\n      end\n\n      stop_parallel_old_supervisor_after_delay if start_in_parallel\n    end\n\n    def after_run\n      stop_windows_event_thread if Fluent.windows?\n      stop_rpc_server if @rpc_endpoint\n      stop_counter_server if @counter\n      cleanup_lock_dir\n      Fluent::Supervisor.cleanup_socketmanager_path unless @starting_new_supervisor_with_zero_downtime\n    ensure\n      notify_new_supervisor_that_old_one_has_stopped if @starting_new_supervisor_with_zero_downtime\n    end\n\n    def cleanup_lock_dir\n      FileUtils.rm(Dir.glob(File.join(@fluentd_lock_dir, \"fluentd-*.lock\")))\n      FileUtils.rmdir(@fluentd_lock_dir)\n    end\n\n    def run_rpc_server\n      @rpc_server = RPC::Server.new(@rpc_endpoint, $log)\n\n      # built-in RPC for signals\n      @rpc_server.mount_proc('/api/processes.interruptWorkers') { |req, res|\n        $log.debug \"fluentd RPC got /api/processes.interruptWorkers request\"\n        Process.kill :INT, Process.pid\n        nil\n      }\n      @rpc_server.mount_proc('/api/processes.killWorkers') { |req, res|\n        $log.debug \"fluentd RPC got /api/processes.killWorkers request\"\n        Process.kill :TERM, Process.pid\n        nil\n      }\n      @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|\n        $log.debug \"fluentd RPC got /api/processes.flushBuffersAndKillWorkers request\"\n        if Fluent.windows?\n          supervisor_sigusr1_handler\n          stop(true)\n        else\n          Process.kill :USR1, Process.pid\n          Process.kill :TERM, Process.pid\n        end\n        nil\n      }\n      unless Fluent.windows?\n        @rpc_server.mount_proc('/api/processes.zeroDowntimeRestart') { |req, res|\n          $log.debug \"fluentd RPC got /api/processes.zeroDowntimeRestart request\"\n          Process.kill :USR2, Process.pid\n          nil\n        }\n      end\n      @rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|\n        $log.debug \"fluentd RPC got /api/plugins.flushBuffers request\"\n        if Fluent.windows?\n          supervisor_sigusr1_handler\n        else\n          Process.kill :USR1, Process.pid\n        end\n        nil\n      }\n      @rpc_server.mount_proc('/api/config.reload') { |req, res|\n        $log.debug \"fluentd RPC got /api/config.reload request\"\n        if Fluent.windows?\n          # restart worker with auto restarting by killing\n          kill_worker\n        else\n          Process.kill :HUP, Process.pid\n        end\n        nil\n      }\n      @rpc_server.mount_proc('/api/config.dump') { |req, res|\n        $log.debug \"fluentd RPC got /api/config.dump request\"\n        $log.info \"dump in-memory config\"\n        supervisor_dump_config_handler\n        nil\n      }\n\n      @rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|\n        $log.debug \"fluentd RPC got /api/config.gracefulReload request\"\n        graceful_reload\n        nil\n      }\n\n      if @enable_get_dump\n        @rpc_server.mount_proc('/api/config.getDump') { |req, res|\n          $log.debug \"fluentd RPC got /api/config.getDump request\"\n          $log.info \"get dump in-memory config via HTTP\"\n          res.body = supervisor_get_dump_config_handler\n          [nil, nil, res]\n        }\n      end\n\n      @rpc_server.start\n    end\n\n    def stop_rpc_server\n      @rpc_server&.shutdown\n    end\n\n    def run_counter_server(counter_conf)\n      @counter = Fluent::Counter::Server.new(\n        counter_conf.scope,\n        {host: counter_conf.bind, port: counter_conf.port, log: $log, path: counter_conf.backup_path}\n      )\n      @counter.start\n    end\n\n    def stop_counter_server\n      @counter.stop\n    end\n\n    def stop_parallel_old_supervisor_after_delay\n      Thread.new do\n        # Delay to wait the new workers to start up.\n        # Even if it takes a long time to start the new workers and stop the old Fluentd first,\n        # it is no problem because the socket buffer works, as long as the capacity is not exceeded.\n        sleep 10\n        old_pid = ENV[\"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\"]&.to_i\n        if old_pid\n          $log.info \"zero-downtime-restart: stop the old supervisor\"\n          Process.kill :TERM, old_pid\n        end\n      rescue => e\n        $log.warn \"zero-downtime-restart: failed to stop the old supervisor.\" +\n                  \" If the old one does not exist, please send SIGWINCH to this new process to start to work fully.\" +\n                  \" If it exists, something went wrong. Please kill the old one manually.\",\n                  error: e\n      end\n    end\n\n    def notify_new_supervisor_that_old_one_has_stopped\n      if config[:pid_path]\n        new_pid = File.read(config[:pid_path]).to_i\n      else\n        raise \"[BUG] new_supervisor_pid is not saved\" unless @new_supervisor_pid\n        new_pid = @new_supervisor_pid\n      end\n\n      $log.info \"zero-downtime-restart: notify the new supervisor (pid: #{new_pid}) that old one has stopped\"\n      Process.kill :WINCH, new_pid\n    rescue => e\n      $log.error(\n        \"zero-downtime-restart: failed to notify the new supervisor.\" +\n        \" Please send SIGWINCH to the new supervisor process manually\" +\n        \" if it does not start to work fully.\",\n        error: e\n      )\n    end\n\n    def install_supervisor_signal_handlers\n      return if Fluent.windows?\n\n      trap :HUP do\n        $log.debug \"fluentd supervisor process get SIGHUP\"\n        supervisor_sighup_handler\n      end\n\n      trap :USR1 do\n        $log.debug \"fluentd supervisor process get SIGUSR1\"\n        supervisor_sigusr1_handler\n      end\n\n      trap :USR2 do\n        $log.debug 'fluentd supervisor process got SIGUSR2'\n        if Fluent.windows?\n          graceful_reload\n        else\n          zero_downtime_restart\n        end\n      end\n\n      trap :WINCH do\n        $log.debug 'fluentd supervisor process got SIGWINCH'\n        cancel_source_only\n      end\n    end\n\n    if Fluent.windows?\n      # Override some methods of ServerEngine::MultiSpawnWorker\n      # Since Fluentd's Supervisor doesn't use ServerEngine's HUP, USR1 and USR2\n      # handlers (see install_supervisor_signal_handlers), they should be\n      # disabled also on Windows, just send commands to workers instead.\n      def restart(graceful)\n        @monitors.each do |m|\n          m.send_command(graceful ? \"GRACEFUL_RESTART\\n\" : \"IMMEDIATE_RESTART\\n\")\n        end\n      end\n\n      def reload\n        @monitors.each do |m|\n          m.send_command(\"RELOAD\\n\")\n        end\n      end\n    end\n\n    def install_windows_event_handler\n      return unless Fluent.windows?\n\n      @pid_signame = \"fluentd_#{Process.pid}\"\n      @signame = config[:signame]\n\n      Thread.new do\n        ipc = Win32::Ipc.new(nil)\n        events = [\n          {win32_event: Win32::Event.new(\"#{@pid_signame}_STOP_EVENT_THREAD\"), action: :stop_event_thread},\n          {win32_event: Win32::Event.new(\"#{@pid_signame}\"), action: :stop},\n          {win32_event: Win32::Event.new(\"#{@pid_signame}_HUP\"), action: :hup},\n          {win32_event: Win32::Event.new(\"#{@pid_signame}_USR1\"), action: :usr1},\n          {win32_event: Win32::Event.new(\"#{@pid_signame}_USR2\"), action: :usr2},\n          {win32_event: Win32::Event.new(\"#{@pid_signame}_CONT\"), action: :cont},\n        ]\n        if @signame\n          signame_events = [\n            {win32_event: Win32::Event.new(\"#{@signame}\"), action: :stop},\n            {win32_event: Win32::Event.new(\"#{@signame}_HUP\"), action: :hup},\n            {win32_event: Win32::Event.new(\"#{@signame}_USR1\"), action: :usr1},\n            {win32_event: Win32::Event.new(\"#{@signame}_USR2\"), action: :usr2},\n            {win32_event: Win32::Event.new(\"#{@signame}_CONT\"), action: :cont},\n          ]\n          events.concat(signame_events)\n        end\n        begin\n          loop do\n            infinite = 0xFFFFFFFF\n            ipc_idx = ipc.wait_any(events.map {|e| e[:win32_event]}, infinite)\n            event_idx = ipc_idx - 1\n\n            if event_idx >= 0 && event_idx < events.length\n              $log.debug(\"Got Win32 event \\\"#{events[event_idx][:win32_event].name}\\\"\")\n            else\n              $log.warn(\"Unexpected return value of Win32::Ipc#wait_any: #{ipc_idx}\")\n            end\n            case events[event_idx][:action]\n            when :stop\n              stop(true)\n            when :hup\n              supervisor_sighup_handler\n            when :usr1\n              supervisor_sigusr1_handler\n            when :usr2\n              graceful_reload\n            when :cont\n              supervisor_dump_handler_for_windows\n            when :stop_event_thread\n              break\n            end\n          end\n        ensure\n          events.each { |event| event[:win32_event].close }\n        end\n      end\n    end\n\n    def stop_windows_event_thread\n      if Fluent.windows?\n        ev = Win32::Event.open(\"#{@pid_signame}_STOP_EVENT_THREAD\")\n        ev.set\n        ev.close\n      end\n    end\n\n    def supervisor_sighup_handler\n      kill_worker\n    end\n\n    def supervisor_sigusr1_handler\n      reopen_log\n      send_signal_to_workers(:USR1)\n    end\n\n    def graceful_reload\n      conf = nil\n      t = Thread.new do\n        $log.info 'Reloading new config'\n\n        # Validate that loading config is valid at first\n        conf = Fluent::Config.build(\n          config_path: config[:config_path],\n          encoding: config[:conf_encoding],\n          additional_config: config[:inline_config],\n          use_v1_config: config[:use_v1_config],\n        )\n\n        Fluent::VariableStore.try_to_reset do\n          Fluent::Engine.reload_config(conf, supervisor: true)\n        end\n      end\n\n      t.report_on_exception = false # Error is handled by myself\n      t.join\n\n      reopen_log\n      send_signal_to_workers(:USR2)\n      @fluentd_conf = conf.to_s\n    rescue => e\n      $log.error \"Failed to reload config file: #{e}\"\n    end\n\n    def zero_downtime_restart\n      Thread.new do\n        @zero_downtime_restart_mutex.synchronize do\n          $log.info \"start zero-downtime-restart sequence\"\n\n          if @starting_new_supervisor_with_zero_downtime\n            $log.warn \"zero-downtime-restart: canceled because it is already starting\"\n            Thread.exit\n          end\n          if ENV.key?(\"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\")\n            $log.warn \"zero-downtime-restart: canceled because the previous sequence is still running\"\n            Thread.exit\n          end\n\n          @starting_new_supervisor_with_zero_downtime = true\n          commands = [ServerEngine.ruby_bin_path, $0] + ARGV\n          env_to_add = {\n            \"SERVERENGINE_SOCKETMANAGER_INTERNAL_TOKEN\" => ServerEngine::SocketManager::INTERNAL_TOKEN,\n            \"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\" => \"#{Process.pid}\",\n          }\n          pid = Process.spawn(env_to_add, commands.join(\" \"))\n          @new_supervisor_pid = pid unless config[:daemonize]\n\n          if config[:daemonize]\n            Thread.new(pid) do |pid|\n              _, status = Process.wait2(pid)\n              # check if `ServerEngine::Daemon#daemonize_with_double_fork` succeeded or not\n              unless status.success?\n                @starting_new_supervisor_with_zero_downtime = false\n                $log.error \"zero-downtime-restart: failed because new supervisor exits unexpectedly\"\n              end\n            end\n          else\n            Thread.new(pid) do |pid|\n              _, status = Process.wait2(pid)\n              @starting_new_supervisor_with_zero_downtime = false\n              $log.error \"zero-downtime-restart: failed because new supervisor exits unexpectedly\", status: status\n            end\n          end\n        end\n      rescue => e\n        $log.error \"zero-downtime-restart: failed\", error: e\n        @starting_new_supervisor_with_zero_downtime = false\n      end\n    end\n\n    def cancel_source_only\n      if ENV.key?(\"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\")\n        if config[:rpc_endpoint]\n          begin\n            @rpc_endpoint = config[:rpc_endpoint]\n            @enable_get_dump = config[:enable_get_dump]\n            run_rpc_server\n          rescue => e\n            $log.error \"failed to start RPC server\", error: e\n          end\n        end\n\n        if counter = config[:counter_server]\n          begin\n            run_counter_server(counter)\n          rescue => e\n            $log.error \"failed to start counter server\", error: e\n          end\n        end\n\n        $log.info \"zero-downtime-restart: done all sequences, now new processes start to work fully\"\n        ENV.delete(\"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\")\n      end\n\n      send_signal_to_workers(:WINCH)\n    end\n\n    def supervisor_dump_handler_for_windows\n      # As for UNIX-like, SIGCONT signal to each process makes the process output its dump-file,\n      # and it is implemented before the implementation of the function for Windows.\n      # It is possible to trap SIGCONT and handle it here also on UNIX-like,\n      # but for backward compatibility, this handler is currently for a Windows-only.\n      raise \"[BUG] This function is for Windows ONLY.\" unless Fluent.windows?\n\n      Thread.new do\n        begin\n          FluentSigdump.dump_windows\n        rescue => e\n          $log.error \"failed to dump: #{e}\"\n        end\n      end\n\n      send_signal_to_workers(:CONT)\n    rescue => e\n      $log.error \"failed to dump: #{e}\"\n    end\n\n    def kill_worker\n      if config[:worker_pid]\n        pids = config[:worker_pid].clone\n        config[:worker_pid].clear\n        pids.each_value do |pid|\n          if Fluent.windows?\n            Process.kill :KILL, pid\n          else\n            Process.kill :TERM, pid\n          end\n        end\n      end\n    end\n\n    def supervisor_dump_config_handler\n      $log.info @fluentd_conf\n    end\n\n    def supervisor_get_dump_config_handler\n      { conf: @fluentd_conf }\n    end\n\n    def dump\n      super unless @stop\n    end\n\n    private\n\n    def reopen_log\n      if $log\n        # Creating new thread due to mutex can't lock\n        # in main thread during trap context\n        Thread.new do\n          $log.reopen!\n        end\n      end\n    end\n\n    def send_signal_to_workers(signal)\n      return unless config[:worker_pid]\n\n      if Fluent.windows?\n        send_command_to_workers(signal)\n      else\n        config[:worker_pid].each_value do |pid|\n          # don't rescue Errno::ESRCH here (invalid status)\n          Process.kill(signal, pid)\n        end\n      end\n    end\n\n    def send_command_to_workers(signal)\n      # Use SeverEngine's CommandSender on Windows\n      case signal\n      when :HUP\n        restart(false)\n      when :USR1\n        restart(true)\n      when :USR2\n        reload\n      when :CONT\n        dump_all_windows_workers\n      end\n    end\n\n    def dump_all_windows_workers\n      @monitors.each do |m|\n        m.send_command(\"DUMP\\n\")\n      end\n    end\n  end\n\n  module WorkerModule\n    def spawn(process_manager)\n      main_cmd = config[:main_cmd]\n      env = {\n        'SERVERENGINE_WORKER_ID' => @worker_id.to_i.to_s,\n        'FLUENT_INSTANCE_ID' => Fluent::INSTANCE_ID,\n      }\n      @pm = process_manager.spawn(env, *main_cmd)\n    end\n\n    def after_start\n      (config[:worker_pid] ||= {})[@worker_id] = @pm.pid\n    end\n\n    def dump\n      super unless @stop\n    end\n  end\n\n  class Supervisor\n    def self.serverengine_config(params = {})\n      # ServerEngine's \"daemonize\" option is boolean, and path of pid file is brought by \"pid_path\"\n      pid_path = params['daemonize']\n      daemonize = !!params['daemonize']\n\n      se_config = {\n        worker_type: 'spawn',\n        workers: params['workers'],\n        log_stdin: false,\n        log_stdout: false,\n        log_stderr: false,\n        enable_heartbeat: true,\n        auto_heartbeat: false,\n        unrecoverable_exit_codes: [2],\n        stop_immediately_at_unrecoverable_exit: true,\n        root_dir: params['root_dir'],\n        logger: $log,\n        log: $log&.out,\n        log_level: params['log_level'],\n        chuser: params['chuser'],\n        chgroup: params['chgroup'],\n        chumask: params['chumask'].is_a?(Integer) ? params['chumask'] : params['chumask']&.to_i(8),\n        daemonize: daemonize,\n        rpc_endpoint: params['rpc_endpoint'],\n        counter_server: params['counter_server'],\n        enable_get_dump: params['enable_get_dump'],\n        windows_daemon_cmdline: [ServerEngine.ruby_bin_path,\n                                 File.join(File.dirname(__FILE__), 'daemon.rb'),\n                                 ServerModule.name,\n                                 WorkerModule.name,\n                                 JSON.dump(params)],\n        command_sender: Fluent.windows? ? \"pipe\" : \"signal\",\n        config_path: params['fluentd_conf_path'],\n        fluentd_conf: params['fluentd_conf'],\n        conf_encoding: params['conf_encoding'],\n        inline_config: params['inline_config'],\n        main_cmd: params['main_cmd'],\n        signame: params['signame'],\n        disable_shared_socket: params['disable_shared_socket'],\n        restart_worker_interval: params['restart_worker_interval'],\n      }\n      se_config[:pid_path] = pid_path if daemonize\n\n      se_config\n    end\n\n    def self.default_options\n      {\n        config_path: Fluent::DEFAULT_CONFIG_PATH,\n        plugin_dirs: [Fluent::DEFAULT_PLUGIN_DIR],\n        log_level: Fluent::Log::LEVEL_INFO,\n        log_path: nil,\n        daemonize: nil,\n        libs: [],\n        setup_path: nil,\n        chuser: nil,\n        chgroup: nil,\n        chumask: \"0\",\n        root_dir: nil,\n        suppress_interval: 0,\n        suppress_repeated_stacktrace: true,\n        ignore_repeated_log_interval: nil,\n        without_source: nil,\n        with_source_only: nil,\n        enable_input_metrics: true,\n        enable_size_metrics: nil,\n        use_v1_config: true,\n        strict_config_value: nil,\n        supervise: true,\n        standalone_worker: false,\n        signame: nil,\n        conf_encoding: 'utf-8',\n        disable_shared_socket: nil,\n        config_file_type: :guess,\n      }\n    end\n\n    def self.cleanup_socketmanager_path\n      return if Fluent.windows?\n      return unless ENV.key?('SERVERENGINE_SOCKETMANAGER_PATH')\n\n      FileUtils.rm_f(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])\n    end\n\n    def initialize(cl_opt)\n      @cl_opt = cl_opt\n      opt = self.class.default_options.merge(cl_opt)\n\n      @config_file_type = opt[:config_file_type]\n      @daemonize = opt[:daemonize]\n      @standalone_worker= opt[:standalone_worker]\n      @config_path = opt[:config_path]\n      @inline_config = opt[:inline_config]\n      @use_v1_config = opt[:use_v1_config]\n      @conf_encoding = opt[:conf_encoding]\n      @show_plugin_config = opt[:show_plugin_config]\n      @libs = opt[:libs]\n      @plugin_dirs = opt[:plugin_dirs]\n      @chgroup = opt[:chgroup]\n      @chuser = opt[:chuser]\n      @chumask = opt[:chumask]\n      @signame = opt[:signame]\n\n      # TODO: `@log_path`, `@log_rotate_age` and `@log_rotate_size` should be removed\n      # since it should be merged with SystemConfig in `build_system_config()`.\n      # We should always use `system_config.log.path`, `system_config.log.rotate_age`\n      # and `system_config.log.rotate_size`.\n      # However, currently, there is a bug that `system_config.log` parameters\n      # are not in `Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS`, and these\n      # parameters are not merged in `build_system_config()`.\n      # Until we fix the bug of `Fluent::SystemConfig`, we need to use these instance variables.\n      @log_path = opt[:log_path]\n      @log_rotate_age = opt[:log_rotate_age]\n      @log_rotate_size = opt[:log_rotate_size]\n\n      @finished = false\n    end\n\n    def run_supervisor(dry_run: false)\n      if dry_run\n        $log.info \"starting fluentd-#{Fluent::VERSION} as dry run mode\", ruby: RUBY_VERSION\n      end\n\n      if @system_config.workers < 1\n        raise Fluent::ConfigError, \"invalid number of workers (must be > 0):#{@system_config.workers}\"\n      end\n\n      if Fluent.windows? && @system_config.with_source_only\n        raise Fluent::ConfigError, \"with-source-only is not supported on Windows\"\n      end\n\n      root_dir = @system_config.root_dir\n      if root_dir\n        if File.exist?(root_dir)\n          unless Dir.exist?(root_dir)\n            raise Fluent::InvalidRootDirectory, \"non directory entry exists:#{root_dir}\"\n          end\n        else\n          begin\n            FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION)\n          rescue => e\n            raise Fluent::InvalidRootDirectory, \"failed to create root directory:#{root_dir}, #{e.inspect}\"\n          end\n        end\n      end\n\n      begin\n        ServerEngine::Privilege.change(@chuser, @chgroup)\n        MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)\n        Fluent::Engine.init(@system_config, supervisor_mode: true, start_in_parallel: ENV.key?(\"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\"))\n        Fluent::Engine.run_configure(@conf, dry_run: dry_run)\n      rescue Fluent::ConfigError => e\n        $log.error 'config error', file: @config_path, error: e\n        $log.debug_backtrace\n        exit!(1)\n      rescue ScriptError => e # LoadError, NotImplementedError, SyntaxError\n        if e.respond_to?(:path)\n          $log.error e.message, path: e.path, error: e\n        else\n          $log.error e.message, error: e\n        end\n        $log.debug_backtrace\n        exit!(1)\n      rescue => e\n        $log.error \"unexpected error\", error: e\n        $log.debug_backtrace\n        exit!(1)\n      end\n\n      if dry_run\n        $log.info 'finished dry run mode'\n        exit 0\n      else\n        supervise\n      end\n    end\n\n    def options\n      {\n        'config_path' => @config_path,\n        'pid_file' => @daemonize,\n        'plugin_dirs' => @plugin_dirs,\n        'log_path' => @log_path,\n        'root_dir' => @system_config.root_dir,\n      }\n    end\n\n    def run_worker\n      Process.setproctitle(\"worker:#{@system_config.process_name}\") if @process_name\n\n      if @standalone_worker && @system_config.workers != 1\n        raise Fluent::ConfigError, \"invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@system_config.workers}\"\n      end\n\n      if Fluent.windows? && @system_config.with_source_only\n        raise Fluent::ConfigError, \"with-source-only is not supported on Windows\"\n      end\n\n      install_main_process_signal_handlers\n\n      # This is the only log message for @standalone_worker\n      $log.info \"starting fluentd-#{Fluent::VERSION} without supervision\", pid: Process.pid, ruby: RUBY_VERSION if @standalone_worker\n\n      main_process do\n        create_socket_manager if @standalone_worker\n        if @standalone_worker\n          ServerEngine::Privilege.change(@chuser, @chgroup)\n          File.umask(@chumask.to_i(8))\n        end\n        MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)\n        Fluent::Engine.init(@system_config, start_in_parallel: ENV.key?(\"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\"))\n        Fluent::Engine.run_configure(@conf)\n        Fluent::Engine.run\n        self.class.cleanup_socketmanager_path if @standalone_worker\n        exit 0\n      end\n    end\n\n    def configure(supervisor: false)\n      setup_global_logger(supervisor: supervisor)\n\n      if @show_plugin_config\n        show_plugin_config\n      end\n\n      if @inline_config == '-'\n        $log.warn('the value \"-\" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')\n        @inline_config = STDIN.read\n      end\n      parsed_files = Set.new\n      @conf = Fluent::Config.build(\n        config_path: @config_path,\n        encoding: @conf_encoding,\n        additional_config: @inline_config,\n        use_v1_config: @use_v1_config,\n        type: @config_file_type,\n        on_file_parsed: ->(path) { parsed_files << path },\n      )\n      @system_config = build_system_config(@conf)\n\n      $log.info :supervisor, 'parsing config file is succeeded', path: @config_path\n\n      build_additional_configurations(parsed_files) do |additional_conf|\n        @conf += additional_conf\n      end\n\n      if Fluent.windows?\n        @conf.elements.each do |element|\n          next unless element.name == 'source'\n          if element['@type'] == 'tail' && element['pos_file']\n            $log.warn(\"Recommend adding #{File.dirname(element['pos_file'])} to the exclusion path of your antivirus software on Windows\")\n          else\n            storage = element.elements.find { |v| v.name == 'storage' and v['@type'] == 'local' }\n            if storage\n              $log.warn(\"Recommend adding #{File.dirname(storage['path'])} to the exclusion path of your antivirus software on Windows\")\n            end\n          end\n        end\n      end\n\n      @libs.each do |lib|\n        require lib\n      end\n\n      @plugin_dirs.each do |dir|\n        if Dir.exist?(dir)\n          dir = File.expand_path(dir)\n          Fluent::Plugin.add_plugin_dir(dir)\n        end\n      end\n\n      if supervisor\n        # plugins / configuration dumps\n        Gem::Specification.find_all.select { |x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/ }.each do |spec|\n          $log.info(\"gem '#{spec.name}' version '#{spec.version}'\")\n        end\n      end\n    end\n\n    private\n\n    def setup_global_logger(supervisor: false)\n      if supervisor\n        worker_id = 0\n        process_type = :supervisor\n      else\n        worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i\n        process_type = case\n                       when @standalone_worker then :standalone\n                       when worker_id == 0 then :worker0\n                       else :workers\n                       end\n      end\n\n      # Parse configuration immediately to initialize logger in early stage.\n      # Since we can't confirm the log messages in this parsing process,\n      # we must parse the config again after initializing logger.\n      conf = Fluent::Config.build(\n        config_path: @config_path,\n        encoding: @conf_encoding,\n        additional_config: @inline_config,\n        use_v1_config: @use_v1_config,\n        type: @config_file_type,\n        on_file_parsed: nil,\n      )\n      system_config = build_system_config(conf)\n\n      # TODO: we should remove this logic. This merging process should be done\n      # in `build_system_config()`.\n      @log_path ||= system_config.log.path\n      @log_rotate_age ||= system_config.log.rotate_age\n      @log_rotate_size ||= system_config.log.rotate_size\n\n      rotate = @log_rotate_age || @log_rotate_size\n      actual_log_path = @log_path\n\n      # We need to prepare a unique path for each worker since Windows locks files.\n      if Fluent.windows? && rotate && @log_path && @log_path != \"-\"\n        actual_log_path = Fluent::Log.per_process_path(@log_path, process_type, worker_id)\n      end\n\n      if actual_log_path && actual_log_path != \"-\"\n        FileUtils.mkdir_p(File.dirname(actual_log_path)) unless File.exist?(actual_log_path)\n        if rotate\n          logdev = Fluent::LogDeviceIO.new(\n            actual_log_path,\n            shift_age: @log_rotate_age,\n            shift_size: @log_rotate_size,\n          )\n        else\n          logdev = File.open(actual_log_path, \"a\")\n        end\n\n        if @chuser || @chgroup\n          chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil\n          chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil\n          File.chown(chuid, chgid, actual_log_path)\n        end\n\n        if system_config.dir_permission\n          File.chmod(system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(actual_log_path))\n        end\n      else\n        logdev = STDOUT\n      end\n\n      $log = Fluent::Log.new(\n        # log_level: subtract 1 to match serverengine daemon logger side logging severity.\n        ServerEngine::DaemonLogger.new(logdev, log_level: system_config.log_level - 1),\n        path: actual_log_path,\n        process_type: process_type,\n        worker_id: worker_id,\n        format: system_config.log.format,\n        time_format: system_config.log.time_format,\n        suppress_repeated_stacktrace: system_config.suppress_repeated_stacktrace,\n        ignore_repeated_log_interval: system_config.ignore_repeated_log_interval,\n        ignore_same_log_interval: system_config.ignore_same_log_interval,\n      )\n      $log.force_stacktrace_level(system_config.log.forced_stacktrace_level) if system_config.force_stacktrace_level?\n      $log.enable_color(false) if actual_log_path\n      $log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG\n\n      $log.info \"init #{process_type} logger\",\n                path: actual_log_path,\n                rotate_age: @log_rotate_age,\n                rotate_size: @log_rotate_size\n    end\n\n    def create_socket_manager\n      server = ServerEngine::SocketManager::Server.open\n      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s\n    end\n\n    def show_plugin_config\n      name, type = @show_plugin_config.split(\":\") # input:tail\n      $log.info \"show_plugin_config option is deprecated. Use fluent-plugin-config-format --format=txt #{name} #{type}\"\n      exit 0\n    end\n\n    def supervise\n      Process.setproctitle(\"supervisor:#{@system_config.process_name}\") if @system_config.process_name\n      $log.info \"starting fluentd-#{Fluent::VERSION}\", pid: Process.pid, ruby: RUBY_VERSION\n\n      fluentd_spawn_cmd = build_spawn_command\n      $log.info \"spawn command to main: \", cmdline: fluentd_spawn_cmd\n\n      params = {\n        'main_cmd' => fluentd_spawn_cmd,\n        'daemonize' => @daemonize,\n        'inline_config' => @inline_config,\n        'chuser' => @chuser,\n        'chgroup' => @chgroup,\n        'chumask' => @chumask,\n        'fluentd_conf_path' => @config_path,\n        'fluentd_conf' => @conf.to_s,\n        'use_v1_config' => @use_v1_config,\n        'conf_encoding' => @conf_encoding,\n        'signame' => @signame,\n\n        'workers' => @system_config.workers,\n        'root_dir' => @system_config.root_dir,\n        'log_level' => @system_config.log_level,\n        'rpc_endpoint' => @system_config.rpc_endpoint,\n        'enable_get_dump' => @system_config.enable_get_dump,\n        'counter_server' => @system_config.counter_server,\n        'disable_shared_socket' => @system_config.disable_shared_socket,\n        'restart_worker_interval' => @system_config.restart_worker_interval,\n      }\n\n      se = ServerEngine.create(ServerModule, WorkerModule) {\n        # Note: This is called only at the initialization of ServerEngine, since\n        # Fluentd overwrites all related SIGNAL(HUP,USR1,USR2) and have own reloading feature.\n        Fluent::Supervisor.serverengine_config(params)\n      }\n\n      se.run\n    end\n\n    def install_main_process_signal_handlers\n      # Fluentd worker process (worker of ServerEngine) don't use code in serverengine to set signal handlers,\n      # because it does almost nothing.\n      # This method is the only method to set signal handlers in Fluentd worker process.\n\n      # When user use Ctrl + C not SIGINT, SIGINT is sent to all process in same process group.\n      # ServerEngine server process will send SIGTERM to child(spawned) processes by that SIGINT, so\n      # worker process SHOULD NOT do anything with SIGINT, SHOULD just ignore.\n      trap :INT do\n        $log.debug \"fluentd main process get SIGINT\"\n\n        # When Fluentd is launched without supervisor, worker should handle ctrl-c by itself\n        if @standalone_worker\n          @finished = true\n          $log.debug \"getting start to shutdown main process\"\n          Fluent::Engine.stop\n        end\n      end\n\n      trap :TERM do\n        $log.debug \"fluentd main process get SIGTERM\"\n        unless @finished\n          @finished = true\n          $log.debug \"getting start to shutdown main process\"\n          Fluent::Engine.stop\n        end\n      end\n\n      if Fluent.windows?\n        install_main_process_command_handlers\n      else\n        trap :USR1 do\n          flush_buffer\n        end\n\n        trap :USR2 do\n          # Leave the old GracefulReload feature, just in case.\n          # We can send SIGUSR2 to the worker process to use this old GracefulReload feature.\n          # (Note: Normally, we can send SIGUSR2 to the supervisor process to use\n          #  zero-downtime-restart feature as GracefulReload on non-Windows.)\n          reload_config\n        end\n\n        trap :CONT do\n          dump_non_windows\n        end\n\n        trap :WINCH do\n          cancel_source_only\n        end\n      end\n    end\n\n    def install_main_process_command_handlers\n      command_pipe = $stdin.dup\n      $stdin.reopen(File::NULL, \"rb\")\n      command_pipe.binmode\n      command_pipe.sync = true\n\n      Thread.new do\n        loop do\n          cmd = command_pipe.gets\n          break unless cmd\n\n          case cmd.chomp!\n          when \"GRACEFUL_STOP\", \"IMMEDIATE_STOP\"\n            $log.debug \"fluentd main process get #{cmd} command\"\n            @finished = true\n            $log.debug \"getting start to shutdown main process\"\n            Fluent::Engine.stop\n            break\n          when \"GRACEFUL_RESTART\"\n            $log.debug \"fluentd main process get #{cmd} command\"\n            flush_buffer\n          when \"RELOAD\"\n            $log.debug \"fluentd main process get #{cmd} command\"\n            reload_config\n          when \"DUMP\"\n            $log.debug \"fluentd main process get #{cmd} command\"\n            dump_windows\n          else\n            $log.warn \"fluentd main process get unknown command [#{cmd}]\"\n          end\n        end\n      end\n    end\n\n    def flush_buffer\n      # Creating new thread due to mutex can't lock\n      # in main thread during trap context\n      Thread.new do\n        begin\n          $log.debug \"fluentd main process get SIGUSR1\"\n          $log.info \"force flushing buffered events\"\n          $log.reopen!\n          Fluent::Engine.flush!\n          $log.debug \"flushing thread: flushed\"\n        rescue Exception => e\n          $log.warn \"flushing thread error: #{e}\"\n        end\n      end\n    end\n\n    def cancel_source_only\n      Thread.new do\n        begin\n          $log.debug \"fluentd main process get SIGWINCH\"\n          $log.info \"try to cancel with-source-only mode\"\n          Fluent::Engine.cancel_source_only!\n        rescue Exception => e\n          $log.warn \"failed to cancel source only\", error: e\n        end\n      end\n    end\n\n    def reload_config\n      Thread.new do\n        $log.debug('worker got SIGUSR2')\n\n        begin\n          parsed_files = Set.new\n          conf = Fluent::Config.build(\n            config_path: @config_path,\n            encoding: @conf_encoding,\n            additional_config: @inline_config,\n            use_v1_config: @use_v1_config,\n            type: @config_file_type,\n            on_file_parsed: ->(path) { parsed_files << path },\n          )\n\n          build_additional_configurations(parsed_files) do |additional_conf|\n            conf += additional_conf\n          end\n\n          Fluent::VariableStore.try_to_reset do\n            Fluent::Engine.reload_config(conf)\n          end\n        rescue => e\n          # it is guaranteed that config file is valid by supervisor side. but it's not atomic because of using signals to commnicate between worker and super\n          # So need this rescue code\n          $log.error(\"failed to reload config: #{e}\")\n          next\n        end\n\n        @conf = conf\n      end\n    end\n\n    def dump_non_windows\n      begin\n        Sigdump.dump unless @finished\n      rescue => e\n        $log.error(\"failed to dump: #{e}\")\n      end\n    end\n\n    def dump_windows\n      Thread.new do\n        begin\n          FluentSigdump.dump_windows\n        rescue => e\n          $log.error(\"failed to dump: #{e}\")\n        end\n      end\n    end\n\n    def logging_with_console_output\n      yield $log\n      unless $log.stdout?\n        logger = ServerEngine::DaemonLogger.new(STDOUT)\n        log = Fluent::Log.new(logger)\n        log.level = @system_config.log_level\n        console = log.enable_debug\n        yield console\n      end\n    end\n\n    def main_process(&block)\n      if @system_config.process_name\n        if @system_config.workers > 1\n          Process.setproctitle(\"worker:#{@system_config.process_name}#{ENV['SERVERENGINE_WORKER_ID']}\")\n        else\n          Process.setproctitle(\"worker:#{@system_config.process_name}\")\n        end\n      end\n\n      unrecoverable_error = false\n\n      begin\n        block.call\n      rescue Fluent::ConfigError => e\n        logging_with_console_output do |log|\n          log.error \"config error\", file: @config_path, error: e\n          log.debug_backtrace\n        end\n        unrecoverable_error = true\n      rescue Fluent::UnrecoverableError => e\n        logging_with_console_output do |log|\n          log.error e.message, error: e\n          log.error_backtrace\n        end\n        unrecoverable_error = true\n      rescue ScriptError => e # LoadError, NotImplementedError, SyntaxError\n        logging_with_console_output do |log|\n          if e.respond_to?(:path)\n            log.error e.message, path: e.path, error: e\n          else\n            log.error e.message, error: e\n          end\n          log.error_backtrace\n        end\n        unrecoverable_error = true\n      rescue => e\n        logging_with_console_output do |log|\n          log.error \"unexpected error\", error: e\n          log.error_backtrace\n        end\n      end\n\n      exit!(unrecoverable_error ? 2 : 1)\n    end\n\n    def build_system_config(conf)\n      system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])\n      # Prefer the options explicitly specified in the command line\n      #\n      # TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are\n      # not merged with the command line options since they are not in\n      # `SYSTEM_CONFIG_PARAMETERS`.\n      # We have to fix this bug.\n      opt = {}\n      Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|\n        if @cl_opt.key?(param) && !@cl_opt[param].nil?\n          opt[param] = @cl_opt[param]\n        end\n      end\n      system_config.overwrite_variables(**opt)\n      system_config\n    end\n\n    def build_additional_configurations(parsed_files)\n      if @system_config.config_include_dir&.empty?\n        $log.info :supervisor, 'configuration include directory is disabled'\n        return\n      end\n      begin\n        supported_suffixes = [\".conf\", \".yaml\", \".yml\"]\n        Find.find(@system_config.config_include_dir) do |path|\n          next if File.directory?(path)\n          next unless supported_suffixes.include?(File.extname(path))\n          # NOTE: both types of normal config (.conf) and YAML will be loaded.\n          # Thus, it does not care whether @config_path is .conf or .yml.\n          if parsed_files.include?(path)\n            $log.info :supervisor, 'skip auto loading, it was already loaded', path: path\n            next\n          end\n\n          $log.info :supervisor, 'loading additional configuration file', path: path\n          yield Fluent::Config.build(config_path: path,\n                                     encoding: @conf_encoding,\n                                     use_v1_config: @use_v1_config,\n                                     type: :guess,\n                                     on_file_parsed: nil)\n        end\n      rescue Errno::ENOENT\n        $log.info :supervisor, 'inaccessible include directory was specified', path: @system_config.config_include_dir\n      end\n    end\n\n    RUBY_ENCODING_OPTIONS_REGEX = %r{\\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze\n\n    def build_spawn_command\n      if ENV['TEST_RUBY_PATH']\n        fluentd_spawn_cmd = [ENV['TEST_RUBY_PATH']]\n      else\n        fluentd_spawn_cmd = [ServerEngine.ruby_bin_path]\n      end\n\n      rubyopt = ENV['RUBYOPT']\n      if rubyopt\n        encodes, others = rubyopt.split(' ').partition { |e| e.match?(RUBY_ENCODING_OPTIONS_REGEX) }\n        fluentd_spawn_cmd.concat(others)\n\n        adopted_encodes = encodes.empty? ? ['-Eascii-8bit:ascii-8bit'] : encodes\n        fluentd_spawn_cmd.concat(adopted_encodes)\n      else\n        fluentd_spawn_cmd << '-Eascii-8bit:ascii-8bit'\n      end\n\n      if @system_config.enable_jit\n        $log.info \"enable Ruby JIT for workers (--jit)\"\n        fluentd_spawn_cmd << '--jit'\n      end\n\n      # Adding `-h` so that it can avoid ruby's command blocking\n      # e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.\n      _, e, s = Open3.capture3(*fluentd_spawn_cmd, \"-h\")\n      if s.exitstatus != 0\n        $log.error('Invalid option is passed to RUBYOPT', command: fluentd_spawn_cmd, error: e)\n        exit s.exitstatus\n      end\n\n      fluentd_spawn_cmd << $0\n      fluentd_spawn_cmd += $fluentdargv\n      fluentd_spawn_cmd << '--under-supervisor'\n\n      fluentd_spawn_cmd\n    end\n  end\n\n  module FluentSigdump\n    def self.dump_windows\n      raise \"[BUG] WindowsSigdump::dump is for Windows ONLY.\" unless Fluent.windows?\n\n      # Sigdump outputs under `/tmp` dir without `SIGDUMP_PATH` specified,\n      # but `/tmp` dir may not exist on Windows by default.\n      # So use the systemroot-temp-dir instead.\n      dump_filepath = ENV['SIGDUMP_PATH'].nil? || ENV['SIGDUMP_PATH'].empty? \\\n        ? \"#{ENV['windir']}/Temp/fluentd-sigdump-#{Process.pid}.log\"\n        : get_path_with_pid(ENV['SIGDUMP_PATH'])\n\n      require 'sigdump'\n      Sigdump.dump(dump_filepath)\n\n      $log.info \"dump to #{dump_filepath}.\"\n    end\n\n    def self.get_path_with_pid(raw_path)\n      path = Pathname.new(raw_path)\n      path.sub_ext(\"-#{Process.pid}#{path.extname}\").to_s\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/system_config.rb",
    "content": "#\n# Fluent\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/configurable'\nrequire 'fluent/config/element'\nrequire 'fluent/env'\n\nmodule Fluent\n  class SystemConfig\n    include Configurable\n\n    SYSTEM_CONFIG_PARAMETERS = [\n      :workers, :restart_worker_interval, :root_dir, :log_level,\n      :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,\n      :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,\n      :without_source, :with_source_only, :rpc_endpoint, :enable_get_dump, :process_name,\n      :file_permission, :dir_permission, :counter_server, :counter_client,\n      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,\n      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer,\n      :config_include_dir\n    ]\n\n    config_param :workers,   :integer, default: 1\n    config_param :restart_worker_interval, :time, default: 0\n    config_param :root_dir,  :string, default: nil\n    config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'\n    config_param :suppress_repeated_stacktrace, :bool, default: nil\n    config_param :ignore_repeated_log_interval, :time, default: nil\n    config_param :ignore_same_log_interval, :time, default: nil\n    config_param :emit_error_log_interval,      :time, default: nil\n    config_param :suppress_config_dump, :bool, default: nil\n    config_param :log_event_verbose,    :bool, default: nil\n    config_param :without_source, :bool, default: nil\n    config_param :with_source_only, :bool, default: nil\n    config_param :rpc_endpoint,    :string, default: nil\n    config_param :enable_get_dump, :bool, default: nil\n    config_param :process_name,    :string, default: nil\n    config_param :strict_config_value, :bool, default: nil\n    config_param :enable_msgpack_time_support, :bool, default: nil\n    config_param :disable_shared_socket, :bool, default: nil\n    config_param :enable_input_metrics, :bool, default: true\n    config_param :enable_size_metrics, :bool, default: nil\n    config_param :enable_jit, :bool, default: false\n    config_param :file_permission, default: nil do |v|\n      v.to_i(8)\n    end\n    config_param :dir_permission, default: nil do |v|\n      v.to_i(8)\n    end\n    config_param :config_include_dir, default: Fluent::DEFAULT_CONFIG_INCLUDE_DIR\n    config_section :log, required: false, init: true, multi: false do\n      config_param :path, :string, default: nil\n      config_param :format, :enum, list: [:text, :json], default: :text\n      config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'\n      config_param :rotate_age, default: nil do |v|\n        if Fluent::Log::LOG_ROTATE_AGE.include?(v)\n          v\n        else\n          begin\n            Integer(v)\n          rescue ArgumentError => e\n            raise Fluent::ConfigError, e.message\n          else\n            v.to_i\n          end\n        end\n      end\n      config_param :rotate_size, :size, default: nil\n      config_param :forced_stacktrace_level, :enum, list: [:none, :trace, :debug, :info, :warn, :error, :fatal], default: :none\n    end\n\n    config_section :counter_server, multi: false do\n      desc 'scope name of counter server'\n      config_param :scope, :string\n\n      desc 'the port of counter server to listen to'\n      config_param :port, :integer, default: nil\n      desc 'the bind address of counter server to listen to'\n      config_param :bind, :string, default: nil\n\n      desc 'backup file path of counter values'\n      config_param :backup_path, :string\n    end\n\n    config_section :counter_client, multi: false do\n      desc 'the port of counter server'\n      config_param :port, :integer, default: nil\n      desc 'the IP address or hostname of counter server'\n      config_param :host, :string\n      desc 'the timeout of each operation'\n      config_param :timeout, :time, default: nil\n    end\n\n    config_section :metrics, multi: false do\n      config_param :@type, :string, default: \"local\"\n      config_param :labels, :hash, default: {}\n    end\n\n    config_section :source_only_buffer, init: true, multi: false do\n      config_param :flush_thread_count, :integer, default: 1\n      config_param :overflow_action, :enum, list: [:throw_exception, :block, :drop_oldest_chunk], default: :drop_oldest_chunk\n      config_param :path, :string, default: nil\n      config_param :flush_interval, :time, default: nil\n      config_param :chunk_limit_size, :size, default: nil\n      config_param :total_limit_size, :size, default: nil\n      config_param :compress, :enum, list: [:text, :gzip], default: nil\n    end\n\n    def force_stacktrace_level?\n      @log.forced_stacktrace_level != :none\n    end\n\n    def self.create(conf, strict_config_value=false)\n      systems = conf.elements(name: 'system')\n      return SystemConfig.new if systems.empty?\n      raise Fluent::ConfigError, \"<system> is duplicated. <system> should be only one\" if systems.size > 1\n\n      SystemConfig.new(systems.first, strict_config_value)\n    end\n\n    def self.blank_system_config\n      Fluent::Config::Element.new('<SYSTEM>', '', {}, [])\n    end\n\n    def self.overwrite_system_config(hash)\n      older = defined?($_system_config) ? $_system_config : nil\n      begin\n        $_system_config = SystemConfig.new(Fluent::Config::Element.new('system', '', hash, []))\n        yield\n      ensure\n        $_system_config = older\n      end\n    end\n\n    def initialize(conf=nil, strict_config_value=false)\n      super()\n      conf ||= SystemConfig.blank_system_config\n      configure(conf, strict_config_value)\n    end\n\n    def configure(conf, strict_config_value=false)\n      strict = strict_config_value\n      if !strict && conf && conf.has_key?(\"strict_config_value\")\n        strict = Fluent::Config.bool_value(conf[\"strict_config_value\"])\n      end\n\n      begin\n        super(conf, strict)\n      rescue ConfigError => e\n        $log.error \"config error in:\\n#{conf}\"\n        $log.error 'config error', error: e\n        $log.debug_backtrace\n        exit!(1)\n      end\n\n      @log_level = Log.str_to_level(@log_level.to_s) if @log_level\n      @log[:forced_stacktrace_level] = Log.str_to_level(@log.forced_stacktrace_level.to_s) if force_stacktrace_level?\n    end\n\n    def dup\n      s = SystemConfig.new\n      SYSTEM_CONFIG_PARAMETERS.each do |param|\n        s.__send__(\"#{param}=\", instance_variable_get(\"@#{param}\"))\n      end\n      s\n    end\n\n    def overwrite_variables(**opt)\n      SYSTEM_CONFIG_PARAMETERS.each do |param|\n        if opt.key?(param) && !opt[param].nil? && instance_variable_defined?(\"@#{param}\")\n          instance_variable_set(\"@#{param}\", opt[param])\n        end\n      end\n    end\n\n    module Mixin\n      def system_config\n        require 'fluent/engine'\n        unless defined?($_system_config)\n          $_system_config = nil\n        end\n        (instance_variable_defined?(:@_system_config) && @_system_config) ||\n          $_system_config || Fluent::Engine.system_config\n      end\n\n      def system_config_override(opts={})\n        require 'fluent/engine'\n        if !instance_variable_defined?(:@_system_config) || @_system_config.nil?\n          @_system_config = (defined?($_system_config) && $_system_config ? $_system_config : Fluent::Engine.system_config).dup\n        end\n        opts.each_pair do |key, value|\n          @_system_config.__send__(:\"#{key.to_s}=\", value)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/base.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config'\nrequire 'fluent/engine'\nrequire 'fluent/system_config'\nrequire 'fluent/test/log'\nrequire 'serverengine'\n\nmodule Fluent\n  module Test\n    class TestDriver\n      include ::Test::Unit::Assertions\n\n      def initialize(klass, &block)\n        if klass.is_a?(Class)\n          if block\n            # Create new class for test w/ overwritten methods\n            #   klass.dup is worse because its ancestors does NOT include original class name\n            klass_name = klass.name\n            klass = Class.new(klass)\n            klass.define_singleton_method(:name) { klass_name }\n            klass.module_eval(&block)\n          end\n          @instance = klass.new\n        else\n          @instance = klass\n        end\n        @instance.router = Engine.root_agent.event_router\n        @instance.log = TestLogger.new\n        Engine.root_agent.instance_variable_set(:@log, @instance.log)\n\n        @config = Config.new\n      end\n\n      attr_reader :instance, :config\n\n      def configure(str, use_v1 = false)\n        if str.is_a?(Fluent::Config::Element)\n          @config = str\n        else\n          @config = Config.parse(str, \"(test)\", \"(test_dir)\", use_v1)\n        end\n        if label_name = @config['@label']\n          Engine.root_agent.add_label(label_name)\n        end\n        @instance.configure(@config)\n        self\n      end\n\n      # num_waits is for checking thread status. This will be removed after improved plugin API\n      def run(num_waits = 10, &block)\n        @instance.start\n        @instance.after_start\n        begin\n          # wait until thread starts\n          num_waits.times { sleep 0.05 }\n          return yield\n        ensure\n          @instance.shutdown\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/base.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config'\nrequire 'fluent/config/element'\nrequire 'fluent/env'\nrequire 'fluent/log'\nrequire 'fluent/clock'\n\nrequire 'serverengine/socket_manager'\nrequire 'fileutils'\nrequire 'timeout'\nrequire 'logger'\n\nmodule Fluent\n  module Test\n    module Driver\n      class TestTimedOut < RuntimeError; end\n\n      class Base\n        attr_reader :instance, :logs\n\n        DEFAULT_TIMEOUT = 300\n\n        def initialize(klass, opts: {}, &block)\n          if klass.is_a?(Class)\n            @instance = klass.new\n            if block\n              @instance.singleton_class.module_eval(&block)\n              @instance.send(:initialize)\n            end\n          else\n            @instance = klass\n          end\n          @instance.under_plugin_development = true\n\n          @socket_manager_server = nil\n\n          @logs = []\n\n          @run_post_conditions = []\n          @run_breaking_conditions = []\n          @broken = false\n        end\n\n        def configure(conf, syntax: :v1)\n          raise NotImplementedError\n        end\n\n        def end_if(&block)\n          raise ArgumentError, \"block is not given\" unless block_given?\n          @run_post_conditions << block\n        end\n\n        def break_if(&block)\n          raise ArgumentError, \"block is not given\" unless block_given?\n          @run_breaking_conditions << block\n        end\n\n        def broken?\n          @broken\n        end\n\n        def run(timeout: nil, start: true, shutdown: true, &block)\n          instance_start if start\n\n          timeout ||= DEFAULT_TIMEOUT\n          stop_at = Fluent::Clock.now + timeout\n          @run_breaking_conditions << ->(){ Fluent::Clock.now >= stop_at }\n\n          if !block_given? && @run_post_conditions.empty? && @run_breaking_conditions.empty?\n            raise ArgumentError, \"no stop conditions nor block specified\"\n          end\n\n          sleep_with_watching_threads = ->(){\n            if @instance.respond_to?(:_threads)\n              @instance._threads.values.each{|t| t.join(0) }\n            end\n            sleep 0.1\n          }\n\n          begin\n            retval = run_actual(timeout: timeout, &block)\n            sleep_with_watching_threads.call until stop?\n            retval\n          ensure\n            instance_shutdown if shutdown\n          end\n        end\n\n        def instance_start\n          if @instance.respond_to?(:server_wait_until_start)\n            if Fluent.windows?\n              @socket_manager_server = ServerEngine::SocketManager::Server.open\n              @socket_manager_path = @socket_manager_server.path\n            else\n              @socket_manager_path = ServerEngine::SocketManager::Server.generate_path\n              if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)\n                FileUtils.rm_f @socket_manager_path\n              end\n              @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)\n            end\n            ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s\n          end\n\n          unless @instance.started?\n            @instance.start\n          end\n          unless @instance.after_started?\n            @instance.after_start\n          end\n\n          if @instance.respond_to?(:thread_wait_until_start)\n            @instance.thread_wait_until_start\n          end\n\n          if @instance.respond_to?(:event_loop_wait_until_start)\n            @instance.event_loop_wait_until_start\n          end\n\n          instance_hook_after_started\n        end\n\n        def instance_hook_after_started\n          # insert hooks for tests available after instance.start\n        end\n\n        def instance_hook_before_stopped\n          # same with above\n        end\n\n        def instance_shutdown(log: Logger.new($stdout))\n          instance_hook_before_stopped\n\n          show_errors_if_exists = ->(label, block){\n            begin\n              block.call\n            rescue => e\n              log.error \"unexpected error while #{label}, #{e.class}:#{e.message}\"\n              e.backtrace.each do |bt|\n                log.error \"\\t#{bt}\"\n              end\n            end\n          }\n\n          show_errors_if_exists.call(:stop,            ->(){ @instance.stop unless @instance.stopped? })\n          show_errors_if_exists.call(:before_shutdown, ->(){ @instance.before_shutdown unless @instance.before_shutdown? })\n          show_errors_if_exists.call(:shutdown,        ->(){ @instance.shutdown unless @instance.shutdown? })\n          show_errors_if_exists.call(:after_shutdown,  ->(){ @instance.after_shutdown unless @instance.after_shutdown? })\n\n          if @instance.respond_to?(:server_wait_until_stop)\n            @instance.server_wait_until_stop\n          end\n\n          if @instance.respond_to?(:event_loop_wait_until_stop)\n            @instance.event_loop_wait_until_stop\n          end\n\n          show_errors_if_exists.call(:close, ->(){ @instance.close unless @instance.closed? })\n\n          if @instance.respond_to?(:thread_wait_until_stop)\n            @instance.thread_wait_until_stop\n          end\n\n          show_errors_if_exists.call(:terminate, ->(){ @instance.terminate unless @instance.terminated? })\n\n          if @socket_manager_server\n            @socket_manager_server.close\n            if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)\n              FileUtils.rm_f @socket_manager_path\n            end\n          end\n        end\n\n        def run_actual(timeout: DEFAULT_TIMEOUT)\n          if @instance.respond_to?(:_threads)\n            sleep 0.1 until @instance._threads.values.all?(&:alive?)\n          end\n\n          if @instance.respond_to?(:event_loop_running?)\n            sleep 0.1 until @instance.event_loop_running?\n          end\n\n          if @instance.respond_to?(:_child_process_processes)\n            sleep 0.1 until @instance._child_process_processes.values.all?{|pinfo| pinfo.alive }\n          end\n\n          return_value = nil\n          begin\n            Timeout.timeout(timeout * 2) do |sec|\n              return_value = yield if block_given?\n            end\n          rescue Timeout::Error\n            raise TestTimedOut, \"Test case timed out with hard limit.\"\n          end\n          return_value\n        end\n\n        def stop?\n          # Should stop running if post conditions are not registered.\n          return true unless @run_post_conditions || @run_post_conditions.empty?\n\n          # Should stop running if all of the post conditions are true.\n          return true if @run_post_conditions.all? {|proc| proc.call }\n\n          # Should stop running if some of the breaking conditions is true.\n          # In this case, some post conditions may be not true.\n          if @run_breaking_conditions.any? {|proc| proc.call }\n            @broken = true\n            return true\n          end\n\n          false\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/base_owned.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/test/driver/base'\n\nrequire 'fluent/plugin/base'\nrequire 'fluent/plugin_id'\nrequire 'fluent/log'\nrequire 'fluent/plugin_helper'\n\nmodule Fluent\n  module Test\n    module Driver\n      class OwnerDummy < Fluent::Plugin::Base\n        include PluginId\n        include PluginLoggerMixin\n        include PluginHelper::Mixin\n      end\n\n      class BaseOwned < Base\n        attr_accessor :section_name\n\n        def initialize(klass, opts: {}, &block)\n          super\n\n          owner = OwnerDummy.new\n          if opts\n            owner.system_config_override(opts)\n          end\n          owner.log = TestLogger.new\n\n          if @instance.respond_to?(:owner=)\n            @instance.owner = owner\n            if opts\n              @instance.system_config_override(opts)\n            end\n          end\n\n          @logs = owner.log.out.logs\n          @section_name = ''\n        end\n\n        def configure(conf)\n          if conf.is_a?(Fluent::Config::Element)\n            @config = conf\n          elsif conf.is_a?(Hash)\n            @config = Fluent::Config::Element.new(@section_name, \"\", Hash[conf.map{|k,v| [k.to_s, v]}], [])\n          else\n            @config = Fluent::Config.parse(conf, @section_name, \"\", syntax: :v1)\n          end\n          @instance.configure(@config)\n          self\n        end\n\n        # this is special method for v0 and should be deleted\n        def configure_v0(conf)\n          if conf.is_a?(Fluent::Config::Element)\n            @config = conf\n          elsif conf.is_a?(Hash)\n            @config = Fluent::Config::Element.new(@section_name, \"\", Hash[conf.map{|k,v| [k.to_s, v]}], [])\n          else\n            @config = Fluent::Config.parse(conf, @section_name, \"\", syntax: :v0)\n          end\n          @instance.configure(@config)\n          self\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/base_owner.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/test/driver/base'\nrequire 'fluent/test/driver/test_event_router'\n\nmodule Fluent\n  module Test\n    module Driver\n      class BaseOwner < Base\n        def initialize(klass, opts: {}, &block)\n          super\n\n          if opts\n            @instance.system_config_override(opts)\n          end\n          @instance.log = TestLogger.new\n          @logs = @instance.log.out.logs\n\n          @event_streams = nil\n          @error_events = nil\n        end\n\n        def configure(conf, syntax: :v1)\n          if conf.is_a?(Fluent::Config::Element)\n            @config = conf\n          else\n            @config = Config.parse(conf, \"(test)\", \"(test_dir)\", syntax: syntax)\n          end\n\n          if @instance.respond_to?(:router=)\n            @event_streams = []\n            @error_events = []\n\n            driver = self\n            mojule = Module.new do\n              define_method(:event_emitter_router) do |label_name|\n                TestEventRouter.new(driver)\n              end\n            end\n            @instance.singleton_class.prepend mojule\n          end\n\n          @instance.configure(@config)\n          self\n        end\n\n        Emit = Struct.new(:tag, :es)\n        ErrorEvent = Struct.new(:tag, :time, :record, :error)\n\n        # via TestEventRouter\n        def emit_event_stream(tag, es)\n          @event_streams << Emit.new(tag, es)\n        end\n\n        def emit_error_event(tag, time, record, error)\n          @error_events << ErrorEvent.new(tag, time, record, error)\n        end\n\n        def events(tag: nil)\n          if block_given?\n            event_streams(tag: tag) do |t, es|\n              es.each do |time, record|\n                yield t, time, record\n              end\n            end\n          else\n            list = []\n            event_streams(tag: tag) do |t, es|\n              es.each do |time, record|\n                list << [t, time, record]\n              end\n            end\n            list\n          end\n        end\n\n        def event_streams(tag: nil)\n          return [] if @event_streams.nil?\n          selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }\n          if block_given?\n            selected.each do |e|\n              yield e.tag, e.es\n            end\n          else\n            selected.map{|e| [e.tag, e.es] }\n          end\n        end\n\n        def emit_count\n          @event_streams.size\n        end\n\n        def record_count\n          @event_streams.reduce(0) {|a, e| a + e.es.size }\n        end\n\n        def error_events(tag: nil)\n          selected = @error_events.select{|e| tag.nil? ? true : e.tag == tag }\n          if block_given?\n            selected.each do |e|\n              yield e.tag, e.time, e.record, e.error\n            end\n          else\n            selected.map{|e| [e.tag, e.time, e.record, e.error] }\n          end\n        end\n\n        def run(expect_emits: nil, expect_records: nil, timeout: nil, start: true, shutdown: true, &block)\n          if expect_emits\n            @run_post_conditions << ->(){ emit_count >= expect_emits }\n          end\n          if expect_records\n            @run_post_conditions << ->(){ record_count >= expect_records }\n          end\n\n          super(timeout: timeout, start: start, shutdown: shutdown, &block)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/event_feeder.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/event'\nrequire 'fluent/time'\n\nmodule Fluent\n  module Test\n    module Driver\n      module EventFeeder\n        def initialize(klass, opts: {}, &block)\n          super\n          @default_tag = nil\n          @feed_method = nil\n        end\n\n        def run(default_tag: nil, **kwargs, &block)\n          @feed_method = if @instance.respond_to?(:filter_stream)\n                           :filter_stream\n                         else\n                           :emit_events\n                         end\n          if default_tag\n            @default_tag = default_tag\n          end\n          super(**kwargs, &block)\n        end\n\n        def feed_to_plugin(tag, es)\n          @instance.__send__(@feed_method, tag, es)\n        end\n\n        # d.run do\n        #   d.feed('tag', time, {record})\n        #   d.feed('tag', [ [time, {record}], [time, {record}], ... ])\n        #   d.feed('tag', es)\n        # end\n        # d.run(default_tag: 'tag') do\n        #   d.feed({record})\n        #   d.feed(time, {record})\n        #   d.feed([ [time, {record}], [time, {record}], ... ])\n        #   d.feed(es)\n        # end\n        def feed(*args)\n          case args.size\n          when 1\n            raise ArgumentError, \"tag not specified without default_tag\" unless @default_tag\n            case args.first\n            when Fluent::EventStream\n              feed_to_plugin(@default_tag, args.first)\n            when Array\n              feed_to_plugin(@default_tag, ArrayEventStream.new(args.first))\n            when Hash\n              record = args.first\n              time = Fluent::EventTime.now\n              feed_to_plugin(@default_tag, OneEventStream.new(time, record))\n            else\n              raise ArgumentError, \"unexpected events object (neither event(Hash), EventStream nor Array): #{args.first.class}\"\n            end\n          when 2\n            if args[0].is_a?(String) && (args[1].is_a?(Array) || args[1].is_a?(Fluent::EventStream))\n              tag, es = args\n              es = ArrayEventStream.new(es) if es.is_a?(Array)\n              feed_to_plugin(tag, es)\n            elsif @default_tag && (args[0].is_a?(Fluent::EventTime) || args[0].is_a?(Integer)) && args[1].is_a?(Hash)\n              time, record = args\n              feed_to_plugin(@default_tag, OneEventStream.new(time, record))\n            else\n              raise ArgumentError, \"unexpected values of argument: #{args[0].class}, #{args[1].class}\"\n            end\n          when 3\n            tag, time, record = args\n            if tag.is_a?(String) && (time.is_a?(Fluent::EventTime) || time.is_a?(Integer)) && record.is_a?(Hash)\n              feed_to_plugin(tag, OneEventStream.new(time, record))\n            else\n              raise ArgumentError, \"unexpected values of argument: #{tag.class}, #{time.class}, #{record.class}\"\n            end\n          else\n            raise ArgumentError, \"unexpected number of arguments: #{args}\"\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/filter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/test/driver/base_owner'\nrequire 'fluent/test/driver/event_feeder'\n\nrequire 'fluent/plugin/filter'\n\nmodule Fluent\n  module Test\n    module Driver\n      class Filter < BaseOwner\n        include EventFeeder\n\n        attr_reader :filtered\n\n        def initialize(klass, opts: {}, &block)\n          super\n          raise ArgumentError, \"plugin is not an instance of Fluent::Plugin::Filter\" unless @instance.is_a? Fluent::Plugin::Filter\n          @filtered = []\n        end\n\n        def filtered_records\n          @filtered.map {|_time, record| record }\n        end\n\n        def filtered_time\n          @filtered.map {|time, _record| time }\n        end\n\n        def instance_hook_after_started\n          super\n          filter_hook = ->(time, record) { @filtered << [time, record] }\n          m = Module.new do\n            define_method(:filter_stream) do |tag, es|\n              new_es = super(tag, es)\n              new_es.each do |time, record|\n                filter_hook.call(time, record)\n              end\n              new_es\n            end\n          end\n          @instance.singleton_class.prepend(m)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/formatter.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/test/driver/base_owned'\n\nmodule Fluent\n  module Test\n    module Driver\n      class Formatter < BaseOwned\n        def initialize(klass, **kwargs, &block)\n          super\n          @section_name = \"format\"\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/input.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/test/driver/base_owner'\nrequire 'fluent/plugin/input'\n\nmodule Fluent\n  module Test\n    module Driver\n      class Input < BaseOwner\n        def initialize(klass, opts: {}, &block)\n          super\n          raise ArgumentError, \"plugin is not an instance of Fluent::Plugin::Input\" unless @instance.is_a? Fluent::Plugin::Input\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/multi_output.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/test/driver/base_owner'\nrequire 'fluent/test/driver/event_feeder'\n\nrequire 'fluent/plugin/multi_output'\n\nmodule Fluent\n  module Test\n    module Driver\n      class MultiOutput < BaseOwner\n        include EventFeeder\n\n        def initialize(klass, opts: {}, &block)\n          super\n          raise ArgumentError, \"plugin is not an instance of Fluent::Plugin::MultiOutput\" unless @instance.is_a? Fluent::Plugin::MultiOutput\n          @flush_buffer_at_cleanup = nil\n        end\n\n        def run(flush: true, **kwargs, &block)\n          @flush_buffer_at_cleanup = flush\n          super(**kwargs, &block)\n        end\n\n        def run_actual(**kwargs, &block)\n          val = super(**kwargs, &block)\n          if @flush_buffer_at_cleanup\n            @instance.outputs.each{|o| o.force_flush }\n          end\n          val\n        end\n\n        def flush\n          @instance.outputs.each{|o| o.force_flush }\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/output.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/test/driver/base_owner'\nrequire 'fluent/test/driver/event_feeder'\n\nrequire 'fluent/plugin/output'\nrequire 'timeout'\n\nmodule Fluent\n  module Test\n    module Driver\n      class Output < BaseOwner\n        include EventFeeder\n\n        def initialize(klass, opts: {}, &block)\n          super\n          raise ArgumentError, \"plugin is not an instance of Fluent::Plugin::Output\" unless @instance.is_a? Fluent::Plugin::Output\n          @flush_buffer_at_cleanup = nil\n          @wait_flush_completion = nil\n          @force_flush_retry = nil\n          @format_hook = nil\n          @format_results = []\n        end\n\n        def run(flush: true, wait_flush_completion: true, force_flush_retry: false, **kwargs, &block)\n          @flush_buffer_at_cleanup = flush\n          @wait_flush_completion = wait_flush_completion\n          @force_flush_retry = force_flush_retry\n          super(**kwargs, &block)\n        end\n\n        def run_actual(**kwargs, &block)\n          if @force_flush_retry\n            @instance.retry_for_error_chunk = true\n          end\n          val = super(**kwargs, &block)\n          if @flush_buffer_at_cleanup\n            self.flush\n          end\n          val\n        ensure\n          @instance.retry_for_error_chunk = false\n        end\n\n        def formatted\n          @format_results\n        end\n\n        def flush\n          @instance.force_flush\n          wait_flush_completion if @wait_flush_completion\n        end\n\n        def wait_flush_completion\n          buffer_queue = ->(){ @instance.buffer && @instance.buffer.queue.size > 0 }\n          dequeued_chunks = ->(){\n            @instance.dequeued_chunks_mutex &&\n            @instance.dequeued_chunks &&\n            @instance.dequeued_chunks_mutex.synchronize{ @instance.dequeued_chunks.size > 0 }\n          }\n\n          Timeout.timeout(10) do\n            while buffer_queue.call || dequeued_chunks.call\n              sleep 0.1\n            end\n          end\n        end\n\n        def instance_hook_after_started\n          super\n\n          # it's decided after #start whether output plugin instances use @custom_format or not.\n          if @instance.instance_eval{ @custom_format }\n            @format_hook = format_hook = ->(result){ @format_results << result }\n            m = Module.new do\n              define_method(:format) do |tag, time, record|\n                result = super(tag, time, record)\n                format_hook.call(result)\n                result\n              end\n            end\n            @instance.singleton_class.prepend m\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/parser.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/test/driver/base_owned'\n\nmodule Fluent\n  module Test\n    module Driver\n      class Parser < BaseOwned\n        def initialize(klass, **kwargs, &block)\n          super\n          @section_name = \"parse\"\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/storage.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/test/driver/base_owned'\n\nmodule Fluent\n  module Test\n    module Driver\n      class Storage < BaseOwned\n        def initialize(klass, **kwargs, &block)\n          super\n          @section_name = \"storage\"\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/driver/test_event_router.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/event'\n\nmodule Fluent\n  module Test\n    module Driver\n      class TestEventRouter\n        def initialize(driver)\n          @driver = driver\n        end\n\n        def emit(tag, time, record)\n          @driver.emit_event_stream(tag, OneEventStream.new(time, record))\n        end\n\n        def emit_array(tag, array)\n          @driver.emit_event_stream(tag, ArrayEventStream.new(array))\n        end\n\n        def emit_stream(tag, es)\n          @driver.emit_event_stream(tag, es)\n        end\n\n        def emit_error_event(tag, time, record, error)\n          @driver.emit_error_event(tag, time, record, error)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/filter_test.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/test/base'\nrequire 'fluent/event'\n\nmodule Fluent\n  module Test\n    class FilterTestDriver < TestDriver\n      def initialize(klass, tag = 'filter.test', &block)\n        super(klass, &block)\n        @tag = tag\n        @events = {}\n        @filtered = MultiEventStream.new\n      end\n\n      attr_reader :filtered\n      attr_accessor :tag\n\n      def emit(record, time = EventTime.now)\n        emit_with_tag(@tag, record, time)\n      end\n      alias_method :filter, :emit\n\n      def emit_with_tag(tag, record, time = EventTime.now)\n        @events[tag] ||= MultiEventStream.new\n        @events[tag].add(time, record)\n      end\n      alias_method :filter_with_tag, :emit_with_tag\n\n      def filter_stream(es)\n        filter_stream_with_tag(@tag, es)\n      end\n\n      def filter_stream_with_tag(tag, es)\n        @events[tag] = es\n      end\n\n      def filtered_as_array\n        all = []\n        @filtered.each { |time, record|\n          all << [@tag, time, record]\n        }\n        all\n      end\n      alias_method :emits, :filtered_as_array # emits is for consistent with other drivers\n\n      # Almost filters don't use threads so default is 0. It reduces test time.\n      def run(num_waits = 0)\n        super(num_waits) {\n          yield if block_given?\n\n          @events.each { |tag, es|\n            processed = @instance.filter_stream(tag, es)\n            processed.each { |time, record|\n              @filtered.add(time, record)\n            }\n          }\n        }\n        self\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/formatter_test.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/formatter'\nrequire 'fluent/config'\nrequire 'fluent/plugin'\n\nmodule Fluent\n  module Test\n    class FormatterTestDriver\n      def initialize(klass_or_str, proc=nil, &block)\n        if klass_or_str.is_a?(Class)\n          if block\n            # Create new class for test w/ overwritten methods\n            #   klass.dup is worse because its ancestors does NOT include original class name\n            klass_name = klass_or_str.name\n            klass_or_str = Class.new(klass_or_str)\n            klass_or_str.define_singleton_method(:name) { klass_name }\n            klass_or_str.module_eval(&block)\n          end\n          @instance = klass_or_str.new\n        elsif klass_or_str.is_a?(String)\n          @instance = Fluent::Plugin.new_formatter(klass_or_str)\n        else\n          @instance = klass_or_str\n        end\n        @config = Config.new\n      end\n\n      attr_reader :instance, :config\n\n      def configure(conf)\n        case conf\n        when Fluent::Config::Element\n          @config = conf\n        when String\n          @config = Config.parse(conf, 'fluent.conf')\n        when Hash\n          @config = Config::Element.new('ROOT', '', conf, [])\n        else\n          raise \"Unknown type... #{conf}\"\n        end\n        @instance.configure(@config)\n        self\n      end\n\n      def format(tag, time, record)\n        @instance.format(tag, time, record)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/helpers.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/config/element'\nrequire 'fluent/msgpack_factory'\nrequire 'fluent/time'\n\nmodule Fluent\n  module Test\n    module Helpers\n      # See \"Example Custom Assertion: https://test-unit.github.io/test-unit/en/Test/Unit/Assertions.html\n      def assert_equal_event_time(expected, actual, message = nil)\n        expected_s = \"#{Time.at(expected.sec)} (nsec #{expected.nsec})\"\n        actual_s   = \"#{Time.at(actual.sec)  } (nsec #{actual.nsec})\"\n        message = build_message(message, <<EOT, expected_s, actual_s)\n<?> expected but was\n<?>.\nEOT\n        assert_block(message) do\n          expected.is_a?(Fluent::EventTime) && actual.is_a?(Fluent::EventTime) && expected.sec == actual.sec && expected.nsec == actual.nsec\n        end\n      end\n\n      def config_element(name = 'test', argument = '', params = {}, elements = [])\n        Fluent::Config::Element.new(name, argument, params, elements)\n      end\n\n      def event_time(str=nil, format: nil)\n        if str\n          if format\n            Fluent::EventTime.from_time(Time.strptime(str, format))\n          else\n            Fluent::EventTime.parse(str)\n          end\n        else\n          Fluent::EventTime.now\n        end\n      end\n\n      def event_time_without_nsec(str=nil, format: nil)\n        Fluent::EventTime.new(event_time(str, format: format))\n      end\n\n      def with_timezone(tz)\n        oldtz, ENV['TZ'] = ENV['TZ'], tz\n        yield\n      ensure\n        ENV['TZ'] = oldtz\n      end\n\n      def with_worker_config(root_dir: nil, workers: nil, worker_id: nil, &block)\n        if workers\n          if worker_id\n            if worker_id >= workers\n              raise \"worker_id must be between 0 and (workers - 1)\"\n            end\n          else\n            worker_id = 0\n          end\n        end\n\n        opts = {}\n        opts['root_dir'] = root_dir if root_dir\n        opts['workers'] = workers if workers\n\n        ENV['SERVERENGINE_WORKER_ID'] = worker_id.to_s\n        Fluent::SystemConfig.overwrite_system_config(opts, &block)\n      ensure\n        ENV.delete('SERVERENGINE_WORKER_ID')\n      end\n\n      def time2str(time, localtime: false, format: nil)\n        if format\n          if localtime\n            Time.at(time).strftime(format)\n          else\n            Time.at(time).utc.strftime(format)\n          end\n        else\n          if localtime\n            Time.at(time).iso8601\n          else\n            Time.at(time).utc.iso8601\n          end\n        end\n      end\n\n      def msgpack(type)\n        case type\n        when :factory\n          Fluent::MessagePackFactory.factory\n        when :packer\n          Fluent::MessagePackFactory.packer\n        when :unpacker\n          Fluent::MessagePackFactory.unpacker\n        else\n          raise ArgumentError, \"unknown msgpack object type '#{type}'\"\n        end\n      end\n\n      #\n      # Use this method with v0.12 compatibility layer.\n      #\n      # For v0.14 API, use `driver.logs` instead.\n      #\n      def capture_log(driver)\n        tmp = driver.instance.log.out\n        driver.instance.log.out = StringIO.new\n        yield\n        return driver.instance.log.out.string\n      ensure\n        driver.instance.log.out = tmp\n      end\n\n      def capture_stdout\n        out = StringIO.new\n        $stdout = out\n        yield\n        out.string.force_encoding('utf-8')\n      ensure\n        $stdout = STDOUT\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/input_test.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/engine'\nrequire 'fluent/time'\nrequire 'fluent/test/base'\n\nmodule Fluent\n  module Test\n    class InputTestDriver < TestDriver\n      def initialize(klass, &block)\n        super(klass, &block)\n        @emit_streams = []\n        @event_streams = []\n        @expects = nil\n        # for checking only the number of emitted records during run\n        @expected_emits_length = nil\n        @run_timeout = 5\n        @run_post_conditions = []\n      end\n\n      def expect_emit(tag, time, record)\n        (@expects ||= []) << [tag, time, record]\n        self\n      end\n\n      def expected_emits\n        @expects ||= []\n      end\n\n      attr_accessor :expected_emits_length\n      attr_accessor :run_timeout\n      attr_reader :emit_streams, :event_streams\n\n      def emits\n        all = []\n        @emit_streams.each {|tag,events|\n          events.each {|time,record|\n            all << [tag, time, record]\n          }\n        }\n        all\n      end\n\n      def events\n        all = []\n        @emit_streams.each {|tag,events|\n          all.concat events\n        }\n        all\n      end\n\n      def records\n        all = []\n        @emit_streams.each {|tag,events|\n          events.each {|time,record|\n            all << record\n          }\n        }\n        all\n      end\n\n      def register_run_post_condition(&block)\n        if block\n          @run_post_conditions << block\n        end\n      end\n\n      def register_run_breaking_condition(&block)\n        if block\n          @run_breaking_conditions ||= []\n          @run_breaking_conditions << block\n        end\n      end\n\n      def run_should_stop?\n        # Should stop running if post conditions are not registered.\n        return true unless @run_post_conditions\n\n        # Should stop running if all of the post conditions are true.\n        return true if @run_post_conditions.all? {|proc| proc.call }\n\n        # Should stop running if any of the breaking conditions is true.\n        # In this case, some post conditions may be not true.\n        return true if @run_breaking_conditions && @run_breaking_conditions.any? {|proc| proc.call }\n\n        false\n      end\n\n      module EmitStreamWrapper\n        def emit_stream_callee=(method)\n          @emit_stream_callee = method\n        end\n        def emit_stream(tag, es)\n          @emit_stream_callee.call(tag, es)\n        end\n      end\n\n      def run(num_waits = 10)\n        m = method(:emit_stream)\n        unless Engine.singleton_class.ancestors.include?(EmitStreamWrapper)\n          Engine.singleton_class.prepend EmitStreamWrapper\n        end\n        Engine.emit_stream_callee = m\n        unless instance.router.singleton_class.ancestors.include?(EmitStreamWrapper)\n          instance.router.singleton_class.prepend EmitStreamWrapper\n        end\n        instance.router.emit_stream_callee = m\n\n        super(num_waits) {\n          yield if block_given?\n\n          if @expected_emits_length || @expects || @run_post_conditions\n            # counters for emits and emit_streams\n            i, j = 0, 0\n\n            # Events of expected length will be emitted at the end.\n            max_length = @expected_emits_length\n            max_length ||= @expects.length if @expects\n            if max_length\n              register_run_post_condition do\n                i == max_length\n              end\n            end\n\n            # Set running timeout to avoid infinite loop caused by some errors.\n            started_at = Time.now\n            register_run_breaking_condition do\n              Time.now >= started_at + @run_timeout\n            end\n\n            until run_should_stop?\n              if j >= @emit_streams.length\n                sleep 0.01\n                next\n              end\n\n              tag, events = @emit_streams[j]\n              events.each do |time, record|\n                if @expects\n                  assert_equal(@expects[i], [tag, time, record])\n                  assert_equal_event_time(@expects[i][1], time) if @expects[i][1].is_a?(Fluent::EventTime)\n                end\n                i += 1\n              end\n              j += 1\n            end\n            assert_equal(@expects.length, i) if @expects\n          end\n        }\n        self\n      end\n\n      private\n      def emit_stream(tag, es)\n        @event_streams << es\n        @emit_streams << [tag, es.to_a]\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/log.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'serverengine'\nrequire 'fluent/log'\n\nmodule Fluent\n  module Test\n    class DummyLogDevice\n      attr_reader :logs\n      attr_accessor :flush_logs\n\n      def initialize\n        @logs = []\n        @flush_logs = true\n        @use_stderr = false\n      end\n\n      def reset\n        @logs = [] if @flush_logs\n      end\n\n      def tty?\n        false\n      end\n\n      def puts(*args)\n        args.each{ |arg| write(arg + \"\\n\") }\n      end\n\n      def write(message)\n        if @use_stderr\n          STDERR.write message\n        end\n        @logs.push message\n      end\n\n      def flush\n        true\n      end\n\n      def close\n        true\n      end\n    end\n\n    class TestLogger < Fluent::PluginLogger\n      def initialize\n        @logdev = DummyLogDevice.new\n        dl_opts = {}\n        dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO\n        logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)\n        log = Fluent::Log.new(logger)\n        super(log)\n      end\n\n      def reset\n        @logdev.reset\n      end\n\n      def logs\n        @logdev.logs\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/output_test.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/engine'\nrequire 'fluent/event'\nrequire 'fluent/test/input_test'\n\nmodule Fluent\n  module Test\n    class TestOutputChain\n      def initialize\n        @called = 0\n      end\n\n      def next\n        @called += 1\n      end\n\n      attr_reader :called\n    end\n\n\n    class OutputTestDriver < InputTestDriver\n      def initialize(klass, tag='test', &block)\n        super(klass, &block)\n        @tag = tag\n      end\n\n      attr_accessor :tag\n\n      def emit(record, time=EventTime.now)\n        es = OneEventStream.new(time, record)\n        @instance.emit_events(@tag, es)\n      end\n    end\n\n\n    class BufferedOutputTestDriver < InputTestDriver\n      def initialize(klass, tag='test', &block)\n        super(klass, &block)\n        @entries = []\n        @expected_buffer = nil\n        @tag = tag\n\n        def @instance.buffer\n          @buffer\n        end\n      end\n\n      attr_accessor :tag\n\n      def emit(record, time=EventTime.now)\n        @entries << [time, record]\n        self\n      end\n\n      def expect_format(str)\n        (@expected_buffer ||= '') << str\n      end\n\n      def run(num_waits = 10)\n        result = nil\n        super(num_waits) {\n          yield if block_given?\n\n          es = ArrayEventStream.new(@entries)\n          buffer = @instance.format_stream(@tag, es)\n\n          if @expected_buffer\n            assert_equal(@expected_buffer, buffer)\n          end\n\n          chunk = if @instance.instance_eval{ @chunk_key_tag }\n                    @instance.buffer.generate_chunk(@instance.metadata(@tag, nil, nil)).staged!\n                  else\n                    @instance.buffer.generate_chunk(@instance.metadata(nil, nil, nil)).staged!\n                  end\n          chunk.concat(buffer, es.size)\n\n          begin\n            result = @instance.write(chunk)\n          ensure\n            chunk.purge\n          end\n        }\n        result\n      end\n    end\n\n    class TimeSlicedOutputTestDriver < InputTestDriver\n      def initialize(klass, tag='test', &block)\n        super(klass, &block)\n        @entries = []\n        @expected_buffer = nil\n        @tag = tag\n      end\n\n      attr_accessor :tag\n\n      def emit(record, time=EventTime.now)\n        @entries << [time, record]\n        self\n      end\n\n      def expect_format(str)\n        (@expected_buffer ||= '') << str\n      end\n\n      def run\n        result = []\n        super {\n          yield if block_given?\n\n          buffer = ''\n          lines = {}\n          # v0.12 TimeSlicedOutput doesn't call #format_stream\n          @entries.each do |time, record|\n            meta = @instance.metadata(@tag, time, record)\n            line = @instance.format(@tag, time, record)\n            buffer << line\n            lines[meta] ||= []\n            lines[meta] << line\n          end\n\n          if @expected_buffer\n            assert_equal(@expected_buffer, buffer)\n          end\n\n          lines.each_key do |meta|\n            chunk = @instance.buffer.generate_chunk(meta).staged!\n            chunk.append(lines[meta])\n            begin\n              result.push(@instance.write(chunk))\n            ensure\n              chunk.purge\n            end\n          end\n        }\n        result\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/parser_test.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/parser'\nrequire 'fluent/config'\n\nmodule Fluent\n  module Test\n    class ParserTestDriver\n      def initialize(klass_or_str, format=nil, conf={}, &block)\n        if klass_or_str.is_a?(Class)\n          if block\n            # Create new class for test w/ overwritten methods\n            #   klass.dup is worse because its ancestors does NOT include original class name\n            klass_name = klass_or_str.name\n            klass_or_str = Class.new(klass_or_str)\n            klass_or_str.define_singleton_method(:name) { klass_name }\n            klass_or_str.module_eval(&block)\n          end\n          case klass_or_str.instance_method(:initialize).arity\n          when 0\n            @instance = klass_or_str.new\n          when -2\n            # for RegexpParser\n            @instance = klass_or_str.new(format, conf)\n          end\n        elsif klass_or_str.is_a?(String)\n          @instance = Fluent::Plugin.new_parser(klass_or_str)\n        else\n          @instance = klass_or_str\n        end\n        @config = Config.new\n      end\n\n      attr_reader :instance, :config\n\n      def configure(conf)\n        case conf\n        when Fluent::Config::Element\n          @config = conf\n        when String\n          @config = Config.parse(conf, 'fluent.conf')\n        when Hash\n          @config = Config::Element.new('ROOT', '', conf, [])\n        else\n          raise \"Unknown type... #{conf}\"\n        end\n        @instance.configure(@config)\n        self\n      end\n\n      def parse(text, &block)\n        @instance.parse(text, &block)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test/startup_shutdown.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'serverengine'\nrequire 'fileutils'\n\nmodule Fluent\n  module Test\n    module StartupShutdown\n      def startup\n        @server = ServerEngine::SocketManager::Server.open\n        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s\n      end\n\n      def shutdown\n        @server.close\n      end\n\n      def self.setup\n        @server = ServerEngine::SocketManager::Server.open\n        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s\n      end\n\n      def self.teardown\n        @server.close\n        # on Windows, the path is a TCP port number\n        FileUtils.rm_f @server.path unless Fluent.windows?\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/test.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'test/unit'\nrequire 'fluent/env' # for Fluent.windows?\nrequire 'fluent/test/log'\nrequire 'fluent/test/base'\nrequire 'fluent/test/input_test'\nrequire 'fluent/test/output_test'\nrequire 'fluent/test/filter_test'\nrequire 'fluent/test/parser_test'\nrequire 'fluent/test/formatter_test'\nrequire 'serverengine'\n\n\nmodule Fluent\n  module Test\n    def self.dummy_logger\n      dl_opts = {log_level: ServerEngine::DaemonLogger::INFO}\n      logdev = Fluent::Test::DummyLogDevice.new\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      Fluent::Log.new(logger)\n    end\n\n    def self.setup\n      ENV['SERVERENGINE_WORKER_ID'] = '0'\n\n      $log = dummy_logger\n\n      old_engine = Fluent.__send__(:remove_const, :Engine)\n      # Ensure that GC can remove the objects of the old engine.\n      # Some objects can still exist after `remove_const`. See https://github.com/fluent/fluentd/issues/5054.\n      old_engine.instance_variable_set(:@root_agent, nil)\n\n      engine = Fluent.const_set(:Engine, EngineClass.new).init(SystemConfig.new)\n      engine.define_singleton_method(:now=) {|n|\n        @now = n\n      }\n      engine.define_singleton_method(:now) {\n        @now ||= super()\n      }\n\n      nil\n    end\n  end\nend\n\n$log ||= Fluent::Test.dummy_logger\n"
  },
  {
    "path": "lib/fluent/time.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'time'\nrequire 'msgpack'\nrequire 'strptime'\nrequire 'fluent/timezone'\nrequire 'fluent/configurable'\nrequire 'fluent/config/error'\n\nmodule Fluent\n  class EventTime\n    TYPE = 0\n    FORMATTER = Strftime.new('%Y-%m-%d %H:%M:%S.%N %z')\n\n    def initialize(sec, nsec = 0)\n      @sec = sec\n      @nsec = nsec\n    end\n\n    def ==(other)\n      if other.is_a?(Fluent::EventTime)\n        @sec == other.sec\n      else\n        @sec == other\n      end\n    end\n\n    def sec\n      @sec\n    end\n\n    def nsec\n      @nsec\n    end\n\n    def to_int\n      @sec\n    end\n    alias :to_i :to_int\n\n    def to_f\n      @sec + @nsec / 1_000_000_000.0\n    end\n\n    # for Time.at\n    def to_r\n      Rational(@sec * 1_000_000_000 + @nsec, 1_000_000_000)\n    end\n\n    # for > and others\n    def coerce(other)\n      [other, @sec]\n    end\n\n    def to_s\n      @sec.to_s\n    end\n\n    begin\n      # ruby 2.5 or later\n      Time.at(0, 0, :nanosecond)\n\n      def to_time\n        Time.at(@sec, @nsec, :nanosecond)\n      end\n    rescue\n      def to_time\n        Time.at(Rational(@sec * 1_000_000_000 + @nsec, 1_000_000_000))\n      end\n    end\n\n    def to_json(*args)\n      @sec.to_s\n    end\n\n    def to_msgpack(io = nil)\n      @sec.to_msgpack(io)\n    end\n\n    def to_msgpack_ext\n      [@sec, @nsec].pack('NN')\n    end\n\n    def self.from_msgpack_ext(data)\n      new(*data.unpack('NN'))\n    end\n\n    def self.from_time(time)\n      Fluent::EventTime.new(time.to_i, time.nsec)\n    end\n\n    def self.eq?(a, b)\n      if a.is_a?(Fluent::EventTime) && b.is_a?(Fluent::EventTime)\n        a.sec == b.sec && a.nsec == b.nsec\n      else\n        a == b\n      end\n    end\n\n    def self.now\n      # This method is called many time. so call Process.clock_gettime directly instead of Fluent::Clock.real_now\n      now = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond)\n      Fluent::EventTime.new(now / 1_000_000_000, now % 1_000_000_000)\n    end\n\n    def self.parse(*args)\n      from_time(Time.parse(*args))\n    end\n\n    ## TODO: For performance, implement +, -, and so on\n    def method_missing(name, *args, &block)\n      @sec.send(name, *args, &block)\n    end\n\n    def inspect\n      FORMATTER.exec(Time.at(self))\n    end\n  end\n\n  module TimeMixin\n    TIME_TYPES = ['string', 'unixtime', 'float', 'mixed']\n\n    TIME_PARAMETERS = [\n      [:time_format, :string, {default: nil}],\n      [:localtime, :bool, {default: true}],  # UTC if :localtime is false and :timezone is nil\n      [:utc,       :bool, {default: false}], # to turn :localtime false\n      [:timezone, :string, {default: nil}],\n      [:time_format_fallbacks, :array, {default: []}], # try time_format, then try fallbacks\n    ]\n    TIME_FULL_PARAMETERS = [\n      # To avoid to define :time_type twice (in plugin_helper/inject)\n      [:time_type, :enum, {default: :string, list: TIME_TYPES.map(&:to_sym)}],\n    ] + TIME_PARAMETERS\n\n    module TimeParameters\n      include Fluent::Configurable\n      TIME_FULL_PARAMETERS.each do |name, type, opts|\n        config_param(name, type, **opts)\n      end\n\n      def configure(conf)\n        if conf.has_key?('localtime') || conf.has_key?('utc')\n          if conf.has_key?('localtime')\n            conf['localtime'] = Fluent::Config.bool_value(conf['localtime'])\n          elsif conf.has_key?('utc')\n            conf['localtime'] = !(Fluent::Config.bool_value(conf['utc']))\n            # Specifying \"localtime false\" means using UTC in TimeFormatter\n            # And specifying \"utc\" is different from specifying \"timezone +0000\"(it's not always UTC).\n            # There are difference between \"Z\" and \"+0000\" in timezone formatting.\n            # TODO: add kwargs to TimeFormatter to specify \"using localtime\", \"using UTC\" or \"using specified timezone\" in more explicit way\n          end\n        end\n\n        super\n\n        if conf.has_key?('localtime') && conf.has_key?('utc') && !(@localtime ^ @utc)\n          raise Fluent::ConfigError, \"both of utc and localtime are specified, use only one of them\"\n        end\n\n        if conf.has_key?('time_type') and @time_type == :mixed\n          if @time_format.nil? and @time_format_fallbacks.empty?\n            raise Fluent::ConfigError, \"time_type is :mixed but time_format and time_format_fallbacks is empty.\"\n          end\n        end\n\n        Fluent::Timezone.validate!(@timezone) if @timezone\n      end\n    end\n\n    module Parser\n      def self.included(mod)\n        mod.include TimeParameters\n      end\n\n      def time_parser_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)\n        return MixedTimeParser.new(type, format, @localtime, timezone, @utc, force_localtime, @time_format_fallbacks) if type == :mixed\n        return NumericTimeParser.new(type) if type != :string\n        return TimeParser.new(format, true, nil) if force_localtime\n\n        localtime = @localtime && (timezone.nil? && !@utc)\n        TimeParser.new(format, localtime, timezone)\n      end\n    end\n\n    module Formatter\n      def self.included(mod)\n        mod.include TimeParameters\n      end\n\n      def time_formatter_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)\n        return NumericTimeFormatter.new(type) if type != :string\n        return TimeFormatter.new(format, true, nil) if force_localtime\n\n        localtime = @localtime && (timezone.nil? && !@utc)\n        TimeFormatter.new(format, localtime, timezone)\n      end\n    end\n  end\n\n  class TimeParser\n    class TimeParseError < StandardError; end\n\n    def initialize(format = nil, localtime = true, timezone = nil)\n      if format.nil? && (timezone || !localtime)\n        raise Fluent::ConfigError, \"specifying timezone requires time format\"\n      end\n\n      @cache1_key = nil\n      @cache1_time = nil\n      @cache2_key = nil\n      @cache2_time = nil\n\n      format_with_timezone = format && (format.include?(\"%z\") || format.include?(\"%Z\"))\n\n      utc_offset = case\n                   when format_with_timezone then\n                     nil\n                   when timezone then\n                     Fluent::Timezone.utc_offset(timezone)\n                   when localtime then\n                     nil\n                   else\n                     0 # utc\n                   end\n\n      strptime = format && (Strptime.new(format) rescue nil)\n\n      @parse = case\n               when format_with_timezone && strptime then ->(v){ Fluent::EventTime.from_time(strptime.exec(v)) }\n               when format_with_timezone             then ->(v){ Fluent::EventTime.from_time(Time.strptime(v, format)) }\n               when format == '%iso8601'             then ->(v){ Fluent::EventTime.from_time(Time.iso8601(v)) }\n               when strptime then\n                 if utc_offset.nil?\n                   ->(v){ t = strptime.exec(v); Fluent::EventTime.new(t.to_i, t.nsec) }\n                 elsif utc_offset.respond_to?(:call)\n                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset.call(t), t.nsec) }\n                 else\n                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset, t.nsec) }\n                 end\n               when format then\n                 if utc_offset.nil?\n                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i, t.nsec) }\n                 elsif utc_offset.respond_to?(:call)\n                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset.call(t), t.nsec) }\n                 else\n                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset, t.nsec) }\n                 end\n               else ->(v){ Fluent::EventTime.parse(v) }\n               end\n    end\n\n    # TODO: new cache mechanism using format string\n    def parse(value)\n      unless value.is_a?(String)\n        raise TimeParseError, \"value must be string: #{value}\"\n      end\n\n      if @cache1_key == value\n        return @cache1_time\n      elsif @cache2_key == value\n        return @cache2_time\n      else\n        begin\n          time = @parse.call(value)\n        rescue => e\n          raise TimeParseError, \"invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}\"\n        end\n        @cache1_key = @cache2_key\n        @cache1_time = @cache2_time\n        @cache2_key = value\n        @cache2_time = time\n        return time\n      end\n    end\n    alias :call :parse\n  end\n\n  class NumericTimeParser < TimeParser # to include TimeParseError\n    def initialize(type, localtime = nil, timezone = nil)\n      @cache1_key = @cache1_time = @cache2_key = @cache2_time = nil\n\n      if type == :unixtime\n        define_singleton_method(:parse, method(:parse_unixtime))\n        define_singleton_method(:call, method(:parse_unixtime))\n      else # :float\n        define_singleton_method(:parse, method(:parse_float))\n        define_singleton_method(:call, method(:parse_float))\n      end\n    end\n\n    def parse_unixtime(value)\n      unless value.is_a?(String) || value.is_a?(Numeric)\n        raise TimeParseError, \"value must be a string or a number: #{value}(#{value.class})\"\n      end\n\n      if @cache1_key == value\n        return @cache1_time\n      elsif @cache2_key == value\n        return @cache2_time\n      end\n\n      begin\n        time = Fluent::EventTime.new(value.to_i)\n      rescue => e\n        raise TimeParseError, \"invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}\"\n      end\n      @cache1_key = @cache2_key\n      @cache1_time = @cache2_time\n      @cache2_key = value\n      @cache2_time = time\n      time\n    end\n\n    # rough benchmark result to compare handmade parser vs Fluent::EventTime.from_time(Time.at(value.to_r))\n    # full: with 9-digits of nsec after dot\n    # msec: with 3-digits of msec after dot\n    # 10_000_000 times loop on MacBookAir\n    ## parse_by_myself(full): 12.162475 sec\n    ## parse_by_myself(msec): 15.050435 sec\n    ## parse_by_to_r  (full): 28.722362 sec\n    ## parse_by_to_r  (msec): 28.232856 sec\n    def parse_float(value)\n      unless value.is_a?(String) || value.is_a?(Numeric)\n        raise TimeParseError, \"value must be a string or a number: #{value}(#{value.class})\"\n      end\n\n      if @cache1_key == value\n        return @cache1_time\n      elsif @cache2_key == value\n        return @cache2_time\n      end\n\n      begin\n        sec_s, nsec_s, _ = value.to_s.split('.', 3) # throw away second-dot and later\n        nsec_s = nsec_s && nsec_s[0..9] || '0'\n        nsec_s += '0' * (9 - nsec_s.size) if nsec_s.size < 9\n        time = Fluent::EventTime.new(sec_s.to_i, nsec_s.to_i)\n      rescue => e\n        raise TimeParseError, \"invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}\"\n      end\n      @cache1_key = @cache2_key\n      @cache1_time = @cache2_time\n      @cache2_key = value\n      @cache2_time = time\n      time\n    end\n  end\n\n  class TimeFormatter\n    def initialize(format = nil, localtime = true, timezone = nil)\n      @tc1 = 0\n      @tc1_str = nil\n      @tc2 = 0\n      @tc2_str = nil\n\n      strftime = format && (Strftime.new(format) rescue nil)\n      if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\\d*N/\n        define_singleton_method(:format, method(:format_with_subsec))\n        define_singleton_method(:call, method(:format_with_subsec))\n      else\n        define_singleton_method(:format, method(:format_without_subsec))\n        define_singleton_method(:call, method(:format_without_subsec))\n      end\n\n      formatter = Fluent::Timezone.formatter(timezone, strftime ? strftime : format)\n      @format_nocache = case\n                        when formatter             then formatter\n                        when strftime && localtime then ->(time){ strftime.exec(Time.at(time)) }\n                        when format && localtime   then ->(time){ Time.at(time).strftime(format) }\n                        when strftime              then ->(time){ strftime.exec(Time.at(time).utc) }\n                        when format                then ->(time){ Time.at(time).utc.strftime(format) }\n                        when localtime             then ->(time){ Time.at(time).iso8601 }\n                        else                            ->(time){ Time.at(time).utc.iso8601 }\n                        end\n    end\n\n    def format_without_subsec(time)\n      if @tc1 == time\n        return @tc1_str\n      elsif @tc2 == time\n        return @tc2_str\n      else\n        str = format_nocache(time)\n        if @tc1 < @tc2\n          @tc1 = time\n          @tc1_str = str\n        else\n          @tc2 = time\n          @tc2_str = str\n        end\n        return str\n      end\n    end\n\n    def format_with_subsec(time)\n      if Fluent::EventTime.eq?(@tc1, time)\n        return @tc1_str\n      elsif Fluent::EventTime.eq?(@tc2, time)\n        return @tc2_str\n      else\n        str = format_nocache(time)\n        if @tc1 < @tc2\n          @tc1 = time\n          @tc1_str = str\n        else\n          @tc2 = time\n          @tc2_str = str\n        end\n        return str\n      end\n    end\n\n    ## Dynamically defined in #initialize\n    # def format(time)\n    # end\n\n    def format_nocache(time)\n      @format_nocache.call(time)\n    end\n  end\n\n  class NumericTimeFormatter < TimeFormatter\n    def initialize(type, localtime = nil, timezone = nil)\n      @cache1_key = @cache1_time = @cache2_key = @cache2_time = nil\n\n      if type == :unixtime\n        define_singleton_method(:format, method(:format_unixtime))\n        define_singleton_method(:call, method(:format_unixtime))\n      else # :float\n        define_singleton_method(:format, method(:format_float))\n        define_singleton_method(:call, method(:format_float))\n      end\n    end\n\n    def format_unixtime(time)\n      time.to_i.to_s\n    end\n\n    def format_float(time)\n      if time.is_a?(Fluent::EventTime) || time.is_a?(Time)\n        # 10.015 secs for 10_000_000 times call on MacBookAir\n        nsec_s = time.nsec.to_s\n        nsec_s = '0' * (9 - nsec_s.size) if nsec_s.size < 9\n        \"#{time.sec}.#{nsec_s}\"\n      else # integer (or float?)\n        time.to_f.to_s\n      end\n    end\n  end\n\n  # MixedTimeParser is available when time_type is set to :mixed\n  #\n  # Use Case 1: primary format is specified explicitly in time_format\n  #  time_type mixed\n  #  time_format %iso8601\n  #  time_format_fallbacks unixtime\n  # Use Case 2: time_format is omitted\n  #  time_type mixed\n  #  time_format_fallbacks %iso8601, unixtime\n  #\n  class MixedTimeParser < TimeParser # to include TimeParseError\n    def initialize(type, format = nil, localtime = nil, timezone = nil, utc = nil, force_localtime = nil, fallbacks = [])\n      @parsers = []\n      fallbacks.unshift(format).each do |fallback|\n        next unless fallback\n        case fallback\n        when 'unixtime', 'float'\n          @parsers << NumericTimeParser.new(fallback, localtime, timezone)\n        else\n          if force_localtime\n            @parsers << TimeParser.new(fallback, true, nil)\n          else\n            localtime = localtime && (timezone.nil? && !utc)\n            @parsers << TimeParser.new(fallback, localtime, timezone)\n          end\n        end\n      end\n    end\n\n    def parse(value)\n      @parsers.each do |parser|\n        begin\n          Float(value) if parser.class == Fluent::NumericTimeParser\n        rescue\n          next\n        end\n        begin\n          return parser.parse(value)\n        rescue\n          # skip TimeParseError\n        end\n      end\n      fallback_class = @parsers.collect do |parser| parser.class end.join(\",\")\n      raise TimeParseError, \"invalid time format: value = #{value}, even though fallbacks: #{fallback_class}\"\n    end\n  end\n\nend\n"
  },
  {
    "path": "lib/fluent/timezone.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'tzinfo'\n\nrequire 'fluent/config/error'\n\n# For v0.12. Will be removed after v2\nmodule IntegerExt\n  refine Integer do\n    def to_time\n      Time.at(self)\n    end\n  end\nend\n\nmodule Fluent\n  class Timezone\n    # [+-]HH:MM, [+-]HHMM, [+-]HH\n    NUMERIC_PATTERN = %r{\\A[+-]\\d\\d(:?\\d\\d)?\\z}\n\n    # Region/Zone, Region/Zone/Zone\n    NAME_PATTERN = %r{\\A[^/]+/[^/]+(/[^/]+)?\\z}\n\n    # Validate the format of the specified timezone.\n    #\n    # Valid formats are as follows. Note that timezone abbreviations\n    # such as PST and JST are not supported intentionally.\n    #\n    #   1. [+-]HH:MM         (e.g. \"+09:00\")\n    #   2. [+-]HHMM          (e.g. \"+0900\")\n    #   3. [+-]HH            (e.g. \"+09\")\n    #   4. Region/Zone       (e.g. \"Asia/Tokyo\")\n    #   5. Region/Zone/Zone  (e.g. \"America/Argentina/Buenos_Aires\")\n    #\n    # In the 4th and 5th cases, it is checked whether the specified\n    # timezone exists in the timezone database.\n    #\n    # When the given timezone is valid, true is returned. Otherwise,\n    # false is returned. When nil is given, false is returned.\n    def self.validate(timezone)\n      # If the specified timezone is nil.\n      if timezone.nil?\n        # Invalid.\n        return false\n      end\n\n      # [+-]HH:MM, [+-]HHMM, [+-]HH\n      if NUMERIC_PATTERN === timezone\n        # Valid. It can be parsed by Time.zone_offset method.\n        return true\n      end\n\n      # Region/Zone, Region/Zone/Zone\n      if NAME_PATTERN === timezone\n        begin\n          # Get a Timezone instance for the specified timezone.\n          TZInfo::Timezone.get(timezone)\n        rescue\n          # Invalid. The string does not exist in the timezone database.\n          return false\n        else\n          # Valid. The string was found in the timezone database.\n          return true\n        end\n      else\n        # Invalid. Timezone abbreviations are not supported.\n        return false\n      end\n    end\n\n    # Validate the format of the specified timezone.\n    #\n    # The implementation of this method calls validate(timezone) method\n    # to check whether the given timezone is valid. When invalid, this\n    # method raises a ConfigError.\n    def self.validate!(timezone)\n      unless validate(timezone)\n        raise ConfigError, \"Unsupported timezone '#{timezone}'\"\n      end\n    end\n\n    using IntegerExt\n\n    # Create a formatter for a timezone and optionally a format.\n    #\n    # An Proc object is returned. If the given timezone is invalid,\n    # nil is returned.\n    def self.formatter(timezone = nil, format = nil)\n      if timezone.nil?\n        return nil\n      end\n\n      # [+-]HH:MM, [+-]HHMM, [+-]HH\n      if NUMERIC_PATTERN === timezone\n        offset = Time.zone_offset(timezone)\n\n        case\n        when format.is_a?(String)\n          return Proc.new {|time|\n            time.to_time.localtime(offset).strftime(format)\n          }\n        when format.is_a?(Strftime)\n          return Proc.new {|time|\n            format.exec(time.to_time.localtime(offset))\n          }\n        else\n          return Proc.new {|time|\n            time.to_time.localtime(offset).iso8601\n          }\n        end\n      end\n\n      # Region/Zone, Region/Zone/Zone\n      if NAME_PATTERN === timezone\n        begin\n          tz = TZInfo::Timezone.get(timezone)\n        rescue\n          return nil\n        end\n\n        case\n        when format.is_a?(String)\n          return Proc.new {|time|\n            time = time.to_time\n            time.localtime(tz.period_for_utc(time).utc_total_offset).strftime(format)\n          }\n        when format.is_a?(Strftime)\n          return Proc.new {|time|\n            time = time.to_time\n            format.exec(time.localtime(tz.period_for_utc(time).utc_total_offset))\n          }\n        else\n          return Proc.new {|time|\n            time = time.to_time\n            time.localtime(tz.period_for_utc(time).utc_total_offset).iso8601\n          }\n        end\n      end\n\n      return nil\n    end\n\n    def self.utc_offset(timezone)\n      return 0 if timezone.nil?\n\n      case timezone\n      when NUMERIC_PATTERN\n        Time.zone_offset(timezone)\n      when NAME_PATTERN\n        tz = TZInfo::Timezone.get(timezone)\n        ->(time) {\n          tz.period_for_utc(time.to_time).utc_total_offset\n        }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/tls.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'openssl'\nrequire 'fluent/config/error'\n\nmodule Fluent\n  module TLS\n    DEFAULT_VERSION = :'TLSv1_2'\n    SUPPORTED_VERSIONS = if defined?(OpenSSL::SSL::TLS1_3_VERSION)\n                           [:'TLSv1_1', :'TLSv1_2', :'TLSv1_3', :'TLS1_1', :'TLS1_2', :'TLS1_3'].freeze\n                         else\n                           [:'TLSv1_1', :'TLSv1_2', :'TLS1_1', :'TLS1_2'].freeze\n                         end\n    ### follow httpclient configuration by nahi\n    # OpenSSL 0.9.8 default: \"ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH\"\n    CIPHERS_DEFAULT = \"ALL:!aNULL:!eNULL:!SSLv2\".freeze # OpenSSL >1.0.0 default\n\n    METHODS_MAP = begin\n                    map = {\n                      TLSv1: OpenSSL::SSL::TLS1_VERSION,\n                      TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,\n                      TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION\n                    }\n                    map[:'TLSv1_3'] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)\n                    MIN_MAX_AVAILABLE = true\n                    map.freeze\n                  rescue NameError\n                    # ruby 2.4 doesn't have OpenSSL::SSL::TLSXXX constants and min_version=/max_version= methods\n                    map = {\n                      TLS1: :'TLSv1',\n                      TLS1_1: :'TLSv1_1',\n                      TLS1_2: :'TLSv1_2',\n                    }.freeze\n                    MIN_MAX_AVAILABLE = false\n                    map\n                  end\n    private_constant :METHODS_MAP\n\n    # Helper for old syntax/method support:\n    # ruby 2.4 uses ssl_version= but this method is now deprecated.\n    # min_version=/max_version= use 'TLS1_2' but ssl_version= uses 'TLSv1_2'\n    def set_version_to_context(ctx, version, min_version, max_version)\n      if MIN_MAX_AVAILABLE\n        case\n        when min_version.nil? && max_version.nil?\n          min_version = METHODS_MAP[version] || version\n          max_version = METHODS_MAP[version] || version\n        when min_version.nil? && max_version\n          raise Fluent::ConfigError, \"When you set max_version, must set min_version together\"\n        when min_version && max_version.nil?\n          raise Fluent::ConfigError, \"When you set min_version, must set max_version together\"\n        else\n          min_version = METHODS_MAP[min_version] || min_version\n          max_version = METHODS_MAP[max_version] || max_version\n        end\n        ctx.min_version = min_version\n        ctx.max_version = max_version\n      else\n        ctx.ssl_version = METHODS_MAP[version] || version\n      end\n\n      ctx\n    end\n    module_function :set_version_to_context\n\n    def set_version_to_options(opt, version, min_version, max_version)\n      if MIN_MAX_AVAILABLE\n        case\n        when min_version.nil? && max_version.nil?\n          min_version = METHODS_MAP[version] || version\n          max_version = METHODS_MAP[version] || version\n        when min_version.nil? && max_version\n          raise Fluent::ConfigError, \"When you set max_version, must set min_version together\"\n        when min_version && max_version.nil?\n          raise Fluent::ConfigError, \"When you set min_version, must set max_version together\"\n        else\n          min_version = METHODS_MAP[min_version] || min_version\n          max_version = METHODS_MAP[max_version] || max_version\n        end\n        opt[:min_version] = min_version\n        opt[:max_version] = max_version\n      else\n        opt[:ssl_version] = METHODS_MAP[version] || version\n      end\n\n      opt\n    end\n    module_function :set_version_to_options\n  end\nend\n\n"
  },
  {
    "path": "lib/fluent/unique_id.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  module UniqueId\n    def self.generate\n      now = Time.now.utc\n      u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | rand(0xfff))\n      [u1 >> 32, u1 & 0xffffffff, rand(0xffffffff), rand(0xffffffff)].pack('NNNN')\n    end\n\n    def self.hex(unique_id)\n      unique_id.unpack1('H*')\n    end\n\n    module Mixin\n      def generate_unique_id\n        Fluent::UniqueId.generate\n      end\n\n      def dump_unique_id_hex(unique_id)\n        Fluent::UniqueId.hex(unique_id)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/variable_store.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n  # VariableStore provides all plugins with the way to shared variable without using class variable\n  # it's for safe reloading mechanism\n  class VariableStore\n    @data = {}\n\n    class << self\n      def fetch_or_build(namespace, default_value: {})\n        @data[namespace] ||= default_value\n      end\n\n      def try_to_reset\n        @data, old = {}, @data\n\n        begin\n          yield\n        rescue\n          @data = old\n          raise\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "lib/fluent/version.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent\n\n  VERSION = '1.19.0'\n\nend\n"
  },
  {
    "path": "lib/fluent/win32api.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/env'\n\nmodule Fluent\n  module Win32API\n    require 'fiddle/import'\n    require 'fiddle/types'\n    extend Fiddle::Importer\n\n    if RUBY_PLATFORM.split('-')[-1] == \"ucrt\"\n      MSVCRT_DLL = 'ucrtbase.dll'\n    else\n      MSVCRT_DLL = 'msvcrt.dll'\n    end\n\n    dlload MSVCRT_DLL, \"kernel32.dll\"\n    include Fiddle::Win32Types\n\n    extern \"intptr_t _get_osfhandle(int)\"\n    extern \"BOOL GetFileInformationByHandle(HANDLE, void *)\"\n    extern \"BOOL GetFileInformationByHandleEx(HANDLE, int, void *, DWORD)\"\n  end if Fluent.windows?\nend\n"
  },
  {
    "path": "lib/fluent/winsvc.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nbegin\n\n  require 'optparse'\n  require 'win32/daemon'\n  require 'win32/event'\n  require 'win32/registry'\n  require 'serverengine'\n\n  include Win32\n\n  op = OptionParser.new\n  opts = {service_name: nil}\n  op.on('--service-name NAME', \"The name of the Windows Service\") {|name|\n    opts[:service_name] = name\n  }\n  op.parse(ARGV)\n  if opts[:service_name] == nil\n    raise \"Error: No Windows Service name set. Use '--service-name'\"\n  end\n\n  def read_fluentdopt(service_name)\n    Win32::Registry::HKEY_LOCAL_MACHINE.open(\"SYSTEM\\\\CurrentControlSet\\\\Services\\\\#{service_name}\") do |reg|\n      reg.read(\"fluentdopt\")[1] rescue \"\"\n    end\n  end\n\n  def service_main_start(service_name)\n    ruby_path = ServerEngine.ruby_bin_path\n    rubybin_dir = ruby_path[0, ruby_path.rindex(\"/\")]\n    opt = read_fluentdopt(service_name)\n    Process.spawn(\"\\\"#{rubybin_dir}/ruby.exe\\\" \\\"#{rubybin_dir}/fluentd\\\" #{opt} -x #{service_name}\")\n  end\n\n  class FluentdService < Daemon\n    ERROR_WAIT_NO_CHILDREN = 128\n\n    @pid = 0\n    @service_name = ''\n\n    def initialize(service_name)\n      @service_name = service_name\n    end\n\n    def service_main\n      @pid = service_main_start(@service_name)\n      begin\n        loop do\n          sleep 5\n          break unless running?\n          raise Errno::ECHILD if Process.waitpid(@pid, Process::WNOHANG)\n        end\n      rescue Errno::ECHILD\n        @pid = 0\n        SetEvent(@@hStopEvent)\n        SetTheServiceStatus.call(SERVICE_STOPPED, ERROR_WAIT_NO_CHILDREN, 0, 0)\n      end\n    end\n\n    def service_stop\n      wait_supervisor_finished if @pid > 0\n    end\n\n    def service_paramchange\n      set_event(\"#{@service_name}_USR2\")\n    end\n\n    def service_user_defined_control(code)\n      case code\n      when 128\n        set_event(\"#{@service_name}_HUP\")\n      when 129\n        set_event(\"#{@service_name}_USR1\")\n      when 130\n        set_event(\"#{@service_name}_CONT\")\n      end\n    end\n\n    private\n\n    def set_event(event_name)\n      ev = Win32::Event.open(event_name)\n      ev.set\n      ev.close\n    end\n\n    def repeat_set_event_several_times_until_success(event_name)\n      retries = 0\n      max_retries = 10\n      delay_sec = 3\n\n      begin\n        set_event(event_name)\n      rescue Errno::ENOENT\n        # This error occurs when the supervisor process has not yet created the event.\n        # If STOP is immediately executed, this state will occur.\n        # Retry `set_event' to wait for the initialization of the supervisor.\n        retries += 1\n        raise if max_retries < retries\n        sleep(delay_sec)\n        retry\n      end\n    end\n\n    def wait_supervisor_finished\n      repeat_set_event_several_times_until_success(@service_name)\n      Process.waitpid(@pid)\n    end\n  end\n\n  FluentdService.new(opts[:service_name]).mainloop\n\nrescue Exception => err\n  raise\nend\n"
  },
  {
    "path": "tasks/backport/backporter.rb",
    "content": "require 'open-uri'\nrequire 'json'\nrequire 'optparse'\nrequire 'logger'\n\nclass PullRequestBackporter\n\n  def initialize\n    @logger = Logger.new(STDOUT)\n    @options = {\n      upstream: \"fluent/fluentd\",\n      branch: \"v1.16\",\n      dry_run: false,\n      log_level: Logger::Severity::INFO,\n      remote: 'origin'\n    }\n  end\n\n  def current_branch\n    branch = IO.popen([\"git\", \"branch\", \"--contains\"]) do |io|\n      io.read\n    end\n    branch.split.last\n  end\n\n  def parse_command_line(argv)\n    opt = OptionParser.new\n    opt.on('--upstream REPOSITORY',\n           'Specify upstream repository (e.g. fluent/fluentd)') {|v| @options[:upstream] = v }\n    opt.on('--branch BRANCH') {|v| @options[:branch] = v }\n    opt.on('--dry-run') {|v| @options[:dry_run] = true }\n    opt.on('--log-level LOG_LEVEL (e.g. debug,info)') {|v|\n      @options[:log_level] = case v\n                             when \"error\"\n                               Logger::Severity::ERROR\n                             when \"warn\"\n                               Logger::Severity::WARN\n                             when \"debug\"\n                               Logger::Severity::DEBUG\n                             when \"info\"\n                               Logger::Severity::INFO\n                             else\n                               puts \"unknown log level: <#{v}>\"\n                               exit 1\n                             end\n    }\n    opt.on('--remote REMOTE') {|v| @options[:remote] = v }\n    opt.parse!(argv)\n  end\n\n  def collect_backports\n    backports = []\n    pages = 5\n    pages.times.each do |page|\n      @logger.debug \"Collecting backport information (#{page + 1}/#{pages})\"\n      URI.open(\"https://api.github.com/repos/#{@options[:upstream]}/pulls?state=closed&per_page=100&page=#{page+1}\",\n               \"Accept\" => \"application/vnd.github+json\",\n               \"Authorization\" => \"Bearer #{ENV['GITHUB_TOKEN']}\",\n               \"X-GitHub-Api-Version\" => \"2022-11-28\") do |request|\n        JSON.parse(request.read).each do |pull_request|\n          unless pull_request[\"labels\"].empty?\n            labels = pull_request[\"labels\"].collect { |label| label[\"name\"] }\n            unless labels.include?(\"backport to #{@options[:branch]}\")\n              next\n            end\n            if labels.include?(\"backported\")\n              @logger.info \"[DONE] \\##{pull_request['number']} #{pull_request['title']} LABELS: #{pull_request['labels'].collect { |label| label['name'] }}\"\n              next\n            end\n            @logger.info \"* \\##{pull_request['number']} #{pull_request['title']} LABELS: #{pull_request['labels'].collect { |label| label['name'] }}\"\n            # merged into this commit\n            @logger.debug \"MERGE_COMMIT_SHA: #{pull_request['merge_commit_sha']}\"\n            body = pull_request[\"body\"].gsub(/\\*\\*Which issue\\(s\\) this PR fixes\\*\\*: \\r\\n/,\n                                             \"**Which issue(s) this PR fixes**: \\r\\nBackport \\##{pull_request['number']}\\r\\n\")\n            backports << {\n              number: pull_request[\"number\"],\n              merge_commit_sha: pull_request[\"merge_commit_sha\"],\n              title: \"Backport(#{@options[:branch]}): #{pull_request['title']} (\\##{pull_request['number']})\",\n              body: body\n            }\n          end\n        end\n      end\n    end\n    backports\n  end\n\n  def create_pull_requests\n    backports = collect_backports\n    if backports.empty?\n      @logger.info \"No need to backport pull requests\"\n      return\n    end\n\n    failed = []\n    original_branch = current_branch\n    backports.each do |backport|\n      @logger.info \"Backport #{backport[:number]} #{backport[:title]}\"\n      if @options[:dry_run]\n        @logger.info \"DRY_RUN: PR was created: \\##{backport[:number]} #{backport[:title]}\"\n        next\n      end\n      begin\n        branch = \"backport-to-#{@options[:branch]}/pr#{backport[:number]}\"\n        @logger.debug \"git switch --create #{branch} --track #{@options[:remote]}/#{@options[:branch]}\"\n        IO.popen([\"git\", \"switch\", \"--create\",  branch, \"--track\",  \"#{@options[:remote]}/#{@options[:branch]}\"]) do |io|\n          @logger.debug io.read\n        end\n        @logger.info `git branch`\n        @logger.info \"cherry-pick for #{backport[:number]}\"\n        @logger.debug \"git cherry-pick --signoff #{backport[:merge_commit_sha]}\"\n        IO.popen([\"git\", \"cherry-pick\", \"--signoff\", backport[:merge_commit_sha]]) do |io|\n          @logger.debug io.read\n        end\n        if $? != 0\n          @logger.warn \"Give up cherry-pick for #{backport[:number]}\"\n          @logger.debug `git cherry-pick --abort`\n          failed << backport\n          next\n        else\n          @logger.info \"Push branch: #{branch}\"\n          @logger.debug `git push origin #{branch}`\n        end\n\n        upstream_repo = \"/repos/#{@options[:upstream]}/pulls\"\n        owner = @options[:upstream].split('/').first\n        head = \"#{owner}:#{branch}\"\n        @logger.debug \"Create pull request repo: #{upstream_repo} head: #{head} base: #{@options[:branch]}\"\n        IO.popen([\"gh\", \"api\", \"--method\", \"POST\",\n                  \"-H\", \"Accept: application/vnd.github+json\",\n                  \"-H\", \"X-GitHub-Api-Version: 2022-11-28\",\n                  upstream_repo,\n                  \"-f\", \"title=#{backport[:title]}\",\n                  \"-f\", \"body=#{backport[:body]}\",\n                  \"-f\", \"head=#{head}\",\n                  \"-f\", \"base=#{@options[:branch]}\"]) do |io|\n          json = JSON.parse(io.read)\n          @logger.info \"PR was created: #{json['url']}\"\n        end\n      rescue => e\n        @logger.error \"ERROR: #{backport[:number]} #{e.message}\"\n      ensure\n        IO.popen([\"git\", \"checkout\", original_branch]) do |io|\n          @logger.debug io.read\n        end\n      end\n    end\n    failed.each do |backport|\n      @logger.error \"FAILED: #{backport[:number]} #{backport[:title]}\"\n    end\n    failed.empty?\n  end\n\n  def run(argv)\n    parse_command_line(argv)\n    @logger.info(\"Target upstream: #{@options[:upstream]} target branch: #{@options[:branch]}\")\n    create_pull_requests\n  end\nend\n"
  },
  {
    "path": "tasks/backport.rb",
    "content": "require_relative 'backport/backporter'\n\n=begin\n\nWhen you want to manually execute backporting, set the following\nenvironment variables:\n\n* GITHUB_REPOSITORY: fluent/fluentd\n* GITHUB_TOKEN: ${PERSONAL_ACCESS_TOKEN}\n\nOptional:\n\n* REPOSITORY_REMOTE: origin\n  If you execute in forked repository, it might be 'upstream'\n\n=end\n\ndef append_additional_arguments(commands)\n  if ENV['DRY_RUN']\n    commands << '--dry-run'\n  end\n  if ENV['GITHUB_REPOSITORY']\n    commands << '--upstream'\n    commands << ENV['GITHUB_REPOSITORY']\n  end\n  if ENV['REPOSITORY_REMOTE']\n    commands << '--remote'\n    commands << ENV['REPOSITORY_REMOTE']\n  end\n  commands\nend\n\nnamespace :backport do\n\n  desc \"Backport PR to v1.19 branch\"\n  task :v1_19 do\n    commands = ['--branch', 'v1.19', '--log-level', 'debug']\n    commands = append_additional_arguments(commands)\n    backporter = PullRequestBackporter.new\n    exit(backporter.run(commands))\n  end\nend\n"
  },
  {
    "path": "tasks/benchmark/conf/in_tail.conf",
    "content": "<source>\n  @type tail\n  tag benchmark\n  path \"#{File.expand_path './tmp/benchmark/data.log'}\"\n  read_from_head true\n  <parse>\n    @type json\n  </parse>\n</source>\n\n<match **>\n  @type file\n  path \"#{File.expand_path './tmp/benchmark/in_tail'}\"\n</match>\n"
  },
  {
    "path": "tasks/benchmark/patch_in_tail.rb",
    "content": "require 'benchmark'\nrequire 'fluent/plugin/in_tail'\n\nclass Fluent::Plugin::TailInput::TailWatcher::IOHandler\n  alias_method :original_with_io, :with_io\n\n  def with_io(&block)\n    @benchmark_measured_in_tail ||= false\n    # Measure the benchmark only once.\n    return original_with_io(&block) if @benchmark_measured_in_tail\n\n    Benchmark.bm do |x|\n      x.report {\n        original_with_io(&block)\n        @benchmark_measured_in_tail = true\n      }\n    end\n\n    exit 0\n  end\nend\n"
  },
  {
    "path": "tasks/benchmark.rb",
    "content": "require \"json\"\nrequire \"fileutils\"\n\nBENCHMARK_FILE_SIZE = 1 * 1024 * 1024 * 1024\nBENCHMARK_FILE_PATH = File.expand_path(\"./tmp/benchmark/data.log\")\n\nnamespace :benchmark do\n  task :init do\n    # Synchronize stdout because the output order is not as intended on Windows environment\n    STDOUT.sync = true\n  end\n\n  task :prepare_1GB do\n    FileUtils.mkdir_p(File.dirname(BENCHMARK_FILE_PATH))\n    File.open(BENCHMARK_FILE_PATH, \"w\") do |f|\n      data = { \"message\": \"a\" * 1024 }.to_json\n\n      loop do\n        f.puts data\n        break if f.size > BENCHMARK_FILE_SIZE\n      end\n    end\n  end\n\n  task :show_info do\n    # Output the information with markdown format\n    puts \"### Environment\"\n    puts \"```\"\n    system \"bundle exec ruby --version\"\n    system \"bundle exec ruby bin/fluentd --version\"\n    puts \"```\\n\"\n  end\n\n  desc \"Run in_tail benchmark\"\n  task :\"run:in_tail\" => [:init, :prepare_1GB, :show_info] do\n    # Output the results with markdown format\n    puts \"### in_tail with 1 GB file\"\n    puts \"```\"\n    system \"bundle exec ruby bin/fluentd -r ./tasks/benchmark/patch_in_tail.rb --no-supervisor -c ./tasks/benchmark/conf/in_tail.conf -o ./tmp/benchmark/fluent.log\"\n    puts \"```\"\n\n    Rake::Task[\"benchmark:clean\"].invoke\n  end\n\n  task :clean do\n    FileUtils.rm_rf(File.dirname(BENCHMARK_FILE_PATH))\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/Gemfile",
    "content": "source \"https://rubygems.org\"\n\ngemspec\n"
  },
  {
    "path": "templates/new_gem/README.md.erb",
    "content": "# <%= gem_name %>\n\n[Fluentd](https://fluentd.org/) <%= type %> plugin to do something.\n\nTODO: write description for you plugin.\n\n## Installation\n\n### RubyGems\n\n```\n$ gem install <%= gem_name %>\n```\n\n### Bundler\n\nAdd following line to your Gemfile:\n\n```ruby\ngem \"<%= gem_name %>\"\n```\n\nAnd then execute:\n\n```\n$ bundle\n```\n\n## Configuration\n\nYou can generate configuration template:\n\n```\n$ fluent-plugin-config-format <%= type %> <%= name %>\n```\n\nYou can copy and paste generated documents here.\n\n## Copyright\n\n* Copyright(c) <%= Date.today.year %>- <%= user_name %>\n* License\n  * <%= @license.full_name %>\n"
  },
  {
    "path": "templates/new_gem/Rakefile",
    "content": "require \"bundler\"\nBundler::GemHelper.install_tasks\n\nrequire \"rake/testtask\"\n\nRake::TestTask.new(:test) do |t|\n  t.libs.push(\"lib\", \"test\")\n  t.test_files = FileList[\"test/**/test_*.rb\"]\n  t.verbose = true\n  t.warning = true\nend\n\ntask default: [:test]\n"
  },
  {
    "path": "templates/new_gem/fluent-plugin.gemspec.erb",
    "content": "lib = File.expand_path(\"../lib\", __FILE__)\n$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)\n\nGem::Specification.new do |spec|\n  spec.name    = \"<%= gem_name %>\"\n  spec.version = \"0.1.0\"\n  spec.authors = [\"<%= user_name %>\"]\n  spec.email   = [\"<%= user_email %>\"]\n\n  spec.summary       = %q{TODO: Write a short summary, because Rubygems requires one.}\n  spec.description   = %q{TODO: Write a longer description or delete this line.}\n  spec.homepage      = \"TODO: Put your gem's website or public repo URL here.\"\n  spec.license       = \"<%= @license.name %>\"\n\n  spec.files         = Dir.chdir(__dir__) do\n    `git ls-files -z`.split(\"\\x0\").reject do |f|\n      (File.expand_path(f) == __FILE__) ||\n        f.start_with?(*%w[test/ spec/ features/ .git .circleci appveyor Gemfile])\n    end\n  end\n  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }\n  spec.require_paths = [\"lib\"]\n\n  spec.add_development_dependency \"bundler\", \"~> <%= bundler_version %>\"\n  spec.add_development_dependency \"rake\", \"~> <%= rake_version %>\"\n  spec.add_development_dependency \"test-unit\", \"~> <%= test_unit_version %>\"\n  spec.add_runtime_dependency \"fluentd\", [\">= 0.14.10\", \"< 2\"]\nend\n"
  },
  {
    "path": "templates/new_gem/lib/fluent/plugin/filter.rb.erb",
    "content": "<%= preamble %>\n\nrequire \"fluent/plugin/filter\"\n\nmodule Fluent\n  module Plugin\n    class <%= class_name %> < Fluent::Plugin::Filter\n      Fluent::Plugin.register_filter(\"<%= plugin_name %>\", self)\n\n      def filter(tag, time, record)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/lib/fluent/plugin/formatter.rb.erb",
    "content": "<%= preamble %>\n\nrequire \"fluent/plugin/formatter\"\n\nmodule Fluent\n  module Plugin\n    class <%= class_name %> < Fluent::Plugin::Formatter\n      Fluent::Plugin.register_formatter(\"<%= name %>\", self)\n\n      def format(tag, time, record)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/lib/fluent/plugin/input.rb.erb",
    "content": "<%= preamble %>\n\nrequire \"fluent/plugin/input\"\n\nmodule Fluent\n  module Plugin\n    class <%= class_name %> < Fluent::Plugin::Input\n      Fluent::Plugin.register_input(\"<%= plugin_name %>\", self)\n    end\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/lib/fluent/plugin/output.rb.erb",
    "content": "<%= preamble %>\n\nrequire \"fluent/plugin/output\"\n\nmodule Fluent\n  module Plugin\n    class <%= class_name %> < Fluent::Plugin::Output\n      Fluent::Plugin.register_output(\"<%= plugin_name %>\", self)\n    end\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/lib/fluent/plugin/parser.rb.erb",
    "content": "<%= preamble %>\n\nrequire \"fluent/plugin/parser\"\n\nmodule Fluent\n  module Plugin\n    class <%= class_name %> < Fluent::Plugin::Parser\n      Fluent::Plugin.register_parser(\"<%= plugin_name %>\", self)\n\n      def parse(text)\n        yield time, record\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/lib/fluent/plugin/storage.rb.erb",
    "content": "<%= preamble %>\n\nrequire \"fluent/plugin/storage\"\n\nmodule Fluent\n  module Plugin\n    class <%= class_name %> < Fluent::Plugin::Storage\n      Fluent::Plugin.register_storage(\"<%= plugin_name %>\", self)\n\n      def initialize\n        super\n      end\n\n      def configure(conf)\n        super\n      end\n\n      def load\n      end\n\n      def save\n      end\n\n      def get(key)\n      end\n\n      def fetch(key, defval)\n      end\n\n      def put(key, value)\n      end\n\n      def delete(key)\n      end\n\n      def update(key, &block)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/test/helper.rb.erb",
    "content": "require \"test-unit\"\nrequire \"fluent/test\"\nrequire \"fluent/test/driver/<%= type %>\"\nrequire \"fluent/test/helpers\"\n\nTest::Unit::TestCase.include(Fluent::Test::Helpers)\nTest::Unit::TestCase.extend(Fluent::Test::Helpers)\n"
  },
  {
    "path": "templates/new_gem/test/plugin/test_filter.rb.erb",
    "content": "require \"helper\"\nrequire \"fluent/plugin/<%= plugin_filename %>\"\n\nclass <%= class_name %>Test < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n  end\n\n  test \"failure\" do\n    flunk\n  end\n\n  private\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Filter.new(Fluent::Plugin::<%= class_name %>).configure(conf)\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/test/plugin/test_formatter.rb.erb",
    "content": "require \"helper\"\nrequire \"fluent/plugin/<%= plugin_filename %>\"\n\nclass <%= class_name %>Test < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n  end\n\n  test \"failure\" do\n    flunk\n  end\n\n  private\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::<%= class_name %>).configure(conf)\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/test/plugin/test_input.rb.erb",
    "content": "require \"helper\"\nrequire \"fluent/plugin/<%= plugin_filename %>\"\n\nclass <%= class_name %>Test < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n  end\n\n  test \"failure\" do\n    flunk\n  end\n\n  private\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::<%= class_name %>).configure(conf)\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/test/plugin/test_output.rb.erb",
    "content": "require \"helper\"\nrequire \"fluent/plugin/<%= plugin_filename %>\"\n\nclass <%= class_name %>Test < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n  end\n\n  test \"failure\" do\n    flunk\n  end\n\n  private\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Output.new(Fluent::Plugin::<%= class_name %>).configure(conf)\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/test/plugin/test_parser.rb.erb",
    "content": "require \"helper\"\nrequire \"fluent/plugin/<%= plugin_filename %>\"\n\nclass <%= class_name %>Test < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n  end\n\n  test \"failure\" do\n    flunk\n  end\n\n  private\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Parser.new(Fluent::Plugin::<%= class_name %>).configure(conf)\n  end\nend\n"
  },
  {
    "path": "templates/new_gem/test/plugin/test_storage.rb.erb",
    "content": "require \"helper\"\nrequire \"fluent/plugin/<%= plugin_filename %>\"\n\nclass <%= class_name %>Test < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n  end\n\n  test \"failure\" do\n    flunk\n  end\n\n  private\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Storage.new(Fluent::Plugin::<%= class_name %>).configure(conf)\n  end\nend\n"
  },
  {
    "path": "templates/plugin_config_formatter/param.md-compact.erb",
    "content": "<%-\ntype = config[:type]\nrequired_label = config[:required] ? \"required\" : \"optional\"\ndefault = config[:default]\nalias_name = config[:alias]\ndeprecated = config[:deprecated]\nobsoleted = config[:obsoleted]\ndescription = config[:desc]\n-%>\n* **<%= name %>** (<%= type %>) (<%= required_label %>): <%= description %>\n<%- if type == :enum -%>\n  * Available values: <%= config[:list].join(\", \") %>\n<%- end -%>\n<%- if default -%>\n  * Default value: `<%= default %>`.\n<%- end -%>\n<%- if alias_name -%>\n  * Alias: <%= alias_name %>\n<%- end -%>\n<%- if deprecated -%>\n  * Deprecated: <%= deprecated %>\n<%- end -%>\n<%- if obsoleted -%>\n  * Obsoleted: <%= :obsoleted %>\n<%- end -%>\n"
  },
  {
    "path": "templates/plugin_config_formatter/param.md-table.erb",
    "content": "<%-\ntype = config[:type]\nrequired_label = config[:required] ? \"required\" : \"optional\"\ndefault = config[:default]\nalias_name = config[:alias]\ndeprecated = config[:deprecated]\nobsoleted = config[:obsoleted]\ndescription = config[:desc]\n-%>\n|<%= name %>|<%= type %> (<%= required_label %>)|<%= description %><%- if type == :enum -%> (<%= config[:list].map{|x| \"`#{x}`\"}.join(\", \") %>)<%- end -%><%- if alias_name -%><br>Alias: <%= alias_name %><%- end -%><%- if deprecated -%><br>Deprecated: <%= deprecated %><%- end -%><%- if obsoleted -%><br>Obsoleted: <%= :obsoleted %><%- end -%>|<%- if default -%>`<%= default %>`<%- end -%>|\n"
  },
  {
    "path": "templates/plugin_config_formatter/param.md.erb",
    "content": "<%-\ntype = config[:type]\nrequired_label = config[:required] ? \"required\" : \"optional\"\ndefault = config[:default]\nalias_name = config[:alias]\ndeprecated = config[:deprecated]\nobsoleted = config[:obsoleted]\ndescription = config[:desc]\nparam_header = \"#\" * (3 + level)\n-%>\n<%= param_header %> <%= name %> (<%= type %>) (<%= required_label %>)\n\n<%= description %>\n<%- if type == :enum -%>\n\nAvailable values: <%= config[:list].join(\", \") %>\n<%- end -%>\n<%- if default -%>\n\nDefault value: `<%= default %>`.\n<%- end -%>\n<%- if alias_name -%>\n\nAlias: <%= alias_name %>\n<%- end -%>\n<%- if deprecated -%>\n\nDeprecated: <%= deprecated %>\n<%- end -%>\n<%- if obsoleted -%>\n\nObsoleted: <%= :obsoleted %>\n<%- end -%>\n\n"
  },
  {
    "path": "templates/plugin_config_formatter/section.md.erb",
    "content": "<%-\nsection_header = \"#\" * (3 + level)\nrequired_label = required ? \"required\" : \"optional\"\nmultiple_label = multi ? \"multiple\" : \"single\"\n-%>\n<%= section_header %> \\<<%= section_name %>\\> section (<%= required_label %>) (<%= multiple_label %>)\n<%- if alias_name -%>\n\nAlias: <%= alias_name %>\n<%- end -%>\n\n<%= dump_section_markdown(sub_section, level + 1) %>\n"
  },
  {
    "path": "test/command/test_binlog_reader.rb",
    "content": "require_relative '../helper'\n\nrequire 'json'\nrequire 'flexmock/test_unit'\n\nrequire 'fluent/command/binlog_reader'\nrequire 'fluent/event'\n\nclass TestFluentBinlogReader < ::Test::Unit::TestCase\n  module ::BinlogReaderCommand\n    class Dummy < Base\n      def call; end\n    end\n  end\n\n  def suppress_stdout\n    out = StringIO.new\n    $stdout = out\n    yield\n  ensure\n    $stdout = STDOUT\n  end\n\n  sub_test_case 'call' do\n    data(\n      empty: [],\n      invalid: %w(invalid packed.log),\n    )\n    test 'should fail when invalid command' do |argv|\n      fu = FluentBinlogReader.new(argv)\n\n      assert_raise(SystemExit) do\n        suppress_stdout { fu.call }\n      end\n    end\n\n    data(\n      cat: %w(cat packed.log),\n      head: %w(head packed.log),\n      formats: %w(formats packed.log)\n    )\n    test 'should succeed when valid command' do |argv|\n      fu = FluentBinlogReader.new(argv)\n\n      flexstub(::BinlogReaderCommand) do |command|\n        command.should_receive(:const_get).once.and_return(::BinlogReaderCommand::Dummy)\n        assert_nothing_raised do\n          fu.call\n        end\n      end\n    end\n  end\nend\n\nclass TestBaseCommand < ::Test::Unit::TestCase\n  TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/../tmp/command/binlog_reader#{ENV['TEST_ENV_NUMBER']}\")\n\n  def create_message_packed_file(path, times = [event_time], records = [{ 'message' => 'dummy' }])\n    es = Fluent::MultiEventStream.new(times, records)\n    v = es.to_msgpack_stream\n    out_path = \"#{TMP_DIR}/#{path}\"\n    File.open(out_path, 'wb') do |f|\n      f.print(v)\n    end\n    waiting(5) do\n      sleep 0.5 until File.size(out_path) == v.bytesize\n    end\n  end\n\n  def setup\n    FileUtils.rm_rf(TMP_DIR)\n    FileUtils.mkdir_p(TMP_DIR)\n  end\n\n  def timezone(timezone = 'UTC')\n    old = ENV['TZ']\n    ENV['TZ'] = timezone\n    yield\n  ensure\n    ENV['TZ'] = old\n  end\nend\n\nclass TestHead < TestBaseCommand\n  setup do\n    @default_newline = if Fluent.windows?\n                         \"\\r\\n\"\n                       else\n                         \"\\n\"\n                       end\n  end\n\n  sub_test_case 'initialize' do\n    data(\n      'file is not passed' => %w(),\n      'file is not found' => %w(invalid_path.log)\n    )\n    test 'should fail if file is invalid' do |argv|\n      assert_raise(SystemExit) do\n        capture_stdout { BinlogReaderCommand::Head.new(argv) }\n      end\n    end\n\n    test 'should succeed if a file is valid' do\n      file_name = 'packed.log'\n      argv = [\"#{TMP_DIR}/#{file_name}\"]\n      create_message_packed_file(file_name)\n\n      assert_nothing_raised do\n        BinlogReaderCommand::Head.new(argv)\n      end\n    end\n\n    test 'should fail when config_params format is invalid' do\n      file_name = 'packed.log'\n      argv = [\"#{TMP_DIR}/#{file_name}\", '--format=csv', '-e', 'only_key']\n      create_message_packed_file(file_name)\n\n      assert_raise(SystemExit) do\n        capture_stdout { BinlogReaderCommand::Head.new(argv) }\n      end\n    end\n\n    test 'should succeed if config_params format is valid' do\n      file_name = 'packed.log'\n      argv = [\"#{TMP_DIR}/#{file_name}\", '--format=csv', '-e', 'fields=message']\n      create_message_packed_file(file_name)\n\n      assert_nothing_raised do\n        capture_stdout { BinlogReaderCommand::Head.new(argv) }\n      end\n    end\n  end\n\n  sub_test_case 'call' do\n    setup do\n      @file_name = 'packed.log'\n      @t = '2011-01-02 13:14:15 UTC'\n      @record = { 'message' => 'dummy' }\n    end\n\n    test 'should output the beginning of the file with default format (out_file)' do\n      argv = [\"#{TMP_DIR}/#{@file_name}\"]\n\n      timezone do\n        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)\n        head = BinlogReaderCommand::Head.new(argv)\n        out = capture_stdout { head.call }\n        assert_equal \"2011-01-02T13:14:15+00:00\\t#{TMP_DIR}/#{@file_name}\\t#{JSON.generate(@record)}#{@default_newline}\" * 5, out\n      end\n    end\n\n    test 'should set the number of lines to display' do\n      argv = [\"#{TMP_DIR}/#{@file_name}\", '-n', '1']\n\n      timezone do\n        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)\n        head = BinlogReaderCommand::Head.new(argv)\n        out = capture_stdout { head.call }\n        assert_equal \"2011-01-02T13:14:15+00:00\\t#{TMP_DIR}/#{@file_name}\\t#{JSON.generate(@record)}#{@default_newline}\", out\n      end\n    end\n\n    test 'should fail when the number of lines is invalid' do\n      argv = [\"#{TMP_DIR}/#{@file_name}\", '-n', '0']\n\n      create_message_packed_file(@file_name)\n      assert_raise(SystemExit) do\n        capture_stdout { BinlogReaderCommand::Head.new(argv) }\n      end\n    end\n\n    test 'should output content of a file with json format' do\n      argv = [\"#{TMP_DIR}/#{@file_name}\", '--format=json']\n\n      timezone do\n        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])\n        head = BinlogReaderCommand::Head.new(argv)\n        out = capture_stdout { head.call }\n        assert_equal \"#{JSON.generate(@record)}#{@default_newline}\", out\n      end\n    end\n\n    test 'should fail with an invalid format' do\n      argv = [\"#{TMP_DIR}/#{@file_name}\", '--format=invalid']\n\n      timezone do\n        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])\n        head = BinlogReaderCommand::Head.new(argv)\n\n        assert_raise(SystemExit) do\n          capture_stdout { head.call }\n        end\n      end\n    end\n\n    test 'should succeed if multiple config_params format' do\n      file_name = 'packed.log'\n      argv = [\"#{TMP_DIR}/#{file_name}\", '--format=csv', '-e', 'fields=message,fo', '-e', 'delimiter=|']\n      create_message_packed_file(file_name, [event_time], [{ 'message' => 'dummy', 'fo' => 'dummy2' }])\n\n      head = BinlogReaderCommand::Head.new(argv)\n      assert_equal \"\\\"dummy\\\"|\\\"dummy2\\\"\\n\", capture_stdout { head.call }\n    end\n  end\nend\n\nclass TestCat < TestBaseCommand\n  setup do\n    @default_newline = if Fluent.windows?\n                         \"\\r\\n\"\n                       else\n                         \"\\n\"\n                       end\n  end\n\n  sub_test_case 'initialize' do\n    data(\n      'file is not passed' => [],\n      'file is not found' => %w(invalid_path.log)\n    )\n    test 'should fail if a file is invalid' do |argv|\n      assert_raise(SystemExit) do\n        capture_stdout { BinlogReaderCommand::Head.new(argv) }\n      end\n    end\n\n    test 'should succeed if a file is valid' do\n      file_name = 'packed.log'\n      argv = [\"#{TMP_DIR}/#{file_name}\"]\n      create_message_packed_file(file_name)\n\n      assert_nothing_raised do\n        BinlogReaderCommand::Cat.new(argv)\n      end\n    end\n\n    test 'should fail when config_params format is invalid' do\n      file_name = 'packed.log'\n      argv = [\"#{TMP_DIR}/#{file_name}\", '--format=json', '-e', 'only_key']\n      create_message_packed_file(file_name)\n\n      assert_raise(SystemExit) do\n        capture_stdout { BinlogReaderCommand::Cat.new(argv) }\n      end\n    end\n\n    test 'should succeed when config_params format is valid' do\n      file_name = 'packed.log'\n      argv = [\"#{TMP_DIR}/#{file_name}\", '--format=csv', '-e', 'fields=message']\n      create_message_packed_file(file_name)\n\n      assert_nothing_raised do\n        capture_stdout { BinlogReaderCommand::Cat.new(argv) }\n      end\n    end\n  end\n\n  sub_test_case 'call' do\n    setup do\n      @file_name = 'packed.log'\n      @t = '2011-01-02 13:14:15 UTC'\n      @record = { 'message' => 'dummy' }\n    end\n\n    test 'should output the file with default format(out_file)' do\n      argv = [\"#{TMP_DIR}/#{@file_name}\"]\n\n      timezone do\n        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)\n        head = BinlogReaderCommand::Cat.new(argv)\n        out = capture_stdout { head.call }\n        assert_equal \"2011-01-02T13:14:15+00:00\\t#{TMP_DIR}/#{@file_name}\\t#{JSON.generate(@record)}#{@default_newline}\" * 6, out\n      end\n    end\n\n    test 'should set the number of lines to display' do\n      argv = [\"#{TMP_DIR}/#{@file_name}\", '-n', '1']\n\n      timezone do\n        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)\n        head = BinlogReaderCommand::Cat.new(argv)\n        out = capture_stdout { head.call }\n        assert_equal \"2011-01-02T13:14:15+00:00\\t#{TMP_DIR}/#{@file_name}\\t#{JSON.generate(@record)}#{@default_newline}\", out\n      end\n    end\n\n    test 'should output content of a file with json format' do\n      argv = [\"#{TMP_DIR}/#{@file_name}\", '--format=json']\n\n      timezone do\n        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])\n        head = BinlogReaderCommand::Cat.new(argv)\n        out = capture_stdout { head.call }\n        assert_equal \"#{JSON.generate(@record)}#{@default_newline}\", out\n      end\n    end\n\n    test 'should fail with an invalid format' do\n      argv = [\"#{TMP_DIR}/#{@file_name}\", '--format=invalid']\n\n      timezone do\n        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])\n        head = BinlogReaderCommand::Cat.new(argv)\n\n        assert_raise(SystemExit) do\n          capture_stdout { head.call }\n        end\n      end\n    end\n\n    test 'should succeed if multiple config_params format' do\n      file_name = 'packed.log'\n      argv = [\"#{TMP_DIR}/#{file_name}\", '--format=csv', '-e', 'fields=message,fo', '-e', 'delimiter=|']\n      create_message_packed_file(file_name, [event_time], [{ 'message' => 'dummy', 'fo' => 'dummy2' }])\n\n      head = BinlogReaderCommand::Cat.new(argv)\n      assert_equal \"\\\"dummy\\\"|\\\"dummy2\\\"\\n\", capture_stdout { head.call }\n    end\n  end\nend\n\nclass TestFormats < TestBaseCommand\n  test 'parse_option!' do\n    assert_raise(SystemExit) do\n      capture_stdout do\n        BinlogReaderCommand::Formats.new(['--plugin=invalid_dir_path'])\n      end\n    end\n  end\n\n  sub_test_case 'call' do\n    test 'display available plugins' do\n      f = BinlogReaderCommand::Formats.new\n      out = capture_stdout { f.call }\n      assert out.include?('json')\n      assert out.include?('csv')\n    end\n\n    test 'add new plugins using --plugin option' do\n      dir_path = File.expand_path(File.dirname(__FILE__) + '/../scripts/fluent/plugin/formatter1')\n\n      f = BinlogReaderCommand::Formats.new([\"--plugin=#{dir_path}\"])\n      out = capture_stdout { f.call }\n      assert out.include?('json')\n      assert out.include?('csv')\n      assert out.include?('test1')\n    end\n\n    test 'add multiple plugins using --plugin option' do\n      dir_path1 = File.expand_path(File.dirname(__FILE__) + '/../scripts/fluent/plugin/formatter1')\n      dir_path2 = File.expand_path(File.dirname(__FILE__) + '/../scripts/fluent/plugin/formatter2')\n\n      f = BinlogReaderCommand::Formats.new([\"--plugin=#{dir_path1}\", '-p', dir_path2])\n      out = capture_stdout { f.call }\n      assert out.include?('json')\n      assert out.include?('csv')\n      assert out.include?('test1')\n      assert out.include?('test2')\n    end\n  end\nend\n"
  },
  {
    "path": "test/command/test_ca_generate.rb",
    "content": "require_relative '../helper'\n\nrequire 'flexmock/test_unit'\nrequire 'tmpdir'\n\nrequire 'fluent/command/ca_generate'\nrequire 'fluent/event'\n\nclass TestFluentCaGenerate < ::Test::Unit::TestCase\n  def test_generate_ca_pair\n    cert, key = Fluent::CaGenerate.generate_ca_pair(Fluent::CaGenerate::DEFAULT_OPTIONS)\n    assert_equal(OpenSSL::X509::Certificate, cert.class)\n    assert_true(key.private?)\n  end\n\n  def test_ca_generate\n    dumped_output = capture_stdout do\n      Dir.mktmpdir do |dir|\n        Fluent::CaGenerate.new([dir, \"fluentd\"]).call\n        assert_true(File.exist?(File.join(dir, \"ca_key.pem\")))\n        assert_true(File.exist?(File.join(dir, \"ca_cert.pem\")))\n      end\n    end\n    expected = <<TEXT\nsuccessfully generated: ca_key.pem, ca_cert.pem\ncopy and use ca_cert.pem to client(out_forward)\nTEXT\n    assert_equal(expected, dumped_output)\n  end\n\n  sub_test_case \"configure options\" do\n    test \"should respond multiple options\" do\n      dumped_output = capture_stdout do\n        Dir.mktmpdir do |dir|\n          Fluent::CaGenerate.new([dir, \"fluentd\",\n                                  \"--country\", \"JP\", \"--key-length\", \"4096\",\n                                  \"--state\", \"Tokyo\", \"--locality\", \"Chiyoda-ku\",\n                                  \"--common-name\", \"Forward CA\"]).call\n          assert_true(File.exist?(File.join(dir, \"ca_key.pem\")))\n          assert_true(File.exist?(File.join(dir, \"ca_cert.pem\")))\n        end\n      end\n      expected = <<TEXT\nsuccessfully generated: ca_key.pem, ca_cert.pem\ncopy and use ca_cert.pem to client(out_forward)\nTEXT\n      assert_equal(expected, dumped_output)\n    end\n\n    test \"invalid options\" do\n      Dir.mktmpdir do |dir|\n        assert_raise(OptionParser::InvalidOption) do\n          Fluent::CaGenerate.new([dir, \"fluentd\",\n                                  \"--invalid\"]).call\n        end\n        assert_false(File.exist?(File.join(dir, \"ca_key.pem\")))\n        assert_false(File.exist?(File.join(dir, \"ca_cert.pem\")))\n      end\n    end\n\n    test \"empty options\" do\n      assert_raise(SystemExit) do\n        capture_stdout do\n          Fluent::CaGenerate.new([]).call\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/command/test_cap_ctl.rb",
    "content": "require_relative '../helper'\n\nrequire 'tempfile'\nrequire 'fluent/command/cap_ctl'\n\nclass TestFluentCapCtl < Test::Unit::TestCase\n  setup do\n    omit \"This environment does not handle Linux capability\" unless defined?(CapNG)\n  end\n\n  sub_test_case \"success\" do\n    test \"clear capability\" do\n      logs = capture_stdout do\n        Fluent::CapCtl.new([\"--clear\"]).call\n      end\n      expression = /\\AClear capabilities .*\\n/m\n      assert_match expression, logs\n    end\n\n    test \"add capability\" do\n      logs = capture_stdout do\n        Fluent::CapCtl.new([\"--add\", \"dac_override\"]).call\n      end\n      expression = /\\AUpdating .* done.\\nAdding .*\\n/m\n      assert_match expression, logs\n    end\n\n    test \"drop capability\" do\n      logs = capture_stdout do\n        Fluent::CapCtl.new([\"--drop\", \"chown\"]).call\n      end\n      expression = /\\AUpdating .* done.\\nDropping .*\\n/m\n      assert_match expression, logs\n    end\n\n    test \"get capability\" do\n      logs = capture_stdout do\n        Fluent::CapCtl.new([\"--get\"]).call\n      end\n      expression = /\\ACapabilities in .*,\\nEffective:   .*\\nInheritable: .*\\nPermitted:   .*/m\n      assert_match expression, logs\n    end\n  end\n\n  sub_test_case \"success with file\" do\n    test \"clear capability\" do\n      logs = capture_stdout do\n        Tempfile.create(\"fluent-cap-\") do |tempfile|\n          Fluent::CapCtl.new([\"--clear-cap\", \"-f\", tempfile.path]).call\n        end\n      end\n      expression = /\\AClear capabilities .*\\n/m\n      assert_match expression, logs\n    end\n\n    test \"add capability\" do\n      logs = capture_stdout do\n        Tempfile.create(\"fluent-cap-\") do |tempfile|\n          Fluent::CapCtl.new([\"--add\", \"dac_override\", \"-f\", tempfile.path]).call\n        end\n      end\n      expression = /\\AUpdating .* done.\\nAdding .*\\n/m\n      assert_match expression, logs\n    end\n\n    test \"drop capability\" do\n      logs = capture_stdout do\n        Tempfile.create(\"fluent-cap-\") do |tempfile|\n          Fluent::CapCtl.new([\"--drop\", \"chown\", \"-f\", tempfile.path]).call\n        end\n      end\n      expression = /\\AUpdating .* done.\\nDropping .*\\n/m\n      assert_match expression, logs\n    end\n\n    test \"get capability\" do\n      logs = capture_stdout do\n        Tempfile.create(\"fluent-cap-\") do |tempfile|\n          Fluent::CapCtl.new([\"--get\", \"-f\", tempfile.path]).call\n        end\n      end\n      expression = /\\ACapabilities in .*,\\nEffective:   .*\\nInheritable: .*\\nPermitted:   .*/m\n      assert_match expression, logs\n    end\n  end\n\n  sub_test_case \"invalid\" do\n    test \"add capability\" do\n      assert_raise(ArgumentError) do\n        Fluent::CapCtl.new([\"--add\", \"nonexitent\"]).call\n      end\n    end\n\n    test \"drop capability\" do\n      assert_raise(ArgumentError) do\n        Fluent::CapCtl.new([\"--drop\", \"invalid\"]).call\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/command/test_cat.rb",
    "content": "require_relative '../helper'\n\nrequire 'test-unit'\nrequire 'open3'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/in_forward'\nrequire 'fluent/plugin/out_secondary_file'\nrequire 'fluent/test/driver/output'\nrequire 'fluent/test/driver/input'\n\nclass TestFluentCat < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    FileUtils.mkdir_p(TMP_DIR)\n    @record = { 'key' => 'value' }\n    @time = event_time\n    @es = Fluent::OneEventStream.new(@time, @record)\n    @primary = create_primary\n    metadata = @primary.buffer.new_metadata\n    @chunk = create_chunk(@primary, metadata, @es)\n    @port = unused_port(protocol: :all)\n  end\n\n  def teardown\n    FileUtils.rm_rf(TMP_DIR)\n    @port = nil\n  end\n\n  TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/../tmp/command/fluent_cat#{ENV['TEST_ENV_NUMBER']}\")\n  FLUENT_CAT_COMMAND = File.expand_path(File.dirname(__FILE__) + \"/../../bin/fluent-cat\")\n\n  def config\n    %[\n      port #{@port}\n      bind 127.0.0.1\n    ]\n  end\n\n  SECONDARY_CONFIG = %[\n    directory #{TMP_DIR}\n  ]\n\n  class DummyOutput < Fluent::Plugin::Output\n    def write(chunk); end\n  end\n\n  def create_driver(conf=config)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput).configure(conf)\n  end\n\n  def create_primary(buffer_config = config_element('buffer'))\n    DummyOutput.new.configure(config_element('ROOT','',{}, [buffer_config]))\n  end\n\n  def create_secondary_driver(conf=SECONDARY_CONFIG)\n    c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)\n    c.instance.acts_as_secondary(@primary)\n    c.configure(conf)\n  end\n\n  def create_chunk(primary, metadata, es)\n    primary.buffer.generate_chunk(metadata).tap do |c|\n      c.concat(es.to_msgpack_stream, es.size)\n      c.commit\n    end\n  end\n\n  sub_test_case \"json\" do\n    def test_cat_json\n      d = create_driver\n      d.run(expect_records: 1) do\n        Open3.pipeline_w(\"#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} json\") do |stdin|\n          stdin.puts('{\"key\":\"value\"}')\n          stdin.close\n        end\n      end\n      event = d.events.first\n      assert_equal([1, \"json\", @record],\n                   [d.events.size, event.first, event.last])\n    end\n  end\n\n  sub_test_case \"msgpack\" do\n    def test_cat_secondary_file\n      d = create_secondary_driver\n      path = d.instance.write(@chunk)\n      d = create_driver\n      d.run(expect_records: 1) do\n        Open3.pipeline_w(\"#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary\") do |stdin|\n          stdin.write(File.read(path, File.size(path)))\n          stdin.close\n        end\n      end\n      event = d.events.first\n      assert_equal([1, \"secondary\", @record],\n                   [d.events.size, event.first, event.last])\n    end\n  end\n\n  sub_test_case \"send specific event time\" do\n    def test_without_event_time\n      event_time = Fluent::EventTime.now\n      d = create_driver\n      d.run(expect_records: 1) do\n        Open3.pipeline_w(\"#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} tag\") do |stdin|\n          stdin.puts('{\"key\":\"value\"}')\n          stdin.close\n        end\n      end\n      event = d.events.first\n      assert_in_delta(event_time.to_f, event[1].to_f, 3.0) # expect command to be finished in 3 seconds\n      assert_equal([1, \"tag\", true, @record],\n                   [d.events.size, event.first, event_time.to_f < event[1].to_f, event.last])\n    end\n\n    def test_with_event_time\n      event_time = \"2021-01-02 13:14:15.0+00:00\"\n      d = create_driver\n      d.run(expect_records: 1) do\n        Open3.pipeline_w(\"#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --event-time '#{event_time}' tag\") do |stdin|\n          stdin.puts('{\"key\":\"value\"}')\n          stdin.close\n        end\n      end\n      assert_equal([[\"tag\", Fluent::EventTime.parse(event_time), @record]], d.events)\n    end\n  end\nend\n"
  },
  {
    "path": "test/command/test_ctl.rb",
    "content": "require_relative '../helper'\n\nrequire 'test-unit'\nrequire 'win32/event' if Fluent.windows?\n\nrequire 'fluent/command/ctl'\n\nclass TestFluentdCtl < ::Test::Unit::TestCase\n  def assert_win32_event(event_name, command, pid_or_svcname)\n    event = Win32::Event.new(event_name)\n    ipc = Win32::Ipc.new(event.handle)\n    ret = Win32::Ipc::TIMEOUT\n\n    wait_thread = Thread.new do\n      ret = ipc.wait(1)\n    end\n    Fluent::Ctl.new([command, pid_or_svcname]).call\n    wait_thread.join\n    assert_equal(Win32::Ipc::SIGNALED, ret)\n  end\n\n  data(\"shutdown\" => [\"shutdown\", \"TERM\", \"\"],\n       \"restart\" => [\"restart\", \"HUP\", \"HUP\"],\n       \"flush\" => [\"flush\", \"USR1\", \"USR1\"],\n       \"reload\" => [\"reload\", \"USR2\", \"USR2\"])\n  def test_commands(data)\n    command, signal, event_suffix = data\n\n    if Fluent.windows?\n      event_name = \"fluentd_54321\"\n      event_name << \"_#{event_suffix}\" unless event_suffix.empty?\n      assert_win32_event(event_name, command, \"54321\")\n    else\n      got_signal = false\n      Signal.trap(signal) do\n        got_signal = true\n      end\n      Fluent::Ctl.new([command, Process.pid.to_s]).call\n      assert_true(got_signal)\n    end\n  end\n\n  data(\"shutdown\" => [\"shutdown\", \"\"],\n       \"restart\" => [\"restart\", \"HUP\"],\n       \"flush\" => [\"flush\", \"USR1\"],\n       \"reload\" => [\"reload\", \"USR2\"])\n  def test_commands_with_winsvcname(data)\n    omit \"Only for Windows\" unless Fluent.windows?\n\n    command, event_suffix = data\n    event_name = \"testfluentdwinsvc\"\n    event_name << \"_#{event_suffix}\" unless event_suffix.empty?\n\n    assert_win32_event(event_name, command, \"testfluentdwinsvc\")\n  end\nend\n"
  },
  {
    "path": "test/command/test_fluentd.rb",
    "content": "require_relative '../helper'\n\n# require 'fluent/command/fluentd'\n# don't require it... it runs immediately\n\nrequire 'fileutils'\nrequire 'timeout'\nrequire 'securerandom'\nrequire 'fluent/file_wrapper'\n\nclass TestFluentdCommand < ::Test::Unit::TestCase\n  SUPERVISOR_PID_PATTERN = /starting fluentd-[.0-9]+ pid=(\\d+)/\n  WORKER_PID_PATTERN = /starting fluentd worker pid=(\\d+) /\n\n  def tmp_dir\n    File.join(File.dirname(__FILE__), \"..\", \"tmp\", \"command\" \"fluentd#{ENV['TEST_ENV_NUMBER']}\", SecureRandom.hex(10))\n  end\n\n  setup do\n    @tmp_dir = tmp_dir\n    FileUtils.mkdir_p(@tmp_dir)\n    @supervisor_pid = nil\n    @worker_pids = []\n    ENV[\"TEST_RUBY_PATH\"] = nil\n  end\n\n  teardown do\n    begin\n      FileUtils.rm_rf(@tmp_dir)\n    rescue Errno::EACCES\n      # It may occur on Windows because of delete pending state due to delayed GC.\n      # Ruby 3.2 or later doesn't ignore Errno::EACCES:\n      # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81\n    end\n  end\n\n  def process_exist?(pid)\n    begin\n      r = Process.waitpid(pid, Process::WNOHANG)\n      return true if r.nil?\n      false\n    rescue SystemCallError\n      false\n    end\n  end\n\n  def create_conf_file(name, content, ext_enc = 'utf-8')\n    conf_path = File.join(@tmp_dir, name)\n    Fluent::FileWrapper.open(conf_path, \"w:#{ext_enc}:utf-8\") do |file|\n      file.write content\n    end\n    conf_path\n  end\n\n  def create_plugin_file(name, content)\n    file_path = File.join(@tmp_dir, 'plugin', name)\n    FileUtils.mkdir_p(File.dirname(file_path))\n    Fluent::FileWrapper.open(file_path, 'w') do |file|\n      file.write content\n    end\n    file_path\n  end\n\n  def create_cmdline(conf_path, *fluentd_options)\n    if Fluent.windows?\n      cmd_path = File.expand_path(File.dirname(__FILE__) + \"../../../bin/fluentd\")\n      [\"bundle\", \"exec\", ServerEngine.ruby_bin_path, cmd_path, \"-c\", conf_path, *fluentd_options]\n    else\n      cmd_path = File.expand_path(File.dirname(__FILE__) + \"../../../bin/fluentd\")\n      [\"bundle\", \"exec\", cmd_path, \"-c\", conf_path, *fluentd_options]\n    end\n  end\n\n  def process_kill(pid)\n    if Fluent.windows?\n      Process.kill(:KILL, pid) rescue nil\n      return\n    end\n\n    begin\n      Process.kill(:TERM, pid) rescue nil\n      Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }\n    rescue Timeout::Error\n      Process.kill(:KILL, pid) rescue nil\n    end\n  end\n\n  def execute_command(cmdline, chdir=@tmp_dir, env = {})\n    null_stream = Fluent::FileWrapper.open(File::NULL, 'w')\n    gemfile_path = File.expand_path(File.dirname(__FILE__) + \"../../../Gemfile\")\n\n    env = { \"BUNDLE_GEMFILE\" => gemfile_path }.merge(env)\n    cmdname = cmdline.shift\n    arg0 = \"testing-fluentd\"\n    # p(here: \"executing process\", env: env, cmdname: cmdname, arg0: arg0, args: cmdline)\n    IO.popen(env, [[cmdname, arg0], *cmdline], chdir: chdir, err: [:child, :out]) do |io|\n      pid = io.pid\n      begin\n        yield pid, io\n        # p(here: \"execute command\", pid: pid, worker_pids: @worker_pids)\n      ensure\n        process_kill(pid)\n        if @supervisor_pid\n          process_kill(@supervisor_pid)\n        end\n        @worker_pids.each do |cpid|\n          process_kill(cpid)\n        end\n        # p(here: \"execute command\", pid: pid, exist: process_exist?(pid), worker_pids: @worker_pids, exists: @worker_pids.map{|i| process_exist?(i) })\n        Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }\n      end\n    end\n  ensure\n    null_stream.close rescue nil\n  end\n\n  def eager_read(io)\n    buf = +''\n\n    loop do\n      b = io.read_nonblock(1024, nil, exception: false)\n      if b == :wait_readable || b.nil?\n        return buf\n      end\n      buf << b\n    end\n  end\n\n  # ATTENTION: This stops taking logs when all `pattern_list` match or timeout,\n  # so `patterns_not_match` can test only logs up to that point.\n  # You can pass a block to assert something after log matching.\n  def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})\n    matched = false\n    matched_wrongly = false\n    error_msg_match = \"\"\n    stdio_buf = \"\"\n    succeeded_block = true\n    error_msg_block = \"\"\n    begin\n      execute_command(cmdline, @tmp_dir, env) do |pid, stdout|\n        begin\n          waiting(timeout) do\n            while process_exist?(pid)\n              readables, _, _ = IO.select([stdout], nil, nil, 1)\n              next unless readables\n              break if readables.first.eof?\n\n              buf = eager_read(readables.first)\n              # puts buf\n              stdio_buf << buf\n              lines = stdio_buf.split(\"\\n\")\n              if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }\n                matched = true\n              end\n\n              if Fluent.windows?\n                # https://github.com/fluent/fluentd/issues/4095\n                # On Windows, the initial process is different from the supervisor process,\n                # so we need to wait until `SUPERVISOR_PID_PATTERN` appears in the logs to get the pid.\n                # (Worker processes will be killed by the supervisor process, so we don't need it-)\n                break if matched && SUPERVISOR_PID_PATTERN =~ stdio_buf\n              else\n                # On Non-Windows, the initial process is the supervisor process,\n                # so we don't need to wait `SUPERVISOR_PID_PATTERN`.\n                break if matched\n              end\n            end\n          end\n\n          begin\n            yield if block_given?\n          rescue => e\n            succeeded_block = false\n            error_msg_block = \"failed block execution after matching: #{e}\"\n          end\n        ensure\n          if SUPERVISOR_PID_PATTERN =~ stdio_buf\n            @supervisor_pid = $1.to_i\n          end\n          stdio_buf.scan(WORKER_PID_PATTERN) do |worker_pid|\n            @worker_pids << worker_pid.first.to_i\n          end\n        end\n      end\n    rescue Timeout::Error\n      error_msg_match = \"execution timeout\"\n      # https://github.com/fluent/fluentd/issues/4095\n      # On Windows, timeout without `@supervisor_pid` means that the test is invalid,\n      # since the supervisor process will survive without being killed correctly.\n      flunk(\"Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.\") if Fluent.windows? && @supervisor_pid.nil?\n    rescue => e\n      error_msg_match = \"unexpected error in launching fluentd: #{e.inspect}\"\n    else\n      error_msg_match = \"log doesn't match\" unless matched\n    end\n\n    if patterns_not_match.empty?\n      error_msg_match = build_message(error_msg_match,\n                                       \"<?>\\nwas expected to include:\\n<?>\",\n                                       stdio_buf, pattern_list)\n    else\n      lines = stdio_buf.split(\"\\n\")\n      patterns_not_match.each do |ptn|\n        matched_wrongly = if ptn.is_a? Regexp\n                            lines.any?{|line| ptn.match(line) }\n                          else\n                            lines.any?{|line| line.include?(ptn) }\n                          end\n        if matched_wrongly\n          error_msg_match << \"\\n\" unless error_msg_match.empty?\n          error_msg_match << \"pattern exists in logs wrongly: #{ptn}\"\n        end\n      end\n      error_msg_match = build_message(error_msg_match,\n                                       \"<?>\\nwas expected to include:\\n<?>\\nand not include:\\n<?>\",\n                                       stdio_buf, pattern_list, patterns_not_match)\n    end\n\n    assert matched && !matched_wrongly, error_msg_match\n    assert succeeded_block, error_msg_block if block_given?\n  end\n\n  def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)\n    # empty_list.all?{ ... } is always true\n    matched = false\n    running = false\n    assert_error_msg = \"failed to start correctly\"\n    stdio_buf = \"\"\n    begin\n      execute_command(cmdline) do |pid, stdout|\n        begin\n          waiting(timeout) do\n            while process_exist?(pid) && !running\n              readables, _, _ = IO.select([stdout], nil, nil, 1)\n              next unless readables\n              next if readables.first.eof?\n\n              stdio_buf << eager_read(readables.first)\n              lines = stdio_buf.split(\"\\n\")\n              if lines.any?{|line| line.include?(\"fluentd worker is now running\") }\n                running = true\n              end\n              if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }\n                matched = true\n              end\n            end\n          end\n        ensure\n          if SUPERVISOR_PID_PATTERN =~ stdio_buf\n            @supervisor_pid = $1.to_i\n          end\n          stdio_buf.scan(WORKER_PID_PATTERN) do |worker_pid|\n            @worker_pids << worker_pid.first.to_i\n          end\n        end\n      end\n    rescue Timeout::Error\n      assert_error_msg = \"execution timeout with command out:\\n\" + stdio_buf\n      # https://github.com/fluent/fluentd/issues/4095\n      # On Windows, timeout without `@supervisor_pid` means that the test is invalid,\n      # since the supervisor process will survive without being killed correctly.\n      flunk(\"Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.\") if Fluent.windows? && @supervisor_pid.nil?\n    rescue => e\n      assert_error_msg = \"unexpected error in launching fluentd: #{e.inspect}\\n\" + stdio_buf\n      assert false, assert_error_msg\n    end\n    assert !running, \"fluentd started to run incorrectly:\\n\" + stdio_buf\n    unless matched\n      assert_error_msg = \"fluentd failed to start, without specified regular expressions:\\n\" + stdio_buf\n    end\n    assert matched, assert_error_msg\n  end\n\n  sub_test_case 'with valid configuration' do\n    test 'runs successfully' do\n      conf = <<CONF\n<source>\n  @type dummy\n  @id dummy\n  @label @dummydata\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('valid.conf', conf)\n      assert File.exist?(conf_path)\n\n      assert_log_matches(create_cmdline(conf_path), \"fluentd worker is now running\", 'worker=0')\n    end\n  end\n\n  sub_test_case 'with --conf-encoding' do\n    test 'runs successfully' do\n      conf = <<CONF\n# テスト\n<source>\n  @type dummy\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<match dummy>\n  @type null\n</match>\nCONF\n      conf_path = create_conf_file('shift_jis.conf', conf, 'shift_jis')\n      assert_log_matches(create_cmdline(conf_path, '--conf-encoding', 'shift_jis'), \"fluentd worker is now running\", 'worker=0')\n    end\n\n    test 'failed to run by invalid encoding' do\n      conf = <<CONF\n# テスト\n<source>\n  @type dummy\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<match dummy>\n  @type null\n</match>\nCONF\n      conf_path = create_conf_file('shift_jis.conf', conf, 'shift_jis')\n      assert_fluentd_fails_to_start(create_cmdline(conf_path), \"invalid byte sequence in UTF-8\")\n    end\n  end\n\n  sub_test_case 'with system configuration about root directory' do\n    setup do\n      @root_path = File.join(@tmp_dir, \"rootpath\")\n      FileUtils.rm_rf(@root_path)\n      @conf = <<CONF\n<system>\n  root_dir #{@root_path}\n</system>\n<source>\n  @type dummy\n  @id dummy\n  @label @dummydata\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n    end\n\n    test 'use the specified existing directory as root' do\n      FileUtils.mkdir_p(@root_path)\n      conf_path = create_conf_file('existing_root_dir.conf', @conf)\n      assert Dir.exist?(@root_path)\n\n      assert_log_matches(create_cmdline(conf_path), \"fluentd worker is now running\", 'worker=0')\n    end\n\n    test 'creates the specified root directory if missing' do\n      conf_path = create_conf_file('missing_root_dir.conf', @conf)\n      assert_false Dir.exist?(@root_path)\n\n      assert_log_matches(create_cmdline(conf_path), \"fluentd worker is now running\", 'worker=0')\n      assert Dir.exist?(@root_path)\n    end\n\n    test 'fails to launch fluentd if specified root path is invalid path for directory' do\n      Fluent::FileWrapper.open(@root_path, 'w') do |_|\n        # create file and close it\n      end\n      conf_path = create_conf_file('existing_root_dir.conf', @conf)\n\n      assert_fluentd_fails_to_start(\n        create_cmdline(conf_path),\n        \"non directory entry exists:#{@root_path}\",\n      )\n    end\n  end\n\n  sub_test_case 'configured to route log events to plugins' do\n    setup do\n      @basic_conf = <<CONF\n<source>\n  @type dummy\n  @id dummy\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<match dummy>\n  @type null\n  @id   blackhole\n</match>\nCONF\n    end\n\n    test 'by top level <match fluent.*> section' do\n      conf = @basic_conf + <<CONF\n<match fluent.**>\n  @type stdout\n</match>\nCONF\n      conf_path = create_conf_file('logevent_1.conf', conf)\n      assert_log_matches(\n        create_cmdline(conf_path),\n        \"fluentd worker is now running\",\n        'fluent.info: {\"worker\":0,\"message\":\"fluentd worker is now running worker=0\"}',\n        \"define <match fluent.**> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead\",\n        patterns_not_match: ['[warn]: some tags for log events are not defined in top level (to be ignored) tags=[\"fluent.trace\", \"fluent.debug\"]'],\n      )\n    end\n\n    test 'by top level <match> section with warning for missing log levels (and warnings for each log event records)' do\n      conf = @basic_conf + <<CONF\n<match fluent.warn fluent.error fluent.fatal>\n  @type stdout\n</match>\nCONF\n      conf_path = create_conf_file('logevent_2.conf', conf)\n      assert_log_matches(\n        create_cmdline(conf_path),\n        \"fluentd worker is now running\",\n        '[warn]: #0 match for some tags of log events are not defined in top level (to be ignored) tags=[\"fluent.trace\", \"fluent.debug\", \"fluent.info\"]',\n        \"define <match fluent.warn>, <match fluent.error>, <match fluent.fatal> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead\",\n        '[warn]: #0 no patterns matched tag=\"fluent.info\"',\n      )\n    end\n\n    test 'by <label @FLUENT_LOG> section' do\n      conf = @basic_conf + <<CONF\n<label @FLUENT_LOG>\n  <match **>\n    @type stdout\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('logevent_3.conf', conf)\n      assert_log_matches(\n        create_cmdline(conf_path),\n        \"fluentd worker is now running\",\n        'fluent.info: {\"worker\":0,\"message\":\"fluentd worker is now running worker=0\"}',\n        patterns_not_match: ['[warn]: some tags for log events are not defined in @FLUENT_LOG label (to be ignored)'],\n      )\n    end\n\n    test 'by <label> section with warning for missing log levels' do\n      conf = @basic_conf + <<CONF\n<label @FLUENT_LOG>\n  <match fluent.{trace,debug}>\n    @type null\n  </match>\n  <match fluent.warn fluent.error>\n    @type stdout\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('logevent_4.conf', conf)\n      assert_log_matches(\n        create_cmdline(conf_path),\n        \"fluentd worker is now running\",\n        '[warn]: #0 match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored) tags=[\"fluent.info\", \"fluent.fatal\"]',\n        patterns_not_match: ['[warn]: no patterns matched tag=\"fluent.info\"'],\n      )\n    end\n  end\n\n  sub_test_case 'configured to suppress configuration dump' do\n    setup do\n      @basic_conf = <<CONF\n<source>\n  @type dummy\n  @id dummy\n  @label @dummydata\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n    end\n\n    test 'configured by system config' do\n      conf = <<SYSTEM + @basic_conf\n<system>\n  suppress_config_dump\n</system>\nSYSTEM\n      conf_path = create_conf_file('suppress_conf_dump_1.conf', conf)\n      assert_log_matches(create_cmdline(conf_path), \"fluentd worker is now running\", patterns_not_match: [\"tag dummy\"])\n    end\n\n    test 'configured by command line option' do\n      conf_path = create_conf_file('suppress_conf_dump_2.conf', @basic_conf)\n      assert_log_matches(create_cmdline(conf_path, '--suppress-config-dump'), \"fluentd worker is now running\", patterns_not_match: [\"tag dummy\"])\n    end\n\n    test 'configured as false by system config, but overridden as true by command line option' do\n      conf = <<SYSTEM + @basic_conf\n<system>\n  suppress_config_dump false\n</system>\nSYSTEM\n      conf_path = create_conf_file('suppress_conf_dump_3.conf', conf)\n      assert_log_matches(create_cmdline(conf_path, '--suppress-config-dump'), \"fluentd worker is now running\", patterns_not_match: [\"tag dummy\"])\n    end\n  end\n\n  sub_test_case 'configuration with wrong plugin type' do\n    test 'failed to start' do\n      conf = <<CONF\n<source>\n  @type\n  @id dummy\n  @label @dummydata\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('type_missing.conf', conf)\n      assert File.exist?(conf_path)\n\n      assert_fluentd_fails_to_start(\n        create_cmdline(conf_path),\n        \"config error\",\n        \"error=\\\"Unknown input plugin ''. Run 'gem search -rd fluent-plugin' to find plugins\",\n      )\n    end\n  end\n\n  sub_test_case 'configuration to load plugin file with syntax error' do\n    test 'failed to start' do\n      script =  \"require 'fluent/plugin/input'\\n\"\n      script << \"module Fluent::Plugin\\n\"\n      script << \"  class BuggyInput < Input\\n\"\n      script << \"    Fluent::Plugin.register_input('buggy', self)\\n\"\n      script << \"  end\\n\"\n      plugin_path = create_plugin_file('in_buggy.rb', script)\n\n      conf = <<CONF\n<source>\n  @type buggy\n  @id dummy\n  @label @dummydata\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('buggy_plugin.conf', conf)\n      assert File.exist?(conf_path)\n\n      assert_fluentd_fails_to_start(\n        create_cmdline(conf_path, \"-p\", File.dirname(plugin_path)),\n        /\\[error\\]: .+in_buggy.rb:\\d+: syntax error/\n      )\n    end\n  end\n\n  sub_test_case 'configuration to load plugin which raises unrecoverable error in #start' do\n    test 'failed to start' do\n      script =  \"require 'fluent/plugin/input'\\n\"\n      script << \"require 'fluent/error'\\n\"\n      script << \"module Fluent::Plugin\\n\"\n      script << \"  class CrashingInput < Input\\n\"\n      script << \"    Fluent::Plugin.register_input('crashing', self)\\n\"\n      script << \"    def start\\n\"\n      script << \"      raise Fluent::UnrecoverableError\"\n      script << \"    end\\n\"\n      script << \"  end\\n\"\n      script << \"end\\n\"\n      plugin_path = create_plugin_file('in_crashing.rb', script)\n\n      conf = <<CONF\n<source>\n  @type crashing\n  @id dummy\n  @label @dummydata\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('crashing_plugin.conf', conf)\n      assert File.exist?(conf_path)\n\n      assert_fluentd_fails_to_start(\n        create_cmdline(conf_path, \"-p\", File.dirname(plugin_path)),\n        'unexpected error error_class=Fluent::UnrecoverableError error=\"an unrecoverable error occurs in Fluentd process\"',\n      )\n    end\n  end\n\n  sub_test_case 'configured to run 2 workers' do\n    setup do\n      @root_path = File.join(@tmp_dir, \"rootpath\")\n      FileUtils.rm_rf(@root_path)\n      FileUtils.mkdir_p(@root_path)\n    end\n\n    test 'success to start the number of workers specified in configuration' do\n      conf = <<'CONF'\n<system>\n  workers 2\n  root_dir #{@root_path}\n</system>\n<source>\n  @type dummy\n  @id \"dummy#{worker_id}\" # check worker_id works or not with actual command\n  @label @dummydata\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('workers1.conf', conf)\n      assert Dir.exist?(@root_path)\n\n      assert_log_matches(\n        create_cmdline(conf_path),\n        \"#0 fluentd worker is now running worker=0\",\n        \"#1 fluentd worker is now running worker=1\"\n      )\n    end\n\n    sub_test_case \"YAML config format\" do\n      test 'success to start the number of workers specified in configuration' do\n        conf = <<'CONF'\n        system:\n          workers: 2\n          root_dir: \"#{@root_path}\"\n        config:\n          - source:\n              $type: dummy\n              $id: !fluent/s \"dummy.#{worker_id}\" # check worker_id works or not with actual command\n              $label: '@dummydata'\n              tag: dummy\n              dummy: !fluent/json {\"message\": !fluent/s \"yay from #{hostname}!\"}\n\n          - label:\n              $name: '@dummydata'\n              config:\n               - match:\n                  $tag: dummy\n                  $type: \"null\"\n                  $id: blackhole\nCONF\n        conf_path = create_conf_file('workers1.yaml', conf)\n        assert Dir.exist?(@root_path)\n\n        assert_log_matches(\n          create_cmdline(conf_path),\n          \"#0 fluentd worker is now running worker=0\",\n          \"#1 fluentd worker is now running worker=1\"\n        )\n      end\n    end\n\n    test 'success to start the number of workers specified by command line option' do\n      conf = <<CONF\n<system>\n  root_dir #{@root_path}\n</system>\n<source>\n  @type dummy\n  @id dummy\n  @label @dummydata\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('workers2.conf', conf)\n      assert_log_matches(\n        create_cmdline(conf_path, '--workers', '2'),\n        \"#0 fluentd worker is now running worker=0\",\n        \"#1 fluentd worker is now running worker=1\"\n      )\n    end\n\n    test 'failed to start workers when configured plugins do not support multi worker configuration' do\n      script =  \"require 'fluent/plugin/input'\\n\"\n      script << \"module Fluent::Plugin\\n\"\n      script << \"  class SingleInput < Input\\n\"\n      script << \"    Fluent::Plugin.register_input('single', self)\\n\"\n      script << \"    def multi_workers_ready?\\n\"\n      script << \"      false\\n\"\n      script << \"    end\\n\"\n      script << \"  end\\n\"\n      script << \"end\\n\"\n      plugin_path = create_plugin_file('in_single.rb', script)\n\n      conf = <<CONF\n<system>\n  workers 2\n</system>\n<source>\n  @type single\n  @id single\n  @label @dummydata\n</source>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('workers_invalid1.conf', conf)\n      assert_fluentd_fails_to_start(\n        create_cmdline(conf_path, \"-p\", File.dirname(plugin_path)),\n        \"Plugin 'single' does not support multi workers configuration (Fluent::Plugin::SingleInput)\",\n      )\n    end\n\n    test 'failed to start workers when file buffer is configured in non-workers way' do\n      conf = <<CONF\n<system>\n  workers 2\n</system>\n<source>\n  @type dummy\n  tag dummy\n  @id single\n  @label @dummydata\n</source>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n    <buffer>\n      @type file\n      path #{File.join(@root_path, \"buf\", \"file.*.log\")}\n    </buffer>\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('workers_invalid2.conf', conf)\n      assert_fluentd_fails_to_start(\n        create_cmdline(conf_path),\n        \"[blackhole] file buffer with multi workers should be configured to use directory 'path', or system root_dir and plugin id\",\n        \"config error file=\\\"#{conf_path}\\\" error_class=Fluent::ConfigError error=\\\"Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)\\\"\",\n      )\n    end\n\n    test 'failed to start workers when configured plugins as children of MultiOutput do not support multi worker configuration' do\n      script = <<-EOC\nrequire 'fluent/plugin/output'\nmodule Fluent::Plugin\n  class SingleOutput < Output\n    Fluent::Plugin.register_output('single', self)\n    def multi_workers_ready?\n      false\n    end\n    def write(chunk)\n    end\n  end\nend\nEOC\n      plugin_path = create_plugin_file('out_single.rb', script)\n\n      conf = <<CONF\n<system>\n  workers 2\n</system>\n<source>\n  @type dummy\n  tag dummy\n  @id single\n  @label @dummydata\n</source>\n<label @dummydata>\n  <match dummy>\n    @type copy\n    <store>\n      @type single\n    </store>\n    <store>\n      @type single\n    </store>\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('workers_invalid3.conf', conf)\n      assert_fluentd_fails_to_start(\n        create_cmdline(conf_path, \"-p\", File.dirname(plugin_path)),\n        \"Plugin 'single' does not support multi workers configuration (Fluent::Plugin::SingleOutput)\",\n      )\n    end\n\n    test 'success to start a worker2 with worker specific configuration' do\n      conf = <<CONF\n<system>\n  root_dir #{@root_path}\n  dir_permission 0744\n</system>\nCONF\n      conf_path = create_conf_file('worker_section0.conf', conf)\n\n      FileUtils.rm_rf(@root_path) rescue nil\n\n      assert_path_not_exist(@root_path)\n      assert_log_matches(create_cmdline(conf_path), 'spawn command to main') # any message is ok\n      assert_path_exist(@root_path)\n      if Fluent.windows?\n        # In Windows, dir permission is always 755.\n        assert_equal '755', File.stat(@root_path).mode.to_s(8)[-3, 3]\n      else\n        assert_equal '744', File.stat(@root_path).mode.to_s(8)[-3, 3]\n      end\n    end\n\n    test 'success to start a worker with worker specific configuration' do\n      conf = <<CONF\n<system>\n  workers 2\n  root_dir #{@root_path}\n</system>\n<source>\n  @type dummy\n  @id dummy\n  @label @dummydata\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<worker 1>\n  <source>\n    @type dummy\n    @id dummy_in_worker\n    @label @dummydata\n    tag dummy\n    dummy {\"message\": \"yay!\"}\n  </source>\n</worker>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('worker_section0.conf', conf)\n      assert Dir.exist?(@root_path)\n\n      assert_log_matches(\n        create_cmdline(conf_path),\n        \"#0 fluentd worker is now running worker=0\",\n        \"#1 fluentd worker is now running worker=1\",\n        /(?!#\\d) adding source type=\"dummy\"/,\n        '#1 adding source type=\"dummy\"'\n      )\n    end\n\n    test 'success to start workers when configured plugins only for specific worker do not support multi worker configuration' do\n      script =  <<-EOC\nrequire 'fluent/plugin/input'\nmodule Fluent::Plugin\n  class SingleInput < Input\n    Fluent::Plugin.register_input('single', self)\n    def multi_workers_ready?\n      false\n    end\n  end\nend\nEOC\n      plugin_path = create_plugin_file('in_single.rb', script)\n\n      conf = <<CONF\n<system>\n  workers 2\n</system>\n<worker 1>\n  <source>\n    @type single\n    @id single\n    @label @dummydata\n  </source>\n</worker>\n<label @dummydata>\n  <match dummy>\n    @type null\n    @id   blackhole\n  </match>\n</label>\nCONF\n      conf_path = create_conf_file('worker_section1.conf', conf)\n      assert Dir.exist?(@root_path)\n\n      assert_log_matches(\n        create_cmdline(conf_path, \"-p\", File.dirname(plugin_path)),\n        \"#0 fluentd worker is now running worker=0\",\n        \"#1 fluentd worker is now running worker=1\",\n        '#1 adding source type=\"single\"'\n      )\n    end\n\n    test \"multiple values are set to RUBYOPT\" do\n      conf = <<CONF\n<source>\n  @type dummy\n  tag dummy\n</source>\n<match>\n  @type null\n</match>\nCONF\n      conf_path = create_conf_file('rubyopt_test.conf', conf)\n      assert_log_matches(\n        create_cmdline(conf_path),\n        '#0 fluentd worker is now running worker=0',\n        patterns_not_match: ['(LoadError)'],\n        env: { 'RUBYOPT' => '-rtest-unit -rbundler/setup' },\n      )\n    end\n\n    data(\n      '-E' => '-Eutf-8',\n      '-encoding' => '--encoding=utf-8',\n      '-external-encoding' => '--external-encoding=utf-8',\n      '-internal-encoding' => '--internal-encoding=utf-8',\n    )\n    test \"-E option is set to RUBYOPT\" do |base_opt|\n      conf = <<CONF\n<source>\n  @type dummy\n  tag dummy\n</source>\n<match>\n  @type null\n</match>\nCONF\n      conf_path = create_conf_file('rubyopt_test.conf', conf)\n      opt = base_opt.dup\n      opt << \" #{ENV['RUBYOPT']}\" if ENV['RUBYOPT']\n      assert_log_matches(\n        create_cmdline(conf_path),\n        *opt.split(' '),\n        patterns_not_match: ['-Eascii-8bit:ascii-8bit'],\n        env: { 'RUBYOPT' => opt },\n      )\n    end\n\n    test \"without RUBYOPT\" do\n      saved_ruby_opt = ENV[\"RUBYOPT\"]\n      ENV[\"RUBYOPT\"] = nil\n      conf = <<CONF\n<source>\n  @type dummy\n  tag dummy\n</source>\n<match>\n  @type null\n</match>\nCONF\n      conf_path = create_conf_file('rubyopt_test.conf', conf)\n      assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit')\n    ensure\n      ENV[\"RUBYOPT\"] = saved_ruby_opt\n    end\n\n    test 'invalid values are set to RUBYOPT' do\n      omit \"hard to run correctly because RUBYOPT=-r/path/to/bundler/setup is required on Windows while this test set invalid RUBYOPT\" if Fluent.windows?\n      conf = <<CONF\n<source>\n  @type dummy\n  tag dummy\n</source>\n<match>\n  @type null\n</match>\nCONF\n      conf_path = create_conf_file('rubyopt_invalid_test.conf', conf)\n      if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')\n        expected_phrase = 'ruby: invalid switch in RUBYOPT'\n      else\n        expected_phrase = 'Invalid option is passed to RUBYOPT'\n      end\n      assert_log_matches(\n        create_cmdline(conf_path),\n        expected_phrase,\n        env: { 'RUBYOPT' => 'a' },\n      )\n    end\n\n    # https://github.com/fluent/fluentd/issues/2915\n    test \"ruby path contains spaces\" do\n      saved_ruby_opt = ENV[\"RUBYOPT\"]\n      ENV[\"RUBYOPT\"] = nil\n      conf = <<CONF\n<source>\n  @type dummy\n  tag dummy\n</source>\n<match>\n  @type null\n</match>\nCONF\n      ruby_path = ServerEngine.ruby_bin_path\n      tmp_ruby_path = File.join(@tmp_dir, \"ruby with spaces\")\n      if Fluent.windows?\n        tmp_ruby_path << \".bat\"\n        Fluent::FileWrapper.open(tmp_ruby_path, \"w\") do |file|\n          file.write \"#{ruby_path} %*\"\n        end\n      else\n        FileUtils.ln_sf(ruby_path, tmp_ruby_path)\n      end\n      ENV[\"TEST_RUBY_PATH\"] = tmp_ruby_path\n      cmd_path = File.expand_path(File.dirname(__FILE__) + \"../../../bin/fluentd\")\n      conf_path = create_conf_file('space_mixed_ruby_path_test.conf', conf)\n      args = [\"bundle\", \"exec\", tmp_ruby_path, cmd_path, \"-c\", conf_path]\n      assert_log_matches(\n        args,\n        'spawn command to main:',\n        '-Eascii-8bit:ascii-8bit'\n      )\n    ensure\n      ENV[\"RUBYOPT\"] = saved_ruby_opt\n    end\n\n    test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do\n      conf = <<CONF\n<system>\n  workers 2\n</system>\n<source>\n  @type dummy\n  @id dummy\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<worker 1>\n  <match dummy>\n    @type null\n    @id   blackhole\n    <buffer>\n      @type file\n      path #{File.join(@root_path, \"buf\")}\n    </buffer>\n  </match>\n</worker>\nCONF\n      conf_path = create_conf_file('worker_section2.conf', conf)\n      assert_log_matches(\n        create_cmdline(conf_path),\n        \"#0 fluentd worker is now running worker=0\",\n        \"#1 fluentd worker is now running worker=1\",\n        '#1 adding match pattern=\"dummy\" type=\"null\"'\n      )\n    end\n\n    test 'success to start workers when configured plugins as a children of MultiOutput only for specific worker do not support multi worker configuration' do\n      script = <<-EOC\nrequire 'fluent/plugin/output'\nmodule Fluent::Plugin\n  class SingleOutput < Output\n    Fluent::Plugin.register_output('single', self)\n    def multi_workers_ready?\n      false\n    end\n    def write(chunk)\n    end\n  end\nend\nEOC\n      plugin_path = create_plugin_file('out_single.rb', script)\n\n      conf = <<CONF\n<system>\n  workers 2\n</system>\n<source>\n  @type dummy\n  @id dummy\n  tag dummy\n  dummy {\"message\": \"yay!\"}\n</source>\n<worker 1>\n  <match dummy>\n    @type copy\n    <store>\n      @type single\n    </store>\n    <store>\n      @type single\n    </store>\n  </match>\n</worker>\nCONF\n      conf_path = create_conf_file('worker_section3.conf', conf)\n      assert_log_matches(\n        create_cmdline(conf_path, \"-p\", File.dirname(plugin_path)),\n        \"#0 fluentd worker is now running worker=0\",\n        \"#1 fluentd worker is now running worker=1\",\n        '#1 adding match pattern=\"dummy\" type=\"copy\"'\n      )\n    end\n  end\n\n  sub_test_case 'config dump' do\n    test 'all secret parameters in worker section is sealed' do\n      script =  <<-EOC\nrequire 'fluent/plugin/input'\nmodule Fluent::Plugin\n  class FakeInput < Input\n    Fluent::Plugin.register_input('fake', self)\n    config_param :secret, :string, secret: true\n    def multi_workers_ready?; true; end\n  end\nend\nEOC\n      plugin_path = create_plugin_file('in_fake.rb', script)\n\n      conf = <<CONF\n<system>\n  workers 2\n</system>\n<worker 0>\n  <source>\n    @type fake\n    secret secret0\n  </source>\n  <match>\n    @type null\n  </match>\n</worker>\n<worker 1>\n  <source>\n    @type fake\n    secret secret1\n  </source>\n  <match>\n    @type null\n  </match>\n</worker>\nCONF\n      conf_path = create_conf_file('secret_in_worker.conf', conf)\n      assert File.exist?(conf_path)\n\n      assert_log_matches(create_cmdline(conf_path, \"-p\", File.dirname(plugin_path)),\n                         \"secret xxxxxx\", patterns_not_match: [\"secret secret0\", \"secret secret1\"])\n    end\n  end\n\n  sub_test_case 'shared socket options' do\n    test 'enable shared socket by default' do\n      conf = \"\"\n      conf_path = create_conf_file('empty.conf', conf)\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path),\n                         patterns_not_match: [\"shared socket for multiple workers is disabled\"])\n    end\n\n    test 'disable shared socket by command line option' do\n      conf = \"\"\n      conf_path = create_conf_file('empty.conf', conf)\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path, \"--disable-shared-socket\"),\n                         \"shared socket for multiple workers is disabled\",)\n    end\n\n    test 'disable shared socket by system config' do\n      conf = <<CONF\n<system>\n  disable_shared_socket\n</system>\nCONF\n      conf_path = create_conf_file('empty.conf', conf)\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path, \"--disable-shared-socket\"),\n                         \"shared socket for multiple workers is disabled\",)\n    end\n  end\n\n  # TODO: `patterns_not_match` can test only logs up to `pattern_list`,\n  # so we need to fix some meaningless `patterns_not_match` conditions.\n  sub_test_case 'log_level by command line option' do\n    test 'info' do\n      conf = \"\"\n      conf_path = create_conf_file('empty.conf', conf)\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path),\n                         \"[info]\",\n                         patterns_not_match: [\"[debug]\"])\n    end\n\n    test 'debug' do\n      conf = \"\"\n      conf_path = create_conf_file('empty.conf', conf)\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path, \"-v\"),\n                         \"[debug]\",\n                         patterns_not_match: [\"[trace]\"])\n    end\n\n    data(\"Trace\" => \"-vv\")\n    data(\"Invalid low level should be treated as Trace level\": \"-vvv\")\n    test 'trace' do |option|\n      conf = <<CONF\n<source>\n  @type sample\n  tag test\n</source>\nCONF\n      conf_path = create_conf_file('sample.conf', conf)\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path, option),\n                         \"[trace]\",)\n    end\n\n    test 'warn' do\n      omit \"Can't run on Windows since there is no way to take pid of the supervisor.\" if Fluent.windows?\n      conf = <<CONF\n<source>\n  @type sample\n  tag test\n</source>\nCONF\n      conf_path = create_conf_file('sample.conf', conf)\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path, \"-q\"),\n                         \"[warn]\",\n                         patterns_not_match: [\"[info]\"])\n    end\n\n    data(\"Error\" => \"-qq\")\n    data(\"Fatal should be treated as Error level\" => \"-qqq\")\n    data(\"Invalid high level should be treated as Error level\": \"-qqqq\")\n    test 'error' do |option|\n      # This test can run on Windows correctly,\n      # since the process will stop automatically with an error.\n      conf = <<CONF\n<source>\n  @type plugin_not_found\n  tag test\n</source>\nCONF\n      conf_path = create_conf_file('plugin_not_found.conf', conf)\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path, option),\n                         \"[error]\",\n                         patterns_not_match: [\"[warn]\"])\n    end\n\n    test 'system config one should not be overwritten when cmd line one is not specified' do\n      conf = <<CONF\n<system>\n  log_level debug\n</system>\nCONF\n      conf_path = create_conf_file('debug.conf', conf)\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path),\n                         \"[debug]\")\n    end\n  end\n\n  sub_test_case \"inline_config\" do\n    test \"can change log_level by --inline-config\" do\n      # Since we can't define multiple `<system>` directives, this use-case is not recommended.\n      # This is just for this test.\n      inline_conf = '<system>\\nlog_level debug\\n</system>'\n      conf_path = create_conf_file('test.conf', \"\")\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path, \"--inline-config\", inline_conf),\n                         \"[debug]\")\n    end\n  end\n\n  sub_test_case \"plugin option\" do\n    test \"should be the default value when not specifying\" do\n      conf_path = create_conf_file('test.conf', <<~CONF)\n        <source>\n          @type monitor_agent\n        </source>\n      CONF\n      assert File.exist?(conf_path)\n      cmdline = create_cmdline(conf_path)\n\n      assert_log_matches(cmdline, \"fluentd worker is now running\") do\n        response = Net::HTTP.get(URI.parse(\"http://localhost:24220/api/config.json\"))\n        actual_conf = JSON.parse(response)\n        assert_equal Fluent::Supervisor.default_options[:plugin_dirs], actual_conf[\"plugin_dirs\"]\n      end\n    end\n\n    data(short: \"-p\")\n    data(long: \"--plugin\")\n    test \"can be added by specifying the option\" do |option_name|\n      conf_path = create_conf_file('test.conf', <<~CONF)\n        <source>\n          @type monitor_agent\n        </source>\n      CONF\n      assert File.exist?(conf_path)\n      cmdline = create_cmdline(conf_path, option_name, @tmp_dir, option_name, @tmp_dir)\n\n      assert_log_matches(cmdline, \"fluentd worker is now running\") do\n        response = Net::HTTP.get(URI.parse(\"http://localhost:24220/api/config.json\"))\n        actual_conf = JSON.parse(response)\n        assert_equal Fluent::Supervisor.default_options[:plugin_dirs] + [@tmp_dir, @tmp_dir], actual_conf[\"plugin_dirs\"]\n      end\n    end\n  end\n\n  test \"--umask should be recognized as command line\" do\n    conf_path = create_conf_file(\"empty.conf\", \"\")\n    assert File.exist?(conf_path)\n    assert_log_matches(create_cmdline(conf_path, \"--umask\", \"222\"),\n                       \"spawn command to main:\", '\"--umask\", \"222\"',\n                       patterns_not_match: [\"[error]\"])\n  end\n\n  sub_test_case \"--with-source-only\" do\n    setup do\n      omit \"Not supported on Windows\" if Fluent.windows?\n    end\n\n    test \"should work without error\" do\n      conf_path = create_conf_file(\"empty.conf\", \"\")\n      assert File.exist?(conf_path)\n      assert_log_matches(create_cmdline(conf_path, \"--with-source-only\"),\n                         \"with-source-only: the emitted data will be stored in the buffer files under\",\n                         \"fluentd worker is now running\",\n                         patterns_not_match: [\"[error]\"])\n    end\n  end\n\n  sub_test_case \"zero_downtime_restart\" do\n    setup do\n      omit \"Not supported on Windows\" if Fluent.windows?\n    end\n\n    def conf(udp_port, tcp_port, syslog_port)\n      <<~CONF\n        <system>\n          rpc_endpoint localhost:24444\n        </system>\n        <source>\n          @type monitor_agent\n        </source>\n        <source>\n          @type udp\n          tag test.udp\n          port #{udp_port}\n          <parse>\n            @type none\n          </parse>\n        </source>\n        <source>\n          @type tcp\n          tag test.tcp\n          port #{tcp_port}\n          <parse>\n            @type none\n          </parse>\n        </source>\n        <source>\n          @type syslog\n          tag test.syslog\n          port #{syslog_port}\n        </source>\n        <filter test.**>\n          @type record_transformer\n          <record>\n            foo foo\n          </record>\n        </filter>\n        <match test.**>\n          @type stdout\n        </match>\n      CONF\n    end\n\n    def run_fluentd(config)\n      conf_path = create_conf_file(\"test.conf\", config)\n      assert File.exist?(conf_path)\n      cmdline = create_cmdline(conf_path)\n\n      stdio_buf = \"\"\n      execute_command(cmdline) do |pid, stdout|\n        begin\n          waiting(60) do\n            while true\n              readables, _, _ = IO.select([stdout], nil, nil, 1)\n              next unless readables\n              break if readables.first.eof?\n\n              buf = eager_read(readables.first)\n              stdio_buf << buf\n              logs = buf.split(\"\\n\")\n\n              yield logs\n\n              break if buf.include? \"finish test\"\n            end\n          end\n        ensure\n          supervisor_pids = stdio_buf.scan(SUPERVISOR_PID_PATTERN)\n          @supervisor_pid = supervisor_pids.last.first.to_i if supervisor_pids.size >= 2\n          stdio_buf.scan(WORKER_PID_PATTERN) do |worker_pid|\n            @worker_pids << worker_pid.first.to_i\n          end\n        end\n      end\n    end\n\n    def send_udp(port, count:, interval_sec:)\n      count.times do |i|\n        s = UDPSocket.new\n        s.send(\"udp-#{i}\", 0, \"localhost\", port)\n        s.close\n        sleep interval_sec\n      end\n    end\n\n    def send_tcp(port, count:, interval_sec:)\n      count.times do |i|\n        s = TCPSocket.new(\"localhost\", port)\n        s.write(\"tcp-#{i}\\n\")\n        s.close\n        sleep interval_sec\n      end\n    end\n\n    def send_syslog(port, count:, interval_sec:)\n      count.times do |i|\n        s = UDPSocket.new\n        s.send(\"<6>Sep 10 00:00:00 localhost test: syslog-#{i}\", 0, \"localhost\", port)\n        s.close\n        sleep interval_sec\n      end\n    end\n\n    def send_end(port)\n      s = TCPSocket.new(\"localhost\", port)\n      s.write(\"finish test\\n\")\n      s.close\n    end\n\n    test \"should restart with zero downtime (no data loss)\" do\n      udp_port, syslog_port = unused_port(2, protocol: :udp)\n      tcp_port = unused_port(protocol: :tcp)\n\n      client_threads = []\n      end_thread = nil\n      records_by_type = {\n        \"udp\" => [],\n        \"tcp\" => [],\n        \"syslog\" => [],\n      }\n\n      phase = \"startup\"\n      run_fluentd(conf(udp_port, tcp_port, syslog_port)) do |logs|\n        logs.each do |log|\n          next unless /\"message\":\"(udp|tcp|syslog)-(\\d+)\",\"foo\":\"foo\"}/ =~ log\n          type = $1\n          num = $2.to_i\n          assert_true records_by_type.key?(type)\n          records_by_type[type].append(num)\n        end\n\n        if phase == \"startup\" and logs.any? { |log| log.include?(\"fluentd worker is now running worker\") }\n          phase = \"zero-downtime-restart\"\n\n          client_threads << Thread.new do\n            send_udp(udp_port, count: 500, interval_sec: 0.01)\n          end\n          client_threads << Thread.new do\n            send_tcp(tcp_port, count: 500, interval_sec: 0.01)\n          end\n          client_threads << Thread.new do\n            send_syslog(syslog_port, count: 500, interval_sec: 0.01)\n          end\n\n          sleep 1\n          response = Net::HTTP.get(URI.parse(\"http://localhost:24444/api/processes.zeroDowntimeRestart\"))\n          assert_equal '{\"ok\":true}', response\n        elsif phase == \"zero-downtime-restart\" and logs.any? { |log| log.include?(\"zero-downtime-restart: done all sequences\") }\n          phase = \"flush\"\n          response = Net::HTTP.get(URI.parse(\"http://localhost:24444/api/plugins.flushBuffers\"))\n          assert_equal '{\"ok\":true}', response\n        elsif phase == \"flush\"\n          phase = \"done\"\n          end_thread = Thread.new do\n            client_threads.each(&:join)\n            sleep 5 # make sure to flush each chunk (1s flush interval for 1chunk)\n            send_end(tcp_port)\n          end\n        end\n      end\n\n      assert_equal(\n        [(0..499).to_a, (0..499).to_a, (0..499).to_a],\n        [\n          records_by_type[\"udp\"].sort,\n          records_by_type[\"tcp\"].sort,\n          records_by_type[\"syslog\"].sort,\n        ]\n      )\n    ensure\n      client_threads.each(&:kill)\n      end_thread&.kill\n    end\n  end\n\n  def create_config_include_dir_configuration(config_path, config_dir, yaml_format = false)\n    if yaml_format\n      conf = <<CONF\n      system:\n        config_include_dir: \"#{config_dir}\"\nCONF\n    else\n      conf = <<CONF\n      <system>\n        config_include_dir #{config_dir}\n      </system>\nCONF\n    end\n    create_conf_file(config_path, conf)\n  end\n\n  sub_test_case \"test additional configuration directory\" do\n    setup do\n      FileUtils.mkdir_p(File.join(@tmp_dir, \"conf.d\"))\n    end\n\n    test \"disable additional configuration directory\" do\n      conf_path = create_config_include_dir_configuration(\"disabled_config_include_dir.conf\", \"\")\n      assert_log_matches(create_cmdline(conf_path),\n                         \"[info]: configuration include directory is disabled\")\n    end\n\n    test \"inaccessible include directory error\" do\n      conf_path = create_config_include_dir_configuration(\"inaccessible_include.conf\", \"/nonexistent\")\n      assert_log_matches(create_cmdline(conf_path),\n                         \"[info]: inaccessible include directory was specified\")\n    end\n\n    data(\"include additional configuration with relative conf.d\" => {\"relative_path\" => true},\n         \"include additional configuration with full-path conf.d\" => {\"relative_path\" => false})\n    test \"additional configuration file (conf.d/child.conf) was loaded\" do |option|\n      conf_dir = option[\"relative_path\"] ? \"conf.d\" : \"#{@tmp_dir}/conf.d\"\n      conf_path = create_config_include_dir_configuration(\"parent.conf\", conf_dir)\n      create_conf_file('conf.d/child.conf', \"\")\n      assert_log_matches(create_cmdline(conf_path),\n                         \"[info]: loading additional configuration file path=\\\"#{conf_dir}/child.conf\\\"\")\n    end\n  end\n\n  sub_test_case \"test additional configuration directory (YAML)\" do\n    setup do\n      FileUtils.mkdir_p(File.join(@tmp_dir, \"conf.d\"))\n    end\n\n    test \"disable additional configuration directory\" do\n      conf_path = create_config_include_dir_configuration(\"disabled_config_include_dir.yml\", \"\", true)\n      assert_log_matches(create_cmdline(conf_path),\n                         \"[info]: configuration include directory is disabled\")\n    end\n\n    test \"inaccessible include directory error\" do\n      conf_path = create_config_include_dir_configuration(\"inaccessible_include.yml\", \"/nonexistent\", true)\n      assert_log_matches(create_cmdline(conf_path),\n                         \"[info]: inaccessible include directory was specified\")\n    end\n\n    data(\"include additional YAML configuration with relative conf.d\" => {\"relative_path\" => true},\n         \"include additional YAML configuration with full path conf.d\" => {\"relative_path\" => false})\n    test \"additional relative configuration file (conf.d/child.yml) was loaded\" do |option|\n      conf_dir = option[\"relative_path\"] ? \"conf.d\" : \"#{@tmp_dir}/conf.d\"\n      conf_path = create_config_include_dir_configuration(\"parent.yml\", conf_dir, true)\n      create_conf_file('conf.d/child.yml', \"\")\n      assert_log_matches(create_cmdline(conf_path),\n                         \"[info]: loading additional configuration file path=\\\"#{conf_dir}/child.yml\\\"\")\n    end\n  end\n\n  test \"allow --disable-input-metrics option\" do\n    conf_path = create_conf_file('empty.conf', '')\n    assert_log_matches(\n      create_cmdline(conf_path, '--disable-input-metrics'),\n      \"#0 fluentd worker is now running worker=0\"\n    )\n  end\n\n  sub_test_case \"test suspicious harmful backed-up configuration\" do\n    data('suspicious .bak.conf' => 'dummy.bak.conf',\n         'suspicious .old.conf' => 'dummy.old.conf',\n         'suspicious .backup.conf' => 'dummy.backup.conf',\n         'suspicious .orig.conf' => 'dummy.orig.conf',\n         'suspicious .prev.conf' => 'dummy.prev.conf',\n         'suspicious .conf.conf' => 'dummy.conf.conf',\n         'suspicious .tmp.conf' => 'dummy.tmp.conf',\n         'suspicious .temp.conf' => 'dummy.temp.conf',\n         'suspicious .debug.conf' => 'dummy.debug.conf',\n         'suspicious .wip.conf' => 'dummy.wip.conf'\n        )\n    test \"warn suspicious backed-up file will be loaded\" do |suspicious_conf|\n      create_conf_file(\"dummy.conf\", <<~EOF)\n        <source>\n          @type forward\n        </source>\n      EOF\n      create_conf_file(suspicious_conf, <<~EOF)\n        <source>\n          @type forward\n        </source>\n      EOF\n      working_dir = File.join(@tmp_dir, 'working')\n      FileUtils.mkdir_p(working_dir)\n      conf_path = create_conf_file(\"working/fluent.conf\", <<~EOF)\n        <system>\n          config_include_dir \"\"\n        </system>\n        @include #{@tmp_dir}/*.conf\n      EOF\n      expected_warning_message = \"[warn]: There is a possibility that '@include #{@tmp_dir}/*.conf' includes duplicated backed-up config file such as <#{suspicious_conf}>\"\n      assert_log_matches(create_cmdline(conf_path, '--dry-run'),\n                         expected_warning_message)\n    end\n\n    data('non suspicious bar.conf' => 'bar.conf')\n    test \"no warn message\" do |non_suspicious_conf|\n      create_conf_file(\"foo.conf\", <<~EOF)\n        <source>\n          @type forward\n        </source>\n      EOF\n      create_conf_file(non_suspicious_conf, <<~EOF)\n        <source>\n          @type forward\n        </source>\n      EOF\n      working_dir = File.join(@tmp_dir, 'working')\n      FileUtils.mkdir_p(working_dir)\n      conf_path = create_conf_file(\"working/fluent.conf\", <<~EOF)\n        <system>\n          config_include_dir \"\"\n        </system>\n        @include #{@tmp_dir}/*.conf\n      EOF\n      expected_warning_message = \"[warn]: There is a possibility that '@include #{@tmp_dir}/*.conf' includes duplicated backed-up config file such as <#{non_suspicious_conf}>\"\n      assert_log_matches(create_cmdline(conf_path, '--dry-run'),\n                         \"as dry run mode\", patterns_not_match: [expected_warning_message])\n    end\n  end\n\n  sub_test_case \"test suspicious harmful antivirus exclusion path\" do\n    test \"warn pos_file path\" do\n      omit \"skip recommendation warnings about exclusion path for antivirus\" unless Fluent.windows?\n      conf_path = create_conf_file(\"foo.conf\", <<~EOF)\n        <source>\n          @type tail\n          tag t1\n          path #{@tmp_dir}/test.log\n          pos_file #{@tmp_dir}/test.pos\n        </source>\n      EOF\n      expected_warning_message = \"[warn]: Recommend adding #{@tmp_dir} to the exclusion path of your antivirus software on Windows\"\n      assert_log_matches(create_cmdline(conf_path, '--dry-run'),\n                         expected_warning_message)\n    end\n\n    test \"warn storage path\" do\n      omit \"skip recommendation warnings about exclusion path for antivirus\" unless Fluent.windows?\n      conf_path = create_conf_file(\"foo.conf\", <<~EOF)\n        <source>\n          @type sample\n          tag t1\n          <storage>\n            @type local\n            path #{@tmp_dir}/test.pos\n          </storage>\n        </source>\n      EOF\n      expected_warning_message = \"[warn]: Recommend adding #{@tmp_dir} to the exclusion path of your antivirus software on Windows\"\n      assert_log_matches(create_cmdline(conf_path, '--dry-run'),\n                         expected_warning_message)\n    end\n  end\n\n  sub_test_case \"test suspicious timekey interval (1d) in out_file configuration\" do\n    data('without buffer' => {buffer: nil, buffer_arg: nil, message: :missing_buffer},\n         'buffer' => {buffer: true, buffer_arg: '', message: :warning},\n         'buffer time' => {buffer: true, buffer_arg: 'time', message: :warning},\n         'buffer []' => {buffer: true, buffer_arg: '[]', message: nil})\n    test 'warn the default value of timekey (1d) is used as-is' do |data|\n      prefix = \"default timekey interval (1d) will be used\"\n      advice = \"To change the output frequency, please modify the timekey value\"\n      missing_warning = \"#{prefix} because of missing <buffer> section. #{advice}\"\n      warning = \"#{prefix}. #{advice}\"\n\n      conf_path = if data[:buffer]\n                    create_conf_file(\"warning.conf\", <<~EOF)\n                      <system>\n                        config_include_dir \"\"\n                      </system>\n                      <match>\n                        @type file\n                        tag test\n                        path #{@tmp_dir}/test.log\n                        <buffer #{data[:buffer_arg]}>\n                          @type file\n                        </buffer>\n                      </match>\n                    EOF\n                  else\n                    create_conf_file(\"warning.conf\", <<~EOF)\n                      <system>\n                        config_include_dir \"\"\n                      </system>\n                      <match>\n                        @type file\n                        tag test\n                        path #{@tmp_dir}/test.log\n                      </match>\n                    EOF\n      end\n      case data[:message]\n      when :missing_buffer\n        assert_log_matches(create_cmdline(conf_path, '--dry-run'), missing_warning, patterns_not_match: [warning])\n      when :warning\n        assert_log_matches(create_cmdline(conf_path, '--dry-run'), \"warning\", patterns_not_match: [missing_warning])\n      else\n        assert_log_matches(create_cmdline(conf_path, '--dry-run'), patterns_not_match: [warning, missing_warning])\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/command/test_plugin_config_formatter.rb",
    "content": "require_relative '../helper'\n\nrequire 'pathname'\nrequire 'fluent/command/plugin_config_formatter'\nrequire 'fluent/plugin/input'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/filter'\nrequire 'fluent/plugin/parser'\nrequire 'fluent/plugin/formatter'\n\nclass TestFluentPluginConfigFormatter < Test::Unit::TestCase\n  class FakeInput < ::Fluent::Plugin::Input\n    ::Fluent::Plugin.register_input(\"fake\", self)\n\n    desc \"path to something\"\n    config_param :path, :string\n  end\n\n  class FakeOutput < ::Fluent::Plugin::Output\n    ::Fluent::Plugin.register_output(\"fake\", self)\n\n    desc \"path to something\"\n    config_param :path, :string\n\n    def process(tag, es)\n    end\n  end\n\n  class FakeFilter < ::Fluent::Plugin::Filter\n    ::Fluent::Plugin.register_filter(\"fake\", self)\n\n    desc \"path to something\"\n    config_param :path, :string\n\n    def filter(tag, time, record)\n    end\n  end\n\n  class FakeParser < ::Fluent::Plugin::Parser\n    ::Fluent::Plugin.register_parser(\"fake\", self)\n\n    desc \"path to something\"\n    config_param :path, :string\n\n    def parse(text)\n    end\n  end\n\n  class FakeFormatter < ::Fluent::Plugin::Formatter\n    ::Fluent::Plugin.register_formatter(\"fake\", self)\n\n    desc \"path to something\"\n    config_param :path, :string\n\n    def format(tag, time, record)\n    end\n  end\n\n  class FakeStorage < ::Fluent::Plugin::Storage\n    ::Fluent::Plugin.register_storage('fake', self)\n\n    def get(key)\n    end\n\n    def fetch(key, defval)\n    end\n\n    def put(key, value)\n    end\n\n    def delete(key)\n    end\n\n    def update(key, &block)\n    end\n  end\n\n  class FakeServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery\n    ::Fluent::Plugin.register_sd('fake', self)\n\n    desc \"hostname\"\n    config_param :hostname, :string\n  end\n\n  class SimpleInput < ::Fluent::Plugin::Input\n    ::Fluent::Plugin.register_input(\"simple\", self)\n    helpers :inject, :compat_parameters\n\n    desc \"path to something\"\n    config_param :path, :string\n  end\n\n  class ComplexOutput < ::Fluent::Plugin::Output\n    ::Fluent::Plugin.register_output(\"complex\", self)\n    helpers :inject, :compat_parameters\n\n    config_section :authentication, required: true, multi: false do\n      desc \"username\"\n      config_param :username, :string\n      desc \"password\"\n      config_param :password, :string, secret: true\n    end\n\n    config_section :parent do\n      config_section :child do\n        desc \"names\"\n        config_param :names, :array\n        desc \"difficulty\"\n        config_param :difficulty, :enum, list: [:easy, :normal, :hard], default: :normal\n      end\n    end\n\n    def process(tag, es)\n    end\n  end\n\n  class SimpleServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery\n    ::Fluent::Plugin.register_sd('simple', self)\n\n    desc \"servers\"\n    config_param :servers, :array\n  end\n\n  sub_test_case \"json\" do\n    data(input: [FakeInput, \"input\"],\n         output: [FakeOutput, \"output\"],\n         filter: [FakeFilter, \"filter\"],\n         parser: [FakeParser, \"parser\"],\n         formatter: [FakeFormatter, \"formatter\"])\n    test \"dumped config should be valid JSON\" do |(klass, type)|\n      dumped_config = capture_stdout do\n        FluentPluginConfigFormatter.new([\"--format=json\", type, \"fake\"]).call\n      end\n      expected = {\n        path: {\n          desc: \"path to something\",\n          type: \"string\",\n          required: true\n        }\n      }\n      assert_equal(expected, JSON.parse(dumped_config, symbolize_names: true)[klass.name.to_sym])\n    end\n  end\n\n  sub_test_case \"text\" do\n    test \"input simple\" do\n      dumped_config = capture_stdout do\n        FluentPluginConfigFormatter.new([\"--format=txt\", \"input\", \"simple\"]).call\n      end\n      expected = <<TEXT\nhelpers: inject,compat_parameters\n@log_level: string: (nil)\npath: string: (nil)\nTEXT\n      assert_equal(expected, dumped_config)\n    end\n\n    test \"output complex\" do\n      dumped_config = capture_stdout do\n        FluentPluginConfigFormatter.new([\"--format=txt\", \"output\", \"complex\"]).call\n      end\n      expected = <<TEXT\nhelpers: inject,compat_parameters\n@log_level: string: (nil)\ntime_as_integer: bool: (false)\nslow_flush_log_threshold: float: (20.0)\n<buffer>: optional, single\n chunk_keys: array: ([])\n @type: string: (\"memory\")\n timekey: time: (nil)\n timekey_wait: time: (600)\n timekey_use_utc: bool: (false)\n timekey_zone: string: (\"#{Time.now.strftime('%z')}\")\n flush_at_shutdown: bool: (nil)\n flush_mode: enum: (:default)\n flush_interval: time: (60)\n flush_thread_count: integer: (1)\n flush_thread_interval: float: (1.0)\n flush_thread_burst_interval: float: (1.0)\n delayed_commit_timeout: time: (60)\n overflow_action: enum: (:throw_exception)\n retry_forever: bool: (false)\n retry_timeout: time: (259200)\n retry_max_times: integer: (nil)\n retry_secondary_threshold: float: (0.8)\n retry_type: enum: (:exponential_backoff)\n retry_wait: time: (1)\n retry_exponential_backoff_base: float: (2)\n retry_max_interval: time: (nil)\n retry_randomize: bool: (true)\n<secondary>: optional, single\n @type: string: (nil)\n <buffer>: optional, single\n <secondary>: optional, single\n<authentication>: required, single\n username: string: (nil)\n password: string: (nil)\n<parent>: optional, multiple\n <child>: optional, multiple\n  names: array: (nil)\n  difficulty: enum: (:normal)\nTEXT\n      assert_equal(expected, dumped_config)\n    end\n  end\n\n  sub_test_case \"markdown\" do\n    test \"input simple\" do\n      dumped_config = capture_stdout do\n        FluentPluginConfigFormatter.new([\"--format=markdown\", \"input\", \"simple\"]).call\n      end\n      expected = <<TEXT\n## Plugin helpers\n\n* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)\n* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)\n\n* See also: [Input Plugin Overview](https://docs.fluentd.org/v/1.0/input#overview)\n\n## TestFluentPluginConfigFormatter::SimpleInput\n\n### path (string) (required)\n\npath to something\n\n\nTEXT\n      assert_equal(expected, dumped_config)\n    end\n\n    test \"input simple (table)\" do\n      dumped_config = capture_stdout do\n        FluentPluginConfigFormatter.new([\"--format=markdown\", \"--table\", \"input\", \"simple\"]).call\n      end\n      expected = <<TEXT\n## Plugin helpers\n\n* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)\n* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)\n\n* See also: [Input Plugin Overview](https://docs.fluentd.org/v/1.0/input#overview)\n\n## TestFluentPluginConfigFormatter::SimpleInput\n\n### Configuration\n\n|parameter|type|description|default|\n|---|---|---|---|\n|path|string (required)|path to something||\n\nTEXT\n      assert_equal(expected, dumped_config)\n    end\n\n    data(\"abbrev\" => \"sd\",\n         \"normal\" => \"service_discovery\")\n    test \"service_discovery simple\" do |data|\n      plugin_type = data\n      dumped_config = capture_stdout do\n        FluentPluginConfigFormatter.new([\"--format=markdown\", plugin_type, \"simple\"]).call\n      end\n      expected = <<TEXT\n* See also: [ServiceDiscovery Plugin Overview](https://docs.fluentd.org/v/1.0/servicediscovery#overview)\n\n## TestFluentPluginConfigFormatter::SimpleServiceDiscovery\n\n### servers (array) (required)\n\nservers\n\n\nTEXT\n      assert_equal(expected, dumped_config)\n    end\n\n\n    test \"output complex\" do\n      dumped_config = capture_stdout do\n        FluentPluginConfigFormatter.new([\"--format=markdown\", \"output\", \"complex\"]).call\n      end\n      expected = <<TEXT\n## Plugin helpers\n\n* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)\n* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)\n\n* See also: [Output Plugin Overview](https://docs.fluentd.org/v/1.0/output#overview)\n\n## TestFluentPluginConfigFormatter::ComplexOutput\n\n\n### \\\\<authentication\\\\> section (required) (single)\n\n#### username (string) (required)\n\nusername\n\n#### password (string) (required)\n\npassword\n\n\n\n### \\\\<parent\\\\> section (optional) (multiple)\n\n\n#### \\\\<child\\\\> section (optional) (multiple)\n\n##### names (array) (required)\n\nnames\n\n##### difficulty (enum) (optional)\n\ndifficulty\n\nAvailable values: easy, normal, hard\n\nDefault value: `normal`.\n\n\n\n\nTEXT\n      assert_equal(expected, dumped_config)\n    end\n\n    test \"output complex (table)\" do\n      dumped_config = capture_stdout do\n        FluentPluginConfigFormatter.new([\"--format=markdown\", \"--table\", \"output\", \"complex\"]).call\n      end\n      expected = <<TEXT\n## Plugin helpers\n\n* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)\n* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)\n\n* See also: [Output Plugin Overview](https://docs.fluentd.org/v/1.0/output#overview)\n\n## TestFluentPluginConfigFormatter::ComplexOutput\n\n\n### \\\\<authentication\\\\> section (required) (single)\n\n### Configuration\n\n|parameter|type|description|default|\n|---|---|---|---|\n|username|string (required)|username||\n|password|string (required)|password||\n\n\n### \\\\<parent\\\\> section (optional) (multiple)\n\n\n#### \\\\<child\\\\> section (optional) (multiple)\n\n### Configuration\n\n|parameter|type|description|default|\n|---|---|---|---|\n|names|array (required)|names||\n|difficulty|enum (optional)|difficulty (`easy`, `normal`, `hard`)|`normal`|\n\n\n\nTEXT\n      assert_equal(expected, dumped_config)\n    end\n  end\n\n  sub_test_case \"arguments\" do\n    data do\n      hash = {}\n      [\"input\", \"output\", \"filter\", \"parser\", \"formatter\", \"storage\", \"service_discovery\"].each do |type|\n        [\"txt\", \"json\", \"markdown\"].each do |format|\n          argv = [\"--format=#{format}\"]\n          [\n            [\"--verbose\", \"--compact\"],\n            [\"--verbose\"],\n            [\"--compact\"]\n          ].each do |options|\n            hash[\"[#{type}] \" + (argv + options).join(\" \")] = argv + options + [type, \"fake\"]\n          end\n        end\n      end\n      hash\n    end\n    test \"dump txt\" do |argv|\n      capture_stdout do\n        assert_nothing_raised do\n          FluentPluginConfigFormatter.new(argv).call\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/command/test_plugin_generator.rb",
    "content": "require_relative '../helper'\n\nrequire 'pathname'\nrequire 'fluent/command/plugin_generator'\n\nclass TestFluentPluginGenerator < Test::Unit::TestCase\n  TEMP_DIR = \"tmp/plugin_generator\"\n  setup do\n    FileUtils.mkdir_p(TEMP_DIR)\n    @pwd = Dir.pwd\n    Dir.chdir(TEMP_DIR)\n  end\n\n  teardown do\n    Dir.chdir(@pwd)\n    FileUtils.rm_rf(TEMP_DIR)\n  end\n\n  def stub_git_process(target)\n    stub(target).spawn do |cmd, arg1, arg2|\n      assert_equal %w[git init .], [cmd, arg1, arg2]\n      -1\n    end\n    stub(Process).wait { |pid| assert_equal(pid, -1) }\n  end\n\n  sub_test_case \"generate plugin\" do\n    data(input: [\"input\", \"in\"],\n         output: [\"output\", \"out\"],\n         filter: [\"filter\", \"filter\"],\n         parser: [\"parser\", \"parser\"],\n         formatter: [\"formatter\", \"formatter\"],\n         storage: [\"storage\", \"storage\"])\n    test \"generate plugin\" do |(type, part)|\n      generator = FluentPluginGenerator.new([type, \"fake\"])\n      stub_git_process(generator)\n      capture_stdout do\n        generator.call\n      end\n      plugin_base_dir = Pathname(\"fluent-plugin-fake\")\n      assert { plugin_base_dir.directory? }\n      expected = [\n        \"fluent-plugin-fake\",\n        \"fluent-plugin-fake/Gemfile\",\n        \"fluent-plugin-fake/LICENSE\",\n        \"fluent-plugin-fake/README.md\",\n        \"fluent-plugin-fake/Rakefile\",\n        \"fluent-plugin-fake/fluent-plugin-fake.gemspec\",\n        \"fluent-plugin-fake/lib\",\n        \"fluent-plugin-fake/lib/fluent\",\n        \"fluent-plugin-fake/lib/fluent/plugin\",\n        \"fluent-plugin-fake/lib/fluent/plugin/#{part}_fake.rb\",\n        \"fluent-plugin-fake/test\",\n        \"fluent-plugin-fake/test/helper.rb\",\n        \"fluent-plugin-fake/test/plugin\",\n        \"fluent-plugin-fake/test/plugin/test_#{part}_fake.rb\",\n      ]\n      actual = plugin_base_dir.find.reject {|f| f.fnmatch(\"*/.git*\") }.map(&:to_s).sort\n      assert_equal(expected, actual)\n    end\n\n    test \"no license\" do\n      generator = FluentPluginGenerator.new([\"--no-license\", \"filter\", \"fake\"])\n      stub_git_process(generator)\n      capture_stdout do\n        generator.call\n      end\n      assert { !Pathname(\"fluent-plugin-fake/LICENSE\").exist? }\n      assert { Pathname(\"fluent-plugin-fake/Gemfile\").exist? }\n    end\n\n    test \"unknown license\" do\n      out = capture_stdout do\n        assert_raise(SystemExit) do\n          FluentPluginGenerator.new([\"--license=unknown\", \"filter\", \"fake\"]).call\n        end\n      end\n      assert { out.lines.include?(\"License: unknown\\n\") }\n    end\n  end\n\n  sub_test_case \"unify plugin name\" do\n    data(\"word\" => [\"fake\", \"fake\"],\n         \"underscore\" => [\"rewrite_tag_filter\", \"rewrite_tag_filter\"],\n         \"dash\" => [\"rewrite-tag-filter\", \"rewrite_tag_filter\"])\n    test \"plugin_name\" do |(name, plugin_name)|\n      generator = FluentPluginGenerator.new([\"filter\", name])\n      stub_git_process(generator)\n      stub(Process).wait { |pid| assert_equal(pid, -1) }\n      capture_stdout do\n        generator.call\n      end\n      assert_equal(plugin_name, generator.__send__(:plugin_name))\n    end\n\n    data(\"word\" => [\"fake\", \"fluent-plugin-fake\"],\n         \"underscore\" => [\"rewrite_tag_filter\", \"fluent-plugin-rewrite-tag-filter\"],\n         \"dash\" => [\"rewrite-tag-filter\", \"fluent-plugin-rewrite-tag-filter\"])\n    test \"gem_name\" do |(name, gem_name)|\n      generator = FluentPluginGenerator.new([\"output\", name])\n      stub_git_process(generator)\n      stub(Process).wait { |pid| assert_equal(pid, -1) }\n      capture_stdout do\n        generator.call\n      end\n      assert_equal(gem_name, generator.__send__(:gem_name))\n    end\n  end\nend\n"
  },
  {
    "path": "test/compat/test_calls_super.rb",
    "content": "require_relative '../helper'\n\n# these are Fluent::Compat::* in fact\nrequire 'fluent/input'\nrequire 'fluent/output'\nrequire 'fluent/filter'\n\nclass CompatCallsSuperTest < Test::Unit::TestCase\n  class DummyGoodInput < Fluent::Input\n    def configure(conf); super; end\n    def start; super; end\n    def before_shutdown; super; end\n    def shutdown; super; end\n  end\n  class DummyBadInput < Fluent::Input\n    def configure(conf); super; end\n    def start; end\n    def before_shutdown; end\n    def shutdown; end\n  end\n  class DummyGoodOutput < Fluent::Output\n    def configure(conf); super; end\n    def start; super; end\n    def before_shutdown; super; end\n    def shutdown; super; end\n  end\n  class DummyBadOutput < Fluent::Output\n    def configure(conf); super; end\n    def start; end\n    def before_shutdown; end\n    def shutdown; end\n  end\n  class DummyGoodFilter < Fluent::Filter\n    def configure(conf); super; end\n    def filter(tag, time, record); end\n    def start; super; end\n    def before_shutdown; super; end\n    def shutdown; super; end\n  end\n  class DummyBadFilter < Fluent::Filter\n    def configure(conf); super; end\n    def filter(tag, time, record); end\n    def start; end\n    def before_shutdown; end\n    def shutdown; end\n  end\n\n  setup do\n    Fluent::Test.setup\n  end\n\n  sub_test_case 'old API plugin which calls super properly' do\n    test 'Input#start, #before_shutdown and #shutdown calls all superclass methods properly' do\n      i = DummyGoodInput.new\n      i.configure(config_element())\n      assert i.configured?\n\n      i.start\n      assert i.started?\n\n      i.before_shutdown\n      assert i.before_shutdown?\n\n      i.shutdown\n      assert i.shutdown?\n\n      assert i.log.out.logs.empty?\n    end\n\n    test 'Output#start, #before_shutdown and #shutdown calls all superclass methods properly' do\n      i = DummyGoodOutput.new\n      i.configure(config_element())\n      assert i.configured?\n\n      i.start\n      assert i.started?\n\n      i.before_shutdown\n      assert i.before_shutdown?\n\n      i.shutdown\n      assert i.shutdown?\n\n      assert i.log.out.logs.empty?\n    end\n\n    test 'Filter#start, #before_shutdown and #shutdown calls all superclass methods properly' do\n      i = DummyGoodFilter.new\n      i.configure(config_element())\n      assert i.configured?\n\n      i.start\n      assert i.started?\n\n      i.before_shutdown\n      assert i.before_shutdown?\n\n      i.shutdown\n      assert i.shutdown?\n\n      assert i.log.out.logs.empty?\n    end\n  end\n\n  sub_test_case 'old API plugin which does not call super' do\n    test 'Input#start, #before_shutdown and #shutdown calls superclass methods forcedly with logs' do\n      i = DummyBadInput.new\n      i.configure(config_element())\n      assert i.configured?\n\n      i.start\n      assert i.started?\n\n      i.before_shutdown\n      assert i.before_shutdown?\n\n      i.shutdown\n      assert i.shutdown?\n\n      logs = i.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"[warn]: super was not called in #start: called it forcedly plugin=CompatCallsSuperTest::DummyBadInput\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: super was not called in #before_shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadInput\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: super was not called in #shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadInput\") } }\n    end\n\n    test 'Output#start, #before_shutdown and #shutdown calls superclass methods forcedly with logs' do\n      i = DummyBadOutput.new\n      i.configure(config_element())\n      assert i.configured?\n\n      i.start\n      assert i.started?\n\n      i.before_shutdown\n      assert i.before_shutdown?\n\n      i.shutdown\n      assert i.shutdown?\n\n      logs = i.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"[warn]: super was not called in #start: called it forcedly plugin=CompatCallsSuperTest::DummyBadOutput\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: super was not called in #before_shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadOutput\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: super was not called in #shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadOutput\") } }\n    end\n\n    test 'Filter#start, #before_shutdown and #shutdown calls superclass methods forcedly with logs' do\n      i = DummyBadFilter.new\n      i.configure(config_element())\n      assert i.configured?\n\n      i.start\n      assert i.started?\n\n      i.before_shutdown\n      assert i.before_shutdown?\n\n      i.shutdown\n      assert i.shutdown?\n\n      logs = i.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"[warn]: super was not called in #start: called it forcedly plugin=CompatCallsSuperTest::DummyBadFilter\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: super was not called in #before_shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadFilter\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: super was not called in #shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadFilter\") } }\n    end\n  end\nend\n"
  },
  {
    "path": "test/compat/test_parser.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/parser'\n\nclass TextParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  class MultiEventTestParser < ::Fluent::Parser\n    include Fluent::Configurable\n\n    def parse(text)\n      2.times { |i|\n        record = {}\n        record['message'] = text\n        record['number'] = i\n        yield Fluent::Engine.now, record\n      }\n    end\n  end\n\n  Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })\n\n  def test_lookup_unknown_format\n    assert_raise Fluent::NotFoundPluginError do\n      Fluent::Plugin.new_parser('unknown')\n    end\n  end\n\n  data('register_formatter' => 'known', 'register_template' => 'known_old')\n  def test_lookup_known_parser(data)\n    $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), '..', 'scripts'))\n    assert_nothing_raised Fluent::ConfigError do\n      Fluent::Plugin.new_parser(data)\n    end\n    $LOAD_PATH.shift\n  end\n\n  def test_parse_with_return\n    parser = Fluent::TextParser.new\n    parser.configure(config_element('test', '', 'format' => 'none'))\n    _time, record = parser.parse('log message!')\n    assert_equal({'message' => 'log message!'}, record)\n  end\n\n  def test_parse_with_block\n    parser = Fluent::TextParser.new\n    parser.configure(config_element('test', '', 'format' => 'none'))\n    parser.parse('log message!') { |time, record|\n      assert_equal({'message' => 'log message!'}, record)\n    }\n  end\n\n  def test_multi_event_parser\n    parser = Fluent::TextParser.new\n    parser.configure(config_element('test', '', 'format' => 'multi_event_test'))\n    i = 0\n    parser.parse('log message!') { |time, record|\n      assert_equal('log message!', record['message'])\n      assert_equal(i, record['number'])\n      i += 1\n    }\n  end\n\n  def test_setting_estimate_current_event_value\n    p1 = Fluent::TextParser.new\n    assert_nil p1.estimate_current_event\n    assert_nil p1.parser\n\n    p1.configure(config_element('test', '', 'format' => 'none'))\n    assert_equal true, p1.parser.estimate_current_event\n\n    p2 = Fluent::TextParser.new\n    assert_nil p2.estimate_current_event\n    assert_nil p2.parser\n\n    p2.estimate_current_event = false\n\n    p2.configure(config_element('test', '', 'format' => 'none'))\n    assert_equal false, p2.parser.estimate_current_event\n  end\n\n  data(ignorecase: Regexp::IGNORECASE,\n       multiline: Regexp::MULTILINE,\n       both: Regexp::IGNORECASE & Regexp::MULTILINE)\n  def test_regexp_parser_config(options)\n    source = \"(?<test>.*)\"\n    parser = Fluent::TextParser::RegexpParser.new(Regexp.new(source, options), { \"dummy\" => \"dummy\" })\n    regexp = parser.instance_variable_get(:@regexp)\n    assert_equal(options, regexp.options)\n  end\nend\n"
  },
  {
    "path": "test/config/assertions.rb",
    "content": "require 'test/unit/assertions'\n\nmodule Test::Unit::Assertions\n  def assert_text_parsed_as(expected, actual)\n    msg = parse_text(actual).inspect rescue 'failed'\n    msg = \"expected that #{actual.inspect} would be a parsed as #{expected.inspect} but got #{msg}\"\n    assert_block(msg) {\n      v = parse_text(actual)\n      if expected.is_a?(Float)\n        v.is_a?(Float) && (v == obj || (v.nan? && obj.nan?) || (v - obj).abs < 0.000001)\n      else\n        v == expected\n      end\n    }\n  end\n\n  def assert_text_parsed_as_json(expected, actual)\n    msg = parse_text(actual).inspect rescue 'failed'\n    msg = \"expected that #{actual.inspect} would be a parsed as #{expected.inspect} but got #{msg}\"\n    assert_block(msg) {\n      v = JSON.parse(parse_text(actual))\n      v == expected\n    }\n  end\n\n  def assert_parse_error(actual)\n    msg = begin\n            parse_text(actual).inspect\n          rescue => e\n            e.inspect\n          end\n    msg = \"expected that #{actual.inspect} would cause a parse error but got #{msg}\"\n    assert_block(msg) {\n      begin\n        parse_text(actual)\n        false\n      rescue Fluent::ConfigParseError\n        true\n      end\n    }\n  end\nend\n"
  },
  {
    "path": "test/config/test_config_parser.rb",
    "content": "require_relative '../helper'\nrequire_relative \"assertions\"\nrequire \"json\"\nrequire \"fluent/config/error\"\nrequire \"fluent/config/basic_parser\"\nrequire \"fluent/config/literal_parser\"\nrequire \"fluent/config/v1_parser\"\nrequire 'fluent/config/parser'\n\nmodule Fluent::Config\n  module V1TestHelper\n    def root(*elements)\n      if elements.first.is_a?(Fluent::Config::Element)\n        attrs = {}\n      else\n        attrs = elements.shift || {}\n      end\n      Fluent::Config::Element.new('ROOT', '', attrs, elements)\n    end\n\n    def e(name, arg='', attrs={}, elements=[])\n      Fluent::Config::Element.new(name, arg, attrs, elements)\n    end\n  end\n\n  class AllTypes\n    include Fluent::Configurable\n\n    config_param :param_string, :string\n    config_param :param_enum, :enum, list: [:foo, :bar, :baz]\n    config_param :param_integer, :integer\n    config_param :param_float, :float\n    config_param :param_size, :size\n    config_param :param_bool, :bool\n    config_param :param_time, :time\n    config_param :param_hash, :hash\n    config_param :param_array, :array\n    config_param :param_regexp, :regexp\n  end\n\n  class TestV1Parser < ::Test::Unit::TestCase\n    def read_config(path)\n      path = File.expand_path(path)\n      data = File.read(path)\n      Fluent::Config::V1Parser.parse(data, File.basename(path), File.dirname(path))\n    end\n\n    def parse_text(text)\n      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')\n      Fluent::Config::V1Parser.parse(text, '(test)', basepath, nil)\n    end\n\n    include V1TestHelper\n    extend V1TestHelper\n\n    sub_test_case 'attribute parsing' do\n      test \"parses attributes\" do\n        assert_text_parsed_as(e('ROOT', '', {\"k1\"=>\"v1\", \"k2\"=>\"v2\"}), %[\n          k1 v1\n          k2 v2\n        ])\n      end\n\n      test \"allows attribute without value\" do\n        assert_text_parsed_as(e('ROOT', '', {\"k1\"=>\"\", \"k2\"=>\"v2\"}), %[\n          k1\n          k2 v2\n        ])\n      end\n\n      test \"parses attribute key always string\" do\n        assert_text_parsed_as(e('ROOT', '', {\"1\" => \"1\"}), \"1 1\")\n      end\n\n      data(\"_.%$!,\"     => \"_.%$!,\",\n           \"/=~-~@\\`:?\" => \"/=~-~@\\`:?\",\n           \"()*{}.[]\"   => \"()*{}.[]\")\n      test \"parses a value with symbols\" do |v|\n        assert_text_parsed_as(e('ROOT', '', {\"k\" => v}), \"k #{v}\")\n      end\n\n      test \"ignores spacing around value\" do\n        assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"a\"}), \"  k1     a    \")\n      end\n\n      test \"allows spaces in value\" do\n        assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"a  b  c\"}), \"k1 a  b  c\")\n      end\n\n      test \"parses value into empty string if only key exists\" do\n        # value parser parses empty string as true for bool type\n        assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"\"}), \"k1\\n\")\n        assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"\"}), \"k1\")\n      end\n\n      sub_test_case 'non-quoted string' do\n        test \"remains text starting with '#'\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"#not_comment\"}), \"  k1 #not_comment\")\n        end\n\n        test \"remains text just after '#'\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"a#not_comment\"}), \"  k1 a#not_comment\")\n        end\n\n        test \"remove text after ` #` (comment)\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"a\"}), \"  k1 a #comment\")\n        end\n\n        test \"does not require escaping backslash\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"\\\\\\\\\"}), \"  k1 \\\\\\\\\")\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"\\\\\"}), \"  k1 \\\\\")\n        end\n\n        test \"remains backslash in front of a normal character\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => '\\['}), \"  k1 \\\\[\")\n        end\n\n        test \"does not accept escape characters\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => '\\n'}), \"  k1 \\\\n\")\n        end\n      end\n\n      sub_test_case 'double quoted string' do\n        test \"allows # in value\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"a#comment\"}), '  k1 \"a#comment\"')\n        end\n\n        test \"rejects characters after double quoted string\" do\n          assert_parse_error('  k1 \"a\" 1')\n        end\n\n        test \"requires escaping backslash\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"\\\\\"}), '  k1 \"\\\\\\\\\"')\n          assert_parse_error('  k1 \"\\\\\"')\n        end\n\n        test \"requires escaping double quote\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => '\"'}), '  k1 \"\\\\\"\"')\n          assert_parse_error('  k1 \"\"\"')\n          assert_parse_error('  k1 \"\"\\'')\n        end\n\n        test \"removes backslash in front of a normal character\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => '['}), '  k1 \"\\\\[\"')\n        end\n\n        test \"accepts escape characters\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"\\n\"}), '  k1 \"\\\\n\"')\n        end\n\n        test \"support multiline string\" do\n          assert_text_parsed_as(e('ROOT', '',\n            {\"k1\" => %[line1\n                       line2]\n            }),\n            %[k1      \"line1\n                       line2\"]\n          )\n          assert_text_parsed_as(e('ROOT', '',\n            {\"k1\" => %[line1                       line2]\n            }),\n            %[k1      \"line1\\\\\n                       line2\"]\n          )\n          assert_text_parsed_as(e('ROOT', '',\n            {\"k1\" => %[line1\n                       line2\n                       line3]\n            }),\n            %[k1      \"line1\n                       line2\n                       line3\"]\n          )\n          assert_text_parsed_as(e('ROOT', '',\n            {\"k1\" => %[line1\n                       line2                       line3]\n            }),\n            %[k1      \"line1\n                       line2\\\\\n                       line3\"]\n          )\n        end\n      end\n\n      sub_test_case 'single quoted string' do\n        test \"allows # in value\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"a#comment\"}), \"  k1 'a#comment'\")\n        end\n\n        test \"rejects characters after single quoted string\" do\n          assert_parse_error(\"  k1 'a' 1\")\n        end\n\n        test \"requires escaping backslash\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"\\\\\"}), \"  k1 '\\\\\\\\'\")\n          assert_parse_error(\"  k1 '\\\\'\")\n        end\n\n        test \"requires escaping single quote\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"'\"}), \"  k1 '\\\\''\")\n          assert_parse_error(\"  k1 '''\")\n        end\n\n        test \"remains backslash in front of a normal character\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => '\\\\['}), \"  k1 '\\\\['\")\n        end\n\n        test \"does not accept escape characters\" do\n          assert_text_parsed_as(e('ROOT', '', {\"k1\" => \"\\\\n\"}), \"  k1 '\\\\n'\")\n        end\n      end\n\n      data(\n        \"in match\" => %[\n          <match>\n            @k v\n          </match>\n        ],\n        \"in source\" => %[\n          <source>\n            @k v\n          </source>\n        ],\n        \"in filter\" => %[\n          <filter>\n            @k v\n          </filter>\n        ],\n        \"in top-level\" => '  @k v '\n        )\n      def test_rejects_at_prefix_in_the_parameter_name(data)\n        assert_parse_error(data)\n      end\n\n      data(\n        \"in nested\" => %[\n          <match>\n            <record>\n              @k v\n            </record>\n          </match>\n        ]\n        )\n      def test_not_reject_at_prefix_in_the_parameter_name(data)\n        assert_nothing_raised { parse_text(data) }\n      end\n    end\n\n    sub_test_case 'element parsing' do\n      data(\n        'root' => [root, \"\"],\n        \"accepts empty element\" => [root(e(\"test\")), %[\n          <test>\n          </test>\n        ]],\n        \"accepts argument and attributes\" => [root(e(\"test\", 'var', {'key'=>\"val\"})), %[\n          <test var>\n            key val\n          </test>\n        ]],\n        \"accepts nested elements\" => [root(\n          e(\"test\", 'var', {'key'=>'1'}, [\n            e('nested1'),\n            e('nested2')\n          ])), %[\n          <test var>\n            key 1\n            <nested1>\n            </nested1>\n            <nested2>\n            </nested2>\n          </test>\n        ]],\n        \"accepts multiline json values\" => [root(e(\"test\", 'var', {'key'=>\"[\\\"a\\\",\\\"b\\\",\\\"c\\\",\\\"d\\\"]\"})), %[\n          <test var>\n            key [\"a\",\n\"b\", \"c\",\n\"d\"]\n          </test>\n        ]],\n        \"parses empty element argument to nil\" => [root(e(\"test\", '')), %[\n          <test >\n          </test>\n        ]],\n        \"ignores spacing around element argument\" => [root(e(\"test\", \"a\")), %[\n          <test    a    >\n          </test>\n        ]],\n        \"accepts spacing inside element argument (for multiple tags)\" => [root(e(\"test\", \"a.** b.**\")), %[\n          <test    a.** b.** >\n          </test>\n        ]])\n      def test_parse_element(data)\n        expected, target = data\n        assert_text_parsed_as(expected, target)\n      end\n\n      [\n        \"**\",\n        \"*.*\",\n        \"1\",\n        \"_.%$!\",\n        \"/\",\n        \"()*{}.[]\",\n      ].each do |arg|\n        test \"parses symbol element argument:#{arg}\" do\n          assert_text_parsed_as(root(e(\"test\", arg)), %[\n            <test #{arg}>\n            </test>\n          ])\n        end\n      end\n\n      data(\n        \"considers comments in element argument\" => %[\n          <test #a>\n          </test>\n        ],\n        \"requires line_end after begin tag\" => %[\n          <test></test>\n        ],\n        \"requires line_end after end tag\" => %[\n          <test>\n          </test><test>\n          </test>\n        ])\n      def test_parse_error(data)\n        assert_parse_error(data)\n      end\n    end\n\n    sub_test_case \"Embedded Ruby Code in section attributes\" do\n      setup do\n        ENV[\"EMBEDDED_VAR\"] = \"embedded\"\n        ENV[\"NESTED_EMBEDDED_VAR\"] = \"nested-embedded\"\n        @hostname = Socket.gethostname\n      end\n\n      teardown do\n        ENV[\"EMBEDDED_VAR\"] = nil\n        ENV[\"NESTED_EMBEDDED_VAR\"] = nil\n      end\n\n      test \"embedded Ruby code should be expanded\" do\n        assert_text_parsed_as(root(\n          e(\"test\", 'embedded', {'key'=>'1'}, [\n            e('nested1', 'nested-embedded'),\n            e('nested2', \"#{@hostname}\")\n          ])), <<-EOF\n          <test \"#{ENV[\"EMBEDDED_VAR\"]}\">\n            key 1\n            <nested1 \"#{ENV[\"NESTED_EMBEDDED_VAR\"]}\">\n            </nested1>\n            <nested2 \"#{Socket.gethostname}\">\n            </nested2>\n          </test>\n        EOF\n        )\n      end\n    end\n\n    # port from test_config.rb\n    sub_test_case '@include parsing' do\n      TMP_DIR = File.dirname(__FILE__) + \"/tmp/v1_config#{ENV['TEST_ENV_NUMBER']}\"\n      TMP_DIR_WITH_SPACES = File.dirname(__FILE__) + \"/tmp/folder with spaces/v1_config#{ENV['TEST_ENV_NUMBER']}\"\n\n      def write_config(path, data)\n        FileUtils.mkdir_p(File.dirname(path))\n        File.open(path, \"w\") { |f| f.write data }\n      end\n\n      def prepare_config(tmp_dir)\n        write_config \"#{tmp_dir}/config_test_1.conf\", %[\n        k1 root_config\n        include dir/config_test_2.conf  #\n        @include #{tmp_dir}/config_test_4.conf\n        include file://#{tmp_dir}/config_test_5.conf\n        @include config.d/*.conf\n      ]\n        write_config \"#{tmp_dir}/dir/config_test_2.conf\", %[\n        k2 relative_path_include\n        @include ../config_test_3.conf\n      ]\n        write_config \"#{tmp_dir}/config_test_3.conf\", %[\n        k3 relative_include_in_included_file\n      ]\n        write_config \"#{tmp_dir}/config_test_4.conf\", %[\n        k4 absolute_path_include\n      ]\n        write_config \"#{tmp_dir}/config_test_5.conf\", %[\n        k5 uri_include\n      ]\n        write_config \"#{tmp_dir}/config.d/config_test_6.conf\", %[\n        k6 wildcard_include_1\n        <elem1 name>\n          include normal_parameter\n        </elem1>\n      ]\n        write_config \"#{tmp_dir}/config.d/config_test_7.conf\", %[\n        k7 wildcard_include_2\n      ]\n        write_config \"#{tmp_dir}/config.d/config_test_8.conf\", %[\n        <elem2 name>\n          @include ../dir/config_test_9.conf\n        </elem2>\n      ]\n        write_config \"#{tmp_dir}/dir/config_test_9.conf\", %[\n        k9 embedded\n        <elem3 name>\n          nested nested_value\n          include hoge\n        </elem3>\n      ]\n        write_config \"#{tmp_dir}/config.d/00_config_test_8.conf\", %[\n        k8 wildcard_include_3\n        <elem4 name>\n          include normal_parameter\n        </elem4>\n      ]\n      end\n\n      data(\"TMP_DIR without spaces\" => TMP_DIR,\n           \"TMP_DIR with spaces\" => TMP_DIR_WITH_SPACES)\n      test 'parses @include / include correctly' do |data|\n        prepare_config(data)\n        c = read_config(\"#{data}/config_test_1.conf\")\n        assert_equal('root_config', c['k1'])\n        assert_equal('relative_path_include', c['k2'])\n        assert_equal('relative_include_in_included_file', c['k3'])\n        assert_equal('absolute_path_include', c['k4'])\n        assert_equal('uri_include', c['k5'])\n        assert_equal('wildcard_include_1', c['k6'])\n        assert_equal('wildcard_include_2', c['k7'])\n        assert_equal('wildcard_include_3', c['k8'])\n        assert_equal([\n            'k1',\n            'k2',\n            'k3',\n            'k4',\n            'k5',\n            'k8', # Because of the file name this comes first.\n            'k6',\n            'k7',\n          ], c.keys)\n\n        elem1 = c.elements.find { |e| e.name == 'elem1' }\n        assert(elem1)\n        assert_equal('name', elem1.arg)\n        assert_equal('normal_parameter', elem1['include'])\n\n        elem2 = c.elements.find { |e| e.name == 'elem2' }\n        assert(elem2)\n        assert_equal('name', elem2.arg)\n        assert_equal('embedded', elem2['k9'])\n        assert_not_include(elem2, 'include')\n\n        elem3 = elem2.elements.find { |e| e.name == 'elem3' }\n        assert(elem3)\n        assert_equal('nested_value', elem3['nested'])\n        assert_equal('hoge', elem3['include'])\n      end\n\n      # TODO: Add uri based include spec\n    end\n\n    sub_test_case '#to_s' do\n      test 'parses dumpped configuration' do\n        original = %q!a\\\\\\n\\r\\f\\b'\"z!\n        expected = %q!a\\\\\\n\\r\\f\\b'\"z!\n\n        conf = parse_text(%[k1 #{original}])\n        assert_equal(expected, conf['k1']) # escape check\n        conf2 = parse_text(conf.to_s) # use dumpped configuration to check unescape\n        assert_equal(expected, conf2.elements.first['k1'])\n      end\n\n      test 'all types' do\n        conf = parse_text(%[\n          param_string \"value\"\n          param_enum foo\n          param_integer 999\n          param_float 55.55\n          param_size 4k\n          param_bool true\n          param_time 10m\n          param_hash { \"key1\": \"value1\", \"key2\": 2 }\n          param_array [\"value1\", \"value2\", 100]\n          param_regexp /pattern/\n        ])\n        target = AllTypes.new.configure(conf)\n        assert_equal(conf.to_s, target.config.to_s)\n        expected = <<DUMP\n<ROOT>\n  param_string \"value\"\n  param_enum foo\n  param_integer 999\n  param_float 55.55\n  param_size 4k\n  param_bool true\n  param_time 10m\n  param_hash {\"key1\":\"value1\",\"key2\":2}\n  param_array [\"value1\",\"value2\",100]\n  param_regexp /pattern/\n</ROOT>\nDUMP\n        assert_equal(expected, conf.to_s)\n      end\n    end\n  end\n\n  class TestV0Parser < ::Test::Unit::TestCase\n    def parse_text(text)\n      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')\n      Fluent::Config::Parser.parse(StringIO.new(text), '(test)', basepath)\n    end\n\n    sub_test_case \"Fluent::Config::Element#to_s\" do\n      test 'all types' do\n        conf = parse_text(%[\n          param_string value\n          param_enum foo\n          param_integer 999\n          param_float 55.55\n          param_size 4k\n          param_bool true\n          param_time 10m\n          param_hash { \"key1\": \"value1\", \"key2\": 2 }\n          param_array [\"value1\", \"value2\", 100]\n          param_regexp /pattern/\n        ])\n        target = AllTypes.new.configure(conf)\n        assert_equal(conf.to_s, target.config.to_s)\n        expected = <<DUMP\n<ROOT>\n  param_string value\n  param_enum foo\n  param_integer 999\n  param_float 55.55\n  param_size 4k\n  param_bool true\n  param_time 10m\n  param_hash { \"key1\": \"value1\", \"key2\": 2 }\n  param_array [\"value1\", \"value2\", 100]\n  param_regexp /pattern/\n</ROOT>\nDUMP\n        assert_equal(expected, conf.to_s)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/config/test_configurable.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/configurable'\nrequire 'fluent/config/element'\nrequire 'fluent/config/section'\n\nmodule ConfigurableSpec\n  class Base1\n    include Fluent::Configurable\n\n    config_param :node, :string, default: \"node\"\n    config_param :flag1, :bool, default: false\n    config_param :flag2, :bool, default: true\n\n    config_param :name1, :string\n    config_param :name2, :string\n    config_param :name3, :string, default: \"base1\"\n    config_param :name4, :string, default: \"base1\"\n\n    config_param :opt1, :enum, list: [:foo, :bar, :baz]\n    config_param :opt2, :enum, list: [:foo, :bar, :baz], default: :foo\n\n    def get_all\n      [@node, @flag1, @flag2, @name1, @name2, @name3, @name4]\n    end\n  end\n\n  class Base1Safe < Base1\n    config_set_default :name1, \"basex1\"\n    config_set_default :name2, \"basex2\"\n    config_set_default :opt1, :baz\n  end\n\n  class Base1Nil < Base1\n    config_set_default :name1, nil\n    config_set_default :name2, nil\n    config_set_default :opt1, nil\n    config_param :name5, :string, default: nil\n  end\n\n  class Base2 < Base1\n    config_set_default :name2, \"base2\"\n    config_set_default :name4, \"base2\"\n    config_set_default :opt1, :bar\n    config_param :name5, :string\n    config_param :name6, :string, default: \"base2\"\n    config_param :opt3, :enum, list: [:a, :b]\n\n    def get_all\n      ary = super\n      ary + [@name5, @name6]\n    end\n  end\n\n  class Base3 < Base2\n    config_set_default :opt3, :a\n    config_section :node do\n      config_param :name, :string, default: \"node\"\n      config_param :type, :string\n    end\n    config_section :branch, required: true, multi: true do\n      config_argument :name, :string\n      config_param :size, :integer, default: 10\n      config_section :leaf, required: false, multi: true do\n        config_param :weight, :integer\n        config_section :worm, param_name: 'worms', multi: true do\n          config_param :type, :string, default: 'ladybird'\n        end\n      end\n    end\n\n    def get_all\n      ary = super\n      ary + [@branch]\n    end\n  end\n\n  class Base4 < Base2\n    config_set_default :opt3, :a\n    config_section :node, param_name: :nodes do\n      config_argument :num, :integer\n      config_param :name, :string, default: \"node\"\n      config_param :type, :string, default: \"b4\"\n    end\n    config_section :description1, required: false, multi: false do\n      config_argument :note, :string, default: \"desc1\"\n      config_param :text, :string\n    end\n    config_section :description2, required: true, multi: false do\n      config_argument :note, :string, default: \"desc2\"\n      config_param :text, :string\n    end\n    config_section :description3, required: true, multi: true do\n      config_argument :note, default: \"desc3\" do |val|\n        \"desc3: #{val}\"\n      end\n      config_param :text, :string\n    end\n\n    def get_all\n      ary = super\n      ary + [@nodes, @description1, @description2, @description3]\n    end\n  end\n\n  class Base4Safe < Base4\n    # config_section :node, param_name: :nodes do\n    #   config_argument :num, :integer\n    #   config_param :name, :string, default: \"node\"\n    #   config_param :type, :string, default: \"b4\"\n    # end\n    # config_section :description1, required: false, multi: false do\n    #   config_argument :note, :string, default: \"desc1\"\n    #   config_param :text, :string\n    # end\n    # config_section :description2, required: true, multi: false do\n    #   config_argument :note, :string, default: \"desc2\"\n    #   config_param :text, :string\n    # end\n    # config_section :description3, required: true, multi: true do\n    #   config_argument :note, default: \"desc3\" do |val|\n    #     \"desc3: #{val}\"\n    #   end\n    #   config_param :text, :string\n    # end\n    config_section :node do\n      config_set_default :num, 0\n    end\n    config_section :description1 do\n      config_set_default :text, \"teeeext\"\n    end\n    config_section :description2 do\n      config_set_default :text, nil\n    end\n    config_section :description3 do\n      config_set_default :text, \"yay\"\n    end\n  end\n\n  class Init0\n    include Fluent::Configurable\n    config_section :sec1, init: true, multi: false do\n      config_param :name, :string, default: 'sec1'\n    end\n    config_section :sec2, init: true, multi: true do\n      config_param :name, :string, default: 'sec1'\n    end\n  end\n\n  class Example0\n    include Fluent::Configurable\n\n    config_param :stringvalue, :string\n    config_param :boolvalue, :bool\n    config_param :integervalue, :integer\n    config_param :sizevalue, :size\n    config_param :timevalue, :time\n    config_param :floatvalue, :float\n    config_param :hashvalue, :hash\n    config_param :arrayvalue, :array\n  end\n\n  class ExampleWithAlias\n    include Fluent::Configurable\n\n    config_param :name, :string, alias: :fullname\n    config_param :bool, :bool, alias: :flag\n    config_section :detail, required: false, multi: false, alias: \"information\" do\n      config_param :address, :string, default: \"x\"\n    end\n\n    def get_all\n      [@name, @detail]\n    end\n  end\n\n  class ExampleWithSecret\n    include Fluent::Configurable\n\n    config_param :normal_param, :string\n    config_param :secret_param, :string, secret: true\n\n    config_section :section  do\n      config_param :normal_param2, :string\n      config_param :secret_param2, :string, secret: true\n    end\n  end\n\n  class ExampleWithDefaultHashAndArray\n    include Fluent::Configurable\n    config_param :obj1, :hash, default: {}\n    config_param :obj2, :array, default: []\n  end\n\n  class ExampleWithSkipAccessor\n    include Fluent::Configurable\n    config_param :name, :string, default: 'example7', skip_accessor: true\n  end\n\n  class ExampleWithCustomSection\n    include Fluent::Configurable\n    config_param :name_param, :string\n    config_section :normal_section  do\n      config_param :normal_section_param, :string\n    end\n\n    class CustomSection\n      include Fluent::Configurable\n      config_param :custom_section_param, :string\n    end\n\n    class AnotherElement\n      include Fluent::Configurable\n    end\n\n    def configure(conf)\n      super\n      conf.elements.each do |e|\n        next if e.name != 'custom_section'\n        CustomSection.new.configure(e)\n      end\n    end\n  end\n\n  class ExampleWithIntFloat\n    include Fluent::Configurable\n    config_param :int1, :integer\n    config_param :float1, :float\n  end\n\n  module Overwrite\n    class Base\n      include Fluent::Configurable\n\n      config_param :name, :string, alias: :fullname\n      config_param :bool, :bool, alias: :flag\n      config_section :detail, required: false, multi: false, alias: \"information\" do\n        config_param :address, :string, default: \"x\"\n      end\n    end\n\n    class Required < Base\n      config_section :detail, required: true do\n        config_param :address, :string, default: \"x\"\n      end\n    end\n\n    class Multi < Base\n      config_section :detail, multi: true do\n        config_param :address, :string, default: \"x\"\n      end\n    end\n\n    class Alias < Base\n      config_section :detail, alias: \"information2\" do\n        config_param :address, :string, default: \"x\"\n      end\n    end\n\n    class DefaultOptions < Base\n      config_section :detail do\n        config_param :address, :string, default: \"x\"\n      end\n    end\n\n    class DetailAddressDefault < Base\n      config_section :detail do\n        config_param :address, :string, default: \"y\"\n      end\n    end\n\n    class AddParam < Base\n      config_section :detail do\n        config_param :phone_no, :string\n      end\n    end\n\n    class AddParamOverwriteAddress < Base\n      config_section :detail do\n        config_param :address, :string, default: \"y\"\n        config_param :phone_no, :string\n      end\n    end\n  end\n\n  module Final\n    # Show what is allowed in finalized sections\n    # InheritsFinalized < Finalized < Base\n    class Base\n      include Fluent::Configurable\n      config_section :appendix, multi: false, final: false do\n        config_param :code, :string\n        config_param :name, :string\n        config_param :address, :string, default: \"\"\n      end\n    end\n\n    class Finalized < Base\n      # to non-finalized section\n      # subclass can change type (code)\n      #          add default value (name)\n      #          change default value (address)\n      #          add field (age)\n      config_section :appendix, final: true do\n        config_param :code, :integer\n        config_set_default :name, \"y\"\n        config_set_default :address, \"-\"\n        config_param :age, :integer, default: 10\n      end\n    end\n\n    class InheritsFinalized < Finalized\n      # to finalized section\n      # subclass can add default value (code)\n      #              change default value (age)\n      #              add field (phone_no)\n      config_section :appendix do\n        config_set_default :code, 2\n        config_set_default :age, 0\n        config_param :phone_no, :string\n      end\n    end\n\n    # Show what is allowed/prohibited for finalized sections\n    class FinalizedBase\n      include Fluent::Configurable\n      config_section :appendix, param_name: :apd, init: false, required: true, multi: false, alias: \"options\", final: true do\n        config_param :name, :string\n      end\n    end\n\n    class FinalizedBase2\n      include Fluent::Configurable\n      config_section :appendix, param_name: :apd, init: false, required: false, multi: false, alias: \"options\", final: true do\n        config_param :name, :string\n      end\n    end\n\n    # subclass can change init with adding default values\n    class OverwriteInit < FinalizedBase2\n      config_section :appendix, init: true do\n        config_set_default :name, \"moris\"\n        config_param :code, :integer, default: 0\n      end\n    end\n\n    # subclass cannot change type (name)\n    class Subclass < FinalizedBase\n      config_section :appendix do\n        config_param :name, :integer\n      end\n    end\n\n    # subclass cannot change param_name\n    class OverwriteParamName < FinalizedBase\n      config_section :appendix, param_name: :adx do\n      end\n    end\n\n    # subclass cannot change final (section)\n    class OverwriteFinal < FinalizedBase\n      config_section :appendix, final: false do\n        config_param :name, :integer\n      end\n    end\n\n    # subclass cannot change required\n    class OverwriteRequired < FinalizedBase\n      config_section :appendix, required: false do\n      end\n    end\n\n    # subclass cannot change multi\n    class OverwriteMulti < FinalizedBase\n      config_section :appendix, multi: true do\n      end\n    end\n\n    # subclass cannot change alias\n    class OverwriteAlias < FinalizedBase\n      config_section :appendix, alias: \"options2\" do\n      end\n    end\n  end\n\n  module OverwriteDefaults\n    class Owner\n      include Fluent::Configurable\n      config_set_default :key1, \"V1\"\n      config_section :buffer do\n        config_set_default :size_of_something, 1024\n      end\n    end\n\n    class SubOwner < Owner\n      config_section :buffer do\n        config_set_default :size_of_something, 2048\n      end\n    end\n\n    class NilOwner < Owner\n      config_section :buffer do\n        config_set_default :size_of_something, nil\n      end\n    end\n\n    class FlatChild\n      include Fluent::Configurable\n      attr_accessor :owner\n      config_param :key1, :string, default: \"v1\"\n    end\n\n    class BufferChild\n      include Fluent::Configurable\n      attr_accessor :owner\n      configured_in :buffer\n      config_param :size_of_something, :size, default: 128\n    end\n\n    class BufferBase\n      include Fluent::Configurable\n    end\n\n    class BufferSubclass < BufferBase\n      attr_accessor :owner\n      configured_in :buffer\n      config_param :size_of_something, :size, default: 512\n    end\n\n    class BufferSubSubclass < BufferSubclass\n    end\n  end\n  class UnRecommended\n    include Fluent::Configurable\n    attr_accessor :log\n    config_param :key1, :string, default: 'deprecated', deprecated: \"key1 will be removed.\"\n    config_param :key2, :string, default: 'obsoleted', obsoleted: \"key2 has been removed.\"\n  end\nend\n\nmodule Fluent::Config\n  class TestConfigurable < ::Test::Unit::TestCase\n    sub_test_case 'class defined without config_section' do\n      sub_test_case '#initialize' do\n        test 'create instance methods and default values by config_param and config_set_default' do\n          obj1 = ConfigurableSpec::Base1.new\n          assert_equal(\"node\", obj1.node)\n          assert_false(obj1.flag1)\n          assert_true(obj1.flag2)\n          assert_nil(obj1.name1)\n          assert_nil(obj1.name2)\n          assert_equal(\"base1\", obj1.name3)\n          assert_equal(\"base1\", obj1.name4)\n          assert_nil(obj1.opt1)\n          assert_equal(:foo, obj1.opt2)\n        end\n\n        test 'create instance methods and default values overwritten by sub class definition' do\n          obj2 = ConfigurableSpec::Base2.new\n          assert_equal(\"node\", obj2.node)\n          assert_false(obj2.flag1)\n          assert_true(obj2.flag2)\n          assert_nil(obj2.name1)\n          assert_equal(\"base2\", obj2.name2)\n          assert_equal(\"base1\", obj2.name3)\n          assert_equal(\"base2\", obj2.name4)\n          assert_nil(obj2.name5)\n          assert_equal(\"base2\", obj2.name6)\n          assert_equal(:bar, obj2.opt1)\n          assert_equal(:foo, obj2.opt2)\n        end\n      end\n\n      sub_test_case '#configured_section_create' do\n        test 'raises configuration error if required param exists but no configuration element is specified' do\n          obj = ConfigurableSpec::Base1.new\n          assert_raise(Fluent::ConfigError.new(\"'name1' parameter is required\")) do\n            obj.configured_section_create(nil)\n          end\n        end\n\n        test 'creates root section with default values if name and config are specified with nil' do\n          obj = ConfigurableSpec::Base1Safe.new\n          root = obj.configured_section_create(nil)\n\n          assert_equal \"node\", root.node\n          assert_false root.flag1\n          assert_true  root.flag2\n          assert_equal \"basex1\", root.name1\n          assert_equal \"basex2\", root.name2\n          assert_equal \"base1\", root.name3\n          assert_equal \"base1\", root.name4\n          assert_equal :baz, root.opt1\n          assert_equal :foo, root.opt2\n        end\n\n        test 'creates root section with default values if name is nil and config is empty element' do\n          obj = ConfigurableSpec::Base1Safe.new\n          root = obj.configured_section_create(nil, config_element())\n\n          assert_equal \"node\", root.node\n          assert_false root.flag1\n          assert_true  root.flag2\n          assert_equal \"basex1\", root.name1\n          assert_equal \"basex2\", root.name2\n          assert_equal \"base1\", root.name3\n          assert_equal \"base1\", root.name4\n          assert_equal :baz, root.opt1\n          assert_equal :foo, root.opt2\n        end\n\n        test 'creates root section with specified value if name is nil and configuration element is specified' do\n          obj = ConfigurableSpec::Base1Safe.new\n          root = obj.configured_section_create(nil, config_element('match', '', {'node' => \"nodename\", 'flag1' => 'true', 'name1' => 'fixed1', 'opt1' => 'foo'}))\n\n          assert_equal \"nodename\", root.node\n          assert_equal \"fixed1\", root.name1\n          assert_true root.flag1\n          assert_equal :foo, root.opt1\n\n          assert_true  root.flag2\n          assert_equal \"basex2\", root.name2\n          assert_equal \"base1\", root.name3\n          assert_equal \"base1\", root.name4\n          assert_equal :foo, root.opt2\n        end\n      end\n\n      sub_test_case '#configure' do\n        test 'returns configurable object itself' do\n          b2 = ConfigurableSpec::Base2.new\n          assert_instance_of(ConfigurableSpec::Base2, b2.configure(config_element(\"\", \"\", {\"name1\" => \"t1\", \"name5\" => \"t5\", \"opt3\" => \"a\"})))\n        end\n\n        test 'can accept frozen string' do\n          b2 = ConfigurableSpec::Base2.new\n          assert_instance_of(ConfigurableSpec::Base2, b2.configure(config_element(\"\", \"\", {\"name1\" => \"t1\".freeze, \"name5\" => \"t5\", \"opt3\" => \"a\"})))\n        end\n\n        test 'raise errors without any specifications for param without defaults' do\n          b2 = ConfigurableSpec::Base2.new\n          assert_raise(Fluent::ConfigError) { b2.configure(config_element(\"\", \"\", {})) }\n          assert_raise(Fluent::ConfigError) { b2.configure(config_element(\"\", \"\", {\"name1\" => \"t1\"})) }\n          assert_raise(Fluent::ConfigError) { b2.configure(config_element(\"\", \"\", {\"name5\" => \"t5\"})) }\n          assert_raise(Fluent::ConfigError) { b2.configure(config_element(\"\", \"\", {\"name1\" => \"t1\", \"name5\" => \"t5\"})) }\n          assert_nothing_raised { b2.configure(config_element(\"\", \"\", {\"name1\" => \"t1\", \"name5\" => \"t5\", \"opt3\" => \"a\"})) }\n\n          assert_equal([\"node\", false, true, \"t1\", \"base2\", \"base1\", \"base2\", \"t5\", \"base2\"], b2.get_all)\n          assert_equal(:a, b2.opt3)\n        end\n\n        test 'can configure bool values' do\n          b2a = ConfigurableSpec::Base2.new\n          assert_nothing_raised { b2a.configure(config_element(\"\", \"\", {\"flag1\" => \"true\", \"flag2\" => \"yes\", \"name1\" => \"t1\", \"name5\" => \"t5\", \"opt3\" => \"a\"})) }\n          assert_true(b2a.flag1)\n          assert_true(b2a.flag2)\n\n          b2b = ConfigurableSpec::Base2.new\n          assert_nothing_raised { b2b.configure(config_element(\"\", \"\", {\"flag1\" => false, \"flag2\" => \"no\", \"name1\" => \"t1\", \"name5\" => \"t5\", \"opt3\" => \"a\"})) }\n          assert_false(b2b.flag1)\n          assert_false(b2b.flag2)\n        end\n\n        test 'overwrites values of defaults' do\n          b2 = ConfigurableSpec::Base2.new\n          b2.configure(config_element(\"\", \"\", {\"name1\" => \"t1\", \"name2\" => \"t2\", \"name3\" => \"t3\", \"name4\" => \"t4\", \"name5\" => \"t5\", \"opt1\" => \"foo\", \"opt3\" => \"b\"}))\n          assert_equal(\"t1\", b2.name1)\n          assert_equal(\"t2\", b2.name2)\n          assert_equal(\"t3\", b2.name3)\n          assert_equal(\"t4\", b2.name4)\n          assert_equal(\"t5\", b2.name5)\n          assert_equal(\"base2\", b2.name6)\n          assert_equal(:foo, b2.opt1)\n          assert_equal(:b, b2.opt3)\n\n          assert_equal([\"node\", false, true, \"t1\", \"t2\", \"t3\", \"t4\", \"t5\", \"base2\"], b2.get_all)\n        end\n\n        test 'enum type rejects values which does not exist in list' do\n          default = config_element(\"\", \"\", {\"name1\" => \"t1\", \"name2\" => \"t2\", \"name3\" => \"t3\", \"name4\" => \"t4\", \"name5\" => \"t5\", \"opt1\" => \"foo\", \"opt3\" => \"b\"})\n\n          b2 = ConfigurableSpec::Base2.new\n          assert_nothing_raised { b2.configure(default) }\n          assert_raise(Fluent::ConfigError) { b2.configure(default.merge({\"opt1\" => \"bazz\"})) }\n          assert_raise(Fluent::ConfigError) { b2.configure(default.merge({\"opt2\" => \"fooooooo\"})) }\n          assert_raise(Fluent::ConfigError) { b2.configure(default.merge({\"opt3\" => \"c\"})) }\n        end\n\n        sub_test_case 'default values should be duplicated before touched in plugin code' do\n          test 'default object should be dupped for cases configured twice' do\n            x6a = ConfigurableSpec::ExampleWithDefaultHashAndArray.new\n            assert_nothing_raised { x6a.configure(config_element(\"\")) }\n            assert_equal({}, x6a.obj1)\n            assert_equal([], x6a.obj2)\n\n            x6b = ConfigurableSpec::ExampleWithDefaultHashAndArray.new\n            assert_nothing_raised { x6b.configure(config_element(\"\")) }\n            assert_equal({}, x6b.obj1)\n            assert_equal([], x6b.obj2)\n\n            assert { x6a.obj1.object_id != x6b.obj1.object_id }\n            assert { x6a.obj2.object_id != x6b.obj2.object_id }\n\n            x6c = ConfigurableSpec::ExampleWithDefaultHashAndArray.new\n            assert_nothing_raised { x6c.configure(config_element(\"\")) }\n            assert_equal({}, x6c.obj1)\n            assert_equal([], x6c.obj2)\n\n            x6c.obj1['k'] = 'v'\n            x6c.obj2 << 'v'\n\n            assert_equal({'k' => 'v'}, x6c.obj1)\n            assert_equal(['v'], x6c.obj2)\n\n            assert_equal({}, x6a.obj1)\n            assert_equal([], x6a.obj2)\n          end\n        end\n\n        test 'strict value type' do\n          default = config_element(\"\", \"\", {\"int1\" => \"1\", \"float1\" => \"\"})\n\n          c = ConfigurableSpec::ExampleWithIntFloat.new\n          assert_nothing_raised { c.configure(default) }\n          assert_raise(Fluent::ConfigError) { c.configure(default, true) }\n        end\n      end\n\n        test 'set nil for a parameter which has no default value' do\n          obj = ConfigurableSpec::Base2.new\n          conf = config_element(\"\", \"\", {\"name1\" => nil, \"name5\" => \"t5\", \"opt3\" => \"a\"})\n          assert_raise(Fluent::ConfigError.new(\"'name1' parameter is required but nil is specified\")) do\n            obj.configure(conf)\n          end\n        end\n\n        test 'set nil for a parameter which has non-nil default value' do\n          obj = ConfigurableSpec::Base2.new\n          conf = config_element(\"\", \"\", {\"name1\" => \"t1\", \"name3\" => nil, \"name5\" => \"t5\", \"opt3\" => \"a\"})\n          assert_raise(Fluent::ConfigError.new(\"'name3' parameter is required but nil is specified\")) do\n            obj.configure(conf)\n          end\n        end\n\n        test 'set nil for a parameter whose default value is nil' do\n          obj = ConfigurableSpec::Base1Nil.new\n          conf = config_element(\"\", \"\", {\"name5\" => nil})\n          obj.configure(conf)\n          assert_nil obj.name5\n        end\n\n        test 'set nil for parameters whose default values are overwritten by nil' do\n          obj = ConfigurableSpec::Base1Nil.new\n          conf = config_element(\"\", \"\", {\"name1\" => nil, \"name2\" => nil, \"opt1\" => nil})\n          obj.configure(conf)\n          assert_nil obj.name1\n          assert_nil obj.name2\n          assert_nil obj.opt1\n        end\n\n        test 'set :default' do\n          obj = ConfigurableSpec::Base2.new\n          conf = config_element(\"\", \"\", {\"name1\" => \"t1\", \"name3\" => :default, \"name5\" => \"t5\", \"opt3\" => \"a\"})\n          obj.configure(conf)\n          assert_equal \"base1\", obj.name3\n        end\n\n        test 'set :default for a parameter which has no default value' do\n          obj = ConfigurableSpec::Base2.new\n          conf = config_element(\"\", \"\", {\"name1\" => :default, \"name5\" => \"t5\", \"opt3\" => \"a\"})\n          assert_raise(Fluent::ConfigError.new(\"'name1' doesn't have default value\")) do\n            obj.configure(conf)\n          end\n        end\n\n        test 'set :default for a parameter which has an overwritten default value' do\n          obj = ConfigurableSpec::Base2.new\n          conf = config_element(\"\", \"\", {\"name1\" => \"t1\", \"name3\" => \"t3\", \"name4\" => :default, \"name5\" => \"t5\", \"opt3\" => \"a\"})\n          obj.configure(conf)\n          assert_equal \"base2\", obj.name4\n        end\n    end\n\n    sub_test_case 'class defined with config_section' do\n      sub_test_case '#initialize' do\n        test 'create instance methods and default values as nil for params from config_section specified as non-multi' do\n          b4 = ConfigurableSpec::Base4.new\n          assert_nil(b4.description1)\n          assert_nil(b4.description2)\n        end\n\n        test 'create instance methods and default values as [] for params from config_section specified as multi' do\n          b4 = ConfigurableSpec::Base4.new\n          assert_equal([], b4.description3)\n        end\n\n        test 'overwrite base class definition by config_section of sub class definition' do\n          b3 = ConfigurableSpec::Base3.new\n          assert_equal([], b3.node)\n        end\n\n        test 'create instance methods and default values by param_name' do\n          b4 = ConfigurableSpec::Base4.new\n          assert_equal([], b4.nodes)\n          assert_equal(\"node\", b4.node)\n        end\n\n        test 'create non-required and multi without any specifications' do\n          b3 = ConfigurableSpec::Base3.new\n          assert_false(b3.class.merged_configure_proxy.sections[:node].required?)\n          assert_true(b3.class.merged_configure_proxy.sections[:node].multi?)\n        end\n      end\n\n      sub_test_case '#configured_section_create' do\n        test 'raises configuration error if required param exists but no configuration element is specified' do\n          obj = ConfigurableSpec::Base4.new\n          assert_raise(Fluent::ConfigError.new(\"'<node ARG>' section requires argument\")) do\n            obj.configured_section_create(:node)\n          end\n          assert_raise(Fluent::ConfigError.new(\"'text' parameter is required\")) do\n            obj.configured_section_create(:description1)\n          end\n        end\n\n        test 'creates any defined section with default values if name is nil and config is not specified' do\n          obj = ConfigurableSpec::Base4Safe.new\n          node = obj.configured_section_create(:node)\n          assert_equal 0, node.num\n          assert_equal \"node\", node.name\n          assert_equal \"b4\", node.type\n\n          desc1 = obj.configured_section_create(:description1)\n          assert_equal \"desc1\", desc1.note\n          assert_equal \"teeeext\", desc1.text\n        end\n\n        test 'creates any defined section with default values if name is nil and config is empty element' do\n          obj = ConfigurableSpec::Base4Safe.new\n          node = obj.configured_section_create(:node, config_element())\n          assert_equal 0, node.num\n          assert_equal \"node\", node.name\n          assert_equal \"b4\", node.type\n\n          desc1 = obj.configured_section_create(:description1, config_element())\n          assert_equal \"desc1\", desc1.note\n          assert_equal \"teeeext\", desc1.text\n        end\n\n        test 'creates any defined section with specified value if name is nil and configuration element is specified' do\n          obj = ConfigurableSpec::Base4Safe.new\n          node = obj.configured_section_create(:node, config_element('node', '1', {'name' => 'node1', 'type' => 'b1'}))\n          assert_equal 1, node.num\n          assert_equal \"node1\", node.name\n          assert_equal \"b1\", node.type\n\n          desc1 = obj.configured_section_create(:description1, config_element('description1', 'desc one', {'text' => 't'}))\n          assert_equal \"desc one\", desc1.note\n          assert_equal \"t\", desc1.text\n        end\n\n        test 'creates a defined section instance even if it is defined as multi:true' do\n          obj = ConfigurableSpec::Base4Safe.new\n          desc3 = obj.configured_section_create(:description3)\n          assert_equal \"desc3\", desc3.note\n          assert_equal \"yay\", desc3.text\n\n          desc3 = obj.configured_section_create(:description3, config_element('description3', 'foo'))\n          assert_equal \"desc3: foo\", desc3.note\n          assert_equal \"yay\", desc3.text\n        end\n      end\n\n      sub_test_case '#configure' do\n        BASE_ATTRS = {\n          \"name1\" => \"1\", \"name2\" => \"2\", \"name3\" => \"3\",\n          \"name4\" => \"4\", \"name5\" => \"5\", \"name6\" => \"6\",\n        }\n        test 'checks required subsections' do\n          b3 = ConfigurableSpec::Base3.new\n          # branch sections required\n          assert_raise(Fluent::ConfigError) { b3.configure(config_element('ROOT', '', BASE_ATTRS, [])) }\n\n          # branch argument required\n          msg = \"'<branch ARG>' section requires argument, in section branch\"\n          #expect{ b3.configure(e('ROOT', '', BASE_ATTRS, [e('branch', '')])) }.to raise_error(Fluent::ConfigError, msg)\n          assert_raise(Fluent::ConfigError.new(msg)) { b3.configure(config_element('ROOT', '', BASE_ATTRS, [config_element('branch', '')])) }\n\n          # leaf is not required\n          assert_nothing_raised { b3.configure(config_element('ROOT', '', BASE_ATTRS, [config_element('branch', 'branch_name')])) }\n\n          # leaf weight required\n          msg = \"'weight' parameter is required, in section branch > leaf\"\n          branch1 = config_element('branch', 'branch_name', {size: 1}, [config_element('leaf', '10', {\"weight\" => 1})])\n          assert_nothing_raised { b3.configure(config_element('ROOT', '', BASE_ATTRS, [branch1])) }\n          branch2 = config_element('branch', 'branch_name', {size: 1}, [config_element('leaf', '20')])\n          assert_raise(Fluent::ConfigError.new(msg)) { b3.configure(config_element('ROOT', '', BASE_ATTRS, [branch1, branch2])) }\n          branch3 = config_element('branch', 'branch_name', {size: 1}, [config_element('leaf', '10', {\"weight\" =>  3}), config_element('leaf', '20')])\n          assert_raise(Fluent::ConfigError.new(msg)) { b3.configure(config_element('ROOT', '', BASE_ATTRS, [branch3])) }\n\n          ### worm not required\n\n          b4 = ConfigurableSpec::Base4.new\n\n          d1 = config_element('description1', '', {\"text\" => \"d1\"})\n          d2 = config_element('description2', '', {\"text\" => \"d2\"})\n          d3 = config_element('description3', '', {\"text\" => \"d3\"})\n          assert_nothing_raised { b4.configure(config_element('ROOT', '', BASE_ATTRS, [d1.dup, d2.dup, d3.dup])) }\n\n          # description1 cannot be specified 2 or more\n          msg = \"'<description1>' section cannot be written twice or more\"\n          assert_raise(Fluent::ConfigError.new(msg)) { b4.configure(config_element('ROOT', '', BASE_ATTRS, [d1.dup, d2.dup, d1.dup, d3.dup])) }\n\n          # description2 cannot be specified 2 or more\n          msg = \"'<description2>' section cannot be written twice or more\"\n          assert_raise(Fluent::ConfigError.new(msg)) { b4.configure(config_element('ROOT', '', BASE_ATTRS, [d1.dup, d2.dup, d3.dup, d2.dup])) }\n\n          # description3 can be specified 2 or more\n          assert_nothing_raised { b4.configure(config_element('ROOT', '', BASE_ATTRS, [d1.dup, d2.dup, d3.dup, d3.dup])) }\n        end\n\n        test 'constructs configuration object tree for Base3' do\n          conf = config_element(\n            'ROOT',\n            '',\n            BASE_ATTRS,\n            [\n              config_element('node', '', {\"type\" => \"1\"}), config_element('node', '', {\"name\" => \"node2\",\"type\" => \"2\"}),\n              config_element('branch', 'b1.*', {}, []),\n              config_element('branch',\n                'b2.*',\n                {\"size\" => 5},\n                [\n                  config_element('leaf', 'THIS IS IGNORED', {\"weight\" =>  55}, []),\n                  config_element('leaf', 'THIS IS IGNORED', {\"weight\" =>  50}, [ config_element('worm', '', {}) ]),\n                  config_element('leaf', 'THIS IS IGNORED', {\"weight\" =>  50}, [ config_element('worm', '', {\"type\" => \"w1\"}), config_element('worm', '', {\"type\" => \"w2\"}) ]),\n                ]\n                ),\n              config_element('branch',\n                'b3.*',\n                {\"size\" => \"503\"},\n                [\n                  config_element('leaf', 'THIS IS IGNORED', {\"weight\" =>  55}, []),\n                ]\n                )\n            ],\n            )\n          b3 = ConfigurableSpec::Base3.new.configure(conf)\n\n          assert_not_equal(\"node\", b3.node) # overwritten\n\n          assert_equal(\"1\", b3.name1)\n          assert_equal(\"2\", b3.name2)\n          assert_equal(\"3\", b3.name3)\n          assert_equal(\"4\", b3.name4)\n          assert_equal(\"5\", b3.name5)\n          assert_equal(\"6\", b3.name6)\n\n          assert_instance_of(Array, b3.node)\n          assert_equal(2, b3.node.size)\n\n          assert_equal(\"node\", b3.node[0].name)\n          assert_equal(\"1\", b3.node[0].type)\n          assert_equal(b3.node[0].type, b3.node[0][:type])\n          assert_equal(\"node2\", b3.node[1].name)\n          assert_equal(\"2\", b3.node[1].type)\n          assert_equal(b3.node[1].type, b3.node[1][:type])\n\n          assert_instance_of(Array, b3.branch)\n          assert_equal(3, b3.branch.size)\n\n          assert_equal('b1.*', b3.branch[0].name)\n          assert_equal(10, b3.branch[0].size)\n          assert_equal([], b3.branch[0].leaf)\n\n          assert_equal('b2.*', b3.branch[1].name)\n          assert_equal(5, b3.branch[1].size)\n          assert_equal(3, b3.branch[1].leaf.size)\n          assert_equal(b3.branch[1].leaf, b3.branch[1][:leaf])\n\n          assert_equal(55, b3.branch[1].leaf[0].weight)\n          assert_equal(0, b3.branch[1].leaf[0].worms.size)\n\n          assert_equal(50, b3.branch[1].leaf[1].weight)\n          assert_equal(1, b3.branch[1].leaf[1].worms.size)\n          assert_equal(\"ladybird\", b3.branch[1].leaf[1].worms[0].type)\n\n          assert_equal(50, b3.branch[1].leaf[2].weight)\n          assert_equal(2, b3.branch[1].leaf[2].worms.size)\n          assert_equal(\"w1\", b3.branch[1].leaf[2].worms[0].type)\n          assert_equal(\"w2\", b3.branch[1].leaf[2].worms[1].type)\n\n          assert_equal('b3.*', b3.branch[2].name)\n          assert_equal(503, b3.branch[2].size)\n          assert_equal(1, b3.branch[2].leaf.size)\n          assert_equal(55, b3.branch[2].leaf[0].weight)\n        end\n\n        test 'constructs configuration object tree for Base4' do\n          conf = config_element(\n            'ROOT',\n            '',\n            BASE_ATTRS,\n            [\n              config_element('node', '1', {\"type\" => \"1\"}), config_element('node', '2', {\"name\" => \"node2\"}),\n              config_element('description3', '', {\"text\" => \"dddd3-1\"}),\n              config_element('description2', 'd-2', {\"text\" => \"dddd2\"}),\n              config_element('description1', '', {\"text\" => \"dddd1\"}),\n              config_element('description3', 'd-3', {\"text\" => \"dddd3-2\"}),\n              config_element('description3', 'd-3a', {\"text\" => \"dddd3-3\"}),\n              config_element('node', '4', {\"type\" => \"four\"}),\n            ],\n            )\n          b4 = ConfigurableSpec::Base4.new.configure(conf)\n\n          assert_equal(\"node\", b4.node)\n\n          assert_equal(\"1\", b4.name1)\n          assert_equal(\"2\", b4.name2)\n          assert_equal(\"3\", b4.name3)\n          assert_equal(\"4\", b4.name4)\n          assert_equal(\"5\", b4.name5)\n          assert_equal(\"6\", b4.name6)\n\n          assert_instance_of(Array, b4.nodes)\n          assert_equal(3, b4.nodes.size)\n          assert_equal(1, b4.nodes[0].num)\n          assert_equal(\"node\", b4.nodes[0].name)\n          assert_equal(\"1\", b4.nodes[0].type)\n          assert_equal(2, b4.nodes[1].num)\n          assert_equal(\"node2\", b4.nodes[1].name)\n          assert_equal(\"b4\", b4.nodes[1].type)\n          assert_equal(4, b4.nodes[2].num)\n          assert_equal(\"node\", b4.nodes[2].name)\n          assert_equal(\"four\", b4.nodes[2].type)\n\n          # config_element('description3', '', {\"text\" => \"dddd3-1\"}),\n          # config_element('description3', 'd-3', {\"text\" => \"dddd3-2\"}),\n          # config_element('description3', 'd-3a', {\"text\" => \"dddd3-3\"}),\n\n          # NoMethodError: undefined method `class' for <Fluent::Config::Section {...}>:Fluent::Config::Section occurred. Should we add class method to Section?\n          #assert_equal('Fluent::Config::Section', b4.description1.class.name)\n          assert_equal(\"desc1\", b4.description1.note)\n          assert_equal(\"dddd1\", b4.description1.text)\n\n          # same with assert_equal('Fluent::Config::Section', b4.description1)\n          #assert_equal('Fluent::Config::Section', b4.description2)\n          assert_equal(\"d-2\", b4.description2.note)\n          assert_equal(\"dddd2\", b4.description2.text)\n\n          assert_instance_of(Array, b4.description3)\n          assert_equal(3, b4.description3.size)\n          assert_equal(\"desc3\", b4.description3[0].note)\n          assert_equal(\"dddd3-1\", b4.description3[0].text)\n          assert_equal('desc3: d-3', b4.description3[1].note)\n          assert_equal('dddd3-2', b4.description3[1].text)\n          assert_equal('desc3: d-3a', b4.description3[2].note)\n          assert_equal('dddd3-3', b4.description3[2].text)\n        end\n\n        test 'checks missing of specifications' do\n          conf0 = config_element('ROOT', '', {}, [])\n          ex01 = ConfigurableSpec::Example0.new\n          assert_raise(Fluent::ConfigError) { ex01.configure(conf0) }\n\n          complete = config_element('ROOT', '', {\n            \"stringvalue\" => \"s1\", \"boolvalue\" => \"yes\", \"integervalue\" => \"10\",\n            \"sizevalue\" => \"10m\", \"timevalue\" => \"100s\", \"floatvalue\" => \"1.001\",\n            \"hashvalue\" => '{\"foo\":1, \"bar\":2}',\n            \"arrayvalue\" => '[1,\"ichi\"]',\n          })\n\n          checker = lambda { |conf| ConfigurableSpec::Example0.new.configure(conf) }\n\n          assert_nothing_raised { checker.call(complete) }\n          assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete(\"stringvalue\"); checker.call(c) }\n          assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete(\"boolvalue\"); checker.call(c) }\n          assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete(\"integervalue\"); checker.call(c) }\n          assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete(\"sizevalue\"); checker.call(c) }\n          assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete(\"timevalue\"); checker.call(c) }\n          assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete(\"floatvalue\"); checker.call(c) }\n          assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete(\"hashvalue\"); checker.call(c) }\n          assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete(\"arrayvalue\"); checker.call(c) }\n        end\n\n        test 'generates section with default values for init:true sections' do\n          conf = config_element('ROOT', '', {}, [])\n          init0 = ConfigurableSpec::Init0.new\n          assert_nothing_raised { init0.configure(conf) }\n          assert init0.sec1\n          assert_equal \"sec1\", init0.sec1.name\n          assert_equal 1, init0.sec2.size\n          assert_equal \"sec1\", init0.sec2.first.name\n        end\n\n        test 'accepts configuration values as string representation' do\n          conf = config_element('ROOT', '', {\n            \"stringvalue\" => \"s1\", \"boolvalue\" => \"yes\", \"integervalue\" => \"10\",\n            \"sizevalue\" => \"10m\", \"timevalue\" => \"10m\", \"floatvalue\" => \"1.001\",\n            \"hashvalue\" => '{\"foo\":1, \"bar\":2}',\n            \"arrayvalue\" => '[1,\"ichi\"]',\n          })\n          ex = ConfigurableSpec::Example0.new.configure(conf)\n          assert_equal(\"s1\", ex.stringvalue)\n          assert_true(ex.boolvalue)\n          assert_equal(10, ex.integervalue)\n          assert_equal(10 * 1024 * 1024, ex.sizevalue)\n          assert_equal(10 * 60, ex.timevalue)\n          assert_equal(1.001, ex.floatvalue)\n          assert_equal({\"foo\" => 1, \"bar\" => 2}, ex.hashvalue)\n          assert_equal([1, \"ichi\"], ex.arrayvalue)\n        end\n\n        test 'accepts configuration values as ruby value representation (especially for DSL)' do\n          conf = config_element('ROOT', '', {\n            \"stringvalue\" => \"s1\", \"boolvalue\" => true, \"integervalue\" => 10,\n            \"sizevalue\" => 10 * 1024 * 1024, \"timevalue\" => 10 * 60, \"floatvalue\" => 1.001,\n            \"hashvalue\" => {\"foo\" => 1, \"bar\" => 2},\n            \"arrayvalue\" => [1,\"ichi\"],\n          })\n          ex = ConfigurableSpec::Example0.new.configure(conf)\n          assert_equal(\"s1\", ex.stringvalue)\n          assert_true(ex.boolvalue)\n          assert_equal(10, ex.integervalue)\n          assert_equal(10 * 1024 * 1024, ex.sizevalue)\n          assert_equal(10 * 60, ex.timevalue)\n          assert_equal(1.001, ex.floatvalue)\n          assert_equal({\"foo\" => 1, \"bar\" => 2}, ex.hashvalue)\n          assert_equal([1, \"ichi\"], ex.arrayvalue)\n        end\n\n        test 'gets both of true(yes) and false(no) for bool value parameter' do\n          conf = config_element('ROOT', '', {\n            \"stringvalue\" => \"s1\", \"integervalue\" => 10,\n            \"sizevalue\" => 10 * 1024 * 1024, \"timevalue\" => 10 * 60, \"floatvalue\" => 1.001,\n            \"hashvalue\" => {\"foo\" => 1, \"bar\" => 2},\n            \"arrayvalue\" => [1,\"ichi\"],\n          })\n          ex0 = ConfigurableSpec::Example0.new.configure(conf.merge({\"boolvalue\" => \"true\"}))\n          assert_true(ex0.boolvalue)\n\n          ex1 = ConfigurableSpec::Example0.new.configure(conf.merge({\"boolvalue\" => \"yes\"}))\n          assert_true(ex1.boolvalue)\n\n          ex2 = ConfigurableSpec::Example0.new.configure(conf.merge({\"boolvalue\" => true}))\n          assert_true(ex2.boolvalue)\n\n          ex3 = ConfigurableSpec::Example0.new.configure(conf.merge({\"boolvalue\" => \"false\"}))\n          assert_false(ex3.boolvalue)\n\n          ex4 = ConfigurableSpec::Example0.new.configure(conf.merge({\"boolvalue\" => \"no\"}))\n          assert_false(ex4.boolvalue)\n\n          ex5 = ConfigurableSpec::Example0.new.configure(conf.merge({\"boolvalue\" => false}))\n          assert_false(ex5.boolvalue)\n        end\n      end\n\n      sub_test_case '.config_section' do\n        CONF1 = config_element('ROOT', '', {\n                                 'name' => 'tagomoris',\n                                 'bool' => true,\n                               })\n\n        CONF2 = config_element('ROOT', '', {\n                                 'name' => 'tagomoris',\n                                 'bool' => true,\n                               },\n                               [config_element('detail', '', { 'phone_no' => \"+81-00-0000-0000\" }, [])])\n\n        CONF3 = config_element('ROOT', '', {\n                                 'name' => 'tagomoris',\n                                 'bool' => true,\n                               },\n                               [config_element('detail', '', { 'address' => \"Chiyoda Tokyo Japan\" }, [])])\n\n        CONF4 = config_element('ROOT', '', {\n                                 'name' => 'tagomoris',\n                                 'bool' => true,\n                               },\n                               [\n                                 config_element('detail', '', {\n                                                  'address' => \"Chiyoda Tokyo Japan\",\n                                                  'phone_no' => '+81-00-0000-0000'\n                                                },\n                                                [])\n                               ])\n\n        data(conf1: CONF1,\n             conf2: CONF2,\n             conf3: CONF3,\n             conf4: CONF4,)\n        test 'base class' do |data|\n          assert_nothing_raised { ConfigurableSpec::Overwrite::Base.new.configure(data) }\n        end\n\n        test 'subclass cannot overwrite required' do\n          assert_raise(Fluent::ConfigError.new(\"BUG: subclass cannot overwrite base class's config_section: required\")) do\n            ConfigurableSpec::Overwrite::Required.new.configure(CONF1)\n          end\n        end\n\n        test 'subclass cannot overwrite multi' do\n          assert_raise(Fluent::ConfigError.new(\"BUG: subclass cannot overwrite base class's config_section: multi\")) do\n            ConfigurableSpec::Overwrite::Multi.new.configure(CONF1)\n          end\n        end\n\n        test 'subclass cannot overwrite alias' do\n          assert_raise(Fluent::ConfigError.new(\"BUG: subclass cannot overwrite base class's config_section: alias\")) do\n            ConfigurableSpec::Overwrite::Alias.new.configure(CONF1)\n          end\n        end\n\n        test 'subclass uses superclass default options' do\n          base = ConfigurableSpec::Overwrite::Base.new.configure(CONF2)\n          sub = ConfigurableSpec::Overwrite::DefaultOptions.new.configure(CONF2)\n          detail_base = base.class.merged_configure_proxy.sections[:detail]\n          detail_sub = sub.class.merged_configure_proxy.sections[:detail]\n          detail_base_attributes = {\n            required: detail_base.required,\n            multi: detail_base.multi,\n            alias: detail_base.alias,\n          }\n          detail_sub_attributes = {\n            required: detail_sub.required,\n            multi: detail_sub.multi,\n            alias: detail_sub.alias,\n          }\n          assert_equal(detail_base_attributes, detail_sub_attributes)\n        end\n\n        test 'subclass can overwrite detail.address' do\n          base = ConfigurableSpec::Overwrite::Base.new.configure(CONF2)\n          target = ConfigurableSpec::Overwrite::DetailAddressDefault.new.configure(CONF2)\n          expected_addresses = [\"x\", \"y\"]\n          actual_addresses = [base.detail.address, target.detail.address]\n          assert_equal(expected_addresses, actual_addresses)\n        end\n\n        test 'subclass can add param' do\n          assert_raise(Fluent::ConfigError.new(\"'phone_no' parameter is required, in section detail\")) do\n            ConfigurableSpec::Overwrite::AddParam.new.configure(CONF3)\n          end\n          target = ConfigurableSpec::Overwrite::AddParam.new.configure(CONF4)\n          expected = {\n            address: \"Chiyoda Tokyo Japan\",\n            phone_no: \"+81-00-0000-0000\"\n          }\n          actual = {\n            address: target.detail.address,\n            phone_no: target.detail.phone_no\n          }\n          assert_equal(expected, actual)\n        end\n\n        test 'subclass can add param with overwriting address' do\n          assert_raise(Fluent::ConfigError.new(\"'phone_no' parameter is required, in section detail\")) do\n            ConfigurableSpec::Overwrite::AddParamOverwriteAddress.new.configure(CONF3)\n          end\n          target = ConfigurableSpec::Overwrite::AddParamOverwriteAddress.new.configure(CONF4)\n          expected = {\n            address: \"Chiyoda Tokyo Japan\",\n            phone_no: \"+81-00-0000-0000\"\n          }\n          actual = {\n            address: target.detail.address,\n            phone_no: target.detail.phone_no\n          }\n          assert_equal(expected, actual)\n        end\n\n        sub_test_case 'final' do\n          test 'base class has designed params and default values' do\n            b = ConfigurableSpec::Final::Base.new\n            appendix_conf = config_element('appendix', '', {\"code\" => \"b\", \"name\" => \"base\"})\n            b.configure(config_element('ROOT', '', {}, [appendix_conf]))\n\n            assert_equal \"b\", b.appendix.code\n            assert_equal \"base\", b.appendix.name\n            assert_equal \"\", b.appendix.address\n          end\n\n          test 'subclass can change type, add default value, change default value of parameters, and add parameters to non-finalized section' do\n            f = ConfigurableSpec::Final::Finalized.new\n            appendix_conf = config_element('appendix', '', {\"code\" => 1})\n            f.configure(config_element('ROOT', '', {}, [appendix_conf]))\n\n            assert_equal 1, f.appendix.code\n            assert_equal 'y', f.appendix.name\n            assert_equal \"-\", f.appendix.address\n            assert_equal 10, f.appendix.age\n          end\n\n          test 'subclass can add default value, change default value of parameters, and add parameters to finalized section' do\n            i = ConfigurableSpec::Final::InheritsFinalized.new\n            appendix_conf = config_element('appendix', '', {\"phone_no\" => \"00-0000-0000\"})\n            i.configure(config_element('ROOT', '', {}, [appendix_conf]))\n\n            assert_equal 2, i.appendix.code\n            assert_equal 0, i.appendix.age\n            assert_equal \"00-0000-0000\", i.appendix.phone_no\n          end\n\n          test 'finalized base class works as designed' do\n            b = ConfigurableSpec::Final::FinalizedBase.new\n            appendix_conf = config_element('options', '', {\"name\" => \"moris\"})\n\n            assert_nothing_raised do\n              b.configure(config_element('ROOT', '', {}, [appendix_conf]))\n            end\n            assert b.apd\n            assert_equal \"moris\", b.apd.name\n          end\n\n          test 'subclass can change init' do\n            n = ConfigurableSpec::Final::OverwriteInit.new\n\n            assert_nothing_raised do\n              n.configure(config_element('ROOT', ''))\n            end\n            assert n.apd\n            assert_equal \"moris\", n.apd.name\n            assert_equal 0, n.apd.code\n          end\n\n          test 'subclass cannot change parameter types in finalized sections' do\n            s = ConfigurableSpec::Final::Subclass.new\n            appendix_conf = config_element('options', '', {\"name\" => \"1\"})\n\n            assert_nothing_raised do\n              s.configure(config_element('ROOT', '', {}, [appendix_conf]))\n            end\n            assert_equal \"1\", s.apd.name\n          end\n\n          test 'subclass cannot change param_name of finalized section' do\n            assert_raise(Fluent::ConfigError.new(\"BUG: subclass cannot overwrite base class's config_section: param_name\")) do\n              ConfigurableSpec::Final::OverwriteParamName.new\n            end\n          end\n\n          test 'subclass cannot change final of finalized section' do\n            assert_raise(Fluent::ConfigError.new(\"BUG: subclass cannot overwrite finalized base class's config_section\")) do\n              ConfigurableSpec::Final::OverwriteFinal.new\n            end\n          end\n\n          test 'subclass cannot change required of finalized section' do\n            assert_raise(Fluent::ConfigError.new(\"BUG: subclass cannot overwrite base class's config_section: required\")) do\n              ConfigurableSpec::Final::OverwriteRequired.new\n            end\n          end\n\n          test 'subclass cannot change multi of finalized section' do\n            assert_raise(Fluent::ConfigError.new(\"BUG: subclass cannot overwrite base class's config_section: multi\")) do\n              ConfigurableSpec::Final::OverwriteMulti.new\n            end\n          end\n\n          test 'subclass cannot change alias of finalized section' do\n            assert_raise(Fluent::ConfigError.new(\"BUG: subclass cannot overwrite base class's config_section: alias\")) do\n              ConfigurableSpec::Final::OverwriteAlias.new\n            end\n          end\n        end\n      end\n    end\n\n    sub_test_case 'class defined with config_param/config_section having :alias' do\n      sub_test_case '#initialize' do\n        test 'does not create methods for alias' do\n          ex1 = ConfigurableSpec::ExampleWithAlias.new\n          assert_nothing_raised { ex1.name }\n          assert_raise(NoMethodError) { ex1.fullname }\n          assert_nothing_raised { ex1.bool }\n          assert_raise(NoMethodError) { ex1.flag }\n          assert_nothing_raised { ex1.detail }\n          assert_raise(NoMethodError) { ex1.information}\n        end\n      end\n\n      sub_test_case '#configure' do\n        test 'provides accessible data for alias attribute keys' do\n          ex1 = ConfigurableSpec::ExampleWithAlias.new\n          conf = config_element('ROOT', '', {\n                                  \"fullname\" => \"foo bar\",\n                                  \"bool\" => false\n                                },\n                                [config_element('information', '', {\"address\" => \"Mountain View 0\"})])\n          ex1.configure(conf)\n          assert_equal(\"foo bar\", ex1.name)\n          assert_not_nil(ex1.bool)\n          assert_false(ex1.bool)\n          assert_not_nil(ex1.detail)\n          assert_equal(\"Mountain View 0\", ex1.detail.address)\n        end\n      end\n    end\n\n    sub_test_case 'defaults can be overwritten by owner' do\n      test 'for feature plugin which has flat parameters with parent' do\n        owner = ConfigurableSpec::OverwriteDefaults::Owner.new\n        child = ConfigurableSpec::OverwriteDefaults::FlatChild.new\n        assert_nil child.class.merged_configure_proxy.configured_in_section\n\n        child.owner = owner\n        child.configure(config_element('ROOT', '', {}, []))\n        assert_equal \"V1\", child.key1\n      end\n\n      test 'for feature plugin which has parameters in subsection of parent' do\n        owner = ConfigurableSpec::OverwriteDefaults::Owner.new\n        child = ConfigurableSpec::OverwriteDefaults::BufferChild.new\n        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section\n\n        child.owner = owner\n        child.configure(config_element('ROOT', '', {}, []))\n        assert_equal 1024, child.size_of_something\n      end\n\n      test 'even in subclass of owner' do\n        owner = ConfigurableSpec::OverwriteDefaults::SubOwner.new\n        child = ConfigurableSpec::OverwriteDefaults::BufferChild.new\n        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section\n\n        child.owner = owner\n        child.configure(config_element('ROOT', '', {}, []))\n        assert_equal 2048, child.size_of_something\n      end\n\n      test 'default values can be overwritten with nil' do\n        owner = ConfigurableSpec::OverwriteDefaults::NilOwner.new\n        child = ConfigurableSpec::OverwriteDefaults::BufferChild.new\n        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section\n\n        child.owner = owner\n        child.configure(config_element('ROOT', '', {}, []))\n        assert_nil child.size_of_something\n      end\n\n      test 'the first configured_in (in the order from base class) will be applied' do\n        child = ConfigurableSpec::OverwriteDefaults::BufferSubclass.new\n        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section\n\n        child.configure(config_element('ROOT', '', {}, []))\n        assert_equal 512, child.size_of_something\n      end\n\n      test 'the first configured_in is valid with owner classes' do\n        owner = ConfigurableSpec::OverwriteDefaults::Owner.new\n        child = ConfigurableSpec::OverwriteDefaults::BufferSubclass.new\n        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section\n\n        child.owner = owner\n        child.configure(config_element('ROOT', '', {}, []))\n        assert_equal 1024, child.size_of_something\n      end\n\n      test 'the only first configured_in is valid even in subclasses of a class with configured_in' do\n        child = ConfigurableSpec::OverwriteDefaults::BufferSubSubclass.new\n        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section\n\n        child.configure(config_element('ROOT', '', {}, []))\n        assert_equal 512, child.size_of_something\n      end\n    end\n\n    sub_test_case ':secret option' do\n      setup do\n        @conf = config_element('ROOT', '',\n                               {\n                                 'normal_param' => 'normal',\n                                 'secret_param' => 'secret'\n                               },\n                               [config_element('section', '', {'normal_param2' => 'normal', 'secret_param2' => 'secret'} )])\n        @example = ConfigurableSpec::ExampleWithSecret.new\n        @example.configure(@conf)\n      end\n\n      test 'to_s hides secret config_param' do\n        @conf.to_s.each_line { |line|\n          key, value = line.strip.split(' ', 2)\n          assert_secret_param(key, value)\n        }\n      end\n\n      test 'config returns masked configuration' do\n        conf = @example.config\n        conf.each_pair { |key, value|\n          assert_secret_param(key, value)\n        }\n        conf.elements.each { |element|\n          element.each_pair { |key, value|\n            assert_secret_param(key, value)\n          }\n        }\n      end\n\n      def assert_secret_param(key, value)\n        case key\n        when 'normal_param', 'normal_param2'\n          assert_equal 'normal', value\n        when 'secret_param', 'secret_param2'\n          assert_equal 'xxxxxx', value\n        end\n      end\n    end\n\n    sub_test_case 'unused section' do\n      test 'get plugin name when found unknown section' do\n        conf = config_element('ROOT', '',\n                              {\n                                'name_param' => 'name',\n                              },\n                              [config_element('unknown', '', {'name_param' => 'normal'} )])\n        example = ConfigurableSpec::ExampleWithCustomSection.new\n        example.configure(conf)\n        conf.elements.each { |e|\n          assert_equal(['ROOT', nil], e.unused_in)\n        }\n      end\n\n      test 'get an empty array when the section is defined without using config_section' do\n        conf = config_element('ROOT', '',\n                              {\n                                'name_param' => 'name',\n                              },\n                              [config_element('custom_section', '', {'custom_section_param' => 'custom'} )])\n        example = ConfigurableSpec::ExampleWithCustomSection.new\n        example.configure(conf)\n        conf.elements.each { |e|\n          assert_equal([], e.unused_in)\n        }\n      end\n\n      test 'get an empty array when the configuration is used in another element without any sections' do\n        conf = config_element('ROOT', '',\n                              {\n                                'name_param' => 'name',\n                              },\n                              [config_element('normal_section', '', {'normal_section_param' => 'normal'} )])\n        example = ConfigurableSpec::ExampleWithCustomSection.new\n        example.configure(conf)\n        ConfigurableSpec::ExampleWithCustomSection::AnotherElement.new.configure(conf)\n        conf.elements.each { |e|\n          assert_equal([], e.unused_in)\n        }\n      end\n    end\n\n    sub_test_case ':skip_accessor option' do\n      test 'it does not create accessor methods for parameters' do\n        @example = ConfigurableSpec::ExampleWithSkipAccessor.new\n        @example.configure(config_element('ROOT'))\n        assert_equal 'example7', @example.instance_variable_get(:@name)\n        assert_raise NoMethodError do\n          @example.name\n        end\n      end\n    end\n\n    sub_test_case 'non-required options for config_param' do\n      test 'desc must be a string if specified' do\n        assert_raise ArgumentError.new(\"key: desc must be a String, but Symbol\") do\n          class InvalidDescClass\n            include Fluent::Configurable\n            config_param :key, :string, default: '', desc: :invalid_description\n          end\n        end\n      end\n      test 'alias must be a symbol if specified' do\n        assert_raise ArgumentError.new(\"key: alias must be a Symbol, but String\") do\n          class InvalidAliasClass\n            include Fluent::Configurable\n            config_param :key, :string, default: '', alias: 'yay'\n          end\n        end\n      end\n      test 'secret must be true or false if specified' do\n        assert_raise ArgumentError.new(\"key: secret must be true or false, but NilClass\") do\n          class InvalidSecretClass\n            include Fluent::Configurable\n            config_param :key, :string, default: '', secret: nil\n          end\n        end\n        assert_raise ArgumentError.new(\"key: secret must be true or false, but String\") do\n          class InvalidSecret2Class\n            include Fluent::Configurable\n            config_param :key, :string, default: '', secret: 'yes'\n          end\n        end\n      end\n      test 'deprecated must be a string if specified' do\n        assert_raise ArgumentError.new(\"key: deprecated must be a String, but TrueClass\") do\n          class InvalidDeprecatedClass\n            include Fluent::Configurable\n            config_param :key, :string, default: '', deprecated: true\n          end\n        end\n      end\n      test 'obsoleted must be a string if specified' do\n        assert_raise ArgumentError.new(\"key: obsoleted must be a String, but TrueClass\") do\n          class InvalidObsoletedClass\n            include Fluent::Configurable\n            config_param :key, :string, default: '', obsoleted: true\n          end\n        end\n      end\n      test 'value_type for hash must be a symbol' do\n        assert_raise ArgumentError.new(\"key: value_type must be a Symbol, but String\") do\n          class InvalidValueTypeOfHashClass\n            include Fluent::Configurable\n            config_param :key, :hash, value_type: 'yay'\n          end\n        end\n      end\n      test 'value_type for array must be a symbol' do\n        assert_raise ArgumentError.new(\"key: value_type must be a Symbol, but String\") do\n          class InvalidValueTypeOfArrayClass\n            include Fluent::Configurable\n            config_param :key, :array, value_type: 'yay'\n          end\n        end\n      end\n      test 'skip_accessor must be true or false if specified' do\n        assert_raise ArgumentError.new(\"key: skip_accessor must be true or false, but NilClass\") do\n          class InvalidSkipAccessorClass\n            include Fluent::Configurable\n            config_param :key, :string, default: '', skip_accessor: nil\n          end\n        end\n        assert_raise ArgumentError.new(\"key: skip_accessor must be true or false, but String\") do\n          class InvalidSkipAccessor2Class\n            include Fluent::Configurable\n            config_param :key, :string, default: '', skip_accessor: 'yes'\n          end\n        end\n      end\n    end\n    sub_test_case 'enum parameters' do\n      test 'list must be specified as an array of symbols'\n    end\n    sub_test_case 'deprecated/obsoleted parameters' do\n      test 'both cannot be specified at once' do\n        assert_raise ArgumentError.new(\"param1: both of deprecated and obsoleted cannot be specified at once\") do\n          class Buggy1\n            include Fluent::Configurable\n            config_param :param1, :string, default: '', deprecated: 'yay', obsoleted: 'foo!'\n          end\n        end\n      end\n\n      test 'warned if deprecated parameter is configured' do\n        obj = ConfigurableSpec::UnRecommended.new\n        obj.log = Fluent::Test::TestLogger.new\n        obj.configure(config_element('ROOT', '', {'key1' => 'yay'}, []))\n\n        assert_equal 'yay', obj.key1\n        first_log = obj.log.logs.first\n        assert{ first_log && first_log.include?(\"[warn]\") && first_log.include?(\"'key1' parameter is deprecated: key1 will be removed.\") }\n      end\n\n      test 'error raised if obsoleted parameter is configured' do\n        obj = ConfigurableSpec::UnRecommended.new\n        obj.log = Fluent::Test::TestLogger.new\n\n        assert_raise Fluent::ObsoletedParameterError.new(\"'key2' parameter is already removed: key2 has been removed.\") do\n          obj.configure(config_element('ROOT', '', {'key2' => 'yay'}, []))\n        end\n        first_log = obj.log.logs.first\n        assert{ first_log && first_log.include?(\"[error]\") && first_log.include?(\"config error in:\\n<ROOT>\\n  key2 yay\\n</ROOT>\") }\n      end\n\n      sub_test_case 'logger is nil' do\n        test 'nothing raised if deprecated parameter is configured' do\n          obj = ConfigurableSpec::UnRecommended.new\n          obj.log = nil\n          obj.configure(config_element('ROOT', '', {'key1' => 'yay'}, []))\n          assert_nil(obj.log)\n        end\n\n        test 'NoMethodError is not raised if obsoleted parameter is configured' do\n          obj = ConfigurableSpec::UnRecommended.new\n          obj.log = nil\n          assert_raise Fluent::ObsoletedParameterError.new(\"'key2' parameter is already removed: key2 has been removed.\") do\n            obj.configure(config_element('ROOT', '', {'key2' => 'yay'}, []))\n          end\n          assert_nil(obj.log)\n        end\n      end\n    end\n\n    sub_test_case '#config_param without default values cause error if section is configured as init:true' do\n      setup do\n        @type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }\n        @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)\n      end\n\n      test 'with simple config_param with default value' do\n        class InitTestClass01\n          include Fluent::Configurable\n          config_section :subsection, init: true do\n            config_param :param1, :integer, default: 1\n          end\n        end\n        c = InitTestClass01.new\n        c.configure(config_element('root', ''))\n\n        assert_equal 1, c.subsection.size\n        assert_equal 1, c.subsection.first.param1\n      end\n\n      test 'with simple config_param without default value' do\n        class InitTestClass02\n          include Fluent::Configurable\n          config_section :subsection, init: true do\n            config_param :param1, :integer\n          end\n        end\n        c = InitTestClass02.new\n        assert_raises ArgumentError.new(\"subsection: init is specified, but there're parameters without default values:param1\") do\n          c.configure(config_element('root', ''))\n        end\n\n        c.configure(config_element('root', '', {}, [config_element('subsection', '', {'param1' => '1'})]))\n\n        assert_equal 1, c.subsection.size\n        assert_equal 1, c.subsection.first.param1\n      end\n\n      test 'with config_param with config_set_default' do\n        module InitTestModule03\n          include Fluent::Configurable\n          config_section :subsection, init: true do\n            config_param :param1, :integer\n          end\n        end\n        class InitTestClass03\n          include Fluent::Configurable\n          include InitTestModule03\n          config_section :subsection do\n            config_set_default :param1, 1\n          end\n        end\n\n        c = InitTestClass03.new\n        c.configure(config_element('root', ''))\n\n        assert_equal 1, c.subsection.size\n        assert_equal 1, c.subsection.first.param1\n      end\n\n      test 'with config_argument with default value' do\n        class InitTestClass04\n          include Fluent::Configurable\n          config_section :subsection, init: true do\n            config_argument :param0, :string, default: 'yay'\n          end\n        end\n\n        c = InitTestClass04.new\n        c.configure(config_element('root', ''))\n\n        assert_equal 1, c.subsection.size\n        assert_equal 'yay', c.subsection.first.param0\n      end\n\n      test 'with config_argument without default value' do\n        class InitTestClass04\n          include Fluent::Configurable\n          config_section :subsection, init: true do\n            config_argument :param0, :string\n          end\n        end\n\n        c = InitTestClass04.new\n        assert_raise ArgumentError.new(\"subsection: init is specified, but default value of argument is missing\") do\n          c.configure(config_element('root', ''))\n        end\n      end\n\n      test 'with config_argument with config_set_default' do\n        module InitTestModule05\n          include Fluent::Configurable\n          config_section :subsection, init: true do\n            config_argument :param0, :string\n          end\n        end\n        class InitTestClass05\n          include Fluent::Configurable\n          include InitTestModule05\n          config_section :subsection do\n            config_set_default :param0, 'foo'\n          end\n        end\n\n        c = InitTestClass05.new\n        c.configure(config_element('root', ''))\n\n        assert_equal 1, c.subsection.size\n        assert_equal 'foo', c.subsection.first.param0\n      end\n    end\n\n    sub_test_case '#config_argument' do\n      test 'with strict_config_value' do\n        class TestClass01\n          include Fluent::Configurable\n          config_section :subsection do\n            config_argument :param1, :integer\n          end\n        end\n\n        c = TestClass01.new\n        subsection = config_element('subsection', \"hoge\", { })\n        assert_raise(Fluent::ConfigError.new('param1: invalid value for Integer(): \"hoge\"')) do\n          c.configure(config_element('root', '', {}, [subsection]), true)\n        end\n      end\n\n      test 'with nil' do\n        class TestClass02\n          include Fluent::Configurable\n          config_section :subsection do\n            config_argument :param1, :integer\n          end\n        end\n\n        c = TestClass02.new\n        subsection = config_element('subsection', nil, { })\n        assert_raise(Fluent::ConfigError.new(\"'<subsection ARG>' section requires argument, in section subsection\")) do\n          c.configure(config_element('root', '', {}, [subsection]))\n        end\n      end\n\n      test 'with nil for an argument whose default value is nil' do\n        class TestClass03\n          include Fluent::Configurable\n          config_section :subsection do\n            config_argument :param1, :integer, default: nil\n          end\n        end\n\n        c = TestClass03.new\n        subsection = config_element('subsection', nil, { })\n        c.configure(config_element('root', '', {}, [subsection]))\n\n        assert_equal 1, c.subsection.size\n        assert_equal nil, c.subsection.first.param1\n      end\n\n      test 'with :default' do\n        class TestClass04\n          include Fluent::Configurable\n          config_section :subsection do\n            config_argument :param1, :integer, default: 3\n          end\n        end\n\n        c = TestClass04.new\n        subsection = config_element('subsection', :default, { })\n        c.configure(config_element('root', '', {}, [subsection]))\n\n        assert_equal 1, c.subsection.size\n        assert_equal 3, c.subsection.first.param1\n      end\n\n      test 'with :default for an argument which does not have default value' do\n        class TestClass05\n          include Fluent::Configurable\n          config_section :subsection do\n            config_argument :param1, :integer\n          end\n        end\n\n        c = TestClass05.new\n        subsection = config_element('subsection', :default, { })\n        assert_raise(Fluent::ConfigError.new(\"'param1' doesn\\'t have default value\")) do\n          c.configure(config_element('root', '', {}, [subsection]))\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/config/test_configure_proxy.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/config/configure_proxy'\n\nmodule Fluent::Config\n  class TestConfigureProxy < ::Test::Unit::TestCase\n    setup do\n      @type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }\n    end\n\n    sub_test_case 'to generate a instance' do\n      sub_test_case '#initialize' do\n        test 'has default values' do\n          proxy = Fluent::Config::ConfigureProxy.new('section', type_lookup: @type_lookup)\n          assert_equal(:section, proxy.name)\n\n          proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)\n          assert_equal(:section, proxy.name)\n          assert_nil(proxy.param_name)\n          assert_equal(:section, proxy.variable_name)\n          assert_false(proxy.root?)\n          assert_nil(proxy.init)\n          assert_nil(proxy.required)\n          assert_false(proxy.required?)\n          assert_nil(proxy.multi)\n          assert_true(proxy.multi?)\n        end\n\n        test 'can specify param_name/required/multi with optional arguments' do\n          proxy = Fluent::Config::ConfigureProxy.new(:section, param_name: 'sections', init: true, required: false, multi: true, type_lookup: @type_lookup)\n          assert_equal(:section, proxy.name)\n          assert_equal(:sections, proxy.param_name)\n          assert_equal(:sections, proxy.variable_name)\n          assert_false(proxy.required)\n          assert_false(proxy.required?)\n          assert_true(proxy.multi)\n          assert_true(proxy.multi?)\n\n          proxy = Fluent::Config::ConfigureProxy.new(:section, param_name: :sections, init: false, required: true, multi: false, type_lookup: @type_lookup)\n          assert_equal(:section, proxy.name)\n          assert_equal(:sections, proxy.param_name)\n          assert_equal(:sections, proxy.variable_name)\n          assert_true(proxy.required)\n          assert_true(proxy.required?)\n          assert_false(proxy.multi)\n          assert_false(proxy.multi?)\n        end\n        test 'raise error if both of init and required are true' do\n          assert_raise RuntimeError.new(\"init and required are exclusive\") do\n            Fluent::Config::ConfigureProxy.new(:section, init: true, required: true, type_lookup: @type_lookup)\n          end\n        end\n      end\n\n      sub_test_case '#merge' do\n        test 'generate a new instance which values are overwritten by the argument object' do\n          proxy = p1 = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)\n          assert_equal(:section, proxy.name)\n          assert_nil(proxy.param_name)\n          assert_equal(:section, proxy.variable_name)\n          assert_nil(proxy.init)\n          assert_nil(proxy.required)\n          assert_false(proxy.required?)\n          assert_nil(proxy.multi)\n          assert_true(proxy.multi?)\n          assert_nil(proxy.configured_in_section)\n\n          p2 = Fluent::Config::ConfigureProxy.new(:section, init: false, required: true, multi: false, type_lookup: @type_lookup)\n          proxy = p1.merge(p2)\n          assert_equal(:section, proxy.name)\n          assert_nil(proxy.param_name)\n          assert_equal(:section, proxy.variable_name)\n          assert_false(proxy.init)\n          assert_false(proxy.init?)\n          assert_true(proxy.required)\n          assert_true(proxy.required?)\n          assert_false(proxy.multi)\n          assert_false(proxy.multi?)\n          assert_nil(proxy.configured_in_section)\n        end\n\n        test 'does not overwrite with argument object without any specifications of required/multi' do\n          p1 = Fluent::Config::ConfigureProxy.new(:section1, param_name: :sections, type_lookup: @type_lookup)\n          p1.configured_in_section = :subsection\n          p2 = Fluent::Config::ConfigureProxy.new(:section2, init: false, required: true, multi: false, type_lookup: @type_lookup)\n          p3 = Fluent::Config::ConfigureProxy.new(:section3, type_lookup: @type_lookup)\n          proxy = p1.merge(p2).merge(p3)\n          assert_equal(:section1, proxy.name)\n          assert_equal(:sections, proxy.param_name)\n          assert_equal(:sections, proxy.variable_name)\n          assert_false(proxy.init)\n          assert_false(proxy.init?)\n          assert_true(proxy.required)\n          assert_true(proxy.required?)\n          assert_false(proxy.multi)\n          assert_false(proxy.multi?)\n          assert_equal :subsection, proxy.configured_in_section\n        end\n\n        test \"does overwrite name of proxy for root sections which are used for plugins\" do\n          # latest plugin class shows actual plugin implementation\n          p1 = Fluent::Config::ConfigureProxy.new('Fluent::Plugin::MyP1'.to_sym, root: true, required: true, multi: false, type_lookup: @type_lookup)\n          p1.config_param :key1, :integer\n\n          p2 = Fluent::Config::ConfigureProxy.new('Fluent::Plugin::MyP2'.to_sym, root: true, required: true, multi: false, type_lookup: @type_lookup)\n          p2.config_param :key2, :string, default: \"value2\"\n\n          merged = p1.merge(p2)\n\n          assert_equal 'Fluent::Plugin::MyP2'.to_sym, merged.name\n          assert_true merged.root?\n        end\n      end\n\n      sub_test_case '#overwrite_defaults' do\n        test 'overwrites only defaults with others defaults' do\n          type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }\n          p1 = Fluent::Config::ConfigureProxy.new(:mychild, type_lookup: type_lookup)\n          p1.configured_in_section = :child\n          p1.config_param(:k1a, :string)\n          p1.config_param(:k1b, :string)\n          p1.config_param(:k2a, :integer, default: 0)\n          p1.config_param(:k2b, :integer, default: 0)\n          p1.config_section(:sub1) do\n            config_param :k3, :time, default: 30\n          end\n\n          p0 = Fluent::Config::ConfigureProxy.new(:myparent, type_lookup: type_lookup)\n          p0.config_section(:child) do\n            config_set_default :k1a, \"v1a\"\n            config_param :k1b, :string, default: \"v1b\"\n            config_set_default :k2a, 21\n            config_param :k2b, :integer, default: 22\n            config_section :sub1 do\n              config_set_default :k3, 60\n            end\n          end\n\n          p1.overwrite_defaults(p0.sections[:child])\n\n          assert_equal \"v1a\", p1.defaults[:k1a]\n          assert_equal \"v1b\", p1.defaults[:k1b]\n          assert_equal 21, p1.defaults[:k2a]\n          assert_equal 22, p1.defaults[:k2b]\n          assert_equal 60, p1.sections[:sub1].defaults[:k3]\n        end\n      end\n\n      sub_test_case '#configured_in' do\n        test 'sets a section name which have configuration parameters of target plugin in owners configuration' do\n          proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)\n          proxy.configured_in(:mysection)\n          assert_equal :mysection, proxy.configured_in_section\n        end\n\n        test 'do not permit to be called twice' do\n          proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)\n          proxy.configured_in(:mysection)\n          assert_raise(ArgumentError) { proxy.configured_in(:myothersection) }\n        end\n      end\n\n      sub_test_case '#config_param / #config_set_default / #config_argument' do\n        setup do\n          @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)\n        end\n\n        test 'handles configuration parameters without type as string' do\n          @proxy.config_argument(:label)\n          @proxy.config_param(:name)\n          assert_equal :label, @proxy.argument[0]\n          assert_equal :string, @proxy.argument[2][:type]\n          assert_equal :string, @proxy.params[:name][1][:type]\n        end\n\n        data(\n          default: [:default, nil],\n          alias: [:alias, :alias_name_in_config],\n          secret: [:secret, true],\n          skip_accessor: [:skip_accessor, true],\n          deprecated: [:deprecated, 'it is deprecated'],\n          obsoleted: [:obsoleted, 'it is obsoleted'],\n          desc: [:desc, \"description\"],\n        )\n        test 'always allow options for all types' do |(option, value)|\n          opt = {option => value}\n          assert_nothing_raised{ @proxy.config_argument(:param0, **opt) }\n          assert_nothing_raised{ @proxy.config_param(:p1, :string, **opt) }\n          assert_nothing_raised{ @proxy.config_param(:p2, :enum, list: [:a, :b, :c], **opt) }\n          assert_nothing_raised{ @proxy.config_param(:p3, :integer, **opt) }\n          assert_nothing_raised{ @proxy.config_param(:p4, :float, **opt) }\n          assert_nothing_raised{ @proxy.config_param(:p5, :size, **opt) }\n          assert_nothing_raised{ @proxy.config_param(:p6, :bool, **opt) }\n          assert_nothing_raised{ @proxy.config_param(:p7, :time, **opt) }\n          assert_nothing_raised{ @proxy.config_param(:p8, :hash, **opt) }\n          assert_nothing_raised{ @proxy.config_param(:p9, :array, **opt) }\n          assert_nothing_raised{ @proxy.config_param(:pa, :regexp, **opt) }\n        end\n\n        data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, hash: :hash, array: :array, regexp: :regexp)\n        test 'deny list for non-enum types' do |type|\n          assert_raise ArgumentError.new(\":list is valid only for :enum type, but #{type}: arg\") do\n            @proxy.config_argument(:arg, type, list: [:a, :b])\n          end\n          assert_raise ArgumentError.new(\":list is valid only for :enum type, but #{type}: p1\") do\n            @proxy.config_param(:p1, type, list: [:a, :b])\n          end\n        end\n\n        data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, regexp: :regexp)\n        test 'deny value_type for non-hash/array types' do |type|\n          assert_raise ArgumentError.new(\":value_type is valid only for :hash and :array, but #{type}: arg\") do\n            @proxy.config_argument(:arg, type, value_type: :string)\n          end\n          assert_raise ArgumentError.new(\":value_type is valid only for :hash and :array, but #{type}: p1\") do\n            @proxy.config_param(:p1, type, value_type: :integer)\n          end\n        end\n\n        data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, array: :array, regexp: :regexp)\n        test 'deny symbolize_keys for non-hash types' do |type|\n          assert_raise ArgumentError.new(\":symbolize_keys is valid only for :hash, but #{type}: arg\") do\n            @proxy.config_argument(:arg, type, symbolize_keys: true)\n          end\n          assert_raise ArgumentError.new(\":symbolize_keys is valid only for :hash, but #{type}: p1\") do\n            @proxy.config_param(:p1, type, symbolize_keys: true)\n          end\n        end\n\n        data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, hash: :hash, array: :array)\n        test 'deny unknown options' do |type|\n          assert_raise ArgumentError.new(\"unknown option 'required' for configuration parameter: arg\") do\n            @proxy.config_argument(:arg, type, required: true)\n          end\n          assert_raise ArgumentError.new(\"unknown option 'param_name' for configuration parameter: p1\") do\n            @proxy.config_argument(:p1, type, param_name: :yay)\n          end\n        end\n\n        test 'desc gets string' do\n          assert_nothing_raised do\n            @proxy.config_param(:name, :string, desc: \"it is description\")\n          end\n          assert_raise ArgumentError.new(\"name1: desc must be a String, but Symbol\") do\n            @proxy.config_param(:name1, :string, desc: :yaaaaaaaay)\n          end\n        end\n\n        test 'alias gets symbol' do\n          assert_nothing_raised do\n            @proxy.config_param(:name, :string, alias: :label)\n          end\n          assert_raise ArgumentError.new(\"name1: alias must be a Symbol, but String\") do\n            @proxy.config_param(:name1, :string, alias: 'label1')\n          end\n        end\n\n        test 'secret gets true/false' do\n          assert_nothing_raised do\n            @proxy.config_param(:name1, :string, secret: false)\n          end\n          assert_nothing_raised do\n            @proxy.config_param(:name2, :string, secret: true)\n          end\n          assert_raise ArgumentError.new(\"name3: secret must be true or false, but String\") do\n            @proxy.config_param(:name3, :string, secret: 'yes')\n          end\n          assert_raise ArgumentError.new(\"name4: secret must be true or false, but NilClass\") do\n            @proxy.config_param(:name4, :string, secret: nil)\n          end\n        end\n\n        test 'symbolize_keys gets true/false' do\n          assert_nothing_raised do\n            @proxy.config_param(:data1, :hash, symbolize_keys: false)\n          end\n          assert_nothing_raised do\n            @proxy.config_param(:data2, :hash, symbolize_keys: true)\n          end\n          assert_raise ArgumentError.new(\"data3: symbolize_keys must be true or false, but NilClass\") do\n            @proxy.config_param(:data3, :hash, symbolize_keys: nil)\n          end\n        end\n\n        test 'value_type gets symbol' do\n          assert_nothing_raised do\n            @proxy.config_param(:data1, :array, value_type: :integer)\n          end\n          assert_raise ArgumentError.new(\"data2: value_type must be a Symbol, but Class\") do\n            @proxy.config_param(:data2, :array, value_type: Integer)\n          end\n        end\n\n        test 'list gets an array of symbols' do\n          assert_nothing_raised do\n            @proxy.config_param(:proto1, :enum, list: [:a, :b])\n          end\n          assert_raise ArgumentError.new(\"proto2: enum parameter requires :list of Symbols\") do\n            @proxy.config_param(:proto2, :enum, list: nil)\n          end\n          assert_raise ArgumentError.new(\"proto3: enum parameter requires :list of Symbols\") do\n            @proxy.config_param(:proto3, :enum, list: ['a', 'b'])\n          end\n          assert_raise ArgumentError.new(\"proto4: enum parameter requires :list of Symbols\") do\n            @proxy.config_param(:proto4, :enum, list: [])\n          end\n        end\n\n        test 'deprecated gets string' do\n          assert_nothing_raised do\n            @proxy.config_param(:name1, :string, deprecated: \"use name2 instead\")\n          end\n          assert_raise ArgumentError.new(\"name2: deprecated must be a String, but TrueClass\") do\n            @proxy.config_param(:name2, :string, deprecated: true)\n          end\n        end\n\n        test 'obsoleted gets string' do\n          assert_nothing_raised do\n            @proxy.config_param(:name1, :string, obsoleted: \"use name2 instead\")\n          end\n          assert_raise ArgumentError.new(\"name2: obsoleted must be a String, but TrueClass\") do\n            @proxy.config_param(:name2, :string, obsoleted: true)\n          end\n        end\n\n        test 'skip_accessor gets true/false' do\n          assert_nothing_raised do\n            @proxy.config_param(:format1, :string, skip_accessor: false)\n          end\n          assert_nothing_raised do\n            @proxy.config_param(:format2, :string, skip_accessor: true)\n          end\n          assert_raise ArgumentError.new(\"format2: skip_accessor must be true or false, but String\") do\n            @proxy.config_param(:format2, :string, skip_accessor: 'yes')\n          end\n        end\n\n        test 'list is required for :enum' do\n          assert_nothing_raised do\n            @proxy.config_param(:proto1, :enum, list: [:a, :b])\n          end\n          assert_raise ArgumentError.new(\"proto1: enum parameter requires :list of Symbols\") do\n            @proxy.config_param(:proto1, :enum, default: :a)\n          end\n        end\n\n        test 'does not permit config_set_default for param w/ :default option' do\n          @proxy.config_param(:name, :string, default: \"name1\")\n          assert_raise(ArgumentError) { @proxy.config_set_default(:name, \"name2\") }\n        end\n\n        test 'does not permit default value specification twice' do\n          @proxy.config_param(:name, :string)\n          @proxy.config_set_default(:name, \"name1\")\n          assert_raise(ArgumentError) { @proxy.config_set_default(:name, \"name2\") }\n        end\n\n        test 'does not permit default value specification twice, even on config_argument' do\n          @proxy.config_param(:name, :string)\n          @proxy.config_set_default(:name, \"name1\")\n\n          @proxy.config_argument(:name)\n          assert_raise(ArgumentError) { @proxy.config_argument(:name, default: \"name2\") }\n        end\n      end\n\n      sub_test_case '#config_set_desc' do\n        setup do\n          @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)\n        end\n\n        test 'does not permit description specification twice w/ :desc option' do\n          @proxy.config_param(:name, :string, desc: \"description\")\n          assert_raise(ArgumentError) { @proxy.config_set_desc(:name, \"description2\") }\n        end\n\n        test 'does not permit description specification twice' do\n          @proxy.config_param(:name, :string)\n          @proxy.config_set_desc(:name, \"description\")\n          assert_raise(ArgumentError) { @proxy.config_set_desc(:name, \"description2\") }\n        end\n      end\n\n      sub_test_case '#desc' do\n        setup do\n          @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)\n        end\n\n        test 'permit to specify description twice' do\n          @proxy.desc(\"description1\")\n          @proxy.desc(\"description2\")\n          @proxy.config_param(:name, :string)\n          assert_equal(\"description2\", @proxy.descriptions[:name])\n        end\n\n        test 'does not permit description specification twice' do\n          @proxy.desc(\"description1\")\n          assert_raise(ArgumentError) do\n            @proxy.config_param(:name, :string, desc: \"description2\")\n          end\n        end\n      end\n\n      sub_test_case '#dump_config_definition' do\n        setup do\n          @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)\n        end\n\n        test 'empty proxy' do\n          assert_equal({}, @proxy.dump_config_definition)\n        end\n\n        test 'plain proxy w/o default value' do\n          @proxy.config_param(:name, :string)\n          expected = {\n            name: { type: :string, required: true }\n          }\n          assert_equal(expected, @proxy.dump_config_definition)\n        end\n\n        test 'plain proxy w/ default value' do\n          @proxy.config_param(:name, :string, default: \"name1\")\n          expected = {\n            name: { type: :string, default: \"name1\", required: false }\n          }\n          assert_equal(expected, @proxy.dump_config_definition)\n        end\n\n        test 'plain proxy w/ default value using config_set_default' do\n          @proxy.config_param(:name, :string)\n          @proxy.config_set_default(:name, \"name1\")\n          expected = {\n            name: { type: :string, default: \"name1\", required: false }\n          }\n          assert_equal(expected, @proxy.dump_config_definition)\n        end\n\n        test 'plain proxy w/ argument' do\n          @proxy.instance_eval do\n            config_argument(:argname, :string)\n            config_param(:name, :string, default: \"name1\")\n          end\n          expected = {\n            argname: { type: :string, required: true, argument: true },\n            name: { type: :string, default: \"name1\", required: false }\n          }\n          assert_equal(expected, @proxy.dump_config_definition)\n        end\n\n        test 'plain proxy w/ argument default value' do\n          @proxy.instance_eval do\n            config_argument(:argname, :string, default: \"value\")\n            config_param(:name, :string, default: \"name1\")\n          end\n          expected = {\n            argname: { type: :string, default: \"value\", required: false, argument: true },\n            name: { type: :string, default: \"name1\", required: false }\n          }\n          assert_equal(expected, @proxy.dump_config_definition)\n        end\n\n        test 'plain proxy w/ argument overwriting default value' do\n          @proxy.instance_eval do\n            config_argument(:argname, :string)\n            config_param(:name, :string, default: \"name1\")\n            config_set_default(:argname, \"value1\")\n          end\n          expected = {\n            argname: { type: :string, default: \"value1\", required: false, argument: true },\n            name: { type: :string, default: \"name1\", required: false }\n          }\n          assert_equal(expected, @proxy.dump_config_definition)\n        end\n\n        test 'single sub proxy' do\n          @proxy.config_section(:sub) do\n            config_param(:name, :string, default: \"name1\")\n          end\n          expected = {\n            sub: {\n              alias: nil,\n              multi: true,\n              required: false,\n              section: true,\n              name: { type: :string, default: \"name1\", required: false }\n            }\n          }\n          assert_equal(expected, @proxy.dump_config_definition)\n        end\n\n        test 'nested sub proxy' do\n          @proxy.config_section(:sub) do\n            config_param(:name1, :string, default: \"name1\")\n            config_param(:name2, :string, default: \"name2\")\n            config_section(:sub2) do\n              config_param(:name3, :string, default: \"name3\")\n              config_param(:name4, :string, default: \"name4\")\n            end\n          end\n          expected = {\n            sub: {\n              alias: nil,\n              multi: true,\n              required: false,\n              section: true,\n              name1: { type: :string, default: \"name1\", required: false },\n              name2: { type: :string, default: \"name2\", required: false },\n              sub2: {\n                alias: nil,\n                multi: true,\n                required: false,\n                section: true,\n                name3: { type: :string, default: \"name3\", required: false },\n                name4: { type: :string, default: \"name4\", required: false },\n              }\n            }\n          }\n          assert_equal(expected, @proxy.dump_config_definition)\n        end\n\n        sub_test_case 'w/ description' do\n          test 'single proxy' do\n            @proxy.config_param(:name, :string, desc: \"description for name\")\n            expected = {\n              name: { type: :string, desc: \"description for name\", required: true }\n            }\n            assert_equal(expected, @proxy.dump_config_definition)\n          end\n\n          test 'single proxy using config_set_desc' do\n            @proxy.config_param(:name, :string)\n            @proxy.config_set_desc(:name, \"description for name\")\n            expected = {\n              name: { type: :string, desc: \"description for name\", required: true }\n            }\n            assert_equal(expected, @proxy.dump_config_definition)\n          end\n\n          test 'sub proxy' do\n            @proxy.config_section(:sub) do\n              config_param(:name1, :string, default: \"name1\", desc: \"desc1\")\n              config_param(:name2, :string, default: \"name2\", desc: \"desc2\")\n              config_section(:sub2) do\n                config_param(:name3, :string, default: \"name3\")\n                config_param(:name4, :string, default: \"name4\", desc: \"desc4\")\n              end\n            end\n            expected = {\n              sub: {\n                alias: nil,\n                multi: true,\n                required: false,\n                section: true,\n                name1: { type: :string, default: \"name1\", desc: \"desc1\", required: false },\n                name2: { type: :string, default: \"name2\", desc: \"desc2\", required: false },\n                sub2: {\n                  alias: nil,\n                  multi: true,\n                  required: false,\n                  section: true,\n                  name3: { type: :string, default: \"name3\", required: false },\n                  name4: { type: :string, default: \"name4\", desc: \"desc4\", required: false },\n                }\n              }\n            }\n            assert_equal(expected, @proxy.dump_config_definition)\n          end\n\n          test 'sub proxy w/ desc method' do\n            @proxy.config_section(:sub) do\n              desc(\"desc1\")\n              config_param(:name1, :string, default: \"name1\")\n              config_param(:name2, :string, default: \"name2\", desc: \"desc2\")\n              config_section(:sub2) do\n                config_param(:name3, :string, default: \"name3\")\n                desc(\"desc4\")\n                config_param(:name4, :string, default: \"name4\")\n              end\n            end\n            expected = {\n              sub: {\n                alias: nil,\n                multi: true,\n                required: false,\n                section: true,\n                name1: { type: :string, default: \"name1\", desc: \"desc1\", required: false },\n                name2: { type: :string, default: \"name2\", desc: \"desc2\", required: false },\n                sub2: {\n                  alias: nil,\n                  multi: true,\n                  required: false,\n                  section: true,\n                  name3: { type: :string, default: \"name3\", required: false },\n                  name4: { type: :string, default: \"name4\", desc: \"desc4\", required: false },\n                }\n              }\n            }\n            assert_equal(expected, @proxy.dump_config_definition)\n          end\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/config/test_dsl.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/config/element'\nrequire \"fluent/config/dsl\"\nrequire 'tempfile'\n\nTMP_DIR = File.dirname(__FILE__) + \"/tmp/config_dsl#{ENV['TEST_ENV_NUMBER']}\"\ndef write_config(path, data)\n  FileUtils.mkdir_p(File.dirname(path))\n  File.open(path, \"w\") {|f| f.write data }\nend\n\ndef prepare_config1\n  write_config \"#{TMP_DIR}/config_test_1.conf\", %[\n  k1 root_config\n  include dir/config_test_2.conf  #\n  @include #{TMP_DIR}/config_test_4.conf\n  include file://#{TMP_DIR}/config_test_5.conf\n  @include config.d/*.conf\n]\n  write_config \"#{TMP_DIR}/dir/config_test_2.conf\", %[\n  k2 relative_path_include\n  @include ../config_test_3.conf\n]\n  write_config \"#{TMP_DIR}/config_test_3.conf\", %[\n  k3 relative_include_in_included_file\n]\n  write_config \"#{TMP_DIR}/config_test_4.conf\", %[\n  k4 absolute_path_include\n]\n  write_config \"#{TMP_DIR}/config_test_5.conf\", %[\n  k5 uri_include\n]\n  write_config \"#{TMP_DIR}/config.d/config_test_6.conf\", %[\n  k6 wildcard_include_1\n  <elem1 name>\n    include normal_parameter\n  </elem1>\n]\n  write_config \"#{TMP_DIR}/config.d/config_test_7.conf\", %[\n  k7 wildcard_include_2\n]\n  write_config \"#{TMP_DIR}/config.d/config_test_8.conf\", %[\n  <elem2 name>\n    @include ../dir/config_test_9.conf\n  </elem2>\n]\n  write_config \"#{TMP_DIR}/dir/config_test_9.conf\", %[\n  k9 embedded\n  <elem3 name>\n    nested nested_value\n    include hoge\n  </elem3>\n]\n  write_config \"#{TMP_DIR}/config.d/00_config_test_8.conf\", %[\n  k8 wildcard_include_3\n  <elem4 name>\n    include normal_parameter\n  </elem4>\n]\nend\n\ndef prepare_config2\n  write_config \"#{TMP_DIR}/config_test_1.rb\", DSL_CONFIG_EXAMPLE\nend\n\nDSL_CONFIG_EXAMPLE = %q[\nworker {\n  hostname = \"myhostname\"\n\n  (0..9).each { |i|\n    source {\n      type :tail\n      path \"/var/log/httpd/access.part#{i}.log\"\n\n      filter ('bar.**') {\n        type :hoge\n        val1 \"moge\"\n        val2 [\"foo\", \"bar\", \"baz\"]\n        val3 10\n        id :hoge\n\n        subsection {\n          foo \"bar\"\n        }\n        subsection {\n          foo \"baz\"\n        }\n      }\n\n      filter ('foo.**') {\n        type \"pass\"\n      }\n\n      match ('{foo,bar}.**') {\n        type \"file\"\n        path \"/var/log/httpd/access.#{hostname}.#{i}.log\"\n      }\n    }\n  }\n}\n]\n\nDSL_CONFIG_EXAMPLE_WITHOUT_WORKER = %q[\nhostname = \"myhostname\"\n\nsource {\n  type :tail\n  path \"/var/log/httpd/access.part.log\"\n\n  element {\n    name \"foo\"\n  }\n\n  match ('{foo,bar}.**') {\n    type \"file\"\n    path \"/var/log/httpd/access.full.log\"\n  }\n}\n]\n\nDSL_CONFIG_EXAMPLE_FOR_INCLUDE_CONF = %q[\ninclude \"#{TMP_DIR}/config_test_1.conf\"\n]\n\nDSL_CONFIG_EXAMPLE_FOR_INCLUDE_RB = %q[\ninclude \"#{TMP_DIR}/config_test_1.rb\"\n]\n\nDSL_CONFIG_RETURNS_NON_ELEMENT = %q[\nworker {\n}\n[]\n]\n\nDSL_CONFIG_WRONG_SYNTAX1 = %q[\nmatch\n]\nDSL_CONFIG_WRONG_SYNTAX2 = %q[\nmatch('aa','bb'){\n  type :null\n}\n]\nDSL_CONFIG_WRONG_SYNTAX3 = %q[\nmatch('aa','bb')\n]\nDSL_CONFIG_WRONG_SYNTAX4 = %q[\ninclude\n]\n\nmodule Fluent::Config\n  class TestDSLParser < ::Test::Unit::TestCase\n    sub_test_case 'with worker tag on top level' do\n      def setup\n        @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE, 'dsl_config.rb')\n      end\n\n      sub_test_case '.parse' do\n        test 'makes root element' do\n          assert_equal('ROOT', @root.name)\n          assert_predicate(@root.arg, :empty?)\n          assert_equal(0, @root.keys.size)\n        end\n\n        test 'makes worker element for worker tag' do\n          assert_equal(1, @root.elements.size)\n\n          worker = @root.elements.first\n          assert_equal('worker', worker.name)\n          assert_predicate(worker.arg, :empty?)\n          assert_equal(0, worker.keys.size)\n          assert_equal(10, worker.elements.size)\n        end\n\n        test 'makes subsections for blocks, with variable substitution' do\n          ele4 = @root.elements.first.elements[4]\n\n          assert_equal('source', ele4.name)\n          assert_predicate(ele4.arg, :empty?)\n          assert_equal(2, ele4.keys.size)\n          assert_equal('tail', ele4['@type'])\n          assert_equal(\"/var/log/httpd/access.part4.log\", ele4['path'])\n        end\n\n        test 'makes user-defined sections with blocks' do\n          filter0 = @root.elements.first.elements[4].elements.first\n\n          assert_equal('filter', filter0.name)\n          assert_equal('bar.**', filter0.arg)\n          assert_equal('hoge', filter0['@type'])\n          assert_equal('moge', filter0['val1'])\n          assert_equal(JSON.dump(['foo', 'bar', 'baz']), filter0['val2'])\n          assert_equal('10', filter0['val3'])\n          assert_equal('hoge', filter0['@id'])\n\n          assert_equal(2, filter0.elements.size)\n          assert_equal('subsection', filter0.elements[0].name)\n          assert_equal('bar', filter0.elements[0]['foo'])\n          assert_equal('subsection', filter0.elements[1].name)\n          assert_equal('baz', filter0.elements[1]['foo'])\n        end\n\n        test 'makes values with user-assigned variable substitutions' do\n          match0 = @root.elements.first.elements[4].elements.last\n\n          assert_equal('match', match0.name)\n          assert_equal('{foo,bar}.**', match0.arg)\n          assert_equal('file', match0['@type'])\n          assert_equal('/var/log/httpd/access.myhostname.4.log', match0['path'])\n        end\n      end\n    end\n\n    sub_test_case 'without worker tag on top level' do\n      def setup\n        @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_WITHOUT_WORKER, 'dsl_config_without_worker.rb')\n      end\n\n      sub_test_case '.parse' do\n        test 'makes root element' do\n          assert_equal('ROOT', @root.name)\n          assert_predicate(@root.arg, :empty?)\n          assert_equal(0, @root.keys.size)\n        end\n\n        test 'does not make worker element implicitly because DSL configuration does not support v10 compat mode' do\n          assert_equal(1, @root.elements.size)\n          assert_equal('source', @root.elements.first.name)\n          refute(@root.elements.find { |e| e.name == 'worker' })\n        end\n      end\n    end\n\n    sub_test_case 'with include conf' do\n      def setup\n        prepare_config1\n        @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_FOR_INCLUDE_CONF, 'dsl_config_for_include.conf')\n      end\n      test 'include config' do\n        assert_equal('root_config', @root['k1'])\n        assert_equal('relative_path_include', @root['k2'])\n        assert_equal('relative_include_in_included_file', @root['k3'])\n        assert_equal('absolute_path_include', @root['k4'])\n        assert_equal('uri_include', @root['k5'])\n        assert_equal('wildcard_include_1', @root['k6'])\n        assert_equal('wildcard_include_2', @root['k7'])\n        assert_equal('wildcard_include_3', @root['k8'])\n        assert_equal([\n            'k1',\n            'k2',\n            'k3',\n            'k4',\n            'k5',\n            'k8', # Because of the file name this comes first.\n            'k6',\n            'k7',\n          ], @root.keys)\n\n        elem1 = @root.elements.find { |e| e.name == 'elem1' }\n        assert(elem1)\n        assert_equal('name', elem1.arg)\n        assert_equal('normal_parameter', elem1['include'])\n\n        elem2 = @root.elements.find { |e| e.name == 'elem2' }\n        assert(elem2)\n        assert_equal('name', elem2.arg)\n        assert_equal('embedded', elem2['k9'])\n        assert_not_include(elem2, 'include')\n\n        elem3 = elem2.elements.find { |e| e.name == 'elem3' }\n        assert(elem3)\n        assert_equal('nested_value', elem3['nested'])\n        assert_equal('hoge', elem3['include'])\n      end\n\n      # TODO: Add uri based include spec\n    end\n\n    sub_test_case 'with include rb' do\n      def setup\n        prepare_config2\n        @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_FOR_INCLUDE_RB, 'dsl_config_for_include.rb')\n      end\n      sub_test_case '.parse' do\n        test 'makes root element' do\n          assert_equal('ROOT', @root.name)\n          assert_predicate(@root.arg, :empty?)\n          assert_equal(0, @root.keys.size)\n        end\n\n        test 'makes worker element for worker tag' do\n          assert_equal(1, @root.elements.size)\n\n          worker = @root.elements.first\n          assert_equal('worker', worker.name)\n          assert_predicate(worker.arg, :empty?)\n          assert_equal(0, worker.keys.size)\n          assert_equal(10, worker.elements.size)\n        end\n\n        test 'makes subsections for blocks, with variable substitution' do\n          ele4 = @root.elements.first.elements[4]\n\n          assert_equal('source', ele4.name)\n          assert_predicate(ele4.arg, :empty?)\n          assert_equal(2, ele4.keys.size)\n          assert_equal('tail', ele4['@type'])\n          assert_equal(\"/var/log/httpd/access.part4.log\", ele4['path'])\n        end\n\n        test 'makes user-defined sections with blocks' do\n          filter0 = @root.elements.first.elements[4].elements.first\n\n          assert_equal('filter', filter0.name)\n          assert_equal('bar.**', filter0.arg)\n          assert_equal('hoge', filter0['@type'])\n          assert_equal('moge', filter0['val1'])\n          assert_equal(JSON.dump(['foo', 'bar', 'baz']), filter0['val2'])\n          assert_equal('10', filter0['val3'])\n          assert_equal('hoge', filter0['@id'])\n\n          assert_equal(2, filter0.elements.size)\n          assert_equal('subsection', filter0.elements[0].name)\n          assert_equal('bar', filter0.elements[0]['foo'])\n          assert_equal('subsection', filter0.elements[1].name)\n          assert_equal('baz', filter0.elements[1]['foo'])\n        end\n\n        test 'makes values with user-assigned variable substitutions' do\n          match0 = @root.elements.first.elements[4].elements.last\n\n          assert_equal('match', match0.name)\n          assert_equal('{foo,bar}.**', match0.arg)\n          assert_equal('file', match0['@type'])\n          assert_equal('/var/log/httpd/access.myhostname.4.log', match0['path'])\n        end\n      end\n    end\n\n    sub_test_case 'with configuration that returns non element on top' do\n      sub_test_case '.parse' do\n        test 'does not crash' do\n          Fluent::Config::DSL::Parser.parse(DSL_CONFIG_RETURNS_NON_ELEMENT, 'dsl_config_returns_non_element.rb')\n        end\n      end\n    end\n\n    sub_test_case 'with configuration with wrong arguments for specific elements' do\n      sub_test_case '.parse' do\n        test 'raises ArgumentError correctly' do\n          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX1, 'dsl_config_wrong_syntax1') }\n          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX2, 'dsl_config_wrong_syntax2') }\n          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX3, 'dsl_config_wrong_syntax3') }\n          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX4, 'dsl_config_wrong_syntax4') }\n        end\n      end\n    end\n\n    sub_test_case 'with ruby keyword, that provides ruby Kernel module features' do\n      sub_test_case '.parse' do\n        test 'can get result of Kernel.open() by ruby.open()' do\n          uname_string = `uname -a`\n          tmpfile = Tempfile.create('fluentd-test')\n          tmpfile.write(uname_string)\n          tmpfile.close\n\n          root = Fluent::Config::DSL::Parser.parse(<<DSL)\nworker {\n  uname_str = ruby.open(\"#{tmpfile.path}\"){|out| out.read}\n  source {\n    uname uname_str\n  }\n}\nDSL\n          worker = root.elements.first\n          assert_equal('worker', worker.name)\n          source = worker.elements.first\n          assert_equal('source', source.name)\n          assert_equal(1, source.keys.size)\n          assert_equal(uname_string, source['uname'])\n        ensure\n          File.delete(tmpfile.path)\n        end\n\n        test 'accepts ruby keyword with block, which allow to use methods included from ::Kernel' do\n          root = Fluent::Config::DSL::Parser.parse(<<DSL)\nworker {\n  ruby_version = ruby {\n    require 'erb'\n    ERB.new('<%= RUBY_VERSION %> from erb').result\n  }\n  source {\n    version ruby_version\n  }\n}\nDSL\n          worker = root.elements.first\n          assert_equal('worker', worker.name)\n          source = worker.elements.first\n          assert_equal('source', source.name)\n          assert_equal(1, source.keys.size)\n          assert_equal(\"#{RUBY_VERSION} from erb\", source['version'])\n        end\n\n        test 'raises NoMethodError when configuration DSL elements are written in ruby block' do\n          conf = <<DSL\nworker {\n  ruby {\n    source {\n      type \"tail\"\n    }\n  }\n  source {\n    uname uname_str\n  }\n}\nDSL\n          assert_raise(NoMethodError) { Fluent::Config::DSL::Parser.parse(conf) }\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/config/test_element.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/config/element'\nrequire 'fluent/config/configure_proxy'\nrequire 'fluent/configurable'\nrequire 'pp'\n\nclass TestConfigElement < ::Test::Unit::TestCase\n  def element(name = 'ROOT', arg = '', attrs = {}, elements = [], unused = nil)\n    Fluent::Config::Element.new(name, arg, attrs, elements, unused)\n  end\n\n  sub_test_case '#elements=' do\n    test 'elements can be set by others' do\n      e = element()\n      assert_equal [], e.elements\n\n      e1 = element('e1')\n      e2 = element('e2')\n      e.elements = [e1, e2]\n      assert_equal [e1, e2], e.elements\n    end\n  end\n\n  sub_test_case '#elements' do\n    setup do\n      @c1 = element('source')\n      @c2 = element('source', 'yay')\n      @c3 = element('match', 'test.**')\n      @c4 = element('label', '@mytest', {}, [ element('filter', '**'), element('match', '**') ])\n      children = [@c1, @c2, @c3, @c4]\n      @e = Fluent::Config::Element.new('ROOT', '', {}, children)\n    end\n\n    test 'returns all elements without arguments' do\n      assert_equal [@c1, @c2, @c3, @c4], @e.elements\n    end\n\n    test 'returns elements with specified names' do\n      assert_equal [@c1, @c2, @c3], @e.elements('source', 'match')\n      assert_equal [@c3, @c4], @e.elements('match', 'label')\n    end\n\n    test 'returns elements with specified name, and arg if specified' do\n      assert_equal [@c1, @c2], @e.elements(name: 'source')\n      assert_equal [@c2], @e.elements(name: 'source', arg: 'yay')\n    end\n\n    test 'keyword argument name/arg and names are exclusive' do\n      assert_raise ArgumentError do\n        @e.elements('source', name: 'match')\n      end\n      assert_raise ArgumentError do\n        @e.elements('source', 'match', name: 'label', arg: '@mytest')\n      end\n    end\n\n    test 'specifying only arg without name is invalid' do\n      assert_raise ArgumentError do\n        @e.elements(arg: '@mytest')\n      end\n    end\n  end\n\n  sub_test_case '#initialize' do\n    test 'creates object with blank attrs and elements' do\n      e = element('ROOT', '', {}, [])\n      assert_equal([], e.elements)\n    end\n\n    test 'creates object which contains attrs and elements' do\n      e = element('ROOT', '', {\"k1\" => \"v1\", \"k2\" => \"v2\"}, [\n                    element('test', 'mydata', {'k3' => 'v3'}, [])\n                  ])\n      assert_equal('ROOT', e.name)\n      assert_equal('',  e.arg)\n      assert_equal('v1', e[\"k1\"])\n      assert_equal('v2', e[\"k2\"])\n      assert_equal(1, e.elements.size)\n      e.each_element('test') do |el|\n        assert_equal('test', el.name)\n        assert_equal('mydata', el.arg)\n        assert_equal('v3', el[\"k3\"])\n      end\n    end\n\n    test 'creates object which contains attrs, elements and unused' do\n      e = element('ROOT', '', {\"k1\" => \"v1\", \"k2\" => \"v2\", \"k4\" => \"v4\"}, [\n                    element('test', 'mydata', {'k3' => 'v3'}, [])\n                  ], \"k3\")\n      assert_equal(\"k3\", e.unused)\n      assert_equal('ROOT', e.name)\n      assert_equal('',  e.arg)\n      assert_equal('v1', e[\"k1\"])\n      assert_equal('v2', e[\"k2\"])\n      assert_equal('v4', e[\"k4\"])\n      assert_equal(1, e.elements.size)\n      e.each_element('test') do |el|\n        assert_equal('test', el.name)\n        assert_equal('mydata', el.arg)\n        assert_equal('v3', el[\"k3\"])\n      end\n      assert_equal(\"k3\", e.unused)\n    end\n  end\n\n  sub_test_case \"@unused\" do\n    sub_test_case '#[] has side effect for @unused' do\n      test 'without unused argument' do\n        e = element('ROOT', '', {\"k1\" => \"v1\", \"k2\" => \"v2\", \"k4\" => \"v4\"}, [\n                      element('test', 'mydata', {'k3' => 'v3'}, [])\n                    ])\n        assert_equal([\"k1\", \"k2\", \"k4\"], e.unused)\n        assert_equal('v1', e[\"k1\"])\n        assert_equal([\"k2\", \"k4\"], e.unused)\n        assert_equal('v2', e[\"k2\"])\n        assert_equal([\"k4\"], e.unused)\n        assert_equal('v4', e[\"k4\"])\n        assert_equal([], e.unused)\n      end\n\n      test 'with unused argument' do\n        e = element('ROOT', '', {\"k1\" => \"v1\", \"k2\" => \"v2\", \"k4\" => \"v4\"}, [\n                      element('test', 'mydata', {'k3' => 'v3'}, [])\n                    ], [\"k4\"])\n        assert_equal([\"k4\"], e.unused)\n        assert_equal('v1', e[\"k1\"])\n        assert_equal([\"k4\"], e.unused)\n        assert_equal('v2', e[\"k2\"])\n        assert_equal([\"k4\"], e.unused)\n        assert_equal('v4', e[\"k4\"])\n        # only consume for \"k4\"\n        assert_equal([], e.unused)\n      end\n    end\n  end\n\n  sub_test_case '#add_element' do\n    test 'elements can be set by #add_element' do\n      e = element()\n      assert_equal [], e.elements\n\n      e.add_element('e1', '')\n      e.add_element('e2', '')\n      assert_equal [element('e1', ''), element('e2', '')], e.elements\n    end\n  end\n\n  sub_test_case '#==' do\n    sub_test_case 'compare with two element objects' do\n      test 'equal' do\n        e1 = element('ROOT', '', {}, [])\n        e2 = element('ROOT', '', {}, [])\n        assert_true(e1 == e2)\n      end\n\n      data(\"differ args\" => [Fluent::Config::Element.new('ROOT', '', {}, []),\n                             Fluent::Config::Element.new('ROOT', 'mydata', {}, [])],\n           \"differ keys\" => [Fluent::Config::Element.new('ROOT', 'mydata', {}, []),\n                             Fluent::Config::Element.new('ROOT', 'mydata', {\"k1\" => \"v1\"}, [])],\n           \"differ elements\" =>\n           [Fluent::Config::Element.new('ROOT', 'mydata', {\"k1\" => \"v1\"}, []),\n            Fluent::Config::Element.new('ROOT', 'mydata', {\"k1\" => \"v1\"}, [\n              Fluent::Config::Element.new('test', 'mydata', {'k3' => 'v3'}, [])\n            ])])\n      test 'not equal' do |data|\n        e1, e2 = data\n        assert_false(e1 == e2)\n      end\n    end\n  end\n\n  sub_test_case '#+' do\n    test 'can merge 2 elements: object side is primary' do\n      e1 = element('ROOT', 'mydata', {\"k1\" => \"v1\"}, [])\n      e2 = element('ROOT', 'mydata2', {\"k1\" => \"ignored\", \"k2\" => \"v2\"}, [\n                     element('test', 'ext', {'k3' => 'v3'}, [])\n                   ])\n      e = e1 + e2\n\n      assert_equal('ROOT', e.name)\n      assert_equal('mydata', e.arg)\n      assert_equal('v1', e['k1'])\n      assert_equal('v2', e['k2'])\n      assert_equal(1, e.elements.size)\n      e.each_element('test') do |el|\n        assert_equal('test', el.name)\n        assert_equal('ext', el.arg)\n        assert_equal('v3', el[\"k3\"])\n      end\n    end\n  end\n\n  sub_test_case '#check_not_fetched' do\n    sub_test_case 'without unused' do\n      test 'can get attribute keys and original Config::Element' do\n        e = element('ROOT', 'mydata', {\"k1\" => \"v1\"}, [])\n        e.check_not_fetched { |key, elem|\n          assert_equal(\"k1\", key)\n          assert_equal(e, elem)\n        }\n      end\n    end\n\n    sub_test_case 'with unused' do\n      test 'can get unused marked attribute keys and original Config::Element' do\n        e = element('ROOT', 'mydata', {\"k1\" => \"v1\", \"k2\" => \"unused\", \"k3\" => \"k3\"})\n        e.unused = \"k2\"\n        e.check_not_fetched { |key, elem|\n          assert_equal(\"k2\", key)\n          assert_equal(e, elem)\n        }\n      end\n    end\n  end\n\n  sub_test_case '#has_key?' do\n    test 'can get boolean with key name' do\n      e = element('ROOT', 'mydata', {\"k1\" => \"v1\"}, [])\n      assert_true(e.has_key?(\"k1\"))\n      assert_false(e.has_key?(\"noexistent\"))\n    end\n  end\n\n\n  sub_test_case '#to_s' do\n    data(\"without v1_config\" => [false, <<-CONF\n<ROOT>\n  k1 v1\n  k2 \"stringVal\"\n  <test ext>\n    k2 v2\n  </test>\n</ROOT>\nCONF\n                                ],\n         \"with v1_config\" => [true, <<-CONF\n<ROOT>\n  k1 v1\n  k2 \"stringVal\"\n  <test ext>\n    k2 v2\n  </test>\n</ROOT>\nCONF\n                             ],\n        )\n    test 'dump config element with #to_s' do |data|\n      v1_config, expected = data\n      e = element('ROOT', '', {'k1' => 'v1', \"k2\" =>\"\\\"stringVal\\\"\"}, [\n                    element('test', 'ext', {'k2' => 'v2'}, [])\n                  ])\n      e.v1_config = v1_config\n      dump = expected\n      assert_not_equal(e.inspect, e.to_s)\n      assert_equal(dump, e.to_s)\n    end\n\n    test 'dump nil and default for v1' do\n      expected = <<-CONF\n<ROOT>\n  str1 \n  str2 defstring\n</ROOT>\nCONF\n      e = element('ROOT', '', {'str1' => nil, \"str2\" => :default}, [])\n      type_lookup = ->(type){ Fluent::Configurable.lookup_type(type) }\n      p = Fluent::Config::ConfigureProxy.new(\"test\", type_lookup: type_lookup)\n      p.config_param :str1, :string\n      p.config_param :str2, :string, default: \"defstring\"\n      e.corresponding_proxies << p\n      e.v1_config = true\n      assert_not_equal(e.inspect, e.to_s)\n      assert_equal(expected, e.to_s)\n    end\n  end\n\n  sub_test_case '#inspect' do\n    test 'dump config element with #inspect' do\n      e = element('ROOT', '', {'k1' => 'v1'}, [\n                   element('test', 'ext', {'k2' => 'v2'}, [])\n                  ])\n      dump = <<-CONF\n<name:ROOT, arg:, attrs:{\\\"k1\\\"=>\\\"v1\\\"}, elements:[<name:test, arg:ext, attrs:{\\\"k2\\\"=>\\\"v2\\\"}, elements:[]>]>\nCONF\n      assert_not_equal(e.to_s, e.inspect.gsub(' => ', '=>'))\n      assert_equal(dump.chomp, e.inspect.gsub(' => ', '=>'))\n    end\n  end\n\n  sub_test_case 'for sections which has secret parameter' do\n    setup do\n      @type_lookup = ->(type){ Fluent::Configurable.lookup_type(type) }\n      p1 = Fluent::Config::ConfigureProxy.new(:match, type_lookup: @type_lookup)\n      p1.config_param :str1, :string\n      p1.config_param :str2, :string, secret: true\n      p1.config_param :enum1, :enum, list: [:a, :b, :c]\n      p1.config_param :enum2, :enum, list: [:a, :b, :c], secret: true\n      p1.config_param :bool1, :bool\n      p1.config_param :bool2, :bool, secret: true\n      p1.config_param :int1, :integer\n      p1.config_param :int2, :integer, secret: true\n      p1.config_param :float1, :float\n      p1.config_param :float2, :float, secret: true\n      p2 = Fluent::Config::ConfigureProxy.new(:match, type_lookup: @type_lookup)\n      p2.config_param :size1, :size\n      p2.config_param :size2, :size, secret: true\n      p2.config_param :time1, :time\n      p2.config_param :time2, :time, secret: true\n      p2.config_param :array1, :array\n      p2.config_param :array2, :array, secret: true\n      p2.config_param :hash1, :hash\n      p2.config_param :hash2, :hash, secret: true\n      p1.config_section :mysection do\n        config_param :str1, :string\n        config_param :str2, :string, secret: true\n        config_param :enum1, :enum, list: [:a, :b, :c]\n        config_param :enum2, :enum, list: [:a, :b, :c], secret: true\n        config_param :bool1, :bool\n        config_param :bool2, :bool, secret: true\n        config_param :int1, :integer\n        config_param :int2, :integer, secret: true\n        config_param :float1, :float\n        config_param :float2, :float, secret: true\n        config_param :size1, :size\n        config_param :size2, :size, secret: true\n        config_param :time1, :time\n        config_param :time2, :time, secret: true\n        config_param :array1, :array\n        config_param :array2, :array, secret: true\n        config_param :hash1, :hash\n        config_param :hash2, :hash, secret: true\n      end\n      params = {\n        'str1' => 'aaa', 'str2' => 'bbb', 'enum1' => 'a', 'enum2' => 'b', 'bool1' => 'true', 'bool2' => 'yes',\n        'int1' => '1', 'int2' => '2', 'float1' => '1.0', 'float2' => '0.5', 'size1' => '1k', 'size2' => '1m',\n        'time1' => '5m', 'time2' => '3h', 'array1' => 'a,b,c', 'array2' => 'd,e,f',\n        'hash1' => 'a:1,b:2', 'hash2' => 'a:2,b:4',\n        'unknown1' => 'yay', 'unknown2' => 'boo',\n      }\n      e2 = Fluent::Config::Element.new('mysection', '', params.dup, [])\n      e2.corresponding_proxies << p1.sections.values.first\n      @e = Fluent::Config::Element.new('match', '**', params, [e2])\n      @e.corresponding_proxies << p1\n      @e.corresponding_proxies << p2\n    end\n\n    sub_test_case '#to_masked_element' do\n      test 'returns a new element object which has masked values for secret parameters and elements' do\n        e = @e.to_masked_element\n        assert_equal 'aaa',    e['str1']\n        assert_equal 'xxxxxx', e['str2']\n        assert_equal 'a',      e['enum1']\n        assert_equal 'xxxxxx', e['enum2']\n        assert_equal 'true',   e['bool1']\n        assert_equal 'xxxxxx', e['bool2']\n        assert_equal '1',      e['int1']\n        assert_equal 'xxxxxx', e['int2']\n        assert_equal '1.0',    e['float1']\n        assert_equal 'xxxxxx', e['float2']\n        assert_equal '1k',     e['size1']\n        assert_equal 'xxxxxx', e['size2']\n        assert_equal '5m',     e['time1']\n        assert_equal 'xxxxxx', e['time2']\n        assert_equal 'a,b,c',  e['array1']\n        assert_equal 'xxxxxx', e['array2']\n        assert_equal 'a:1,b:2', e['hash1']\n        assert_equal 'xxxxxx',  e['hash2']\n        assert_equal 'yay', e['unknown1']\n        assert_equal 'boo', e['unknown2']\n        e2 = e.elements.first\n        assert_equal 'aaa',    e2['str1']\n        assert_equal 'xxxxxx', e2['str2']\n        assert_equal 'a',      e2['enum1']\n        assert_equal 'xxxxxx', e2['enum2']\n        assert_equal 'true',   e2['bool1']\n        assert_equal 'xxxxxx', e2['bool2']\n        assert_equal '1',      e2['int1']\n        assert_equal 'xxxxxx', e2['int2']\n        assert_equal '1.0',    e2['float1']\n        assert_equal 'xxxxxx', e2['float2']\n        assert_equal '1k',     e2['size1']\n        assert_equal 'xxxxxx', e2['size2']\n        assert_equal '5m',     e2['time1']\n        assert_equal 'xxxxxx', e2['time2']\n        assert_equal 'a,b,c',  e2['array1']\n        assert_equal 'xxxxxx', e2['array2']\n        assert_equal 'a:1,b:2', e2['hash1']\n        assert_equal 'xxxxxx',  e2['hash2']\n        assert_equal 'yay', e2['unknown1']\n        assert_equal 'boo', e2['unknown2']\n      end\n    end\n\n    sub_test_case '#secret_param?' do\n      test 'returns boolean which shows values of given key will be masked' do\n        assert !@e.secret_param?('str1')\n        assert @e.secret_param?('str2')\n        assert !@e.elements.first.secret_param?('str1')\n        assert @e.elements.first.secret_param?('str2')\n      end\n    end\n\n    sub_test_case '#param_type' do\n      test 'returns parameter type which are registered in corresponding proxy' do\n        assert_equal :string, @e.param_type('str1')\n        assert_equal :string, @e.param_type('str2')\n        assert_equal :enum, @e.param_type('enum1')\n        assert_equal :enum, @e.param_type('enum2')\n        assert_nil @e.param_type('unknown1')\n        assert_nil @e.param_type('unknown2')\n      end\n    end\n\n    # sub_test_case '#dump_value'\n    sub_test_case '#dump_value' do\n      test 'dumps parameter_name and values with leading indentation' do\n        assert_equal \"str1 aaa\\n\", @e.dump_value(\"str1\", @e[\"str1\"], \"\")\n        assert_equal \"str2 xxxxxx\\n\", @e.dump_value(\"str2\", @e[\"str2\"], \"\")\n      end\n    end\n  end\n\n  sub_test_case '#set_target_worker' do\n    test 'set target_worker_id recursively' do\n      e = element('label', '@mytest', {}, [ element('filter', '**'), element('match', '**', {}, [ element('store'), element('store') ]) ])\n      e.set_target_worker_id(1)\n      assert_equal [1], e.target_worker_ids\n      assert_equal [1], e.elements[0].target_worker_ids\n      assert_equal [1], e.elements[1].target_worker_ids\n      assert_equal [1], e.elements[1].elements[0].target_worker_ids\n      assert_equal [1], e.elements[1].elements[1].target_worker_ids\n    end\n  end\n\n  sub_test_case '#for_every_workers?' do\n    test 'has target_worker_id' do\n      e = element()\n      e.set_target_worker_id(1)\n      assert_false e.for_every_workers?\n    end\n\n    test \"doesn't have target_worker_id\" do\n      e = element()\n      assert e.for_every_workers?\n    end\n  end\n\n  sub_test_case '#for_this_workers?' do\n    test 'target_worker_id == current worker_id' do\n      e = element()\n      e.set_target_worker_id(0)\n      assert e.for_this_worker?\n    end\n\n    test 'target_worker_ids includes current worker_id' do\n      e = element()\n      e.set_target_worker_ids([0])\n      assert e.for_this_worker?\n    end\n\n    test 'target_worker_id != current worker_id' do\n      e = element()\n      e.set_target_worker_id(1)\n      assert_false e.for_this_worker?\n    end\n\n    test 'target_worker_ids does not includes current worker_id' do\n      e = element()\n      e.set_target_worker_ids([1, 2])\n      assert_false e.for_this_worker?\n    end\n\n    test \"doesn't have target_worker_id\" do\n      e = element()\n      assert_false e.for_this_worker?\n    end\n  end\n\n  sub_test_case '#for_another_worker?' do\n    test 'target_worker_id == current worker_id' do\n      e = element()\n      e.set_target_worker_id(0)\n      assert_false e.for_another_worker?\n    end\n\n    test 'target_worker_ids contains current worker_id' do\n      e = element()\n      e.set_target_worker_ids([0, 1])\n      assert_false e.for_another_worker?\n    end\n\n    test 'target_worker_id != current worker_id' do\n      e = element()\n      e.set_target_worker_id(1)\n      assert e.for_another_worker?\n    end\n\n    test 'target_worker_ids does not contains current worker_id' do\n      e = element()\n      e.set_target_worker_ids([1, 2])\n      assert e.for_another_worker?\n    end\n\n    test \"doesn't have target_worker_id\" do\n      e = element()\n      assert_false e.for_another_worker?\n    end\n  end\n\n  sub_test_case '#pretty_print' do\n    test 'prints inspect to pp object' do\n      q = PP.new\n      e = element()\n      e.pretty_print(q)\n      assert_equal e.inspect, q.output\n    end\n  end\nend\n"
  },
  {
    "path": "test/config/test_literal_parser.rb",
    "content": "require_relative \"../helper\"\nrequire_relative 'assertions'\nrequire \"fluent/config/error\"\nrequire \"fluent/config/literal_parser\"\nrequire \"fluent/config/v1_parser\"\nrequire 'json'\n\nmodule Fluent::Config\n  class TestLiteralParser < ::Test::Unit::TestCase\n    def parse_text(text)\n      basepath = File.expand_path(File.dirname(__FILE__)+'/../../')\n      ss = StringScanner.new(text)\n      parser = Fluent::Config::V1Parser.new(ss, basepath, \"(test)\", eval_context)\n      parser.parse_literal\n    end\n\n    TestLiteralParserContext = Struct.new(:v1, :v2, :v3)\n\n    def v1\n      :test\n    end\n\n    def v2\n      true\n    end\n\n    def v3\n      nil\n    end\n\n    def eval_context\n      @eval_context ||= TestLiteralParserContext.new(v1, v2, v3)\n    end\n\n    sub_test_case 'boolean parsing' do\n      def test_true\n        assert_text_parsed_as('true', \"true\")\n      end\n      def test_false\n        assert_text_parsed_as('false', \"false\")\n      end\n      def test_trueX\n        assert_text_parsed_as('trueX', \"trueX\")\n      end\n      def test_falseX\n        assert_text_parsed_as('falseX', \"falseX\")\n      end\n    end\n\n    sub_test_case 'integer parsing' do\n      test('0') { assert_text_parsed_as('0', \"0\") }\n      test('1') { assert_text_parsed_as('1', \"1\") }\n      test('10') { assert_text_parsed_as('10', \"10\") }\n      test('-1') { assert_text_parsed_as('-1', \"-1\") }\n      test('-10') { assert_text_parsed_as('-10', \"-10\") }\n      test('0 ') { assert_text_parsed_as('0', \"0 \") }\n      test(' -1 ') { assert_text_parsed_as(\"-1\", ' -1 ') }\n      # string\n      test('01') { assert_text_parsed_as('01', \"01\") }\n      test('00') { assert_text_parsed_as('00', \"00\") }\n      test('-01') { assert_text_parsed_as('-01', \"-01\") }\n      test('-00') { assert_text_parsed_as('-00', \"-00\") }\n      test('0x61') { assert_text_parsed_as('0x61', \"0x61\") }\n      test('0s') { assert_text_parsed_as('0s', \"0s\") }\n    end\n\n    sub_test_case 'float parsing' do\n      test('1.1') { assert_text_parsed_as('1.1', \"1.1\") }\n      test('0.1') { assert_text_parsed_as('0.1', \"0.1\") }\n      test('0.0') { assert_text_parsed_as('0.0', \"0.0\") }\n      test('-1.1') { assert_text_parsed_as('-1.1', \"-1.1\") }\n      test('-0.1') { assert_text_parsed_as('-0.1', \"-0.1\") }\n      test('1.10') { assert_text_parsed_as('1.10', \"1.10\") }\n      # string\n      test('12e8') { assert_text_parsed_as('12e8', \"12e8\") }\n      test('12.1e7') { assert_text_parsed_as('12.1e7', \"12.1e7\") }\n      test('-12e8') { assert_text_parsed_as('-12e8', \"-12e8\") }\n      test('-12.1e7') { assert_text_parsed_as('-12.1e7', \"-12.1e7\") }\n      test('.0') { assert_text_parsed_as('.0', \".0\") }\n      test('.1') { assert_text_parsed_as('.1', \".1\") }\n      test('0.') { assert_text_parsed_as('0.', \"0.\") }\n      test('1.') { assert_text_parsed_as('1.', \"1.\") }\n      test('.0a') { assert_text_parsed_as('.0a', \".0a\") }\n      test('1.a') { assert_text_parsed_as('1.a', \"1.a\") }\n      test('0@') { assert_text_parsed_as('0@', \"0@\") }\n    end\n\n    sub_test_case 'float keywords parsing' do\n      test('NaN') { assert_text_parsed_as('NaN', \"NaN\") }\n      test('Infinity') { assert_text_parsed_as('Infinity', \"Infinity\") }\n      test('-Infinity') { assert_text_parsed_as('-Infinity', \"-Infinity\") }\n      test('NaNX') { assert_text_parsed_as('NaNX', \"NaNX\") }\n      test('InfinityX') { assert_text_parsed_as('InfinityX', \"InfinityX\") }\n      test('-InfinityX') { assert_text_parsed_as('-InfinityX', \"-InfinityX\") }\n    end\n\n    sub_test_case 'double quoted string' do\n      test('\"\"') { assert_text_parsed_as(\"\", '\"\"') }\n      test('\"text\"') { assert_text_parsed_as(\"text\", '\"text\"') }\n      test('\"\\\\\"\"') { assert_text_parsed_as(\"\\\"\", '\"\\\\\"\"') }\n      test('\"\\\\t\"') { assert_text_parsed_as(\"\\t\", '\"\\\\t\"') }\n      test('\"\\\\n\"') { assert_text_parsed_as(\"\\n\", '\"\\\\n\"') }\n      test('\"\\\\r\\\\n\"') { assert_text_parsed_as(\"\\r\\n\", '\"\\\\r\\\\n\"') }\n      test('\"\\\\f\\\\b\"') { assert_text_parsed_as(\"\\f\\b\", '\"\\\\f\\\\b\"') }\n      test('\"\\\\.t\"') { assert_text_parsed_as(\".t\", '\"\\\\.t\"') }\n      test('\"\\\\$t\"') { assert_text_parsed_as(\"$t\", '\"\\\\$t\"') }\n      test('\"\\\\\"') { assert_text_parsed_as(\"#t\", '\"\\\\#t\"') }\n      test('\"\\\\0\"') { assert_text_parsed_as(\"\\0\", '\"\\\\0\"') }\n      test('\"\\\\z\"') { assert_parse_error('\"\\\\z\"') }  # unknown escaped character\n      test('\"\\\\1\"') { assert_parse_error('\"\\\\1\"') }  # unknown escaped character\n      test('\"t') { assert_parse_error('\"t') }  # non-terminated quoted character\n      test(\"\\\"t\\nt\\\"\") { assert_text_parsed_as(\"t\\nt\", \"\\\"t\\nt\\\"\" ) } # multiline string\n      test(\"\\\"t\\\\\\nt\\\"\") { assert_text_parsed_as(\"tt\", \"\\\"t\\\\\\nt\\\"\" ) } # multiline string\n      test('t\"') { assert_text_parsed_as('t\"', 't\"') }\n      test('\".\"') { assert_text_parsed_as('.', '\".\"') }\n      test('\"*\"') { assert_text_parsed_as('*', '\"*\"') }\n      test('\"@\"') { assert_text_parsed_as('@', '\"@\"') }\n      test('\"\\\\#{test}\"') { assert_text_parsed_as(\"\\#{test}\", '\"\\\\#{test}\"') }\n      test('\"$\"') { assert_text_parsed_as('$', '\"$\"') }\n      test('\"$t\"') { assert_text_parsed_as('$t', '\"$t\"') }\n      test('\"$}\"') { assert_text_parsed_as('$}', '\"$}\"') }\n      test('\"\\\\\\\\\"') { assert_text_parsed_as(\"\\\\\", '\"\\\\\\\\\"') }\n      test('\"\\\\[\"') { assert_text_parsed_as(\"[\", '\"\\\\[\"') }\n    end\n\n    sub_test_case 'single quoted string' do\n      test(\"''\") { assert_text_parsed_as(\"\", \"''\") }\n      test(\"'text'\") { assert_text_parsed_as(\"text\", \"'text'\") }\n      test(\"'\\\\''\") { assert_text_parsed_as('\\'', \"'\\\\''\") }\n      test(\"'\\\\t'\") { assert_text_parsed_as('\\t', \"'\\\\t'\") }\n      test(\"'\\\\n'\") { assert_text_parsed_as('\\n', \"'\\\\n'\") }\n      test(\"'\\\\r\\\\n'\") { assert_text_parsed_as('\\r\\n', \"'\\\\r\\\\n'\") }\n      test(\"'\\\\f\\\\b'\") { assert_text_parsed_as('\\f\\b', \"'\\\\f\\\\b'\") }\n      test(\"'\\\\.t'\") { assert_text_parsed_as('\\.t', \"'\\\\.t'\") }\n      test(\"'\\\\$t'\") { assert_text_parsed_as('\\$t', \"'\\\\$t'\") }\n      test(\"'\\\\#t'\") { assert_text_parsed_as('\\#t', \"'\\\\#t'\") }\n      test(\"'\\\\z'\") { assert_text_parsed_as('\\z', \"'\\\\z'\") }\n      test(\"'\\\\0'\") { assert_text_parsed_as('\\0', \"'\\\\0'\") }\n      test(\"'\\\\1'\") { assert_text_parsed_as('\\1', \"'\\\\1'\") }\n      test(\"'t\") { assert_parse_error(\"'t\") }  # non-terminated quoted character\n      test(\"t'\") { assert_text_parsed_as(\"t'\", \"t'\") }\n      test(\"'.'\") { assert_text_parsed_as('.', \"'.'\") }\n      test(\"'*'\") { assert_text_parsed_as('*', \"'*'\") }\n      test(\"'@'\") { assert_text_parsed_as('@', \"'@'\") }\n      test(%q['#{test}']) { assert_text_parsed_as('#{test}', %q['#{test}']) }\n      test(\"'$'\") { assert_text_parsed_as('$', \"'$'\") }\n      test(\"'$t'\") { assert_text_parsed_as('$t', \"'$t'\") }\n      test(\"'$}'\") { assert_text_parsed_as('$}', \"'$}'\") }\n      test(\"'\\\\\\\\'\") { assert_text_parsed_as('\\\\', \"'\\\\\\\\'\") }\n      test(\"'\\\\['\") { assert_text_parsed_as('\\[', \"'\\\\['\") }\n    end\n\n    sub_test_case 'nonquoted string parsing' do\n      test(\"''\") { assert_text_parsed_as(nil, '') }\n      test('text') { assert_text_parsed_as('text', 'text') }\n      test('\\\"') { assert_text_parsed_as('\\\"', '\\\"') }\n      test('\\t') { assert_text_parsed_as('\\t', '\\t') }\n      test('\\n') { assert_text_parsed_as('\\n', '\\n') }\n      test('\\r\\n') { assert_text_parsed_as('\\r\\n', '\\r\\n') }\n      test('\\f\\b') { assert_text_parsed_as('\\f\\b', '\\f\\b') }\n      test('\\.t') { assert_text_parsed_as('\\.t', '\\.t') }\n      test('\\$t') { assert_text_parsed_as('\\$t', '\\$t') }\n      test('\\#t') { assert_text_parsed_as('\\#t', '\\#t') }\n      test('\\z') { assert_text_parsed_as('\\z', '\\z') }\n      test('\\0') { assert_text_parsed_as('\\0', '\\0') }\n      test('\\1') { assert_text_parsed_as('\\1', '\\1') }\n      test('.') { assert_text_parsed_as('.', '.') }\n      test('*') { assert_text_parsed_as('*', '*') }\n      test('@') { assert_text_parsed_as('@', '@') }\n      test('#{test}') { assert_text_parsed_as('#{test}', '#{test}') }\n      test('$') { assert_text_parsed_as('$', '$') }\n      test('$t') { assert_text_parsed_as('$t', '$t') }\n      test('$}') { assert_text_parsed_as('$}', '$}') }\n      test('\\\\\\\\') { assert_text_parsed_as('\\\\\\\\', '\\\\\\\\') }\n      test('\\[') { assert_text_parsed_as('\\[', '\\[') }\n      test('#foo') { assert_text_parsed_as('#foo', '#foo') } # not comment out\n      test('foo#bar') { assert_text_parsed_as('foo#bar', 'foo#bar') } # not comment out\n      test(' text') { assert_text_parsed_as('text', ' text') } # remove starting spaces\n      test(' #foo') { assert_text_parsed_as('#foo', ' #foo') } # remove starting spaces\n      test('foo #bar') { assert_text_parsed_as('foo', 'foo #bar') } # comment out\n      test('foo\\t#bar') { assert_text_parsed_as('foo', \"foo\\t#bar\") } # comment out\n\n      test('t') { assert_text_parsed_as('t', 't') }\n      test('T') { assert_text_parsed_as('T', 'T') }\n      test('_') { assert_text_parsed_as('_', '_') }\n      test('T1') { assert_text_parsed_as('T1', 'T1') }\n      test('_2') { assert_text_parsed_as('_2', '_2') }\n      test('t0') { assert_text_parsed_as('t0', 't0') }\n      test('t@') { assert_text_parsed_as('t@', 't@') }\n      test('t-') { assert_text_parsed_as('t-', 't-') }\n      test('t.') { assert_text_parsed_as('t.', 't.') }\n      test('t+') { assert_text_parsed_as('t+', 't+') }\n      test('t/') { assert_text_parsed_as('t/', 't/') }\n      test('t=') { assert_text_parsed_as('t=', 't=') }\n      test('t,') { assert_text_parsed_as('t,', 't,') }\n      test('0t') { assert_text_parsed_as('0t', \"0t\") }\n      test('@1t') { assert_text_parsed_as('@1t', '@1t') }\n      test('-1t') { assert_text_parsed_as('-1t', '-1t') }\n      test('.1t') { assert_text_parsed_as('.1t', '.1t') }\n      test(',1t') { assert_text_parsed_as(',1t', ',1t') }\n      test('.t') { assert_text_parsed_as('.t', '.t') }\n      test('*t') { assert_text_parsed_as('*t', '*t') }\n      test('@t') { assert_text_parsed_as('@t', '@t') }\n      test('{t') { assert_parse_error('{t') }  # '{' begins map\n      test('t{') { assert_text_parsed_as('t{', 't{') }\n      test('}t') { assert_text_parsed_as('}t', '}t') }\n      test('[t') { assert_parse_error('[t') }  # '[' begins array\n      test('t[') { assert_text_parsed_as('t[', 't[') }\n      test(']t') { assert_text_parsed_as(']t', ']t') }\n      test('t:') { assert_text_parsed_as('t:', 't:') }\n      test('t;') { assert_text_parsed_as('t;', 't;') }\n      test('t?') { assert_text_parsed_as('t?', 't?') }\n      test('t^') { assert_text_parsed_as('t^', 't^') }\n      test('t`') { assert_text_parsed_as('t`', 't`') }\n      test('t~') { assert_text_parsed_as('t~', 't~') }\n      test('t|') { assert_text_parsed_as('t|', 't|') }\n      test('t>') { assert_text_parsed_as('t>', 't>') }\n      test('t<') { assert_text_parsed_as('t<', 't<') }\n      test('t(') { assert_text_parsed_as('t(', 't(') }\n    end\n\n    sub_test_case 'embedded ruby code parsing' do\n      test('\"#{v1}\"') { assert_text_parsed_as(\"#{v1}\", '\"#{v1}\"') }\n      test('\"#{v2}\"') { assert_text_parsed_as(\"#{v2}\", '\"#{v2}\"') }\n      test('\"#{v3}\"') { assert_text_parsed_as(\"#{v3}\", '\"#{v3}\"') }\n      test('\"#{1+1}\"') { assert_text_parsed_as(\"2\", '\"#{1+1}\"') }\n      test('\"#{}\"') { assert_text_parsed_as(\"\", '\"#{}\"') }\n      test('\"t#{v1}\"') { assert_text_parsed_as(\"t#{v1}\", '\"t#{v1}\"') }\n      test('\"t#{v1}t\"') { assert_text_parsed_as(\"t#{v1}t\", '\"t#{v1}t\"') }\n      test('\"#{\"}\"}\"') { assert_text_parsed_as(\"}\", '\"#{\"}\"}\"') }\n      test('\"#{#}\"') { assert_parse_error('\"#{#}\"') }  # error in embedded ruby code\n      test(\"\\\"\\#{\\n=begin\\n}\\\"\") { assert_parse_error(\"\\\"\\#{\\n=begin\\n}\\\"\") }  # error in embedded ruby code\n      test('\"#{v1}foo#{v2}\"') { assert_text_parsed_as(\"#{v1}foo#{v2}\", '\"#{v1}foo#{v2}\"') }\n      test('\"#{1+1}foo#{2+2}bar\"') { assert_text_parsed_as(\"#{1+1}foo#{2+2}bar\", '\"#{1+1}foo#{2+2}bar\"') }\n      test('\"foo#{hostname}\"') { assert_text_parsed_as(\"foo#{Socket.gethostname}\", '\"foo#{hostname}\"') }\n      test('\"foo#{worker_id}\"') {\n        ENV.delete('SERVERENGINE_WORKER_ID')\n        assert_text_parsed_as(\"foo\", '\"foo#{worker_id}\"')\n        ENV['SERVERENGINE_WORKER_ID'] = '1'\n        assert_text_parsed_as(\"foo1\", '\"foo#{worker_id}\"')\n        ENV.delete('SERVERENGINE_WORKER_ID')\n      }\n      test('nil') { assert_text_parsed_as(nil, '\"#{raise SetNil}\"') }\n      test('default') { assert_text_parsed_as(:default, '\"#{raise SetDefault}\"') }\n      test('nil helper') { assert_text_parsed_as(nil, '\"#{use_nil}\"') }\n      test('default helper') { assert_text_parsed_as(:default, '\"#{use_default}\"') }\n    end\n\n    sub_test_case 'array parsing' do\n      test('[]') { assert_text_parsed_as_json([], '[]') }\n      test('[1]') { assert_text_parsed_as_json([1], '[1]') }\n      test('[1,2]') { assert_text_parsed_as_json([1,2], '[1,2]') }\n      test('[1, 2]') { assert_text_parsed_as_json([1,2], '[1, 2]') }\n      test('[ 1 , 2 ]') { assert_text_parsed_as_json([1,2], '[ 1 , 2 ]') }\n      test('[1,2,]') { assert_parse_error('[1,2,]') } # TODO: Need trailing commas support?\n      test(\"[\\n1\\n,\\n2\\n]\") { assert_text_parsed_as_json([1,2], \"[\\n1\\n,\\n2\\n]\") }\n      test('[\"a\"]') { assert_text_parsed_as_json([\"a\"], '[\"a\"]') }\n      test('[\"a\",\"b\"]') { assert_text_parsed_as_json([\"a\",\"b\"], '[\"a\",\"b\"]') }\n      test('[ \"a\" , \"b\" ]') { assert_text_parsed_as_json([\"a\",\"b\"], '[ \"a\" , \"b\" ]') }\n      test(\"[\\n\\\"a\\\"\\n,\\n\\\"b\\\"\\n]\") { assert_text_parsed_as_json([\"a\",\"b\"], \"[\\n\\\"a\\\"\\n,\\n\\\"b\\\"\\n]\") }\n      test('[\"ab\",\"cd\"]') { assert_text_parsed_as_json([\"ab\",\"cd\"], '[\"ab\",\"cd\"]') }\n      test('[\"a\",\"#{v1}\"') { assert_text_parsed_as_json([\"a\",\"#{v1}\"], '[\"a\",\"#{v1}\"]') }\n      test('[\"a\",\"#{v1}\",\"#{v2}\"]') { assert_text_parsed_as_json([\"a\",\"#{v1}\",\"#{v2}\"], '[\"a\",\"#{v1}\",\"#{v2}\"]') }\n      test('[\"a\",\"#{v1} #{v2}\"]') { assert_text_parsed_as_json([\"a\",\"#{v1} #{v2}\"], '[\"a\",\"#{v1} #{v2}\"]') }\n      test('[\"a\",\"#{hostname}\"]') { assert_text_parsed_as_json([\"a\",\"#{Socket.gethostname}\"], '[\"a\",\"#{hostname}\"]') }\n      test('[\"a\",\"foo#{worker_id}\"]') {\n        ENV.delete('SERVERENGINE_WORKER_ID')\n        assert_text_parsed_as('[\"a\",\"foo\"]', '[\"a\",\"foo#{worker_id}\"]')\n        ENV['SERVERENGINE_WORKER_ID'] = '1'\n        assert_text_parsed_as('[\"a\",\"foo1\"]', '[\"a\",\"foo#{worker_id}\"]')\n        ENV.delete('SERVERENGINE_WORKER_ID')\n      }\n      json_array_with_js_comment = <<EOA\n[\n \"a\", // this is a\n \"b\", // this is b\n \"c\"  // this is c\n]\nEOA\n      test(json_array_with_js_comment) { assert_text_parsed_as_json([\"a\",\"b\",\"c\"], json_array_with_js_comment) }\n      json_array_with_comment = <<EOA\n[\n \"a\", # this is a\n \"b\", # this is b\n \"c\"  # this is c\n]\nEOA\n      test(json_array_with_comment) { assert_text_parsed_as_json([\"a\",\"b\",\"c\"], json_array_with_comment) }\n      json_array_with_tailing_comma = <<EOA\n[\n \"a\", # this is a\n \"b\", # this is b\n \"c\", # this is c\n]\nEOA\n      test(json_array_with_tailing_comma) { assert_parse_error(json_array_with_tailing_comma) }\n    end\n\n    sub_test_case 'map parsing' do\n      test('{}') { assert_text_parsed_as_json({}, '{}') }\n      test('{\"a\":1}') { assert_text_parsed_as_json({\"a\"=>1}, '{\"a\":1}') }\n      test('{\"a\":1,\"b\":2}') { assert_text_parsed_as_json({\"a\"=>1,\"b\"=>2}, '{\"a\":1,\"b\":2}') }\n      test('{ \"a\" : 1 , \"b\" : 2 }') { assert_text_parsed_as_json({\"a\"=>1,\"b\"=>2}, '{ \"a\" : 1 , \"b\" : 2 }') }\n      test('{\"a\":1,\"b\":2,}') { assert_parse_error('{\"a\":1,\"b\":2,}') } # TODO: Need trailing commas support?\n      test('{\\n\\\"a\\\"\\n:\\n1\\n,\\n\\\"b\\\"\\n:\\n2\\n}') { assert_text_parsed_as_json({\"a\"=>1,\"b\"=>2}, \"{\\n\\\"a\\\"\\n:\\n1\\n,\\n\\\"b\\\"\\n:\\n2\\n}\") }\n      test('{\"a\":\"b\"}') { assert_text_parsed_as_json({\"a\"=>\"b\"}, '{\"a\":\"b\"}') }\n      test('{\"a\":\"b\",\"c\":\"d\"}') { assert_text_parsed_as_json({\"a\"=>\"b\",\"c\"=>\"d\"}, '{\"a\":\"b\",\"c\":\"d\"}') }\n      test('{ \"a\" : \"b\" , \"c\" : \"d\" }') { assert_text_parsed_as_json({\"a\"=>\"b\",\"c\"=>\"d\"}, '{ \"a\" : \"b\" , \"c\" : \"d\" }') }\n      test('{\\n\\\"a\\\"\\n:\\n\\\"b\\\"\\n,\\n\\\"c\\\"\\n:\\n\\\"d\\\"\\n}') { assert_text_parsed_as_json({\"a\"=>\"b\",\"c\"=>\"d\"}, \"{\\n\\\"a\\\"\\n:\\n\\\"b\\\"\\n,\\n\\\"c\\\"\\n:\\n\\\"d\\\"\\n}\") }\n      test('{\"a\":\"b\",\"c\":\"#{v1}\"}') { assert_text_parsed_as_json({\"a\"=>\"b\",\"c\"=>\"#{v1}\"}, '{\"a\":\"b\",\"c\":\"#{v1}\"}') }\n      test('{\"a\":\"b\",\"#{v1}\":\"d\"}') { assert_text_parsed_as_json({\"a\"=>\"b\",\"#{v1}\"=>\"d\"}, '{\"a\":\"b\",\"#{v1}\":\"d\"}') }\n      test('{\"a\":\"#{v1}\",\"c\":\"#{v2}\"}') { assert_text_parsed_as_json({\"a\"=>\"#{v1}\",\"c\"=>\"#{v2}\"}, '{\"a\":\"#{v1}\",\"c\":\"#{v2}\"}') }\n      test('{\"a\":\"b\",\"c\":\"d #{v1} #{v2}\"}') { assert_text_parsed_as_json({\"a\"=>\"b\",\"c\"=>\"d #{v1} #{v2}\"}, '{\"a\":\"b\",\"c\":\"d #{v1} #{v2}\"}') }\n      test('{\"a\":\"#{hostname}\"}') { assert_text_parsed_as_json({\"a\"=>\"#{Socket.gethostname}\"}, '{\"a\":\"#{hostname}\"}') }\n      test('{\"a\":\"foo#{worker_id}\"}') {\n        ENV.delete('SERVERENGINE_WORKER_ID')\n        assert_text_parsed_as('{\"a\":\"foo\"}', '{\"a\":\"foo#{worker_id}\"}')\n        ENV['SERVERENGINE_WORKER_ID'] = '1'\n        assert_text_parsed_as('{\"a\":\"foo1\"}', '{\"a\":\"foo#{worker_id}\"}')\n        ENV.delete('SERVERENGINE_WORKER_ID')\n      }\n      test('no quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'test'}, '{\"a\":\"b\",\"c\":\"#{v1}\"}') }\n      test('single quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'#{v1}'}, '\\'{\"a\":\"b\",\"c\":\"#{v1}\"}\\'') }\n      test('double quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'test'}, '\"{\\\"a\\\":\\\"b\\\",\\\"c\\\":\\\"#{v1}\\\"}\"') }\n      json_hash_with_comment = <<EOH\n{\n \"a\": 1, # this is a\n \"b\": 2, # this is b\n \"c\": 3  # this is c\n}\nEOH\n      test(json_hash_with_comment) { assert_text_parsed_as_json({\"a\"=>1,\"b\"=>2,\"c\"=>3}, json_hash_with_comment) }\n    end\n  end\nend\n"
  },
  {
    "path": "test/config/test_plugin_configuration.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/input'\nrequire 'fluent/test/driver/input'\n\nmodule ConfigurationForPlugins\n  class AllBooleanParams < Fluent::Plugin::Input\n    config_param :flag1, :bool, default: true\n    config_param :flag2, :bool, default: true\n    config_param :flag3, :bool, default: false\n    config_param :flag4, :bool, default: false\n\n    config_section :child, param_name: :children, multi: true, required: true do\n      config_param :flag1, :bool, default: true\n      config_param :flag2, :bool, default: true\n      config_param :flag3, :bool, default: false\n      config_param :flag4, :bool, default: false\n    end\n  end\n\n  class BooleanParamsWithoutValue < ::Test::Unit::TestCase\n    CONFIG = <<CONFIG\n    flag1\n    flag2 # yaaaaaaaaaay\n    flag3\n    flag4 # yaaaaaaaaaay\n    <child>\n      flag1\n      flag2 # yaaaaaaaaaay\n      flag3\n      flag4 # yaaaaaaaaaay\n    </child>\n    <child>\n      flag1 # yaaaaaaaaaay\n      flag2\n      flag3 # yaaaaaaaaaay\n      flag4\n    </child>\n    # with following whitespace\n    <child>\n      flag1\n      flag2\n      flag3\n      flag4\n    </child>\nCONFIG\n\n    test 'create plugin via driver' do\n      d = Fluent::Test::Driver::Input.new(AllBooleanParams)\n      d.configure(CONFIG)\n      assert_equal([true] * 4, [d.instance.flag1, d.instance.flag2, d.instance.flag3, d.instance.flag4])\n      num_of_sections = 3\n      assert_equal num_of_sections, d.instance.children.size\n      assert_equal([true] * (num_of_sections * 4), d.instance.children.map{|c| [c.flag1, c.flag2, c.flag3, c.flag4]}.flatten)\n    end\n  end\nend\n"
  },
  {
    "path": "test/config/test_section.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/config/section'\nrequire 'pp'\n\nmodule Fluent::Config\n  class TestSection < ::Test::Unit::TestCase\n    sub_test_case Fluent::Config::Section do\n      sub_test_case 'class' do\n        sub_test_case '.name' do\n          test 'returns its full module name as String' do\n            assert_equal('Fluent::Config::Section', Fluent::Config::Section.name)\n          end\n        end\n      end\n\n      sub_test_case 'instance object' do\n        sub_test_case '#initialize' do\n          test 'creates blank object without argument' do\n            s = Fluent::Config::Section.new\n            assert_equal({}, s.instance_eval{ @params })\n          end\n\n          test 'creates object which contains specified hash object itself' do\n            hash = {\n              name: 'tagomoris',\n              age: 34,\n              send: 'email',\n              klass: 'normal',\n              keys: 5,\n            }\n            s1 = Fluent::Config::Section.new(hash)\n            assert_equal(hash, s1.instance_eval { @params })\n            assert_equal(\"tagomoris\", s1[:name])\n            assert_equal(34, s1[:age])\n            assert_equal(\"email\", s1[:send])\n            assert_equal(\"normal\", s1[:klass])\n            assert_equal(5, s1[:keys])\n\n            assert_equal(\"tagomoris\", s1.name)\n            assert_equal(34, s1.age)\n            assert_equal(\"email\", s1.send)\n            assert_equal(\"normal\", s1.klass)\n            assert_equal(5, s1.keys)\n          end\n\n          test 'creates object which contains specified hash object itself, including fields with at prefix' do\n            hash = {\n              name: 'tagomoris',\n              age: 34,\n              send: 'email',\n              klass: 'normal',\n              keys: 5,\n            }\n            hash['@id'.to_sym] = 'myid'\n\n            s1 = Fluent::Config::Section.new(hash)\n            assert_equal('myid', s1['@id'])\n            assert_equal('myid', s1['@id'.to_sym])\n            assert_equal('myid', s1.__send__('@id'.to_sym))\n          end\n\n          test 'creates object and config element which corresponds to section object itself' do\n            hash = {\n              name: 'tagomoris',\n              age: 34,\n              send: 'email',\n              klass: 'normal',\n              keys: 5,\n            }\n            hash['@id'.to_sym] = 'myid'\n            conf = config_element('section', '', {'name' => 'tagomoris', 'age' => 34, 'send' => 'email', 'klass' => 'normal', 'keys' => 5})\n            s2 = Fluent::Config::Section.new(hash, conf)\n            assert s2.corresponding_config_element.is_a?(Fluent::Config::Element)\n          end\n        end\n\n        sub_test_case '#class' do\n          test 'returns class constant' do\n            assert_equal Fluent::Config::Section, Fluent::Config::Section.new({}).class\n          end\n        end\n\n        sub_test_case '#object_id' do\n          test 'returns its object id' do\n            s1 = Fluent::Config::Section.new({})\n            assert s1.object_id\n            s2 = Fluent::Config::Section.new({})\n            assert s2.object_id\n            assert_not_equal s1.object_id, s2.object_id\n          end\n        end\n\n        sub_test_case '#to_h' do\n          test 'returns internal hash itself' do\n            hash = {\n              name: 'tagomoris',\n              age: 34,\n              send: 'email',\n              klass: 'normal',\n              keys: 5,\n            }\n            s = Fluent::Config::Section.new(hash)\n            assert_equal(hash, s.to_h)\n            assert_instance_of(Hash, s.to_h)\n          end\n        end\n\n        sub_test_case '#instance_of?' do\n          test 'can judge whether it is a Section object or not' do\n            s = Fluent::Config::Section.new\n            assert_true(s.instance_of?(Fluent::Config::Section))\n            assert_false(s.instance_of?(BasicObject))\n          end\n        end\n\n        sub_test_case '#is_a?' do\n          test 'can judge whether it belongs to or not' do\n            s = Fluent::Config::Section.new\n            assert_true(s.is_a?(Fluent::Config::Section))\n            assert_true(s.kind_of?(Fluent::Config::Section))\n            assert_true(s.is_a?(BasicObject))\n          end\n        end\n\n        sub_test_case '#+' do\n          test 'can merge 2 sections: argument side is primary, internal hash is newly created' do\n            h1 = {name: \"s1\", num: 10, klass: \"A\"}\n            s1 = Fluent::Config::Section.new(h1)\n\n            h2 = {name: \"s2\", klass: \"A\", num2: \"5\", num3: \"8\"}\n            s2 = Fluent::Config::Section.new(h2)\n            s = s1 + s2\n\n            assert_not_equal(h1.object_id, s.to_h.object_id)\n            assert_not_equal(h2.object_id, s.to_h.object_id)\n\n            assert_equal(\"s2\", s.name)\n            assert_equal(10, s.num)\n            assert_equal(\"A\", s.klass)\n            assert_equal(\"5\", s.num2)\n            assert_equal(\"8\", s.num3)\n          end\n        end\n\n        sub_test_case '#to_s' do\n          test '#to_s == #inspect' do\n            h1 = {name: \"s1\", num: 10, klass: \"A\"}\n            s1 = Fluent::Config::Section.new(h1)\n\n            assert_equal(s1.to_s, s1.inspect)\n          end\n        end\n\n        data(\"inspect\" => [:inspect, true],\n             \"nil?\" => [:nil?, true],\n             \"to_h\" => [:to_h, true],\n             \"+\" =>   [:+, true],\n             \"instance_of?\" => [:instance_of?, true],\n             \"kind_of?\" => [:kind_of?, true],\n             \"[]\" => [:[], true],\n             \"respond_to?\" => [:respond_to?, true],\n             \"respond_to_missing?\" => [:respond_to_missing?, true],\n             \"!\" => [:!, true],\n             \"!=\" => [:!=, true],\n             \"==\" => [:==, true],\n             \"equal?\" => [:equal?, true],\n             \"instance_eval\" => [:instance_eval, true],\n             \"instance_exec\" => [:instance_exec, true],\n             \"method_missing\" => [:method_missing, false],\n             \"singleton_method_added\" => [:singleton_method_added, false],\n             \"singleton_method_removed\" => [:singleton_method_removed, false],\n             \"singleton_method_undefined\" => [:singleton_method_undefined, false],\n             \"no_such_method\" => [:no_such_method, false])\n        test '#respond_to?' do |data|\n          method, expected = data\n          h1 = {name: \"s1\", num: 10, klass: \"A\"}\n          s1 = Fluent::Config::Section.new(h1)\n          assert_equal(expected, s1.respond_to?(method))\n        end\n\n        test '#pretty_print' do\n          q = PP.new\n          h1 = {name: \"s1\", klass: \"A\"}\n          s1 = Fluent::Config::Section.new(h1)\n          s1.pretty_print(q)\n          assert_equal s1.inspect, q.output\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/config/test_system_config.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/configurable'\nrequire 'fluent/config/element'\nrequire 'fluent/config/section'\nrequire 'fluent/system_config'\n\nmodule Fluent::Config\n  class FakeSupervisor\n    attr_writer :log_level\n\n    def initialize(**opt)\n      @system_config = nil\n      @cl_opt = {\n        workers: nil,\n        restart_worker_interval: nil,\n        root_dir: nil,\n        log_level: Fluent::Log::LEVEL_INFO,\n        suppress_interval: nil,\n        suppress_config_dump: nil,\n        suppress_repeated_stacktrace: nil,\n        log_event_label: nil,\n        log_event_verbose: nil,\n        without_source: nil,\n        with_source_only: nil,\n        enable_input_metrics: nil,\n        enable_size_metrics: nil,\n        emit_error_log_interval: nil,\n        file_permission: nil,\n        dir_permission: nil,\n      }.merge(opt)\n    end\n\n    def for_system_config\n      opt = {}\n      # this is copy from Supervisor#build_system_config\n      Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|\n        if @cl_opt.key?(param) && !@cl_opt[param].nil?\n          if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO\n            # info level can't be specified via command line option.\n            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.\n            next\n          end\n\n          opt[param] = @cl_opt[param]\n        end\n      end\n\n      opt\n    end\n  end\n\n  class TestSystemConfig < ::Test::Unit::TestCase\n    TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/tmp/system_config/#{ENV['TEST_ENV_NUMBER']}\")\n\n    def parse_text(text)\n      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')\n      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }\n    end\n\n    test 'should not override default configurations when no parameters' do\n      conf = parse_text(<<-EOS)\n        <system>\n        </system>\n      EOS\n      s = FakeSupervisor.new\n      sc = Fluent::SystemConfig.new(conf)\n      sc.overwrite_variables(**s.for_system_config)\n      assert_equal(1, sc.workers)\n      assert_equal(0, sc.restart_worker_interval)\n      assert_nil(sc.root_dir)\n      assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)\n      assert_nil(sc.suppress_repeated_stacktrace)\n      assert_nil(sc.ignore_repeated_log_interval)\n      assert_nil(sc.emit_error_log_interval)\n      assert_nil(sc.suppress_config_dump)\n      assert_nil(sc.without_source)\n      assert_nil(sc.with_source_only)\n      assert_true(sc.enable_input_metrics)\n      assert_nil(sc.enable_size_metrics)\n      assert_nil(sc.enable_msgpack_time_support)\n      assert(!sc.enable_jit)\n      assert_nil(sc.log.path)\n      assert_equal(:text, sc.log.format)\n      assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)\n      assert_equal(:none, sc.log.forced_stacktrace_level)\n    end\n\n    data(\n      'workers' => ['workers', 3],\n      'restart_worker_interval' => ['restart_worker_interval', 60],\n      'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],\n      'log_level' => ['log_level', 'error'],\n      'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],\n      'ignore_repeated_log_interval' => ['ignore_repeated_log_interval', 10],\n      'log_event_verbose' => ['log_event_verbose', true],\n      'suppress_config_dump' => ['suppress_config_dump', true],\n      'without_source' => ['without_source', true],\n      'with_source_only' => ['with_source_only', true],\n      'strict_config_value' => ['strict_config_value', true],\n      'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],\n      'enable_input_metrics' => ['enable_input_metrics', false],\n      'enable_size_metrics' => ['enable_size_metrics', true],\n      'enable_jit' => ['enable_jit', true],\n    )\n    test \"accepts parameters\" do |(k, v)|\n      conf = parse_text(<<-EOS)\n          <system>\n            #{k} #{v}\n          </system>\n      EOS\n      s = FakeSupervisor.new\n      sc = Fluent::SystemConfig.new(conf)\n      sc.overwrite_variables(**s.for_system_config)\n      if k == 'log_level'\n        assert_equal(Fluent::Log::LEVEL_ERROR, sc.__send__(k))\n      else\n        assert_equal(v, sc.__send__(k))\n      end\n    end\n\n    test \"log parameters\" do\n      conf = parse_text(<<-EOS)\n          <system>\n            <log>\n              path /tmp/fluentd.log\n              format json\n              time_format %Y\n              forced_stacktrace_level info\n            </log>\n          </system>\n      EOS\n      s = FakeSupervisor.new\n      sc = Fluent::SystemConfig.new(conf)\n      sc.overwrite_variables(**s.for_system_config)\n      assert_equal('/tmp/fluentd.log', sc.log.path)\n      assert_equal(:json, sc.log.format)\n      assert_equal('%Y', sc.log.time_format)\n      assert_equal(Fluent::Log::LEVEL_INFO, sc.log.forced_stacktrace_level)\n    end\n\n    # info is removed because info level can't be specified via command line\n    data('trace' => Fluent::Log::LEVEL_TRACE,\n         'debug' => Fluent::Log::LEVEL_DEBUG,\n         'warn' => Fluent::Log::LEVEL_WARN,\n         'error' => Fluent::Log::LEVEL_ERROR,\n         'fatal' => Fluent::Log::LEVEL_FATAL)\n    test 'log_level is ignored when log_level related command line option is passed' do |level|\n      conf = parse_text(<<-EOS)\n        <system>\n          log_level info\n        </system>\n      EOS\n      s = FakeSupervisor.new(log_level: level)\n      sc = Fluent::SystemConfig.new(conf)\n      sc.overwrite_variables(**s.for_system_config)\n      assert_equal(level, sc.log_level)\n    end\n\n    sub_test_case \"log rotation\" do\n      data('daily' => \"daily\",\n           'weekly' => 'weekly',\n           'monthly' => 'monthly')\n      test \"strings for rotate_age\" do |age|\n        conf = parse_text(<<-EOS)\n          <system>\n            <log>\n              rotate_age #{age}\n            </log>\n          </system>\n        EOS\n        sc = Fluent::SystemConfig.new(conf)\n        assert_equal(age, sc.log.rotate_age)\n      end\n\n      test \"numeric number for rotate age\" do\n        conf = parse_text(<<-EOS)\n          <system>\n            <log>\n              rotate_age 3\n            </log>\n          </system>\n        EOS\n        sc = Fluent::SystemConfig.new(conf)\n        assert_equal(3, sc.log.rotate_age)\n      end\n\n      data(h: ['100', 100],\n           k: ['1k', 1024],\n           m: ['1m', 1024 * 1024],\n           g: ['1g', 1024 * 1024 * 1024])\n      test \"numeric and SI prefix for rotate_size\" do |(label, size)|\n        conf = parse_text(<<-EOS)\n          <system>\n            <log>\n              rotate_size #{label}\n            </log>\n          </system>\n        EOS\n        sc = Fluent::SystemConfig.new(conf)\n        assert_equal(size, sc.log.rotate_size)\n      end\n    end\n\n    test \"source-only-buffer parameters\" do\n      conf = parse_text(<<~EOS)\n        <system>\n          <source_only_buffer>\n            flush_thread_count 4\n            overflow_action throw_exception\n            path /tmp/source-only-buffer\n            flush_interval 1\n            chunk_limit_size 100\n            total_limit_size 1000\n            compress gzip\n          </source_only_buffer>\n        </system>\n      EOS\n      s = FakeSupervisor.new\n      sc = Fluent::SystemConfig.new(conf)\n      sc.overwrite_variables(**s.for_system_config)\n\n      assert_equal(\n        [\n          4,\n          :throw_exception,\n          \"/tmp/source-only-buffer\",\n          1,\n          100,\n          1000,\n          :gzip,\n        ],\n        [\n          sc.source_only_buffer.flush_thread_count,\n          sc.source_only_buffer.overflow_action,\n          sc.source_only_buffer.path,\n          sc.source_only_buffer.flush_interval,\n          sc.source_only_buffer.chunk_limit_size,\n          sc.source_only_buffer.total_limit_size,\n          sc.source_only_buffer.compress,\n        ]\n      )\n    end\n  end\nend\n"
  },
  {
    "path": "test/config/test_types.rb",
    "content": "require 'helper'\nrequire 'fluent/config/types'\n\nclass TestConfigTypes < ::Test::Unit::TestCase\n  include Fluent\n\n  sub_test_case 'Config.size_value' do\n    data(\"2k\" => [2048, \"2k\"],\n         \"2K\" => [2048, \"2K\"],\n         \"3m\" => [3145728, \"3m\"],\n         \"3M\" => [3145728, \"3M\"],\n         \"4g\" => [4294967296, \"4g\"],\n         \"4G\" => [4294967296, \"4G\"],\n         \"5t\" => [5497558138880, \"5t\"],\n         \"5T\" => [5497558138880, \"5T\"],\n         \"6\"  => [6, \"6\"])\n    test 'normal case' do |(expected, val)|\n      assert_equal(expected, Config.size_value(val))\n      assert_equal(expected, Config.size_value(val, { strict: true }))\n    end\n\n    data(\"integer\" => [6, 6],\n         \"hoge\" => [0, \"hoge\"],\n         \"empty\" => [0, \"\"])\n    test 'not assumed case' do |(expected, val)|\n      assert_equal(expected, Config.size_value(val))\n    end\n\n    test 'nil' do\n      assert_equal(nil, Config.size_value(nil))\n    end\n\n    data(\"integer\" => [6, 6],\n         \"hoge\" => [Fluent::ConfigError.new('name1: invalid value for Integer(): \"hoge\"'), \"hoge\"],\n         \"empty\" => [Fluent::ConfigError.new('name1: invalid value for Integer(): \"\"'), \"\"])\n    test 'not assumed case with strict' do |(expected, val)|\n      if expected.kind_of? Exception\n        assert_raise(expected) do\n          Config.size_value(val, { strict: true }, \"name1\")\n        end\n      else\n        assert_equal(expected, Config.size_value(val, { strict: true }, \"name1\"))\n      end\n    end\n\n    test 'nil with strict' do\n      assert_equal(nil, Config.size_value(nil, { strict: true }))\n    end\n  end\n\n  sub_test_case 'Config.time_value' do\n    data(\"10s\" => [10, \"10s\"],\n         \"10sec\" => [10, \"10sec\"],\n         \"2m\" => [120, \"2m\"],\n         \"3h\" => [10800, \"3h\"],\n         \"4d\" => [345600, \"4d\"])\n    test 'normal case' do |(expected, val)|\n      assert_equal(expected, Config.time_value(val))\n      assert_equal(expected, Config.time_value(val, { strict: true }))\n    end\n\n    data(\"integer\" => [4.0, 4],\n         \"float\" => [0.4, 0.4],\n         \"hoge\" => [0.0, \"hoge\"],\n         \"empty\" => [0.0, \"\"])\n    test 'not assumed case' do |(expected, val)|\n      assert_equal(expected, Config.time_value(val))\n    end\n\n    test 'nil' do\n      assert_equal(nil, Config.time_value(nil))\n    end\n\n    data(\"integer\" => [6, 6],\n         \"hoge\" => [Fluent::ConfigError.new('name1: invalid value for Float(): \"hoge\"'), \"hoge\"],\n         \"empty\" => [Fluent::ConfigError.new('name1: invalid value for Float(): \"\"'), \"\"])\n    test 'not assumed case with strict' do |(expected, val)|\n      if expected.kind_of? Exception\n        assert_raise(expected) do\n          Config.time_value(val, { strict: true }, \"name1\")\n        end\n      else\n        assert_equal(expected, Config.time_value(val, { strict: true }, \"name1\"))\n      end\n    end\n\n    test 'nil with strict' do\n      assert_equal(nil, Config.time_value(nil, { strict: true }))\n    end\n  end\n\n  sub_test_case 'Config.bool_value' do\n    data(\"true\" => [true, \"true\"],\n         \"yes\" => [true, \"yes\"],\n         \"empty\" => [true, \"\"],\n         \"false\" => [false, \"false\"],\n         \"no\" => [false, \"no\"])\n    test 'normal case' do |(expected, val)|\n      assert_equal(expected, Config.bool_value(val))\n    end\n\n    data(\"true\" =>  [true,  true],\n         \"false\" => [false, false],\n         \"hoge\" => [nil, \"hoge\"],\n         \"nil\" => [nil, nil],\n         \"integer\" => [nil, 10])\n    test 'not assumed case' do |(expected, val)|\n      assert_equal(expected, Config.bool_value(val))\n    end\n\n    data(\"true\" =>  [true,  true],\n         \"false\" => [false, false],\n         \"hoge\" => [Fluent::ConfigError.new(\"name1: invalid bool value: hoge\"), \"hoge\"],\n         \"nil\" => [nil, nil],\n         \"integer\" => [Fluent::ConfigError.new(\"name1: invalid bool value: 10\"), 10])\n    test 'not assumed case with strict' do |(expected, val)|\n      if expected.kind_of? Exception\n        assert_raise(expected) do\n          Config.bool_value(val, { strict: true }, \"name1\")\n        end\n      else\n        assert_equal(expected, Config.bool_value(val, { strict: true }, \"name1\"))\n      end\n    end\n  end\n\n  sub_test_case 'Config.regexp_value' do\n    data(\"empty\" => [//, \"//\"],\n         \"plain\" => [/regexp/, \"/regexp/\"],\n         \"zero width\" => [/^$/, \"/^$/\"],\n         \"character classes\" => [/[a-z]/, \"/[a-z]/\"],\n         \"meta charactersx\" => [/.+.*?\\d\\w\\s\\S/, '/.+.*?\\d\\w\\s\\S/'])\n    test 'normal case' do |(expected, str)|\n      assert_equal(expected, Config.regexp_value(str))\n    end\n\n    data(\"empty\" => [//, \"\"],\n         \"plain\" => [/regexp/, \"regexp\"],\n         \"zero width\" => [/^$/, \"^$\"],\n         \"character classes\" => [/[a-z]/, \"[a-z]\"],\n         \"meta charactersx\" => [/.+.*?\\d\\w\\s\\S/, '.+.*?\\d\\w\\s\\S'])\n    test 'w/o slashes' do |(expected, str)|\n      assert_equal(expected, Config.regexp_value(str))\n    end\n\n    data(\"missing right slash\" => \"/regexp\",\n         \"too many options\" => \"/regexp/imx\",)\n    test 'invalid regexp' do |(str)|\n      assert_raise(Fluent::ConfigError.new(\"invalid regexp: missing right slash: #{str}\")) do\n        Config.regexp_value(str)\n      end\n    end\n\n    test 'nil' do\n      assert_equal nil, Config.regexp_value(nil)\n    end\n  end\n\n  sub_test_case 'type converters for config_param definitions' do\n    data(\"test\" => ['test', 'test'],\n         \"1\" =>  ['1',    '1'],\n         \"spaces\" =>  ['   ',  '   '])\n    test 'string' do |(expected, val)|\n      assert_equal expected, Config::STRING_TYPE.call(val, {})\n      assert_equal Encoding::UTF_8, Config::STRING_TYPE.call(val, {}).encoding\n    end\n\n    test 'string nil' do\n      assert_equal nil, Config::STRING_TYPE.call(nil, {})\n    end\n\n    data('latin' => 'Märch',\n         'ascii' => 'ascii',\n         'space' => '     ',\n         'number' => '1',\n         'Hiragana' => 'あいうえお')\n    test 'string w/ binary' do |str|\n      actual = Config::STRING_TYPE.call(str.b, {})\n      assert_equal str, actual\n      assert_equal Encoding::UTF_8, actual.encoding\n    end\n\n    data('starts_with_semicolon' => [:conor, ':conor'],\n         'simple_string' => [:conor, 'conor'],\n         'empty_string' => [nil, ''])\n    test 'symbol' do |(expected, val)|\n      assert_equal Config::SYMBOL_TYPE.call(val, {}), expected\n    end\n\n    data(\"val\" => [:val, 'val'],\n         \"v\" => [:v, 'v'],\n         \"value\" => [:value, 'value'])\n    test 'enum' do |(expected, val)|\n      assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})\n    end\n\n    test 'enum: pick unknown choice' do\n      assert_raises(Fluent::ConfigError.new(\"valid options are val,value,v but got x\")) do\n        Config::ENUM_TYPE.call('x', {list: [:val, :value, :v]})\n      end\n    end\n\n    data(\"empty list\"  => {},\n         \"string list\" => {list: [\"val\", \"value\", \"v\"]})\n    test 'enum: invalid choices' do | list |\n      assert_raises(RuntimeError.new(\"Plugin BUG: config type 'enum' requires :list of symbols\")) do\n        Config::ENUM_TYPE.call('val', list)\n      end\n    end\n\n    test 'enum: nil' do\n      assert_equal nil, Config::ENUM_TYPE.call(nil)\n    end\n\n    data(\"1\" => [1, '1'],\n         \"1.0\" => [1, '1.0'],\n         \"1_000\" => [1000, '1_000'],\n         \"1x\" => [1, '1x'])\n    test 'integer' do |(expected, val)|\n      assert_equal expected, Config::INTEGER_TYPE.call(val, {})\n    end\n\n    data(\"integer\" => [6, 6],\n         \"hoge\" => [0, \"hoge\"],\n         \"empty\" => [0, \"\"])\n    test 'integer: not assumed case' do |(expected, val)|\n      assert_equal expected, Config::INTEGER_TYPE.call(val, {})\n    end\n\n    test 'integer: nil' do\n      assert_equal nil, Config::INTEGER_TYPE.call(nil, {})\n    end\n\n    data(\"integer\" => [6, 6],\n         \"hoge\" => [Fluent::ConfigError.new('name1: invalid value for Integer(): \"hoge\"'), \"hoge\"],\n         \"empty\" => [Fluent::ConfigError.new('name1: invalid value for Integer(): \"\"'), \"\"])\n    test 'integer: not assumed case with strict' do |(expected, val)|\n      if expected.kind_of? Exception\n        assert_raise(expected) do\n          Config::INTEGER_TYPE.call(val, { strict: true }, \"name1\")\n        end\n      else\n        assert_equal expected, Config::INTEGER_TYPE.call(val, { strict: true }, \"name1\")\n      end\n    end\n\n    test 'integer: nil with strict' do\n      assert_equal nil, Config::INTEGER_TYPE.call(nil, { strict: true })\n    end\n\n    data(\"1\" => [1.0, '1'],\n         \"1.0\" => [1.0, '1.0'],\n         \"1.00\" => [1.0, '1.00'],\n         \"1e0\" => [1.0, '1e0'])\n    test 'float' do |(expected, val)|\n      assert_equal expected, Config::FLOAT_TYPE.call(val, {})\n    end\n\n    data(\"integer\" => [6, 6],\n         \"hoge\" => [0, \"hoge\"],\n         \"empty\" => [0, \"\"])\n    test 'float: not assumed case' do |(expected, val)|\n      assert_equal expected, Config::FLOAT_TYPE.call(val, {})\n    end\n\n    test 'float: nil' do\n      assert_equal nil, Config::FLOAT_TYPE.call(nil, {})\n    end\n\n    data(\"integer\" => [6, 6],\n         \"hoge\" => [Fluent::ConfigError.new('name1: invalid value for Float(): \"hoge\"'), \"hoge\"],\n         \"empty\" => [Fluent::ConfigError.new('name1: invalid value for Float(): \"\"'), \"\"])\n    test 'float: not assumed case with strict' do |(expected, val)|\n      if expected.kind_of? Exception\n        assert_raise(expected) do\n          Config::FLOAT_TYPE.call(val, { strict: true }, \"name1\")\n        end\n      else\n        assert_equal expected, Config::FLOAT_TYPE.call(val, { strict: true }, \"name1\")\n      end\n    end\n\n    test 'float: nil with strict' do\n      assert_equal nil, Config::FLOAT_TYPE.call(nil, { strict: true })\n    end\n\n    data(\"1000\" => [1000, '1000'],\n         \"1k\" => [1024, '1k'],\n         \"1m\" => [1024*1024, '1m'])\n    test 'size' do |(expected, val)|\n      assert_equal expected, Config::SIZE_TYPE.call(val, {})\n    end\n\n    data(\"true\" => [true, 'true'],\n         \"yes\" => [true, 'yes'],\n         \"no\" => [false, 'no'],\n         \"false\" => [false, 'false'],\n         \"TRUE\" => [nil, 'TRUE'],\n         \"True\" => [nil, 'True'],\n         \"Yes\" => [nil, 'Yes'],\n         \"No\" => [nil, 'No'],\n         \"empty\" => [true, ''],\n         \"unexpected_string\" => [nil, 'unexpected_string'])\n    test 'bool' do |(expected, val)|\n      assert_equal expected, Config::BOOL_TYPE.call(val, {})\n    end\n\n    data(\"0\" => [0, '0'],\n         \"1\" => [1.0, '1'],\n         \"1.01\" => [1.01,  '1.01'],\n         \"1s\" => [1, '1s'],\n         \"1,\" => [60, '1m'],\n         \"1h\" => [3600,  '1h'],\n         \"1d\" => [86400, '1d'])\n    test 'time' do |(expected, val)|\n      assert_equal expected, Config::TIME_TYPE.call(val, {})\n    end\n\n    data(\"empty\" => [//, \"//\"],\n         \"plain\" => [/regexp/, \"/regexp/\"],\n         \"zero width\" => [/^$/, \"/^$/\"],\n         \"character classes\" => [/[a-z]/, \"/[a-z]/\"],\n         \"meta charactersx\" => [/.+.*?\\d\\w\\s\\S/, '/.+.*?\\d\\w\\s\\S/'])\n    test 'regexp' do |(expected, str)|\n      assert_equal(expected, Config::REGEXP_TYPE.call(str, {}))\n    end\n\n    data(\"string and integer\" => [{\"x\"=>\"v\",\"k\"=>1}, '{\"x\":\"v\",\"k\":1}', {}],\n         \"strings\" => [{\"x\"=>\"v\",\"k\"=>\"1\"}, 'x:v,k:1', {}],\n         \"w/ space\" => [{\"x\"=>\"v\",\"k\"=>\"1\"}, 'x:v, k:1', {}],\n         \"heading space\" => [{\"x\"=>\"v\",\"k\"=>\"1\"}, ' x:v, k:1 ', {}],\n         \"trailing space\" => [{\"x\"=>\"v\",\"k\"=>\"1\"}, 'x:v , k:1 ', {}],\n         \"multiple colons\" => [{\"x\"=>\"v:v\",\"k\"=>\"1\"}, 'x:v:v, k:1', {}],\n         \"symbolize keys\" => [{x: \"v\", k: 1}, '{\"x\":\"v\",\"k\":1}', {symbolize_keys: true}],\n         \"value_type: :string\" => [{x: \"v\", k: \"1\"}, 'x:v,k:1', {symbolize_keys: true, value_type: :string}],\n         \"value_type: :string 2\" => [{x: \"v\", k: \"1\"}, '{\"x\":\"v\",\"k\":1}', {symbolize_keys: true, value_type: :string}],\n         \"value_type: :integer\" => [{x: 0, k: 1}, 'x:0,k:1', {symbolize_keys: true, value_type: :integer}],\n         \"time 1\" => [{\"x\"=>1,\"y\"=>60,\"z\"=>3600}, '{\"x\":\"1s\",\"y\":\"1m\",\"z\":\"1h\"}', {value_type: :time}],\n         \"time 2\" => [{\"x\"=>1,\"y\"=>60,\"z\"=>3600}, 'x:1s,y:1m,z:1h', {value_type: :time}])\n    test 'hash' do |(expected, val, opts)|\n      assert_equal(expected, Config::HASH_TYPE.call(val, opts))\n    end\n\n    test 'hash w/ unknown type' do\n      assert_raise(RuntimeError.new(\"unknown type in REFORMAT: foo\")) do\n        Config::HASH_TYPE.call(\"x:1,y:2\", {value_type: :foo})\n      end\n    end\n\n    test 'hash w/ strict option' do\n      assert_raise(Fluent::ConfigError.new('y: invalid value for Integer(): \"hoge\"')) do\n        Config::HASH_TYPE.call(\"x:1,y:hoge\", {value_type: :integer, strict: true})\n      end\n    end\n\n    data('latin' => ['3:Märch', {\"3\"=>\"Märch\"}],\n         'ascii' => ['ascii:ascii', {\"ascii\"=>\"ascii\"}],\n         'number' => ['number:1', {\"number\"=>\"1\"}],\n         'Hiragana' => ['hiragana:あいうえお', {\"hiragana\"=>\"あいうえお\"}])\n    test 'hash w/ binary' do |(target, expected)|\n      assert_equal(expected, Config::HASH_TYPE.call(target.b, { value_type: :string }))\n    end\n\n    test 'hash w/ nil' do\n      assert_equal(nil, Config::HASH_TYPE.call(nil))\n    end\n\n    data(\"strings and integer\" => [[\"1\",\"2\",1],   '[\"1\",\"2\",1]', {}],\n         \"number strings\" => [[\"1\",\"2\",\"1\"], '1,2,1', {}],\n         \"alphabets\" => [[\"a\",\"b\",\"c\"], '[\"a\",\"b\",\"c\"]', {}],\n         \"alphabets w/o quote\" => [[\"a\",\"b\",\"c\"], 'a,b,c', {}],\n         \"w/ spaces\" => [[\"a\",\"b\",\"c\"], 'a, b, c', {}],\n         \"w/ space before comma\" => [[\"a\",\"b\",\"c\"], 'a , b , c', {}],\n         \"comma or space w/ qupte\" => [[\"a a\",\"b,b\",\" c \"], '[\"a a\",\"b,b\",\" c \"]', {}],\n         \"space in a value w/o qupte\" => [[\"a a\",\"b\",\"c\"], 'a a,b,c', {}],\n         \"integers\" => [[1,2,1], '[1,2,1]', {}],\n         \"value_type: :integer w/ quote\" => [[1,2,1], '[\"1\",\"2\",\"1\"]', {value_type: :integer}],\n         \"value_type: :integer w/o quote\" => [[1,2,1], '1,2,1', {value_type: :integer}])\n    test 'array' do |(expected, val, opts)|\n      assert_equal(expected, Config::ARRAY_TYPE.call(val, opts))\n    end\n\n    data('[\"1\",\"2\"]' => [[\"1\",\"2\"], '[\"1\",\"2\"]'],\n         '[\"3\"]' => [[\"3\"], '[\"3\"]'])\n    test 'array w/ default values' do |(expected, val)|\n      array_options = {\n        default: [],\n      }\n      assert_equal(expected, Config::ARRAY_TYPE.call(val, array_options))\n    end\n\n    test 'array w/ unknown type' do\n      assert_raise(RuntimeError.new(\"unknown type in REFORMAT: foo\")) do\n        Config::ARRAY_TYPE.call(\"1,2\", {value_type: :foo})\n      end\n    end\n\n    test 'array w/ strict option' do\n      assert_raise(Fluent::ConfigError.new(': invalid value for Integer(): \"hoge\"')) do\n        Config::ARRAY_TYPE.call(\"1,hoge\", {value_type: :integer, strict: true}, \"name1\")\n      end\n    end\n\n    test 'array w/ nil' do\n      assert_equal(nil, Config::ARRAY_TYPE.call(nil))\n    end\n  end\nend\n"
  },
  {
    "path": "test/config/test_yaml_parser.rb",
    "content": "require 'helper'\nrequire 'fluent/config/yaml_parser'\nrequire 'socket'\nrequire 'json'\nrequire 'date'\n\nclass YamlParserTest < Test::Unit::TestCase\n  TMP_DIR = File.dirname(__FILE__) + \"/tmp/yaml_config#{ENV['TEST_ENV_NUMBER']}\"\n\n  def write_config(path, data, encoding: 'utf-8')\n    FileUtils.mkdir_p(File.dirname(path))\n    File.open(path, \"w:#{encoding}:utf-8\") {|f| f.write data }\n  end\n\n  sub_test_case 'Special YAML elements' do\n    def test_special_yaml_elements_dollar\n      write_config \"#{TMP_DIR}/test_special_yaml_elements_dollar_source.yaml\", <<~EOS\n        config:\n          - source:\n              $type: dummy_type\n              $label: dummy_label\n              $id: dummy_id\n              $log_level: debug\n              $unknown: unknown\n      EOS\n      config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_special_yaml_elements_dollar_source.yaml\")\n      assert_equal('source', config.elements[0].name)\n      assert_equal('dummy_type', config.elements[0]['@type'])\n      assert_equal('dummy_label', config.elements[0]['@label'])\n      assert_equal('dummy_id', config.elements[0]['@id'])\n      assert_equal('debug', config.elements[0]['@log_level'])\n      assert_nil(config.elements[0]['@unknown'])\n\n      write_config \"#{TMP_DIR}/test_special_yaml_elements_dollar_match.yaml\", <<~EOS\n        config:\n          - match:\n              $tag: dummy_tag\n      EOS\n      config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_special_yaml_elements_dollar_match.yaml\")\n      assert_equal('match', config.elements[0].name)\n      assert_equal('dummy_tag', config.elements[0].arg)\n\n\n      write_config \"#{TMP_DIR}/test_special_yaml_elements_dollar_worker.yaml\", <<~EOS\n        config:\n          - worker:\n              $arg: dummy_arg\n              config:\n                - source:\n                    $type: dummy\n      EOS\n      config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_special_yaml_elements_dollar_worker.yaml\")\n      assert_equal('worker', config.elements[0].name)\n      assert_equal('dummy_arg', config.elements[0].arg)\n    end\n\n    def test_embedded_ruby_code\n      write_config \"#{TMP_DIR}/test_embedded_ruby_code.yaml\", <<~EOS\n        config:\n          - source:\n              host: !fluent/s \"#{Socket.gethostname}\"\n      EOS\n      config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_embedded_ruby_code.yaml\")\n      assert_equal(Socket.gethostname, config.elements[0]['host'])\n    end\n\n    def test_fluent_json_format\n      write_config \"#{TMP_DIR}/test_fluent_json_format.yaml\", <<~EOS\n        config:\n          - source:\n              hash_param: !fluent/json {\n                \"k\": \"v\",\n                \"k1\": 10\n              }\n      EOS\n      config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_fluent_json_format.yaml\")\n      assert_equal({'k': 'v', 'k1': 10}.to_json, config.elements[0]['hash_param'])\n    end\n  end\n\n  sub_test_case 'root elements' do\n    def test_root_elements\n      write_config \"#{TMP_DIR}/test_root_elements.yaml\", <<~EOS\n        config:\n          - source:\n              $type: dummy\n        system:\n          dummy: dummy\n        unknown:\n          dummy: dummy\n      EOS\n      config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_root_elements.yaml\")\n      assert_equal(2, config.elements.size)\n      # the first element is system section.\n      assert_equal(\"system\", config.elements[0].name)\n      # the second element is source in config section.\n      assert_equal(\"source\", config.elements[1].name)\n    end\n\n    def test_root_elements_only_config_section\n      write_config \"#{TMP_DIR}/test_root_elements_only_config_section.yaml\", <<~EOS\n        config:\n          - source:\n              $type: dummy\n      EOS\n      config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_root_elements_only_config_section.yaml\")\n      assert_equal(1, config.elements.size)\n      assert_equal(\"source\", config.elements[0].name)\n    end\n\n    def test_root_elements_only_system_section\n      write_config \"#{TMP_DIR}/test_root_elements_only_system_section.yaml\", <<~EOS\n        system:\n          dummy: dummy\n      EOS\n      config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_root_elements_only_system_section.yaml\")\n      assert_equal(1, config.elements.size)\n      assert_equal(\"system\", config.elements[0].name)\n    end\n  end\n\n  sub_test_case 'config section' do\n    def test_config_section_directives\n      write_config \"#{TMP_DIR}/dummy.yaml\", <<~EOS\n        - filter:\n            $type: dummy\n      EOS\n      write_config \"#{TMP_DIR}/test_config_section_directives.yaml\", <<~EOS\n        config:\n          - source:\n              $type: dummy\n          - filter:\n              $type: dummy\n          - match:\n              $tag: dummy\n          - worker:\n              $arg: 0\n              config:\n                - source:\n                    $type: dummy\n          - label:\n              $name: dummy\n              config:\n                - filter:\n                    $type: dummy\n          - !include dummy.yaml\n      EOS\n      config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_config_section_directives.yaml\")\n      assert_equal(6, config.elements.size)\n      assert_equal(%w(source filter match worker label filter), config.elements.map(&:name))\n    end\n\n    def test_config_section_unknown_directives\n      write_config \"#{TMP_DIR}/test_config_section_unknown_directives.yaml\", <<~EOS\n        config:\n          - source:\n              $type: dummy\n          - unknown:\n              $type: dummy\n      EOS\n\n      # TODO: it should raise Fluent::ConfigError instead of NoMethodError, or drop unknown directives\n      assert_raise(NoMethodError) do\n        Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_config_section_unknown_directives.yaml\")\n      end\n    end\n\n    sub_test_case 'label' do\n      def test_label_section\n        write_config \"#{TMP_DIR}/test_label_section.yaml\", <<~EOS\n          config:\n            - label:\n                $name: dummy_label\n                config:\n                  - filter:\n                      $type: dummy\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_label_section.yaml\")\n        assert_equal('label', config.elements[0].name)\n        assert_equal('dummy_label', config.elements[0].arg)\n        assert_equal('filter', config.elements[0].elements[0].name)\n      end\n\n      def test_label_section_missing_name\n        write_config \"#{TMP_DIR}/test_label_section_missing_name.yaml\", <<~EOS\n          config:\n            - label:\n                config:\n                  - filter:\n                      $type: dummy\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_label_section_missing_name.yaml\")\n        assert_equal('label', config.elements[0].name)\n        assert_equal('', config.elements[0].arg)\n        assert_equal('filter', config.elements[0].elements[0].name)\n      end\n\n      def test_label_section_missing_config\n        write_config \"#{TMP_DIR}/test_label_section_missing_config.yaml\", <<~EOS\n          config:\n            - label:\n                $name: dummy_label\n        EOS\n\n        # TODO: it should raise Fluent::ConfigError instead of NoMethodError\n        assert_raise(NoMethodError) do\n          Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_label_section_missing_config.yaml\")\n        end\n      end\n    end\n\n    sub_test_case 'worker' do\n      def test_worker_section\n        write_config \"#{TMP_DIR}/test_worker_section.yaml\", <<~EOS\n          config:\n            - worker:\n                $arg: 0\n                config:\n                  - source:\n                      $type: dummy\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_worker_section.yaml\")\n        assert_equal('worker', config.elements[0].name)\n        assert_equal('0', config.elements[0].arg)\n        assert_equal('source', config.elements[0].elements[0].name)\n      end\n\n      def test_worker_section_missing_arg\n        write_config \"#{TMP_DIR}/test_worker_section_missing_arg.yaml\", <<~EOS\n          config:\n            - worker:\n                config:\n                  - source:\n                      $type: dummy\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_worker_section_missing_arg.yaml\")\n        assert_equal('worker', config.elements[0].name)\n        assert_equal('', config.elements[0].arg)\n        assert_equal('source', config.elements[0].elements[0].name)\n      end\n\n      def test_worker_section_missing_config\n        write_config \"#{TMP_DIR}/test_worker_section_missing_config.yaml\", <<~EOS\n          config:\n            - worker:\n                $arg: 0\n        EOS\n\n        # TODO: it should raise Fluent::ConfigError instead of NoMethodError\n        assert_raise(NoMethodError) do\n          Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_worker_section_missing_config.yaml\")\n        end\n      end\n    end\n\n    sub_test_case 'source' do\n      def test_source_section\n        write_config \"#{TMP_DIR}/test_source_section.yaml\", <<~EOS\n          config:\n            - source:\n                $type: dummy_type\n                port: 8888\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_source_section.yaml\")\n        assert_equal('source', config.elements[0].name)\n        assert_equal('dummy_type', config.elements[0]['@type'])\n        assert_equal(8888, config.elements[0]['port'])\n      end\n\n      def test_source_section_missing_type\n        write_config \"#{TMP_DIR}/test_source_section_missing_type.yaml\", <<~EOS\n          config:\n            - source:\n                port: 8888\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_source_section_missing_type.yaml\")\n        assert_equal('source', config.elements[0].name)\n        assert_equal(8888, config.elements[0]['port'])\n        assert_nil(config.elements[0]['@type'])\n      end\n    end\n\n    sub_test_case 'filter' do\n      def test_filter_section\n        write_config \"#{TMP_DIR}/test_filter_section.yaml\", <<~EOS\n          config:\n            - filter:\n                $tag: dummy_tag\n                $type: dummy_type\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_filter_section.yaml\")\n        assert_equal('filter', config.elements[0].name)\n        assert_equal('dummy_tag', config.elements[0].arg)\n        assert_equal('dummy_type', config.elements[0]['@type'])\n      end\n\n      def test_filter_section_multiple_tags\n        write_config \"#{TMP_DIR}/test_filter_section_multiple_tags_1.yaml\", <<~EOS\n          config:\n            - filter:\n                $tag: ['dummy_tag_A', 'dummy_tag_B']\n                $type: dummy_type\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_filter_section_multiple_tags_1.yaml\")\n        assert_equal('filter', config.elements[0].name)\n        assert_equal('dummy_tag_A,dummy_tag_B', config.elements[0].arg)\n        assert_equal('dummy_type', config.elements[0]['@type'])\n\n        write_config \"#{TMP_DIR}/test_filter_section_multiple_tags_2.yaml\", <<~EOS\n          config:\n            - filter:\n                $tag:\n                  - dummy_tag_C\n                  - dummy_tag_D\n                $type: dummy_type\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_filter_section_multiple_tags_2.yaml\")\n        assert_equal('filter', config.elements[0].name)\n        assert_equal('dummy_tag_C,dummy_tag_D', config.elements[0].arg)\n        assert_equal('dummy_type', config.elements[0]['@type'])\n      end\n\n      def test_filter_section_missing_tag\n        write_config \"#{TMP_DIR}/test_filter_section_missing_tag.yaml\", <<~EOS\n          config:\n            - filter:\n                $type: dummy_type\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_filter_section_missing_tag.yaml\")\n        assert_equal('filter', config.elements[0].name)\n        assert_equal('', config.elements[0].arg)\n        assert_equal('dummy_type', config.elements[0]['@type'])\n      end\n\n      def test_filter_section_missing_type\n        write_config \"#{TMP_DIR}/test_filter_section_missing_tag.yaml\", <<~EOS\n          config:\n            - filter:\n                $tag: dummy_tag\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_filter_section_missing_tag.yaml\")\n        assert_equal('filter', config.elements[0].name)\n        assert_equal('dummy_tag', config.elements[0].arg)\n        assert_nil(config.elements[0]['@type'])\n      end\n    end\n\n    sub_test_case 'match' do\n      def test_match_section\n        write_config \"#{TMP_DIR}/test_match_section.yaml\", <<~EOS\n          config:\n            - match:\n                $tag: dummy_tag\n                $type: dummy_type\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_match_section.yaml\")\n        assert_equal('match', config.elements[0].name)\n        assert_equal('dummy_tag', config.elements[0].arg)\n        assert_equal('dummy_type', config.elements[0]['@type'])\n      end\n\n      def test_match_section_multiple_tags\n        write_config \"#{TMP_DIR}/test_match_section_multiple_tags_1.yaml\", <<~EOS\n          config:\n            - match:\n                $tag: ['dummy_tag_A', 'dummy_tag_B']\n                $type: dummy_type\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_match_section_multiple_tags_1.yaml\")\n        assert_equal('match', config.elements[0].name)\n        assert_equal('dummy_tag_A,dummy_tag_B', config.elements[0].arg)\n        assert_equal('dummy_type', config.elements[0]['@type'])\n\n        write_config \"#{TMP_DIR}/test_match_section_multiple_tags_2.yaml\", <<~EOS\n          config:\n            - match:\n                $tag:\n                  - dummy_tag_C\n                  - dummy_tag_D\n                $type: dummy_type\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_match_section_multiple_tags_2.yaml\")\n        assert_equal('match', config.elements[0].name)\n        assert_equal('dummy_tag_C,dummy_tag_D', config.elements[0].arg)\n        assert_equal('dummy_type', config.elements[0]['@type'])\n      end\n\n      def test_match_section_missing_tag\n        write_config \"#{TMP_DIR}/test_match_section_missing_tag.yaml\", <<~EOS\n          config:\n            - match:\n                $type: dummy_type\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_match_section_missing_tag.yaml\")\n        assert_equal('match', config.elements[0].name)\n        assert_equal('', config.elements[0].arg)\n        assert_equal('dummy_type', config.elements[0]['@type'])\n      end\n\n      def test_match_section_missing_type\n        write_config \"#{TMP_DIR}/test_match_section_missing_type.yaml\", <<~EOS\n          config:\n            - match:\n                $tag: dummy_tag\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_match_section_missing_type.yaml\")\n        assert_equal('match', config.elements[0].name)\n        assert_equal('dummy_tag', config.elements[0].arg)\n        assert_nil(config.elements[0]['@type'])\n      end\n    end\n\n    sub_test_case '!include' do\n      def test_include\n        write_config \"#{TMP_DIR}/dummy.yaml\", <<~EOS\n        - filter:\n            $type: dummy\n        EOS\n        write_config \"#{TMP_DIR}/test_include.yaml\", <<~EOS\n        config:\n          - !include dummy.yaml\n        EOS\n        config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_include.yaml\")\n        assert_equal(1, config.elements.size)\n        assert_equal(\"filter\", config.elements[0].name) # included section\n      end\n\n      def test_include_normal_config_file\n        write_config \"#{TMP_DIR}/dummy.conf\", <<~EOS\n          <filter **>\n            type dummy\n          </filter>\n        EOS\n        write_config \"#{TMP_DIR}/test_include_normal_config_file.yaml\", <<~EOS\n          config:\n            - !include dummy.conf\n        EOS\n\n        # TODO: it should raise Fluent::ConfigError instead of TypeError, or parse normal config file\n        assert_raise(TypeError) do\n          Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_include_normal_config_file.yaml\")\n        end\n      end\n    end\n  end\n\n  def test_merge_common_parameter\n    write_config \"#{TMP_DIR}/test_merge_common_parameter.yaml\", <<~EOS\n      common_parameter: &common_parameter\n        common_param: foobarbaz\n      \n      config:\n        - match:\n            $tag: dummy_tag_1\n            $type: dummy_type_1\n            <<: *common_parameter\n        - match:\n            $tag: dummy_tag_2\n            $type: dummy_type_2\n            <<: *common_parameter\n    EOS\n    config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_merge_common_parameter.yaml\")\n    assert_equal(2, config.elements.size)\n    assert_equal('foobarbaz', config.elements[0]['common_param'])\n    assert_equal('foobarbaz', config.elements[1]['common_param'])\n  end\n\n  def test_override_merged_common_parameter\n    write_config \"#{TMP_DIR}/test_merge_common_parameter.yaml\", <<~EOS\n      common_parameter: &common_parameter\n        common_param1: foobarbaz\n        common_param2: 12345\n      \n      config:\n        - match:\n            $tag: dummy_tag_1\n            $type: dummy_type_1\n            <<: *common_parameter\n        - match:\n            $tag: dummy_tag_2\n            $type: dummy_type_2\n            <<: *common_parameter\n            common_param: override\n    EOS\n    config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_merge_common_parameter.yaml\")\n    assert_equal(2, config.elements.size)\n    assert_equal('foobarbaz', config.elements[0]['common_param1'])\n    assert_equal(12345, config.elements[0]['common_param2'])\n    assert_equal('override', config.elements[1]['common_param'])\n    assert_equal(12345, config.elements[1]['common_param2'])\n  end\n\n  def test_merge_common_parameter_using_include\n    write_config \"#{TMP_DIR}/fluent-common.yaml\", <<~EOS\n      common_param: foobarbaz\n    EOS\n\n    write_config \"#{TMP_DIR}/test_merge_common_parameter_using_include.yaml\", <<~EOS\n      config:\n        - match:\n            $tag: dummy_tag_1\n            $type: dummy_type_1\n            <<: !include fluent-common.yaml\n        - match:\n            $tag: dummy_tag_2\n            $type: dummy_type_2\n            <<: !include fluent-common.yaml\n    EOS\n\n    # TODO: Fix exception\n    assert_raise(TypeError) do\n      Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_merge_common_parameter_using_include.yaml\")\n    end\n  end\n\n  def test_unknown_anchor\n    write_config \"#{TMP_DIR}/test_unknown_anchor.yaml\", <<~EOS\n      common_parameter: &common_parameter\n        common_param: foobarbaz\n      \n      config:\n        - match:\n            $tag: dummy_tag_1\n            $type: dummy_type_1\n            <<: *unknown_anchor\n    EOS\n\n    # TODO: it should raise Fluent::ConfigError instead of Psych::AnchorNotDefined\n    assert_raises(Psych::AnchorNotDefined) do\n      Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_unknown_anchor.yaml\")\n    end\n  end\n\n  def test_yaml_values\n    write_config \"#{TMP_DIR}/test_yaml_values.yaml\", <<~EOS\n      config:\n        - source:\n            mode: 0644\n            float_value: 3.40\n            flag_on: On\n            flag_off: Off\n            null: null\n            date: 2026-01-01\n            timestamp: 2026-01-01 00:00:00 +09:00\n            symbol1: :foo\n            symbol2: !ruby/symbol bar\n            regexp: !ruby/regexp /^$/\n    EOS\n    config = Fluent::Config::YamlParser.parse(\"#{TMP_DIR}/test_yaml_values.yaml\")\n    assert_equal(420, config.elements[0]['mode'])\n    assert_equal(3.4, config.elements[0]['float_value'])\n    assert_equal(true, config.elements[0]['flag_on'])\n    assert_equal(false, config.elements[0]['flag_off'])\n    assert_nil(config.elements[0]['null'])\n    assert_equal(Date.new(2026, 1, 1), config.elements[0]['date'])\n    assert_equal(Time.parse(\"2026-01-01 00:00:00 +09:00\"), config.elements[0]['timestamp'])\n    assert_equal(:foo, config.elements[0]['symbol1'])\n    assert_equal(:bar, config.elements[0]['symbol2'])\n    assert_equal(/^$/, config.elements[0]['regexp'])\n  end\nend\n"
  },
  {
    "path": "test/counter/test_client.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/counter/client'\nrequire 'fluent/counter/store'\nrequire 'fluent/counter/server'\nrequire 'flexmock/test_unit'\nrequire 'timecop'\n\nclass CounterClientTest < ::Test::Unit::TestCase\n  TEST_ADDR = '127.0.0.1'\n  TEST_PORT = '8277'\n\n  setup do\n    # timecop isn't compatible with EventTime\n    t = Time.parse('2016-09-22 16:59:59 +0900')\n    Timecop.freeze(t)\n    @now = Fluent::EventTime.now\n\n    @options = {\n      addr: TEST_ADDR,\n      port: TEST_PORT,\n      log: $log,\n    }\n\n    @server_name = 'server1'\n    @scope = \"worker1\\tplugin1\"\n    @loop = Coolio::Loop.new\n    @server = Fluent::Counter::Server.new(@server_name, @options).start\n    @client = Fluent::Counter::Client.new(@loop, @options).start\n  end\n\n  teardown do\n    Timecop.return\n    @server.stop\n    @client.stop\n  end\n\n  test 'Callable API' do\n    [:establish, :init, :delete, :inc, :reset, :get].each do |m|\n      assert_true @client.respond_to?(m)\n    end\n  end\n\n  sub_test_case 'on_message' do\n    setup do\n      @future = flexmock('future')\n      @client.instance_variable_set(:@responses, { 1 => @future })\n    end\n\n    test 'call a set method to a corresponding object' do\n      @future.should_receive(:set).once.with(Hash)\n      @client.send(:on_message, { 'id' => 1 })\n    end\n\n    test \"output a warning log when passed id doesn't exist\" do\n      data = { 'id' => 2 }\n      mock($log).warn(\"Receiving missing id data: #{data}\")\n      @client.send(:on_message, data)\n    end\n  end\n\n  def extract_value_from_server(server, scope, name)\n    store = server.instance_variable_get(:@store).instance_variable_get(:@storage).instance_variable_get(:@store)\n    key = Fluent::Counter::Store.gen_key(scope, name)\n    store[key]\n  end\n\n  def travel(sec)\n    # Since Timecop.travel() causes test failures on Windows/AppVeyor by inducing\n    # rounding errors to Time.now, we need to use Timecop.freeze() instead.\n    Timecop.freeze(Time.now + sec)\n  end\n\n  sub_test_case 'establish' do\n    test 'establish a scope' do\n      @client.establish(@scope)\n      assert_equal \"#{@server_name}\\t#{@scope}\", @client.instance_variable_get(:@scope)\n    end\n\n    data(\n      empty: '',\n      invalid_string: '_scope',\n      invalid_string2: 'Scope'\n    )\n    test 'raise an error when passed scope is invalid' do |scope|\n      assert_raise do\n        @client.establish(scope)\n      end\n    end\n  end\n\n  sub_test_case 'init' do\n    setup do\n      @client.instance_variable_set(:@scope, @scope)\n    end\n\n    data(\n      numeric_type: [\n        { name: 'key', reset_interval: 20, type: 'numeric' }, 0\n      ],\n      float_type: [\n        { name: 'key', reset_interval: 20, type: 'float' }, 0.0\n      ],\n      integer_type: [\n        { name: 'key', reset_interval: 20, type: 'integer' }, 0\n      ]\n    )\n    test 'create a value' do |(param, initial_value)|\n      assert_nil extract_value_from_server(@server, @scope, param[:name])\n\n      response = @client.init(param).get\n      data = response.data.first\n\n      assert_nil response.errors\n      assert_equal param[:name], data['name']\n      assert_equal param[:reset_interval], data['reset_interval']\n      assert_equal param[:type], data['type']\n      assert_equal initial_value, data['current']\n      assert_equal initial_value, data['total']\n\n      v = extract_value_from_server(@server, @scope, param[:name])\n      assert_equal param[:name], v['name']\n      assert_equal param[:reset_interval], v['reset_interval']\n      assert_equal param[:type], v['type']\n      assert_equal initial_value, v['total']\n      assert_equal initial_value, v['current']\n    end\n\n    test 'raise an error when @scope is nil' do\n      @client.instance_variable_set(:@scope, nil)\n      assert_raise 'Call `establish` method to get a `scope` before calling this method' do\n        params = { name: 'key1', reset_interval: 10 }\n        @client.init(params).get\n      end\n    end\n\n    data(\n      already_exist_key: [\n        { name: 'key1', reset_interval: 10 },\n        { 'code' => 'invalid_params', 'message' => \"worker1\\tplugin1\\tkey1 already exists in counter\" }\n      ],\n      missing_name: [\n        { reset_interval: 10 },\n        { 'code' => 'invalid_params', 'message' => '`name` is required' },\n      ],\n      missing_reset_interval: [\n        { name: 'key' },\n        { 'code' => 'invalid_params', 'message' => '`reset_interval` is required' },\n      ],\n      invalid_name: [\n        { name: '\\tkey' },\n        { 'code' => 'invalid_params', 'message' => '`name` is the invalid format' }\n      ]\n    )\n    test 'return an error object' do |(param, expected_error)|\n      params = { name: 'key1', reset_interval: 10 }\n      @client.init(params).get\n      response = @client.init(param).get\n      errors = response.errors.first\n\n      assert_empty response.data\n      assert_equal expected_error, errors\n\n      assert_raise {\n        @client.init(param).wait\n      }\n    end\n\n    test 'return an existing value when passed key already exists and ignore option is true' do\n      params = { name: 'key1', reset_interval: 10 }\n      res1 = @client.init(params).get\n      res2 = nil\n      assert_nothing_raised do\n        res2 = @client.init({ name: 'key1', reset_interval: 10 }, options: { ignore: true }).get\n      end\n      assert_equal res1.data, res2.data\n    end\n\n    test 'return an error object and data object' do\n      param = { name: 'key1', reset_interval: 10 }\n      param2 = { name: 'key2', reset_interval: 10 }\n      @client.init(param).get\n\n      response = @client.init([param2, param]).get\n      data = response.data.first\n      error = response.errors.first\n\n      assert_equal param2[:name], data['name']\n      assert_equal param2[:reset_interval], data['reset_interval']\n\n      assert_equal 'invalid_params', error['code']\n      assert_equal \"#{@scope}\\t#{param[:name]} already exists in counter\", error['message']\n    end\n\n    test 'return a future object when async call' do\n      param = { name: 'key', reset_interval: 10 }\n      r = @client.init(param)\n      assert_true r.is_a?(Fluent::Counter::Future)\n      assert_nil r.errors\n    end\n  end\n\n  sub_test_case 'delete' do\n    setup do\n      @client.instance_variable_set(:@scope, @scope)\n      @name = 'key'\n      @key = Fluent::Counter::Store.gen_key(@scope, @name)\n\n      @init_obj = { name: @name, reset_interval: 20, type: 'numeric' }\n      @client.init(@init_obj).get\n    end\n\n    test 'delete a value' do\n      assert extract_value_from_server(@server, @scope, @name)\n\n      response = @client.delete(@name).get\n      v = response.data.first\n\n      assert_nil response.errors\n      assert_equal @init_obj[:name], v['name']\n      assert_equal @init_obj[:type], v['type']\n      assert_equal @init_obj[:reset_interval], v['reset_interval']\n\n      assert_nil extract_value_from_server(@server, @scope, @name)\n    end\n\n    test 'raise an error when @scope is nil' do\n      @client.instance_variable_set(:@scope, nil)\n      assert_raise 'Call `establish` method to get a `scope` before calling this method' do\n        @client.delete(@name).get\n      end\n    end\n\n    data(\n      key_not_found: [\n        'key2',\n        { 'code' => 'unknown_key', 'message' => \"`worker1\\tplugin1\\tkey2` doesn't exist in counter\" }\n      ],\n      invalid_key: [\n        '\\tkey',\n        { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' }\n      ]\n    )\n    test 'return an error object' do |(param, expected_error)|\n      response = @client.delete(param).get\n      errors = response.errors.first\n\n      assert_empty response.data\n      assert_equal expected_error, errors\n    end\n\n    test 'return an error object and data object' do\n      unknown_name = 'key2'\n\n      response = @client.delete(@name, unknown_name).get\n      data = response.data.first\n      error = response.errors.first\n\n      assert_equal @name, data['name']\n      assert_equal @init_obj[:reset_interval], data['reset_interval']\n\n      assert_equal 'unknown_key', error['code']\n      assert_equal \"`#{@scope}\\t#{unknown_name}` doesn't exist in counter\", error['message']\n\n      assert_nil extract_value_from_server(@server, @scope, @name)\n    end\n\n    test 'return a future object when async call' do\n      r = @client.delete(@name)\n      assert_true r.is_a?(Fluent::Counter::Future)\n      assert_nil r.errors\n    end\n  end\n\n  sub_test_case 'inc' do\n    setup do\n      @client.instance_variable_set(:@scope, @scope)\n      @name = 'key'\n      @key = Fluent::Counter::Store.gen_key(@scope, @name)\n\n      @init_obj = { name: @name, reset_interval: 20, type: 'numeric' }\n      @client.init(@init_obj).get\n    end\n\n    test 'increment a value' do\n      v = extract_value_from_server(@server, @scope, @name)\n      assert_equal 0, v['total']\n      assert_equal 0, v['current']\n\n      travel(1)\n      inc_obj = { name: @name, value: 10 }\n      @client.inc(inc_obj).get\n\n      v = extract_value_from_server(@server, @scope, @name)\n      assert_equal inc_obj[:value], v['total']\n      assert_equal inc_obj[:value], v['current']\n      assert_equal (@now + 1), Fluent::EventTime.new(*v['last_modified_at'])\n    end\n\n    test 'create and increment a value when force option is true' do\n      name = 'new_key'\n      param = { name: name, value: 11, reset_interval: 1 }\n\n      assert_nil extract_value_from_server(@server, @scope, name)\n\n      @client.inc(param, options: { force: true }).get\n\n      v = extract_value_from_server(@server, @scope, name)\n      assert v\n      assert_equal param[:name], v['name']\n      assert_equal 1, v['reset_interval']\n      assert_equal param[:value], v['current']\n      assert_equal param[:value], v['total']\n    end\n\n    test 'raise an error when @scope is nil' do\n      @client.instance_variable_set(:@scope, nil)\n      assert_raise 'Call `establish` method to get a `scope` before calling this method' do\n        params = { name: 'name', value: 1 }\n        @client.inc(params).get\n      end\n    end\n\n    data(\n      not_exist_key: [\n        { name: 'key2', value: 10 },\n        { 'code' => 'unknown_key', 'message' => \"`worker1\\tplugin1\\tkey2` doesn't exist in counter\" }\n      ],\n      missing_name: [\n        { value: 10 },\n        { 'code' => 'invalid_params', 'message' => '`name` is required' },\n      ],\n      missing_value: [\n        { name: 'key' },\n        { 'code' => 'invalid_params', 'message' => '`value` is required' },\n      ],\n      invalid_name: [\n        { name: '\\tkey' },\n        { 'code' => 'invalid_params', 'message' => '`name` is the invalid format' }\n      ]\n    )\n    test 'return an error object' do |(param, expected_error)|\n      response = @client.inc(param).get\n      errors = response.errors.first\n      assert_empty response.data\n      assert_equal expected_error, errors\n    end\n\n    test 'return an error object and data object' do\n      parmas = [\n        { name: @name, value: 10 },\n        { name: 'unknown_key', value: 9 },\n      ]\n      response = @client.inc(parmas).get\n\n      data = response.data.first\n      error = response.errors.first\n\n      assert_equal @name, data['name']\n      assert_equal 10, data['current']\n      assert_equal 10, data['total']\n\n      assert_equal 'unknown_key', error['code']\n      assert_equal \"`#{@scope}\\tunknown_key` doesn't exist in counter\", error['message']\n    end\n\n    test 'return a future object when async call' do\n      param = { name: 'key', value: 10 }\n      r = @client.inc(param)\n      assert_true r.is_a?(Fluent::Counter::Future)\n      assert_nil r.errors\n    end\n  end\n\n  sub_test_case 'get' do\n    setup do\n      @client.instance_variable_set(:@scope, @scope)\n      @name = 'key'\n\n      @init_obj = { name: @name, reset_interval: 20, type: 'numeric' }\n      @client.init(@init_obj).get\n    end\n\n    test 'get a value' do\n      v1 = extract_value_from_server(@server, @scope, @name)\n      v2 = @client.get(@name).data.first\n\n      assert_equal v1['name'], v2['name']\n      assert_equal v1['current'], v2['current']\n      assert_equal v1['total'], v2['total']\n      assert_equal v1['type'], v2['type']\n    end\n\n    test 'raise an error when @scope is nil' do\n      @client.instance_variable_set(:@scope, nil)\n      assert_raise 'Call `establish` method to get a `scope` before calling this method' do\n        @client.get(@name).get\n      end\n    end\n\n    data(\n      key_not_found: [\n        'key2',\n        { 'code' => 'unknown_key', 'message' => \"`worker1\\tplugin1\\tkey2` doesn't exist in counter\" }\n      ],\n      invalid_key: [\n        '\\tkey',\n        { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' }\n      ]\n    )\n    test 'return an error object' do |(param, expected_error)|\n      response = @client.get(param).get\n      errors = response.errors.first\n      assert_empty response.data\n      assert_equal expected_error, errors\n    end\n\n    test 'return an error object and data object' do\n      unknown_name = 'key2'\n\n      response = @client.get(@name, unknown_name).get\n      data = response.data.first\n      error = response.errors.first\n\n      assert_equal @name, data['name']\n      assert_equal @init_obj[:reset_interval], data['reset_interval']\n\n      assert_equal 'unknown_key', error['code']\n      assert_equal \"`#{@scope}\\t#{unknown_name}` doesn't exist in counter\", error['message']\n    end\n\n    test 'return a future object when async call' do\n      r = @client.get(@name)\n      assert_true r.is_a?(Fluent::Counter::Future)\n      assert_nil r.errors\n    end\n  end\n\n  sub_test_case 'reset' do\n    setup do\n      @client.instance_variable_set(:@scope, @scope)\n      @name = 'key'\n      @key = Fluent::Counter::Store.gen_key(@scope, @name)\n\n      @init_obj = { name: @name, reset_interval: 5, type: 'numeric' }\n      @client.init(@init_obj).get\n      @inc_obj = { name: @name, value: 10 }\n      @client.inc(@inc_obj).get\n    end\n\n    test 'reset a value after `reset_interval` passed' do\n      v1 = extract_value_from_server(@server, @scope, @name)\n      assert_equal @inc_obj[:value], v1['total']\n      assert_equal @inc_obj[:value], v1['current']\n      assert_equal @now, Fluent::EventTime.new(*v1['last_reset_at'])\n\n      travel_sec = 6            # greater than reset_interval\n      travel(travel_sec)\n\n      v2 = @client.reset(@name).get\n      data = v2.data.first\n\n      c = data['counter_data']\n\n      assert_equal travel_sec, data['elapsed_time']\n      assert_true data['success']\n\n      assert_equal @inc_obj[:value], c['current']\n      assert_equal @inc_obj[:value], c['total']\n      assert_equal @now, c['last_reset_at']\n\n      v1 = extract_value_from_server(@server, @scope, @name)\n      assert_equal 0, v1['current']\n      assert_equal @inc_obj[:value], v1['total']\n      assert_equal (@now + travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])\n      assert_equal (@now + travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])\n    end\n\n    test 'return a value object before `reset_interval` passed' do\n      v1 = extract_value_from_server(@server, @scope, @name)\n      assert_equal @inc_obj[:value], v1['total']\n      assert_equal @inc_obj[:value], v1['current']\n      assert_equal @now, Fluent::EventTime.new(*v1['last_reset_at'])\n\n      travel_sec = 4            # less than reset_interval\n      travel(travel_sec)\n\n      v2 = @client.reset(@name).get\n      data = v2.data.first\n\n      c = data['counter_data']\n\n      assert_equal travel_sec, data['elapsed_time']\n      assert_equal false, data['success']\n\n      assert_equal @inc_obj[:value], c['current']\n      assert_equal @inc_obj[:value], c['total']\n      assert_equal @now, c['last_reset_at']\n\n      v1 = extract_value_from_server(@server, @scope, @name)\n      assert_equal @inc_obj[:value], v1['current']\n      assert_equal @inc_obj[:value], v1['total']\n      assert_equal @now, Fluent::EventTime.new(*v1['last_reset_at'])\n    end\n\n    test 'raise an error when @scope is nil' do\n      @client.instance_variable_set(:@scope, nil)\n      assert_raise 'Call `establish` method to get a `scope` before calling this method' do\n        @client.reset(@name).get\n      end\n    end\n\n    data(\n      key_not_found: [\n        'key2',\n        { 'code' => 'unknown_key', 'message' => \"`worker1\\tplugin1\\tkey2` doesn't exist in counter\" }\n      ],\n      invalid_key: [\n        '\\tkey',\n        { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' }\n      ]\n    )\n    test 'return an error object' do |(param, expected_error)|\n      response = @client.reset(param).get\n      errors = response.errors.first\n      assert_empty response.data\n      assert_equal expected_error, errors\n    end\n\n    test 'return an error object and data object' do\n      unknown_name = 'key2'\n\n      travel_sec = 6            # greater than reset_interval\n      travel(travel_sec)\n\n      response = @client.reset(@name, unknown_name).get\n      data = response.data.first\n      error = response.errors.first\n      counter = data['counter_data']\n\n      assert_true data['success']\n      assert_equal travel_sec, data['elapsed_time']\n      assert_equal @name, counter['name']\n      assert_equal @init_obj[:reset_interval], counter['reset_interval']\n      assert_equal @inc_obj[:value], counter['total']\n      assert_equal @inc_obj[:value], counter['current']\n\n      assert_equal 'unknown_key', error['code']\n      assert_equal \"`#{@scope}\\t#{unknown_name}` doesn't exist in counter\", error['message']\n\n      v1 = extract_value_from_server(@server, @scope, @name)\n      assert_equal 0, v1['current']\n      assert_equal @inc_obj[:value], v1['total']\n      assert_equal (@now + travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])\n      assert_equal (@now + travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])\n    end\n\n    test 'return a future object when async call' do\n      r = @client.reset(@name)\n      assert_true r.is_a?(Fluent::Counter::Future)\n      assert_nil r.errors\n    end\n  end\nend\n"
  },
  {
    "path": "test/counter/test_error.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/counter/error'\n\nclass CounterErrorTest < ::Test::Unit::TestCase\n  setup do\n    @message = 'error message'\n  end\n\n  test 'invalid_params' do\n    error = Fluent::Counter::InvalidParams.new(@message)\n    expected = { 'code' => 'invalid_params', 'message' => @message }\n    assert_equal expected, error.to_hash\n  end\n\n  test 'unknown_key' do\n    error = Fluent::Counter::UnknownKey.new(@message)\n    expected = { 'code' => 'unknown_key', 'message' => @message }\n    assert_equal expected, error.to_hash\n  end\n\n  test 'parse_error' do\n    error = Fluent::Counter::ParseError.new(@message)\n    expected = { 'code' => 'parse_error', 'message' => @message }\n    assert_equal expected, error.to_hash\n  end\n\n  test 'method_not_found' do\n    error = Fluent::Counter::MethodNotFound.new(@message)\n    expected = { 'code' => 'method_not_found', 'message' => @message }\n    assert_equal expected, error.to_hash\n  end\n\n  test 'invalid_request' do\n    error = Fluent::Counter::InvalidRequest.new(@message)\n    expected = { 'code' => 'invalid_request', 'message' => @message }\n    assert_equal expected, error.to_hash\n  end\n\n  test 'internal_server_error' do\n    error = Fluent::Counter::InternalServerError.new(@message)\n    expected = { 'code' => 'internal_server_error', 'message' => @message }\n    assert_equal expected, error.to_hash\n  end\nend\n"
  },
  {
    "path": "test/counter/test_mutex_hash.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/counter/mutex_hash'\nrequire 'fluent/counter/store'\nrequire 'flexmock/test_unit'\nrequire 'timecop'\n\nclass MutexHashTest < ::Test::Unit::TestCase\n  setup do\n    @store = {}\n    @value = 'sample value'\n    @counter_store_mutex = Fluent::Counter::MutexHash.new(@store)\n  end\n\n  sub_test_case 'synchronize' do\n    test \"create new mutex values if keys don't exist\" do\n      keys = ['key', 'key1']\n\n      @counter_store_mutex.synchronize(*keys) do |store, k|\n        store[k] = @value\n      end\n\n      mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)\n      keys.each do |key|\n        assert_true mhash[key].is_a?(Mutex)\n        assert_equal @value, @store[key]\n      end\n    end\n\n    test 'nothing to do when an empty array passed' do\n      @counter_store_mutex.synchronize(*[]) do |store, k|\n        store[k] = @value\n      end\n\n      mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)\n      assert_true mhash.empty?\n      assert_true @store.empty?\n    end\n\n    test 'use a one mutex value when the same key specified' do\n      key = 'key'\n      @counter_store_mutex.synchronize(key) do |store, k|\n        store[k] = @value\n      end\n\n      mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)\n      m = mhash[key]\n      assert_true m.is_a?(Mutex)\n      assert_equal @value, @store[key]\n\n      # access the same key once again\n      value2 = 'test value2'\n      @counter_store_mutex.synchronize(key) do |store, k|\n        store[k] = value2\n      end\n\n      mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)\n      m2 = mhash[key]\n      assert_true m2.is_a?(Mutex)\n      assert_equal value2, @store[key]\n\n      assert_equal m, m2\n    end\n  end\n\n  sub_test_case 'synchronize_key' do\n    test \"create new mutex values if keys don't exist\" do\n      keys = ['key', 'key1']\n\n      @counter_store_mutex.synchronize_keys(*keys) do |store, k|\n        store[k] = @value\n      end\n\n      mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)\n      keys.each do |key|\n        assert_true mhash[key].is_a?(Mutex)\n        assert_equal @value, @store[key]\n      end\n    end\n\n    test 'nothing to do when an empty array passed' do\n      @counter_store_mutex.synchronize_keys(*[]) do |store, k|\n        store[k] = @value\n      end\n\n      mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)\n      assert_true mhash.empty?\n      assert_true @store.empty?\n    end\n\n    test 'use a one mutex value when the same key specified' do\n      key = 'key'\n      @counter_store_mutex.synchronize_keys(key) do |store, k|\n        store[k] = @value\n      end\n\n      mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)\n      m = mhash[key]\n      assert_true m.is_a?(Mutex)\n      assert_equal @value, @store[key]\n\n      # access the same key once again\n      value2 = 'test value2'\n      @counter_store_mutex.synchronize_keys(key) do |store, k|\n        store[k] = value2\n      end\n\n      mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)\n      m2 = mhash[key]\n      assert_true m2.is_a?(Mutex)\n      assert_equal value2, @store[key]\n\n      assert_equal m, m2\n    end\n  end\nend\n\nclass CleanupThreadTest < ::Test::Unit::TestCase\n  StoreValue = Struct.new(:last_modified_at)\n\n  setup do\n    # timecop isn't compatible with EventTime\n    t = Time.parse('2016-09-22 16:59:59 +0900')\n    Timecop.freeze(t)\n\n    @store = Fluent::Counter::Store.new\n    @mhash = Fluent::Counter::MutexHash.new(@store)\n\n    # stub sleep method to avoid waiting CLEANUP_INTERVAL\n    ct = @mhash.instance_variable_get(:@cleanup_thread)\n    flexstub(ct).should_receive(:sleep)\n  end\n\n  teardown do\n    @mhash.stop\n    Timecop.return\n  end\n\n  test 'clean up unused mutex' do\n    name = 'key1'\n    init_obj = { 'name' => name, 'reset_interval' => 2 }\n\n    @mhash.synchronize(init_obj['name']) do\n      @store.init(name, init_obj)\n    end\n\n    ct = @mhash.instance_variable_get(:@mutex_hash)\n    assert ct[name]\n\n    Timecop.travel(15 * 60 + 1)     # 15 min\n\n    @mhash.start                # start cleanup\n    sleep 1\n\n    ct = @mhash.instance_variable_get(:@mutex_hash)\n    assert_empty ct\n\n    @mhash.stop\n  end\n\n  test \"don't remove when `last_modified_at` is greater than (Time.now - CLEANUP_INTERVAL)\" do\n    name = 'key1'\n    init_obj = { 'name' => name, 'reset_interval' => 2 }\n\n    @mhash.synchronize(init_obj['name']) do\n      @store.init(name, init_obj)\n    end\n\n    ct = @mhash.instance_variable_get(:@mutex_hash)\n    assert ct[name]\n\n    @mhash.start                # start cleanup\n    sleep 1\n\n    ct = @mhash.instance_variable_get(:@mutex_hash)\n    assert ct[name]\n\n    @mhash.stop\n  end\nend\n"
  },
  {
    "path": "test/counter/test_server.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/counter/server'\nrequire 'fluent/counter/store'\nrequire 'fluent/time'\nrequire 'flexmock/test_unit'\nrequire 'timecop'\n\nclass CounterServerTest < ::Test::Unit::TestCase\n  setup do\n    # timecop isn't compatible with EventTime\n    t = Time.parse('2016-09-22 16:59:59 +0900')\n    Timecop.freeze(t)\n    @now = Fluent::EventTime.now\n\n    @scope = \"server\\tworker\\tplugin\"\n    @server_name = 'server1'\n    @server = Fluent::Counter::Server.new(@server_name, opt: { log: $log })\n    @server.instance_eval { @server.close }\n  end\n\n  teardown do\n    Timecop.return\n  end\n\n  def extract_value_from_counter(counter, scope, name)\n    store = counter.instance_variable_get(:@store).instance_variable_get(:@storage).instance_variable_get(:@store)\n    key = Fluent::Counter::Store.gen_key(scope, name)\n    store[key]\n  end\n\n  def travel(sec)\n    # Since Timecop.travel() causes test failures on Windows/AppVeyor by inducing\n    # rounding errors to Time.now, we need to use Timecop.freeze() instead.\n    Timecop.freeze(Time.now + sec)\n  end\n\n  test 'raise an error when server name is invalid' do\n    assert_raise do\n      Fluent::Counter::Server.new(\"\\tinvalid_name\")\n    end\n  end\n\n  sub_test_case 'on_message' do\n    data(\n      establish: 'establish',\n      init: 'init',\n      delete: 'delete',\n      inc: 'inc',\n      get: 'get',\n      reset: 'reset',\n    )\n    test 'call valid methods' do |method|\n      stub(@server).send do |_m, params, scope, options|\n        { 'data' => [params, scope, options] }\n      end\n\n      request = { 'id' => 0, 'method' => method }\n      expected = { 'id' => 0, 'data' => [nil, nil, nil] }\n      assert_equal expected, @server.on_message(request)\n    end\n\n    data(\n      missing_id: [\n        { 'method' => 'init' },\n        { 'code' => 'invalid_request', 'message' => 'Request should include `id`' }\n      ],\n      missing_method: [\n        { 'id' => 0 },\n        { 'code' => 'invalid_request', 'message' => 'Request should include `method`' }\n      ],\n      invalid_method: [\n        { 'id' => 0, 'method' => 'invalid_method' },\n        { 'code' => 'method_not_found', 'message' => 'Unknown method name passed: invalid_method' }\n      ]\n    )\n    test 'invalid request' do |(request, error)|\n      expected = {\n        'id' => request['id'],\n        'data' => [],\n        'errors' => [error],\n      }\n\n      assert_equal expected, @server.on_message(request)\n    end\n\n    test 'return an `internal_server_error` error object when an error raises in safe_run' do\n      stub(@server).send do |_m, _params, _scope, _options|\n        raise 'Error in safe_run'\n      end\n\n      request = { 'id' => 0, 'method' => 'init' }\n      expected = {\n        'id' => request['id'],\n        'data' => [],\n        'errors' => [\n          { 'code' => 'internal_server_error', 'message' => 'Error in safe_run' }\n        ]\n      }\n      assert_equal expected, @server.on_message(request)\n    end\n\n    test 'output an error log when passed data is not Hash' do\n      data = 'this is not a hash'\n      mock($log).error(\"Received data is not Hash: #{data}\")\n      @server.on_message(data)\n    end\n  end\n\n  sub_test_case 'establish' do\n    test 'establish a scope in a counter' do\n      result = @server.send('establish', ['key'], nil, nil)\n      expected = { 'data' => [\"#{@server_name}\\tkey\"] }\n      assert_equal expected, result\n    end\n\n    data(\n      empty: [[], 'One or more `params` are required'],\n      empty_key: [[''], '`scope` is the invalid format'],\n      invalid_key: [['_key'], '`scope` is the invalid format'],\n    )\n    test 'raise an error: invalid_params' do |(params, msg)|\n      result = @server.send('establish', params, nil, nil)\n      expected = {\n        'data' => [],\n        'errors' => [{ 'code' => 'invalid_params', 'message' => msg }]\n      }\n      assert_equal expected, result\n    end\n  end\n\n  sub_test_case 'init' do\n    setup do\n      @name = 'key1'\n      @key = Fluent::Counter::Store.gen_key(@scope, @name)\n    end\n\n    test 'create new value in a counter' do\n      assert_nil extract_value_from_counter(@server, @scope, @name)\n      result = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {})\n      assert_nil result['errors']\n\n      counter = result['data'].first\n      assert_equal 'numeric', counter['type']\n      assert_equal @name, counter['name']\n      assert_equal 0, counter['current']\n      assert_equal 0, counter['total']\n      assert_equal @now, counter['last_reset_at']\n      assert extract_value_from_counter(@server, @scope, @name)\n    end\n\n    data(\n      numeric: 'numeric',\n      integer: 'integer',\n      float: 'float'\n    )\n    test 'set the type of a counter value' do |type|\n      result = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1, 'type' => type }], @scope, {})\n      counter = result['data'].first\n      assert_equal type, counter['type']\n\n      v = extract_value_from_counter(@server, @scope, @name)\n      assert_equal type, v['type']\n    end\n\n    data(\n      empty: [[], 'One or more `params` are required'],\n      missing_name: [\n        [{ 'rest_interval' => 20 }],\n        '`name` is required'\n      ],\n      invalid_name: [\n        [{ 'name' => '_test', 'reset_interval' => 20 }],\n        '`name` is the invalid format'\n      ],\n      missing_interval: [\n        [{ 'name' => 'test' }],\n        '`reset_interval` is required'\n      ],\n      minus_interval: [\n        [{ 'name' => 'test', 'reset_interval' => -1 }],\n        '`reset_interval` should be a positive number'\n      ],\n      invalid_type: [\n        [{ 'name' => 'test', 'reset_interval' => 1, 'type' => 'invalid_type' }],\n        '`type` should be integer, float, or numeric'\n      ]\n    )\n    test 'return an error object: invalid_params' do |(params, msg)|\n      result = @server.send('init', params, @scope, {})\n      assert_empty result['data']\n      error = result['errors'].first\n      assert_equal 'invalid_params', error['code']\n      assert_equal msg, error['message']\n    end\n\n    test 'return error objects when passed key already exists' do\n      @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {})\n\n      # call `init` to same key twice\n      result = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {})\n      assert_empty result['data']\n      error = result['errors'].first\n\n      expected = { 'code' => 'invalid_params', 'message' => \"#{@key} already exists in counter\" }\n      assert_equal expected, error\n    end\n\n    test 'return existing value when passed key already exists and ignore option is true' do\n      v1 = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {})\n\n      # call `init` to same key twice\n      v2 = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, { 'ignore' => true })\n      assert_nil v2['errors']\n      assert_equal v1['data'], v2['data']\n    end\n\n    test 'call `synchronize_keys` when random option is true' do\n      mhash = @server.instance_variable_get(:@mutex_hash)\n      mock(mhash).synchronize(@key).once\n      @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {})\n\n      mhash = @server.instance_variable_get(:@mutex_hash)\n      mock(mhash).synchronize_keys(@key).once\n      @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, { 'random' => true })\n    end\n  end\n\n  sub_test_case 'delete' do\n    setup do\n      @name = 'key1'\n      @key = Fluent::Counter::Store.gen_key(@scope, @name)\n      @server.send('init', [{ 'name' => @name, 'reset_interval' => 20 }], @scope, {})\n    end\n\n    test 'delete a value from a counter' do\n      assert extract_value_from_counter(@server, @scope, @name)\n\n      result = @server.send('delete', [@name], @scope, {})\n      assert_nil result['errors']\n\n      counter = result['data'].first\n      assert_equal 0, counter['current']\n      assert_equal 0, counter['total']\n      assert_equal 'numeric', counter['type']\n      assert_equal @name, counter['name']\n      assert_equal @now, counter['last_reset_at']\n\n      assert_nil extract_value_from_counter(@server, @scope, @name)\n    end\n\n    data(\n      empty: [[], 'One or more `params` are required'],\n      empty_key: [[''], '`key` is the invalid format'],\n      invalid_key: [['_key'], '`key` is the invalid format'],\n    )\n    test 'return an error object: invalid_params' do |(params, msg)|\n      result = @server.send('delete', params, @scope, {})\n\n      assert_empty result['data']\n      error = result['errors'].first\n      assert_equal 'invalid_params', error['code']\n      assert_equal msg, error['message']\n    end\n\n    test 'return an error object: unknown_key' do\n      unknown_key = 'unknown_key'\n      result = @server.send('delete', [unknown_key], @scope, {})\n\n      assert_empty result['data']\n      error = result['errors'].first\n      assert_equal unknown_key, error['code']\n      assert_equal \"`#{@scope}\\t#{unknown_key}` doesn't exist in counter\", error['message']\n    end\n\n    test 'call `synchronize_keys` when random option is true' do\n      mhash = @server.instance_variable_get(:@mutex_hash)\n      mock(mhash).synchronize(@key).once\n      @server.send('delete', [@name], @scope, {})\n\n      mhash = @server.instance_variable_get(:@mutex_hash)\n      mock(mhash).synchronize_keys(@key).once\n      @server.send('delete', [@name], @scope, { 'random' => true })\n    end\n  end\n\n  sub_test_case 'inc' do\n    setup do\n      @name1 = 'key1'\n      @name2 = 'key2'\n      @key1 = Fluent::Counter::Store.gen_key(@scope, @name1)\n      inc_objects = [\n        { 'name' => @name1, 'reset_interval' => 20 },\n        { 'name' => @name2, 'type' => 'integer', 'reset_interval' => 20 }\n      ]\n      @server.send('init', inc_objects, @scope, {})\n    end\n\n    test 'increment or decrement a value in counter' do\n      result = @server.send('inc', [{ 'name' => @name1, 'value' => 10 }], @scope, {})\n      assert_nil result['errors']\n\n      counter = result['data'].first\n      assert_equal 10, counter['current']\n      assert_equal 10, counter['total']\n      assert_equal 'numeric', counter['type']\n      assert_equal @name1, counter['name']\n      assert_equal @now, counter['last_reset_at']\n\n      c = extract_value_from_counter(@server, @scope, @name1)\n      assert_equal 10, c['current']\n      assert_equal 10, c['total']\n      assert_equal @now, Fluent::EventTime.new(*c['last_reset_at'])\n      assert_equal @now, Fluent::EventTime.new(*c['last_modified_at'])\n    end\n\n    test 'create new value and increment/decrement its value when `force` option is true' do\n      new_name = 'new_key'\n      assert_nil extract_value_from_counter(@server, @scope, new_name)\n\n      v1 = @server.send('inc', [{ 'name' => new_name, 'value' => 10 }], @scope, {})\n      assert_empty v1['data']\n      error = v1['errors'].first\n      assert_equal 'unknown_key', error['code']\n\n      assert_nil extract_value_from_counter(@server, @scope, new_name)\n\n      v2 = @server.send(\n        'inc',\n        [{ 'name' => new_name, 'value' => 10, 'reset_interval' => 20 }],\n        @scope,\n        { 'force' => true }\n      )\n\n      assert_nil v2['errors']\n\n      counter = v2['data'].first\n      assert_equal 10, counter['current']\n      assert_equal 10, counter['total']\n      assert_equal 'numeric', counter['type']\n      assert_equal new_name, counter['name']\n      assert_equal @now, counter['last_reset_at']\n\n      assert extract_value_from_counter(@server, @scope, new_name)\n    end\n\n    data(\n      empty: [[], 'One or more `params` are required', {}],\n      missing_name: [\n        [{ 'value' => 10 }],\n        '`name` is required', {}\n      ],\n      missing_value: [\n        [{ 'name' => 'key1' }],\n        '`value` is required', {}\n      ],\n      invalid_type: [\n        [{ 'name' => 'key2', 'value' => 10.0 }],\n        '`type` is integer. You should pass integer value as a `value`', {}\n      ],\n      missing_reset_interval: [\n        [{ 'name' => 'key1', 'value' => 1 }],\n        '`reset_interval` is required',\n        { 'force' => true }\n      ]\n    )\n    test 'return an error object: invalid_params' do |(params, msg, opt)|\n      result = @server.send('inc', params, @scope, opt)\n      assert_empty result['data']\n\n      error = result['errors'].first\n      assert_equal 'invalid_params', error['code']\n      assert_equal msg, error['message']\n    end\n\n    test 'call `synchronize_keys` when random option is true' do\n      mhash = @server.instance_variable_get(:@mutex_hash)\n      mock(mhash).synchronize(@key1).once\n      params = [{ 'name' => @name1, 'value' => 1 }]\n      @server.send('inc', params, @scope, {})\n\n      mhash = @server.instance_variable_get(:@mutex_hash)\n      mock(mhash).synchronize_keys(@key1).once\n      @server.send('inc', params, @scope, { 'random' => true })\n    end\n  end\n\n  sub_test_case 'reset' do\n    setup do\n      @name = 'key'\n      @travel_sec = 10\n      @server.send('init', [{ 'name' => @name, 'reset_interval' => 10 }], @scope, {})\n      @server.send('inc', [{ 'name' => @name, 'value' => 10 }], @scope, {})\n    end\n\n    test 'reset a value in the counter' do\n      travel(@travel_sec)\n\n      result = @server.send('reset', [@name], @scope, {})\n      assert_nil result['errors']\n\n      data = result['data'].first\n      assert_true data['success']\n      assert_equal @travel_sec, data['elapsed_time']\n\n      counter = data['counter_data']\n      assert_equal 10, counter['current']\n      assert_equal 10, counter['total']\n      assert_equal 'numeric', counter['type']\n      assert_equal @name, counter['name']\n      assert_equal @now, counter['last_reset_at']\n\n      v = extract_value_from_counter(@server, @scope, @name)\n      assert_equal 0, v['current']\n      assert_equal 10, v['total']\n      assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v['last_reset_at'])\n      assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v['last_modified_at'])\n    end\n\n    test 'reset a value after `reset_interval` passed' do\n      first_travel_sec = 5\n      travel(first_travel_sec) # jump time less than reset_interval\n      result = @server.send('reset', [@name], @scope, {})\n      v = result['data'].first\n\n      assert_equal false, v['success']\n      assert_equal first_travel_sec, v['elapsed_time']\n\n      store = extract_value_from_counter(@server, @scope, @name)\n      assert_equal 10, store['current']\n      assert_equal @now, Fluent::EventTime.new(*store['last_reset_at'])\n\n      # time is passed greater than reset_interval\n      travel(@travel_sec)\n      result = @server.send('reset', [@name], @scope, {})\n      v = result['data'].first\n\n      assert_true v['success']\n      assert_equal @travel_sec + first_travel_sec, v['elapsed_time']\n\n      v1 = extract_value_from_counter(@server, @scope, @name)\n      assert_equal 0, v1['current']\n      assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])\n      assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])\n    end\n\n    data(\n      empty: [[], 'One or more `params` are required'],\n      empty_key: [[''], '`key` is the invalid format'],\n      invalid_key: [['_key'], '`key` is the invalid format'],\n    )\n    test 'return an error object: invalid_params' do |(params, msg)|\n      result = @server.send('reset', params, @scope, {})\n      assert_empty result['data']\n      assert_equal 'invalid_params', result['errors'].first['code']\n      assert_equal msg, result['errors'].first['message']\n    end\n\n    test 'return an error object: unknown_key' do\n      unknown_key = 'unknown_key'\n      result = @server.send('reset', [unknown_key], @scope, {})\n\n      assert_empty result['data']\n      error = result['errors'].first\n      assert_equal unknown_key, error['code']\n      assert_equal \"`#{@scope}\\t#{unknown_key}` doesn't exist in counter\", error['message']\n    end\n  end\n\n  sub_test_case 'get' do\n    setup do\n      @name1 = 'key1'\n      @name2 = 'key2'\n      init_objects = [\n        { 'name' => @name1, 'reset_interval' => 0 },\n        { 'name' => @name2, 'reset_interval' => 0 },\n      ]\n      @server.send('init', init_objects, @scope, {})\n    end\n\n    test 'get a counter value' do\n      key = @name1\n      result = @server.send('get', [key], @scope, {})\n      assert_nil result['errors']\n\n      counter = result['data'].first\n      assert_equal 0, counter['current']\n      assert_equal 0, counter['total']\n      assert_equal 'numeric', counter['type']\n      assert_equal key, counter['name']\n    end\n\n    test 'get counter values' do\n      result = @server.send('get', [@name1, @name2], @scope, {})\n      assert_nil result['errors']\n\n      counter1 = result['data'][0]\n      assert_equal 0, counter1['current']\n      assert_equal 0, counter1['total']\n      assert_equal 'numeric', counter1['type']\n      assert_equal @name1, counter1['name']\n\n      counter2 = result['data'][1]\n      assert_equal 0, counter2['current']\n      assert_equal 0, counter2['total']\n      assert_equal 'numeric', counter2['type']\n      assert_equal @name2, counter2['name']\n    end\n\n    data(\n      empty: [[], 'One or more `params` are required'],\n      empty_key: [[''], '`key` is the invalid format'],\n      invalid_key: [['_key'], '`key` is the invalid format'],\n    )\n    test 'return an error object: invalid_params' do |(params, msg)|\n      result = @server.send('get', params, @scope, {})\n      assert_empty result['data']\n      assert_equal 'invalid_params', result['errors'].first['code']\n      assert_equal msg, result['errors'].first['message']\n    end\n\n    test 'return an error object: unknown_key' do\n      unknown_key = 'unknown_key'\n      result = @server.send('get', [unknown_key], @scope, {})\n\n      assert_empty result['data']\n      error = result['errors'].first\n      assert_equal unknown_key, error['code']\n      assert_equal \"`#{@scope}\\t#{unknown_key}` doesn't exist in counter\", error['message']\n    end\n  end\nend\n\nclass CounterCounterResponseTest < ::Test::Unit::TestCase\n  setup do\n    @response = Fluent::Counter::Server::Response.new\n    @errors = [\n      StandardError.new('standard error'),\n      Fluent::Counter::InternalServerError.new('internal server error')\n    ]\n    @now = Fluent::EventTime.now\n    value = {\n      'name' => 'name',\n      'total' => 100,\n      'current' => 11,\n      'type' => 'numeric',\n      'reset_interval' => 10,\n      'last_reset_at' => @now,\n    }\n    @values = [value, 'test']\n  end\n\n  test 'push_error' do\n    @errors.each do |e|\n      @response.push_error(e)\n    end\n\n    v = @response.instance_variable_get(:@errors)\n    assert_equal @errors, v\n  end\n\n  test 'push_data' do\n    @values.each do |v|\n      @response.push_data v\n    end\n\n    data = @response.instance_variable_get(:@data)\n    assert_equal @values, data\n  end\n\n  test 'to_hash' do\n    @errors.each do |e|\n      @response.push_error(e)\n    end\n\n    @values.each do |v|\n      @response.push_data v\n    end\n\n    expected_errors = [\n      { 'code' => 'internal_server_error', 'message' => 'standard error' },\n      { 'code' => 'internal_server_error', 'message' => 'internal server error' }\n    ]\n    expected_data = @values\n\n    hash = @response.to_hash\n    assert_equal expected_errors, hash['errors']\n    assert_equal expected_data, hash['data']\n  end\nend\n"
  },
  {
    "path": "test/counter/test_store.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/counter/store'\nrequire 'fluent/time'\nrequire 'timecop'\n\nclass CounterStoreTest < ::Test::Unit::TestCase\n  setup do\n    @name = 'key_name'\n    @scope = \"server\\tworker\\tplugin\"\n\n    # timecop isn't compatible with EventTime\n    t = Time.parse('2016-09-22 16:59:59 +0900')\n    Timecop.freeze(t)\n    @now = Fluent::EventTime.now\n  end\n\n  teardown do\n    Timecop.return\n  end\n\n  def extract_value_from_counter(counter, key)\n    store = counter.instance_variable_get(:@storage).instance_variable_get(:@store)\n    store[key]\n  end\n\n  def travel(sec)\n    # Since Timecop.travel() causes test failures on Windows/AppVeyor by inducing\n    # rounding errors to Time.now, we need to use Timecop.freeze() instead.\n    Timecop.freeze(Time.now + sec)\n  end\n\n  sub_test_case 'init' do\n    setup do\n      @reset_interval = 10\n      @store = Fluent::Counter::Store.new\n      @data = { 'name' => @name, 'reset_interval' => @reset_interval }\n      @key = Fluent::Counter::Store.gen_key(@scope, @name)\n    end\n\n    test 'create new value in the counter' do\n      v = @store.init(@key, @data)\n\n      assert_equal @name, v['name']\n      assert_equal @reset_interval, v['reset_interval']\n\n      v2 = extract_value_from_counter(@store, @key)\n      v2 = @store.send(:build_response, v2)\n      assert_equal v, v2\n    end\n\n    test 'raise an error when a passed key already exists' do\n      @store.init(@key, @data)\n\n      assert_raise Fluent::Counter::InvalidParams do\n        @store.init(@key, @data)\n      end\n    end\n\n    test 'return a value when passed key already exists and a ignore option is true' do\n      v = @store.init(@key, @data)\n      v1 = extract_value_from_counter(@store, @key)\n      v1 = @store.send(:build_response, v1)\n      v2 = @store.init(@key, @data, ignore: true)\n      assert_equal v, v2\n      assert_equal v1, v2\n    end\n  end\n\n  sub_test_case 'get' do\n    setup do\n      @store = Fluent::Counter::Store.new\n      data = { 'name' => @name, 'reset_interval' => 10 }\n      @key = Fluent::Counter::Store.gen_key(@scope, @name)\n      @store.init(@key, data)\n    end\n\n    test 'return a value from the counter' do\n      v = extract_value_from_counter(@store, @key)\n      expected = @store.send(:build_response, v)\n      assert_equal expected, @store.get(@key)\n    end\n\n    test 'return a raw value from the counter when raw option is true' do\n      v = extract_value_from_counter(@store, @key)\n      assert_equal v, @store.get(@key, raw: true)\n    end\n\n    test \"return nil when a passed key doesn't exist\" do\n      assert_equal nil, @store.get('unknown_key')\n    end\n\n    test \"raise a error when a passed key doesn't exist and raise_error option is true\" do\n      assert_raise Fluent::Counter::UnknownKey do\n        @store.get('unknown_key', raise_error: true)\n      end\n    end\n  end\n\n  sub_test_case 'key?' do\n    setup do\n      @store = Fluent::Counter::Store.new\n      data = { 'name' => @name, 'reset_interval' => 10 }\n      @key = Fluent::Counter::Store.gen_key(@scope, @name)\n      @store.init(@key, data)\n    end\n\n    test 'return true when passed key exists' do\n      assert_true @store.key?(@key)\n    end\n\n    test \"return false when passed key doesn't exist\" do\n      assert_true !@store.key?('unknown_key')\n    end\n  end\n\n  sub_test_case 'delete' do\n    setup do\n      @store = Fluent::Counter::Store.new\n      data = { 'name' => @name, 'reset_interval' => 10 }\n      @key = Fluent::Counter::Store.gen_key(@scope, @name)\n      @init_value = @store.init(@key, data)\n    end\n\n    test 'delete a value from the counter' do\n      v = @store.delete(@key)\n      assert_equal @init_value, v\n      assert_nil extract_value_from_counter(@store, @key)\n    end\n\n    test \"raise an error when passed key doesn't exist\" do\n      assert_raise Fluent::Counter::UnknownKey do\n        @store.delete('unknown_key')\n      end\n    end\n  end\n\n  sub_test_case 'inc' do\n    setup do\n      @store = Fluent::Counter::Store.new\n      @init_data = { 'name' => @name, 'reset_interval' => 10 }\n      @travel_sec = 10\n    end\n\n    data(\n      positive: 10,\n      negative: -10\n    )\n    test 'increment or decrement a value in the counter' do |value|\n      key = Fluent::Counter::Store.gen_key(@scope, @name)\n      @store.init(key, @init_data)\n      travel(@travel_sec)\n      v = @store.inc(key, { 'value' => value })\n\n      assert_equal value, v['total']\n      assert_equal value, v['current']\n      assert_equal @now, v['last_reset_at'] # last_reset_at doesn't change\n\n      v1 = extract_value_from_counter(@store, key)\n      v1 = @store.send(:build_response, v1)\n      assert_equal v, v1\n    end\n\n    test \"raise an error when passed key doesn't exist\" do\n      assert_raise Fluent::Counter::UnknownKey do\n        @store.inc('unknown_key', { 'value' => 1 })\n      end\n    end\n\n    test 'raise an error when a type of passed value is incompatible with a stored value' do\n      key1 = Fluent::Counter::Store.gen_key(@scope, @name)\n      key2 = Fluent::Counter::Store.gen_key(@scope, 'name2')\n      key3 = Fluent::Counter::Store.gen_key(@scope, 'name3')\n      v1 = @store.init(key1, @init_data.merge('type' => 'integer'))\n      v2 = @store.init(key2, @init_data.merge('type' => 'float'))\n      v3 = @store.init(key3, @init_data.merge('type' => 'numeric'))\n      assert_equal 'integer', v1['type']\n      assert_equal 'float', v2['type']\n      assert_equal 'numeric', v3['type']\n\n      assert_raise Fluent::Counter::InvalidParams do\n        @store.inc(key1, { 'value' => 1.1 })\n      end\n\n      assert_raise Fluent::Counter::InvalidParams do\n        @store.inc(key2, { 'value' => 1 })\n      end\n\n      assert_nothing_raised do\n        @store.inc(key3, { 'value' => 1 })\n        @store.inc(key3, { 'value' => 1.0 })\n      end\n    end\n  end\n\n  sub_test_case 'reset' do\n    setup do\n      @store = Fluent::Counter::Store.new\n      @travel_sec = 10\n\n      @inc_value = 10\n      @key = Fluent::Counter::Store.gen_key(@scope, @name)\n      @store.init(@key, { 'name' => @name, 'reset_interval' => 10 })\n      @store.inc(@key, { 'value' => 10 })\n    end\n\n    test 'reset a value in the counter' do\n      travel(@travel_sec)\n\n      v = @store.reset(@key)\n      assert_equal @travel_sec, v['elapsed_time']\n      assert_true v['success']\n      counter = v['counter_data']\n\n      assert_equal @name, counter['name']\n      assert_equal @inc_value, counter['total']\n      assert_equal @inc_value, counter['current']\n      assert_equal 'numeric', counter['type']\n      assert_equal @now, counter['last_reset_at']\n      assert_equal 10, counter['reset_interval']\n\n      v1 = extract_value_from_counter(@store, @key)\n      assert_equal 0, v1['current']\n      assert_true v1['current'].is_a?(Integer)\n      assert_equal @inc_value, v1['total']\n      assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])\n      assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])\n    end\n\n    test 'reset a value after `reset_interval` passed' do\n      first_travel_sec = 5\n      travel(first_travel_sec) # jump time less than reset_interval\n      v = @store.reset(@key)\n\n      assert_equal false, v['success']\n      assert_equal first_travel_sec, v['elapsed_time']\n      store = extract_value_from_counter(@store, @key)\n      assert_equal 10, store['current']\n      assert_equal @now, Fluent::EventTime.new(*store['last_reset_at'])\n\n      # time is passed greater than reset_interval\n      travel(@travel_sec)\n      v = @store.reset(@key)\n      assert_true v['success']\n      assert_equal @travel_sec + first_travel_sec, v['elapsed_time']\n\n      v1 = extract_value_from_counter(@store, @key)\n      assert_equal 0, v1['current']\n      assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])\n      assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])\n    end\n\n    test \"raise an error when passed key doesn't exist\" do\n      assert_raise Fluent::Counter::UnknownKey do\n        @store.reset('unknown_key')\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/counter/test_validator.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/counter/validator'\n\nclass CounterValidatorTest < ::Test::Unit::TestCase\n  data(\n    invalid_name1: '',\n    invalid_name3: '_',\n    invalid_name4: 'A',\n    invalid_name5: 'a*',\n    invalid_name6: \"a\\t\",\n    invalid_name7: \"\\n\",\n  )\n  test 'invalid name' do |invalid_name|\n    assert_nil(Fluent::Counter::Validator::VALID_NAME =~ invalid_name)\n  end\n\n  sub_test_case 'request' do\n    test 'return an empty array' do\n      data = { 'id' => 0, 'method' => 'init' }\n      errors = Fluent::Counter::Validator.request(data)\n      assert_empty errors\n    end\n\n    data(\n      missing_id: [\n        { 'method' => 'init' },\n        { 'code' => 'invalid_request', 'message' => 'Request should include `id`' }\n      ],\n      missing_method: [\n        { 'id' => 0 },\n        { 'code' => 'invalid_request', 'message' => 'Request should include `method`' }\n      ],\n      invalid_method: [\n        { 'id' => 0, 'method' => \"A\\t\" },\n        { 'code' => 'invalid_request', 'message' => '`method` is the invalid format' }\n      ],\n      unknown_method: [\n        { 'id' => 0, 'method' => 'unknown_method' },\n        { 'code' => 'method_not_found', 'message' => 'Unknown method name passed: unknown_method' }\n      ]\n    )\n    test 'return an error array' do |(data, expected_error)|\n      errors = Fluent::Counter::Validator.request(data)\n      assert_equal [expected_error], errors\n    end\n  end\n\n  sub_test_case 'call' do\n    test \"return an error hash when passed method doesn't exist\" do\n      v = Fluent::Counter::Validator.new(:unknown)\n      success, errors = v.call(['key1'])\n      assert_empty success\n      assert_equal 'internal_server_error', errors.first.to_hash['code']\n    end\n  end\n\n  test 'validate_empty!' do\n    v = Fluent::Counter::Validator.new(:empty)\n    success, errors = v.call([])\n    assert_empty success\n    assert_equal [Fluent::Counter::InvalidParams.new('One or more `params` are required')], errors\n  end\nend\n\nclass CounterArrayValidatorTest < ::Test::Unit::TestCase\n  test 'validate_key!' do\n    ary = ['key', 100, '_']\n    error_expected = [\n      { 'code' => 'invalid_params', 'message' => 'The type of `key` should be String' },\n      { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' }\n    ]\n    v = Fluent::Counter::ArrayValidator.new(:key)\n    valid_params, errors = v.call(ary)\n\n    assert_equal ['key'], valid_params\n    assert_equal error_expected, errors.map(&:to_hash)\n  end\nend\n\nclass CounterHashValidatorTest < ::Test::Unit::TestCase\n  test 'validate_name!' do\n    hash = [\n      { 'name' => 'key' },\n      {},\n      { 'name' => 10 },\n      { 'name' => '_' }\n    ]\n    error_expected = [\n      { 'code' => 'invalid_params', 'message' => '`name` is required' },\n      { 'code' => 'invalid_params', 'message' => 'The type of `name` should be String' },\n      { 'code' => 'invalid_params', 'message' => '`name` is the invalid format' },\n    ]\n    v = Fluent::Counter::HashValidator.new(:name)\n    success, errors = v.call(hash)\n\n    assert_equal [{ 'name' => 'key' }], success\n    assert_equal error_expected, errors.map(&:to_hash)\n  end\n\n  test 'validate_value!' do\n    hash = [\n      { 'value' => 1 },\n      { 'value' => -1 },\n      {},\n      { 'value' => 'str' }\n    ]\n    error_expected = [\n      { 'code' => 'invalid_params', 'message' => '`value` is required' },\n      { 'code' => 'invalid_params', 'message' => 'The type of `value` type should be Numeric' },\n    ]\n    v = Fluent::Counter::HashValidator.new(:value)\n    valid_params, errors = v.call(hash)\n\n    assert_equal [{ 'value' => 1 }, { 'value' => -1 }], valid_params\n    assert_equal error_expected, errors.map(&:to_hash)\n  end\n\n  test 'validate_reset_interval!' do\n    hash = [\n      { 'reset_interval' => 1 },\n      { 'reset_interval' => 1.0 },\n      {},\n      { 'reset_interval' => -1 },\n      { 'reset_interval' => 'str' }\n    ]\n    error_expected = [\n      { 'code' => 'invalid_params', 'message' => '`reset_interval` is required' },\n      { 'code' => 'invalid_params', 'message' => '`reset_interval` should be a positive number' },\n      { 'code' => 'invalid_params', 'message' => 'The type of `reset_interval` should be Numeric' },\n    ]\n    v = Fluent::Counter::HashValidator.new(:reset_interval)\n    valid_params, errors = v.call(hash)\n\n    assert_equal [{ 'reset_interval' => 1 }, { 'reset_interval' => 1.0 }], valid_params\n    assert_equal error_expected.map(&:to_hash), errors.map(&:to_hash)\n  end\nend\n"
  },
  {
    "path": "test/helper.rb",
    "content": "# simplecov must be loaded before any of target code\nif ENV['SIMPLE_COV']\n  require 'simplecov'\n  if defined?(SimpleCov::SourceFile)\n    mod = SimpleCov::SourceFile\n    def mod.new(*args, &block)\n      m = allocate\n      m.instance_eval do\n        begin\n          initialize(*args, &block)\n        rescue Encoding::UndefinedConversionError\n          @src = \"\".force_encoding('UTF-8')\n        end\n      end\n      m\n    end\n  end\n  unless SimpleCov.running\n    SimpleCov.start do\n      add_filter '/test/'\n      add_filter '/gems/'\n    end\n  end\nend\n\n# Some tests use Hash instead of Element for configure.\n# We should rewrite these tests in the future and remove this ad-hoc code\nclass Hash\n  def corresponding_proxies\n    @corresponding_proxies ||= []\n  end\n\n  def to_masked_element\n    self\n  end\nend\n\nrequire 'rr'\nrequire 'test/unit'\nrequire 'test/unit/rr'\nrequire 'fileutils'\nrequire 'fluent/config/element'\nrequire 'fluent/log'\nrequire 'fluent/test'\nrequire 'fluent/test/helpers'\nrequire 'fluent/plugin/base'\nrequire 'fluent/plugin_id'\nrequire 'fluent/plugin_helper'\nrequire 'fluent/msgpack_factory'\nrequire 'fluent/time'\nrequire 'serverengine'\nrequire_relative 'helpers/fuzzy_assert'\nrequire_relative 'helpers/process_extenstion'\n\nmodule Fluent\n  module Plugin\n    class TestBase < Base\n      # a base plugin class, but not input nor output\n      # mainly for helpers and owned plugins\n      include PluginId\n      include PluginLoggerMixin\n      include PluginHelper::Mixin\n    end\n  end\nend\n\nunless defined?(Test::Unit::AssertionFailedError)\n  class Test::Unit::AssertionFailedError < StandardError\n  end\nend\n\ninclude Fluent::Test::Helpers\n\ndef unused_port(num = 1, protocol:, bind: \"0.0.0.0\")\n  case protocol\n  when :tcp, :tls\n    unused_port_tcp(num)\n  when :udp\n    unused_port_udp(num, bind: bind)\n  when :all\n    unused_port_tcp_udp(num)\n  else\n    raise ArgumentError, \"unknown protocol: #{protocol}\"\n  end\nend\n\ndef unused_port_tcp_udp(num = 1)\n  raise \"not support num > 1\" if num > 1\n\n  # The default maximum number of file descriptors in macOS is 256.\n  # It might need to set num to a smaller value than that.\n  tcp_ports = unused_port_tcp(200)\n  port = unused_port_udp(1, port_list: tcp_ports)\n  raise \"can't find unused port\" unless port\n\n  port\nend\n\ndef unused_port_tcp(num = 1)\n  ports = []\n  sockets = []\n  num.times do\n    s = TCPServer.open(0)\n    sockets << s\n    ports << s.addr[1]\n  end\n  sockets.each(&:close)\n  if num == 1\n    return ports.first\n  else\n    return *ports\n  end\nend\n\nPORT_RANGE_AVAILABLE = (1024...65535)\n\ndef unused_port_udp(num = 1, port_list: [], bind: \"0.0.0.0\")\n  family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6\n  ports = []\n  sockets = []\n\n  use_random_port = port_list.empty?\n  i = 0\n  loop do\n    port = use_random_port ? rand(PORT_RANGE_AVAILABLE) : port_list[i]\n    u = UDPSocket.new(family)\n    if (u.bind(bind, port) rescue nil)\n      ports << port\n      sockets << u\n    else\n      u.close\n    end\n    i += 1\n    break if ports.size >= num\n    break if !use_random_port && i >= port_list.size\n  end\n  sockets.each(&:close)\n  if num == 1\n    return ports.first\n  else\n    return *ports\n  end\nend\n\ndef waiting(seconds, logs: nil, plugin: nil)\n  begin\n    Timeout.timeout(seconds) do\n      yield\n    end\n  rescue Timeout::Error\n    if logs\n      STDERR.print(*logs)\n    elsif plugin\n      STDERR.print(*plugin.log.out.logs)\n    end\n    raise\n  end\nend\n\ndef ipv6_enabled?\n  require 'socket'\n\n  begin\n    # Try to actually bind to an IPv6 address to verify it works\n    sock = Socket.new(Socket::AF_INET6, Socket::SOCK_STREAM, 0)\n    sock.bind(Socket.sockaddr_in(0, '::1'))\n    sock.close\n    \n    # Also test that we can resolve IPv6 addresses\n    # This is needed because some systems can bind but can't connect\n    Socket.getaddrinfo('::1', nil, Socket::AF_INET6)\n    true\n  rescue Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, SocketError\n    false\n  end\nend\n\ndl_opts = {}\ndl_opts[:log_level] = ServerEngine::DaemonLogger::WARN\nlogdev = Fluent::Test::DummyLogDevice.new\nlogger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n$log ||= Fluent::Log.new(logger)\n"
  },
  {
    "path": "test/helpers/fuzzy_assert.rb",
    "content": "require 'test/unit'\n\nclass FuzzyIncludeAssertion\n  include Test::Unit::Assertions\n\n  def self.assert(expected, actual, message = nil)\n    new(expected, actual, message).assert\n  end\n\n  def initialize(expected, actual, message)\n    @expected = expected\n    @actual = actual\n    @message = message\n  end\n\n  def assert\n    if collection?\n      assert_same_collection\n    else\n      assert_same_value\n    end\n  end\n\n  private\n\n  def assert_same_value\n    m = \"expected(#{@expected}) !== actual(#{@actual.inspect})\"\n    if @message\n      m = \"#{@message}: #{m}\"\n    end\n    assert_true(@expected === @actual, m)\n  end\n\n  def assert_same_class\n    if @expected.class != @actual.class\n      if (@expected.class.ancestors | @actual.class.ancestors).empty?\n        assert_equal(@expected.class, @actual.class, @message)\n      end\n    end\n  end\n\n  def assert_same_collection\n    assert_same_class\n    assert_same_values\n  end\n\n  def assert_same_values\n    if @expected.is_a?(Array)\n      @expected.each_with_index do |val, i|\n        self.class.assert(val, @actual[i], @message)\n      end\n    else\n      @expected.each do |key, val|\n        self.class.assert(val, @actual[key], \"#{key}: \")\n      end\n    end\n  end\n\n  def collection?\n    @actual.is_a?(Array) || @actual.is_a?(Hash)\n  end\nend\n\nclass FuzzyAssertion < FuzzyIncludeAssertion\n  private\n\n  def assert_same_collection\n    super\n    assert_same_keys\n  end\n\n  def assert_same_keys\n    if @expected.is_a?(Array)\n      assert_equal(@expected.size, @actual.size, \"expected.size(#{@expected}) != actual.size(#{@expected})\")\n    else\n      assert_equal(@expected.keys.sort, @actual.keys.sort)\n    end\n  end\nend\n\nmodule FuzzyAssert\n  def assert_fuzzy_include(left, right, message = nil)\n    FuzzyIncludeAssertion.new(left, right, message).assert\n  end\n\n  def assert_fuzzy_equal(left, right, message = nil)\n    FuzzyAssertion.new(left, right, message).assert\n  end\nend\n"
  },
  {
    "path": "test/helpers/process_extenstion.rb",
    "content": "require 'timecop'\n\nmodule Process\n  class << self\n    alias_method :clock_gettime_original, :clock_gettime\n\n    def clock_gettime(clock_id, unit = :float_second)\n      # now only support CLOCK_REALTIME\n      if Process::CLOCK_REALTIME == clock_id\n        t = Time.now\n\n        case unit\n        when :float_second\n          t.to_i + t.nsec / 1_000_000_000.0\n        when :float_millisecond\n          t.to_i * 1_000 + t.nsec / 1_000_000.0\n        when :float_microsecond\n          t.to_i * 1_000_000 + t.nsec / 1_000.0\n        when :second\n          t.to_i\n        when :millisecond\n          t.to_i * 1000 + t.nsec / 1_000_000\n        when :microsecond\n          t.to_i * 1_000_000 + t.nsec / 1_000\n        when :nanosecond\n          t.to_i * 1_000_000_000 + t.nsec\n        end\n      else\n        Process.clock_gettime_original(clock_id, unit)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/log/test_console_adapter.rb",
    "content": "require_relative '../helper'\n\nrequire 'fluent/log'\nrequire 'fluent/log/console_adapter'\n\nclass ConsoleAdapterTest < Test::Unit::TestCase\n  def setup\n    @timestamp = Time.parse(\"2023-01-01 15:32:41 +0000\")\n    @timestamp_str = @timestamp.strftime(\"%Y-%m-%d %H:%M:%S %z\")\n    Timecop.freeze(@timestamp)\n\n    @logdev = Fluent::Test::DummyLogDevice.new\n    @logger = ServerEngine::DaemonLogger.new(@logdev)\n    @fluent_log = Fluent::Log.new(@logger)\n    @console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)\n  end\n\n  def teardown\n    Timecop.return\n  end\n\n  def test_expected_log_levels\n    assert_equal({debug: 0, info: 1, warn: 2, error: 3, fatal: 4},\n                 Console::Logger::LEVELS)\n  end\n\n  data(trace: [Fluent::Log::LEVEL_TRACE, :debug],\n       debug: [Fluent::Log::LEVEL_DEBUG, :debug],\n       info: [Fluent::Log::LEVEL_INFO, :info],\n       warn: [Fluent::Log::LEVEL_WARN, :warn],\n       error: [Fluent::Log::LEVEL_ERROR, :error],\n       fatal: [Fluent::Log::LEVEL_FATAL, :fatal])\n  def test_reflect_log_level(data)\n    level, expected = data\n    @fluent_log.level = level\n    console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)\n    assert_equal(Console::Logger::LEVELS[expected],\n                 console_logger.level)\n  end\n\n  data(debug: :debug,\n       info: :info,\n       warn: :warn,\n       error: :error,\n       fatal: :fatal)\n  def test_string_subject(level)\n    @console_logger.send(level, \"subject\")\n    assert_equal([\"#{@timestamp_str} [#{level}]:   0.0s: subject\\n\"],\n                 @logdev.logs)\n  end\n\n  data(debug: :debug,\n       info: :info,\n       warn: :warn,\n       error: :error,\n       fatal: :fatal)\n  def test_args(level)\n    @console_logger.send(level, \"subject\", 1, 2, 3)\n    assert_equal([\n                   \"#{@timestamp_str} [#{level}]:   0.0s: subject\\n\" +\n                   \"      | 1\\n\" +\n                   \"      | 2\\n\" +\n                   \"      | 3\\n\"\n                 ],\n                 @logdev.logs)\n  end\n\n  data(debug: :debug,\n       info: :info,\n       warn: :warn,\n       error: :error,\n       fatal: :fatal)\n  def test_options(level)\n    @console_logger.send(level, \"subject\", kwarg1: \"opt1\", kwarg2: \"opt2\")\n    lines = @logdev.logs[0].split(\"\\n\")\n    args = JSON.parse(lines[1..].collect { |str| str.sub(/\\s+\\|/, \"\") }.join(\"\\n\"))\n    assert_equal([\n                   1,\n                   \"#{@timestamp_str} [#{level}]:   0.0s: subject\",\n                   { \"kwarg1\" => \"opt1\", \"kwarg2\" => \"opt2\" }\n                 ],\n                 [\n                   @logdev.logs.size,\n                   lines[0],\n                   args\n                 ])\n  end\n\n  data(debug: :debug,\n       info: :info,\n       warn: :warn,\n       error: :error,\n       fatal: :fatal)\n  def test_block(level)\n    @console_logger.send(level, \"subject\") { \"block message\" }\n    assert_equal([\n                   \"#{@timestamp_str} [#{level}]:   0.0s: subject\\n\" +\n                   \"      | block message\\n\"\n                 ],\n                 @logdev.logs)\n  end\n\n  data(debug: :debug,\n       info: :info,\n       warn: :warn,\n       error: :error,\n       fatal: :fatal)\n  def test_multiple_entries(level)\n    @console_logger.send(level, \"subject1\")\n    @console_logger.send(level, \"line2\")\n    assert_equal([\n                   \"#{@timestamp_str} [#{level}]:   0.0s: subject1\\n\",\n                   \"#{@timestamp_str} [#{level}]:   0.0s: line2\\n\"\n                 ],\n                 @logdev.logs)\n  end\nend\n"
  },
  {
    "path": "test/plugin/data/2010/01/20100102-030405.log",
    "content": ""
  },
  {
    "path": "test/plugin/data/2010/01/20100102-030406.log",
    "content": ""
  },
  {
    "path": "test/plugin/data/2010/01/20100102.log",
    "content": ""
  },
  {
    "path": "test/plugin/data/log/bar",
    "content": ""
  },
  {
    "path": "test/plugin/data/log/foo/bar.log",
    "content": ""
  },
  {
    "path": "test/plugin/data/log/foo/bar2",
    "content": ""
  },
  {
    "path": "test/plugin/data/log/test.log",
    "content": ""
  },
  {
    "path": "test/plugin/data/log_numeric/01.log",
    "content": ""
  },
  {
    "path": "test/plugin/data/log_numeric/02.log",
    "content": ""
  },
  {
    "path": "test/plugin/data/log_numeric/12.log",
    "content": ""
  },
  {
    "path": "test/plugin/data/log_numeric/14.log",
    "content": ""
  },
  {
    "path": "test/plugin/data/sd_file/config",
    "content": "- 'host': 127.0.0.1\n  'port': 24224\n  'weight': 1\n  'name': test1\n  'standby': false\n  'username': user1\n  'password': pass1\n  'shared_key': key1\n- 'host': 127.0.0.1\n  'port': 24225\n  'weight': 1\n"
  },
  {
    "path": "test/plugin/data/sd_file/config.json",
    "content": "[\n    {\n        \"host\": \"127.0.0.1\",\n        \"port\": 24224,\n        \"weight\": 1,\n        \"name\": \"test1\",\n        \"standby\": false,\n        \"username\": \"user1\",\n        \"password\": \"pass1\",\n        \"shared_key\": \"key1\"\n    },\n    {\n        \"host\": \"127.0.0.1\",\n        \"port\": 24225,\n        \"weight\": 1\n    }\n]\n"
  },
  {
    "path": "test/plugin/data/sd_file/config.yaml",
    "content": "- 'host': 127.0.0.1\n  'port': 24224\n  'weight': 1\n  'name': test1\n  'standby': false\n  'username': user1\n  'password': pass1\n  'shared_key': key1\n- 'host': 127.0.0.1\n  'port': 24225\n  'weight': 1\n"
  },
  {
    "path": "test/plugin/data/sd_file/config.yml",
    "content": "- 'host': 127.0.0.1\n  'port': 24224\n  'weight': 1\n  'name': test1\n  'standby': false\n  'username': user1\n  'password': pass1\n  'shared_key': key1\n- 'host': 127.0.0.1\n  'port': 24225\n  'weight': 1\n"
  },
  {
    "path": "test/plugin/data/sd_file/invalid_config.yml",
    "content": "- 'host': 127.0.0.1\n  'weight': 1\n  'name': test1\n  'standby': false\n  'username': user1\n  'password': pass1\n  'shared_key': key1\n"
  },
  {
    "path": "test/plugin/in_tail/test_fifo.rb",
    "content": "require_relative '../../helper'\n\nrequire 'fluent/plugin/in_tail'\n\nclass IntailFIFO < Test::Unit::TestCase\n  sub_test_case '#read_line' do\n    test 'returns lines splitting per `\\n`' do\n      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log)\n      text = (\"test\\n\" * 3).force_encoding(Encoding::ASCII_8BIT)\n      fifo << text\n      lines = []\n      fifo.read_lines(lines)\n      assert_equal Encoding::ASCII_8BIT, lines[0].encoding\n      assert_equal [\"test\\n\", \"test\\n\", \"test\\n\"], lines\n    end\n\n    test 'concat line when line is separated' do\n      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log)\n      text = (\"test\\n\" * 3 + 'test').force_encoding(Encoding::ASCII_8BIT)\n      fifo << text\n      lines = []\n      fifo.read_lines(lines)\n      assert_equal Encoding::ASCII_8BIT, lines[0].encoding\n      assert_equal [\"test\\n\", \"test\\n\", \"test\\n\"], lines\n\n      fifo << \"2\\n\"\n      fifo.read_lines(lines)\n      assert_equal Encoding::ASCII_8BIT, lines[0].encoding\n      assert_equal [\"test\\n\", \"test\\n\", \"test\\n\", \"test2\\n\"], lines\n    end\n\n    test 'returns lines which convert encoding' do\n      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log, nil, Encoding::UTF_8)\n      text = (\"test\\n\" * 3).force_encoding(Encoding::ASCII_8BIT)\n      fifo << text\n      lines = []\n      fifo.read_lines(lines)\n      assert_equal Encoding::UTF_8, lines[0].encoding\n      assert_equal [\"test\\n\", \"test\\n\", \"test\\n\"], lines\n    end\n\n    test 'reads lines as from_encoding' do\n      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::UTF_8, $log, nil, Encoding::ASCII_8BIT)\n      text = (\"test\\n\" * 3).force_encoding(Encoding::UTF_8)\n      fifo << text\n      lines = []\n      fifo.read_lines(lines)\n      assert_equal Encoding::ASCII_8BIT, lines[0].encoding\n      assert_equal [\"test\\n\", \"test\\n\", \"test\\n\"], lines\n    end\n\n    sub_test_case 'when it includes multi byte chars' do\n      test 'handles it as ascii_8bit' do\n        fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log)\n        text = (\"てすと\\n\" * 3).force_encoding(Encoding::ASCII_8BIT)\n        fifo << text\n        lines = []\n        fifo.read_lines(lines)\n        assert_equal Encoding::ASCII_8BIT, lines[0].encoding\n        assert_equal [\"てすと\\n\", \"てすと\\n\", \"てすと\\n\"].map { |e| e.force_encoding(Encoding::ASCII_8BIT) }, lines\n      end\n\n      test 'replaces character with ? when convert error happens' do\n        fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::UTF_8, $log, nil, Encoding::ASCII_8BIT)\n        text = (\"てすと\\n\" * 3).force_encoding(Encoding::UTF_8)\n        fifo << text\n        lines = []\n        fifo.read_lines(lines)\n        assert_equal Encoding::ASCII_8BIT, lines[0].encoding\n        assert_equal [\"???\\n\", \"???\\n\", \"???\\n\"].map { |e| e.force_encoding(Encoding::ASCII_8BIT) }, lines\n      end\n    end\n\n    test 'returns nothing when buffer is empty' do\n      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log)\n      lines = []\n      fifo.read_lines(lines)\n      assert_equal [], lines\n\n      text = \"test\\n\" * 3\n      fifo << text\n      fifo.read_lines(lines)\n      assert_equal [\"test\\n\", \"test\\n\", \"test\\n\"], lines\n\n      lines = []\n      fifo.read_lines(lines)\n      assert_equal [], lines\n    end\n\n    data('bigger than max_line_size', [\n      [\"test test test\\n\" * 3],\n      [],\n    ])\n    data('less than or equal to max_line_size', [\n      [\"test\\n\" * 2],\n      [\"test\\n\", \"test\\n\"],\n    ])\n    data('mix', [\n      [\"test test test\\ntest\\ntest test test\\ntest\\ntest test test\\n\"],\n      [\"test\\n\", \"test\\n\"],\n    ])\n    data('mix and multiple', [\n      [\n        \"test test test\\ntest\\n\",\n        \"test\",\n        \" test test\\nt\",\n        \"est\\nt\"\n      ],\n      [\"test\\n\", \"test\\n\"],\n    ])\n    data('remaining data bigger than max_line_size should be discarded', [\n      [\n        \"test\\nlong line still not having EOL\",\n        \"following texts to the previous long line\\ntest\\n\",\n      ],\n      [\"test\\n\", \"test\\n\"],\n    ])\n    test 'return lines only that size is less than or equal to max_line_size' do |(input_texts, expected)|\n      max_line_size = 5\n      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log, max_line_size)\n      lines = []\n\n      input_texts.each do |text|\n        fifo << text.force_encoding(Encoding::ASCII_8BIT)\n        fifo.read_lines(lines)\n        # The size of remaining buffer (i.e. a line still not having EOL) must not exceed max_line_size.\n        assert { fifo.buffer.bytesize <= max_line_size }\n      end\n\n      assert_equal expected, lines\n    end\n  end\n\n  sub_test_case '#<<' do\n    test 'does not make any change about encoding to an argument' do\n      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log)\n      text = (\"test\\n\" * 3).force_encoding(Encoding::UTF_8)\n\n      assert_equal Encoding::UTF_8, text.encoding\n      fifo << text\n      assert_equal Encoding::UTF_8, text.encoding\n    end\n  end\n\n  sub_test_case '#reading_bytesize' do\n    test 'returns buffer size' do\n      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log)\n      text = \"test\\n\" * 3 + 'test'\n      fifo << text\n\n      assert_equal text.bytesize, fifo.reading_bytesize\n      lines = []\n      fifo.read_lines(lines)\n      assert_equal [\"test\\n\", \"test\\n\", \"test\\n\"], lines\n\n      assert_equal 'test'.bytesize, fifo.reading_bytesize\n      fifo << \"2\\n\"\n      fifo.read_lines(lines)\n      assert_equal [\"test\\n\", \"test\\n\", \"test\\n\", \"test2\\n\"], lines\n\n      assert_equal 0, fifo.reading_bytesize\n    end\n\n    test 'returns the entire line size even if the size is over max_line_size' do\n      max_line_size = 20\n      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log, max_line_size)\n      lines = []\n\n      text = \"long line still not having EOL\"\n      fifo << text\n      fifo.read_lines(lines)\n      assert_equal [], lines\n      assert_equal 0, fifo.buffer.bytesize\n      assert_equal text.bytesize, fifo.reading_bytesize\n\n      text2 = \" following texts\"\n      fifo << text2\n      fifo.read_lines(lines)\n      assert_equal [], lines\n      assert_equal 0, fifo.buffer.bytesize\n      assert_equal text.bytesize + text2.bytesize, fifo.reading_bytesize\n\n      text3 = \" end of the line\\n\"\n      fifo << text3\n      fifo.read_lines(lines)\n      assert_equal [], lines\n      assert_equal 0, fifo.buffer.bytesize\n      assert_equal 0, fifo.reading_bytesize\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/in_tail/test_io_handler.rb",
    "content": "require_relative '../../helper'\n\nrequire 'fluent/plugin/in_tail'\nrequire 'fluent/plugin/metrics_local'\nrequire 'tempfile'\n\nclass IntailIOHandlerTest < Test::Unit::TestCase\n  def setup\n    Tempfile.create('intail_io_handler') do |file|\n      file.binmode\n      @file = file\n      opened_file_metrics = Fluent::Plugin::LocalMetrics.new\n      opened_file_metrics.configure(config_element('metrics', '', {}))\n      closed_file_metrics = Fluent::Plugin::LocalMetrics.new\n      closed_file_metrics.configure(config_element('metrics', '', {}))\n      rotated_file_metrics = Fluent::Plugin::LocalMetrics.new\n      rotated_file_metrics.configure(config_element('metrics', '', {}))\n      throttling_metrics = Fluent::Plugin::LocalMetrics.new\n      throttling_metrics.configure(config_element('metrics', '', {}))\n      @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics, throttling_metrics)\n      yield\n    end\n  end\n\n  def create_target_info\n    Fluent::Plugin::TailInput::TargetInfo.new(@file.path, Fluent::FileWrapper.stat(@file.path).ino)\n  end\n\n  def create_watcher\n    Fluent::Plugin::TailInput::TailWatcher.new(create_target_info, nil, nil, nil, nil, nil, nil, nil, nil)\n  end\n\n  test '#on_notify load file content and passed it to receive_lines method' do\n    text = \"this line is test\\ntest line is test\\n\"\n    @file.write(text)\n    @file.close\n\n    watcher = create_watcher\n\n    update_pos = 0\n\n    stub(watcher).pe do\n      pe = 'position_file'\n      stub(pe).read_pos { 0 }\n      stub(pe).update_pos { |val| update_pos = val }\n      pe\n    end\n\n    returned_lines = ''\n    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|\n      returned_lines << lines.join\n      true\n    end\n\n    r.on_notify\n    assert_equal text.bytesize, update_pos\n    assert_equal text, returned_lines\n\n    r.on_notify\n\n    assert_equal text.bytesize, update_pos\n    assert_equal text, returned_lines\n  end\n\n  sub_test_case 'when open_on_every_update is true and read_pos returns always 0' do\n    test 'open new IO and change pos to 0 and read it' do\n      text = \"this line is test\\ntest line is test\\n\"\n      @file.write(text)\n      @file.close\n\n      update_pos = 0\n\n      watcher = create_watcher\n      stub(watcher).pe do\n        pe = 'position_file'\n        stub(pe).read_pos { 0 }\n        stub(pe).update_pos { |val| update_pos = val }\n        pe\n      end\n\n      returned_lines = ''\n      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true, metrics: @metrics) do |lines, _watcher|\n        returned_lines << lines.join\n        true\n      end\n\n      r.on_notify\n      assert_equal text.bytesize, update_pos\n      assert_equal text, returned_lines\n\n      r.on_notify\n      assert_equal text * 2, returned_lines\n    end\n  end\n\n  sub_test_case 'when limit is 5' do\n    test 'call receive_lines once when short line(less than 65536)' do\n      text = \"line\\n\" * 8\n      @file.write(text)\n      @file.close\n\n      update_pos = 0\n\n      watcher = create_watcher\n      stub(watcher).pe do\n        pe = 'position_file'\n        stub(pe).read_pos { 0 }\n        stub(pe).update_pos { |val| update_pos = val }\n        pe\n      end\n\n      returned_lines = []\n      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|\n        returned_lines << lines.dup\n        true\n      end\n\n      r.on_notify\n      assert_equal text.bytesize, update_pos\n      assert_equal 8, returned_lines[0].size\n    end\n\n    test 'call receive_lines some times when long line(more than 65536)' do\n      t = 'line' * (65536 / 8)\n      text = \"#{t}\\n\" * 8\n      @file.write(text)\n      @file.close\n\n      update_pos = 0\n\n      watcher = create_watcher\n      stub(watcher).pe do\n        pe = 'position_file'\n        stub(pe).read_pos { 0 }\n        stub(pe).update_pos { |val| update_pos = val }\n        pe\n      end\n\n      returned_lines = []\n      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|\n        returned_lines << lines.dup\n        true\n      end\n\n      r.on_notify\n      assert_equal text.bytesize, update_pos\n      assert_equal 5, returned_lines[0].size\n      assert_equal 3, returned_lines[1].size\n    end\n  end\n\n  sub_test_case 'max_line_size' do\n    test 'does not call receive_lines when line_size exceeds max_line_size' do\n      t = 'x' * (8192)\n      text = \"#{t}\\n\"\n\n      max_line_size = 8192\n\n      @file.write(text)\n      @file.close\n\n      update_pos = 0\n\n      watcher = create_watcher\n      stub(watcher).pe do\n        pe = 'position_file'\n        stub(pe).read_pos {0}\n        stub(pe).update_pos { |val| update_pos = val }\n        pe\n      end\n\n      returned_lines = []\n      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 1000, read_bytes_limit_per_second: -1, max_line_size: max_line_size, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|\n        returned_lines << lines.dup\n        true\n      end\n\n      r.on_notify\n      assert_equal text.bytesize, update_pos\n      assert_equal 0, returned_lines.size\n    end\n\n    data(\n      \"open_on_every_update false\" => false,\n      \"open_on_every_update true\" => true,\n    )\n    test 'manage pos correctly if a long line not having EOL occurs' do |open_on_every_update|\n      max_line_size = 20\n      returned_lines = []\n      pos = 0\n\n      watcher = create_watcher\n      stub(watcher).pe do\n        pe = 'position_file'\n        stub(pe).read_pos { pos }\n        stub(pe).update_pos { |val| pos = val }\n        pe\n      end\n\n      io_handler = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(\n        watcher, path: @file.path, read_lines_limit: 1000, read_bytes_limit_per_second: -1,\n        max_line_size: max_line_size, log: $log, open_on_every_update: open_on_every_update,\n        metrics: @metrics\n      ) do |lines, _watcher|\n        returned_lines << lines.dup\n        true\n      end\n\n      short_line = \"short line\\n\"\n      long_lines = [\n        \"long line still not having EOL\",\n        \" end of the line\\n\",\n      ]\n\n      @file.write(short_line)\n      @file.write(long_lines[0])\n      @file.flush\n      io_handler.on_notify\n\n      assert_equal [[short_line]], returned_lines\n      assert_equal short_line.bytesize, pos\n\n      @file.write(long_lines[1])\n      @file.flush\n      io_handler.on_notify\n\n      assert_equal [[short_line]], returned_lines\n      expected_size = short_line.bytesize + long_lines[0..1].map{|l| l.bytesize}.sum\n      assert_equal expected_size, pos\n\n      io_handler.close\n    end\n\n    data(\n      \"open_on_every_update false\" => false,\n      \"open_on_every_update true\" => true,\n    )\n    test 'discards a subsequent data in a long line even if restarting occurs between' do |open_on_every_update|\n      max_line_size = 20\n      returned_lines = []\n      pos = 0\n\n      watcher = create_watcher\n      stub(watcher).pe do\n        pe = 'position_file'\n        stub(pe).read_pos { pos }\n        stub(pe).update_pos { |val| pos = val }\n        pe\n      end\n\n      io_handler = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(\n        watcher, path: @file.path, read_lines_limit: 1000, read_bytes_limit_per_second: -1,\n        max_line_size: max_line_size, log: $log, open_on_every_update: open_on_every_update,\n        metrics: @metrics\n      ) do |lines, _watcher|\n        returned_lines << lines.dup\n        true\n      end\n\n      short_line = \"short line\\n\"\n      long_lines = [\n        \"long line still not having EOL\",\n        \" end of the line\\n\",\n      ]\n\n      @file.write(short_line)\n      @file.write(long_lines[0])\n      @file.flush\n      io_handler.on_notify\n\n      assert_equal [[short_line]], returned_lines\n\n      io_handler.close\n      io_handler = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(\n        watcher, path: @file.path, read_lines_limit: 1000, read_bytes_limit_per_second: -1,\n        max_line_size: max_line_size, log: $log, open_on_every_update: open_on_every_update,\n        metrics: @metrics\n      ) do |lines, _watcher|\n        returned_lines << lines.dup\n        true\n      end\n\n      @file.write(long_lines[1])\n      @file.flush\n      io_handler.on_notify\n\n      assert_equal [[short_line]], returned_lines\n\n      io_handler.close\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/in_tail/test_position_file.rb",
    "content": "require_relative '../../helper'\nrequire 'fluent/plugin/in_tail/position_file'\nrequire 'fluent/plugin/in_tail'\n\nrequire 'fileutils'\nrequire 'tempfile'\n\nclass IntailPositionFileTest < Test::Unit::TestCase\n  def setup\n    Tempfile.create('intail_position_file_test') do |file|\n      file.binmode\n      @file = file\n      yield\n    end\n  end\n\n  UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION\n  TEST_CONTENT = <<~EOF\n    valid_path\\t0000000000000002\\t0000000000000001\n    inode23bit\\t0000000000000000\\t00000000\n    invalidpath100000000000000000000000000000000\n    unwatched\\t#{UNWATCHED_STR}\\t0000000000000000\n  EOF\n  TEST_CONTENT_PATHS = {\n    \"valid_path\" => Fluent::Plugin::TailInput::TargetInfo.new(\"valid_path\", 1),\n    \"inode23bit\" => Fluent::Plugin::TailInput::TargetInfo.new(\"inode23bit\", 0),\n  }\n  TEST_CONTENT_INODES = {\n    1 => Fluent::Plugin::TailInput::TargetInfo.new(\"valid_path\", 1),\n    0 => Fluent::Plugin::TailInput::TargetInfo.new(\"inode23bit\", 0),\n  }\n\n  def write_data(f, content)\n    f.write(content)\n    f.seek(0)\n  end\n\n  def follow_inodes_block\n    [true, false].each do |follow_inodes|\n      yield follow_inodes\n    end\n  end\n\n  test '.load' do\n    write_data(@file, TEST_CONTENT)\n    Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})\n\n    @file.seek(0)\n    lines = @file.readlines\n    assert_equal 2, lines.size\n    assert_equal \"valid_path\\t0000000000000002\\t0000000000000001\\n\", lines[0]\n    assert_equal \"inode23bit\\t0000000000000000\\t0000000000000000\\n\", lines[1]\n  end\n\n  sub_test_case '#try_compact' do\n    test 'compact invalid and convert 32 bit inode value' do\n      write_data(@file, TEST_CONTENT)\n      Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal 2, lines.size\n      assert_equal \"valid_path\\t0000000000000002\\t0000000000000001\\n\", lines[0]\n      assert_equal \"inode23bit\\t0000000000000000\\t0000000000000000\\n\", lines[1]\n    end\n\n    test 'compact data if duplicated line' do\n      write_data(@file, <<~EOF)\n        valid_path\\t0000000000000002\\t0000000000000001\n        valid_path\\t0000000000000003\\t0000000000000004\n      EOF\n      Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal \"valid_path\\t0000000000000003\\t0000000000000004\\n\", lines[0]\n    end\n\n    test 'does not change when the file is changed' do\n      write_data(@file, TEST_CONTENT)\n      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log})\n\n      mock.proxy(pf).fetch_compacted_entries do |r|\n        @file.write(\"unwatched\\t#{UNWATCHED_STR}\\t0000000000000000\\n\")\n        r\n      end\n\n      pf.try_compact\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal 5, lines.size\n    end\n\n    test 'update seek position of remained position entry' do\n      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log})\n      target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)\n      target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)\n      target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1)\n      pf[target_info1]\n      pf[target_info2]\n      pf[target_info3]\n\n      target_info1_2 = Fluent::Plugin::TailInput::TargetInfo.new('path1', 1234)\n      pf.unwatch(target_info1_2)\n\n      pf.try_compact\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal \"path2\\t0000000000000000\\t0000000000000000\\n\", lines[0]\n      assert_equal \"path3\\t0000000000000000\\t0000000000000000\\n\", lines[1]\n      assert_equal 2, lines.size\n\n      target_info2_2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', 1235)\n      target_info3_2 = Fluent::Plugin::TailInput::TargetInfo.new('path3', 1236)\n      pf.unwatch(target_info2_2)\n      pf.unwatch(target_info3_2)\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal \"path2\\t#{UNWATCHED_STR}\\t0000000000000000\\n\", lines[0]\n      assert_equal \"path3\\t#{UNWATCHED_STR}\\t0000000000000000\\n\", lines[1]\n      assert_equal 2, lines.size\n    end\n\n    test 'should ignore initial existing files on follow_inode' do\n      write_data(@file, TEST_CONTENT)\n      pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_PATHS, **{logger: $log})\n      @file.seek(0)\n      assert_equal([], @file.readlines)\n\n      @file.seek(0)\n      write_data(@file, TEST_CONTENT)\n      pos_file.try_compact\n\n      @file.seek(0)\n      assert_equal([\n                     \"valid_path\\t0000000000000002\\t0000000000000001\\n\",\n                     \"inode23bit\\t0000000000000000\\t0000000000000000\\n\",\n                   ],\n                   @file.readlines)\n    end\n  end\n\n  sub_test_case '#load' do\n    test 'compact invalid and convert 32 bit inode value' do\n      write_data(@file, TEST_CONTENT)\n      Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal 2, lines.size\n      assert_equal \"valid_path\\t0000000000000002\\t0000000000000001\\n\", lines[0]\n      assert_equal \"inode23bit\\t0000000000000000\\t0000000000000000\\n\", lines[1]\n    end\n\n    test 'compact deleted paths' do\n      write_data(@file, TEST_CONTENT)\n      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal [], lines\n    end\n\n    test 'compact data if duplicated line' do\n      write_data(@file, <<~EOF)\n        valid_path\\t0000000000000002\\t0000000000000001\n        valid_path\\t0000000000000003\\t0000000000000004\n      EOF\n      Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).load\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal \"valid_path\\t0000000000000003\\t0000000000000004\\n\", lines[0]\n    end\n  end\n\n  sub_test_case '#[]' do\n    test 'return entry' do\n      write_data(@file, TEST_CONTENT)\n      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})\n\n      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)\n      f = pf[valid_target_info]\n      assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class\n      assert_equal 2, f.read_pos\n      assert_equal 1, f.read_inode\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal 2, lines.size\n\n      nonexistent_target_info = Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)\n      f = pf[nonexistent_target_info]\n      assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class\n      assert_equal 0, f.read_pos\n      assert_equal 0, f.read_inode\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal 3, lines.size\n      assert_equal \"nonexist_path\\t0000000000000000\\t0000000000000000\\n\", lines[2]\n    end\n\n    test 'does not change other value position if other entry try to write' do\n      write_data(@file, TEST_CONTENT)\n      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)\n\n      f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]\n      assert_equal 0, f.read_inode\n      assert_equal 0, f.read_pos\n\n      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)].update(1, 2)\n\n      f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]\n      assert_equal 0, f.read_inode\n      assert_equal 0, f.read_pos\n\n      pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)].update(1, 2)\n      assert_equal 1, f.read_inode\n      assert_equal 2, f.read_pos\n    end\n  end\n\n  sub_test_case '#unwatch' do\n    test 'unwatch entry by path' do\n      write_data(@file, TEST_CONTENT)\n      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)\n      inode1 = File.stat(@file).ino\n      target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)\n      p1 = pf[target_info1]\n      assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class\n\n      pf.unwatch(target_info1)\n      assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION\n\n      inode2 = File.stat(@file).ino\n      target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)\n      p2 = pf[target_info2]\n      assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class\n\n      assert_not_equal p1, p2\n    end\n\n    test 'unwatch entries by inode' do\n      write_data(@file, TEST_CONTENT)\n      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_INODES, logger: $log)\n\n      existing_targets = TEST_CONTENT_INODES.select do |inode, target_info|\n        inode == 1\n      end\n      pe_to_unwatch = pf[TEST_CONTENT_INODES[0]]\n\n      pf.unwatch_removed_targets(existing_targets)\n\n      assert_equal(\n        {\n          map_keys: [TEST_CONTENT_INODES[1].ino],\n          unwatched_pe_pos: Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION,\n        },\n        {\n          map_keys: pf.instance_variable_get(:@map).keys,\n          unwatched_pe_pos: pe_to_unwatch.read_pos,\n        }\n      )\n\n      unwatched_pe_retaken = pf[TEST_CONTENT_INODES[0]]\n      assert_not_equal pe_to_unwatch, unwatched_pe_retaken\n    end\n  end\n\n  sub_test_case 'FilePositionEntry' do\n    FILE_POS_CONTENT = <<~EOF\n      valid_path\\t0000000000000002\\t0000000000000001\n      valid_path2\\t0000000000000003\\t0000000000000002\n    EOF\n\n    def build_files(file)\n      r = {}\n\n      file.each_line do |line|\n        m = /^([^\\t]+)\\t([0-9a-fA-F]+)\\t([0-9a-fA-F]+)/.match(line)\n        path = m[1]\n        pos = m[2].to_i(16)\n        ino = m[3].to_i(16)\n        seek = file.pos - line.bytesize + path.bytesize + 1\n        r[path] = Fluent::Plugin::TailInput::FilePositionEntry.new(@file, Mutex.new, seek, pos, ino)\n      end\n\n      r\n    end\n\n    test '#update' do\n      write_data(@file, FILE_POS_CONTENT)\n      fs = build_files(@file)\n      f = fs['valid_path']\n      f.update(11, 10)\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal 2, lines.size\n      assert_equal \"valid_path\\t000000000000000a\\t000000000000000b\\n\", lines[0]\n      assert_equal \"valid_path2\\t0000000000000003\\t0000000000000002\\n\", lines[1]\n    end\n\n    test '#update_pos' do\n      write_data(@file, FILE_POS_CONTENT)\n      fs = build_files(@file)\n      f = fs['valid_path']\n      f.update_pos(10)\n\n      @file.seek(0)\n      lines = @file.readlines\n      assert_equal 2, lines.size\n      assert_equal \"valid_path\\t000000000000000a\\t0000000000000001\\n\", lines[0]\n      assert_equal \"valid_path2\\t0000000000000003\\t0000000000000002\\n\", lines[1]\n    end\n\n    test '#read_pos' do\n      write_data(@file, FILE_POS_CONTENT)\n      fs = build_files(@file)\n      f = fs['valid_path']\n      assert_equal 2, f.read_pos\n\n      f.update_pos(10)\n      assert_equal 10, f.read_pos\n\n      f.update(2, 11)\n      assert_equal 11, f.read_pos\n    end\n\n    test '#read_inode' do\n      write_data(@file, FILE_POS_CONTENT)\n      fs = build_files(@file)\n      f = fs['valid_path']\n      assert_equal 1, f.read_inode\n      f.update_pos(10)\n      assert_equal 1, f.read_inode\n\n      f.update(2, 11)\n      assert_equal 2, f.read_inode\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/out_forward/test_ack_handler.rb",
    "content": "require_relative '../../helper'\nrequire 'fluent/test/driver/output'\nrequire 'flexmock/test_unit'\n\nrequire 'fluent/plugin/out_forward'\nrequire 'fluent/plugin/out_forward/ack_handler'\n\nclass AckHandlerTest < Test::Unit::TestCase\n  data(\n    'chunk_id is matched' => [MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') }), Fluent::Plugin::ForwardOutput::AckHandler::Result::SUCCESS],\n    'chunk_id is not matched' => [MessagePack.pack({ 'ack' => 'unmatched' }), Fluent::Plugin::ForwardOutput::AckHandler::Result::CHUNKID_UNMATCHED],\n    'chunk_id is empty' => ['', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED],\n  )\n  test 'returns chunk_id, node, sock and result status' do |args|\n    receved, state = args\n    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)\n\n    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log\n    chunk_id = 'chunk_id 111'\n    ack = ack_handler.create_ack(chunk_id, node)\n\n    r, w = IO.pipe\n    begin\n      w.write(chunk_id)\n      mock(r).recv(anything) { |_| receved } # IO does not have recv\n      ack.enqueue(r)\n\n      a1 = a2 = a3 = a4 = nil\n      ack_handler.collect_response(1) do |cid, n, s, ret|\n        # This block is rescued by ack_handler so it needs to invoke assertion outside of this block\n        a1 = cid; a2 = n; a3 = s; a4 = ret\n      end\n\n      assert_equal chunk_id, a1\n      assert_equal node, a2\n      assert_equal r, a3\n      assert_equal state, a4\n    ensure\n      r.close rescue nil\n      w.close rescue nil\n    end\n  end\n\n  test 'returns nil if raise an error' do\n    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)\n\n    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log\n    chunk_id = 'chunk_id 111'\n    ack = ack_handler.create_ack(chunk_id, node)\n\n    r, w = IO.pipe\n    begin\n      w.write(chunk_id)\n      mock(r).recv(anything) { |_| raise 'unexpected error' } # IO does not have recv\n      ack.enqueue(r)\n\n      a1 = a2 = a3 = a4 = nil\n      ack_handler.collect_response(1) do |cid, n, s, ret|\n        # This block is rescued by ack_handler so it needs to invoke assertion outside of this block\n        a1 = cid; a2 = n; a3 = s; a4 = ret\n      end\n\n      assert_nil a1\n      assert_nil a2\n      assert_nil a3\n      assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, a4\n    ensure\n      r.close rescue nil\n      w.close rescue nil\n    end\n  end\n\n  test 'when ack is expired' do\n    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 0, log: $log, read_length: 100)\n\n    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log\n    chunk_id = 'chunk_id 111'\n    ack = ack_handler.create_ack(chunk_id, node)\n\n    r, w = IO.pipe\n    begin\n      w.write(chunk_id)\n      mock(r).recv(anything).never\n      ack.enqueue(r)\n\n      a1 = a2 = a3 = a4 = nil\n      ack_handler.collect_response(1) do |cid, n, s, ret|\n        # This block is rescued by ack_handler so it needs to invoke assertion outside of this block\n        a1 = cid; a2 = n; a3 = s; a4 = ret\n      end\n\n      assert_equal chunk_id, a1\n      assert_equal node, a2\n      assert_equal r, a3\n      assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, a4\n    ensure\n      r.close rescue nil\n      w.close rescue nil\n    end\n  end\n\n  # ForwardOutput uses AckHandler in multiple threads, so we need to assume this case.\n  # If exclusive control for this case is implemented, this test may not be necessary.\n  test 'raises no error when another thread closes a socket' do\n    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)\n\n    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log\n    chunk_id = 'chunk_id 111'\n    ack = ack_handler.create_ack(chunk_id, node)\n\n    r, w = IO.pipe\n    begin\n      w.write(chunk_id)\n      def r.recv(arg)\n        sleep(1) # To ensure that multiple threads select the socket before closing.\n        raise IOError, 'stream closed in another thread' if self.closed?\n        MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') })\n      end\n      ack.enqueue(r)\n\n      threads = []\n      2.times do\n        threads << Thread.new do\n          ack_handler.collect_response(1) do |cid, n, s, ret|\n            s&.close\n          end\n        end\n      end\n\n      assert_true threads.map{ |t| t.join(10) }.all?\n      assert_false(\n        $log.out.logs.any? { |log| log.include?('[error]') },\n        $log.out.logs.select{ |log| log.include?('[error]') }.join('\\n')\n      )\n    ensure\n      r.close rescue nil\n      w.close rescue nil\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/out_forward/test_connection_manager.rb",
    "content": "require_relative '../../helper'\nrequire 'fluent/test/driver/output'\nrequire 'flexmock/test_unit'\n\nrequire 'fluent/plugin/out_forward'\nrequire 'fluent/plugin/out_forward/connection_manager'\nrequire 'fluent/plugin/out_forward/socket_cache'\n\nclass ConnectionManager < Test::Unit::TestCase\n  sub_test_case '#connect' do\n    sub_test_case 'when socket_cache is nil' do\n      test 'creates socket and does not close when block is not given' do\n        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(\n          log: $log,\n          secure: false,\n          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },\n          socket_cache: nil,\n        )\n\n        mock.proxy(cm).connect_keepalive(anything).never\n        sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)\n        assert_equal(sock, 'sock')\n        assert_equal(ri.state, :established)\n      end\n\n      test 'creates socket and calls close when block is given' do\n        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(\n          log: $log,\n          secure: false,\n          connection_factory: -> (_, _, _) {\n            sock = 'sock'\n            mock(sock).close.once\n            mock(sock).close_write.once\n            sock\n          },\n          socket_cache: nil,\n        )\n\n        mock.proxy(cm).connect_keepalive(anything).never\n        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) do |sock, ri|\n          assert_equal(sock, 'sock')\n          assert_equal(ri.state, :established)\n        end\n      end\n\n      test 'when secure is true, state is helo' do\n        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(\n          log: $log,\n          secure: true,\n          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },\n          socket_cache: nil,\n        )\n\n        mock.proxy(cm).connect_keepalive(anything).never\n        sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)\n        assert_equal(sock, 'sock')\n        assert_equal(ri.state, :helo)\n      end\n\n      test 'when passed ack' do\n        sock = 'sock'\n        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(\n          log: $log,\n          secure: false,\n          connection_factory: -> (_, _, _) {\n            mock(sock).close.never\n            mock(sock).close_write.never\n            sock\n          },\n          socket_cache: nil,\n        )\n\n        mock.proxy(cm).connect_keepalive(anything).never\n        ack = mock('ack').enqueue(sock).once.subject\n        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: ack) do |sock, ri|\n          assert_equal(sock, 'sock')\n          assert_equal(ri.state, :established)\n        end\n      end\n    end\n\n    sub_test_case 'when socket_cache exists' do\n      test 'calls connect_keepalive' do\n        cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n        mock(cache).checkin('sock').never\n\n        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(\n          log: $log,\n          secure: false,\n          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },\n          socket_cache: cache,\n        )\n\n        mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: nil).once\n\n        sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)\n        assert_equal(sock, 'sock')\n        assert_equal(ri.state, :established)\n      end\n\n      test 'calls connect_keepalive and closes socket with block' do\n        cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n        mock(cache).checkin('sock').once\n\n        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(\n          log: $log,\n          secure: false,\n          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock); sock },\n          socket_cache: cache,\n        )\n\n        mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: nil).once\n\n        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) do |sock, ri|\n          assert_equal(sock, 'sock')\n          assert_equal(ri.state, :established)\n        end\n      end\n\n      test 'does not call dec_ref when ack is passed' do\n        cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n        mock(cache).checkin('sock').never\n        sock = 'sock'\n        ack = stub('ack').enqueue(sock).once.subject\n\n        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(\n          log: $log,\n          secure: false,\n          connection_factory: -> (_, _, _) {\n            mock(sock).close.never\n            mock(sock).close_write.never\n            sock\n          },\n          socket_cache: cache,\n        )\n\n        mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: ack).once\n        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: ack) do |sock, ri|\n          assert_equal(sock, 'sock')\n          assert_equal(ri.state, :established)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/out_forward/test_handshake_protocol.rb",
    "content": "require_relative '../../helper'\nrequire 'flexmock/test_unit'\n\nrequire 'fluent/plugin/out_forward'\nrequire 'fluent/plugin/out_forward/handshake_protocol'\nrequire 'fluent/plugin/out_forward/connection_manager'\n\nclass HandshakeProtocolTest < Test::Unit::TestCase\n  sub_test_case '#invok when helo state' do\n    test 'sends PING message and change state to pingpong' do\n      hostname = 'hostname'\n      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: hostname, shared_key: 'shared_key', password: nil, username: nil)\n      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)\n\n      sock = StringIO.new('')\n      handshake.invoke(sock, ri, ['HELO', {}])\n\n      assert_equal(ri.state, :pingpong)\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(sock.string) do |ping|\n        assert_equal(ping.size, 6)\n        assert_equal(ping[0], 'PING')\n        assert_equal(ping[1], hostname)\n        assert(ping[2].is_a?(String)) # content is hashed value\n        assert(ping[3].is_a?(String)) # content is hashed value\n        assert_equal(ping[4], '')\n        assert_equal(ping[5], '')\n      end\n    end\n\n    test 'returns PING message with username if auth exists' do\n      hostname = 'hostname'\n      username = 'username'\n      pass = 'pass'\n      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: hostname, shared_key: 'shared_key', password: pass, username: username)\n      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)\n\n      sock = StringIO.new('')\n      handshake.invoke(sock, ri, ['HELO', { 'auth' => 'auth' }])\n\n      assert_equal(ri.state, :pingpong)\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(sock.string) do |ping|\n        assert_equal(ping.size, 6)\n        assert_equal(ping[0], 'PING')\n        assert_equal(ping[1], hostname)\n        assert(ping[2].is_a?(String)) # content is hashed value\n        assert(ping[3].is_a?(String)) # content is hashed value\n        assert_equal(ping[4], username)\n        assert_not_equal(ping[5], pass) # should be hashed\n      end\n    end\n\n    data(\n      lack_of_elem: ['HELO'],\n      wrong_message: ['HELLO!', {}],\n    )\n    test 'raises an error when message is' do |msg|\n      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)\n      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)\n\n      sock = StringIO.new('')\n      assert_raise(Fluent::Plugin::ForwardOutput::HeloError) do\n        handshake.invoke(sock, ri, msg)\n      end\n\n      assert_equal(ri.state, :helo)\n    end\n  end\n\n  sub_test_case '#invok when pingpong state' do\n    test 'sends PING message and change state to pingpong' do\n      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)\n      handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt\n\n      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '')\n      handshake.invoke(\n        '',\n        ri,\n        # 40a3.... = Digest::SHA512.new.update('ce1897b0d3dbd76b90d7fb96010dcac3').update('client_hostname').update('').update('shared_key').hexdigest\n        ['PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f']\n      )\n      assert_equal(ri.state, :established)\n    end\n\n    test 'raises an error when password and username are nil if auth exists' do\n      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)\n      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)\n\n      assert_raise(Fluent::Plugin::ForwardOutput::PingpongError.new('username and password are required')) do\n        handshake.invoke('', ri, ['HELO', { 'auth' => 'auth' }])\n      end\n    end\n\n    data(\n      lack_of_elem: ['PONG', true, '', 'client_hostname'],\n      wrong_message: ['WRONG_PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],\n      error_by_server: ['PONG', false, 'error', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],\n      same_hostname_as_server: ['PONG', true, '', 'hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],\n      wrong_key: ['PONG', true, '', 'hostname', 'wrong_key'],\n    )\n    test 'raises an error when message is' do |msg|\n      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: '', username: '')\n      handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt\n\n      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '')\n      assert_raise(Fluent::Plugin::ForwardOutput::PingpongError) do\n        handshake.invoke('', ri, msg)\n      end\n\n      assert_equal(ri.state, :pingpong)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/out_forward/test_load_balancer.rb",
    "content": "require_relative '../../helper'\nrequire 'flexmock/test_unit'\n\nrequire 'fluent/plugin/out_forward/load_balancer'\n\nclass LoadBalancerTest < Test::Unit::TestCase\n  sub_test_case 'select_healthy_node' do\n    test 'select healthy node' do\n      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)\n      n1 = flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)\n      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)\n\n      lb.rebuild_weight_array([n1, n2])\n      lb.select_healthy_node do |node|\n        assert_equal(node, n2)\n      end\n\n      lb.select_healthy_node do |node|\n        assert_equal(node, n2)\n      end\n    end\n\n    test 'call like round robin' do\n      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)\n      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)\n      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)\n\n      lb.rebuild_weight_array([n1, n2])\n\n      lb.select_healthy_node do |node|\n        # to handle random choice\n        if node == n1\n          lb.select_healthy_node do |node|\n            assert_equal(node, n2)\n          end\n\n          lb.select_healthy_node do |node|\n            assert_equal(node, n1)\n          end\n        else\n          lb.select_healthy_node do |node|\n            assert_equal(node, n1)\n          end\n\n          lb.select_healthy_node do |node|\n            assert_equal(node, n2)\n          end\n        end\n      end\n    end\n\n    test 'call like round robin without weight=0 node' do\n      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)\n      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)\n      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)\n      n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)\n\n      lb.rebuild_weight_array([n1, n2, n3])\n\n      lb.select_healthy_node do |node|\n        # to handle random choice\n        if node == n1\n          lb.select_healthy_node do |node|\n            assert_equal(node, n2)\n          end\n\n          lb.select_healthy_node do |node|\n            assert_equal(node, n1)\n          end\n\n          lb.select_healthy_node do |node|\n            assert_equal(node, n2)\n          end\n        else\n          lb.select_healthy_node do |node|\n            assert_equal(node, n1)\n          end\n\n          lb.select_healthy_node do |node|\n            assert_equal(node, n2)\n          end\n\n          lb.select_healthy_node do |node|\n            assert_equal(node, n1)\n          end\n        end\n      end\n    end\n\n    test 'raise an error if all node are unavailable' do\n      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)\n      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])\n      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do\n        lb.select_healthy_node\n      end\n    end\n\n    test 'it regards weight=0 node as unavailable' do\n      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)\n      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])\n      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do\n        lb.select_healthy_node\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/out_forward/test_socket_cache.rb",
    "content": "require_relative '../../helper'\n\nrequire 'fluent/plugin/out_forward/socket_cache'\nrequire 'timecop'\n\nclass SocketCacheTest < Test::Unit::TestCase\n  sub_test_case 'checkout_or' do\n    test 'when given key does not exist' do\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n      sock = mock!.open { 'socket' }.subject\n      assert_equal('socket', c.checkout_or('key') { sock.open })\n    end\n\n    test 'when given key exists' do\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n      socket = 'socket'\n      assert_equal(socket, c.checkout_or('key') { socket })\n      c.checkin(socket)\n\n      sock = mock!.open.never.subject\n      assert_equal(socket, c.checkout_or('key') { sock.open })\n    end\n\n    test 'when given key exists but used by other' do\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n      assert_equal('sock', c.checkout_or('key') { 'sock' })\n\n      new_sock = 'new sock'\n      sock = mock!.open { new_sock }.subject\n      assert_equal(new_sock, c.checkout_or('key') { sock.open })\n    end\n\n    test \"when given key's value was expired\" do\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(0, $log)\n      assert_equal('sock', c.checkout_or('key') { 'sock' })\n\n      new_sock = 'new sock'\n      sock = mock!.open { new_sock }.subject\n      assert_equal(new_sock, c.checkout_or('key') { sock.open })\n    end\n\n    test 'reuse same hash object after calling purge_obsolete_socks' do\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n      c.checkout_or('key') { 'socket' }\n      c.purge_obsolete_socks\n\n      assert_nothing_raised(NoMethodError) do\n        c.checkout_or('key') { 'new socket' }\n      end\n    end\n  end\n\n  sub_test_case 'checkin' do\n    test 'when value exists' do\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n      socket = 'socket'\n      c.checkout_or('key') { socket }\n      c.checkin(socket)\n\n      assert_equal(socket, c.instance_variable_get(:@available_sockets)['key'].first.sock)\n      assert_equal(1, c.instance_variable_get(:@available_sockets)['key'].size)\n      assert_equal(0, c.instance_variable_get(:@inflight_sockets).size)\n    end\n\n    test 'when value does not exist' do\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n      c.checkout_or('key') { 'sock' }\n      c.checkin('other sock')\n\n      assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size)\n      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)\n    end\n  end\n\n  test 'revoke' do\n    c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n    socket = 'socket'\n    c.checkout_or('key') { socket }\n    c.revoke(socket)\n\n    assert_equal(1, c.instance_variable_get(:@inactive_sockets).size)\n    assert_equal(0, c.instance_variable_get(:@inflight_sockets).size)\n    assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size)\n\n    sock = mock!.open { 1 }.subject\n    assert_equal(1, c.checkout_or('key') { sock.open })\n  end\n\n  sub_test_case 'clear' do\n    test 'when value is in available_sockets' do\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n      m = mock!.close { 'closed' }.subject\n      m2 = mock!.close { 'closed' }.subject\n      m3 = mock!.close { 'closed' }.subject\n      c.checkout_or('key') { m }\n      c.revoke(m)\n      c.checkout_or('key') { m2 }\n      c.checkin(m2)\n      c.checkout_or('key2') { m3 }\n\n      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)\n      assert_equal(1, c.instance_variable_get(:@available_sockets)['key'].size)\n      assert_equal(1, c.instance_variable_get(:@inactive_sockets).size)\n\n      c.clear\n      assert_equal(0, c.instance_variable_get(:@inflight_sockets).size)\n      assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size)\n      assert_equal(0, c.instance_variable_get(:@inactive_sockets).size)\n    end\n  end\n\n  sub_test_case 'purge_obsolete_socks' do\n    def teardown\n      Timecop.return\n    end\n\n    test 'delete key in inactive_socks' do\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n      sock = mock!.close { 'closed' }.subject\n      c.checkout_or('key') { sock }\n      c.revoke(sock)\n      assert_false(c.instance_variable_get(:@inactive_sockets).empty?)\n\n      c.purge_obsolete_socks\n      assert_true(c.instance_variable_get(:@inactive_sockets).empty?)\n    end\n\n    test 'move key from available_sockets to inactive_sockets' do\n      Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900'))\n\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n      sock = mock!.close { 'closed' }.subject\n      sock2 = mock!.close.never.subject\n      stub(sock).inspect\n      stub(sock2).inspect\n\n      c.checkout_or('key') { sock }\n      c.checkin(sock)\n\n      # wait timeout\n      Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900'))\n      c.checkout_or('key') { sock2 }\n\n      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)\n      assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock)\n\n      c.purge_obsolete_socks\n\n      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)\n      assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock)\n    end\n\n    test 'should not purge just after checkin and purge after timeout' do\n      Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900'))\n\n      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)\n      sock = mock!.close.never.subject\n      stub(sock).inspect\n      c.checkout_or('key') { sock }\n\n      Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900'))\n      c.checkin(sock)\n\n      assert_equal(1, c.instance_variable_get(:@available_sockets).size)\n      c.purge_obsolete_socks\n      assert_equal(1, c.instance_variable_get(:@available_sockets).size)\n\n      Timecop.freeze(Time.parse('2016-04-13 14:00:22 +0900'))\n      assert_equal(1, c.instance_variable_get(:@available_sockets).size)\n      c.purge_obsolete_socks\n      assert_equal(0, c.instance_variable_get(:@available_sockets).size)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_bare_output.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/bare_output'\nrequire 'fluent/event'\n\nmodule FluentPluginBareOutputTest\n  class DummyPlugin < Fluent::Plugin::BareOutput\n    attr_reader :store\n    def initialize\n      super\n      @store = []\n    end\n    def process(tag, es)\n      es.each do |time, record|\n        @store << [tag, time, record]\n      end\n    end\n  end\nend\n\nclass BareOutputTest < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n    @p = FluentPluginBareOutputTest::DummyPlugin.new\n  end\n\n  test 'has healthy lifecycle' do\n    assert !@p.configured?\n    @p.configure(config_element())\n    assert @p.configured?\n\n    assert !@p.started?\n    @p.start\n    assert @p.start\n\n    assert !@p.stopped?\n    @p.stop\n    assert @p.stopped?\n\n    assert !@p.before_shutdown?\n    @p.before_shutdown\n    assert @p.before_shutdown?\n\n    assert !@p.shutdown?\n    @p.shutdown\n    assert @p.shutdown?\n\n    assert !@p.after_shutdown?\n    @p.after_shutdown\n    assert @p.after_shutdown?\n\n    assert !@p.closed?\n    @p.close\n    assert @p.closed?\n\n    assert !@p.terminated?\n    @p.terminate\n    assert @p.terminated?\n  end\n\n  test 'has plugin_id automatically generated' do\n    assert @p.respond_to?(:plugin_id_configured?)\n    assert @p.respond_to?(:plugin_id)\n\n    @p.configure(config_element())\n\n    assert !@p.plugin_id_configured?\n    assert @p.plugin_id\n    assert{ @p.plugin_id != 'mytest' }\n  end\n\n  test 'has plugin_id manually configured' do\n    @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))\n    assert @p.plugin_id_configured?\n    assert_equal 'mytest', @p.plugin_id\n  end\n\n  test 'has plugin logger' do\n    assert @p.respond_to?(:log)\n    assert @p.log\n\n    # default logger\n    original_logger = @p.log\n\n    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))\n\n    assert(@p.log.object_id != original_logger.object_id)\n    assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level\n  end\n\n  test 'can load plugin helpers' do\n    assert_nothing_raised do\n      class FluentPluginBareOutputTest::DummyPlugin2 < Fluent::Plugin::BareOutput\n        helpers :storage\n      end\n    end\n  end\n\n  test 'can use metrics plugins and fallback methods' do\n    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))\n\n    %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|\n      assert_true @p.instance_variable_get(:\"@#{metric_name}\").is_a?(Fluent::Plugin::Metrics)\n    end\n\n    assert_equal 0, @p.num_errors\n    assert_equal 0, @p.emit_count\n    assert_equal 0, @p.emit_size\n    assert_equal 0, @p.emit_records\n  end\n\n  test 'can get input event stream to write' do\n    @p.configure(config_element('ROOT'))\n    @p.start\n\n    es1 = Fluent::OneEventStream.new(event_time('2016-05-21 18:37:31 +0900'), {'k1' => 'v1'})\n    es2 = Fluent::ArrayEventStream.new([\n        [event_time('2016-05-21 18:38:33 +0900'), {'k2' => 'v2'}],\n        [event_time('2016-05-21 18:39:10 +0900'), {'k3' => 'v3'}],\n      ])\n    @p.emit_events('mytest1', es1)\n    @p.emit_events('mytest2', es2)\n\n    all_events = [\n      ['mytest1', event_time('2016-05-21 18:37:31 +0900'), {'k1' => 'v1'}],\n      ['mytest2', event_time('2016-05-21 18:38:33 +0900'), {'k2' => 'v2'}],\n      ['mytest2', event_time('2016-05-21 18:39:10 +0900'), {'k3' => 'v3'}],\n    ]\n\n    assert_equal all_events, @p.store\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_base.rb",
    "content": "require_relative '../helper'\nrequire 'tmpdir'\nrequire 'fluent/plugin/base'\n\nmodule FluentPluginBaseTest\n  class DummyPlugin < Fluent::Plugin::Base\n  end\nend\n\nclass BaseTest < Test::Unit::TestCase\n  setup do\n    @p = FluentPluginBaseTest::DummyPlugin.new\n  end\n\n  test 'has methods for phases of plugin life cycle, and methods to know \"super\"s were correctly called or not' do\n    assert !@p.configured?\n    @p.configure(config_element())\n    assert @p.configured?\n\n    assert !@p.started?\n    @p.start\n    assert @p.start\n\n    assert !@p.stopped?\n    @p.stop\n    assert @p.stopped?\n\n    assert !@p.before_shutdown?\n    @p.before_shutdown\n    assert @p.before_shutdown?\n\n    assert !@p.shutdown?\n    @p.shutdown\n    assert @p.shutdown?\n\n    assert !@p.after_shutdown?\n    @p.after_shutdown\n    assert @p.after_shutdown?\n\n    assert !@p.closed?\n    @p.close\n    assert @p.closed?\n\n    assert !@p.terminated?\n    @p.terminate\n    assert @p.terminated?\n  end\n\n  test 'can access system config' do\n    assert @p.system_config\n\n    @p.system_config_override({'process_name' => 'mytest'})\n    assert_equal 'mytest', @p.system_config.process_name\n  end\n\n  test 'does not have router in default' do\n    assert !@p.has_router?\n  end\n\n  sub_test_case '#fluentd_worker_id' do\n    test 'returns 0 in default' do\n      assert_equal 0, @p.fluentd_worker_id\n    end\n\n    test 'returns the value specified via SERVERENGINE_WORKER_ID env variable' do\n      pre_value = ENV['SERVERENGINE_WORKER_ID']\n      begin\n        ENV['SERVERENGINE_WORKER_ID'] = 7.to_s\n        assert_equal 7, @p.fluentd_worker_id\n      ensure\n        ENV['SERVERENGINE_WORKER_ID'] = pre_value\n      end\n    end\n  end\n\n  test 'does not have root dir in default' do\n    assert_nil @p.plugin_root_dir\n  end\n\n  test 'is configurable by config_param and config_section' do\n    assert_nothing_raised do\n      class FluentPluginBaseTest::DummyPlugin2 < Fluent::Plugin::TestBase\n        config_param :myparam1, :string\n        config_section :mysection, multi: false do\n          config_param :myparam2, :integer\n        end\n      end\n    end\n    p2 = FluentPluginBaseTest::DummyPlugin2.new\n    assert_nothing_raised do\n      p2.configure(config_element('ROOT', '', {'myparam1' => 'myvalue1'}, [config_element('mysection', '', {'myparam2' => 99})]))\n    end\n    assert_equal 'myvalue1', p2.myparam1\n    assert_equal 99, p2.mysection.myparam2\n  end\n\n  test 'plugins are available with multi worker configuration in default' do\n    assert @p.multi_workers_ready?\n  end\n\n  test 'provides #string_safe_encoding to scrub invalid sequence string with info logging' do\n    logger = Fluent::Test::TestLogger.new\n    m = Module.new do\n      define_method(:log) do\n        logger\n      end\n    end\n    @p.extend m\n    assert_equal [], logger.logs\n\n    ret = @p.string_safe_encoding(\"abc\\xff.\\x01f\"){|s| s.split(\".\") }\n    assert_equal ['abc?', \"\\u0001f\"], ret\n    assert_equal 1, logger.logs.size\n    assert{ logger.logs.first.include?(\"invalid byte sequence is replaced in \") }\n  end\n\n  test 'generates worker lock path safely' do\n    Dir.mktmpdir(\"test-fluentd-lock-\") do |lock_dir|\n      ENV['FLUENTD_LOCK_DIR'] = lock_dir\n      p = FluentPluginBaseTest::DummyPlugin.new\n      path = p.get_lock_path(\"Aa\\\\|=~/_123\")\n\n      assert_equal lock_dir, File.dirname(path)\n      assert_equal \"fluentd-Aa______123.lock\", File.basename(path)\n    end\n  end\n\n  test 'can acquire inter-worker locking' do\n    Dir.mktmpdir(\"test-fluentd-lock-\") do |lock_dir|\n      ENV['FLUENTD_LOCK_DIR'] = lock_dir\n      p = FluentPluginBaseTest::DummyPlugin.new\n      lock_path = p.get_lock_path(\"test_base\")\n\n      p.acquire_worker_lock(\"test_base\") do\n        # With LOCK_NB set, flock() returns `false` when the\n        # file is already locked.\n        File.open(lock_path, \"w\") do |f|\n          assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)\n        end\n      end\n\n      # Lock should be release by now. In that case, flock\n      # must return 0.\n      File.open(lock_path, \"w\") do |f|\n        assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)\n      end\n    end\n  end\n\n  test '`ArgumentError` when `conf` is not `Fluent::Config::Element`' do\n    assert_raise ArgumentError.new('BUG: type of conf must be Fluent::Config::Element, but Hash is passed.') do\n      @p.configure({})\n    end\n  end\n\n  sub_test_case 'system_config.workers value after configure' do\n    def assert_system_config_workers_value(data)\n      conf = config_element()\n      conf.set_target_worker_ids(data[:target_worker_ids])\n      @p.configure(conf)\n      assert{ @p.system_config.workers == data[:expected] }\n    end\n\n    def stub_supervisor_mode\n      stub(Fluent::Engine).supervisor_mode { true }\n      stub(Fluent::Engine).worker_id { -1 }\n    end\n\n    sub_test_case 'with <system> workers 3 </system>' do\n      setup do\n        system_config = Fluent::SystemConfig.new\n        system_config.workers = 3\n        stub(Fluent::Engine).system_config { system_config }\n      end\n\n      data(\n        'without <worker> directive',\n        {\n          target_worker_ids: [],\n          expected: 3\n        },\n        keep: true\n      )\n      data(\n        'with <worker 0>',\n        {\n          target_worker_ids: [0],\n          expected: 1\n        },\n        keep: true\n      )\n      data(\n        'with <worker 0-1>',\n        {\n          target_worker_ids: [0, 1],\n          expected: 2\n        },\n        keep: true\n      )\n      data(\n        'with <worker 0-2>',\n        {\n          target_worker_ids: [0, 1, 2],\n          expected: 3\n        },\n        keep: true\n      )\n\n      test 'system_config.workers value after configure' do\n        assert_system_config_workers_value(data)\n      end\n\n      test 'system_config.workers value after configure  with supervisor_mode' do\n        stub_supervisor_mode\n        assert_system_config_workers_value(data)\n      end\n    end\n\n    sub_test_case 'without <system> directive' do\n      data(\n        'without <worker> directive',\n        {\n          target_worker_ids: [],\n          expected: 1\n        },\n        keep: true\n      )\n      data(\n        'with <worker 0>',\n        {\n          target_worker_ids: [0],\n          expected: 1\n        },\n        keep: true\n      )\n\n      test 'system_config.workers value after configure' do\n        assert_system_config_workers_value(data)\n      end\n\n      test 'system_config.workers value after configure  with supervisor_mode' do\n        stub_supervisor_mode\n        assert_system_config_workers_value(data)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_buf_file.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/buf_file'\nrequire 'fluent/plugin/output'\nrequire 'fluent/unique_id'\nrequire 'fluent/system_config'\nrequire 'fluent/env'\n\nrequire 'msgpack'\n\nmodule FluentPluginFileBufferTest\n  class DummyOutputPlugin < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('buffer_file_test_output', self)\n    config_section :buffer do\n      config_set_default :@type, 'file'\n    end\n    def multi_workers_ready?\n      true\n    end\n    def write(chunk)\n      # drop\n    end\n  end\n\n  class DummyErrorOutputPlugin < DummyOutputPlugin\n    def register_write(&block)\n      instance_variable_set(:@write, block)\n    end\n\n    def initialize\n      super\n      @should_fail_writing = true\n      @write = nil\n    end\n\n    def recover\n      @should_fail_writing = false\n    end\n\n    def write(chunk)\n      if @should_fail_writing\n        raise \"failed writing chunk\"\n      else\n        @write ? @write.call(chunk) : nil\n      end\n    end\n\n    def format(tag, time, record)\n      [tag, time.to_i, record].to_json + \"\\n\"\n    end\n  end\nend\n\nclass FileBufferTest < Test::Unit::TestCase\n  def metadata(timekey: nil, tag: nil, variables: nil, seq: 0)\n    m = Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n    m.seq = seq\n    m\n  end\n\n  def write_metadata_old(path, chunk_id, metadata, size, ctime, mtime)\n    metadata = {\n      timekey: metadata.timekey, tag: metadata.tag, variables: metadata.variables,\n      id: chunk_id,\n      s: size,\n      c: ctime,\n      m: mtime,\n    }\n    File.open(path, 'wb') do |f|\n      f.write metadata.to_msgpack\n    end\n  end\n\n  def write_metadata(path, chunk_id, metadata, size, ctime, mtime)\n    metadata = {\n      timekey: metadata.timekey, tag: metadata.tag, variables: metadata.variables,\n      seq: metadata.seq,\n      id: chunk_id,\n      s: size,\n      c: ctime,\n      m: mtime,\n    }\n\n    data = metadata.to_msgpack\n    size = [data.size].pack('N')\n    File.open(path, 'wb') do |f|\n      f.write(Fluent::Plugin::Buffer::FileChunk::BUFFER_HEADER + size + data)\n    end\n  end\n\n  sub_test_case 'non configured buffer plugin instance' do\n    setup do\n      Fluent::Test.setup\n\n      @dir = File.expand_path('../../tmp/buffer_file_dir', __FILE__)\n      FileUtils.rm_rf @dir\n      FileUtils.mkdir_p @dir\n    end\n\n    test 'path should include * normally' do\n      d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      p = Fluent::Plugin::FileBuffer.new\n      p.owner = d\n      p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer.*.file')}))\n      assert_equal File.join(@dir, 'buffer.*.file'), p.path\n    end\n\n    data('default' => [nil, 'log'],\n         'conf' => ['.buf', 'buf'])\n    test 'existing directory will be used with additional default file name' do |params|\n      conf, suffix = params\n      d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      p = Fluent::Plugin::FileBuffer.new\n      p.owner = d\n      c = {'path' => @dir}\n      c['path_suffix'] = conf if conf\n      p.configure(config_element('buffer', '', c))\n      assert_equal File.join(@dir, \"buffer.*.#{suffix}\"), p.path\n    end\n\n    data('default' => [nil, 'log'],\n         'conf' => ['.buf', 'buf'])\n    test 'unexisting path without * handled as directory' do |params|\n      conf, suffix = params\n      d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      p = Fluent::Plugin::FileBuffer.new\n      p.owner = d\n      c = {'path' => File.join(@dir, 'buffer')}\n      c['path_suffix'] = conf if conf\n      p.configure(config_element('buffer', '', c))\n      assert_equal File.join(@dir, 'buffer', \"buffer.*.#{suffix}\"), p.path\n    end\n  end\n\n  sub_test_case 'buffer configurations and workers' do\n    setup do\n      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)\n      FileUtils.rm_rf @bufdir\n      Fluent::Test.setup\n\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @p = Fluent::Plugin::FileBuffer.new\n      @p.owner = @d\n    end\n\n    test 'raise error if configured path is of existing file' do\n      @bufpath = File.join(@bufdir, 'buf')\n      FileUtils.mkdir_p @bufdir\n      File.open(@bufpath, 'w'){|f| } # create and close the file\n      assert File.exist?(@bufpath)\n      assert File.file?(@bufpath)\n\n      buf_conf = config_element('buffer', '', {'path' => @bufpath})\n      assert_raise Fluent::ConfigError.new(\"Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)\") do\n        Fluent::SystemConfig.overwrite_system_config('workers' => 4) do\n          @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))\n        end\n      end\n    end\n\n    test 'raise error if fluentd is configured to use file path pattern and multi workers' do\n      @bufpath = File.join(@bufdir, 'testbuf.*.log')\n      buf_conf = config_element('buffer', '', {'path' => @bufpath})\n      assert_raise Fluent::ConfigError.new(\"Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)\") do\n        Fluent::SystemConfig.overwrite_system_config('workers' => 4) do\n          @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))\n        end\n      end\n    end\n\n    test 'enables multi worker configuration with unexisting directory path' do\n      assert_false File.exist?(@bufdir)\n      buf_conf = config_element('buffer', '', {'path' => @bufdir})\n      assert_nothing_raised do\n        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do\n          @d.configure(config_element('ROOT', '', {}, [buf_conf]))\n        end\n      end\n    end\n\n    test 'enables multi worker configuration with existing directory path' do\n      FileUtils.mkdir_p @bufdir\n      buf_conf = config_element('buffer', '', {'path' => @bufdir})\n      assert_nothing_raised do\n        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do\n          @d.configure(config_element('ROOT', '', {}, [buf_conf]))\n        end\n      end\n    end\n\n    test 'enables multi worker configuration with root dir' do\n      buf_conf = config_element('buffer', '')\n      assert_nothing_raised do\n        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do\n          @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))\n        end\n      end\n    end\n  end\n\n  sub_test_case 'buffer plugin configured only with path' do\n    setup do\n      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)\n      @bufpath = File.join(@bufdir, 'testbuf.*.log')\n      FileUtils.rm_r @bufdir if File.exist?(@bufdir)\n\n      Fluent::Test.setup\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @p = Fluent::Plugin::FileBuffer.new\n      @p.owner = @d\n      @p.configure(config_element('buffer', '', {'path' => @bufpath}))\n      @p.start\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      if @bufdir\n        Dir.glob(File.join(@bufdir, '*')).each do |path|\n          next if ['.', '..'].include?(File.basename(path))\n          File.delete(path)\n        end\n      end\n    end\n\n    test 'this is persistent plugin' do\n      assert @p.persistent?\n    end\n\n    test '#start creates directory for buffer chunks' do\n      plugin = Fluent::Plugin::FileBuffer.new\n      plugin.owner = @d\n      rand_num = rand(0..100)\n      bufpath = File.join(File.expand_path(\"../../tmp/buffer_file_#{rand_num}\", __FILE__), 'testbuf.*.log')\n      bufdir = File.dirname(bufpath)\n\n      FileUtils.rm_r bufdir if File.exist?(bufdir)\n      assert !File.exist?(bufdir)\n\n      plugin.configure(config_element('buffer', '', {'path' => bufpath}))\n      assert !File.exist?(bufdir)\n\n      plugin.start\n      assert File.exist?(bufdir)\n      assert{ File.stat(bufdir).mode.to_s(8).end_with?('755') }\n\n      plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate\n      FileUtils.rm_r bufdir\n    end\n\n    test '#start creates directory for buffer chunks with specified permission' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      plugin = Fluent::Plugin::FileBuffer.new\n      plugin.owner = @d\n      rand_num = rand(0..100)\n      bufpath = File.join(File.expand_path(\"../../tmp/buffer_file_#{rand_num}\", __FILE__), 'testbuf.*.log')\n      bufdir = File.dirname(bufpath)\n\n      FileUtils.rm_r bufdir if File.exist?(bufdir)\n      assert !File.exist?(bufdir)\n\n      plugin.configure(config_element('buffer', '', {'path' => bufpath, 'dir_permission' => '0700'}))\n      assert !File.exist?(bufdir)\n\n      plugin.start\n      assert File.exist?(bufdir)\n      assert{ File.stat(bufdir).mode.to_s(8).end_with?('700') }\n\n      plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate\n      FileUtils.rm_r bufdir\n    end\n\n    test '#start creates directory for buffer chunks with specified permission via system config' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      sysconf = {'dir_permission' => '700'}\n      Fluent::SystemConfig.overwrite_system_config(sysconf) do\n        plugin = Fluent::Plugin::FileBuffer.new\n        plugin.owner = @d\n        rand_num = rand(0..100)\n        bufpath = File.join(File.expand_path(\"../../tmp/buffer_file_#{rand_num}\", __FILE__), 'testbuf.*.log')\n        bufdir = File.dirname(bufpath)\n\n        FileUtils.rm_r bufdir if File.exist?(bufdir)\n        assert !File.exist?(bufdir)\n\n        plugin.configure(config_element('buffer', '', {'path' => bufpath}))\n        assert !File.exist?(bufdir)\n\n        plugin.start\n        assert File.exist?(bufdir)\n        assert{ File.stat(bufdir).mode.to_s(8).end_with?('700') }\n\n        plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate\n        FileUtils.rm_r bufdir\n      end\n    end\n\n    test '#generate_chunk generates blank file chunk on path from unique_id of metadata' do\n      m1 = metadata()\n      c1 = @p.generate_chunk(m1)\n      assert c1.is_a? Fluent::Plugin::Buffer::FileChunk\n      assert_equal m1, c1.metadata\n      assert c1.empty?\n      assert_equal :unstaged, c1.state\n      assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission\n      assert_equal @bufpath.gsub('.*.', \".b#{Fluent::UniqueId.hex(c1.unique_id)}.\"), c1.path\n      assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') }\n\n      m2 = metadata(timekey: event_time('2016-04-17 11:15:00 -0700').to_i)\n      c2 = @p.generate_chunk(m2)\n      assert c2.is_a? Fluent::Plugin::Buffer::FileChunk\n      assert_equal m2, c2.metadata\n      assert c2.empty?\n      assert_equal :unstaged, c2.state\n      assert_equal Fluent::DEFAULT_FILE_PERMISSION, c2.permission\n      assert_equal @bufpath.gsub('.*.', \".b#{Fluent::UniqueId.hex(c2.unique_id)}.\"), c2.path\n      assert{ File.stat(c2.path).mode.to_s(8).end_with?('644') }\n\n      c1.purge\n      c2.purge\n    end\n\n    test '#generate_chunk generates blank file chunk with specified permission' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      plugin = Fluent::Plugin::FileBuffer.new\n      plugin.owner = @d\n      rand_num = rand(0..100)\n      bufpath = File.join(File.expand_path(\"../../tmp/buffer_file_#{rand_num}\", __FILE__), 'testbuf.*.log')\n      bufdir = File.dirname(bufpath)\n\n      FileUtils.rm_r bufdir if File.exist?(bufdir)\n      assert !File.exist?(bufdir)\n\n      plugin.configure(config_element('buffer', '', {'path' => bufpath, 'file_permission' => '0600'}))\n      assert !File.exist?(bufdir)\n      plugin.start\n\n      m = metadata()\n      c = plugin.generate_chunk(m)\n      assert c.is_a? Fluent::Plugin::Buffer::FileChunk\n      assert_equal m, c.metadata\n      assert c.empty?\n      assert_equal :unstaged, c.state\n      assert_equal 0600, c.permission\n      assert_equal bufpath.gsub('.*.', \".b#{Fluent::UniqueId.hex(c.unique_id)}.\"), c.path\n      assert{ File.stat(c.path).mode.to_s(8).end_with?('600') }\n\n      c.purge\n\n      plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate\n      FileUtils.rm_r bufdir\n    end\n\n    test '#generate_chunk generates blank file chunk with specified permission with system_config' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      begin\n        plugin = Fluent::Plugin::FileBuffer.new\n        plugin.owner = @d\n        rand_num = rand(0..100)\n        bufpath = File.join(File.expand_path(\"../../tmp/buffer_file_#{rand_num}\", __FILE__), 'testbuf.*.log')\n        bufdir = File.dirname(bufpath)\n\n        FileUtils.rm_r bufdir if File.exist?(bufdir)\n        assert !File.exist?(bufdir)\n\n        plugin.configure(config_element('buffer', '', { 'path' => bufpath }))\n\n        assert !File.exist?(bufdir)\n        plugin.start\n\n        m = metadata()\n        c = nil\n        Fluent::SystemConfig.overwrite_system_config(\"file_permission\" => \"700\") do\n          c = plugin.generate_chunk(m)\n        end\n\n        assert c.is_a? Fluent::Plugin::Buffer::FileChunk\n        assert_equal m, c.metadata\n        assert c.empty?\n        assert_equal :unstaged, c.state\n        assert_equal 0700, c.permission\n        assert_equal bufpath.gsub('.*.', \".b#{Fluent::UniqueId.hex(c.unique_id)}.\"), c.path\n        assert{ File.stat(c.path).mode.to_s(8).end_with?('700') }\n\n        c.purge\n\n        plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate\n      ensure\n        FileUtils.rm_r bufdir\n      end\n    end\n  end\n\n  sub_test_case 'configured with system root directory and plugin @id' do\n    setup do\n      @root_dir = File.expand_path('../../tmp/buffer_file_root', __FILE__)\n      FileUtils.rm_rf @root_dir\n\n      Fluent::Test.setup\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @p = Fluent::Plugin::FileBuffer.new\n      @p.owner = @d\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n    end\n\n    data('default' => [nil, 'log'],\n         'conf' => ['.buf', 'buf'])\n    test '#start creates directory for buffer chunks' do |params|\n      conf, suffix = params\n      c = {}\n      c['path_suffix'] = conf if conf\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do\n        @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))\n        @p.configure(config_element('buffer', '', c))\n      end\n\n      expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', \"buffer.*.#{suffix}\")\n      expected_buffer_dir = File.dirname(expected_buffer_path)\n      assert_equal expected_buffer_path, @p.path\n      assert_false Dir.exist?(expected_buffer_dir)\n\n      @p.start\n\n      assert Dir.exist?(expected_buffer_dir)\n    end\n  end\n\n  sub_test_case 'there are no existing file chunks' do\n    setup do\n      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)\n      @bufpath = File.join(@bufdir, 'testbuf.*.log')\n      FileUtils.rm_r @bufdir if File.exist?(@bufdir)\n\n      Fluent::Test.setup\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @p = Fluent::Plugin::FileBuffer.new\n      @p.owner = @d\n      @p.configure(config_element('buffer', '', {'path' => @bufpath}))\n      @p.start\n    end\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      if @bufdir\n        Dir.glob(File.join(@bufdir, '*')).each do |path|\n          next if ['.', '..'].include?(File.basename(path))\n          File.delete(path)\n        end\n      end\n    end\n\n    test '#resume returns empty buffer state' do\n      ary = @p.resume\n      assert_equal({}, ary[0])\n      assert_equal([], ary[1])\n    end\n  end\n\n  sub_test_case 'there are some existing file chunks' do\n    setup do\n      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)\n      FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir)\n\n      @c1id = Fluent::UniqueId.generate\n      p1 = File.join(@bufdir, \"etest.q#{Fluent::UniqueId.hex(@c1id)}.log\")\n      File.open(p1, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:58:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:58:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 13:58:22 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata(\n        p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),\n        4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i\n      )\n\n      @c2id = Fluent::UniqueId.generate\n      p2 = File.join(@bufdir, \"etest.q#{Fluent::UniqueId.hex(@c2id)}.log\")\n      File.open(p2, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:59:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:59:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:59:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata(\n        p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i),\n        3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i\n      )\n\n      @c3id = Fluent::UniqueId.generate\n      p3 = File.join(@bufdir, \"etest.b#{Fluent::UniqueId.hex(@c3id)}.log\")\n      File.open(p3, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 14:00:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 14:00:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata(\n        p3 + '.meta', @c3id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),\n        4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i\n      )\n\n      @c4id = Fluent::UniqueId.generate\n      p4 = File.join(@bufdir, \"etest.b#{Fluent::UniqueId.hex(@c4id)}.log\")\n      File.open(p4, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:01:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 14:01:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 14:01:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata(\n        p4 + '.meta', @c4id, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i),\n        3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i\n      )\n\n      @bufpath = File.join(@bufdir, 'etest.*.log')\n\n      Fluent::Test.setup\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @p = Fluent::Plugin::FileBuffer.new\n      @p.owner = @d\n      @p.configure(config_element('buffer', '', {'path' => @bufpath}))\n      @p.start\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      if @bufdir\n        Dir.glob(File.join(@bufdir, '*')).each do |path|\n          next if ['.', '..'].include?(File.basename(path))\n          File.delete(path)\n        end\n      end\n    end\n\n    test '#resume returns staged/queued chunks with metadata' do\n      assert_equal 2, @p.stage.size\n      assert_equal 2, @p.queue.size\n\n      stage = @p.stage\n\n      m3 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i)\n      assert_equal @c3id, stage[m3].unique_id\n      assert_equal 4, stage[m3].size\n      assert_equal :staged, stage[m3].state\n\n      m4 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i)\n      assert_equal @c4id, stage[m4].unique_id\n      assert_equal 3, stage[m4].size\n      assert_equal :staged, stage[m4].state\n    end\n\n    test '#resume returns queued chunks ordered by last modified time (FIFO)' do\n      assert_equal 2, @p.stage.size\n      assert_equal 2, @p.queue.size\n\n      queue = @p.queue\n\n      assert{ queue[0].modified_at < queue[1].modified_at }\n\n      assert_equal @c1id, queue[0].unique_id\n      assert_equal :queued, queue[0].state\n      assert_equal event_time('2016-04-17 13:58:00 -0700').to_i, queue[0].metadata.timekey\n      assert_nil queue[0].metadata.tag\n      assert_nil queue[0].metadata.variables\n      assert_equal Time.parse('2016-04-17 13:58:00 -0700').localtime, queue[0].created_at\n      assert_equal Time.parse('2016-04-17 13:58:22 -0700').localtime, queue[0].modified_at\n      assert_equal 4, queue[0].size\n\n      assert_equal @c2id, queue[1].unique_id\n      assert_equal :queued, queue[1].state\n      assert_equal event_time('2016-04-17 13:59:00 -0700').to_i, queue[1].metadata.timekey\n      assert_nil queue[1].metadata.tag\n      assert_nil queue[1].metadata.variables\n      assert_equal Time.parse('2016-04-17 13:59:00 -0700').localtime, queue[1].created_at\n      assert_equal Time.parse('2016-04-17 13:59:23 -0700').localtime, queue[1].modified_at\n      assert_equal 3, queue[1].size\n    end\n  end\n\n  sub_test_case 'there are some existing file chunks with placeholders path' do\n    setup do\n      @bufdir = File.expand_path('../../tmp/buffer_${test}_file', __FILE__)\n      FileUtils.rm_rf(@bufdir)\n      FileUtils.mkdir_p(@bufdir)\n\n      @c1id = Fluent::UniqueId.generate\n      p1 = File.join(@bufdir, \"etest.q#{Fluent::UniqueId.hex(@c1id)}.log\")\n      File.open(p1, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata(\n        p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),\n        1, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i\n      )\n\n      @c2id = Fluent::UniqueId.generate\n      p2 = File.join(@bufdir, \"etest.b#{Fluent::UniqueId.hex(@c2id)}.log\")\n      File.open(p2, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata(\n        p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),\n        1, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i\n      )\n\n      @bufpath = File.join(@bufdir, 'etest.*.log')\n\n      Fluent::Test.setup\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @p = Fluent::Plugin::FileBuffer.new\n      @p.owner = @d\n      @p.configure(config_element('buffer', '', {'path' => @bufpath}))\n      @p.start\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      FileUtils.rm_rf(@bufdir)\n    end\n\n    test '#resume returns staged/queued chunks with metadata' do\n      assert_equal 1, @p.stage.size\n      assert_equal 1, @p.queue.size\n    end\n  end\n\n  sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do\n    setup do\n      @bufdir = File.expand_path('../../tmp/buffer_file/path', __FILE__)\n      @worker0_dir = File.join(@bufdir, \"worker0\")\n      @worker1_dir = File.join(@bufdir, \"worker1\")\n      FileUtils.rm_rf @bufdir\n      FileUtils.mkdir_p @worker0_dir\n      FileUtils.mkdir_p @worker1_dir\n\n      @bufdir_chunk_1 = Fluent::UniqueId.generate\n      bc1 = File.join(@bufdir, \"buffer.q#{Fluent::UniqueId.hex(@bufdir_chunk_1)}.log\")\n      File.open(bc1, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:58:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:58:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 13:58:22 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata(\n        bc1 + '.meta', @bufdir_chunk_1, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),\n        4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i\n      )\n\n      @bufdir_chunk_2 = Fluent::UniqueId.generate\n      bc2 = File.join(@bufdir, \"buffer.q#{Fluent::UniqueId.hex(@bufdir_chunk_2)}.log\")\n      File.open(bc2, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:58:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:58:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 13:58:22 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata(\n        bc2 + '.meta', @bufdir_chunk_2, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),\n        4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i\n      )\n\n      @worker_dir_chunk_1 = Fluent::UniqueId.generate\n      wc0_1 = File.join(@worker0_dir, \"buffer.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.log\")\n      wc1_1 = File.join(@worker1_dir, \"buffer.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.log\")\n      [wc0_1, wc1_1].each do |chunk_path|\n        File.open(chunk_path, 'wb') do |f|\n          f.write [\"t1.test\", event_time('2016-04-17 13:59:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t2.test\", event_time('2016-04-17 13:59:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t3.test\", event_time('2016-04-17 13:59:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        end\n        write_metadata(\n          chunk_path + '.meta', @worker_dir_chunk_1, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i),\n          3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i\n        )\n      end\n\n      @worker_dir_chunk_2 = Fluent::UniqueId.generate\n      wc0_2 = File.join(@worker0_dir, \"buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.log\")\n      wc1_2 = File.join(@worker1_dir, \"buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.log\")\n      [wc0_2, wc1_2].each do |chunk_path|\n        File.open(chunk_path, 'wb') do |f|\n          f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t2.test\", event_time('2016-04-17 14:00:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t3.test\", event_time('2016-04-17 14:00:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        end\n        write_metadata(\n          chunk_path + '.meta', @worker_dir_chunk_2, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),\n          4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i\n        )\n      end\n\n      @worker_dir_chunk_3 = Fluent::UniqueId.generate\n      wc0_3 = File.join(@worker0_dir, \"buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.log\")\n      wc1_3 = File.join(@worker1_dir, \"buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.log\")\n      [wc0_3, wc1_3].each do |chunk_path|\n        File.open(chunk_path, 'wb') do |f|\n          f.write [\"t1.test\", event_time('2016-04-17 14:01:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t2.test\", event_time('2016-04-17 14:01:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t3.test\", event_time('2016-04-17 14:01:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        end\n        write_metadata(\n          chunk_path + '.meta', @worker_dir_chunk_3, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i),\n          3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i\n        )\n      end\n\n      Fluent::Test.setup\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n    end\n\n    test 'worker(id=0) #resume returns staged/queued chunks with metadata, not only in worker dir, including the directory specified by path' do\n      ENV['SERVERENGINE_WORKER_ID'] = '0'\n\n      buf_conf = config_element('buffer', '', {'path' => @bufdir})\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      with_worker_config(workers: 2, worker_id: 0) do\n        @d.configure(config_element('output', '', {}, [buf_conf]))\n      end\n\n      @d.start\n      @p = @d.buffer\n\n      assert_equal 2, @p.stage.size\n      assert_equal 3, @p.queue.size\n\n      stage = @p.stage\n\n      m1 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i)\n      assert_equal @worker_dir_chunk_2, stage[m1].unique_id\n      assert_equal 4, stage[m1].size\n      assert_equal :staged, stage[m1].state\n\n      m2 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i)\n      assert_equal @worker_dir_chunk_3, stage[m2].unique_id\n      assert_equal 3, stage[m2].size\n      assert_equal :staged, stage[m2].state\n\n      queue = @p.queue\n\n      assert_equal [@bufdir_chunk_1, @bufdir_chunk_2, @worker_dir_chunk_1].sort, queue.map(&:unique_id).sort\n      assert_equal [3, 4, 4], queue.map(&:size).sort\n      assert_equal [:queued, :queued, :queued], queue.map(&:state)\n    end\n\n    test 'worker(id=1) #resume returns staged/queued chunks with metadata, only in worker dir' do\n      buf_conf = config_element('buffer', '', {'path' => @bufdir})\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      with_worker_config(workers: 2, worker_id: 1) do\n        @d.configure(config_element('output', '', {}, [buf_conf]))\n      end\n\n      @d.start\n      @p = @d.buffer\n\n      assert_equal 2, @p.stage.size\n      assert_equal 1, @p.queue.size\n\n      stage = @p.stage\n\n      m1 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i)\n      assert_equal @worker_dir_chunk_2, stage[m1].unique_id\n      assert_equal 4, stage[m1].size\n      assert_equal :staged, stage[m1].state\n\n      m2 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i)\n      assert_equal @worker_dir_chunk_3, stage[m2].unique_id\n      assert_equal 3, stage[m2].size\n      assert_equal :staged, stage[m2].state\n\n      queue = @p.queue\n\n      assert_equal @worker_dir_chunk_1, queue[0].unique_id\n      assert_equal 3, queue[0].size\n      assert_equal :queued, queue[0].state\n    end\n  end\n\n  sub_test_case 'there are some existing file chunks with old format metadata' do\n    setup do\n      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)\n      FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir)\n\n      @c1id = Fluent::UniqueId.generate\n      p1 = File.join(@bufdir, \"etest.q#{Fluent::UniqueId.hex(@c1id)}.log\")\n      File.open(p1, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:58:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:58:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 13:58:22 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata_old(\n        p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),\n        4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i\n      )\n\n      @c2id = Fluent::UniqueId.generate\n      p2 = File.join(@bufdir, \"etest.q#{Fluent::UniqueId.hex(@c2id)}.log\")\n      File.open(p2, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:59:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:59:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:59:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata_old(\n        p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i),\n        3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i\n      )\n\n      @c3id = Fluent::UniqueId.generate\n      p3 = File.join(@bufdir, \"etest.b#{Fluent::UniqueId.hex(@c3id)}.log\")\n      File.open(p3, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 14:00:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 14:00:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata_old(\n        p3 + '.meta', @c3id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),\n        4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i\n      )\n\n      @c4id = Fluent::UniqueId.generate\n      p4 = File.join(@bufdir, \"etest.b#{Fluent::UniqueId.hex(@c4id)}.log\")\n      File.open(p4, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:01:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 14:01:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 14:01:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata_old(\n        p4 + '.meta', @c4id, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i),\n        3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i\n      )\n\n      @bufpath = File.join(@bufdir, 'etest.*.log')\n\n      Fluent::Test.setup\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @p = Fluent::Plugin::FileBuffer.new\n      @p.owner = @d\n      @p.configure(config_element('buffer', '', {'path' => @bufpath}))\n      @p.start\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      if @bufdir\n        Dir.glob(File.join(@bufdir, '*')).each do |path|\n          next if ['.', '..'].include?(File.basename(path))\n          File.delete(path)\n        end\n      end\n    end\n\n    test '#resume returns staged/queued chunks with metadata' do\n      assert_equal 2, @p.stage.size\n      assert_equal 2, @p.queue.size\n\n      stage = @p.stage\n\n      m3 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i)\n      assert_equal @c3id, stage[m3].unique_id\n      assert_equal 4, stage[m3].size\n      assert_equal :staged, stage[m3].state\n\n      m4 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i)\n      assert_equal @c4id, stage[m4].unique_id\n      assert_equal 3, stage[m4].size\n      assert_equal :staged, stage[m4].state\n    end\n  end\n\n  sub_test_case 'there are some existing file chunks with old format metadata file' do\n    setup do\n      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)\n\n      @c1id = Fluent::UniqueId.generate\n      p1 = File.join(@bufdir, \"etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log\")\n      File.open(p1, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:58:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:58:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 13:58:22 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      FileUtils.touch(p1, mtime: Time.parse('2016-04-17 13:58:28 -0700'))\n\n      @c2id = Fluent::UniqueId.generate\n      p2 = File.join(@bufdir, \"etest.201604171359.q#{Fluent::UniqueId.hex(@c2id)}.log\")\n      File.open(p2, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:59:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:59:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:59:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      FileUtils.touch(p2, mtime: Time.parse('2016-04-17 13:59:30 -0700'))\n\n      @c3id = Fluent::UniqueId.generate\n      p3 = File.join(@bufdir, \"etest.201604171400.b#{Fluent::UniqueId.hex(@c3id)}.log\")\n      File.open(p3, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 14:00:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 14:00:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      FileUtils.touch(p3, mtime: Time.parse('2016-04-17 14:00:29 -0700'))\n\n      @c4id = Fluent::UniqueId.generate\n      p4 = File.join(@bufdir, \"etest.201604171401.b#{Fluent::UniqueId.hex(@c4id)}.log\")\n      File.open(p4, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:01:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 14:01:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 14:01:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      FileUtils.touch(p4, mtime: Time.parse('2016-04-17 14:01:22 -0700'))\n\n      @bufpath = File.join(@bufdir, 'etest.*.log')\n\n      Fluent::Test.setup\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @p = Fluent::Plugin::FileBuffer.new\n      @p.owner = @d\n      @p.configure(config_element('buffer', '', {'path' => @bufpath}))\n      @p.start\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      if @bufdir\n        Dir.glob(File.join(@bufdir, '*')).each do |path|\n          next if ['.', '..'].include?(File.basename(path))\n          File.delete(path)\n        end\n      end\n    end\n\n    test '#resume returns queued chunks for files without metadata' do\n      assert_equal 0, @p.stage.size\n      assert_equal 4, @p.queue.size\n\n      queue = @p.queue\n\n      m = metadata()\n\n      assert_equal @c1id, queue[0].unique_id\n      assert_equal m, queue[0].metadata\n      assert_equal 0, queue[0].size\n      assert_equal :queued, queue[0].state\n      assert_equal Time.parse('2016-04-17 13:58:28 -0700'), queue[0].modified_at\n\n      assert_equal @c2id, queue[1].unique_id\n      assert_equal m, queue[1].metadata\n      assert_equal 0, queue[1].size\n      assert_equal :queued, queue[1].state\n      assert_equal Time.parse('2016-04-17 13:59:30 -0700'), queue[1].modified_at\n\n      assert_equal @c3id, queue[2].unique_id\n      assert_equal m, queue[2].metadata\n      assert_equal 0, queue[2].size\n      assert_equal :queued, queue[2].state\n      assert_equal Time.parse('2016-04-17 14:00:29 -0700'), queue[2].modified_at\n\n      assert_equal @c4id, queue[3].unique_id\n      assert_equal m, queue[3].metadata\n      assert_equal 0, queue[3].size\n      assert_equal :queued, queue[3].state\n      assert_equal Time.parse('2016-04-17 14:01:22 -0700'), queue[3].modified_at\n    end\n  end\n\n  sub_test_case 'there are the same timekey metadata in stage' do\n    setup do\n      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)\n      @bufpath = File.join(@bufdir, 'testbuf.*.log')\n      FileUtils.rm_r(@bufdir) if File.exist?(@bufdir)\n      FileUtils.mkdir_p(@bufdir)\n\n      m = metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i)\n\n      c1id = Fluent::UniqueId.generate\n      p1 = File.join(@bufdir, \"testbuf.b#{Fluent::UniqueId.hex(c1id)}.log\")\n      File.open(p1, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay1\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay2\"}].to_json + \"\\n\"\n      end\n      write_metadata(p1 + '.meta', c1id, m, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i)\n\n      c2id = Fluent::UniqueId.generate\n      p2 = File.join(@bufdir, \"testbuf.b#{Fluent::UniqueId.hex(c2id)}.log\")\n      File.open(p2, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay3\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay4\"}].to_json + \"\\n\"\n      end\n      m2 = m.dup_next\n      write_metadata(p2 + '.meta', c2id, m2, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i)\n\n      c3id = Fluent::UniqueId.generate\n      p3 = File.join(@bufdir, \"testbuf.b#{Fluent::UniqueId.hex(c3id)}.log\")\n      File.open(p3, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay5\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay6\"}].to_json + \"\\n\"\n      end\n      m3 = m2.dup_next\n      write_metadata(p3 + '.meta', c3id, m3, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i)\n\n      c4id = Fluent::UniqueId.generate\n      p4 = File.join(@bufdir, \"testbuf.b#{Fluent::UniqueId.hex(c4id)}.log\")\n      File.open(p4, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay5\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay6\"}].to_json + \"\\n\"\n      end\n      write_metadata(p4 + '.meta', c4id, m3, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i)\n\n      Fluent::Test.setup\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @p = Fluent::Plugin::FileBuffer.new\n      @p.owner = @d\n      @p.configure(config_element('buffer', '', {'path' => @bufpath}))\n      @p.start\n    end\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n\n      if @bufdir\n        Dir.glob(File.join(@bufdir, '*')).each do |path|\n          next if ['.', '..'].include?(File.basename(path))\n          # Windows does not permit to delete files which are used in another process.\n          # Just ignore for removing failure.\n          File.delete(path) rescue nil\n        end\n      end\n    end\n\n    test '#resume returns each chunks' do\n      s, e = @p.resume\n      assert_equal 3, s.size\n      assert_equal [0, 1, 2], s.keys.map(&:seq).sort\n      assert_equal 1, e.size\n      assert_equal [0], e.map { |e| e.metadata.seq }\n    end\n  end\n\n  sub_test_case 'there are some non-buffer chunk files, with a path without buffer chunk ids' do\n    setup do\n      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)\n\n      FileUtils.rm_rf @bufdir\n      FileUtils.mkdir_p @bufdir\n\n      @c1id = Fluent::UniqueId.generate\n      p1 = File.join(@bufdir, \"etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log\")\n      File.open(p1, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:58:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:58:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 13:58:22 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      FileUtils.touch(p1, mtime: Time.parse('2016-04-17 13:58:28 -0700'))\n\n      @not_chunk = File.join(@bufdir, 'etest.20160416.log')\n      File.open(@not_chunk, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-16 23:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-16 23:58:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-16 23:58:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-16 23:58:22 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      FileUtils.touch(@not_chunk, mtime: Time.parse('2016-04-17 00:00:00 -0700'))\n\n      @bufpath = File.join(@bufdir, 'etest.*.log')\n\n      Fluent::Test.setup\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @p = Fluent::Plugin::FileBuffer.new\n      @p.owner = @d\n      @p.configure(config_element('buffer', '', {'path' => @bufpath}))\n      @p.start\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      if @bufdir\n        Dir.glob(File.join(@bufdir, '*')).each do |path|\n          next if ['.', '..'].include?(File.basename(path))\n          File.delete(path)\n        end\n      end\n    end\n\n    test '#resume returns queued chunks for files without metadata, while ignoring non-chunk looking files' do\n      assert_equal 0, @p.stage.size\n      assert_equal 1, @p.queue.size\n\n      queue = @p.queue\n\n      m = metadata()\n\n      assert_equal @c1id, queue[0].unique_id\n      assert_equal m, queue[0].metadata\n      assert_equal 0, queue[0].size\n      assert_equal :queued, queue[0].state\n      assert_equal Time.parse('2016-04-17 13:58:28 -0700'), queue[0].modified_at\n\n      assert File.exist?(@not_chunk)\n    end\n  end\n\n  sub_test_case 'there are existing broken file chunks' do\n    setup do\n      @id_output = 'backup_test'\n      @bufdir = File.expand_path('../../tmp/broken_buffer_file', __FILE__)\n      FileUtils.rm_rf @bufdir rescue nil\n      FileUtils.mkdir_p @bufdir\n      @bufpath = File.join(@bufdir, 'broken_test.*.log')\n\n      Fluent::Test.setup\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n    end\n\n    def setup_plugins(buf_conf)\n      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new\n      @d.configure(config_element('ROOT', '', {'@id' => @id_output}, [config_element('buffer', '', buf_conf)]))\n      @p = @d.buffer\n    end\n\n    def create_first_chunk(mode)\n      cid = Fluent::UniqueId.generate\n      path = File.join(@bufdir, \"broken_test.#{mode}#{Fluent::UniqueId.hex(cid)}.log\")\n      File.open(path, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 14:00:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 14:00:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata(\n        path + '.meta', cid, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),\n        4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i\n      )\n\n      return cid, path\n    end\n\n    def create_second_chunk(mode)\n      cid = Fluent::UniqueId.generate\n      path = File.join(@bufdir, \"broken_test.#{mode}#{Fluent::UniqueId.hex(cid)}.log\")\n      File.open(path, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:01:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 14:01:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 14:01:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      write_metadata(\n        path + '.meta', cid, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i),\n        3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i\n      )\n\n      return cid, path\n    end\n\n    def compare_staged_chunk(staged, id, time, num, mode)\n      assert_equal 1, staged.size\n      m = metadata(timekey: event_time(time).to_i)\n      assert_equal id, staged[m].unique_id\n      assert_equal num, staged[m].size\n      assert_equal mode, staged[m].state\n    end\n\n    def compare_queued_chunk(queued, id, num, mode)\n      assert_equal 1, queued.size\n      assert_equal id, queued[0].unique_id\n      assert_equal num, queued[0].size\n      assert_equal mode, queued[0].state\n    end\n\n    def compare_log(plugin, msg)\n      logs = plugin.log.out.logs\n      assert { logs.any? { |log| log.include?(msg) } }\n    end\n\n    test '#resume backups staged empty chunk' do\n      setup_plugins({'path' => @bufpath})\n      c1id, p1 = create_first_chunk('b')\n      File.open(p1, 'wb') { |f| } # create staged empty chunk file\n      c2id, _ = create_second_chunk('b')\n\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do\n        @p.start\n      end\n\n      compare_staged_chunk(@p.stage, c2id, '2016-04-17 14:01:00 -0700', 3, :staged)\n      compare_log(@p, 'staged file chunk is empty')\n      assert { not File.exist?(p1) }\n      assert { File.exist?(\"#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log\") }\n    end\n\n    test '#resume backups staged broken metadata' do\n      setup_plugins({'path' => @bufpath})\n      c1id, _ = create_first_chunk('b')\n      c2id, p2 = create_second_chunk('b')\n      File.open(p2 + '.meta', 'wb') { |f| f.write(\"\\0\" * 70) } # create staged broken meta file\n\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do\n        @p.start\n      end\n\n      compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)\n      compare_log(@p, 'staged meta file is broken')\n      assert { not File.exist?(p2) }\n      assert { File.exist?(\"#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log\") }\n    end\n\n    test '#resume backups enqueued empty chunk' do\n      setup_plugins({'path' => @bufpath})\n      c1id, p1 = create_first_chunk('q')\n      File.open(p1, 'wb') { |f| } # create enqueued empty chunk file\n      c2id, _ = create_second_chunk('q')\n\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do\n        @p.start\n      end\n\n      compare_queued_chunk(@p.queue, c2id, 3, :queued)\n      compare_log(@p, 'enqueued file chunk is empty')\n      assert { not File.exist?(p1) }\n      assert { File.exist?(\"#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log\") }\n    end\n\n    test '#resume backups enqueued broken metadata' do\n      setup_plugins({'path' => @bufpath})\n      c1id, _ = create_first_chunk('q')\n      c2id, p2 = create_second_chunk('q')\n      File.open(p2 + '.meta', 'wb') { |f| f.write(\"\\0\" * 70) } # create enqueued broken meta file\n\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do\n        @p.start\n      end\n\n      compare_queued_chunk(@p.queue, c1id, 4, :queued)\n      compare_log(@p, 'enqueued meta file is broken')\n      assert { not File.exist?(p2) }\n      assert { File.exist?(\"#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log\") }\n    end\n\n    test '#resume backups enqueued broken metadata which has broken id, c, m fields' do\n      setup_plugins({'path' => @bufpath})\n      cid, path = create_first_chunk('q')\n      metadata = File.read(path + '.meta')\n      File.open(path + '.meta', 'wb') { |f| f.write(metadata[0..6] + \"\\0\" * (metadata.size - 6)) } # create enqueued broken meta file\n\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do\n        @p.start\n      end\n\n      compare_log(@p, 'enqueued meta file is broken')\n      assert { not File.exist?(path) }\n      assert { File.exist?(\"#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(cid)}.log\") }\n    end\n\n    test '#resume backups enqueued broken metadata by truncated' do\n      setup_plugins({'path' => @bufpath})\n      cid, path = create_first_chunk('q')\n      metadata = File.read(path + '.meta')\n      File.open(path + '.meta', 'wb') { |f| f.write(metadata[0..-2]) } # create enqueued broken meta file with last byte truncated\n\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do\n        @p.start\n      end\n\n      compare_log(@p, 'enqueued meta file is broken')\n      assert { not File.exist?(path) }\n      assert { File.exist?(\"#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(cid)}.log\") }\n    end\n\n    test '#resume throws away broken chunk with disable_chunk_backup' do\n      setup_plugins({'path' => @bufpath, 'disable_chunk_backup' => true})\n      c1id, _ = create_first_chunk('b')\n      c2id, p2 = create_second_chunk('b')\n      File.open(p2 + '.meta', 'wb') { |f| f.write(\"\\0\" * 70) } # create staged broken meta file\n\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do\n        @p.start\n      end\n\n      compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)\n      compare_log(@p, 'staged meta file is broken')\n      compare_log(@p, 'disable_chunk_backup is true')\n      assert { not File.exist?(p2) }\n      assert { not File.exist?(\"#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log\") }\n    end\n  end\n\n  sub_test_case 'evacuate_chunk' do\n    def setup\n      Fluent::Test.setup\n\n      @now = Time.local(2025, 5, 30, 17, 0, 0)\n      @base_dir = File.expand_path(\"../../tmp/evacuate_chunk\", __FILE__)\n      @buf_dir = File.join(@base_dir, \"buffer\")\n      @root_dir = File.join(@base_dir, \"root\")\n      FileUtils.mkdir_p(@root_dir)\n\n      Fluent::SystemConfig.overwrite_system_config(\"root_dir\" => @root_dir) do\n        Timecop.freeze(@now)\n        yield\n      end\n    ensure\n      Timecop.return\n      FileUtils.rm_rf(@base_dir)\n    end\n\n    def start_plugin(plugin)\n      plugin.start\n      plugin.after_start\n    end\n\n    def stop_plugin(plugin)\n      plugin.stop unless plugin.stopped?\n      plugin.before_shutdown unless plugin.before_shutdown?\n      plugin.shutdown unless plugin.shutdown?\n      plugin.after_shutdown unless plugin.after_shutdown?\n      plugin.close unless plugin.closed?\n      plugin.terminate unless plugin.terminated?\n    end\n\n    def configure_output(id, chunk_key, buffer_conf)\n      output = FluentPluginFileBufferTest::DummyErrorOutputPlugin.new\n      output.configure(\n        config_element('ROOT', '', {'@id' => id}, [config_element('buffer', chunk_key, buffer_conf)])\n      )\n      yield output\n    ensure\n      stop_plugin(output)\n    end\n\n    def wait(sec: 4)\n      waiting(sec) do\n        Thread.pass until yield\n      end\n    end\n\n    def emit_events(output, tag, es)\n      output.interrupt_flushes\n      output.emit_events(\"test.1\", dummy_event_stream)\n      @now += 1\n      Timecop.freeze(@now)\n      output.enqueue_thread_wait\n      output.flush_thread_wakeup\n    end\n\n    def proceed_to_next_retry(output)\n      @now += 1\n      Timecop.freeze(@now)\n      output.flush_thread_wakeup\n    end\n\n    def dummy_event_stream\n      Fluent::ArrayEventStream.new([\n        [ event_time(\"2025-05-30 10:00:00\"), {\"message\" => \"data1\"} ],\n        [ event_time(\"2025-05-30 10:10:00\"), {\"message\" => \"data2\"} ],\n        [ event_time(\"2025-05-30 10:20:00\"), {\"message\" => \"data3\"} ],\n      ])\n    end\n\n    def evacuate_dir(plugin_id)\n      File.join(@root_dir, \"buffer\", plugin_id)\n    end\n\n    test 'can recover by putting back evacuated chunk files' do\n      plugin_id = \"test_output\"\n      tag = \"test.1\"\n      buffer_conf = {\n        \"path\" => @buf_dir,\n        \"flush_mode\" => \"interval\",\n        \"flush_interval\" => \"1s\",\n        \"retry_type\" => \"periodic\",\n        \"retry_max_times\" => 1,\n        \"retry_randomize\" => false,\n      }\n\n      # Fail flushing and reach retry limit\n      configure_output(plugin_id, \"tag\", buffer_conf) do |output|\n        start_plugin(output)\n\n        emit_events(output, tag, dummy_event_stream)\n        wait { output.write_count == 1 and output.num_errors == 1 }\n\n        proceed_to_next_retry(output)\n        wait { output.write_count == 2 and output.num_errors == 2 }\n        wait { Dir.empty?(@buf_dir) }\n\n        # Assert evacuated files\n        evacuated_files = Dir.children(evacuate_dir(plugin_id)).map do |child_name|\n          File.join(evacuate_dir(plugin_id), child_name)\n        end\n        assert { evacuated_files.size == 2 } # .log and .log.meta\n\n        # Put back evacuated chunk files for recovery\n        FileUtils.move(evacuated_files, @buf_dir)\n      end\n\n      # Restart plugin to load the chunk files that were put back\n      written_data = []\n      configure_output(plugin_id, \"tag\", buffer_conf) do |output|\n        output.recover\n        output.register_write do |chunk|\n          written_data << chunk.read\n        end\n        start_plugin(output)\n\n        wait { not written_data.empty? }\n      end\n\n      # Assert the recovery success\n      assert { written_data.length == 1 }\n\n      expected_records = []\n      dummy_event_stream.each do |(time, record)|\n        expected_records << [tag, time.to_i, record]\n      end\n\n      actual_records = StringIO.open(written_data.first) do |io|\n        io.each_line.map do |line|\n          JSON.parse(line)\n        end\n      end\n\n      assert_equal(expected_records, actual_records)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_buf_file_single.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/buf_file_single'\nrequire 'fluent/plugin/output'\nrequire 'fluent/unique_id'\nrequire 'fluent/system_config'\nrequire 'fluent/env'\nrequire 'fluent/test/driver/output'\n\nrequire 'msgpack'\n\nmodule FluentPluginFileSingleBufferTest\n  class DummyOutputPlugin < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('buf_file_single_test', self)\n    config_section :buffer do\n      config_set_default :@type, 'file_single'\n    end\n    def multi_workers_ready?\n      true\n    end\n    def write(chunk)\n      # drop\n    end\n  end\n\n  class DummyOutputMPPlugin < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('buf_file_single_mp_test', self)\n    config_section :buffer do\n      config_set_default :@type, 'file_single'\n    end\n    def formatted_to_msgpack_binary?\n      true\n    end\n    def multi_workers_ready?\n      true\n    end\n    def write(chunk)\n      # drop\n    end\n  end\n\n  class DummyErrorOutputPlugin < DummyOutputPlugin\n    def register_write(&block)\n      instance_variable_set(:@write, block)\n    end\n\n    def initialize\n      super\n      @should_fail_writing = true\n      @write = nil\n    end\n\n    def recover\n      @should_fail_writing = false\n    end\n\n    def write(chunk)\n      if @should_fail_writing\n        raise \"failed writing chunk\"\n      else\n        @write ? @write.call(chunk) : nil\n      end\n    end\n\n    def format(tag, time, record)\n      [tag, time.to_i, record].to_json + \"\\n\"\n    end\n  end\nend\n\nclass FileSingleBufferTest < Test::Unit::TestCase\n  def metadata(timekey: nil, tag: 'testing', variables: nil)\n    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n  end\n\n  PATH = File.expand_path('../../tmp/buffer_file_single_dir', __FILE__)\n  TAG_CONF = %[\n    <buffer tag>\n      @type file_single\n      path #{PATH}\n    </buffer>\n  ]\n  FIELD_CONF = %[\n    <buffer k>\n      @type file_single\n      path #{PATH}\n    </buffer>\n  ]\n\n  setup do\n    Fluent::Test.setup\n\n    @d = nil\n    @bufdir = PATH\n    FileUtils.rm_rf(@bufdir) rescue nil\n    FileUtils.mkdir_p(@bufdir)\n  end\n\n  teardown do\n    FileUtils.rm_rf(@bufdir) rescue nil\n  end\n\n  def create_driver(conf = TAG_CONF, klass = FluentPluginFileSingleBufferTest::DummyOutputPlugin)\n    Fluent::Test::Driver::Output.new(klass).configure(conf)\n  end\n\n  sub_test_case 'configuration' do\n    test 'path has \"fsb\" prefix and \"buf\" suffix by default' do\n      @d = create_driver\n      p = @d.instance.buffer\n      assert_equal File.join(@bufdir, 'fsb.*.buf'), p.path\n    end\n\n    data('text based chunk' => [FluentPluginFileSingleBufferTest::DummyOutputPlugin, :text],\n         'msgpack based chunk' => [FluentPluginFileSingleBufferTest::DummyOutputMPPlugin, :msgpack])\n    test 'detect chunk_format' do |param|\n      klass, expected = param\n      @d = create_driver(TAG_CONF, klass)\n      p = @d.instance.buffer\n      assert_equal expected, p.chunk_format\n    end\n\n    test '\"prefix.*.suffix\" path will be replaced with default' do\n      @d = create_driver(%[\n        <buffer tag>\n          @type file_single\n          path #{@bufdir}/foo.*.bar\n        </buffer>\n      ])\n      p = @d.instance.buffer\n      assert_equal File.join(@bufdir, 'fsb.*.buf'), p.path\n    end\n  end\n\n  sub_test_case 'buffer configurations and workers' do\n    setup do\n      @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new\n    end\n\n    test 'enables multi worker configuration with unexisting directory path' do\n      FileUtils.rm_rf(@bufdir)\n      buf_conf = config_element('buffer', '', {'path' => @bufdir})\n      assert_nothing_raised do\n        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do\n          @d.configure(config_element('ROOT', '', {}, [buf_conf]))\n        end\n      end\n    end\n\n    test 'enables multi worker configuration with existing directory path' do\n      FileUtils.mkdir_p @bufdir\n      buf_conf = config_element('buffer', '', {'path' => @bufdir})\n      assert_nothing_raised do\n        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do\n          @d.configure(config_element('ROOT', '', {}, [buf_conf]))\n        end\n      end\n    end\n\n    test 'enables multi worker configuration with root dir' do\n      buf_conf = config_element('buffer', '')\n      assert_nothing_raised do\n        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do\n          @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))\n        end\n      end\n    end\n  end\n\n  test 'raise config error when using same file path' do\n    d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new\n    d2 = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new\n    Fluent::SystemConfig.overwrite_system_config({}) do\n      d.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => File.join(PATH, 'foo.*.bar') })]))\n    end\n\n    any_instance_of(Fluent::Plugin::FileSingleBuffer) do |klass|\n      stub(klass).called_in_test? { false }\n    end\n\n    err = assert_raise(Fluent::ConfigError) do\n      Fluent::SystemConfig.overwrite_system_config({}) do\n        d2.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => PATH })]))\n      end\n    end\n    assert_match(/plugin already uses same buffer path/, err.message)\n  end\n\n  sub_test_case 'buffer plugin configured only with path' do\n    setup do\n      @bufpath = File.join(@bufdir, 'testbuf.*.buf')\n      FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)\n\n      @d = create_driver\n      @p = @d.instance.buffer\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n    end\n\n    test 'this is persistent plugin' do\n      assert @p.persistent?\n    end\n\n    test '#start creates directory for buffer chunks' do\n      @d = create_driver\n      @p = @d.instance.buffer\n\n      FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)\n      assert !File.exist?(@bufdir)\n\n      @p.start\n      assert File.exist?(@bufdir)\n      assert { File.stat(@bufdir).mode.to_s(8).end_with?('755') }\n    end\n\n    test '#start creates directory for buffer chunks with specified permission' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      @d = create_driver(%[\n        <buffer tag>\n          @type file_single\n          path #{PATH}\n          dir_permission 700\n        </buffer>\n      ])\n      @p = @d.instance.buffer\n\n      FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)\n      assert !File.exist?(@bufdir)\n\n      @p.start\n      assert File.exist?(@bufdir)\n      assert { File.stat(@bufdir).mode.to_s(8).end_with?('700') }\n    end\n\n    test '#start creates directory for buffer chunks with specified permission via system config' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      sysconf = {'dir_permission' => '700'}\n      Fluent::SystemConfig.overwrite_system_config(sysconf) do\n        @d = create_driver\n        @p = @d.instance.buffer\n\n        FileUtils.rm_r @bufdir if File.exist?(@bufdir)\n        assert !File.exist?(@bufdir)\n\n        @p.start\n        assert File.exist?(@bufdir)\n        assert { File.stat(@bufdir).mode.to_s(8).end_with?('700') }\n      end\n    end\n\n    test '#generate_chunk generates blank file chunk on path with unique_id and tag' do\n      FileUtils.mkdir_p(@bufdir) unless File.exist?(@bufdir)\n\n      m1 = metadata()\n      c1 = @p.generate_chunk(m1)\n      assert c1.is_a? Fluent::Plugin::Buffer::FileSingleChunk\n      assert_equal m1, c1.metadata\n      assert c1.empty?\n      assert_equal :unstaged, c1.state\n      assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission\n      assert_equal File.join(@bufdir, \"fsb.testing.b#{Fluent::UniqueId.hex(c1.unique_id)}.buf\"), c1.path\n      assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') }\n\n      c1.purge\n    end\n\n    test '#generate_chunk generates blank file chunk on path with unique_id and field key' do\n      FileUtils.mkdir_p(@bufdir) unless File.exist?(@bufdir)\n\n      @d = create_driver(FIELD_CONF)\n      @p = @d.instance.buffer\n\n      m1 = metadata(tag: nil, variables: {:k => 'foo_bar'})\n      c1 = @p.generate_chunk(m1)\n      assert c1.is_a? Fluent::Plugin::Buffer::FileSingleChunk\n      assert_equal m1, c1.metadata\n      assert c1.empty?\n      assert_equal :unstaged, c1.state\n      assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission\n      assert_equal File.join(@bufdir, \"fsb.foo_bar.b#{Fluent::UniqueId.hex(c1.unique_id)}.buf\"), c1.path\n\n      c1.purge\n    end\n\n    test '#generate_chunk generates blank file chunk with specified permission' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      @d = create_driver(%[\n        <buffer tag>\n          @type file_single\n          path #{PATH}\n          file_permission 600\n        </buffer>\n      ])\n      @p = @d.instance.buffer\n\n      FileUtils.rm_r @bufdir if File.exist?(@bufdir)\n      assert !File.exist?(@bufdir)\n\n      @p.start\n\n      m = metadata()\n      c = @p.generate_chunk(m)\n      assert c.is_a? Fluent::Plugin::Buffer::FileSingleChunk\n      assert_equal m, c.metadata\n      assert c.empty?\n      assert_equal :unstaged, c.state\n      assert_equal 0600, c.permission\n      assert_equal File.join(@bufdir, \"fsb.testing.b#{Fluent::UniqueId.hex(c.unique_id)}.buf\"), c.path\n      assert{ File.stat(c.path).mode.to_s(8).end_with?('600') }\n\n      c.purge\n    end\n\n    test '#generate_chunk generates blank file chunk with specified permission with system_config' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      @d = create_driver(%[\n        <buffer tag>\n          @type file_single\n          path #{PATH}\n        </buffer>\n      ])\n      @p = @d.instance.buffer\n\n      FileUtils.rm_r @bufdir if File.exist?(@bufdir)\n      assert !File.exist?(@bufdir)\n\n      @p.start\n\n      m = metadata()\n      c = nil\n      Fluent::SystemConfig.overwrite_system_config(\"file_permission\" => \"700\") do\n        c = @p.generate_chunk(m)\n      end\n      assert c.is_a? Fluent::Plugin::Buffer::FileSingleChunk\n      assert_equal m, c.metadata\n      assert c.empty?\n      assert_equal :unstaged, c.state\n      assert_equal 0700, c.permission\n      assert_equal File.join(@bufdir, \"fsb.testing.b#{Fluent::UniqueId.hex(c.unique_id)}.buf\"), c.path\n      assert{ File.stat(c.path).mode.to_s(8).end_with?('700') }\n\n      c.purge\n    end\n  end\n\n  sub_test_case 'configured with system root directory and plugin @id' do\n    setup do\n      @root_dir = File.expand_path('../../tmp/buffer_file_single_root', __FILE__)\n      FileUtils.rm_rf(@root_dir)\n\n      @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new\n      @p = nil\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n    end\n\n    test '#start creates directory for buffer chunks' do\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do\n        @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [config_element('buffer', '', {})]))\n        @p = @d.buffer\n      end\n\n      expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', \"fsb.*.buf\")\n      expected_buffer_dir = File.dirname(expected_buffer_path)\n      assert_equal expected_buffer_path, @d.buffer.path\n      assert_false Dir.exist?(expected_buffer_dir)\n\n      @p.start\n      assert Dir.exist?(expected_buffer_dir)\n    end\n  end\n\n  sub_test_case 'buffer plugin configuration errors' do\n    data('tag and key' => 'tag,key',\n         'multiple keys' => 'key1,key2')\n    test 'invalid chunk keys' do |param|\n      assert_raise Fluent::ConfigError do\n        @d = create_driver(%[\n          <buffer #{param}>\n            @type file_single\n            path #{PATH}\n            calc_num_records false\n          </buffer>\n        ])\n      end\n    end\n\n    test 'path is not specified' do\n      assert_raise Fluent::ConfigError do\n        @d = create_driver(%[\n          <buffer tag>\n            @type file_single\n          </buffer>\n        ])\n      end\n    end\n  end\n\n  sub_test_case 'there are no existing file chunks' do\n    setup do\n      FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)\n\n      @d = create_driver\n      @p = @d.instance.buffer\n      @p.start\n    end\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      if @bufdir\n        Dir.glob(File.join(@bufdir, '*')).each do |path|\n          next if ['.', '..'].include?(File.basename(path))\n          File.delete(path)\n        end\n      end\n    end\n\n    test '#resume returns empty buffer state' do\n      ary = @p.resume\n      assert_equal({}, ary[0])\n      assert_equal([], ary[1])\n    end\n  end\n\n  sub_test_case 'there are some existing file chunks' do\n    setup do\n      @c1id = Fluent::UniqueId.generate\n      p1 = File.join(@bufdir, \"fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf\")\n      File.open(p1, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:58:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:58:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 13:58:22 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      t = Time.now - 50000\n      File.utime(t, t, p1)\n\n      @c2id = Fluent::UniqueId.generate\n      p2 = File.join(@bufdir, \"fsb.testing.q#{Fluent::UniqueId.hex(@c2id)}.buf\")\n      File.open(p2, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:59:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:59:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:59:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      t = Time.now - 40000\n      File.utime(t, t, p2)\n\n      @c3id = Fluent::UniqueId.generate\n      p3 = File.join(@bufdir, \"fsb.testing.b#{Fluent::UniqueId.hex(@c3id)}.buf\")\n      File.open(p3, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 14:00:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 14:00:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n\n      @c4id = Fluent::UniqueId.generate\n      p4 = File.join(@bufdir, \"fsb.foo.b#{Fluent::UniqueId.hex(@c4id)}.buf\")\n      File.open(p4, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:01:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 14:01:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 14:01:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      if @bufdir\n        Dir.glob(File.join(@bufdir, '*')).each do |path|\n          next if ['.', '..'].include?(File.basename(path))\n          File.delete(path)\n        end\n      end\n    end\n\n    test '#resume returns staged/queued chunks with metadata' do\n      @d = create_driver\n      @p = @d.instance.buffer\n      @p.start\n\n      assert_equal 2, @p.stage.size\n      assert_equal 2, @p.queue.size\n\n      stage = @p.stage\n\n      m3 = metadata()\n      assert_equal @c3id, stage[m3].unique_id\n      assert_equal 4, stage[m3].size\n      assert_equal :staged, stage[m3].state\n\n      m4 = metadata(tag: 'foo')\n      assert_equal @c4id, stage[m4].unique_id\n      assert_equal 3, stage[m4].size\n      assert_equal :staged, stage[m4].state\n    end\n\n    test '#resume returns queued chunks ordered by last modified time (FIFO)' do\n      @d = create_driver\n      @p = @d.instance.buffer\n      @p.start\n\n      assert_equal 2, @p.stage.size\n      assert_equal 2, @p.queue.size\n\n      queue = @p.queue\n\n      assert{ queue[0].modified_at <= queue[1].modified_at }\n\n      assert_equal @c1id, queue[0].unique_id\n      assert_equal :queued, queue[0].state\n      assert_equal 'testing', queue[0].metadata.tag\n      assert_nil queue[0].metadata.variables\n      assert_equal 4, queue[0].size\n\n      assert_equal @c2id, queue[1].unique_id\n      assert_equal :queued, queue[1].state\n      assert_equal 'testing', queue[1].metadata.tag\n      assert_nil queue[1].metadata.variables\n      assert_equal 3, queue[1].size\n    end\n\n    test '#resume returns staged/queued chunks but skips size calculation by calc_num_records' do\n      @d = create_driver(%[\n        <buffer tag>\n         @type file_single\n         path #{PATH}\n         calc_num_records false\n        </buffer>\n      ])\n      @p = @d.instance.buffer\n      @p.start\n\n      assert_equal 2, @p.stage.size\n      assert_equal 2, @p.queue.size\n\n      stage = @p.stage\n\n      m3 = metadata()\n      assert_equal @c3id, stage[m3].unique_id\n      assert_equal 0, stage[m3].size\n      assert_equal :staged, stage[m3].state\n\n      m4 = metadata(tag: 'foo')\n      assert_equal @c4id, stage[m4].unique_id\n      assert_equal 0, stage[m4].size\n      assert_equal :staged, stage[m4].state\n    end\n  end\n\n  sub_test_case 'there are some existing file chunks with placeholders path' do\n    setup do\n      @buf_ph_dir = File.expand_path('../../tmp/buffer_${test}_file_single_dir', __FILE__)\n      FileUtils.rm_rf(@buf_ph_dir)\n      FileUtils.mkdir_p(@buf_ph_dir)\n\n      @c1id = Fluent::UniqueId.generate\n      p1 = File.join(@buf_ph_dir, \"fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf\")\n      File.open(p1, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n      t = Time.now - 50000\n      File.utime(t, t, p1)\n\n      @c2id = Fluent::UniqueId.generate\n      p2 = File.join(@buf_ph_dir, \"fsb.testing.b#{Fluent::UniqueId.hex(@c2id)}.buf\")\n      File.open(p2, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      FileUtils.rm_rf(@buf_ph_dir)\n    end\n\n    test '#resume returns staged/queued chunks with metadata' do\n      @d = create_driver(%[\n        <buffer tag>\n          @type file_single\n          path #{@buf_ph_dir}\n        </buffer>\n      ])\n      @p = @d.instance.buffer\n      @p.start\n\n      assert_equal 1, @p.stage.size\n      assert_equal 1, @p.queue.size\n    end\n  end\n\n  sub_test_case 'there are some existing msgpack file chunks' do\n    setup do\n      packer = Fluent::MessagePackFactory.packer\n      @c1id = Fluent::UniqueId.generate\n      p1 = File.join(@bufdir, \"fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf\")\n      File.open(p1, 'wb') do |f|\n        packer.write([\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}])\n        packer.write([\"t2.test\", event_time('2016-04-17 13:58:17 -0700').to_i, {\"message\" => \"yay\"}])\n        packer.write([\"t3.test\", event_time('2016-04-17 13:58:21 -0700').to_i, {\"message\" => \"yay\"}])\n        packer.write([\"t4.test\", event_time('2016-04-17 13:58:22 -0700').to_i, {\"message\" => \"yay\"}])\n        f.write packer.full_pack\n      end\n      t = Time.now - 50000\n      File.utime(t, t, p1)\n\n      @c2id = Fluent::UniqueId.generate\n      p2 = File.join(@bufdir, \"fsb.testing.q#{Fluent::UniqueId.hex(@c2id)}.buf\")\n      File.open(p2, 'wb') do |f|\n        packer.write([\"t1.test\", event_time('2016-04-17 13:59:15 -0700').to_i, {\"message\" => \"yay\"}])\n        packer.write([\"t2.test\", event_time('2016-04-17 13:59:17 -0700').to_i, {\"message\" => \"yay\"}])\n        packer.write([\"t3.test\", event_time('2016-04-17 13:59:21 -0700').to_i, {\"message\" => \"yay\"}])\n        f.write packer.full_pack\n      end\n      t = Time.now - 40000\n      File.utime(t, t, p2)\n\n      @c3id = Fluent::UniqueId.generate\n      p3 = File.join(@bufdir, \"fsb.testing.b#{Fluent::UniqueId.hex(@c3id)}.buf\")\n      File.open(p3, 'wb') do |f|\n        packer.write([\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay\"}])\n        packer.write([\"t2.test\", event_time('2016-04-17 14:00:17 -0700').to_i, {\"message\" => \"yay\"}])\n        packer.write([\"t3.test\", event_time('2016-04-17 14:00:21 -0700').to_i, {\"message\" => \"yay\"}])\n        packer.write([\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay\"}])\n        f.write packer.full_pack\n      end\n\n      @c4id = Fluent::UniqueId.generate\n      p4 = File.join(@bufdir, \"fsb.foo.b#{Fluent::UniqueId.hex(@c4id)}.buf\")\n      File.open(p4, 'wb') do |f|\n        packer.write([\"t1.test\", event_time('2016-04-17 14:01:15 -0700').to_i, {\"message\" => \"yay\"}])\n        packer.write([\"t2.test\", event_time('2016-04-17 14:01:17 -0700').to_i, {\"message\" => \"yay\"}])\n        packer.write([\"t3.test\", event_time('2016-04-17 14:01:21 -0700').to_i, {\"message\" => \"yay\"}])\n        f.write packer.full_pack\n      end\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n      if @bufdir\n        Dir.glob(File.join(@bufdir, '*')).each do |path|\n          next if ['.', '..'].include?(File.basename(path))\n          File.delete(path)\n        end\n      end\n    end\n\n    test '#resume returns staged/queued chunks with msgpack format' do\n      @d = create_driver(%[\n        <buffer tag>\n         @type file_single\n         path #{PATH}\n         chunk_format msgpack\n        </buffer>\n      ])\n      @p = @d.instance.buffer\n      @p.start\n\n      assert_equal 2, @p.stage.size\n      assert_equal 2, @p.queue.size\n\n      stage = @p.stage\n\n      m3 = metadata()\n      assert_equal @c3id, stage[m3].unique_id\n      assert_equal 4, stage[m3].size\n      assert_equal :staged, stage[m3].state\n\n      m4 = metadata(tag: 'foo')\n      assert_equal @c4id, stage[m4].unique_id\n      assert_equal 3, stage[m4].size\n      assert_equal :staged, stage[m4].state\n    end\n  end\n\n  sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do\n    setup do\n      @worker0_dir = File.join(@bufdir, \"worker0\")\n      @worker1_dir = File.join(@bufdir, \"worker1\")\n      FileUtils.rm_rf(@bufdir)\n      FileUtils.mkdir_p(@worker0_dir)\n      FileUtils.mkdir_p(@worker1_dir)\n\n      @bufdir_chunk_1 = Fluent::UniqueId.generate\n      bc1 = File.join(@bufdir, \"fsb.testing.q#{Fluent::UniqueId.hex(@bufdir_chunk_1)}.buf\")\n      File.open(bc1, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:58:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:58:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 13:58:22 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n\n      @bufdir_chunk_2 = Fluent::UniqueId.generate\n      bc2 = File.join(@bufdir, \"fsb.testing.q#{Fluent::UniqueId.hex(@bufdir_chunk_2)}.buf\")\n      File.open(bc2, 'wb') do |f|\n        f.write [\"t1.test\", event_time('2016-04-17 13:58:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t2.test\", event_time('2016-04-17 13:58:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t3.test\", event_time('2016-04-17 13:58:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        f.write [\"t4.test\", event_time('2016-04-17 13:58:22 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n      end\n\n      @worker_dir_chunk_1 = Fluent::UniqueId.generate\n      wc0_1 = File.join(@worker0_dir, \"fsb.testing.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.buf\")\n      wc1_1 = File.join(@worker1_dir, \"fsb.testing.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.buf\")\n      [wc0_1, wc1_1].each do |chunk_path|\n        File.open(chunk_path, 'wb') do |f|\n          f.write [\"t1.test\", event_time('2016-04-17 13:59:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t2.test\", event_time('2016-04-17 13:59:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t3.test\", event_time('2016-04-17 13:59:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        end\n      end\n\n      @worker_dir_chunk_2 = Fluent::UniqueId.generate\n      wc0_2 = File.join(@worker0_dir, \"fsb.testing.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.buf\")\n      wc1_2 = File.join(@worker1_dir, \"fsb.foo.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.buf\")\n      [wc0_2, wc1_2].each do |chunk_path|\n        File.open(chunk_path, 'wb') do |f|\n          f.write [\"t1.test\", event_time('2016-04-17 14:00:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t2.test\", event_time('2016-04-17 14:00:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t3.test\", event_time('2016-04-17 14:00:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t4.test\", event_time('2016-04-17 14:00:28 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        end\n      end\n\n      @worker_dir_chunk_3 = Fluent::UniqueId.generate\n      wc0_3 = File.join(@worker0_dir, \"fsb.bar.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.buf\")\n      wc1_3 = File.join(@worker1_dir, \"fsb.baz.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.buf\")\n      [wc0_3, wc1_3].each do |chunk_path|\n        File.open(chunk_path, 'wb') do |f|\n          f.write [\"t1.test\", event_time('2016-04-17 14:01:15 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t2.test\", event_time('2016-04-17 14:01:17 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n          f.write [\"t3.test\", event_time('2016-04-17 14:01:21 -0700').to_i, {\"message\" => \"yay\"}].to_json + \"\\n\"\n        end\n      end\n    end\n\n    teardown do\n      if @p\n        @p.stop unless @p.stopped?\n        @p.before_shutdown unless @p.before_shutdown?\n        @p.shutdown unless @p.shutdown?\n        @p.after_shutdown unless @p.after_shutdown?\n        @p.close unless @p.closed?\n        @p.terminate unless @p.terminated?\n      end\n    end\n\n    test 'worker(id=0) #resume returns staged/queued chunks with metadata, not only in worker dir, including the directory specified by path' do\n      ENV['SERVERENGINE_WORKER_ID'] = '0'\n\n      buf_conf = config_element('buffer', '', {'path' => @bufdir})\n      @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new\n      with_worker_config(workers: 2, worker_id: 0) do\n        @d.configure(config_element('output', '', {}, [buf_conf]))\n      end\n\n      @d.start\n      @p = @d.buffer\n\n      assert_equal 2, @p.stage.size\n      assert_equal 3, @p.queue.size\n\n      stage = @p.stage\n\n      m1 = metadata(tag: 'testing')\n      assert_equal @worker_dir_chunk_2, stage[m1].unique_id\n      assert_equal 4, stage[m1].size\n      assert_equal :staged, stage[m1].state\n\n      m2 = metadata(tag: 'bar')\n      assert_equal @worker_dir_chunk_3, stage[m2].unique_id\n      assert_equal 3, stage[m2].size\n      assert_equal :staged, stage[m2].state\n\n      queue = @p.queue\n\n      assert_equal [@bufdir_chunk_1, @bufdir_chunk_2, @worker_dir_chunk_1].sort, queue.map(&:unique_id).sort\n      assert_equal [3, 4, 4], queue.map(&:size).sort\n      assert_equal [:queued, :queued, :queued], queue.map(&:state)\n    end\n\n    test 'worker(id=1) #resume returns staged/queued chunks with metadata, only in worker dir' do\n      buf_conf = config_element('buffer', '', {'path' => @bufdir})\n      @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new\n      with_worker_config(workers: 2, worker_id: 1) do\n        @d.configure(config_element('output', '', {}, [buf_conf]))\n      end\n\n      @d.start\n      @p = @d.buffer\n\n      assert_equal 2, @p.stage.size\n      assert_equal 1, @p.queue.size\n\n      stage = @p.stage\n\n      m1 = metadata(tag: 'foo')\n      assert_equal @worker_dir_chunk_2, stage[m1].unique_id\n      assert_equal 4, stage[m1].size\n      assert_equal :staged, stage[m1].state\n\n      m2 = metadata(tag: 'baz')\n      assert_equal @worker_dir_chunk_3, stage[m2].unique_id\n      assert_equal 3, stage[m2].size\n      assert_equal :staged, stage[m2].state\n\n      queue = @p.queue\n\n      assert_equal @worker_dir_chunk_1, queue[0].unique_id\n      assert_equal 3, queue[0].size\n      assert_equal :queued, queue[0].state\n    end\n  end\n\n  sub_test_case 'there are existing broken file chunks' do\n    setup do\n      FileUtils.rm_rf(@bufdir) rescue nil\n      FileUtils.mkdir_p(@bufdir)\n    end\n\n    teardown do\n      return unless @p\n\n      @p.stop unless @p.stopped?\n      @p.before_shutdown unless @p.before_shutdown?\n      @p.shutdown unless @p.shutdown?\n      @p.after_shutdown unless @p.after_shutdown?\n      @p.close unless @p.closed?\n      @p.terminate unless @p.terminated?\n    end\n\n    test '#resume backups empty chunk' do\n      id_output = 'backup_test'\n      @d = create_driver(%[\n        @id #{id_output}\n        <buffer tag>\n         @type file_single\n         path #{PATH}\n        </buffer>\n      ])\n      @p = @d.instance.buffer\n\n      c1id = Fluent::UniqueId.generate\n      p1 = File.join(@bufdir, \"fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf\")\n      File.open(p1, 'wb') { |f| } # create empty chunk file\n\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do\n        @p.start\n      end\n\n      assert { not File.exist?(p1) }\n      assert { File.exist?(\"#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log\") }\n    end\n\n    test '#resume throws away broken chunk with disable_chunk_backup' do\n      id_output = 'backup_test'\n      @d = create_driver(%[\n        @id #{id_output}\n        <buffer tag>\n         @type file_single\n         path #{PATH}\n         disable_chunk_backup true\n        </buffer>\n      ])\n      @p = @d.instance.buffer\n\n      c1id = Fluent::UniqueId.generate\n      p1 = File.join(@bufdir, \"fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf\")\n      File.open(p1, 'wb') { |f| } # create empty chunk file\n\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do\n        @p.start\n      end\n\n      assert { not File.exist?(p1) }\n      assert { not File.exist?(\"#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log\") }\n    end\n  end\n\n  sub_test_case 'evacuate_chunk' do\n    def setup\n      Fluent::Test.setup\n\n      @now = Time.local(2025, 5, 30, 17, 0, 0)\n      @base_dir = File.expand_path(\"../../tmp/evacuate_chunk\", __FILE__)\n      @buf_dir = File.join(@base_dir, \"buffer\")\n      @root_dir = File.join(@base_dir, \"root\")\n      FileUtils.mkdir_p(@root_dir)\n\n      Fluent::SystemConfig.overwrite_system_config(\"root_dir\" => @root_dir) do\n        Timecop.freeze(@now)\n        yield\n      end\n    ensure\n      Timecop.return\n      FileUtils.rm_rf(@base_dir)\n    end\n\n    def start_plugin(plugin)\n      plugin.start\n      plugin.after_start\n    end\n\n    def stop_plugin(plugin)\n      plugin.stop unless plugin.stopped?\n      plugin.before_shutdown unless plugin.before_shutdown?\n      plugin.shutdown unless plugin.shutdown?\n      plugin.after_shutdown unless plugin.after_shutdown?\n      plugin.close unless plugin.closed?\n      plugin.terminate unless plugin.terminated?\n    end\n\n    def configure_output(id, chunk_key, buffer_conf)\n      output = FluentPluginFileSingleBufferTest::DummyErrorOutputPlugin.new\n      output.configure(\n        config_element('ROOT', '', {'@id' => id}, [config_element('buffer', chunk_key, buffer_conf)])\n      )\n      yield output\n    ensure\n      stop_plugin(output)\n    end\n\n    def wait(sec: 4)\n      waiting(sec) do\n        Thread.pass until yield\n      end\n    end\n\n    def emit_events(output, tag, es)\n      output.interrupt_flushes\n      output.emit_events(\"test.1\", dummy_event_stream)\n      @now += 1\n      Timecop.freeze(@now)\n      output.enqueue_thread_wait\n      output.flush_thread_wakeup\n    end\n\n    def proceed_to_next_retry(output)\n      @now += 1\n      Timecop.freeze(@now)\n      output.flush_thread_wakeup\n    end\n\n    def dummy_event_stream\n      Fluent::ArrayEventStream.new([\n        [ event_time(\"2025-05-30 10:00:00\"), {\"message\" => \"data1\"} ],\n        [ event_time(\"2025-05-30 10:10:00\"), {\"message\" => \"data2\"} ],\n        [ event_time(\"2025-05-30 10:20:00\"), {\"message\" => \"data3\"} ],\n      ])\n    end\n\n    def evacuate_dir(plugin_id)\n      File.join(@root_dir, \"buffer\", plugin_id)\n    end\n\n    test 'can recover by putting back evacuated chunk files' do\n      plugin_id = \"test_output\"\n      tag = \"test.1\"\n      buffer_conf = {\n        \"path\" => @buf_dir,\n        \"flush_mode\" => \"interval\",\n        \"flush_interval\" => \"1s\",\n        \"retry_type\" => \"periodic\",\n        \"retry_max_times\" => 1,\n        \"retry_randomize\" => false,\n      }\n\n      # Fail flushing and reach retry limit\n      configure_output(plugin_id, \"tag\", buffer_conf) do |output|\n        start_plugin(output)\n\n        emit_events(output, tag, dummy_event_stream)\n        wait { output.write_count == 1 and output.num_errors == 1 }\n\n        proceed_to_next_retry(output)\n        wait { output.write_count == 2 and output.num_errors == 2 }\n        wait { Dir.empty?(@buf_dir) }\n\n        # Assert evacuated files\n        evacuated_files = Dir.children(evacuate_dir(plugin_id)).map do |child_name|\n          File.join(evacuate_dir(plugin_id), child_name)\n        end\n        assert { evacuated_files.size == 1 } # .log\n\n        # Put back evacuated chunk files for recovery\n        FileUtils.move(evacuated_files, @buf_dir)\n      end\n\n      # Restart plugin to load the chunk files that were put back\n      written_data = []\n      configure_output(plugin_id, \"tag\", buffer_conf) do |output|\n        output.recover\n        output.register_write do |chunk|\n          written_data << chunk.read\n        end\n        start_plugin(output)\n\n        wait { not written_data.empty? }\n      end\n\n      # Assert the recovery success\n      assert { written_data.length == 1 }\n\n      expected_records = []\n      dummy_event_stream.each do |(time, record)|\n        expected_records << [tag, time.to_i, record]\n      end\n\n      actual_records = StringIO.open(written_data.first) do |io|\n        io.each_line.map do |line|\n          JSON.parse(line)\n        end\n      end\n\n      assert_equal(expected_records, actual_records)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_buf_memory.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/buf_memory'\nrequire 'fluent/plugin/output'\nrequire 'flexmock/test_unit'\n\nmodule FluentPluginMemoryBufferTest\n  class DummyOutputPlugin < Fluent::Plugin::Output\n  end\nend\n\nclass MemoryBufferTest < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n    @d = FluentPluginMemoryBufferTest::DummyOutputPlugin.new\n    @p = Fluent::Plugin::MemoryBuffer.new\n    @p.owner = @d\n  end\n\n  test 'this is non persistent plugin' do\n    assert !@p.persistent?\n  end\n\n  test '#resume always returns empty stage and queue' do\n    ary = @p.resume\n    assert_equal({}, ary[0])\n    assert_equal([], ary[1])\n  end\n\n  test '#generate_chunk returns memory chunk instance' do\n    m1 = Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil)\n    c1 = @p.generate_chunk(m1)\n    assert c1.is_a? Fluent::Plugin::Buffer::MemoryChunk\n    assert_equal m1, c1.metadata\n\n    require 'time'\n    t2 = Time.parse('2016-04-08 19:55:00 +0900').to_i\n    m2 = Fluent::Plugin::Buffer::Metadata.new(t2, 'test.tag', {k1: 'v1', k2: 0})\n    c2 = @p.generate_chunk(m2)\n    assert c2.is_a? Fluent::Plugin::Buffer::MemoryChunk\n    assert_equal m2, c2.metadata\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_buffer.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin/buffer/memory_chunk'\nrequire 'fluent/plugin/compressable'\nrequire 'fluent/plugin/buffer/chunk'\nrequire 'fluent/event'\nrequire 'flexmock/test_unit'\n\nrequire 'fluent/log'\nrequire 'fluent/plugin_id'\n\nrequire 'time'\n\nmodule FluentPluginBufferTest\n  class DummyOutputPlugin < Fluent::Plugin::Base\n    include Fluent::PluginId\n    include Fluent::PluginLoggerMixin\n  end\n  class DummyMemoryChunkError < StandardError; end\n  class DummyMemoryChunk < Fluent::Plugin::Buffer::MemoryChunk\n    attr_reader :append_count, :rollbacked, :closed, :purged, :chunk\n    attr_accessor :failing\n    def initialize(metadata, compress: :text)\n      super\n      @append_count = 0\n      @rollbacked = false\n      @closed = false\n      @purged = false\n      @failing = false\n    end\n    def concat(data, size)\n      @append_count += 1\n      raise DummyMemoryChunkError if @failing\n      super\n    end\n    def rollback\n      super\n      @rollbacked = true\n    end\n    def close\n      super\n      @closed = true\n    end\n    def purge\n      super\n      @purged = true\n    end\n  end\n  class DummyPlugin < Fluent::Plugin::Buffer\n    def create_metadata(timekey=nil, tag=nil, variables=nil)\n      Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n    end\n    def create_chunk(metadata, data)\n      c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata)\n      c.append(data)\n      c.commit\n      c\n    end\n    def create_chunk_es(metadata, es)\n      c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata)\n      c.concat(es.to_msgpack_stream, es.size)\n      c.commit\n      c\n    end\n    def resume\n      dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)\n      dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)\n      dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)\n      dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil)\n      staged = {\n        dm2 => create_chunk(dm2, [\"b\" * 100]).staged!,\n        dm3 => create_chunk(dm3, [\"c\" * 100]).staged!,\n      }\n      queued = [\n        create_chunk(dm0, [\"0\" * 100]).enqueued!,\n        create_chunk(dm1, [\"a\" * 100]).enqueued!,\n        create_chunk(dm1, [\"a\" * 3]).enqueued!,\n      ]\n      return staged, queued\n    end\n    def generate_chunk(metadata)\n      DummyMemoryChunk.new(metadata, compress: @compress)\n    end\n  end\nend\n\nclass BufferTest < Test::Unit::TestCase\n  def create_buffer(hash)\n    buffer_conf = config_element('buffer', '', hash, [])\n    owner = FluentPluginBufferTest::DummyOutputPlugin.new\n    owner.configure(config_element('ROOT', '', {}, [ buffer_conf ]))\n    p = FluentPluginBufferTest::DummyPlugin.new\n    p.owner = owner\n    p.configure(buffer_conf)\n    p\n  end\n\n  def create_metadata(timekey=nil, tag=nil, variables=nil)\n    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n  end\n\n  def create_chunk(metadata, data)\n    c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata)\n    c.append(data)\n    c.commit\n    c\n  end\n\n  def create_chunk_es(metadata, es)\n    c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata)\n    c.concat(es.to_msgpack_stream, es.size)\n    c.commit\n    c\n  end\n\n  setup do\n    Fluent::Test.setup\n  end\n\n  sub_test_case 'using base buffer class' do\n    setup do\n      buffer_conf = config_element('buffer', '', {}, [])\n      owner = FluentPluginBufferTest::DummyOutputPlugin.new\n      owner.configure(config_element('ROOT', '', {}, [ buffer_conf ]))\n      p = Fluent::Plugin::Buffer.new\n      p.owner = owner\n      p.configure(buffer_conf)\n      @p = p\n    end\n\n    test 'default persistency is false' do\n      assert !@p.persistent?\n    end\n\n    test 'chunk bytes limit is 8MB, and total bytes limit is 512MB' do\n      assert_equal 8*1024*1024, @p.chunk_limit_size\n      assert_equal 512*1024*1024, @p.total_limit_size\n    end\n\n    test 'chunk records limit is ignored in default' do\n      assert_nil @p.chunk_limit_records\n    end\n\n    test '#storable? checks total size of staged and enqueued(includes dequeued chunks) against total_limit_size' do\n      assert_equal 512*1024*1024, @p.total_limit_size\n      assert_equal 0, @p.stage_size\n      assert_equal 0, @p.queue_size\n      assert @p.storable?\n\n      @p.stage_size = 256 * 1024 * 1024\n      @p.queue_size = 256 * 1024 * 1024 - 1\n      assert @p.storable?\n\n      @p.queue_size = 256 * 1024 * 1024\n      assert !@p.storable?\n    end\n\n    test '#resume must be implemented by subclass' do\n      assert_raise NotImplementedError do\n        @p.resume\n      end\n    end\n\n    test '#generate_chunk must be implemented by subclass' do\n      assert_raise NotImplementedError do\n        @p.generate_chunk(Object.new)\n      end\n    end\n  end\n\n  sub_test_case 'with default configuration and dummy implementation' do\n    setup do\n      @p = create_buffer({'queued_chunks_limit_size' => 100})\n      @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)\n      @dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)\n      @dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)\n      @dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil)\n      @p.start\n    end\n\n    test '#start resumes buffer states and update queued numbers per metadata' do\n      plugin = create_buffer({})\n\n      assert_equal({}, plugin.stage)\n      assert_equal([], plugin.queue)\n      assert_equal({}, plugin.dequeued)\n      assert_equal({}, plugin.queued_num)\n\n      assert_equal 0, plugin.stage_size\n      assert_equal 0, plugin.queue_size\n      assert_equal [], plugin.timekeys\n\n      # @p is started plugin\n\n      assert_equal [@dm2,@dm3], @p.stage.keys\n      assert_equal \"b\" * 100, @p.stage[@dm2].read\n      assert_equal \"c\" * 100, @p.stage[@dm3].read\n\n      assert_equal 200, @p.stage_size\n\n      assert_equal 3, @p.queue.size\n      assert_equal \"0\" * 100, @p.queue[0].read\n      assert_equal \"a\" * 100, @p.queue[1].read\n      assert_equal \"a\" * 3, @p.queue[2].read\n\n      assert_equal 203, @p.queue_size\n\n      # staged, queued\n      assert_equal 1, @p.queued_num[@dm0]\n      assert_equal 2, @p.queued_num[@dm1]\n    end\n\n    test '#close closes all chunks in dequeued, enqueued and staged' do\n      dmx = create_metadata(Time.parse('2016-04-11 15:50:00 +0000').to_i, nil, nil)\n      cx = create_chunk(dmx, [\"x\" * 1024])\n      @p.dequeued[cx.unique_id] = cx\n\n      staged_chunks = @p.stage.values.dup\n      queued_chunks = @p.queue.dup\n\n      @p.close\n\n      assert cx.closed\n      assert{ staged_chunks.all?{|c| c.closed } }\n      assert{ queued_chunks.all?{|c| c.closed } }\n    end\n\n    test '#terminate initializes all internal states' do\n      dmx = create_metadata(Time.parse('2016-04-11 15:50:00 +0000').to_i, nil, nil)\n      cx = create_chunk(dmx, [\"x\" * 1024])\n      @p.dequeued[cx.unique_id] = cx\n\n      @p.close\n\n      @p.terminate\n\n      assert_nil @p.stage\n      assert_nil @p.queue\n      assert_nil @p.dequeued\n      assert_nil @p.queued_num\n      assert_nil @p.stage_length_metrics\n      assert_nil @p.stage_size_metrics\n      assert_nil @p.queue_length_metrics\n      assert_nil @p.queue_size_metrics\n      assert_nil @p.available_buffer_space_ratios_metrics\n      assert_nil @p.total_queued_size_metrics\n      assert_nil @p.newest_timekey_metrics\n      assert_nil @p.oldest_timekey_metrics\n      assert_equal [], @p.timekeys\n    end\n\n    test '#queued_records returns total number of size in all chunks in queue' do\n      assert_equal 3, @p.queue.size\n\n      r0 = @p.queue[0].size\n      assert_equal 1, r0\n      r1 = @p.queue[1].size\n      assert_equal 1, r1\n      r2 = @p.queue[2].size\n      assert_equal 1, r2\n\n      assert_equal (r0+r1+r2), @p.queued_records\n    end\n\n    test '#queued? returns queue has any chunks or not without arguments' do\n      assert @p.queued?\n\n      @p.queue.reject!{|_c| true }\n      assert !@p.queued?\n    end\n\n    test '#queued? returns queue has chunks for specified metadata with an argument' do\n      assert @p.queued?(@dm0)\n      assert @p.queued?(@dm1)\n      assert !@p.queued?(@dm2)\n    end\n\n    test '#enqueue_chunk enqueues a chunk on stage with specified metadata' do\n      assert_equal 2, @p.stage.size\n      assert_equal [@dm2,@dm3], @p.stage.keys\n      assert_equal 3, @p.queue.size\n      assert_nil @p.queued_num[@dm2]\n\n      assert_equal 200, @p.stage_size\n      assert_equal 203, @p.queue_size\n\n      @p.enqueue_chunk(@dm2)\n\n      assert_equal [@dm3], @p.stage.keys\n      assert_equal @dm2, @p.queue.last.metadata\n      assert_equal 1, @p.queued_num[@dm2]\n      assert_equal 100, @p.stage_size\n      assert_equal 303, @p.queue_size\n    end\n\n    test '#enqueue_chunk ignores empty chunks' do\n      assert_equal 3, @p.queue.size\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n      c = create_chunk(m, [''])\n      @p.stage[m] = c\n      assert @p.stage[m].empty?\n      assert !c.closed\n\n      @p.enqueue_chunk(m)\n\n      assert_nil @p.stage[m]\n      assert_equal 3, @p.queue.size\n      assert_nil @p.queued_num[m]\n      assert c.closed\n    end\n\n    test '#enqueue_chunk calls #enqueued! if chunk responds to it' do\n      assert_equal 3, @p.queue.size\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n      c = create_chunk(m, ['c' * 256])\n      callback_called = false\n      (class << c; self; end).module_eval do\n        define_method(:enqueued!){ callback_called = true }\n      end\n\n      @p.stage[m] = c\n      @p.enqueue_chunk(m)\n\n      assert_equal c, @p.queue.last\n      assert callback_called\n    end\n\n    test '#enqueue_all enqueues chunks on stage which given block returns true with' do\n      m1 = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n      c1 = create_chunk(m1, ['c' * 256])\n      @p.stage[m1] = c1\n      m2 = @p.metadata(timekey: Time.parse('2016-04-11 16:50:00 +0000').to_i)\n      c2 = create_chunk(m2, ['c' * 256])\n      @p.stage[m2] = c2\n\n      assert_equal [@dm2,@dm3,m1,m2], @p.stage.keys\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n\n      @p.enqueue_all{ |m, c| m.timekey < Time.parse('2016-04-11 16:41:00 +0000').to_i }\n\n      assert_equal [m2], @p.stage.keys\n      assert_equal [@dm0,@dm1,@dm1,@dm2,@dm3,m1], @p.queue.map(&:metadata)\n    end\n\n    test '#enqueue_all enqueues all chunks on stage without block' do\n      m1 = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n      c1 = create_chunk(m1, ['c' * 256])\n      @p.stage[m1] = c1\n      m2 = @p.metadata(timekey: Time.parse('2016-04-11 16:50:00 +0000').to_i)\n      c2 = create_chunk(m2, ['c' * 256])\n      @p.stage[m2] = c2\n\n      assert_equal [@dm2,@dm3,m1,m2], @p.stage.keys\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n\n      @p.enqueue_all\n\n      assert_equal [], @p.stage.keys\n      assert_equal [@dm0,@dm1,@dm1,@dm2,@dm3,m1,m2], @p.queue.map(&:metadata)\n    end\n\n    test '#dequeue_chunk dequeues a chunk from queue if a chunk exists' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({}, @p.dequeued)\n\n      m1 = @p.dequeue_chunk\n      assert_equal @dm0, m1.metadata\n      assert_equal @dm0, @p.dequeued[m1.unique_id].metadata\n\n      m2 = @p.dequeue_chunk\n      assert_equal @dm1, m2.metadata\n      assert_equal @dm1, @p.dequeued[m2.unique_id].metadata\n\n      m3 = @p.dequeue_chunk\n      assert_equal @dm1, m3.metadata\n      assert_equal @dm1, @p.dequeued[m3.unique_id].metadata\n\n      m4 = @p.dequeue_chunk\n      assert_nil m4\n    end\n\n    test '#takeback_chunk resumes a chunk from dequeued to queued at the head of queue, and returns true' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({}, @p.dequeued)\n\n      m1 = @p.dequeue_chunk\n      assert_equal @dm0, m1.metadata\n      assert_equal @dm0, @p.dequeued[m1.unique_id].metadata\n      assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({m1.unique_id => m1}, @p.dequeued)\n\n      assert @p.takeback_chunk(m1.unique_id)\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({}, @p.dequeued)\n    end\n\n    test '#purge_chunk removes a chunk specified by argument id from dequeued chunks' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({}, @p.dequeued)\n\n      m0 = @p.dequeue_chunk\n      m1 = @p.dequeue_chunk\n\n      assert @p.takeback_chunk(m0.unique_id)\n\n      assert_equal [@dm0,@dm1], @p.queue.map(&:metadata)\n      assert_equal({m1.unique_id => m1}, @p.dequeued)\n\n      assert !m1.purged\n\n      @p.purge_chunk(m1.unique_id)\n      assert m1.purged\n\n      assert_equal [@dm0,@dm1], @p.queue.map(&:metadata)\n      assert_equal({}, @p.dequeued)\n    end\n\n    test '#purge_chunk removes an argument metadata if no chunks exist on stage or in queue' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({}, @p.dequeued)\n\n      m0 = @p.dequeue_chunk\n\n      assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({m0.unique_id => m0}, @p.dequeued)\n\n      assert !m0.purged\n\n      @p.purge_chunk(m0.unique_id)\n      assert m0.purged\n\n      assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({}, @p.dequeued)\n    end\n\n    test '#takeback_chunk returns false if specified chunk_id is already purged' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({}, @p.dequeued)\n\n      m0 = @p.dequeue_chunk\n\n      assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({m0.unique_id => m0}, @p.dequeued)\n\n      assert !m0.purged\n\n      @p.purge_chunk(m0.unique_id)\n      assert m0.purged\n\n      assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({}, @p.dequeued)\n\n      assert !@p.takeback_chunk(m0.unique_id)\n\n      assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal({}, @p.dequeued)\n    end\n\n    test '#clear_queue! removes all chunks in queue, but leaves staged chunks' do\n      qchunks = @p.queue.dup\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal 2, @p.stage.size\n      assert_equal({}, @p.dequeued)\n\n      @p.clear_queue!\n\n      assert_equal [], @p.queue\n      assert_equal 0, @p.queue_size\n      assert_equal 2, @p.stage.size\n      assert_equal({}, @p.dequeued)\n\n      assert{ qchunks.all?{ |c| c.purged } }\n    end\n\n    test '#write returns immediately if argument data is empty array' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      @p.write({m => []})\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n    end\n\n    test '#write returns immediately if argument data is empty event stream' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      @p.write({m => Fluent::ArrayEventStream.new([])})\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n    end\n\n    test '#write raises BufferOverflowError if buffer is not storable' do\n      @p.stage_size = 256 * 1024 * 1024\n      @p.queue_size = 256 * 1024 * 1024\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      assert_raise Fluent::Plugin::Buffer::BufferOverflowError do\n        @p.write({m => [\"x\" * 256]})\n      end\n    end\n\n    test '#write stores data into an existing chunk with metadata specified' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      dm3data = @p.stage[@dm3].read.dup\n      prev_stage_size = @p.stage_size\n\n      assert_equal 1, @p.stage[@dm3].append_count\n\n      @p.write({@dm3 => [\"x\" * 256, \"y\" * 256, \"z\" * 256]})\n\n      assert_equal 2, @p.stage[@dm3].append_count\n      assert_equal (dm3data + (\"x\" * 256) + (\"y\" * 256) + (\"z\" * 256)), @p.stage[@dm3].read\n      assert_equal (prev_stage_size + 768), @p.stage_size\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n    end\n\n    test '#write creates new chunk and store data into it if there are no chunks for specified metadata' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      timekey = Time.parse('2016-04-11 16:40:00 +0000').to_i\n      assert !@p.timekeys.include?(timekey)\n\n      prev_stage_size = @p.stage_size\n\n      m = @p.metadata(timekey: timekey)\n\n      @p.write({m => [\"x\" * 256, \"y\" * 256, \"z\" * 256]})\n\n      assert_equal 1, @p.stage[m].append_count\n      assert_equal (\"x\" * 256 + \"y\" * 256 + \"z\" * 256), @p.stage[m].read\n      assert_equal (prev_stage_size + 768), @p.stage_size\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3,m], @p.stage.keys\n\n      @p.update_timekeys\n\n      assert @p.timekeys.include?(timekey)\n    end\n\n    test '#write tries to enqueue and store data into a new chunk if existing chunk is full' do\n      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size\n      assert_equal 0.95, @p.chunk_full_threshold\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      row = \"x\" * 1024 * 1024\n      small_row = \"x\" * 1024 * 512\n      @p.write({m => [row] * 7 + [small_row]})\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3,m], @p.stage.keys\n      assert_equal 1, @p.stage[m].append_count\n\n      @p.write({m => [row]})\n\n      assert_equal [@dm0,@dm1,@dm1,m], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3,m], @p.stage.keys\n      assert_equal 1, @p.stage[m].append_count\n      assert_equal 1024*1024, @p.stage[m].bytesize\n      assert_equal 3, @p.queue.last.append_count # 1 -> write (2) -> write_step_by_step (3)\n      assert @p.queue.last.rollbacked\n    end\n\n    test '#write rollbacks if commit raises errors' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      row = \"x\" * 1024\n      @p.write({m => [row] * 8})\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3,m], @p.stage.keys\n\n      target_chunk = @p.stage[m]\n\n      assert_equal 1, target_chunk.append_count\n      assert !target_chunk.rollbacked\n\n      (class << target_chunk; self; end).module_eval do\n        define_method(:commit){ raise \"yay\" }\n      end\n\n      assert_raise RuntimeError.new(\"yay\") do\n        @p.write({m => [row]})\n      end\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3,m], @p.stage.keys\n\n      assert_equal 2, target_chunk.append_count\n      assert target_chunk.rollbacked\n      assert_equal row * 8, target_chunk.read\n    end\n\n    test '#write w/ format raises BufferOverflowError if buffer is not storable' do\n      @p.stage_size = 256 * 1024 * 1024\n      @p.queue_size = 256 * 1024 * 1024\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"xxxxxxxxxxxxxx\"} ] ])\n\n      assert_raise Fluent::Plugin::Buffer::BufferOverflowError do\n        @p.write({m => es}, format: ->(e){e.to_msgpack_stream})\n      end\n    end\n\n    test '#write w/ format stores data into an existing chunk with metadata specified' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      dm3data = @p.stage[@dm3].read.dup\n      prev_stage_size = @p.stage_size\n\n      assert_equal 1, @p.stage[@dm3].append_count\n\n      es = Fluent::ArrayEventStream.new(\n        [\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 128}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"y\" * 128}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"z\" * 128}],\n        ]\n      )\n\n      @p.write({@dm3 => es}, format: ->(e){e.to_msgpack_stream})\n\n      assert_equal 2, @p.stage[@dm3].append_count\n      assert_equal (dm3data + es.to_msgpack_stream), @p.stage[@dm3].read\n      assert_equal (prev_stage_size + es.to_msgpack_stream.bytesize), @p.stage_size\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n    end\n\n    test '#write w/ format creates new chunk and store data into it if there are not chunks for specified metadata' do\n      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      timekey = Time.parse('2016-04-11 16:40:00 +0000').to_i\n      assert !@p.timekeys.include?(timekey)\n\n      m = @p.metadata(timekey: timekey)\n\n      es = Fluent::ArrayEventStream.new(\n        [\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:03 +0000'), {\"message\" => \"z\" * 1024 * 512}],\n        ]\n      )\n      @p.write({m => es}, format: ->(e){e.to_msgpack_stream})\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3,m], @p.stage.keys\n      assert_equal 1, @p.stage[m].append_count\n\n      @p.update_timekeys\n\n      assert @p.timekeys.include?(timekey)\n    end\n\n    test '#write w/ format tries to enqueue and store data into a new chunk if existing chunk does not have enough space' do\n      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      es = Fluent::ArrayEventStream.new(\n        [\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:03 +0000'), {\"message\" => \"z\" * 1024 * 512}],\n        ]\n      )\n      @p.write({m => es}, format: ->(e){e.to_msgpack_stream})\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3,m], @p.stage.keys\n      assert_equal 1, @p.stage[m].append_count\n\n      es2 = Fluent::OneEventStream.new(event_time('2016-04-11 16:40:03 +0000'), {\"message\" => \"z\" * 1024 * 1024})\n      @p.write({m => es2}, format: ->(e){e.to_msgpack_stream})\n\n      assert_equal [@dm0,@dm1,@dm1,m], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3,m], @p.stage.keys\n      assert_equal 1, @p.stage[m].append_count\n      assert_equal es2.to_msgpack_stream.bytesize, @p.stage[m].bytesize\n      assert_equal 2, @p.queue.last.append_count # 1 -> write (2) -> rollback&enqueue\n      assert @p.queue.last.rollbacked\n    end\n\n    test '#write w/ format enqueues chunk if it is already full after adding data' do\n      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n      es = Fluent::ArrayEventStream.new(\n        [\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * (1024 * 1024 - 25)}], # 1024 * 1024 bytes as msgpack stream\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * (1024 * 1024 - 25)}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * (1024 * 1024 - 25)}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * (1024 * 1024 - 25)}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * (1024 * 1024 - 25)}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * (1024 * 1024 - 25)}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * (1024 * 1024 - 25)}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * (1024 * 1024 - 25)}],\n        ]\n      )\n      @p.write({m => es}, format: ->(e){e.to_msgpack_stream})\n\n      assert_equal [@dm0,@dm1,@dm1,m], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n      assert_equal 1, @p.queue.last.append_count\n    end\n\n    test '#write w/ format rollbacks if commit raises errors' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      es = Fluent::ArrayEventStream.new(\n        [\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:01 +0000'), {\"message\" => \"x\" * 1024 * 1024}],\n          [event_time('2016-04-11 16:40:03 +0000'), {\"message\" => \"z\" * 1024 * 512}],\n        ]\n      )\n      @p.write({m => es}, format: ->(e){e.to_msgpack_stream})\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3,m], @p.stage.keys\n\n      target_chunk = @p.stage[m]\n\n      assert_equal 1, target_chunk.append_count\n      assert !target_chunk.rollbacked\n\n      (class << target_chunk; self; end).module_eval do\n        define_method(:commit){ raise \"yay\" }\n      end\n\n      es2 = Fluent::ArrayEventStream.new(\n        [\n          [event_time('2016-04-11 16:40:04 +0000'), {\"message\" => \"z\" * 1024 * 128}],\n        ]\n      )\n      assert_raise RuntimeError.new(\"yay\") do\n        @p.write({m => es2}, format: ->(e){e.to_msgpack_stream})\n      end\n\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3,m], @p.stage.keys\n\n      assert_equal 2, target_chunk.append_count\n      assert target_chunk.rollbacked\n      assert_equal es.to_msgpack_stream, target_chunk.read\n    end\n\n    test '#write writes many metadata and data pairs at once' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      row = \"x\" * 1024\n      @p.write({ @dm0 => [row, row, row], @dm1 => [row, row] })\n\n      assert_equal [@dm2,@dm3,@dm0,@dm1], @p.stage.keys\n    end\n\n    test '#write does not commit on any chunks if any append operation on chunk fails' do\n      assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      row = \"x\" * 1024\n      @p.write({ @dm0 => [row, row, row], @dm1 => [row, row] })\n\n      assert_equal [@dm2,@dm3,@dm0,@dm1], @p.stage.keys\n\n      dm2_size = @p.stage[@dm2].size\n      assert !@p.stage[@dm2].rollbacked\n      dm3_size = @p.stage[@dm3].size\n      assert !@p.stage[@dm3].rollbacked\n\n      assert{ @p.stage[@dm0].size == 3 }\n      assert !@p.stage[@dm0].rollbacked\n      assert{ @p.stage[@dm1].size == 2 }\n      assert !@p.stage[@dm1].rollbacked\n\n      meta_list = [@dm0, @dm1, @dm2, @dm3].sort\n      @p.stage[meta_list.last].failing = true\n\n      assert_raise(FluentPluginBufferTest::DummyMemoryChunkError) do\n        @p.write({ @dm2 => [row], @dm3 => [row], @dm0 => [row, row, row], @dm1 => [row, row] })\n      end\n\n      assert{ @p.stage[@dm2].size == dm2_size }\n      assert @p.stage[@dm2].rollbacked\n      assert{ @p.stage[@dm3].size == dm3_size }\n      assert @p.stage[@dm3].rollbacked\n\n      assert{ @p.stage[@dm0].size == 3 }\n      assert @p.stage[@dm0].rollbacked\n      assert{ @p.stage[@dm1].size == 2 }\n      assert @p.stage[@dm1].rollbacked\n    end\n\n    test '#compress returns :text' do\n      assert_equal :text, @p.compress\n    end\n\n    # https://github.com/fluent/fluentd/issues/3089\n    test \"closed chunk should not be committed\" do\n      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size\n      assert_equal 0.95, @p.chunk_full_threshold\n\n      purge_count = 0\n\n      stub.proxy(@p).generate_chunk(anything) do |chunk|\n        stub.proxy(chunk).purge do |result|\n          purge_count += 1\n          result\n        end\n        stub.proxy(chunk).commit do |result|\n          assert_false(chunk.closed?)\n          result\n        end\n        stub.proxy(chunk).rollback do |result|\n          assert_false(chunk.closed?)\n          result\n        end\n        chunk\n      end\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n      small_row = \"x\" * 1024 * 400\n      big_row = \"x\" * 1024 * 1024 * 8 # just `chunk_size_limit`, it doesn't cause BufferOverFlowError.\n\n      # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry`\n      @p.write({m => [small_row] * 40 + [big_row] + [\"x\"]})\n\n      # Above event stream will be splitted twice by `Buffer#write_step_by_step`\n      #\n      # 1. `write_once`: 42 [events] * 1 [stream]\n      # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]\n      # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]\n      #\n      # The problematic data is built in the 2nd stage.\n      # In the 2nd stage, 5 streams are packed in a chunk.\n      # ((1024 * 400) [bytes] * 4 [events] * 5 [streams] = 8192000 [bytes] < `chunk_limit_size` (8MB)).\n      # So 3 chunks are used to store all data.\n      # The 1st chunk is already staged by `write_once`.\n      # The 2nd & 3rd chunks are newly created as unstaged.\n      # The 3rd chunk is purged before `ShouldRetry`, it's no problem:\n      #   https://github.com/fluent/fluentd/blob/7e9eba736ff40ad985341be800ddc46558be75f2/lib/fluent/plugin/buffer.rb#L850\n      # The 2nd chunk is purged in `rescue ShouldRetry`:\n      #   https://github.com/fluent/fluentd/blob/7e9eba736ff40ad985341be800ddc46558be75f2/lib/fluent/plugin/buffer.rb#L862\n      # It causes the issue described in https://github.com/fluent/fluentd/issues/3089#issuecomment-1811839198\n\n      assert_equal 2, purge_count\n    end\n\n    # https://github.com/fluent/fluentd/issues/4446\n    test \"#write_step_by_step keeps chunks kept in locked in entire #write process\" do\n      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size\n      assert_equal 0.95, @p.chunk_full_threshold\n\n      mon_enter_counts_by_chunk = {}\n      mon_exit_counts_by_chunk = {}\n\n      stub.proxy(@p).generate_chunk(anything) do |chunk|\n        stub(chunk).mon_enter do\n          enter_count = 1 + mon_enter_counts_by_chunk.fetch(chunk, 0)\n          exit_count = mon_exit_counts_by_chunk.fetch(chunk, 0)\n          mon_enter_counts_by_chunk[chunk] = enter_count\n\n          # Assert that chunk is passed to &block of write_step_by_step before exiting the lock.\n          # (i.e. The lock count must be 2 greater than the exit count).\n          # Since ShouldRetry occurs once, the staged chunk takes the lock 3 times when calling the block.\n          if chunk.staged?\n            lock_in_block = enter_count == 3\n            assert_equal(enter_count - 2, exit_count) if lock_in_block\n          else\n            lock_in_block = enter_count == 2\n            assert_equal(enter_count - 2, exit_count) if lock_in_block\n          end\n        end\n        stub(chunk).mon_exit do\n          exit_count = 1 + mon_exit_counts_by_chunk.fetch(chunk, 0)\n          mon_exit_counts_by_chunk[chunk] = exit_count\n        end\n        chunk\n      end\n\n      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)\n      small_row = \"x\" * 1024 * 400\n      big_row = \"x\" * 1024 * 1024 * 8 # just `chunk_size_limit`, it doesn't cause BufferOverFlowError.\n\n      # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry`\n      @p.write({m => [small_row] * 40 + [big_row] + [\"x\"]})\n\n      # Above event stream will be splitted twice by `Buffer#write_step_by_step`\n      #\n      # 1. `write_once`: 42 [events] * 1 [stream]\n      # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]\n      # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]\n      #\n      # Example of staged chunk lock behavior:\n      #\n      # 1. mon_enter in write_step_by_step\n      # 2. ShouldRetry occurs\n      # 3. mon_exit in write_step_by_step\n      # 4. mon_enter again in write_step_by_step (retry)\n      # 5. passed to &block of write_step_by_step\n      # 6. mon_enter in the block (write)\n      # 7. mon_exit in write_step_by_step\n      # 8. mon_exit in write\n\n      assert_equal(mon_enter_counts_by_chunk.values, mon_exit_counts_by_chunk.values)\n    end\n  end\n\n  sub_test_case 'standard format with configuration for test with lower chunk limit size' do\n    setup do\n      @p = create_buffer({\"chunk_limit_size\" => 1_280_000})\n      @format = ->(e){e.to_msgpack_stream}\n      @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)\n      # 1 record is 128bytes in msgpack stream\n      @es0 = es0 = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:01 +0000'), {\"message\" => \"x\" * (128 - 22)}] ] * 5000)\n      (class << @p; self; end).module_eval do\n        define_method(:resume) {\n          staged = {\n            dm0 => create_chunk_es(dm0, es0).staged!,\n          }\n          queued = []\n          return staged, queued\n        }\n      end\n      @p.start\n    end\n\n    test '#write appends event stream into staged chunk' do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"x\" * (128 - 22)}] ] * 1000)\n      @p.write({@dm0 => es}, format: @format)\n\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal (@es0.to_msgpack_stream + es.to_msgpack_stream), @p.stage[@dm0].read\n    end\n\n    test '#write writes event stream into a new chunk with enqueueing existing chunk if event stream is larger than available space of existing chunk' do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"x\" * (128 - 22)}] ] * 8000)\n      @p.write({@dm0 => es}, format: @format)\n\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [@dm0], @p.queue.map(&:metadata)\n\n      assert_equal (es.to_msgpack_stream), @p.stage[@dm0].read\n    end\n\n    test '#write writes event stream into many chunks excluding staged chunk if event stream is larger than chunk limit size' do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"x\" * (128 - 22)}] ] * 45000)\n      @p.write({@dm0 => es}, format: @format)\n\n      # metadata whose seq is 4 is created, but overwrite with original metadata(seq=0) for next use of this chunk https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L357\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal 5400, @p.stage[@dm0].size\n      assert_equal [@dm0, @dm0, @dm0, @dm0, @dm0], @p.queue.map(&:metadata)\n      assert_equal [5000, 9900, 9900, 9900, 9900], @p.queue.map(&:size) # splits: 45000 / 100 => 450 * ...\n      # 9900 * 4 + 5400 == 45000\n    end\n\n    test '#dequeue_chunk succeeds when chunk is splited' do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"x\" * (128 - 22)}] ] * 45000)\n      @p.write({@dm0 => es}, format: @format)\n      @p.enqueue_all(true)\n\n      dequeued_chunks = Array.new(6) { |e| @p.dequeue_chunk } # splits: 45000 / 100 => 450 * ...\n      assert_equal [5000, 9900, 9900, 9900, 9900, 5400], dequeued_chunks.map(&:size)\n      assert_equal [@dm0, @dm0, @dm0, @dm0, @dm0, @dm0], dequeued_chunks.map(&:metadata)\n    end\n\n    test '#write raises BufferChunkOverflowError if a record is bigger than chunk limit size' do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"x\" * 1_280_000}] ])\n      assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError do\n        @p.write({@dm0 => es}, format: @format)\n      end\n    end\n\n    data(\n      first_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"x\" * 1_280_000}],\n                                                 [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"a\"}],\n                                                 [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"b\"}]]),\n      intermediate_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"a\"}],\n                                                        [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"x\" * 1_280_000}],\n                                                        [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"b\"}]]),\n      last_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"a\"}],\n                                                [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"b\"}],\n                                                [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"x\" * 1_280_000}]]),\n      multiple_chunks: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"a\"}],\n                                                     [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"x\" * 1_280_000}],\n                                                     [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"b\"}],\n                                                     [event_time('2016-04-11 16:00:02 +0000'), {\"message\" => \"x\" * 1_280_000}]])\n    )\n    test '#write exceeds chunk_limit_size, raise BufferChunkOverflowError, but not lost whole messages' do |(es)|\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      nth = []\n      es.entries.each_with_index do |entry, index|\n        if entry.last[\"message\"].size == @p.chunk_limit_size\n          nth << index\n        end\n      end\n      messages = []\n      nth.each do |n|\n        messages << \"a 1280025 bytes record (nth: #{n}) is larger than buffer chunk limit size (1280000)\"\n      end\n\n      assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError.new(messages.join(\", \")) do\n        @p.write({@dm0 => es}, format: @format)\n      end\n      # message a and b are concatenated and staged\n      staged_messages = Fluent::MessagePackFactory.msgpack_unpacker.feed_each(@p.stage[@dm0].chunk).collect do |record|\n        record.last\n      end\n      assert_equal([2, [{\"message\" => \"a\"}, {\"message\" => \"b\"}]],\n                   [@p.stage[@dm0].size, staged_messages])\n      # only es0 message is queued\n      assert_equal [@dm0], @p.queue.map(&:metadata)\n      assert_equal [5000], @p.queue.map(&:size)\n    end\n\n    test \"confirm that every message which is smaller than chunk threshold does not raise BufferChunkOverflowError\" do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n      timestamp = event_time('2016-04-11 16:00:02 +0000')\n      es = Fluent::ArrayEventStream.new([[timestamp, {\"message\" => \"a\" * 1_000_000}],\n                                         [timestamp, {\"message\" => \"b\" * 1_000_000}],\n                                         [timestamp, {\"message\" => \"c\" * 1_000_000}]])\n\n      # https://github.com/fluent/fluentd/issues/1849\n      # Even though 1_000_000 < 1_280_000 (chunk_limit_size), it raised BufferChunkOverflowError before.\n      # It should not be raised and message a,b,c should be stored into 3 chunks.\n      assert_nothing_raised do\n        @p.write({@dm0 => es}, format: @format)\n      end\n      messages = []\n      # pick up first letter to check whether chunk is queued in expected order\n      3.times do |index|\n        chunk = @p.queue[index]\n        es = Fluent::MessagePackEventStream.new(chunk.chunk)\n        es.ensure_unpacked!\n        records = es.instance_eval{ @unpacked_records }\n        records.each do |record|\n          messages << record[\"message\"][0]\n        end\n      end\n      es = Fluent::MessagePackEventStream.new(@p.stage[@dm0].chunk)\n      es.ensure_unpacked!\n      staged_message = es.instance_eval{ @unpacked_records }.first[\"message\"]\n      # message a and b are queued, message c is staged\n      assert_equal([\n                     [@dm0],\n                     \"c\" * 1_000_000,\n                     [@dm0, @dm0, @dm0],\n                     [5000, 1, 1],\n                     [[\"x\"] * 5000, \"a\", \"b\"].flatten\n                   ],\n                   [\n                     @p.stage.keys,\n                     staged_message,\n                     @p.queue.map(&:metadata),\n                     @p.queue.map(&:size),\n                     messages\n                   ])\n    end\n  end\n\n  sub_test_case 'custom format with configuration for test with lower chunk limit size' do\n    setup do\n      @p = create_buffer({\"chunk_limit_size\" => 1_280_000})\n      @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)\n      @row = \"x\" * 128\n      @data0 = data0 = [@row] * 5000\n      (class << @p; self; end).module_eval do\n        define_method(:resume) {\n          staged = {\n            dm0 => create_chunk(dm0, data0).staged!,\n          }\n          queued = []\n          return staged, queued\n        }\n      end\n      @p.start\n    end\n\n    test '#write appends event stream into staged chunk' do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      data = [@row] * 1000\n      @p.write({@dm0 => data})\n\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal (@row * 6000), @p.stage[@dm0].read\n    end\n\n    test '#write writes event stream into a new chunk with enqueueing existing chunk if event stream is larger than available space of existing chunk' do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      staged_chunk_object_id = @p.stage[@dm0].object_id\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      data = [@row] * 8000\n      @p.write({@dm0 => data})\n\n      assert_equal [@dm0], @p.queue.map(&:metadata)\n      assert_equal [staged_chunk_object_id], @p.queue.map(&:object_id)\n      assert_equal [@dm0], @p.stage.keys\n\n      assert_equal [9800], @p.queue.map(&:size)\n      assert_equal 3200, @p.stage[@dm0].size\n      # 9800 + 3200 == 5000 + 8000\n    end\n\n    test '#write writes event stream into many chunks including staging chunk if event stream is larger than chunk limit size' do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      staged_chunk_object_id = @p.stage[@dm0].object_id\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      assert_equal 5000, @p.stage[@dm0].size\n\n      data = [@row] * 45000\n      @p.write({@dm0 => data})\n\n      assert_equal staged_chunk_object_id, @p.queue.first.object_id\n\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal 900, @p.stage[@dm0].size\n      assert_equal [@dm0, @dm0, @dm0, @dm0, @dm0], @p.queue.map(&:metadata)\n      assert_equal [9500, 9900, 9900, 9900, 9900], @p.queue.map(&:size) # splits: 45000 / 100 => 450 * ...\n      ##### 900 + 9500 + 9900 * 4 == 5000 + 45000\n    end\n\n    test '#write raises BufferChunkOverflowError if a record is bigger than chunk limit size' do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      es = [\"x\" * 1_280_000 + \"x\" * 300]\n      assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError do\n        @p.write({@dm0 => es})\n      end\n    end\n\n    test 'confirm that every array message which is smaller than chunk threshold does not raise BufferChunkOverflowError' do\n      assert_equal [@dm0], @p.stage.keys\n      assert_equal [], @p.queue.map(&:metadata)\n\n      assert_equal 1_280_000, @p.chunk_limit_size\n\n      es = [\"a\" * 1_000_000, \"b\" * 1_000_000, \"c\" * 1_000_000]\n      assert_nothing_raised do\n        @p.write({@dm0 => es})\n      end\n      queue_messages = @p.queue.collect do |chunk|\n        # collect first character of each message\n        chunk.chunk[0]\n      end\n      assert_equal([\n                     [@dm0],\n                     1,\n                     \"c\",\n                     [@dm0, @dm0, @dm0],\n                     [5000, 1, 1],\n                     [\"x\", \"a\", \"b\"]\n                   ],\n                   [\n                     @p.stage.keys,\n                     @p.stage[@dm0].size,\n                     @p.stage[@dm0].chunk[0],\n                     @p.queue.map(&:metadata),\n                     @p.queue.map(&:size),\n                     queue_messages\n                   ])\n    end\n  end\n\n  sub_test_case 'with configuration for test with lower limits' do\n    setup do\n      @p = create_buffer({\"chunk_limit_size\" => 1024, \"total_limit_size\" => 10240})\n      @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)\n      @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)\n      @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)\n      @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil)\n      (class << @p; self; end).module_eval do\n        define_method(:resume) {\n          staged = {\n            dm2 => create_chunk(dm2, [\"b\" * 128] * 7).staged!,\n            dm3 => create_chunk(dm3, [\"c\" * 128] * 5).staged!,\n          }\n          queued = [\n            create_chunk(dm0, [\"0\" * 128] * 8).enqueued!,\n            create_chunk(dm0, [\"0\" * 128] * 8).enqueued!,\n            create_chunk(dm0, [\"0\" * 128] * 8).enqueued!,\n            create_chunk(dm0, [\"0\" * 128] * 8).enqueued!,\n            create_chunk(dm0, [\"0\" * 128] * 8).enqueued!,\n            create_chunk(dm1, [\"a\" * 128] * 8).enqueued!,\n            create_chunk(dm1, [\"a\" * 128] * 8).enqueued!,\n            create_chunk(dm1, [\"a\" * 128] * 8).enqueued!, # 8th queued chunk\n            create_chunk(dm1, [\"a\" * 128] * 3).enqueued!,\n          ]\n          return staged, queued\n        }\n      end\n      @p.start\n    end\n\n    test '#storable? returns false when too many data exist' do\n      assert_equal [@dm0,@dm0,@dm0,@dm0,@dm0,@dm1,@dm1,@dm1,@dm1], @p.queue.map(&:metadata)\n      assert_equal [@dm2,@dm3], @p.stage.keys\n\n      assert_equal 128*8*8+128*3, @p.queue_size\n      assert_equal 128*7+128*5, @p.stage_size\n\n      assert @p.storable?\n\n      dm3 = @p.metadata(timekey: @dm3.timekey)\n      @p.write({dm3 => [\"c\" * 128]})\n\n      assert_equal 10240, (@p.stage_size + @p.queue_size)\n      assert !@p.storable?\n    end\n\n    test '#chunk_size_over? returns true if chunk size is bigger than limit' do\n      m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      c1 = create_chunk(m, [\"a\" * 128] * 8)\n      assert !@p.chunk_size_over?(c1)\n\n      c2 = create_chunk(m, [\"a\" * 128] * 9)\n      assert @p.chunk_size_over?(c2)\n\n      c3 = create_chunk(m, [\"a\" * 128] * 8 + [\"a\"])\n      assert @p.chunk_size_over?(c3)\n    end\n\n    test '#chunk_size_full? returns true if chunk size is enough big against limit' do\n      m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      c1 = create_chunk(m, [\"a\" * 128] * 7)\n      assert !@p.chunk_size_full?(c1)\n\n      c2 = create_chunk(m, [\"a\" * 128] * 8)\n      assert @p.chunk_size_full?(c2)\n\n      assert_equal 0.95, @p.chunk_full_threshold\n      c3 = create_chunk(m, [\"a\" * 128] * 6 + [\"a\" * 64])\n      assert !@p.chunk_size_full?(c3)\n    end\n  end\n\n  sub_test_case 'with configuration includes chunk_limit_records' do\n    setup do\n      @p = create_buffer({\"chunk_limit_size\" => 1024, \"total_limit_size\" => 10240, \"chunk_limit_records\" => 6})\n      @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)\n      @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)\n      @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)\n      @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil)\n      (class << @p; self; end).module_eval do\n        define_method(:resume) {\n          staged = {\n            dm2 => create_chunk(dm2, [\"b\" * 128] * 1).staged!,\n            dm3 => create_chunk(dm3, [\"c\" * 128] * 2).staged!,\n          }\n          queued = [\n            create_chunk(dm0, [\"0\" * 128] * 6).enqueued!,\n            create_chunk(dm1, [\"a\" * 128] * 6).enqueued!,\n            create_chunk(dm1, [\"a\" * 128] * 6).enqueued!,\n            create_chunk(dm1, [\"a\" * 128] * 3).enqueued!,\n          ]\n          return staged, queued\n        }\n      end\n      @p.start\n    end\n\n    test '#chunk_size_over? returns true if too many records exists in a chunk even if its bytes is less than limit' do\n      assert_equal 6, @p.chunk_limit_records\n\n      m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      c1 = create_chunk(m, [\"a\" * 128] * 6)\n      assert_equal 6, c1.size\n      assert !@p.chunk_size_over?(c1)\n\n      c2 = create_chunk(m, [\"a\" * 128] * 7)\n      assert @p.chunk_size_over?(c2)\n\n      c3 = create_chunk(m, [\"a\" * 128] * 6 + [\"a\"])\n      assert @p.chunk_size_over?(c3)\n    end\n\n    test '#chunk_size_full? returns true if enough many records exists in a chunk even if its bytes is less than limit' do\n      assert_equal 6, @p.chunk_limit_records\n\n      m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)\n\n      c1 = create_chunk(m, [\"a\" * 128] * 5)\n      assert_equal 5, c1.size\n      assert !@p.chunk_size_full?(c1)\n\n      c2 = create_chunk(m, [\"a\" * 128] * 6)\n      assert @p.chunk_size_full?(c2)\n\n      c3 = create_chunk(m, [\"a\" * 128] * 5 + [\"a\"])\n      assert @p.chunk_size_full?(c3)\n    end\n  end\n\n  sub_test_case 'with configuration includes queue_limit_length' do\n    setup do\n      @p = create_buffer({\"chunk_limit_size\" => 1024, \"total_limit_size\" => 10240, \"queue_limit_length\" => 5})\n      @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)\n      @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)\n      @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)\n      @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil)\n      (class << @p; self; end).module_eval do\n        define_method(:resume) {\n          staged = {\n            dm2 => create_chunk(dm2, [\"b\" * 128] * 1).staged!,\n            dm3 => create_chunk(dm3, [\"c\" * 128] * 2).staged!,\n          }\n          queued = [\n            create_chunk(dm0, [\"0\" * 128] * 6).enqueued!,\n            create_chunk(dm1, [\"a\" * 128] * 6).enqueued!,\n            create_chunk(dm1, [\"a\" * 128] * 6).enqueued!,\n            create_chunk(dm1, [\"a\" * 128] * 3).enqueued!,\n          ]\n          return staged, queued\n        }\n      end\n      @p.start\n    end\n\n    test '#configure will overwrite standard configuration if queue_limit_length' do\n      assert_equal 1024, @p.chunk_limit_size\n      assert_equal 5, @p.queue_limit_length\n      assert_equal (1024*5), @p.total_limit_size\n    end\n  end\n\n  sub_test_case 'when compress is gzip' do\n    setup do\n      @p = create_buffer({'compress' => 'gzip'})\n      @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)\n    end\n\n    test '#compress returns :gzip' do\n      assert_equal :gzip, @p.compress\n    end\n\n    test 'create decompressable chunk' do\n      chunk = @p.generate_chunk(create_metadata)\n      assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::GzipDecompressable)\n    end\n\n    test '#write compressed data which exceeds chunk_limit_size, it raises BufferChunkOverflowError' do\n      @p = create_buffer({'compress' => 'gzip', 'chunk_limit_size' => 70})\n      timestamp = event_time('2016-04-11 16:00:02 +0000')\n      es = Fluent::ArrayEventStream.new([[timestamp, {\"message\" => \"012345\"}], # overflow\n                                         [timestamp, {\"message\" => \"aaa\"}],\n                                         [timestamp, {\"message\" => \"bbb\"}]])\n      assert_equal [], @p.queue.map(&:metadata)\n      assert_equal 70, @p.chunk_limit_size\n\n      # calculate the actual boundary value. it varies on machine\n      c = @p.generate_chunk(create_metadata)\n      c.append(Fluent::ArrayEventStream.new([[timestamp, {\"message\" => \"012345\"}]]), compress: :gzip)\n      overflow_bytes = c.bytesize\n\n      messages = \"concatenated/appended a #{overflow_bytes} bytes record (nth: 0) is larger than buffer chunk limit size (70)\"\n      assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError.new(messages) do\n        # test format == nil && compress == :gzip\n        @p.write({@dm0 => es})\n      end\n      # message a and b occupies each chunks in full, so both of messages are queued (no staged chunk)\n      assert_equal([2, [@dm0, @dm0], [1, 1], nil],\n                   [@p.queue.size, @p.queue.map(&:metadata), @p.queue.map(&:size), @p.stage[@dm0]])\n    end\n  end\n\n  sub_test_case '#statistics' do\n    setup do\n      @p = create_buffer({ \"total_limit_size\" => 1024 })\n      dm = create_metadata(Time.parse('2020-03-13 16:00:00 +0000').to_i, nil, nil)\n\n      (class << @p; self; end).module_eval do\n        define_method(:resume) {\n          queued = [create_chunk(dm, [\"a\" * (1024 - 102)]).enqueued!]\n          return {}, queued\n        }\n      end\n\n      @p.start\n    end\n\n    test 'returns available_buffer_space_ratios' do\n      assert_equal 10.0, @p.statistics['buffer']['available_buffer_space_ratios']\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_buffer_chunk.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/buffer/chunk'\n\nclass BufferChunkTest < Test::Unit::TestCase\n  sub_test_case 'blank buffer chunk' do\n    test 'has generated unique id, given metadata, created_at and modified_at' do\n      meta = Object.new\n      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)\n      assert{ chunk.unique_id.bytesize == 16 }\n      assert{ chunk.metadata.object_id == meta.object_id }\n      assert{ chunk.created_at.is_a? Time }\n      assert{ chunk.modified_at.is_a? Time }\n      assert chunk.unstaged?\n      assert !chunk.staged?\n      assert !chunk.queued?\n      assert !chunk.closed?\n    end\n\n    test 'has many methods for chunks, but not implemented' do\n      meta = Object.new\n      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)\n\n      assert chunk.respond_to?(:append)\n      assert chunk.respond_to?(:concat)\n      assert chunk.respond_to?(:commit)\n      assert chunk.respond_to?(:rollback)\n      assert chunk.respond_to?(:bytesize)\n      assert chunk.respond_to?(:size)\n      assert chunk.respond_to?(:length)\n      assert chunk.respond_to?(:empty?)\n      assert chunk.respond_to?(:read)\n      assert chunk.respond_to?(:open)\n      assert chunk.respond_to?(:write_to)\n      assert_raise(NotImplementedError){ chunk.append([]) }\n      assert_raise(NotImplementedError){ chunk.concat(nil, 0) }\n      assert_raise(NotImplementedError){ chunk.commit }\n      assert_raise(NotImplementedError){ chunk.rollback }\n      assert_raise(NotImplementedError){ chunk.bytesize }\n      assert_raise(NotImplementedError){ chunk.size }\n      assert_raise(NotImplementedError){ chunk.length }\n      assert_raise(NotImplementedError){ chunk.empty? }\n      assert_raise(NotImplementedError){ chunk.read }\n      assert_raise(NotImplementedError){ chunk.open(){} }\n      assert_raise(NotImplementedError){ chunk.write_to(nil) }\n      assert !chunk.respond_to?(:msgpack_each)\n    end\n\n    test 'has method #each and #msgpack_each only when extended by ChunkMessagePackEventStreamer' do\n      meta = Object.new\n      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)\n\n      assert !chunk.respond_to?(:each)\n      assert !chunk.respond_to?(:msgpack_each)\n\n      chunk.extend Fluent::ChunkMessagePackEventStreamer\n      assert chunk.respond_to?(:each)\n      assert chunk.respond_to?(:msgpack_each)\n    end\n\n    test 'unpacker arg is not implemented for ChunkMessagePackEventStreamer' do\n      meta = Object.new\n      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)\n      chunk.extend Fluent::ChunkMessagePackEventStreamer\n\n      unpacker = Fluent::MessagePackFactory.thread_local_msgpack_unpacker\n\n      assert_raise(NotImplementedError){ chunk.each(unpacker: unpacker) }\n      assert_raise(NotImplementedError){ chunk.msgpack_each(unpacker: unpacker) }\n    end\n\n    test 'some methods raise ArgumentError with an option of `compressed: :gzip` and without extending Compressble`' do\n      meta = Object.new\n      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)\n\n      assert_raise(ArgumentError){ chunk.read(compressed: :gzip) }\n      assert_raise(ArgumentError){ chunk.open(compressed: :gzip){} }\n      assert_raise(ArgumentError){ chunk.write_to(nil, compressed: :gzip) }\n      assert_raise(ArgumentError){ chunk.append(nil, compress: :gzip) }\n    end\n\n    test 'some methods raise ArgumentError with an option of `compressed: :zstd` and without extending Compressble`' do\n      meta = Object.new\n      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)\n\n      assert_raise(ArgumentError){ chunk.read(compressed: :zstd) }\n      assert_raise(ArgumentError){ chunk.open(compressed: :zstd){} }\n      assert_raise(ArgumentError){ chunk.write_to(nil, compressed: :zstd) }\n      assert_raise(ArgumentError){ chunk.append(nil, compress: :zstd) }\n    end\n  end\n\n  class TestChunk < Fluent::Plugin::Buffer::Chunk\n    attr_accessor :data\n    def initialize(meta)\n      super\n      @data = ''\n    end\n    def size\n      @data.size\n    end\n    def open(**kwargs)\n      require 'stringio'\n      io = StringIO.new(@data)\n      yield io\n    end\n  end\n\n  sub_test_case 'minimum chunk implements #size and #open' do\n    test 'chunk lifecycle' do\n      c = TestChunk.new(Object.new)\n      assert c.unstaged?\n      assert !c.staged?\n      assert !c.queued?\n      assert !c.closed?\n      assert c.writable?\n\n      c.staged!\n\n      assert !c.unstaged?\n      assert c.staged?\n      assert !c.queued?\n      assert !c.closed?\n      assert c.writable?\n\n      c.enqueued!\n\n      assert !c.unstaged?\n      assert !c.staged?\n      assert c.queued?\n      assert !c.closed?\n      assert !c.writable?\n\n      c.close\n\n      assert !c.unstaged?\n      assert !c.staged?\n      assert !c.queued?\n      assert c.closed?\n      assert !c.writable?\n    end\n\n    test 'chunk can be unstaged' do\n      c = TestChunk.new(Object.new)\n      assert c.unstaged?\n      assert !c.staged?\n      assert !c.queued?\n      assert !c.closed?\n      assert c.writable?\n\n      c.staged!\n\n      assert !c.unstaged?\n      assert c.staged?\n      assert !c.queued?\n      assert !c.closed?\n      assert c.writable?\n\n      c.unstaged!\n\n      assert c.unstaged?\n      assert !c.staged?\n      assert !c.queued?\n      assert !c.closed?\n      assert c.writable?\n\n      c.enqueued!\n\n      assert !c.unstaged?\n      assert !c.staged?\n      assert c.queued?\n      assert !c.closed?\n      assert !c.writable?\n\n      c.close\n\n      assert !c.unstaged?\n      assert !c.staged?\n      assert !c.queued?\n      assert c.closed?\n      assert !c.writable?\n    end\n\n    test 'can respond to #empty? correctly' do\n      c = TestChunk.new(Object.new)\n      assert_equal 0, c.size\n      assert c.empty?\n    end\n\n    test 'can write its contents to io object' do\n      c = TestChunk.new(Object.new)\n      c.data << \"my data\\nyour data\\n\"\n      io = StringIO.new\n      c.write_to(io)\n      assert \"my data\\nyour data\\n\", io.to_s\n    end\n\n    test 'can feed objects into blocks with unpacking msgpack if ChunkMessagePackEventStreamer is included' do\n      require 'msgpack'\n      c = TestChunk.new(Object.new)\n      c.extend Fluent::ChunkMessagePackEventStreamer\n      c.data << MessagePack.pack(['my data', 1])\n      c.data << MessagePack.pack(['your data', 2])\n      ary = []\n      c.msgpack_each do |obj|\n        ary << obj\n      end\n      assert_equal ['my data', 1], ary[0]\n      assert_equal ['your data', 2], ary[1]\n    end\n  end\n\n  sub_test_case 'when compress is gzip' do\n    test 'create decompressable chunk' do\n      meta = Object.new\n      chunk = Fluent::Plugin::Buffer::Chunk.new(meta, compress: :gzip)\n      assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::GzipDecompressable)\n    end\n  end\n\n  sub_test_case 'when compress is zstd' do\n    test 'create decompressable chunk' do\n      meta = Object.new\n      chunk = Fluent::Plugin::Buffer::Chunk.new(meta, compress: :zstd)\n      assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::ZstdDecompressable)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_buffer_file_chunk.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/buffer/file_chunk'\nrequire 'fluent/plugin/compressable'\nrequire 'fluent/unique_id'\n\nrequire 'fileutils'\nrequire 'msgpack'\nrequire 'time'\nrequire 'timecop'\n\nclass BufferFileChunkTest < Test::Unit::TestCase\n  include Fluent::Plugin::Compressable\n\n  setup do\n    @klass = Fluent::Plugin::Buffer::FileChunk\n    @chunkdir = File.expand_path('../../tmp/buffer_file_chunk', __FILE__)\n    FileUtils.rm_r @chunkdir rescue nil\n    FileUtils.mkdir_p @chunkdir\n  end\n  teardown do\n    Timecop.return\n  end\n\n  Metadata = Fluent::Plugin::Buffer::Metadata\n\n  def gen_metadata(timekey: nil, tag: nil, variables: nil)\n    Metadata.new(timekey, tag, variables)\n  end\n\n  def read_metadata_file(path)\n    File.open(path, 'rb') do |f|\n      chunk = f.read\n      if chunk.size <= 6 # size of BUFFER_HEADER (2) + size of data(4)\n        return nil\n      end\n\n      data = nil\n      if chunk.slice(0, 2) == Fluent::Plugin::Buffer::FileChunk::BUFFER_HEADER\n        size = chunk.slice(2, 4).unpack('N').first\n        if size\n          data = MessagePack.unpack(chunk.slice(6, size), symbolize_keys: true)\n        end\n      else\n        # old type\n        data = MessagePack.unpack(chunk, symbolize_keys: true)\n      end\n\n      data\n    end\n  end\n\n  def gen_path(path)\n    File.join(@chunkdir, path)\n  end\n\n  def gen_test_chunk_id\n    require 'time'\n    now = Time.parse('2016-04-07 14:31:33 +0900')\n    u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | 1725) # 1725 is one of `rand(0xfff)`\n    u3 = 2979763054 # one of rand(0xffffffff)\n    u4 = 438020492  # ditto\n    [u1 >> 32, u1 & 0xffffffff, u3, u4].pack('NNNN')\n    # unique_id.unpack('N*').map{|n| n.to_s(16)}.join => \"52fde6425d7406bdb19b936e1a1ba98c\"\n  end\n\n  def hex_id(id)\n    id.unpack('N*').map{|n| n.to_s(16)}.join\n  end\n\n  sub_test_case 'classmethods' do\n    data(\n      correct_staged: ['/mydir/mypath/myfile.b00ff.log', :staged],\n      correct_queued: ['/mydir/mypath/myfile.q00ff.log', :queued],\n      incorrect_staged: ['/mydir/mypath/myfile.b00ff.log/unknown', :unknown],\n      incorrect_queued: ['/mydir/mypath/myfile.q00ff.log/unknown', :unknown],\n      output_file: ['/mydir/mypath/myfile.20160716.log', :unknown],\n    )\n    test 'can .assume_chunk_state' do |data|\n      path, expected = data\n      assert_equal expected, @klass.assume_chunk_state(path)\n    end\n\n    test '.generate_stage_chunk_path generates path with staged mark & chunk unique_id' do\n      assert_equal gen_path(\"mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log\"), @klass.generate_stage_chunk_path(gen_path(\"mychunk.*.log\"), gen_test_chunk_id)\n      assert_raise RuntimeError.new(\"BUG: buffer chunk path on stage MUST have '.*.'\") do\n        @klass.generate_stage_chunk_path(gen_path(\"mychunk.log\"), gen_test_chunk_id)\n      end\n      assert_raise RuntimeError.new(\"BUG: buffer chunk path on stage MUST have '.*.'\") do\n        @klass.generate_stage_chunk_path(gen_path(\"mychunk.*\"), gen_test_chunk_id)\n      end\n      assert_raise RuntimeError.new(\"BUG: buffer chunk path on stage MUST have '.*.'\") do\n        @klass.generate_stage_chunk_path(gen_path(\"*.log\"), gen_test_chunk_id)\n      end\n    end\n\n    test '.generate_queued_chunk_path generates path with enqueued mark for staged chunk path' do\n      assert_equal(\n        gen_path(\"mychunk.q52fde6425d7406bdb19b936e1a1ba98c.log\"),\n        @klass.generate_queued_chunk_path(gen_path(\"mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log\"), gen_test_chunk_id)\n      )\n    end\n\n    test '.generate_queued_chunk_path generates special path with chunk unique_id for non staged chunk path' do\n      assert_equal(\n        gen_path(\"mychunk.log.q52fde6425d7406bdb19b936e1a1ba98c.chunk\"),\n        @klass.generate_queued_chunk_path(gen_path(\"mychunk.log\"), gen_test_chunk_id)\n      )\n      assert_equal(\n        gen_path(\"mychunk.q55555555555555555555555555555555.log.q52fde6425d7406bdb19b936e1a1ba98c.chunk\"),\n        @klass.generate_queued_chunk_path(gen_path(\"mychunk.q55555555555555555555555555555555.log\"), gen_test_chunk_id)\n      )\n    end\n\n    test '.unique_id_from_path recreates unique_id from file path to assume unique_id for v0.12 chunks' do\n      assert_equal gen_test_chunk_id, @klass.unique_id_from_path(gen_path(\"mychunk.q52fde6425d7406bdb19b936e1a1ba98c.log\"))\n    end\n  end\n\n  sub_test_case 'newly created chunk' do\n    setup do\n      @chunk_path = File.join(@chunkdir, 'test.*.log')\n      @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :create)\n    end\n\n    def gen_chunk_path(prefix, unique_id)\n      File.join(@chunkdir, \"test.#{prefix}#{Fluent::UniqueId.hex(unique_id)}.log\")\n    end\n\n    teardown do\n      if @c\n        @c.purge rescue nil\n      end\n      if File.exist? @chunk_path\n        File.unlink @chunk_path\n      end\n    end\n\n    test 'creates new files for chunk and metadata with specified path & permission' do\n      assert{ @c.unique_id.size == 16 }\n      assert_equal gen_chunk_path('b', @c.unique_id), @c.path\n\n      assert File.exist?(gen_chunk_path('b', @c.unique_id))\n      assert{ File.stat(gen_chunk_path('b', @c.unique_id)).mode.to_s(8).end_with?(Fluent::DEFAULT_FILE_PERMISSION.to_s(8)) }\n\n      assert File.exist?(gen_chunk_path('b', @c.unique_id) + '.meta')\n      assert{ File.stat(gen_chunk_path('b', @c.unique_id) + '.meta').mode.to_s(8).end_with?(Fluent::DEFAULT_FILE_PERMISSION.to_s(8)) }\n\n      assert_equal :unstaged, @c.state\n      assert @c.empty?\n    end\n\n    test 'can #append, #commit and #read it' do\n      assert @c.empty?\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n\n      content = @c.read\n      ds = content.split(\"\\n\").select{|d| !d.empty? }\n\n      assert_equal 2, ds.size\n      assert_equal d1, JSON.parse(ds[0])\n      assert_equal d2, JSON.parse(ds[1])\n\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n      @c.commit\n\n      content = @c.read\n      ds = content.split(\"\\n\").select{|d| !d.empty? }\n\n      assert_equal 4, ds.size\n      assert_equal d1, JSON.parse(ds[0])\n      assert_equal d2, JSON.parse(ds[1])\n      assert_equal d3, JSON.parse(ds[2])\n      assert_equal d4, JSON.parse(ds[3])\n    end\n\n    test 'can #concat, #commit and #read it' do\n      assert @c.empty?\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"].join\n      @c.concat(data, 2)\n      @c.commit\n\n      content = @c.read\n      ds = content.split(\"\\n\").select{|d| !d.empty? }\n\n      assert_equal 2, ds.size\n      assert_equal d1, JSON.parse(ds[0])\n      assert_equal d2, JSON.parse(ds[1])\n\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.concat([d3.to_json + \"\\n\", d4.to_json + \"\\n\"].join, 2)\n      @c.commit\n\n      content = @c.read\n      ds = content.split(\"\\n\").select{|d| !d.empty? }\n\n      assert_equal 4, ds.size\n      assert_equal d1, JSON.parse(ds[0])\n      assert_equal d2, JSON.parse(ds[1])\n      assert_equal d3, JSON.parse(ds[2])\n      assert_equal d4, JSON.parse(ds[3])\n    end\n\n    test 'has its contents in binary (ascii-8bit)' do\n      data1 = \"aaa bbb ccc\".force_encoding('utf-8')\n      @c.append([data1])\n      @c.commit\n      assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.external_encoding }\n\n      content = @c.read\n      assert_equal Encoding::ASCII_8BIT, content.encoding\n    end\n\n    test 'has #bytesize and #size' do\n      assert @c.empty?\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      @c.commit\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      first_bytesize = @c.bytesize\n\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n\n      assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 4, @c.size\n\n      @c.commit\n\n      assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 4, @c.size\n    end\n\n    test 'can #rollback to revert non-committed data' do\n      assert @c.empty?\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      @c.rollback\n\n      assert @c.empty?\n\n      assert_equal '', File.open(@c.path, 'rb'){|f| f.read }\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      first_bytesize = @c.bytesize\n\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n\n      assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 4, @c.size\n\n      @c.rollback\n\n      assert_equal first_bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\"), File.open(@c.path, 'rb'){|f| f.read }\n    end\n\n    test 'can #rollback to revert non-committed data from #concat' do\n      assert @c.empty?\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"].join\n      @c.concat(data, 2)\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      @c.rollback\n\n      assert @c.empty?\n\n      assert_equal '', File.open(@c.path, 'rb'){|f| f.read }\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      first_bytesize = @c.bytesize\n\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.concat([d3.to_json + \"\\n\", d4.to_json + \"\\n\"].join, 2)\n\n      assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 4, @c.size\n\n      @c.rollback\n\n      assert_equal first_bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\"), File.open(@c.path, 'rb'){|f| f.read }\n    end\n\n    test 'can store its data by #close' do\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n      @c.commit\n\n      content = @c.read\n\n      unique_id = @c.unique_id\n      size = @c.size\n      created_at = @c.created_at\n      modified_at = @c.modified_at\n\n      @c.close\n\n      assert_equal content, File.open(@c.path, 'rb'){|f| f.read }\n\n      stored_meta = {\n        timekey: nil, tag: nil, variables: nil, seq: 0,\n        id: unique_id,\n        s: size,\n        c: created_at.to_i,\n        m: modified_at.to_i,\n      }\n\n      assert_equal stored_meta, read_metadata_file(@c.path + '.meta')\n    end\n\n    test 'deletes all data by #purge' do\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n      @c.commit\n\n      @c.purge\n\n      assert @c.empty?\n      assert_equal 0, @c.bytesize\n      assert_equal 0, @c.size\n\n      assert !File.exist?(@c.path)\n      assert !File.exist?(@c.path + '.meta')\n    end\n\n    test 'can #open its contents as io' do\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n      @c.commit\n\n      lines = []\n      @c.open do |io|\n        assert io\n        io.readlines.each do |l|\n          lines << l\n        end\n      end\n\n      assert_equal d1.to_json + \"\\n\", lines[0]\n      assert_equal d2.to_json + \"\\n\", lines[1]\n      assert_equal d3.to_json + \"\\n\", lines[2]\n      assert_equal d4.to_json + \"\\n\", lines[3]\n    end\n\n    test '#write_metadata tries to store metadata on file' do\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n\n      expected = {\n        timekey: nil, tag: nil, variables: nil, seq: 0,\n        id: @c.unique_id,\n        s: @c.size,\n        c: @c.created_at.to_i,\n        m: @c.modified_at.to_i,\n      }\n      assert_equal expected, read_metadata_file(@c.path + '.meta')\n\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n      # append does write_metadata\n\n      dummy_now = Time.parse('2016-04-07 16:59:59 +0900')\n      Timecop.freeze(dummy_now)\n      @c.write_metadata\n\n      expected = {\n        timekey: nil, tag: nil, variables: nil, seq: 0,\n        id: @c.unique_id,\n        s: @c.size,\n        c: @c.created_at.to_i,\n        m: dummy_now.to_i,\n      }\n      assert_equal expected, read_metadata_file(@c.path + '.meta')\n\n      @c.commit\n\n      expected = {\n        timekey: nil, tag: nil, variables: nil, seq: 0,\n        id: @c.unique_id,\n        s: @c.size,\n        c: @c.created_at.to_i,\n        m: @c.modified_at.to_i,\n      }\n      assert_equal expected, read_metadata_file(@c.path + '.meta')\n\n      content = @c.read\n\n      unique_id = @c.unique_id\n      size = @c.size\n      created_at = @c.created_at\n      modified_at = @c.modified_at\n\n      @c.close\n\n      assert_equal content, File.open(@c.path, 'rb'){|f| f.read }\n\n      stored_meta = {\n        timekey: nil, tag: nil, variables: nil, seq: 0,\n        id: unique_id,\n        s: size,\n        c: created_at.to_i,\n        m: modified_at.to_i,\n      }\n\n      assert_equal stored_meta, read_metadata_file(@c.path + '.meta')\n    end\n  end\n\n  test 'ensure to remove metadata file if #write_metadata raise an error because of disk full' do\n    chunk_path = File.join(@chunkdir, 'test.*.log')\n    stub(Fluent::UniqueId).hex(anything) { 'id' } # to fix chunk id\n\n    any_instance_of(Fluent::Plugin::Buffer::FileChunk) do |klass|\n      stub(klass).write_metadata(anything) do |v|\n        raise 'disk full'\n      end\n    end\n\n    err = assert_raise(Fluent::Plugin::Buffer::BufferOverflowError) do\n      Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, chunk_path, :create)\n    end\n\n    assert_false File.exist?(File.join(@chunkdir, 'test.bid.log.meta'))\n    assert_match(/create buffer metadata/, err.message)\n  end\n\n  sub_test_case 'chunk with file for staged chunk' do\n    setup do\n      @chunk_id = gen_test_chunk_id\n      @chunk_path = File.join(@chunkdir, \"test_staged.b#{hex_id(@chunk_id)}.log\")\n      @enqueued_path = File.join(@chunkdir, \"test_staged.q#{hex_id(@chunk_id)}.log\")\n\n      @d1 = {\"k\" => \"x\", \"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      @d2 = {\"k\" => \"x\", \"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      @d3 = {\"k\" => \"x\", \"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      @d4 = {\"k\" => \"x\", \"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + \"\\n\" }.join\n      File.open(@chunk_path, 'wb') do |f|\n        f.write @d\n      end\n\n      @metadata = {\n        timekey: nil, tag: 'testing', variables: {k: \"x\"}, seq: 0,\n        id: @chunk_id,\n        s: 4,\n        c: Time.parse('2016-04-07 17:44:00 +0900').to_i,\n        m: Time.parse('2016-04-07 17:44:13 +0900').to_i,\n      }\n      File.open(@chunk_path + '.meta', 'wb') do |f|\n        f.write @metadata.to_msgpack\n      end\n\n      @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :staged)\n    end\n\n    teardown do\n      if @c\n        @c.purge rescue nil\n      end\n      [@chunk_path, @chunk_path + '.meta', @enqueued_path, @enqueued_path + '.meta'].each do |path|\n        File.unlink path if File.exist? path\n      end\n    end\n\n    test 'can load as staged chunk from file with metadata' do\n      assert_equal @chunk_path, @c.path\n      assert_equal :staged, @c.state\n\n      assert_nil @c.metadata.timekey\n      assert_equal 'testing', @c.metadata.tag\n      assert_equal({k: \"x\"}, @c.metadata.variables)\n\n      assert_equal 4, @c.size\n      assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at\n      assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at\n\n      content = @c.read\n      assert_equal @d, content\n    end\n\n    test 'can be enqueued' do\n      stage_path = @c.path\n      queue_path = @enqueued_path\n      assert File.exist?(stage_path)\n      assert File.exist?(stage_path + '.meta')\n      assert !File.exist?(queue_path)\n      assert !File.exist?(queue_path + '.meta')\n\n      @c.enqueued!\n\n      assert_equal queue_path, @c.path\n\n      assert !File.exist?(stage_path)\n      assert !File.exist?(stage_path + '.meta')\n      assert File.exist?(queue_path)\n      assert File.exist?(queue_path + '.meta')\n\n      assert_nil @c.metadata.timekey\n      assert_equal 'testing', @c.metadata.tag\n      assert_equal({k: \"x\"}, @c.metadata.variables)\n\n      assert_equal 4, @c.size\n      assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at\n      assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at\n\n      assert_equal @d, File.open(@c.path, 'rb'){|f| f.read }\n      assert_equal @metadata, read_metadata_file(@c.path + '.meta')\n    end\n\n    test '#write_metadata tries to store metadata on file with non-committed data' do\n      d5 = {\"k\" => \"x\", \"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      d5s = d5.to_json + \"\\n\"\n      @c.append([d5s])\n\n      metadata = {\n        timekey: nil, tag: 'testing', variables: {k: \"x\"}, seq: 0,\n        id: @chunk_id,\n        s: 4,\n        c: Time.parse('2016-04-07 17:44:00 +0900').to_i,\n        m: Time.parse('2016-04-07 17:44:13 +0900').to_i,\n      }\n      assert_equal metadata, read_metadata_file(@c.path + '.meta')\n\n      @c.write_metadata\n\n      metadata = {\n        timekey: nil, tag: 'testing', variables: {k: \"x\"}, seq: 0,\n        id: @chunk_id,\n        s: 5,\n        c: Time.parse('2016-04-07 17:44:00 +0900').to_i,\n        m: Time.parse('2016-04-07 17:44:38 +0900').to_i,\n      }\n\n      dummy_now = Time.parse('2016-04-07 17:44:38 +0900')\n      Timecop.freeze(dummy_now)\n      @c.write_metadata\n\n      assert_equal metadata, read_metadata_file(@c.path + '.meta')\n    end\n\n    test '#file_rename can rename chunk files even in windows, and call callback with file size' do\n      data = \"aaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccc\"\n\n      testing_file1 = gen_path('rename1.test')\n      testing_file2 = gen_path('rename2.test')\n      f = File.open(testing_file1, 'wb', @c.permission)\n      f.set_encoding(Encoding::ASCII_8BIT)\n      f.sync = true\n      f.binmode\n      f.write data\n      pos = f.pos\n\n      assert f.binmode?\n      assert f.sync\n      assert_equal data.bytesize, f.size\n\n      io = nil\n      @c.file_rename(f, testing_file1, testing_file2, ->(new_io){ io = new_io })\n      assert io\n      if Fluent.windows?\n        assert{ f != io }\n      else\n        assert_equal f, io\n      end\n      assert_equal Encoding::ASCII_8BIT, io.external_encoding\n      assert io.sync\n      assert io.binmode?\n      assert_equal data.bytesize, io.size\n\n      assert_equal pos, io.pos\n\n      assert_equal '', io.read\n\n      io.rewind\n      assert_equal data, io.read\n    end\n  end\n\n  sub_test_case 'chunk with file for enqueued chunk' do\n    setup do\n      @chunk_id = gen_test_chunk_id\n      @enqueued_path = File.join(@chunkdir, \"test_staged.q#{hex_id(@chunk_id)}.log\")\n\n      @d1 = {\"k\" => \"x\", \"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      @d2 = {\"k\" => \"x\", \"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      @d3 = {\"k\" => \"x\", \"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      @d4 = {\"k\" => \"x\", \"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + \"\\n\" }.join\n      File.open(@enqueued_path, 'wb') do |f|\n        f.write @d\n      end\n\n      @dummy_timekey = Time.parse('2016-04-07 17:40:00 +0900').to_i\n\n      @metadata = {\n        timekey: @dummy_timekey, tag: 'testing', variables: {k: \"x\"}, seq: 0,\n        id: @chunk_id,\n        s: 4,\n        c: Time.parse('2016-04-07 17:44:00 +0900').to_i,\n        m: Time.parse('2016-04-07 17:44:13 +0900').to_i,\n      }\n      File.open(@enqueued_path + '.meta', 'wb') do |f|\n        f.write @metadata.to_msgpack\n      end\n\n      @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @enqueued_path, :queued)\n    end\n\n    teardown do\n      if @c\n        @c.purge rescue nil\n      end\n      [@enqueued_path, @enqueued_path + '.meta'].each do |path|\n        File.unlink path if File.exist? path\n      end\n    end\n\n    test 'can load as queued chunk (read only) with metadata' do\n      assert @c\n      assert_equal @chunk_id, @c.unique_id\n      assert_equal :queued, @c.state\n      assert_equal gen_metadata(timekey: @dummy_timekey, tag: 'testing', variables: {k: \"x\"}), @c.metadata\n      assert_equal Time.at(@metadata[:c]), @c.created_at\n      assert_equal Time.at(@metadata[:m]), @c.modified_at\n      assert_equal @metadata[:s], @c.size\n      assert_equal @d.bytesize, @c.bytesize\n      assert_equal @d, @c.read\n\n      assert_raise RuntimeError.new(\"BUG: concatenating to unwritable chunk, now 'queued'\") do\n        @c.append([\"queued chunk is read only\"])\n      end\n      assert_raise IOError do\n        @c.instance_eval{ @chunk }.write \"chunk io is opened as read only\"\n      end\n    end\n  end\n\n  sub_test_case 'chunk with queued chunk file of v0.12, without metadata' do\n    setup do\n      @chunk_id = gen_test_chunk_id\n      @chunk_path = File.join(@chunkdir, \"test_v12.2016040811.q#{hex_id(@chunk_id)}.log\")\n\n      @d1 = {\"k\" => \"x\", \"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      @d2 = {\"k\" => \"x\", \"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      @d3 = {\"k\" => \"x\", \"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      @d4 = {\"k\" => \"x\", \"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + \"\\n\" }.join\n      File.open(@chunk_path, 'wb') do |f|\n        f.write @d\n      end\n\n      @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :queued)\n    end\n\n    teardown do\n      if @c\n        @c.purge rescue nil\n      end\n      File.unlink @chunk_path if File.exist? @chunk_path\n    end\n\n    test 'can load as queued chunk from file without metadata' do\n      assert @c\n      assert_equal :queued, @c.state\n      assert_equal @chunk_id, @c.unique_id\n      assert_equal gen_metadata, @c.metadata\n      assert_equal @d.bytesize, @c.bytesize\n      assert_equal 0, @c.size\n      assert_equal @d, @c.read\n\n      assert_raise RuntimeError.new(\"BUG: concatenating to unwritable chunk, now 'queued'\") do\n        @c.append([\"queued chunk is read only\"])\n      end\n      assert_raise IOError do\n        @c.instance_eval{ @chunk }.write \"chunk io is opened as read only\"\n      end\n    end\n  end\n\n  sub_test_case 'chunk with staged chunk file of v0.12, without metadata' do\n    setup do\n      @chunk_id = gen_test_chunk_id\n      @chunk_path = File.join(@chunkdir, \"test_v12.2016040811.b#{hex_id(@chunk_id)}.log\")\n\n      @d1 = {\"k\" => \"x\", \"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      @d2 = {\"k\" => \"x\", \"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      @d3 = {\"k\" => \"x\", \"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      @d4 = {\"k\" => \"x\", \"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + \"\\n\" }.join\n      File.open(@chunk_path, 'wb') do |f|\n        f.write @d\n      end\n\n      @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :staged)\n    end\n\n    teardown do\n      if @c\n        @c.purge rescue nil\n      end\n      File.unlink @chunk_path if File.exist? @chunk_path\n    end\n\n    test 'can load as queued chunk from file without metadata even if it was loaded as staged chunk' do\n      assert @c\n      assert_equal :queued, @c.state\n      assert_equal @chunk_id, @c.unique_id\n      assert_equal gen_metadata, @c.metadata\n      assert_equal @d.bytesize, @c.bytesize\n      assert_equal 0, @c.size\n      assert_equal @d, @c.read\n\n      assert_raise RuntimeError.new(\"BUG: concatenating to unwritable chunk, now 'queued'\") do\n        @c.append([\"queued chunk is read only\"])\n      end\n      assert_raise IOError do\n        @c.instance_eval{ @chunk }.write \"chunk io is opened as read only\"\n      end\n    end\n  end\n\n  sub_test_case 'compressed buffer' do\n    setup do\n      @src = 'text data for compressing' * 5\n      @gzipped_src = compress(@src)\n      @zstded_src = compress(@src, type: :zstd)\n    end\n\n    test '#append with compress option writes  compressed data to chunk when compress is gzip' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)\n      c.append([@src, @src], compress: :gzip)\n      c.commit\n\n      # check chunk is compressed\n      assert c.read(compressed: :gzip).size < [@src, @src].join(\"\").size\n\n      assert_equal @src + @src, c.read\n    end\n\n    test '#open passes io object having decompressed data to a block when compress is gzip' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      decomressed_data = c.open do |io|\n        v = io.read\n        assert_equal @src, v\n        v\n      end\n      assert_equal @src, decomressed_data\n    end\n\n    test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      comressed_data = c.open(compressed: :gzip) do |io|\n        v = io.read\n        assert_equal @gzipped_src, v\n        v\n      end\n      assert_equal @gzipped_src, comressed_data\n    end\n\n    test '#write_to writes decompressed data when compress is gzip' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @gzipped_src, c.read(compressed: :gzip)\n\n      io = StringIO.new\n      c.write_to(io)\n      assert_equal @src, io.string\n    end\n\n    test '#write_to with compressed option writes compressed data when compress is gzip' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @gzipped_src, c.read(compressed: :gzip)\n\n      io = StringIO.new\n      io.set_encoding(Encoding::ASCII_8BIT)\n      c.write_to(io, compressed: :gzip)\n      assert_equal @gzipped_src, io.string\n    end\n\n    test '#append with compress option writes  compressed data to chunk when compress is zstd' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd)\n      c.append([@src, @src], compress: :zstd)\n      c.commit\n\n      # check chunk is compressed\n      assert c.read(compressed: :zstd).size < [@src, @src].join(\"\").size\n\n      assert_equal @src + @src, c.read\n    end\n\n    test '#open passes io object having decompressed data to a block when compress is zstd' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      decomressed_data = c.open do |io|\n        v = io.read\n        assert_equal @src, v\n        v\n      end\n      assert_equal @src, decomressed_data\n    end\n\n    test '#open with compressed option passes io object having decompressed data to a block when compress is zstd' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      comressed_data = c.open(compressed: :zstd) do |io|\n        v = io.read\n        assert_equal @zstded_src, v\n        v\n      end\n      assert_equal @zstded_src, comressed_data\n    end\n\n    test '#write_to writes decompressed data when compress is zstd' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @zstded_src, c.read(compressed: :zstd)\n\n      io = StringIO.new\n      c.write_to(io)\n      assert_equal @src, io.string\n    end\n\n    test '#write_to with compressed option writes compressed data when compress is zstd' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @zstded_src, c.read(compressed: :zstd)\n\n      io = StringIO.new\n      io.set_encoding(Encoding::ASCII_8BIT)\n      c.write_to(io, compressed: :zstd)\n      assert_equal @zstded_src, io.string\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_buffer_file_single_chunk.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/buffer/file_single_chunk'\nrequire 'fluent/plugin/compressable'\nrequire 'fluent/unique_id'\n\nrequire 'fileutils'\nrequire 'msgpack'\nrequire 'time'\n\nclass BufferFileSingleChunkTest < Test::Unit::TestCase\n  include Fluent::Plugin::Compressable\n\n  setup do\n    @klass = Fluent::Plugin::Buffer::FileSingleChunk\n    @chunkdir = File.expand_path('../../tmp/buffer_file_single_chunk', __FILE__)\n    FileUtils.rm_r(@chunkdir) rescue nil\n    FileUtils.mkdir_p(@chunkdir)\n  end\n\n  Metadata = Struct.new(:timekey, :tag, :variables)\n  def gen_metadata(timekey: nil, tag: 'testing', variables: nil)\n    Metadata.new(timekey, tag, variables)\n  end\n\n  def gen_path(path)\n    File.join(@chunkdir, path)\n  end\n\n  def gen_test_chunk_id\n    now = Time.parse('2016-04-07 14:31:33 +0900')\n    u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | 1725) # 1725 is one of `rand(0xfff)`\n    u3 = 2979763054 # one of rand(0xffffffff)\n    u4 = 438020492  # ditto\n    [u1 >> 32, u1 & 0xffffffff, u3, u4].pack('NNNN')\n    # unique_id.unpack('N*').map{|n| n.to_s(16)}.join => \"52fde6425d7406bdb19b936e1a1ba98c\"\n  end\n\n  def hex_id(id)\n    id.unpack('N*').map { |n| n.to_s(16) }.join\n  end\n\n  sub_test_case 'classmethods' do\n    data(\n      correct_staged: ['/mydir/mypath/fsb.b00ff.buf', :staged],\n      correct_queued: ['/mydir/mypath/fsb.q00ff.buf', :queued],\n      incorrect_staged: ['/mydir/mypath/fsb.b00ff.buf/unknown', :unknown],\n      incorrect_queued: ['/mydir/mypath/fsb.q00ff.buf/unknown', :unknown],\n      output_file: ['/mydir/mypath/fsb.20160716.buf', :unknown],\n    )\n    test 'can .assume_chunk_state' do |data|\n      path, expected = data\n      assert_equal expected, @klass.assume_chunk_state(path)\n    end\n\n    test '.generate_stage_chunk_path generates path with staged mark & chunk unique_id' do\n      assert_equal gen_path(\"fsb.foo.b52fde6425d7406bdb19b936e1a1ba98c.buf\"), @klass.generate_stage_chunk_path(gen_path(\"fsb.*.buf\"), 'foo', gen_test_chunk_id)\n      assert_raise RuntimeError.new(\"BUG: buffer chunk path on stage MUST have '.*.'\") do\n        @klass.generate_stage_chunk_path(gen_path(\"fsb.buf\"), 'foo', gen_test_chunk_id)\n      end\n      assert_raise RuntimeError.new(\"BUG: buffer chunk path on stage MUST have '.*.'\") do\n        @klass.generate_stage_chunk_path(gen_path(\"fsb.*\"), 'foo', gen_test_chunk_id)\n      end\n      assert_raise RuntimeError.new(\"BUG: buffer chunk path on stage MUST have '.*.'\") do\n        @klass.generate_stage_chunk_path(gen_path(\"*.buf\"), 'foo', gen_test_chunk_id)\n      end\n    end\n\n    test '.generate_queued_chunk_path generates path with enqueued mark for staged chunk path' do\n      assert_equal(\n        gen_path(\"fsb.q52fde6425d7406bdb19b936e1a1ba98c.buf\"),\n        @klass.generate_queued_chunk_path(gen_path(\"fsb.b52fde6425d7406bdb19b936e1a1ba98c.buf\"), gen_test_chunk_id)\n      )\n    end\n\n    test '.generate_queued_chunk_path generates special path with chunk unique_id for non staged chunk path' do\n      assert_equal(\n        gen_path(\"fsb.buf.q52fde6425d7406bdb19b936e1a1ba98c.chunk\"),\n        @klass.generate_queued_chunk_path(gen_path(\"fsb.buf\"), gen_test_chunk_id)\n      )\n      assert_equal(\n        gen_path(\"fsb.q55555555555555555555555555555555.buf.q52fde6425d7406bdb19b936e1a1ba98c.chunk\"),\n        @klass.generate_queued_chunk_path(gen_path(\"fsb.q55555555555555555555555555555555.buf\"), gen_test_chunk_id)\n      )\n    end\n\n    data('1 word tag' => 'foo',\n         '2 words tag' => 'test.log',\n         'empty' => '')\n    test '.unique_id_and_key_from_path recreates unique_id and key from file path' do |key|\n      path = @klass.unique_id_and_key_from_path(gen_path(\"fsb.#{key}.q52fde6425d7406bdb19b936e1a1ba98c.buf\"))\n      assert_equal [gen_test_chunk_id, key], path\n    end\n  end\n\n  sub_test_case 'newly created chunk' do\n    setup do\n      @path_conf = File.join(@chunkdir, 'fsb.*.buf')\n      @chunk_path = File.join(@chunkdir, \"fsb.testing.b#{hex_id(gen_test_chunk_id)}.buf\")\n      @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @path_conf, :create, nil)\n    end\n\n    def gen_chunk_path(prefix, unique_id)\n      File.join(@chunkdir, \"fsb.testing.#{prefix}#{Fluent::UniqueId.hex(unique_id)}.buf\")\n    end\n\n    teardown do\n      if @c\n        @c.purge rescue nil\n      end\n      if File.exist?(@chunk_path)\n        File.unlink(@chunk_path)\n      end\n    end\n\n    test 'creates new files for chunk and metadata with specified path & permission' do\n      assert_equal 16, @c.unique_id.size\n      assert_equal gen_chunk_path('b', @c.unique_id), @c.path\n\n      assert File.exist?(gen_chunk_path('b', @c.unique_id))\n      assert { File.stat(gen_chunk_path('b', @c.unique_id)).mode.to_s(8).end_with?(Fluent::DEFAULT_FILE_PERMISSION.to_s(8)) }\n\n      assert_equal :unstaged, @c.state\n      assert @c.empty?\n    end\n\n    test 'can #append, #commit and #read it' do\n      assert @c.empty?\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n\n      ds = @c.read.split(\"\\n\").select { |d| !d.empty? }\n      assert_equal 2, ds.size\n      assert_equal d1, JSON.parse(ds[0])\n      assert_equal d2, JSON.parse(ds[1])\n\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n      @c.commit\n\n      ds = @c.read.split(\"\\n\").select{|d| !d.empty? }\n      assert_equal 4, ds.size\n      assert_equal d1, JSON.parse(ds[0])\n      assert_equal d2, JSON.parse(ds[1])\n      assert_equal d3, JSON.parse(ds[2])\n      assert_equal d4, JSON.parse(ds[3])\n    end\n\n    test 'can #concat, #commit and #read it' do\n      assert @c.empty?\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"].join\n      @c.concat(data, 2)\n      @c.commit\n\n      ds = @c.read.split(\"\\n\").select{|d| !d.empty? }\n      assert_equal 2, ds.size\n      assert_equal d1, JSON.parse(ds[0])\n      assert_equal d2, JSON.parse(ds[1])\n\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.concat([d3.to_json + \"\\n\", d4.to_json + \"\\n\"].join, 2)\n      @c.commit\n\n      ds = @c.read.split(\"\\n\").select { |d| !d.empty? }\n      assert_equal 4, ds.size\n      assert_equal d1, JSON.parse(ds[0])\n      assert_equal d2, JSON.parse(ds[1])\n      assert_equal d3, JSON.parse(ds[2])\n      assert_equal d4, JSON.parse(ds[3])\n    end\n\n    test 'has its contents in binary (ascii-8bit)' do\n      data1 = \"aaa bbb ccc\".force_encoding('utf-8')\n      @c.append([data1])\n      @c.commit\n      assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.external_encoding }\n      assert_equal Encoding::ASCII_8BIT, @c.read.encoding\n    end\n\n    test 'has #bytesize and #size' do\n      assert @c.empty?\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      @c.commit\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      first_bytesize = @c.bytesize\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n\n      assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 4, @c.size\n\n      @c.commit\n\n      assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 4, @c.size\n    end\n\n    test 'can #rollback to revert non-committed data' do\n      assert @c.empty?\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      @c.rollback\n\n      assert @c.empty?\n      assert_equal '', File.read(@c.path)\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      first_bytesize = @c.bytesize\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n\n      assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 4, @c.size\n\n      @c.rollback\n\n      assert_equal first_bytesize, @c.bytesize\n      assert_equal 2, @c.size\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\"), File.read(@c.path)\n    end\n\n    test 'can #rollback to revert non-committed data from #concat' do\n      assert @c.empty?\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"].join\n      @c.concat(data, 2)\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      @c.rollback\n\n      assert @c.empty?\n      assert_equal '', File.read(@c.path)\n\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 2, @c.size\n\n      first_bytesize = @c.bytesize\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.concat([d3.to_json + \"\\n\", d4.to_json + \"\\n\"].join, 2)\n\n      assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n      assert_equal 4, @c.size\n\n      @c.rollback\n\n      assert_equal first_bytesize, @c.bytesize\n      assert_equal 2, @c.size\n      assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\"), File.read(@c.path)\n    end\n\n    test 'can store its data by #close' do\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n      @c.commit\n      content = @c.read\n      @c.close\n\n      assert_equal content, File.read(@c.path)\n    end\n\n    test 'deletes all data by #purge' do\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n      @c.commit\n      @c.purge\n\n      assert @c.empty?\n      assert_equal 0, @c.bytesize\n      assert_equal 0, @c.size\n      assert !File.exist?(@c.path)\n    end\n\n    test 'can #open its contents as io' do\n      d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n      @c.append(data)\n      @c.commit\n      d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n      @c.commit\n\n      lines = []\n      @c.open do |io|\n        assert io\n        io.readlines.each do |l|\n          lines << l\n        end\n      end\n\n      assert_equal d1.to_json + \"\\n\", lines[0]\n      assert_equal d2.to_json + \"\\n\", lines[1]\n      assert_equal d3.to_json + \"\\n\", lines[2]\n      assert_equal d4.to_json + \"\\n\", lines[3]\n    end\n  end\n\n  sub_test_case 'chunk with file for staged chunk' do\n    setup do\n      @chunk_id = gen_test_chunk_id\n      @staged_path = File.join(@chunkdir, \"fsb.testing.b#{hex_id(@chunk_id)}.buf\")\n      @enqueued_path = File.join(@chunkdir, \"fsb.testing.q#{hex_id(@chunk_id)}.buf\")\n\n      @d1 = {\"k\" => \"x\", \"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      @d2 = {\"k\" => \"x\", \"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      @d3 = {\"k\" => \"x\", \"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      @d4 = {\"k\" => \"x\", \"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @d = [@d1, @d2, @d3, @d4].map{ |d| d.to_json + \"\\n\" }.join\n      File.write(@staged_path, @d, :mode => 'wb')\n\n      @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @staged_path, :staged, nil)\n    end\n\n    teardown do\n      if @c\n        @c.purge rescue nil\n      end\n      [@staged_path, @enqueued_path].each do |path|\n        File.unlink(path) if File.exist?(path)\n      end\n    end\n\n    test 'can load as staged chunk from file with metadata' do\n      assert_equal @staged_path, @c.path\n      assert_equal :staged, @c.state\n\n      assert_nil @c.metadata.timekey\n      assert_equal 'testing', @c.metadata.tag\n      assert_nil @c.metadata.variables\n      assert_equal 0, @c.size\n      assert_equal @d, @c.read\n\n      @c.restore_size(:text)\n      assert_equal 4, @c.size\n    end\n\n    test 'can be enqueued' do\n      stage_path = @c.path\n      queue_path = @enqueued_path\n      assert File.exist?(stage_path)\n      assert !File.exist?(queue_path)\n\n      @c.enqueued!\n\n      assert_equal queue_path, @c.path\n      assert !File.exist?(stage_path)\n      assert File.exist?(queue_path)\n\n      assert_nil @c.metadata.timekey\n      assert_equal 'testing', @c.metadata.tag\n      assert_nil @c.metadata.variables\n\n      assert_equal 0, @c.size\n      assert_equal @d, File.read(@c.path)\n\n      @c.restore_size(:text)\n      assert_equal 4, @c.size\n    end\n\n    test '#file_rename can rename chunk files even in windows, and call callback with file size' do\n      data = \"aaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccc\"\n\n      testing_file1 = gen_path('rename1.test')\n      testing_file2 = gen_path('rename2.test')\n      f = File.open(testing_file1, 'wb', @c.permission)\n      f.set_encoding(Encoding::ASCII_8BIT)\n      f.sync = true\n      f.binmode\n      f.write data\n      pos = f.pos\n\n      assert f.binmode?\n      assert f.sync\n      assert_equal data.bytesize, f.size\n\n      io = nil\n      @c.file_rename(f, testing_file1, testing_file2, ->(new_io){ io = new_io })\n      assert io\n      if Fluent.windows?\n        assert { f != io }\n      else\n        assert_equal f, io\n      end\n      assert_equal Encoding::ASCII_8BIT, io.external_encoding\n      assert io.sync\n      assert io.binmode?\n      assert_equal data.bytesize, io.size\n      assert_equal pos, io.pos\n      assert_equal '', io.read\n\n      io.rewind\n      assert_equal data, io.read\n    end\n  end\n\n  sub_test_case 'chunk with file for enqueued chunk' do\n    setup do\n      @chunk_id = gen_test_chunk_id\n      @enqueued_path = File.join(@chunkdir, \"fsb.testing.q#{hex_id(@chunk_id)}.buf\")\n\n      @d1 = {\"k\" => \"x\", \"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      @d2 = {\"k\" => \"x\", \"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      @d3 = {\"k\" => \"x\", \"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      @d4 = {\"k\" => \"x\", \"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @d = [@d1, @d2, @d3, @d4].map { |d| d.to_json + \"\\n\" }.join\n      File.write(@enqueued_path, @d, :mode => 'wb')\n\n      @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @enqueued_path, :queued, nil)\n    end\n\n    teardown do\n      if @c\n        @c.purge rescue nil\n      end\n      File.unlink(@enqueued_path) if File.exist?(@enqueued_path)\n    end\n\n    test 'can load as queued chunk (read only) with metadata' do\n      assert @c\n      assert_equal @chunk_id, @c.unique_id\n      assert_equal :queued, @c.state\n      stat = File.stat(@enqueued_path)\n      assert_equal stat.ctime.to_i, @c.created_at.to_i\n      assert_equal stat.mtime.to_i, @c.modified_at.to_i\n      assert_equal 0, @c.size\n      assert_equal @d.bytesize, @c.bytesize\n      assert_equal @d, @c.read\n\n      @c.restore_size(:text)\n      assert_equal 4, @c.size\n\n      assert_raise RuntimeError.new(\"BUG: concatenating to unwritable chunk, now 'queued'\") do\n        @c.append([\"queued chunk is read only\"])\n      end\n      assert_raise IOError do\n        @c.instance_eval{ @chunk }.write \"chunk io is opened as read only\"\n      end\n    end\n  end\n\n  sub_test_case 'chunk with queued chunk file' do\n    setup do\n      @chunk_id = gen_test_chunk_id\n      @chunk_path = File.join(@chunkdir, \"fsb.testing.q#{hex_id(@chunk_id)}.buf\")\n\n      @d1 = {\"k\" => \"x\", \"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n      @d2 = {\"k\" => \"x\", \"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n      @d3 = {\"k\" => \"x\", \"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n      @d4 = {\"k\" => \"x\", \"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n      @d = [@d1, @d2, @d3, @d4].map { |d| d.to_json + \"\\n\" }.join\n      File.write(@chunk_path, @d, :mode => 'wb')\n\n      @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @chunk_path, :queued, nil)\n    end\n\n    teardown do\n      if @c\n        @c.purge rescue nil\n      end\n      File.unlink(@chunk_path) if File.exist?(@chunk_path)\n    end\n\n    test 'can load as queued chunk' do\n      assert @c\n      assert_equal :queued, @c.state\n      assert_equal @chunk_id, @c.unique_id\n      assert_equal gen_metadata, @c.metadata\n      assert_equal @d.bytesize, @c.bytesize\n      assert_equal 0, @c.size\n      assert_equal @d, @c.read\n\n      assert_raise RuntimeError.new(\"BUG: concatenating to unwritable chunk, now 'queued'\") do\n        @c.append([\"queued chunk is read only\"])\n      end\n      assert_raise IOError do\n        @c.instance_eval{ @chunk }.write \"chunk io is opened as read only\"\n      end\n    end\n  end\n\n  sub_test_case 'compressed buffer' do\n    setup do\n      @src = 'text data for compressing' * 5\n      @gzipped_src = compress(@src)\n      @zstded_src = compress(@src, type: :zstd)\n    end\n\n    test '#append with compress option writes  compressed data to chunk when compress is gzip' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)\n      c.append([@src, @src], compress: :gzip)\n      c.commit\n\n      # check chunk is compressed\n      assert c.read(compressed: :gzip).size < [@src, @src].join(\"\").size\n\n      assert_equal @src + @src, c.read\n    end\n\n    test '#open passes io object having decompressed data to a block when compress is gzip' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      decomressed_data = c.open do |io|\n        v = io.read\n        assert_equal @src, v\n        v\n      end\n      assert_equal @src, decomressed_data\n    end\n\n    test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      comressed_data = c.open(compressed: :gzip) do |io|\n        v = io.read\n        assert_equal @gzipped_src, v\n        v\n      end\n      assert_equal @gzipped_src, comressed_data\n    end\n\n    test '#write_to writes decompressed data when compress is gzip' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @gzipped_src, c.read(compressed: :gzip)\n\n      io = StringIO.new\n      c.write_to(io)\n      assert_equal @src, io.string\n    end\n\n    test '#write_to with compressed option writes compressed data when compress is gzip' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @gzipped_src, c.read(compressed: :gzip)\n\n      io = StringIO.new\n      io.set_encoding(Encoding::ASCII_8BIT)\n      c.write_to(io, compressed: :gzip)\n      assert_equal @gzipped_src, io.string\n    end\n\n    test '#append with compress option writes  compressed data to chunk when compress is zstd' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd)\n      c.append([@src, @src], compress: :zstd)\n      c.commit\n\n      # check chunk is compressed\n      assert c.read(compressed: :zstd).size < [@src, @src].join(\"\").size\n\n      assert_equal @src + @src, c.read\n    end\n\n    test '#open passes io object having decompressed data to a block when compress is zstd' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      decomressed_data = c.open do |io|\n        v = io.read\n        assert_equal @src, v\n        v\n      end\n      assert_equal @src, decomressed_data\n    end\n\n    test '#open with compressed option passes io object having decompressed data to a block when compress is zstd' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      comressed_data = c.open(compressed: :zstd) do |io|\n        v = io.read\n        assert_equal @zstded_src, v\n        v\n      end\n      assert_equal @zstded_src, comressed_data\n    end\n\n    test '#write_to writes decompressed data when compress is zstd' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @zstded_src, c.read(compressed: :zstd)\n\n      io = StringIO.new\n      c.write_to(io)\n      assert_equal @src, io.string\n    end\n\n    test '#write_to with compressed option writes compressed data when compress is zstd' do\n      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @zstded_src, c.read(compressed: :zstd)\n\n      io = StringIO.new\n      io.set_encoding(Encoding::ASCII_8BIT)\n      c.write_to(io, compressed: :zstd)\n      assert_equal @zstded_src, io.string\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_buffer_memory_chunk.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/buffer/memory_chunk'\nrequire 'fluent/plugin/compressable'\n\nrequire 'json'\n\nclass BufferMemoryChunkTest < Test::Unit::TestCase\n  include Fluent::Plugin::Compressable\n\n  setup do\n    @c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new)\n  end\n\n  test 'has blank chunk initially' do\n    assert @c.empty?\n    assert_equal '', @c.instance_eval{ @chunk }\n    assert_equal 0, @c.instance_eval{ @chunk_bytes }\n    assert_equal 0, @c.instance_eval{ @adding_bytes }\n    assert_equal 0, @c.instance_eval{ @adding_size }\n  end\n\n  test 'can #append, #commit and #read it' do\n    assert @c.empty?\n\n    d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n    d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n    data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n    @c.append(data)\n    @c.commit\n\n    content = @c.read\n    ds = content.split(\"\\n\").select{|d| !d.empty? }\n\n    assert_equal 2, ds.size\n    assert_equal d1, JSON.parse(ds[0])\n    assert_equal d2, JSON.parse(ds[1])\n\n    d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n    d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n    @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n    @c.commit\n\n    content = @c.read\n    ds = content.split(\"\\n\").select{|d| !d.empty? }\n\n    assert_equal 4, ds.size\n    assert_equal d1, JSON.parse(ds[0])\n    assert_equal d2, JSON.parse(ds[1])\n    assert_equal d3, JSON.parse(ds[2])\n    assert_equal d4, JSON.parse(ds[3])\n  end\n\n  test 'can #concat, #commit and #read it' do\n    assert @c.empty?\n\n    d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n    d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n    data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"].join\n    @c.concat(data, 2)\n    @c.commit\n\n    content = @c.read\n    ds = content.split(\"\\n\").select{|d| !d.empty? }\n\n    assert_equal 2, ds.size\n    assert_equal d1, JSON.parse(ds[0])\n    assert_equal d2, JSON.parse(ds[1])\n\n    d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n    d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n    @c.concat([d3.to_json + \"\\n\", d4.to_json + \"\\n\"].join, 2)\n    @c.commit\n\n    content = @c.read\n    ds = content.split(\"\\n\").select{|d| !d.empty? }\n\n    assert_equal 4, ds.size\n    assert_equal d1, JSON.parse(ds[0])\n    assert_equal d2, JSON.parse(ds[1])\n    assert_equal d3, JSON.parse(ds[2])\n    assert_equal d4, JSON.parse(ds[3])\n  end\n\n  test 'has its contents in binary (ascii-8bit)' do\n    data1 = \"aaa bbb ccc\".force_encoding('utf-8')\n    @c.append([data1])\n    @c.commit\n    assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.encoding }\n\n    content = @c.read\n    assert_equal Encoding::ASCII_8BIT, content.encoding\n  end\n\n  test 'has #bytesize and #size' do\n    assert @c.empty?\n\n    d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n    d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n    data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n    @c.append(data)\n\n    assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n    assert_equal 2, @c.size\n\n    @c.commit\n\n    assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n    assert_equal 2, @c.size\n\n    first_bytesize = @c.bytesize\n\n    d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n    d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n    @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n\n    assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n    assert_equal 4, @c.size\n\n    @c.commit\n\n    assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n    assert_equal 4, @c.size\n  end\n\n  test 'can #rollback to revert non-committed data' do\n    assert @c.empty?\n\n    d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n    d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n    data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n    @c.append(data)\n\n    assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n    assert_equal 2, @c.size\n\n    @c.rollback\n\n    assert @c.empty?\n\n    assert @c.empty?\n\n    d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n    d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n    data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n    @c.append(data)\n    @c.commit\n\n    assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n    assert_equal 2, @c.size\n\n    first_bytesize = @c.bytesize\n\n    d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n    d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n    @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n\n    assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n    assert_equal 4, @c.size\n\n    @c.rollback\n\n    assert_equal first_bytesize, @c.bytesize\n    assert_equal 2, @c.size\n  end\n\n  test 'can #rollback to revert non-committed data from #concat' do\n    assert @c.empty?\n\n    d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n    d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n    data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"].join\n    @c.concat(data, 2)\n\n    assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n    assert_equal 2, @c.size\n\n    @c.rollback\n\n    assert @c.empty?\n\n    assert @c.empty?\n\n    d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n    d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n    data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n    @c.append(data)\n    @c.commit\n\n    assert_equal (d1.to_json + \"\\n\" + d2.to_json + \"\\n\").bytesize, @c.bytesize\n    assert_equal 2, @c.size\n\n    first_bytesize = @c.bytesize\n\n    d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n    d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n    @c.concat([d3.to_json + \"\\n\", d4.to_json + \"\\n\"].join, 2)\n\n    assert_equal first_bytesize + (d3.to_json + \"\\n\" + d4.to_json + \"\\n\").bytesize, @c.bytesize\n    assert_equal 4, @c.size\n\n    @c.rollback\n\n    assert_equal first_bytesize, @c.bytesize\n    assert_equal 2, @c.size\n  end\n\n  test 'does nothing for #close' do\n    d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n    d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n    data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n    @c.append(data)\n    @c.commit\n    d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n    d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n    @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n    @c.commit\n\n    content = @c.read\n\n    @c.close\n\n    assert_equal content, @c.read\n  end\n\n  test 'deletes all data by #purge' do\n    d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n    d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n    data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n    @c.append(data)\n    @c.commit\n    d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n    d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n    @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n    @c.commit\n\n    @c.purge\n\n    assert @c.empty?\n    assert_equal 0, @c.bytesize\n    assert_equal 0, @c.size\n    assert_equal '', @c.read\n  end\n\n  test 'can #open its contents as io' do\n    d1 = {\"f1\" => 'v1', \"f2\" => 'v2', \"f3\" => 'v3'}\n    d2 = {\"f1\" => 'vv1', \"f2\" => 'vv2', \"f3\" => 'vv3'}\n    data = [d1.to_json + \"\\n\", d2.to_json + \"\\n\"]\n    @c.append(data)\n    @c.commit\n    d3 = {\"f1\" => 'x', \"f2\" => 'y', \"f3\" => 'z'}\n    d4 = {\"f1\" => 'a', \"f2\" => 'b', \"f3\" => 'c'}\n    @c.append([d3.to_json + \"\\n\", d4.to_json + \"\\n\"])\n    @c.commit\n\n    lines = []\n    @c.open do |io|\n      assert io\n      io.readlines.each do |l|\n        lines << l\n      end\n    end\n\n    assert_equal d1.to_json + \"\\n\", lines[0]\n    assert_equal d2.to_json + \"\\n\", lines[1]\n    assert_equal d3.to_json + \"\\n\", lines[2]\n    assert_equal d4.to_json + \"\\n\", lines[3]\n  end\n\n  sub_test_case 'compressed buffer' do\n    setup do\n      @src = 'text data for compressing' * 5\n      @gzipped_src = compress(@src)\n      @zstded_src = compress(@src, type: :zstd)\n    end\n\n    test '#append with compress option writes  compressed data to chunk when compress is gzip' do\n      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)\n      c.append([@src, @src], compress: :gzip)\n      c.commit\n\n      # check chunk is compressed\n      assert c.read(compressed: :gzip).size < [@src, @src].join(\"\").size\n\n      assert_equal @src + @src, c.read\n    end\n\n    test '#open passes io object having decompressed data to a block when compress is gzip' do\n      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      decomressed_data = c.open do |io|\n        v = io.read\n        assert_equal @src, v\n        v\n      end\n      assert_equal @src, decomressed_data\n    end\n\n    test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do\n      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      comressed_data = c.open(compressed: :gzip) do |io|\n        v = io.read\n        assert_equal @gzipped_src, v\n        v\n      end\n      assert_equal @gzipped_src, comressed_data\n    end\n\n    test '#write_to writes decompressed data when compress is gzip' do\n      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @gzipped_src, c.read(compressed: :gzip)\n\n      io = StringIO.new\n      c.write_to(io)\n      assert_equal @src, io.string\n    end\n\n    test '#write_to with compressed option writes compressed data when compress is gzip' do\n      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)\n      c.concat(@gzipped_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @gzipped_src, c.read(compressed: :gzip)\n\n      io = StringIO.new\n      io.set_encoding(Encoding::ASCII_8BIT)\n      c.write_to(io, compressed: :gzip)\n      assert_equal @gzipped_src, io.string\n    end\n\n    test '#append with compress option writes  compressed data to chunk when compress is zstd' do\n      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd)\n      c.append([@src, @src], compress: :zstd)\n      c.commit\n\n      # check chunk is compressed\n      assert c.read(compressed: :zstd).size < [@src, @src].join(\"\").size\n\n      assert_equal @src + @src, c.read\n    end\n\n    test '#open passes io object having decompressed data to a block when compress is zstd' do\n      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      decomressed_data = c.open do |io|\n        v = io.read\n        assert_equal @src, v\n        v\n      end\n      assert_equal @src, decomressed_data\n    end\n\n    test '#open with compressed option passes io object having decompressed data to a block when compress is zstd' do\n      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      comressed_data = c.open(compressed: :zstd) do |io|\n        v = io.read\n        assert_equal @zstded_src, v\n        v\n      end\n      assert_equal @zstded_src, comressed_data\n    end\n\n    test '#write_to writes decompressed data when compress is zstd' do\n      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @zstded_src, c.read(compressed: :zstd)\n\n      io = StringIO.new\n      c.write_to(io)\n      assert_equal @src, io.string\n    end\n\n    test '#write_to with compressed option writes compressed data when compress is zstd' do\n      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd)\n      c.concat(@zstded_src, @src.size)\n      c.commit\n\n      assert_equal @src, c.read\n      assert_equal @zstded_src, c.read(compressed: :zstd)\n\n      io = StringIO.new\n      io.set_encoding(Encoding::ASCII_8BIT)\n      c.write_to(io, compressed: :zstd)\n      assert_equal @zstded_src, io.string\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_compressable.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/compressable'\n\nclass CompressableTest < Test::Unit::TestCase\n  include Fluent::Plugin::Compressable\n\n  def compress_assert_equal(expected, actual)\n    e = Zlib::GzipReader.new(StringIO.new(expected)).read\n    a = Zlib::GzipReader.new(StringIO.new(actual)).read\n    assert_equal(e, a)\n  end\n\n  sub_test_case '#compress' do\n    setup do\n      @src = 'text data for compressing' * 5\n      @gzipped_src = compress(@src)\n    end\n\n    test 'compress data' do\n      assert compress(@src).size < @src.size\n      assert_not_equal @gzipped_src, @src\n    end\n\n    test 'write compressed data to IO with output_io option' do\n      io = StringIO.new\n      compress(@src, output_io: io)\n      compress_assert_equal @gzipped_src, io.string\n    end\n  end\n\n  sub_test_case '#decompress' do\n    setup do\n      @src = 'text data for compressing' * 5\n      @gzipped_src = compress(@src)\n    end\n\n    test 'decompress compressed data' do\n      assert_equal @src, decompress(@gzipped_src)\n    end\n\n    test 'write decompressed data to IO with output_io option' do\n      io = StringIO.new\n      decompress(@gzipped_src, output_io: io)\n      assert_equal @src, io.string\n    end\n\n    test 'return decompressed string with output_io option' do\n      io = StringIO.new(@gzipped_src)\n      assert_equal @src, decompress(input_io: io)\n    end\n\n    test 'decompress multiple compressed data' do\n      src1 = 'text data'\n      src2 = 'text data2'\n      gzipped_src = compress(src1) + compress(src2)\n\n      assert_equal src1 + src2, decompress(gzipped_src)\n    end\n\n    test 'decompress with input_io and output_io' do\n      input_io = StringIO.new(@gzipped_src)\n      output_io = StringIO.new\n\n      decompress(input_io: input_io, output_io: output_io)\n      assert_equal @src, output_io.string\n    end\n\n    test 'decompress multiple compressed data with input_io and output_io' do\n      src1 = 'text data'\n      src2 = 'text data2'\n      gzipped_src = compress(src1) + compress(src2)\n\n      input_io = StringIO.new(gzipped_src)\n      output_io = StringIO.new\n\n      decompress(input_io: input_io, output_io: output_io)\n      assert_equal src1 + src2, output_io.string\n    end\n\n    test 'return the received value as it is with empty string or nil' do\n      assert_equal nil, decompress\n      assert_equal nil, decompress(nil)\n      assert_equal '', decompress('')\n      assert_equal '', decompress('', output_io: StringIO.new)\n    end\n\n    test 'decompress large zstd compressed data' do\n      src1 = SecureRandom.random_bytes(1024)\n      src2 = SecureRandom.random_bytes(1024)\n      src3 = SecureRandom.random_bytes(1024)\n\n      zstd_compressed_data = compress(src1, type: :zstd) + compress(src2, type: :zstd) + compress(src3, type: :zstd)\n      assert_equal src1 + src2 + src3, decompress(zstd_compressed_data, type: :zstd)\n    end\n\n    test 'decompress large zstd compressed data with input_io and output_io' do\n      src1 = SecureRandom.random_bytes(1024)\n      src2 = SecureRandom.random_bytes(1024)\n      src3 = SecureRandom.random_bytes(1024)\n\n      zstd_compressed_data = compress(src1, type: :zstd) + compress(src2, type: :zstd) + compress(src3, type: :zstd)\n\n      input_io = StringIO.new(zstd_compressed_data)\n      output_io = StringIO.new\n      output_io.set_encoding(src1.encoding)\n\n      decompress(input_io: input_io, output_io: output_io, type: :zstd)\n      assert_equal src1 + src2 + src3, output_io.string\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_file_util.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/file_util'\nrequire 'fileutils'\n\nclass FileUtilTest < Test::Unit::TestCase\n  def setup\n    FileUtils.rm_rf(TEST_DIR)\n    FileUtils.mkdir_p(TEST_DIR)\n  end\n\n  TEST_DIR = File.expand_path(File.dirname(__FILE__) + \"/../tmp/file_util\")\n\n  sub_test_case 'writable?' do\n    test 'file exists and writable' do\n      FileUtils.touch(\"#{TEST_DIR}/test_file\")\n      assert_true Fluent::FileUtil.writable?(\"#{TEST_DIR}/test_file\")\n    end\n\n    test 'file exists and not writable' do\n      FileUtils.touch(\"#{TEST_DIR}/test_file\")\n      File.chmod(0444, \"#{TEST_DIR}/test_file\")\n      assert_false Fluent::FileUtil.writable?(\"#{TEST_DIR}/test_file\")\n    end\n\n    test 'directory exists' do\n      FileUtils.mkdir_p(\"#{TEST_DIR}/test_dir\")\n      assert_false Fluent::FileUtil.writable?(\"#{TEST_DIR}/test_dir\")\n    end\n\n    test 'file does not exist and parent directory is writable' do\n      FileUtils.mkdir_p(\"#{TEST_DIR}/test_dir\")\n      assert_true Fluent::FileUtil.writable?(\"#{TEST_DIR}/test_dir/test_file\")\n    end\n\n    test 'file does not exist and parent directory is not writable' do\n      FileUtils.mkdir_p(\"#{TEST_DIR}/test_dir\")\n      File.chmod(0444, \"#{TEST_DIR}/test_dir\")\n      assert_false Fluent::FileUtil.writable?(\"#{TEST_DIR}/test_dir/test_file\")\n    end\n\n    test 'parent directory does not exist' do\n      FileUtils.rm_rf(\"#{TEST_DIR}/test_dir\")\n      assert_false Fluent::FileUtil.writable?(\"#{TEST_DIR}/test_dir/test_file\")\n    end\n\n    test 'parent file (not directory) exists' do\n      FileUtils.touch(\"#{TEST_DIR}/test_file\")\n      assert_false Fluent::FileUtil.writable?(\"#{TEST_DIR}/test_file/foo\")\n    end\n  end\n\n  sub_test_case 'writable_p?' do\n    test 'file exists and writable' do\n      FileUtils.touch(\"#{TEST_DIR}/test_file\")\n      assert_true Fluent::FileUtil.writable_p?(\"#{TEST_DIR}/test_file\")\n    end\n\n    test 'file exists and not writable' do\n      FileUtils.touch(\"#{TEST_DIR}/test_file\")\n      File.chmod(0444, \"#{TEST_DIR}/test_file\")\n      assert_false Fluent::FileUtil.writable_p?(\"#{TEST_DIR}/test_file\")\n    end\n\n    test 'directory exists' do\n      FileUtils.mkdir_p(\"#{TEST_DIR}/test_dir\")\n      assert_false Fluent::FileUtil.writable_p?(\"#{TEST_DIR}/test_dir\")\n    end\n\n    test 'parent directory exists and writable' do\n      FileUtils.mkdir_p(\"#{TEST_DIR}/test_dir\")\n      assert_true Fluent::FileUtil.writable_p?(\"#{TEST_DIR}/test_dir/test_file\")\n    end\n\n    test 'parent directory exists and not writable' do\n      FileUtils.mkdir_p(\"#{TEST_DIR}/test_dir\")\n      File.chmod(0555, \"#{TEST_DIR}/test_dir\")\n      assert_false Fluent::FileUtil.writable_p?(\"#{TEST_DIR}/test_dir/test_file\")\n    end\n\n    test 'parent of parent (of parent ...) directory exists and writable' do\n      FileUtils.mkdir_p(\"#{TEST_DIR}/test_dir\")\n      assert_true Fluent::FileUtil.writable_p?(\"#{TEST_DIR}/test_dir/foo/bar/baz\")\n    end\n\n    test 'parent of parent (of parent ...) directory exists and not writable' do\n      FileUtils.mkdir_p(\"#{TEST_DIR}/test_dir\")\n      File.chmod(0555, \"#{TEST_DIR}/test_dir\")\n      assert_false Fluent::FileUtil.writable_p?(\"#{TEST_DIR}/test_dir/foo/bar/baz\")\n    end\n\n    test 'parent of parent (of parent ...) file (not directory) exists' do\n      FileUtils.touch(\"#{TEST_DIR}/test_file\")\n      assert_false Fluent::FileUtil.writable_p?(\"#{TEST_DIR}/test_file/foo/bar/baz\")\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_filter.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/filter'\nrequire 'fluent/event'\nrequire 'flexmock/test_unit'\n\nmodule FluentPluginFilterTest\n  class DummyPlugin < Fluent::Plugin::Filter\n  end\n  class NumDoublePlugin < Fluent::Plugin::Filter\n    def filter(tag, time, record)\n      r = record.dup\n      r[\"num\"] = r[\"num\"].to_i * 2\n      r\n    end\n  end\n  class IgnoreForNumPlugin < Fluent::Plugin::Filter\n    def filter(tag, time, record)\n      if record[\"num\"].is_a? Numeric\n        nil\n      else\n        record\n      end\n    end\n  end\n  class RaiseForNumPlugin < Fluent::Plugin::Filter\n    def filter(tag, time, record)\n      if record[\"num\"].is_a? Numeric\n        raise \"Value of num is Number!\"\n      end\n      record\n    end\n  end\n  class NumDoublePluginWithTime < Fluent::Plugin::Filter\n    def filter_with_time(tag, time, record)\n      r = record.dup\n      r[\"num\"] = r[\"num\"].to_i * 2\n      [time, r]\n    end\n  end\n  class IgnoreForNumPluginWithTime < Fluent::Plugin::Filter\n    def filter_with_time(tag, time, record)\n      if record[\"num\"].is_a? Numeric\n        nil\n      else\n        [time, record]\n      end\n    end\n  end\n  class InvalidPlugin < Fluent::Plugin::Filter\n    # Because of implementing `filter_with_time` and `filter` methods\n    def filter_with_time(tag, time, record); end\n    def filter(tag, time, record); end\n  end\nend\n\nclass FilterPluginTest < Test::Unit::TestCase\n  DummyRouter = Struct.new(:emits) do\n    def emit_error_event(tag, time, record, error)\n      self.emits << [tag, time, record, error]\n    end\n  end\n\n  setup do\n    @p = nil\n  end\n\n  teardown do\n    if @p\n      @p.stop unless @p.stopped?\n      @p.before_shutdown unless @p.before_shutdown?\n      @p.shutdown unless @p.shutdown?\n      @p.after_shutdown unless @p.after_shutdown?\n      @p.close unless @p.closed?\n      @p.terminate unless @p.terminated?\n    end\n  end\n\n  sub_test_case 'for basic dummy plugin' do\n    setup do\n      Fluent::Test.setup\n    end\n\n    test 'plugin does not define #filter raises error' do\n      assert_raise NotImplementedError do\n        FluentPluginFilterTest::DummyPlugin.new\n      end\n    end\n  end\n\n  sub_test_case 'normal filter plugin' do\n    setup do\n      Fluent::Test.setup\n      @p = FluentPluginFilterTest::NumDoublePlugin.new\n    end\n\n    test 'has healthy lifecycle' do\n      assert !@p.configured?\n      @p.configure(config_element)\n      assert @p.configured?\n\n      assert !@p.started?\n      @p.start\n      assert @p.start\n\n      assert !@p.stopped?\n      @p.stop\n      assert @p.stopped?\n\n      assert !@p.before_shutdown?\n      @p.before_shutdown\n      assert @p.before_shutdown?\n\n      assert !@p.shutdown?\n      @p.shutdown\n      assert @p.shutdown?\n\n      assert !@p.after_shutdown?\n      @p.after_shutdown\n      assert @p.after_shutdown?\n\n      assert !@p.closed?\n      @p.close\n      assert @p.closed?\n\n      assert !@p.terminated?\n      @p.terminate\n      assert @p.terminated?\n    end\n\n    test 'has plugin_id automatically generated' do\n      assert @p.respond_to?(:plugin_id_configured?)\n      assert @p.respond_to?(:plugin_id)\n\n      @p.configure(config_element)\n\n      assert !@p.plugin_id_configured?\n      assert @p.plugin_id\n      assert{ @p.plugin_id != 'mytest' }\n    end\n\n    test 'has plugin_id manually configured' do\n      @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))\n      assert @p.plugin_id_configured?\n      assert_equal 'mytest', @p.plugin_id\n    end\n\n    test 'has plugin logger' do\n      assert @p.respond_to?(:log)\n      assert @p.log\n\n      # default logger\n      original_logger = @p.log\n\n      @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))\n\n      assert(@p.log.object_id != original_logger.object_id)\n      assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level\n    end\n\n    test 'can load plugin helpers' do\n      assert_nothing_raised do\n        class FluentPluginFilterTest::DummyPlugin2 < Fluent::Plugin::Filter\n          helpers :storage\n        end\n      end\n    end\n\n    test 'can use metrics plugins and fallback methods' do\n      @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))\n\n      %w[emit_size_metrics emit_records_metrics].each do |metric_name|\n        assert_true @p.instance_variable_get(:\"@#{metric_name}\").is_a?(Fluent::Plugin::Metrics)\n      end\n\n      assert_equal 0, @p.emit_size\n      assert_equal 0, @p.emit_records\n    end\n\n    test 'are available with multi worker configuration in default' do\n      assert @p.multi_workers_ready?\n    end\n\n    test 'filters events correctly' do\n      test_es = [\n        [event_time('2016-04-19 13:01:00 -0700'), {\"num\" => \"1\", \"message\" => \"Hello filters!\"}],\n        [event_time('2016-04-19 13:01:03 -0700'), {\"num\" => \"2\", \"message\" => \"Hello filters!\"}],\n        [event_time('2016-04-19 13:01:05 -0700'), {\"num\" => \"3\", \"message\" => \"Hello filters!\"}],\n      ]\n      @p.configure(config_element)\n      es = @p.filter_stream('testing', test_es)\n      assert es.is_a? Fluent::EventStream\n\n      ary = []\n      es.each do |time, r|\n        ary << [time, r]\n      end\n\n      assert_equal 3, ary.size\n\n      assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0]\n      assert_equal \"Hello filters!\", ary[0][1][\"message\"]\n      assert_equal 2, ary[0][1][\"num\"]\n\n      assert_equal event_time('2016-04-19 13:01:03 -0700'), ary[1][0]\n      assert_equal 4, ary[1][1][\"num\"]\n\n      assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[2][0]\n      assert_equal 6, ary[2][1][\"num\"]\n    end\n  end\n\n  sub_test_case 'filter plugin returns nil for some records' do\n    setup do\n      Fluent::Test.setup\n      @p = FluentPluginFilterTest::IgnoreForNumPlugin.new\n    end\n\n    test 'filter_stream ignores records which #filter return nil' do\n      test_es = [\n        [event_time('2016-04-19 13:01:00 -0700'), {\"num\" => \"1\", \"message\" => \"Hello filters!\"}],\n        [event_time('2016-04-19 13:01:03 -0700'), {\"num\" => 2, \"message\" => \"Ignored, yay!\"}],\n        [event_time('2016-04-19 13:01:05 -0700'), {\"num\" => \"3\", \"message\" => \"Hello filters!\"}],\n      ]\n      @p.configure(config_element)\n      es = @p.filter_stream('testing', test_es)\n      assert es.is_a? Fluent::EventStream\n\n      ary = []\n      es.each do |time, r|\n        ary << [time, r]\n      end\n\n      assert_equal 2, ary.size\n\n      assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0]\n      assert_equal \"Hello filters!\", ary[0][1][\"message\"]\n      assert_equal \"1\", ary[0][1][\"num\"]\n\n      assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[1][0]\n      assert_equal \"3\", ary[1][1][\"num\"]\n    end\n  end\n\n  sub_test_case 'filter plugin raises error' do\n    setup do\n      Fluent::Test.setup\n      @p = FluentPluginFilterTest::RaiseForNumPlugin.new\n    end\n\n    test 'has router and can emit events to error streams' do\n      assert @p.has_router?\n      @p.configure(config_element)\n      assert @p.router\n\n      @p.router = DummyRouter.new([])\n\n      test_es = [\n        [event_time('2016-04-19 13:01:00 -0700'), {\"num\" => \"1\", \"message\" => \"Hello filters!\"}],\n        [event_time('2016-04-19 13:01:03 -0700'), {\"num\" => 2, \"message\" => \"Hello error router!\"}],\n        [event_time('2016-04-19 13:01:05 -0700'), {\"num\" => \"3\", \"message\" => \"Hello filters!\"}],\n      ]\n      es = @p.filter_stream('testing', test_es)\n      assert es.is_a? Fluent::EventStream\n\n      ary = []\n      es.each do |time, r|\n        ary << [time, r]\n      end\n\n      assert_equal 2, ary.size\n\n      assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0]\n      assert_equal \"Hello filters!\", ary[0][1][\"message\"]\n      assert_equal \"1\", ary[0][1][\"num\"]\n\n      assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[1][0]\n      assert_equal \"3\", ary[1][1][\"num\"]\n\n      assert_equal 1, @p.router.emits.size\n\n      error_emits = @p.router.emits\n\n      assert_equal \"testing\", error_emits[0][0]\n      assert_equal event_time('2016-04-19 13:01:03 -0700'), error_emits[0][1]\n      assert_equal({\"num\" => 2, \"message\" => \"Hello error router!\"}, error_emits[0][2])\n      assert{ error_emits[0][3].is_a? RuntimeError }\n      assert_equal \"Value of num is Number!\", error_emits[0][3].message\n    end\n  end\n\n  sub_test_case 'filter plugins that is implemented `filter_with_time`' do\n    setup do\n      Fluent::Test.setup\n      @p = FluentPluginFilterTest::NumDoublePluginWithTime.new\n    end\n\n    test 'filters events correctly' do\n      test_es = [\n        [event_time('2016-04-19 13:01:00 -0700'), {\"num\" => \"1\", \"message\" => \"Hello filters!\"}],\n        [event_time('2016-04-19 13:01:03 -0700'), {\"num\" => \"2\", \"message\" => \"Hello filters!\"}],\n        [event_time('2016-04-19 13:01:05 -0700'), {\"num\" => \"3\", \"message\" => \"Hello filters!\"}],\n      ]\n      es = @p.filter_stream('testing', test_es)\n      assert es.is_a? Fluent::EventStream\n\n      ary = []\n      es.each do |time, r|\n        ary << [time, r]\n      end\n\n      assert_equal 3, ary.size\n\n      assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0]\n      assert_equal \"Hello filters!\", ary[0][1][\"message\"]\n      assert_equal 2, ary[0][1][\"num\"]\n\n      assert_equal event_time('2016-04-19 13:01:03 -0700'), ary[1][0]\n      assert_equal 4, ary[1][1][\"num\"]\n\n      assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[2][0]\n      assert_equal 6, ary[2][1][\"num\"]\n    end\n  end\n\n  sub_test_case 'filter plugin that is implemented `filter_with_time` and returns nil for some records' do\n    setup do\n      Fluent::Test.setup\n      @p = FluentPluginFilterTest::IgnoreForNumPluginWithTime.new\n    end\n\n    test 'filter_stream ignores records which #filter_with_time return nil' do\n      test_es = [\n        [event_time('2016-04-19 13:01:00 -0700'), {\"num\" => \"1\", \"message\" => \"Hello filters!\"}],\n        [event_time('2016-04-19 13:01:03 -0700'), {\"num\" => 2, \"message\" => \"Ignored, yay!\"}],\n        [event_time('2016-04-19 13:01:05 -0700'), {\"num\" => \"3\", \"message\" => \"Hello filters!\"}],\n      ]\n      @p.configure(config_element)\n      es = @p.filter_stream('testing', test_es)\n      assert es.is_a? Fluent::EventStream\n\n      ary = []\n      es.each do |time, r|\n        ary << [time, r]\n      end\n\n      assert_equal 2, ary.size\n\n      assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0]\n      assert_equal \"Hello filters!\", ary[0][1][\"message\"]\n      assert_equal \"1\", ary[0][1][\"num\"]\n\n      assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[1][0]\n      assert_equal \"3\", ary[1][1][\"num\"]\n    end\n  end\n\n  sub_test_case 'filter plugins that is implemented both `filter_with_time` and `filter`' do\n    setup do\n      Fluent::Test.setup\n    end\n\n    test 'raises DuplicatedImplementError' do\n      assert_raise do\n        FluentPluginFilterTest::InvalidPlugin.new\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_filter_grep.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/filter_grep'\nrequire 'fluent/test/driver/filter'\n\nclass GrepFilterTest < Test::Unit::TestCase\n  include Fluent\n\n  setup do\n    Fluent::Test.setup\n    @time = event_time\n  end\n\n  def create_driver(conf = '')\n    Fluent::Test::Driver::Filter.new(Fluent::Plugin::GrepFilter).configure(conf)\n  end\n\n  sub_test_case 'configure' do\n    test 'check default' do\n      d = create_driver\n      assert_empty(d.instance.regexps)\n      assert_empty(d.instance.excludes)\n    end\n\n    test \"regexpN can contain a space\" do\n      d = create_driver(%[regexp1 message  foo])\n      d.instance._regexp_and_conditions.each { |value|\n        assert_equal(Regexp.compile(/ foo/), value.pattern)\n      }\n    end\n\n    test \"excludeN can contain a space\" do\n      d = create_driver(%[exclude1 message  foo])\n      d.instance._exclude_or_conditions.each { |value|\n        assert_equal(Regexp.compile(/ foo/), value.pattern)\n      }\n    end\n\n    sub_test_case \"duplicate key\" do\n      test \"flat\" do\n        conf = %[\n          regexp1 message test\n          regexp2 message test2\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n      test \"section\" do\n        conf = %[\n          <regexp>\n            key message\n            pattern test\n          </regexp>\n          <regexp>\n            key message\n            pattern test2\n          </regexp>\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n      test \"mix\" do\n        conf = %[\n          regexp1 message test\n          <regexp>\n            key message\n            pattern test\n          </regexp>\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n\n      test \"and/regexp\" do\n        conf = %[\n          <and>\n            <regexp>\n              key message\n              pattern test\n            </regexp>\n            <regexp>\n              key message\n              pattern test\n            </regexp>\n          </and>\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n\n      test \"and/regexp, and/regexp\" do\n        conf = %[\n          <and>\n            <regexp>\n              key message\n              pattern test\n            </regexp>\n          </and>\n          <and>\n            <regexp>\n              key message\n              pattern test\n            </regexp>\n          </and>\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n\n      test \"regexp, and/regexp\" do\n        conf = %[\n          <regexp>\n            key message\n            pattern test\n          </regexp>\n          <and>\n            <regexp>\n              key message\n              pattern test\n            </regexp>\n          </and>\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n\n      test \"and/exclude\" do\n        conf = %[\n          <and>\n            <exclude>\n              key message\n              pattern test\n            </exclude>\n            <exclude>\n              key message\n              pattern test\n            </exclude>\n          </and>\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n\n      test \"and/exclude, and/exclude\" do\n        conf = %[\n          <and>\n            <exclude>\n              key message\n              pattern test\n            </exclude>\n          </and>\n          <and>\n            <exclude>\n              key message\n              pattern test\n            </exclude>\n          </and>\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n\n      test \"exclude, or/exclude\" do\n        conf = %[\n          <exclude>\n            key message\n            pattern test\n          </exclude>\n          <or>\n            <exclude>\n              key message\n              pattern test\n            </exclude>\n          </or>\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n    end\n\n    sub_test_case \"pattern with slashes\" do\n      test \"start with character classes\" do\n        conf = %[\n          <regexp>\n            key message\n            pattern /[a-z]test/\n          </regexp>\n          <exclude>\n            key message\n            pattern /[A-Z]test/\n          </exclude>\n        ]\n        d = create_driver(conf)\n        assert_equal(/[a-z]test/, d.instance.regexps.first.pattern)\n        assert_equal(/[A-Z]test/, d.instance.excludes.first.pattern)\n      end\n    end\n\n    sub_test_case \"and/or section\" do\n      test \"<and> section cannot include both <regexp> and <exclude>\" do\n        conf = %[\n          <and>\n            <regexp>\n              key message\n              pattern /test/\n            </regexp>\n            <exclude>\n              key level\n              pattern /debug/\n            </exclude>\n          </and>\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n\n      test \"<or> section cannot include both <regexp> and <exclude>\" do\n        conf = %[\n          <or>\n            <regexp>\n              key message\n              pattern /test/\n            </regexp>\n            <exclude>\n              key level\n              pattern /debug/\n            </exclude>\n          </or>\n        ]\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n    end\n  end\n\n  sub_test_case 'filter_stream' do\n    def messages\n      [\n        \"2013/01/13T07:02:11.124202 INFO GET /ping\",\n        \"2013/01/13T07:02:13.232645 WARN POST /auth\",\n        \"2013/01/13T07:02:21.542145 WARN GET /favicon.ico\",\n        \"2013/01/13T07:02:43.632145 WARN POST /login\",\n      ]\n    end\n\n    def filter(config, msgs)\n      d = create_driver(config)\n      d.run {\n        msgs.each { |msg|\n          d.feed(\"filter.test\", @time, {'foo' => 'bar', 'message' => msg})\n        }\n      }\n      d.filtered_records\n    end\n\n    test 'empty config' do\n      filtered_records = filter('', messages)\n      assert_equal(4, filtered_records.size)\n    end\n\n    test 'regexpN' do\n      filtered_records = filter('regexp1 message WARN', messages)\n      assert_equal(3, filtered_records.size)\n      assert_block('only WARN logs') do\n        filtered_records.all? { |r|\n          !r['message'].include?('INFO')\n        }\n      end\n    end\n\n    test 'excludeN' do\n      filtered_records = filter('exclude1 message favicon', messages)\n      assert_equal(3, filtered_records.size)\n      assert_block('remove favicon logs') do\n        filtered_records.all? { |r|\n          !r['message'].include?('favicon')\n        }\n      end\n    end\n\n    test 'regexps' do\n      conf = %[\n        <regexp>\n          key message\n          pattern WARN\n        </regexp>\n      ]\n      filtered_records = filter(conf, messages)\n      assert_equal(3, filtered_records.size)\n      assert_block('only WARN logs') do\n        filtered_records.all? { |r|\n          !r['message'].include?('INFO')\n        }\n      end\n    end\n\n    test 'excludes' do\n      conf = %[\n        <exclude>\n          key message\n          pattern favicon\n        </exclude>\n      ]\n      filtered_records = filter(conf, messages)\n      assert_equal(3, filtered_records.size)\n      assert_block('remove favicon logs') do\n        filtered_records.all? { |r|\n          !r['message'].include?('favicon')\n        }\n      end\n    end\n\n    sub_test_case 'with invalid sequence' do\n      def messages\n        [\n          \"\\xff\".force_encoding('UTF-8'),\n        ]\n      end\n\n      test \"don't raise an exception\" do\n        assert_nothing_raised {\n          filter(%[regexp1 message WARN], [\"\\xff\".force_encoding('UTF-8')])\n        }\n      end\n    end\n\n    sub_test_case \"and/or section\" do\n      def records\n        [\n          { \"time\" => \"2013/01/13T07:02:11.124202\", \"level\" => \"INFO\", \"method\" => \"GET\", \"path\" => \"/ping\" },\n          { \"time\" => \"2013/01/13T07:02:13.232645\", \"level\" => \"WARN\", \"method\" => \"POST\", \"path\" => \"/auth\" },\n          { \"time\" => \"2013/01/13T07:02:21.542145\", \"level\" => \"WARN\", \"method\" => \"GET\", \"path\" => \"/favicon.ico\" },\n          { \"time\" => \"2013/01/13T07:02:43.632145\", \"level\" => \"WARN\", \"method\" => \"POST\", \"path\" => \"/login\" },\n        ]\n      end\n\n      def filter(conf, records)\n        d = create_driver(conf)\n        d.run do\n          records.each do |record|\n            d.feed(\"filter.test\", @time, record)\n          end\n        end\n        d.filtered_records\n      end\n\n      test \"basic and/regexp\" do\n        conf = %[\n          <and>\n            <regexp>\n              key level\n              pattern ^INFO$\n            </regexp>\n            <regexp>\n              key method\n              pattern ^GET$\n            </regexp>\n          </and>\n        ]\n        filtered_records = filter(conf, records)\n        assert_equal(records.values_at(0), filtered_records)\n      end\n\n      test \"basic or/exclude\" do\n        conf = %[\n          <or>\n            <exclude>\n              key level\n              pattern ^INFO$\n            </exclude>\n            <exclude>\n              key method\n              pattern ^GET$\n            </exclude>\n          </or>\n        ]\n        filtered_records = filter(conf, records)\n        assert_equal(records.values_at(1, 3), filtered_records)\n      end\n\n      test \"basic or/regexp\" do\n        conf = %[\n          <or>\n            <regexp>\n              key level\n              pattern ^INFO$\n            </regexp>\n            <regexp>\n              key method\n              pattern ^GET$\n            </regexp>\n          </or>\n        ]\n        filtered_records = filter(conf, records)\n        assert_equal(records.values_at(0, 2), filtered_records)\n      end\n\n      test \"basic and/exclude\" do\n        conf = %[\n          <and>\n            <exclude>\n              key level\n              pattern ^INFO$\n            </exclude>\n            <exclude>\n              key method\n              pattern ^GET$\n            </exclude>\n          </and>\n        ]\n        filtered_records = filter(conf, records)\n        assert_equal(records.values_at(1, 2, 3), filtered_records)\n      end\n\n      sub_test_case \"and/or combo\" do\n        def records\n          [\n            { \"time\" => \"2013/01/13T07:02:11.124202\", \"level\" => \"INFO\", \"method\" => \"GET\", \"path\" => \"/ping\" },\n            { \"time\" => \"2013/01/13T07:02:13.232645\", \"level\" => \"WARN\", \"method\" => \"POST\", \"path\" => \"/auth\" },\n            { \"time\" => \"2013/01/13T07:02:21.542145\", \"level\" => \"WARN\", \"method\" => \"GET\", \"path\" => \"/favicon.ico\" },\n            { \"time\" => \"2013/01/13T07:02:43.632145\", \"level\" => \"WARN\", \"method\" => \"POST\", \"path\" => \"/login\" },\n            { \"time\" => \"2013/01/13T07:02:44.959307\", \"level\" => \"ERROR\", \"method\" => \"POST\", \"path\" => \"/login\" },\n            { \"time\" => \"2013/01/13T07:02:45.444992\", \"level\" => \"ERROR\", \"method\" => \"GET\", \"path\" => \"/ping\" },\n            { \"time\" => \"2013/01/13T07:02:51.247941\", \"level\" => \"WARN\", \"method\" => \"GET\", \"path\" => \"/info\" },\n            { \"time\" => \"2013/01/13T07:02:53.108366\", \"level\" => \"WARN\", \"method\" => \"POST\", \"path\" => \"/ban\" },\n          ]\n        end\n\n        test \"and/regexp, or/exclude\" do\n          conf = %[\n            <and>\n              <regexp>\n                key level\n                pattern ^ERROR|WARN$\n              </regexp>\n              <regexp>\n                key method\n                pattern ^GET|POST$\n              </regexp>\n            </and>\n            <or>\n              <exclude>\n                key level\n                pattern ^WARN$\n              </exclude>\n              <exclude>\n                key method\n                pattern ^GET$\n              </exclude>\n            </or>\n          ]\n          filtered_records = filter(conf, records)\n          assert_equal(records.values_at(4), filtered_records)\n        end\n\n        test \"and/regexp, and/exclude\" do\n          conf = %[\n            <and>\n              <regexp>\n                key level\n                pattern ^ERROR|WARN$\n              </regexp>\n              <regexp>\n                key method\n                pattern ^GET|POST$\n              </regexp>\n            </and>\n            <and>\n              <exclude>\n                key level\n                pattern ^WARN$\n              </exclude>\n              <exclude>\n                key method\n                pattern ^GET$\n              </exclude>\n            </and>\n          ]\n          filtered_records = filter(conf, records)\n          assert_equal(records.values_at(1, 3, 4, 5, 7), filtered_records)\n        end\n\n        test \"or/regexp, and/exclude\" do\n          conf = %[\n            <or>\n              <regexp>\n                key level\n                pattern ^ERROR|WARN$\n              </regexp>\n              <regexp>\n                key method\n                pattern ^GET|POST$\n              </regexp>\n            </or>\n            <and>\n              <exclude>\n                key level\n                pattern ^WARN$\n              </exclude>\n              <exclude>\n                key method\n                pattern ^GET$\n              </exclude>\n            </and>\n          ]\n          filtered_records = filter(conf, records)\n          assert_equal(records.values_at(0, 1, 3, 4, 5, 7), filtered_records)\n        end\n\n        test \"or/regexp, or/exclude\" do\n          conf = %[\n            <or>\n              <regexp>\n                key level\n                pattern ^ERROR|WARN$\n              </regexp>\n              <regexp>\n                key method\n                pattern ^GET|POST$\n              </regexp>\n            </or>\n            <or>\n              <exclude>\n                key level\n                pattern ^WARN$\n              </exclude>\n              <exclude>\n                key method\n                pattern ^GET$\n              </exclude>\n            </or>\n          ]\n          filtered_records = filter(conf, records)\n          assert_equal(records.values_at(4), filtered_records)\n        end\n\n        test \"regexp, and/regexp\" do\n          conf = %[\n            <and>\n              <regexp>\n                key level\n                pattern ^ERROR|WARN$\n              </regexp>\n              <regexp>\n                key method\n                pattern ^GET|POST$\n              </regexp>\n            </and>\n            <regexp>\n              key path\n              pattern ^/login$\n            </regexp>\n          ]\n          filtered_records = filter(conf, records)\n          assert_equal(records.values_at(3, 4), filtered_records)\n        end\n\n        test \"regexp, or/exclude\" do\n          conf = %[\n            <regexp>\n              key level\n              pattern ^ERROR|WARN$\n            </regexp>\n            <regexp>\n              key method\n              pattern ^GET|POST$\n            </regexp>\n            <or>\n              <exclude>\n                key level\n                pattern ^WARN$\n              </exclude>\n              <exclude>\n                key method\n                pattern ^GET$\n              </exclude>\n            </or>\n          ]\n          filtered_records = filter(conf, records)\n          assert_equal(records.values_at(4), filtered_records)\n        end\n\n        test \"regexp, and/exclude\" do\n          conf = %[\n            <regexp>\n              key level\n              pattern ^ERROR|WARN$\n            </regexp>\n            <regexp>\n              key method\n              pattern ^GET|POST$\n            </regexp>\n            <and>\n              <exclude>\n                key level\n                pattern ^WARN$\n              </exclude>\n              <exclude>\n                key method\n                pattern ^GET$\n              </exclude>\n            </and>\n          ]\n          filtered_records = filter(conf, records)\n          assert_equal(records.values_at(1, 3, 4, 5, 7), filtered_records)\n        end\n      end\n    end\n  end\n\n  sub_test_case 'nested keys' do\n    def messages\n      [\n        {\"nest1\" => {\"nest2\" => \"INFO\"}},\n        {\"nest1\" => {\"nest2\" => \"WARN\"}},\n        {\"nest1\" => {\"nest2\" => \"WARN\"}}\n      ]\n    end\n\n    def filter(config, msgs)\n      d = create_driver(config)\n      d.run {\n        msgs.each { |msg|\n          d.feed(\"filter.test\", @time, {'foo' => 'bar', 'message' => msg})\n        }\n      }\n      d.filtered_records\n    end\n\n    test 'regexps' do\n      conf = %[\n        <regexp>\n          key $.message.nest1.nest2\n          pattern WARN\n        </regexp>\n      ]\n      filtered_records = filter(conf, messages)\n      assert_equal(2, filtered_records.size)\n      assert_block('only 2 nested logs') do\n        filtered_records.all? { |r|\n          r['message']['nest1']['nest2'] == 'WARN'\n        }\n      end\n    end\n\n    test 'excludes' do\n      conf = %[\n        <exclude>\n          key $.message.nest1.nest2\n          pattern WARN\n        </exclude>\n      ]\n      filtered_records = filter(conf, messages)\n      assert_equal(1, filtered_records.size)\n      assert_block('only 2 nested logs') do\n        filtered_records.all? { |r|\n          r['message']['nest1']['nest2'] == 'INFO'\n        }\n      end\n    end\n  end\n\n  sub_test_case 'grep non-string jsonable values' do\n    def filter(msg, config = 'regexp1 message 0')\n      d = create_driver(config)\n      d.run do\n        d.feed(\"filter.test\", @time, {'foo' => 'bar', 'message' => msg})\n      end\n      d.filtered_records\n    end\n\n    data(\n      'array' => [\"0\"],\n      'hash' => [\"0\" => \"0\"],\n      'integer' => 0,\n      'float' => 0.1)\n    test \"value\" do |data|\n      filtered_records = filter(data)\n      assert_equal(1, filtered_records.size)\n    end\n\n    test \"value boolean\" do\n      filtered_records = filter(true, %[regexp1 message true])\n      assert_equal(1, filtered_records.size)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_filter_parser.rb",
    "content": "require_relative '../helper'\nrequire 'timecop'\nrequire 'fluent/test/driver/filter'\nrequire 'fluent/plugin/filter_parser'\n\nclass ParserFilterTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    @tag = 'test'\n    @default_time = Time.parse('2010-05-04 03:02:01 UTC')\n    Timecop.freeze(@default_time)\n  end\n\n  def teardown\n    super\n    Timecop.return\n  end\n\n  def assert_equal_parsed_time(expected, actual)\n    if expected.is_a?(Integer)\n      assert_equal(expected, actual.to_i)\n    else\n      assert_equal_event_time(expected, actual)\n    end\n  end\n\n  ParserError = Fluent::Plugin::Parser::ParserError\n  CONFIG = %[\n    key_name     message\n    reserve_data true\n    <parse>\n      @type regexp\n      expression /^(?<x>.)(?<y>.) (?<time>.+)$/\n      time_format %Y%m%d%H%M%S\n    </parse>\n  ]\n\n  def create_driver(conf=CONFIG)\n    Fluent::Test::Driver::Filter.new(Fluent::Plugin::ParserFilter).configure(conf)\n  end\n\n  def test_configure\n    assert_raise(Fluent::ConfigError) {\n      create_driver('')\n    }\n    assert_raise(Fluent::NotFoundPluginError) {\n      create_driver %[\n        key_name foo\n        <parse>\n          @type unknown_format_that_will_never_be_implemented\n        </parse>\n      ]\n    }\n    assert_nothing_raised {\n      create_driver %[\n        key_name foo\n        <parse>\n          @type regexp\n          expression /(?<x>.)/\n        </parse>\n      ]\n    }\n    assert_nothing_raised {\n      create_driver %[\n        key_name foo\n        <parse>\n          @type json\n        </parse>\n      ]\n    }\n    assert_nothing_raised {\n      create_driver %[\n        key_name foo\n        format json\n      ]\n    }\n    assert_nothing_raised {\n      create_driver %[\n        key_name foo\n        <parse>\n          @type ltsv\n        </parse>\n      ]\n    }\n    assert_nothing_raised {\n      create_driver %[\n        key_name message\n        <parse>\n          @type regexp\n          expression /^col1=(?<col1>.+) col2=(?<col2>.+)$/\n        </parse>\n      ]\n    }\n    d = create_driver %[\n      key_name foo\n      <parse>\n        @type regexp\n        expression /(?<x>.)/\n      </parse>\n    ]\n    assert_false d.instance.reserve_data\n  end\n\n  # CONFIG = %[\n  #   remove_prefix test\n  #   add_prefix    parsed\n  #   key_name      message\n  #   format        /^(?<x>.)(?<y>.) (?<time>.+)$/\n  #   time_format   %Y%m%d%H%M%S\n  #   reserve_data  true\n  # ]\n  def test_filter\n    d1 = create_driver(CONFIG)\n    time = event_time(\"2012-01-02 13:14:15\")\n    d1.run(default_tag: @tag) do\n      d1.feed(time, {'message' => '12 20120402182059'})\n      d1.feed(time, {'message' => '34 20120402182100'})\n      d1.feed(time, {'message' => '56 20120402182100'})\n      d1.feed(time, {'message' => '78 20120402182101'})\n      d1.feed(time, {'message' => '90 20120402182100'})\n    end\n    filtered = d1.filtered\n    assert_equal 5, filtered.length\n\n    first = filtered[0]\n    assert_equal_event_time event_time(\"2012-04-02 18:20:59\"), first[0]\n    assert_equal '1', first[1]['x']\n    assert_equal '2', first[1]['y']\n    assert_equal '12 20120402182059', first[1]['message']\n\n    second = filtered[1]\n    assert_equal_event_time event_time(\"2012-04-02 18:21:00\"), second[0]\n    assert_equal '3', second[1]['x']\n    assert_equal '4', second[1]['y']\n\n    third = filtered[2]\n    assert_equal_event_time event_time(\"2012-04-02 18:21:00\"), third[0]\n    assert_equal '5', third[1]['x']\n    assert_equal '6', third[1]['y']\n\n    fourth = filtered[3]\n    assert_equal_event_time event_time(\"2012-04-02 18:21:01\"), fourth[0]\n    assert_equal '7', fourth[1]['x']\n    assert_equal '8', fourth[1]['y']\n\n    fifth = filtered[4]\n    assert_equal_event_time event_time(\"2012-04-02 18:21:00\"), fifth[0]\n    assert_equal '9', fifth[1]['x']\n    assert_equal '0', fifth[1]['y']\n\n    d2 = create_driver(%[\n      key_name data\n      format   /^(?<x>.)(?<y>.) (?<t>.+)$/\n    ])\n    time = Fluent::EventTime.from_time(@default_time) # EventTime emit test\n    d2.run(default_tag: @tag) do\n      d2.feed(time, {'data' => '12 20120402182059'})\n      d2.feed(time, {'data' => '34 20120402182100'})\n    end\n    filtered = d2.filtered\n    assert_equal 2, filtered.length\n\n    first = filtered[0]\n    assert_equal_event_time time, first[0]\n    assert_nil first[1]['data']\n    assert_equal '1', first[1]['x']\n    assert_equal '2', first[1]['y']\n    assert_equal '20120402182059', first[1]['t']\n\n    second = filtered[1]\n    assert_equal_event_time time, second[0]\n    assert_nil second[1]['data']\n    assert_equal '3', second[1]['x']\n    assert_equal '4', second[1]['y']\n    assert_equal '20120402182100', second[1]['t']\n\n    d3 = create_driver(%[\n      key_name data\n      <parse>\n        @type regexp\n        expression /^(?<x>[0-9])(?<y>[0-9]) (?<t>.+)$/\n      </parse>\n    ])\n    time = Time.parse(\"2012-04-02 18:20:59\").to_i\n    d3.run(default_tag: @tag) do\n      d3.feed(time, {'data' => '12 20120402182059'})\n      d3.feed(time, {'data' => '34 20120402182100'})\n      d3.feed(time, {'data' => 'xy 20120402182101'})\n    end\n    filtered = d3.filtered\n    assert_equal 2, filtered.length\n\n    d4 = create_driver(%[\n      key_name data\n      <parse>\n        @type json\n      </parse>\n    ])\n    time = Time.parse(\"2012-04-02 18:20:59\").to_i\n    d4.run(default_tag: @tag) do\n      d4.feed(time, {'data' => '{\"xxx\":\"first\",\"yyy\":\"second\"}', 'xxx' => 'x', 'yyy' => 'y'})\n      d4.feed(time, {'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'})\n    end\n    filtered = d4.filtered\n    assert_equal 1, filtered.length\n\n  end\n\n  def test_filter_with_multiple_records\n    d1 = create_driver(%[\n      key_name data\n      <parse>\n        @type json\n      </parse>\n    ])\n    time = Fluent::EventTime.from_time(@default_time)\n    d1.run(default_tag: @tag) do\n      d1.feed(time, {'data' => '[{\"xxx_1\":\"first\",\"yyy\":\"second\"}, {\"xxx_2\":\"first\", \"yyy_2\":\"second\"}]'})\n    end\n    filtered = d1.filtered\n    assert_equal 2, filtered.length\n    assert_equal ({\"xxx_1\"=>\"first\", \"yyy\"=>\"second\"}), filtered[0][1]\n    assert_equal ({\"xxx_2\"=>\"first\", \"yyy_2\"=>\"second\"}), filtered[1][1]\n  end\n\n  data(:keep_key_name => false,\n       :remove_key_name => true)\n  def test_filter_with_reserved_data(remove_key_name)\n    d1 = create_driver(%[\n      key_name data\n      reserve_data yes\n      remove_key_name_field #{remove_key_name}\n      <parse>\n        @type regexp\n        expression /^(?<x>\\\\d)(?<y>\\\\d) (?<t>.+)$/\n      </parse>\n    ])\n    time = event_time(\"2012-04-02 18:20:59\")\n    d1.run(default_tag: @tag) do\n      d1.feed(time, {'data' => '12 20120402182059'})\n      d1.feed(time, {'data' => '34 20120402182100'})\n      d1.feed(time, {'data' => 'xy 20120402182101'})\n    end\n    filtered = d1.filtered\n    assert_equal 3, filtered.length\n\n    d2 = create_driver(%[\n      key_name data\n      reserve_data yes\n      remove_key_name_field #{remove_key_name}\n      <parse>\n        @type json\n      </parse>\n    ])\n    time = Fluent::EventTime.from_time(@default_time)\n    d2.run(default_tag: @tag) do\n      d2.feed(time, {'data' => '{\"xxx\":\"first\",\"yyy\":\"second\"}', 'xxx' => 'x', 'yyy' => 'y'})\n      d2.feed(time, {'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'})\n    end\n    filtered = d2.filtered\n    assert_equal 2, filtered.length\n\n    first = filtered[0]\n    assert_equal_event_time time, first[0]\n    if remove_key_name\n      assert_not_include first[1], 'data'\n    else\n      assert_equal '{\"xxx\":\"first\",\"yyy\":\"second\"}', first[1]['data']\n    end\n    assert_equal 'first', first[1]['xxx']\n    assert_equal 'second', first[1]['yyy']\n\n    second = filtered[1]\n    assert_equal_event_time time, second[0]\n    assert_equal 'foobar', second[1]['data']\n    assert_equal 'x', second[1]['xxx']\n    assert_equal 'y', second[1]['yyy']\n  end\n\n\n  CONFIG_LTSV =  %[\n    key_name data\n    <parse>\n      @type ltsv\n    </parse>\n  ]\n  CONFIG_LTSV_WITH_TYPES =  %[\n    key_name data\n    <parse>\n      @type ltsv\n      types i:integer,s:string,f:float,b:bool\n    </parse>\n  ]\n  data(:event_time => lambda { |time| Fluent::EventTime.from_time(time) },\n       :int_time => lambda { |time| time.to_i })\n  def test_filter_ltsv(time_parse)\n    d = create_driver(CONFIG_LTSV)\n    time = time_parse.call(@default_time)\n    d.run(default_tag: @tag) do\n      d.feed(time, {'data' => \"xxx:first\\tyyy:second\", 'xxx' => 'x', 'yyy' => 'y'})\n      d.feed(time, {'data' => \"xxx:first\\tyyy:second2\", 'xxx' => 'x', 'yyy' => 'y'})\n    end\n    filtered = d.filtered\n    assert_equal 2, filtered.length\n\n    first = filtered[0]\n    assert_equal_parsed_time time, first[0]\n    assert_nil first[1]['data']\n    assert_equal 'first', first[1]['xxx']\n    assert_equal 'second', first[1]['yyy']\n\n    second = filtered[1]\n    assert_equal_parsed_time time, second[0]\n    assert_nil first[1]['data']\n    assert_equal 'first', second[1]['xxx']\n    assert_equal 'second2', second[1]['yyy']\n\n    d = create_driver(CONFIG_LTSV + %[reserve_data yes])\n    time = @default_time.to_i\n    d.run(default_tag: @tag) do\n      d.feed(time, {'data' => \"xxx:first\\tyyy:second\", 'xxx' => 'x', 'yyy' => 'y'})\n      d.feed(time, {'data' => \"xxx:first\\tyyy:second2\", 'xxx' => 'x', 'yyy' => 'y'})\n    end\n    filtered = d.filtered\n    assert_equal 2, filtered.length\n\n    first = filtered[0]\n    assert_equal_parsed_time time, first[0]\n    assert_equal \"xxx:first\\tyyy:second\", first[1]['data']\n    assert_equal 'first', first[1]['xxx']\n    assert_equal 'second', first[1]['yyy']\n\n    second = filtered[1]\n    assert_equal_parsed_time time, second[0]\n    assert_equal \"xxx:first\\tyyy:second\", first[1]['data']\n    assert_equal 'first', second[1]['xxx']\n    assert_equal 'second2', second[1]['yyy']\n\n    # convert types\n    #d = create_driver(CONFIG_LTSV + %[\n    d = create_driver(CONFIG_LTSV_WITH_TYPES)\n    time = @default_time.to_i\n    d.run do\n      d.feed(@tag, time, {'data' => \"i:1\\ts:2\\tf:3\\tb:true\\tx:123\"})\n    end\n    filtered = d.filtered\n    assert_equal 1, filtered.length\n\n    first = filtered[0]\n    assert_equal_parsed_time time, first[0]\n    assert_equal 1, first[1]['i']\n    assert_equal '2', first[1]['s']\n    assert_equal 3.0, first[1]['f']\n    assert_equal true, first[1]['b']\n    assert_equal '123', first[1]['x']\n  end\n\n  CONFIG_TSV =  %[\n    key_name data\n    <parse>\n      @type tsv\n      keys key1,key2,key3\n    </parse>\n  ]\n  data(:event_time => lambda { |time| Fluent::EventTime.from_time(time) },\n       :int_time => lambda { |time| time.to_i })\n  def test_filter_tsv(time_parse)\n    d = create_driver(CONFIG_TSV)\n    time = time_parse.call(@default_time)\n    d.run do\n      d.feed(@tag, time, {'data' => \"value1\\tvalue2\\tvalueThree\", 'xxx' => 'x', 'yyy' => 'y'})\n    end\n    filtered = d.filtered\n    assert_equal 1, filtered.length\n\n    first = filtered[0]\n    assert_equal_parsed_time time, first[0]\n    assert_nil first[1]['data']\n    assert_equal 'value1', first[1]['key1']\n    assert_equal 'value2', first[1]['key2']\n    assert_equal 'valueThree', first[1]['key3']\n  end\n\n  CONFIG_CSV =  %[\n    key_name data\n    <parse>\n      @type csv\n      keys key1,key2,key3\n    </parse>\n  ]\n  CONFIG_CSV_COMPAT =  %[\n    key_name data\n    format csv\n    keys key1,key2,key3\n  ]\n  data(new_conf: CONFIG_CSV,\n       compat_conf: CONFIG_CSV_COMPAT)\n  def test_filter_csv(conf)\n    d = create_driver(conf)\n    time = @default_time.to_i\n    d.run do\n      d.feed(@tag, time, {'data' => 'value1,\"value2\",\"value\"\"ThreeYes!\"', 'xxx' => 'x', 'yyy' => 'y'})\n    end\n    filtered = d.filtered\n    assert_equal 1, filtered.length\n\n    first = filtered[0]\n    assert_equal time, first[0]\n    assert_nil first[1]['data']\n    assert_equal 'value1', first[1]['key1']\n    assert_equal 'value2', first[1]['key2']\n    assert_equal 'value\"ThreeYes!', first[1]['key3']\n  end\n\n  def test_filter_with_nested_record\n    d = create_driver(%[\n      key_name $.data.log\n      <parse>\n        @type csv\n        keys key1,key2,key3\n      </parse>\n    ])\n    time = @default_time.to_i\n    d.run do\n      d.feed(@tag, time, {'data' => {'log' => 'value1,\"value2\",\"value\"\"ThreeYes!\"'}, 'xxx' => 'x', 'yyy' => 'y'})\n    end\n    filtered = d.filtered\n    assert_equal 1, filtered.length\n\n    first = filtered[0]\n    assert_equal time, first[0]\n    assert_nil first[1]['data']\n    assert_equal 'value1', first[1]['key1']\n    assert_equal 'value2', first[1]['key2']\n    assert_equal 'value\"ThreeYes!', first[1]['key3']\n  end\n\n  CONFIG_HASH_VALUE_FIELD = %[\n    key_name data\n    hash_value_field parsed\n    <parse>\n      @type json\n    </parse>\n  ]\n  CONFIG_HASH_VALUE_FIELD_RESERVE_DATA = %[\n    key_name data\n    reserve_data yes\n    hash_value_field parsed\n    <parse>\n      @type json\n    </parse>\n  ]\n  CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX = %[\n    key_name data\n    hash_value_field parsed\n    inject_key_prefix data.\n    <parse>\n      @type json\n    </parse>\n  ]\n  data(:event_time => lambda { |time| Fluent::EventTime.from_time(time) },\n       :int_time => lambda { |time| time.to_i })\n  def test_filter_inject_hash_value_field(time_parse)\n    original = {'data' => '{\"xxx\":\"first\",\"yyy\":\"second\"}', 'xxx' => 'x', 'yyy' => 'y'}\n\n    d = create_driver(CONFIG_HASH_VALUE_FIELD)\n    time = time_parse.call(@default_time)\n    d.run do\n      d.feed(@tag, time, original)\n    end\n    filtered = d.filtered\n    assert_equal 1, filtered.length\n\n    first = filtered[0]\n    assert_equal_parsed_time time, first[0]\n\n    record = first[1]\n    assert_equal 1, record.keys.size\n    assert_equal({\"xxx\"=>\"first\",\"yyy\"=>\"second\"}, record['parsed'])\n\n    d = create_driver(CONFIG_HASH_VALUE_FIELD_RESERVE_DATA)\n    time = @default_time.to_i\n    d.run do\n      d.feed(@tag, time, original)\n    end\n    filtered = d.filtered\n    assert_equal 1, filtered.length\n\n    first = filtered[0]\n    assert_equal_parsed_time time, first[0]\n\n    record = first[1]\n    assert_equal 4, record.keys.size\n    assert_equal original['data'], record['data']\n    assert_equal original['xxx'], record['xxx']\n    assert_equal original['yyy'], record['yyy']\n    assert_equal({\"xxx\"=>\"first\",\"yyy\"=>\"second\"}, record['parsed'])\n\n    d = create_driver(CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX)\n    time = @default_time.to_i\n    d.run do\n      d.feed(@tag, time, original)\n    end\n    filtered = d.filtered\n    assert_equal 1, filtered.length\n\n    first = filtered[0]\n    assert_equal_parsed_time time, first[0]\n\n    record = first[1]\n    assert_equal 1, record.keys.size\n    assert_equal({\"data.xxx\"=>\"first\",\"data.yyy\"=>\"second\"}, record['parsed'])\n  end\n\n  CONFIG_DONT_PARSE_TIME = %[\n    key_name data\n    reserve_time true\n    <parse>\n      @type json\n      keep_time_key true\n    </parse>\n  ]\n  CONFIG_DONT_PARSE_TIME_COMPAT = %[\n    key_name data\n    reserve_time true\n    format json\n    keep_time_key true\n  ]\n  data(new_conf: CONFIG_DONT_PARSE_TIME,\n       compat_conf: CONFIG_DONT_PARSE_TIME_COMPAT)\n  def test_time_should_be_reserved(conf)\n    t = Time.now.to_i\n    d = create_driver(conf)\n    d.run(default_tag: @tag) do\n      d.feed(t, {'data' => '{\"time\":1383190430, \"f1\":\"v1\"}'})\n      d.feed(t, {'data' => '{\"time\":\"1383190430\", \"f1\":\"v1\"}'})\n      d.feed(t, {'data' => '{\"time\":\"2013-10-31 12:34:03 +0900\", \"f1\":\"v1\"}'})\n    end\n    filtered = d.filtered\n    assert_equal 3, filtered.length\n\n    assert_equal 'v1', filtered[0][1]['f1']\n    assert_equal 1383190430, filtered[0][1]['time']\n    assert_equal t, filtered[0][0]\n\n    assert_equal 'v1', filtered[1][1]['f1']\n    assert_equal \"1383190430\", filtered[1][1]['time']\n    assert_equal t, filtered[1][0]\n\n    assert_equal 'v1', filtered[2][1]['f1']\n    assert_equal '2013-10-31 12:34:03 +0900', filtered[2][1]['time']\n    assert_equal t, filtered[2][0]\n  end\n\n  sub_test_case \"abnormal cases\" do\n    module HashExcept\n      refine Hash do\n        def except(*keys)\n          reject do |key, _|\n            keys.include?(key)\n          end\n        end\n      end\n    end\n\n    # Ruby 2.x does not support Hash#except.\n    using HashExcept unless {}.respond_to?(:except)\n\n    def run_and_assert(driver, records:, expected_records:, expected_error_records:, expected_errors:)\n      driver.run do\n        records.each do |record|\n          driver.feed(@tag, Fluent::EventTime.now.to_i, record)\n        end\n      end\n\n      assert_equal(\n        [\n          expected_records,\n          expected_error_records,\n          expected_errors.collect { |e| [e.class, e.message] },\n        ],\n        [\n          driver.filtered_records,\n          driver.error_events.collect { |_, _, record, _| record },\n          driver.error_events.collect { |_, _, _, e| [e.class, e.message] },\n        ]\n      )\n    end\n\n    data(\n      \"with default\" => {\n        records: [{\"foo\" => \"bar\"}],\n        additional_config: \"\",\n        expected_records: [],\n        expected_error_records: [{\"foo\" => \"bar\"}],\n        expected_errors: [ArgumentError.new(\"data does not exist\")],\n      },\n      \"with reserve_data\" => {\n        records: [{\"foo\" => \"bar\"}],\n        additional_config: \"reserve_data\",\n        expected_records: [{\"foo\" => \"bar\"}],\n        expected_error_records: [{\"foo\" => \"bar\"}],\n        expected_errors: [ArgumentError.new(\"data does not exist\")],\n      },\n      \"with disabled emit_invalid_record_to_error\" => {\n        records: [{\"foo\" => \"bar\"}],\n        additional_config: \"emit_invalid_record_to_error false\",\n        expected_records: [],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n      \"with reserve_data and disabled emit_invalid_record_to_error\" => {\n        records: [{\"foo\" => \"bar\"}],\n        additional_config: [\"reserve_data\", \"emit_invalid_record_to_error false\"].join(\"\\n\"),\n        expected_records: [{\"foo\" => \"bar\"}],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n      \"with reserve_data and hash_value_field\" => {\n        records: [{\"foo\" => \"bar\"}],\n        additional_config: [\"reserve_data\", \"hash_value_field parsed\"].join(\"\\n\"),\n        expected_records: [{\"foo\" => \"bar\", \"parsed\" => {}}],\n        expected_error_records: [{\"foo\" => \"bar\"}],\n        expected_errors: [ArgumentError.new(\"data does not exist\")],\n      },\n    )\n    def test_filter_key_not_exist(data)\n      driver = create_driver(<<~EOC)\n        key_name data\n        #{data[:additional_config]}\n        <parse>\n          @type json\n        </parse>\n      EOC\n\n      run_and_assert(driver, **data.except(:additional_config))\n    end\n\n    data(\n      \"with default\" => {\n        records: [{\"data\" => \"foo bar\"}],\n        additional_config: \"\",\n        expected_records: [],\n        expected_error_records: [{\"data\" => \"foo bar\"}],\n        expected_errors: [Fluent::Plugin::Parser::ParserError.new(\"pattern not matched with data 'foo bar'\")],\n      },\n      \"with reserve_data\" => {\n        records: [{\"data\" => \"foo bar\"}],\n        additional_config: \"reserve_data\",\n        expected_records: [{\"data\" => \"foo bar\"}],\n        expected_error_records: [{\"data\" => \"foo bar\"}],\n        expected_errors: [Fluent::Plugin::Parser::ParserError.new(\"pattern not matched with data 'foo bar'\")],\n      },\n      \"with disabled emit_invalid_record_to_error\" => {\n        records: [{\"data\" => \"foo bar\"}],\n        additional_config: \"emit_invalid_record_to_error false\",\n        expected_records: [],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n      \"with reserve_data and disabled emit_invalid_record_to_error\" => {\n        records: [{\"data\" => \"foo bar\"}],\n        additional_config: [\"reserve_data\", \"emit_invalid_record_to_error false\"].join(\"\\n\"),\n        expected_records: [{\"data\" => \"foo bar\"}],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n      \"with matched pattern\" => {\n        records: [{\"data\" => \"col1=foo col2=bar\"}],\n        additional_config: \"\",\n        expected_records: [{\"col1\" => \"foo\", \"col2\" => \"bar\"}],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n    )\n    def test_pattern_not_matched(data)\n      driver = create_driver(<<~EOC)\n        key_name data\n        #{data[:additional_config]}\n        <parse>\n          @type regexp\n          expression /^col1=(?<col1>.+) col2=(?<col2>.+)$/\n        </parse>\n      EOC\n\n      run_and_assert(driver, **data.except(:additional_config))\n    end\n\n    data(\n      \"invalid format with default\" => {\n        records: [{'data' => '{\"time\":[], \"f1\":\"v1\"}'}],\n        additional_config: \"\",\n        expected_records: [],\n        expected_error_records: [{'data' => '{\"time\":[], \"f1\":\"v1\"}'}],\n        expected_errors: [Fluent::Plugin::Parser::ParserError.new(\"value must be a string or a number: [](Array)\")],\n      },\n      \"invalid format with disabled emit_invalid_record_to_error\" => {\n        records: [{'data' => '{\"time\":[], \"f1\":\"v1\"}'}],\n        additional_config: \"emit_invalid_record_to_error false\",\n        expected_records: [],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n      \"mixed valid and invalid with default\" => {\n        records: [{'data' => '{\"time\":[], \"f1\":\"v1\"}'}, {'data' => '{\"time\":0, \"f1\":\"v1\"}'}],\n        additional_config: \"\",\n        expected_records: [{\"f1\" => \"v1\"}],\n        expected_error_records: [{'data' => '{\"time\":[], \"f1\":\"v1\"}'}],\n        expected_errors: [Fluent::Plugin::Parser::ParserError.new(\"value must be a string or a number: [](Array)\")],\n      },\n    )\n    def test_parser_error(data)\n      driver = create_driver(<<~EOC)\n        key_name data\n        #{data[:additional_config]}\n        <parse>\n          @type json\n        </parse>\n      EOC\n\n      run_and_assert(driver, **data.except(:additional_config))\n    end\n\n    data(\n      \"UTF-8 with default\" => {\n        records: [{\"data\" => \"\\xff\"}],\n        additional_config: \"\",\n        expected_records: [],\n        expected_error_records: [{\"data\" => \"\\xff\"}],\n        expected_errors: [ArgumentError.new(\"invalid byte sequence in UTF-8\")],\n      },\n      \"UTF-8 with replace_invalid_sequence\" => {\n        records: [{\"data\" => \"\\xff\"}],\n        additional_config: \"replace_invalid_sequence\",\n        expected_records: [{\"message\" => \"?\"}],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n      \"UTF-8 with replace_invalid_sequence and reserve_data\" => {\n        records: [{\"data\" => \"\\xff\"}],\n        additional_config: [\"replace_invalid_sequence\", \"reserve_data\"].join(\"\\n\"),\n        expected_records: [{\"message\" => \"?\", \"data\" => \"\\xff\"}],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n      \"US-ASCII with default\" => {\n        records: [{\"data\" => \"\\xff\".force_encoding(\"US-ASCII\")}],\n        additional_config: \"\",\n        expected_records: [],\n        expected_error_records: [{\"data\" => \"\\xff\".force_encoding(\"US-ASCII\")}],\n        expected_errors: [ArgumentError.new(\"invalid byte sequence in US-ASCII\")],\n      },\n      \"US-ASCII with replace_invalid_sequence\" => {\n        records: [{\"data\" => \"\\xff\".force_encoding(\"US-ASCII\")}],\n        additional_config: \"replace_invalid_sequence\",\n        expected_records: [{\"message\" => \"?\".force_encoding(\"US-ASCII\")}],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n      \"US-ASCII with replace_invalid_sequence and reserve_data\" => {\n        records: [{\"data\" => \"\\xff\".force_encoding(\"US-ASCII\")}],\n        additional_config: [\"replace_invalid_sequence\", \"reserve_data\"].join(\"\\n\"),\n        expected_records: [{\"message\" => \"?\".force_encoding(\"US-ASCII\"), \"data\" => \"\\xff\".force_encoding(\"US-ASCII\")}],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n    )\n    def test_invalid_byte(data)\n      driver = create_driver(<<~EOC)\n        key_name data\n        #{data[:additional_config]}\n        <parse>\n          @type regexp\n          expression /^(?<message>.*)$/\n        </parse>\n      EOC\n\n      run_and_assert(driver, **data.except(:additional_config))\n    end\n\n    test \"replace_invalid_sequence should be applied only to invalid byte sequence errors\" do\n      error_msg = \"This is a dummy ArgumentError other than invalid byte sequence\"\n      any_instance_of(Fluent::Plugin::JSONParser) do |klass|\n        stub(klass).parse do\n          raise ArgumentError, error_msg\n        end\n      end\n\n      driver = create_driver(<<~EOC)\n        key_name data\n        replace_invalid_sequence\n        <parse>\n          @type json\n        </parse>\n      EOC\n\n      # replace_invalid_sequence should not applied\n      run_and_assert(\n        driver,\n        records: [{'data' => '{\"foo\":\"bar\"}'}],\n        expected_records: [],\n        expected_error_records: [{'data' => '{\"foo\":\"bar\"}'}],\n        expected_errors: [ArgumentError.new(error_msg)]\n      )\n    end\n\n    data(\n      \"with default\" => {\n        records: [{'data' => '{\"foo\":\"bar\"}'}],\n        additional_config: \"\",\n        expected_records: [],\n        expected_error_records: [{'data' => '{\"foo\":\"bar\"}'}],\n        expected_errors: [Fluent::Plugin::Parser::ParserError.new(\"parse failed This is a dummy unassumed error\")],\n      },\n      \"with disabled emit_invalid_record_to_error\" => {\n        records: [{'data' => '{\"foo\":\"bar\"}'}],\n        additional_config: \"emit_invalid_record_to_error false\",\n        expected_records: [],\n        expected_error_records: [],\n        expected_errors: [],\n      },\n    )\n    def test_unassumed_error(data)\n      any_instance_of(Fluent::Plugin::JSONParser) do |klass|\n        stub(klass).parse do\n          raise RuntimeError, \"This is a dummy unassumed error\"\n        end\n      end\n\n      driver = create_driver(<<~EOC)\n        key_name data\n        #{data[:additional_config]}\n        <parse>\n          @type json\n        </parse>\n      EOC\n\n      run_and_assert(driver, **data.except(:additional_config))\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_filter_record_transformer.rb",
    "content": "require_relative '../helper'\nrequire 'timecop'\nrequire 'fluent/test/driver/filter'\nrequire 'fluent/plugin/filter_record_transformer'\nrequire 'flexmock/test_unit'\n\nclass RecordTransformerFilterTest < Test::Unit::TestCase\n  include Fluent\n  include FlexMock::TestCase\n\n  setup do\n    Test.setup\n    @hostname = Socket.gethostname.chomp\n    @tag = 'test.tag'\n    @tag_parts = @tag.split('.')\n    @time = event_time('2010-05-04 03:02:01 UTC')\n    Timecop.freeze(@time.to_time)\n  end\n\n  teardown do\n    Timecop.return\n  end\n\n  def create_driver(conf = '')\n    Fluent::Test::Driver::Filter.new(Fluent::Plugin::RecordTransformerFilter).configure(conf)\n  end\n\n  sub_test_case 'configure' do\n    test 'check default' do\n      assert_nothing_raised do\n        create_driver\n      end\n    end\n\n    test \"keep_keys must be specified together with renew_record true\" do\n      assert_raise(Fluent::ConfigError) do\n        create_driver(%[keep_keys a])\n      end\n    end\n  end\n\n  sub_test_case \"test options\" do\n    def filter(config, msgs = [''])\n      d = create_driver(config)\n      d.run {\n        msgs.each { |msg|\n          d.feed(@tag, @time, {'foo' => 'bar', 'message' => msg, 'nest' => {'k1' => 'v1', 'k2' => 'v2'}})\n        }\n      }\n      d.filtered\n    end\n\n    CONFIG = %[\n      <record>\n        hostname ${hostname}\n        tag ${tag}\n        time ${time}\n        message ${hostname} ${tag_parts[-1]} ${record[\"message\"]}\n      </record>\n    ]\n\n    test 'typical usage' do\n      msgs = ['1', '2']\n      filtered = filter(CONFIG, msgs)\n      filtered.each_with_index do |(_t, r), i|\n        assert_equal('bar', r['foo'])\n        assert_equal(@hostname, r['hostname'])\n        assert_equal(@tag, r['tag'])\n        assert_equal(Time.at(@time).localtime.to_s, r['time'])\n        assert_equal(\"#{@hostname} #{@tag_parts[-1]} #{msgs[i]}\", r['message'])\n        assert_equal({'k1' => 'v1', 'k2' => 'v2'}, r['nest'])\n      end\n    end\n\n    test 'remove_keys' do\n      config = CONFIG + %[remove_keys foo,message]\n      filtered = filter(config)\n      filtered.each_with_index do |(_t, r), i|\n        assert_not_include(r, 'foo')\n        assert_equal(@hostname, r['hostname'])\n        assert_equal(@tag, r['tag'])\n        assert_equal(Time.at(@time).localtime.to_s, r['time'])\n        assert_not_include(r, 'message')\n      end\n    end\n\n    test 'remove_keys with nested key' do\n      config = CONFIG + %[remove_keys $.nest.k1]\n      filtered = filter(config)\n      filtered.each_with_index do |(_t, r), i|\n        assert_not_include(r['nest'], 'k1')\n      end\n    end\n\n    test 'renew_record' do\n      config = CONFIG + %[renew_record true]\n      msgs = ['1', '2']\n      filtered = filter(config, msgs)\n      filtered.each_with_index do |(_t, r), i|\n        assert_not_include(r, 'foo')\n        assert_equal(@hostname, r['hostname'])\n        assert_equal(@tag, r['tag'])\n        assert_equal(Time.at(@time).localtime.to_s, r['time'])\n        assert_equal(\"#{@hostname} #{@tag_parts[-1]} #{msgs[i]}\", r['message'])\n      end\n    end\n\n    test 'renew_time_key' do\n      config = %[renew_time_key message]\n      times = [ Time.local(2,2,3,4,5,2010,nil,nil,nil,nil), Time.local(3,2,3,4,5,2010,nil,nil,nil,nil) ]\n      msgs = times.map{|t| t.to_f.to_s }\n      filtered = filter(config, msgs)\n      filtered.each_with_index do |(time, _record), i|\n        assert_equal(times[i].to_i, time)\n        assert(time.is_a?(Fluent::EventTime))\n        assert_true(_record.has_key?('message'))\n      end\n    end\n\n    test 'renew_time_key and remove_keys' do\n      config = %[\n                 renew_time_key event_time_key\n                 remove_keys event_time_key\n                 auto_typecast true\n                 <record>\n                   event_time_key ${record[\"message\"]}\n                 </record>\n               ]\n      times = [Time.local(2, 2, 3, 4, 5, 2010, nil, nil, nil, nil), Time.local(3, 2, 3, 4, 5, 2010, nil, nil, nil, nil)]\n      msgs = times.map { |t| t.to_f.to_s }\n      filtered = filter(config, msgs)\n      filtered.each_with_index do |(time, _record), i|\n        assert_equal(times[i].to_i, time)\n        assert(time.is_a?(Fluent::EventTime))\n        assert_false(_record.has_key?('event_time_key'))\n      end\n    end\n\n    test 'keep_keys' do\n      config = %[renew_record true\\nkeep_keys foo,message]\n      msgs = ['1', '2']\n      filtered = filter(config, msgs)\n      filtered.each_with_index do |(_t, r), i|\n        assert_equal('bar', r['foo'])\n        assert_equal(msgs[i], r['message'])\n      end\n    end\n\n    test 'keep_keys that are not present in the original record should not be included in the result record' do\n      config = %[renew_record true\\nkeep_keys foo, bar, baz, message]\n      msgs = ['1', '2', nil]\n      filtered = filter(config, msgs)\n      filtered.each_with_index do |(_t, r), i|\n        assert_equal('bar', r['foo'])\n        assert_equal(msgs[i], r['message'])\n        assert_equal(false, r.has_key?('bar'))\n        assert_equal(false, r.has_key?('baz'))\n      end\n    end\n\n    test 'enable_ruby' do\n      config = %[\n        enable_ruby yes\n        <record>\n          message ${hostname} ${tag_parts.last} ${\"'\" + record[\"message\"] + \"'\"}\n        </record>\n      ]\n      msgs = ['1', '2']\n      filtered = filter(config, msgs)\n      filtered.each_with_index do |(_t, r), i|\n        assert_equal(\"#{@hostname} #{@tag_parts[-1]} '#{msgs[i]}'\", r['message'])\n      end\n    end\n\n    test 'hash_value' do\n      config = %[\n        <record>\n          hash_field {\"k1\":100, \"k2\":\"foobar\"}\n        </record>\n      %]\n      msgs = ['1', '2']\n      filtered = filter(config, msgs)\n      filtered.each_with_index do |(_t, r), i|\n        assert_equal({\"k1\"=>100, \"k2\"=>\"foobar\"}, r['hash_field'])\n      end\n    end\n\n    test 'array_value' do\n      config = %[\n        <record>\n          array_field [1, 2, 3]\n        </record>\n      %]\n      msgs = ['1', '2']\n      filtered = filter(config, msgs)\n      filtered.each_with_index do |(_t, r), i|\n        assert_equal([1,2,3], r['array_field'])\n      end\n    end\n\n    test 'array_hash_mixed' do\n      config = %[\n        <record>\n          mixed_field {\"hello\":[1,2,3], \"world\":{\"foo\":\"bar\"}}\n        </record>\n      %]\n      msgs = ['1', '2']\n      filtered = filter(config, msgs)\n      filtered.each_with_index do |(_t, r), i|\n        assert_equal({\"hello\"=>[1,2,3], \"world\"=>{\"foo\"=>\"bar\"}}, r['mixed_field'])\n      end\n    end\n  end\n\n  sub_test_case 'test placeholders' do\n    def filter(config, msgs = [''])\n      d = create_driver(config)\n      yield d if block_given?\n      d.run {\n        records = msgs.map do |msg|\n          next msg if msg.is_a?(Hash)\n          { 'eventType0' => 'bar', 'message' => msg }\n        end\n        records.each do |record|\n          d.feed(@tag, @time, record)\n        end\n      }\n      d.filtered\n    end\n\n    %w[yes no].each do |enable_ruby|\n      test \"hostname with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          <record>\n            message ${hostname}\n          </record>\n        ]\n        filtered = filter(config)\n        filtered.each do |t, r|\n          assert_equal(@hostname, r['message'])\n        end\n      end\n\n      test \"tag with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          <record>\n            message ${tag}\n          </record>\n        ]\n        filtered = filter(config)\n        filtered.each do |t, r|\n          assert_equal(@tag, r['message'])\n        end\n      end\n\n      test \"tag_parts with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          <record>\n            message ${tag_parts[0]} ${tag_parts[-1]}\n          </record>\n        ]\n        expected = \"#{@tag.split('.').first} #{@tag.split('.').last}\"\n        filtered = filter(config)\n        filtered.each do |t, r|\n          assert_equal(expected, r['message'])\n        end\n      end\n\n      test \"${tag_prefix[N]} and ${tag_suffix[N]} with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          <record>\n            message ${tag_prefix[1]} ${tag_prefix[-2]} ${tag_suffix[2]} ${tag_suffix[-3]}\n          </record>\n        ]\n        @tag = 'prefix.test.tag.suffix'\n        expected = \"prefix.test prefix.test.tag tag.suffix test.tag.suffix\"\n        filtered = filter(config)\n        filtered.each do |t, r|\n          assert_equal(expected, r['message'])\n        end\n      end\n\n      test \"time with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          <record>\n            message ${time}\n          </record>\n        ]\n        filtered = filter(config)\n        filtered.each do |t, r|\n          if enable_ruby == \"yes\"\n            assert_equal(Time.at(@time).localtime, r['message'])\n          else\n            assert_equal(Time.at(@time).localtime.to_s, r['message'])\n          end\n        end\n      end\n\n      test \"record keys with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          remove_keys eventType0\n          <record>\n            message bar ${record[\"message\"]}\n            eventtype ${record[\"eventType0\"]}\n          </record>\n        ]\n        msgs = ['1', '2']\n        filtered = filter(config, msgs)\n        filtered.each_with_index do |(_t, r), i|\n          assert_not_include(r, 'eventType0')\n          assert_equal(\"bar\", r['eventtype'])\n          assert_equal(\"bar #{msgs[i]}\", r['message'])\n        end\n      end\n\n      test \"Prevent overwriting reserved keys such as tag with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          <record>\n            new_tag ${tag}\n            new_record_tag ${record[\"tag\"]}\n          </record>\n        ]\n        records = [{'tag' => 'tag', 'time' => 'time'}]\n        filtered = filter(config, records)\n        filtered.each_with_index do |(_t, r), i|\n          assert_not_equal('tag', r['new_tag'])\n          assert_equal(@tag, r['new_tag'])\n          assert_equal('tag', r['new_record_tag'])\n        end\n      end\n\n      test \"hash values with placeholders with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          <record>\n            hash_field {\n              \"hostname\":\"${hostname}\",\n              \"tag\":\"${tag}\",\n              \"${tag}\":100\n            }\n          </record>\n        ]\n        msgs = ['1', '2']\n        filtered = filter(config, msgs)\n        filtered.each_with_index do |(_t, r), i|\n          assert_equal({\"hostname\" => @hostname, \"tag\" => @tag, \"#{@tag}\" => 100}, r['hash_field'])\n        end\n      end\n\n      test \"array values with placeholders with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          <record>\n            array_field [\"${hostname}\", \"${tag}\"]\n          </record>\n        ]\n        msgs = ['1', '2']\n        filtered = filter(config, msgs)\n        filtered.each_with_index do |(_t, r), i|\n          assert_equal([@hostname, @tag], r['array_field'])\n        end\n      end\n\n      test \"array and hash values with placeholders with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          <record>\n            mixed_field [{\"tag\":\"${tag}\"}]\n          </record>\n        ]\n        msgs = ['1', '2']\n        filtered = filter(config, msgs)\n        filtered.each_with_index do |(_t, r), i|\n          assert_equal([{\"tag\" => @tag}], r['mixed_field'])\n        end\n      end\n\n      test \"keys with placeholders with enable_ruby #{enable_ruby}\" do\n        config = %[\n          enable_ruby #{enable_ruby}\n          renew_record true\n          <record>\n            ${hostname} hostname\n            foo.${tag}  tag\n          </record>\n        ]\n        msgs = ['1', '2']\n        filtered = filter(config, msgs)\n        filtered.each_with_index do |(_t, r), i|\n          assert_equal({@hostname=>'hostname',\"foo.#{@tag}\"=>'tag'}, r)\n        end\n      end\n\n      test \"disabled typecasting of values with enable_ruby #{enable_ruby}\" do\n        config = %[\n          auto_typecast false\n          enable_ruby #{enable_ruby}\n          <record>\n            single      ${record[\"source\"]}\n            multiple    ${record[\"source\"]}${record[\"source\"]}\n            with_prefix prefix-${record[\"source\"]}\n            with_suffix ${record[\"source\"]}-suffix\n            with_quote  record[\"source\"][\"\"]\n          </record>\n        ]\n        msgs = [\n          { \"source\" => \"string\" },\n          { \"source\" => 123 },\n          { \"source\" => [1, 2] },\n          { \"source\" => {a:1, b:2} },\n          { \"source\" => nil },\n        ]\n        expected_results = [\n          { single: \"string\",\n            multiple: \"stringstring\",\n            with_prefix: \"prefix-string\",\n            with_suffix: \"string-suffix\",\n            with_quote: %Q{record[\"source\"][\"\"]} },\n          { single: 123.to_s,\n            multiple: \"#{123.to_s}#{123.to_s}\",\n            with_prefix: \"prefix-#{123.to_s}\",\n            with_suffix: \"#{123.to_s}-suffix\",\n            with_quote: %Q{record[\"source\"][\"\"]} },\n          { single: [1, 2].to_s,\n            multiple: \"#{[1, 2].to_s}#{[1, 2].to_s}\",\n            with_prefix: \"prefix-#{[1, 2].to_s}\",\n            with_suffix: \"#{[1, 2].to_s}-suffix\",\n            with_quote: %Q{record[\"source\"][\"\"]} },\n          { single: {a:1, b:2}.to_s,\n            multiple: \"#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}\",\n            with_prefix: \"prefix-#{{a:1, b:2}.to_s}\",\n            with_suffix: \"#{{a:1, b:2}.to_s}-suffix\",\n            with_quote: %Q{record[\"source\"][\"\"]} },\n          { single: nil.to_s,\n            multiple: \"#{nil.to_s}#{nil.to_s}\",\n            with_prefix: \"prefix-#{nil.to_s}\",\n            with_suffix: \"#{nil.to_s}-suffix\",\n            with_quote: %Q{record[\"source\"][\"\"]} },\n        ]\n        actual_results = []\n        filtered = filter(config, msgs)\n        filtered.each_with_index do |(_t, r), i|\n          actual_results << {\n            single: r[\"single\"],\n            multiple: r[\"multiple\"],\n            with_prefix: r[\"with_prefix\"],\n            with_suffix: r[\"with_suffix\"],\n            with_quote: r[\"with_quote\"],\n          }\n        end\n        assert_equal(expected_results, actual_results)\n      end\n\n      test \"enabled typecasting of values with enable_ruby #{enable_ruby}\" do\n        config = %[\n          auto_typecast yes\n          enable_ruby #{enable_ruby}\n          <record>\n            single      ${record[\"source\"]}\n            multiple    ${record[\"source\"]}${record[\"source\"]}\n            with_prefix prefix-${record[\"source\"]}\n            with_suffix ${record[\"source\"]}-suffix\n          </record>\n        ]\n        msgs = [\n          { \"source\" => \"string\" },\n          { \"source\" => 123 },\n          { \"source\" => [1, 2] },\n          { \"source\" => {a:1, b:2} },\n          { \"source\" => nil },\n        ]\n        expected_results = [\n          { single: \"string\",\n            multiple: \"stringstring\",\n            with_prefix: \"prefix-string\",\n            with_suffix: \"string-suffix\" },\n          { single: 123,\n            multiple: \"#{123.to_s}#{123.to_s}\",\n            with_prefix: \"prefix-#{123.to_s}\",\n            with_suffix: \"#{123.to_s}-suffix\" },\n          { single: [1, 2],\n            multiple: \"#{[1, 2].to_s}#{[1, 2].to_s}\",\n            with_prefix: \"prefix-#{[1, 2].to_s}\",\n            with_suffix: \"#{[1, 2].to_s}-suffix\" },\n          { single: {a:1, b:2},\n            multiple: \"#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}\",\n            with_prefix: \"prefix-#{{a:1, b:2}.to_s}\",\n            with_suffix: \"#{{a:1, b:2}.to_s}-suffix\" },\n          { single: nil,\n            multiple: \"#{nil.to_s}#{nil.to_s}\",\n            with_prefix: \"prefix-#{nil.to_s}\",\n            with_suffix: \"#{nil.to_s}-suffix\" },\n        ]\n        actual_results = []\n        filtered = filter(config, msgs)\n        filtered.each_with_index do |(_t, r), i|\n          actual_results << {\n            single: r[\"single\"],\n            multiple: r[\"multiple\"],\n            with_prefix: r[\"with_prefix\"],\n            with_suffix: r[\"with_suffix\"],\n          }\n        end\n        assert_equal(expected_results, actual_results)\n      end\n    end\n\n    test 'unknown placeholder (enable_ruby no)' do\n      config = %[\n        enable_ruby no\n        <record>\n          message ${unknown}\n        </record>\n      ]\n      filter(config) { |d|\n        mock(d.instance.log).warn(\"unknown placeholder `${unknown}` found\")\n      }\n    end\n\n    test 'expand fields starting with @ (enable_ruby no)' do\n      config = %[\n        enable_ruby no\n        <record>\n          foo ${record[\"@timestamp\"]}\n        </record>\n      ]\n      d = create_driver(config)\n      message = {\"@timestamp\" => \"foo\"}\n      d.run { d.feed(@tag, @time, message) }\n      filtered = d.filtered\n      filtered.each do |t, r|\n        assert_equal(message[\"@timestamp\"], r['foo'])\n      end\n    end\n\n    test 'auto_typecast placeholder containing {} (enable_ruby yes)' do\n      config = %[\n        tag tag\n        enable_ruby yes\n        auto_typecast yes\n        <record>\n          foo ${record.map{|k,v|v}}\n        </record>\n      ]\n      d = create_driver(config)\n      message = {\"@timestamp\" => \"foo\"}\n      d.run { d.feed(@tag, @time, message) }\n      filtered = d.filtered\n      filtered.each do |t, r|\n        assert_equal([message[\"@timestamp\"]], r['foo'])\n      end\n    end\n\n    test 'CGI.escape / CGI.unescape' do\n      config = %[\n        tag tag\n        enable_ruby yes\n        <record>\n          encoded_url ${CGI.escape(record[\"url\"])}\n          decoded_text ${CGI.unescape(record[\"quoted\"])}\n        </record>\n      ]\n      d = create_driver(config)\n      message = {\"url\": \"http://example.com/?q=ruby & cgi\", \"quoted\": \"hello%21\"}\n      d.run { d.feed(@tag, @time, message) }\n      filtered = d.filtered\n      filtered.each do |t, r|\n        assert_equal(\"http%3A%2F%2Fexample.com%2F%3Fq%3Druby+%26+cgi\", r['encoded_url'])\n        assert_equal(\"hello!\", r['decoded_text'])\n      end\n    end\n  end # test placeholders\n\n  sub_test_case 'test error record' do\n    test 'invalid record for placeholders' do\n      d = create_driver(%[\n        enable_ruby yes\n        <record>\n          foo ${record[\"unknown\"][\"key\"]}\n        </record>\n      ])\n      flexmock(d.instance.router).should_receive(:emit_error_event).\n        with(String, Fluent::EventTime, Hash, RuntimeError).once\n      d.run do\n        d.feed(@tag, Fluent::EventTime.now, {'key' => 'value'})\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_filter_stdout.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/filter'\nrequire 'fluent/plugin/filter_stdout'\nrequire 'timecop'\nrequire 'flexmock/test_unit'\n\nclass StdoutFilterTest < Test::Unit::TestCase\n  include FlexMock::TestCase\n\n  def setup\n    Fluent::Test.setup\n    @old_tz = ENV[\"TZ\"]\n    ENV[\"TZ\"] = \"UTC\"\n    Timecop.freeze\n    @default_newline = if Fluent.windows?\n                         \"\\r\\n\"\n                       else\n                         \"\\n\"\n                       end\n  end\n\n  def teardown\n    super # FlexMock::TestCase requires this\n    # http://flexmock.rubyforge.org/FlexMock/TestCase.html\n    Timecop.return\n    ENV[\"TZ\"] = @old_tz\n  end\n\n  CONFIG = config_element('ROOT')\n\n  def create_driver(conf = CONFIG)\n    Fluent::Test::Driver::Filter.new(Fluent::Plugin::StdoutFilter).configure(conf)\n  end\n\n  def filter(d, time, record)\n    d.run {\n      d.feed(\"filter.test\", time, record)\n    }\n    d.filtered_records\n  end\n\n  def test_through_record\n    d = create_driver\n    filtered = filter(d, event_time, {'test' => 'test'})\n    assert_equal([{'test' => 'test'}], filtered)\n  end\n\n  sub_test_case \"flat style parameters\" do\n    sub_test_case \"configure\" do\n      def test_configure_default\n        d = create_driver\n        d.run {}\n        assert_equal 'json', d.instance.formatter.output_type\n      end\n\n      data(json: \"json\",\n           hash: \"hash\",\n           ltsv: \"ltsv\")\n      def test_output_type(data)\n        d = create_driver(CONFIG + config_element(\"\", \"\", { \"output_type\" => data }))\n        d.run {}\n        assert_equal data, d.instance.formatter.output_type\n      end\n\n      def test_invalid_output_type\n        assert_raise(Fluent::NotFoundPluginError) do\n          d = create_driver(CONFIG + config_element(\"\", \"\", { \"output_type\" => \"foo\" }))\n          d.run {}\n        end\n      end\n    end\n\n    def test_output_type_json\n      d = create_driver(CONFIG + config_element(\"\", \"\", { \"output_type\" => \"json\" }))\n      etime = event_time(\"2016-10-07 21:09:31.012345678 UTC\")\n      out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }\n      assert_equal \"2016-10-07 21:09:31.012345678 +0000 filter.test: {\\\"test\\\":\\\"test\\\"}\\n\", out\n\n      # NOTE: Float::NAN is not jsonable\n      d = create_driver(CONFIG + config_element(\"\", \"\", { \"output_type\" => \"json\" }))\n      flexmock(d.instance.router).should_receive(:emit_error_event)\n      filter(d, etime, {'test' => Float::NAN})\n    end\n\n    def test_output_type_hash\n      d = create_driver(CONFIG + config_element(\"\", \"\", { \"output_type\" => \"hash\" }))\n      etime = event_time(\"2016-10-07 21:09:31.012345678 UTC\")\n      out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }\n      assert_equal \"2016-10-07 21:09:31.012345678 +0000 filter.test: {\\\"test\\\"=>\\\"test\\\"}\\n\", out.gsub(' => ', '=>')\n\n      # NOTE: Float::NAN is not jsonable, but hash string can output it.\n      d = create_driver(CONFIG + config_element(\"\", \"\", { \"output_type\" => \"hash\" }))\n      out = capture_log(d) { filter(d, etime, {'test' => Float::NAN}) }\n      assert_equal \"2016-10-07 21:09:31.012345678 +0000 filter.test: {\\\"test\\\"=>NaN}\\n\", out.gsub(' => ', '=>')\n    end\n\n    # Use include_time_key to output the message's time\n    def test_include_time_key\n      config = config_element(\"\", \"\", {\n                                \"output_type\" => \"json\",\n                                \"include_time_key\" => true,\n                                \"localtime\" => false\n                              })\n      d = create_driver(config)\n      etime = event_time(\"2016-10-07 21:09:31.012345678 UTC\")\n      out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }\n      assert_equal \"2016-10-07 21:09:31.012345678 +0000 filter.test: {\\\"test\\\":\\\"test\\\",\\\"time\\\":\\\"2016-10-07T21:09:31Z\\\"}\\n\", out\n    end\n\n    # out_stdout formatter itself can also be replaced\n    def test_format_json\n      d = create_driver(CONFIG + config_element(\"\", \"\", { \"format\" => \"json\" }))\n      out = capture_log(d) { filter(d, event_time, {'test' => 'test'}) }\n      assert_equal \"{\\\"test\\\":\\\"test\\\"}#{@default_newline}\", out\n    end\n  end\n\n  sub_test_case \"with <format> sub section\" do\n    sub_test_case \"configure\" do\n      def test_default\n        conf = config_element\n        conf.elements << config_element(\"format\", \"\", { \"@type\" => \"stdout\"})\n        d = create_driver(conf)\n        d.run {}\n        assert_equal(\"json\", d.instance.formatter.output_type)\n      end\n\n      data(json: \"json\",\n           hash: \"hash\",\n           ltsv: \"ltsv\")\n      def test_output_type(data)\n        conf = config_element\n        conf.elements << config_element(\"format\", \"\", { \"@type\" => \"stdout\", \"output_type\" => data })\n        d = create_driver(conf)\n        d.run {}\n        assert_equal(data, d.instance.formatter.output_type)\n      end\n\n      def test_invalid_output_type\n        conf = config_element\n        conf.elements << config_element(\"format\", \"\", { \"@type\" => \"stdout\", \"output_type\" => \"foo\" })\n        assert_raise(Fluent::NotFoundPluginError) do\n          d = create_driver(conf)\n          d.run {}\n        end\n      end\n    end\n\n    sub_test_case \"output_type\" do\n      def test_json\n        conf = config_element\n        conf.elements << config_element(\"format\", \"\", { \"@type\" => \"stdout\", \"output_type\" => \"json\" })\n        d = create_driver(conf)\n        etime = event_time(\"2016-10-07 21:09:31.012345678 UTC\")\n        out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }\n        assert_equal \"2016-10-07 21:09:31.012345678 +0000 filter.test: {\\\"test\\\":\\\"test\\\"}\\n\", out\n      end\n\n      def test_json_nan\n        # NOTE: Float::NAN is not jsonable\n        conf = config_element\n        conf.elements << config_element(\"format\", \"\", { \"@type\" => \"stdout\", \"output_type\" => \"json\" })\n        d = create_driver(conf)\n        etime = event_time(\"2016-10-07 21:09:31.012345678 UTC\")\n        flexmock(d.instance.router).should_receive(:emit_error_event)\n        filter(d, etime, {'test' => Float::NAN})\n      end\n\n      def test_hash\n        conf = config_element\n        conf.elements << config_element(\"format\", \"\", { \"@type\" => \"stdout\", \"output_type\" => \"hash\" })\n        d = create_driver(conf)\n        etime = event_time(\"2016-10-07 21:09:31.012345678 UTC\")\n        out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }\n        assert_equal \"2016-10-07 21:09:31.012345678 +0000 filter.test: {\\\"test\\\"=>\\\"test\\\"}\\n\", out.gsub(' => ', '=>')\n      end\n\n      def test_hash_nan\n        # NOTE: Float::NAN is not jsonable, but hash string can output it.\n        conf = config_element\n        conf.elements << config_element(\"format\", \"\", { \"@type\" => \"stdout\", \"output_type\" => \"hash\" })\n        d = create_driver(conf)\n        etime = event_time(\"2016-10-07 21:09:31.012345678 UTC\")\n        out = capture_log(d) { filter(d, etime, {'test' => Float::NAN}) }\n        assert_equal \"2016-10-07 21:09:31.012345678 +0000 filter.test: {\\\"test\\\"=>NaN}\\n\", out.gsub(' => ', '=>')\n      end\n\n      # Use include_time_key to output the message's time\n      def test_include_time_key\n        conf = config_element\n        conf.elements << config_element(\"format\", \"\", {\n                                          \"@type\" => \"stdout\",\n                                          \"output_type\" => \"json\"\n                                        })\n        conf.elements << config_element(\"inject\", \"\", {\n                                          \"time_key\" => \"time\",\n                                          \"time_type\" => \"string\",\n                                          \"localtime\" => false\n                                          })\n        d = create_driver(conf)\n        etime = event_time(\"2016-10-07 21:09:31.012345678 UTC\")\n        out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }\n        assert_equal \"2016-10-07 21:09:31.012345678 +0000 filter.test: {\\\"test\\\":\\\"test\\\",\\\"time\\\":\\\"2016-10-07T21:09:31Z\\\"}\\n\", out\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_formatter_csv.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/formatter'\nrequire 'fluent/plugin/formatter_csv'\n\nclass CsvFormatterTest < ::Test::Unit::TestCase\n\n  def setup\n    @time = event_time\n  end\n\n  CONF = %[\n    fields a,b,c\n  ]\n\n  def create_driver(conf = CONF)\n    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::CsvFormatter).configure(conf)\n  end\n\n  def tag\n    \"tag\"\n  end\n\n  def test_config_params\n    d = create_driver\n    assert_equal(',', d.instance.delimiter)\n    assert_equal(true, d.instance.force_quotes)\n    assert_equal(['a', 'b', 'c'], d.instance.fields)\n  end\n\n  data('empty array' => [],\n       'array including empty string' => ['', ''])\n  def test_empty_fields(param)\n    assert_raise Fluent::ConfigError do\n      create_driver('fields' => param)\n    end\n  end\n\n  data(\n    'tab_char' => [\"\\t\", '\\t'],\n    'tab_string' => [\"\\t\", 'TAB'],\n    'pipe' => ['|', '|'])\n  def test_config_params_with_customized_delimiters(data)\n    expected, target = data\n    d = create_driver(\"delimiter\" => target, 'fields' => 'a,b,c')\n    assert_equal expected, d.instance.delimiter\n  end\n\n  def test_format\n    d = create_driver(\"fields\" => \"message,message2\")\n    formatted = d.instance.format(tag, @time, {\n                                    'message' => 'awesome',\n                                    'message2' => 'awesome2'\n                                  })\n    assert_equal(\"\\\"awesome\\\",\\\"awesome2\\\"\\n\", formatted)\n  end\n\n  def test_format_with_nested_fields\n    d = create_driver(\"fields\" => \"message,$.nest.key\")\n    formatted = d.instance.format(tag, @time, {\n                                    'message' => 'awesome',\n                                    'nest' => {'key' => 'awesome2'}\n                                  })\n    assert_equal(\"\\\"awesome\\\",\\\"awesome2\\\"\\n\", formatted)\n  end\n\n  def test_format_without_newline\n    d = create_driver(\"fields\" => \"message,message2\", \"add_newline\" => false)\n    formatted = d.instance.format(tag, @time, {\n                                    'message' => 'awesome',\n                                    'message2' => 'awesome2'\n                                  })\n    assert_equal(\"\\\"awesome\\\",\\\"awesome2\\\"\", formatted)\n  end\n\n  def test_format_with_customized_delimiters\n    d = create_driver(\"fields\" => \"message,message2\",\n                      \"delimiter\" => \"\\t\")\n    formatted = d.instance.format(tag, @time, {\n                                    'message' => 'awesome',\n                                    'message2' => 'awesome2'\n                                  })\n    assert_equal(\"\\\"awesome\\\"\\t\\\"awesome2\\\"\\n\", formatted)\n  end\n\n  def test_format_with_non_quote\n    d = create_driver(\"fields\" => \"message,message2\",\n                      \"force_quotes\" => false)\n    formatted = d.instance.format(tag, @time, {\n                                    'message' => 'awesome',\n                                    'message2' => 'awesome2'\n                                  })\n    assert_equal(\"awesome,awesome2\\n\", formatted)\n  end\n\n  data(\n    'nil' => {\n      'message' => 'awesome',\n      'message2' => nil,\n      'message3' => 'awesome3'\n    },\n    'blank' => {\n      'message' => 'awesome',\n      'message2' => '',\n      'message3' => 'awesome3'\n    })\n  def test_format_with_empty_fields(data)\n    d = create_driver(\"fields\" => \"message,message2,message3\")\n    formatted = d.instance.format(tag, @time, data)\n    assert_equal(\"\\\"awesome\\\",\\\"\\\",\\\"awesome3\\\"\\n\", formatted)\n  end\n\n  data(\n    'normally' => 'one,two,three',\n    'white_space' => 'one , two , three',\n    'blank' => 'one,,two,three')\n  def test_config_params_with_fields(data)\n    d = create_driver('fields' => data)\n    assert_equal %w(one two three), d.instance.fields\n  end\n\n  def test_format_with_multiple_records\n    d = create_driver(\"fields\" => \"message,message2\")\n    r = {'message' => 'hello', 'message2' => 'fluentd'}\n\n    formatted = d.instance.format(tag, @time, r)\n    assert_equal(\"\\\"hello\\\",\\\"fluentd\\\"\\n\", formatted)\n\n    r = {'message' => 'hey', 'message2' => 'ho'}\n    formatted = d.instance.format(tag, @time, r)\n    assert_equal(\"\\\"hey\\\",\\\"ho\\\"\\n\", formatted)\n\n    r = {'message' => 'longer message', 'message2' => 'longer longer message'}\n    formatted = d.instance.format(tag, @time, r)\n    assert_equal(\"\\\"longer message\\\",\\\"longer longer message\\\"\\n\", formatted)\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_formatter_hash.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/formatter'\nrequire 'fluent/plugin/formatter_hash'\n\nclass HashFormatterTest < ::Test::Unit::TestCase\n  def setup\n    @time = event_time\n  end\n\n  def create_driver(conf = \"\")\n    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::HashFormatter).configure(conf)\n  end\n\n  def tag\n    \"tag\"\n  end\n\n  def record\n    {'message' => 'awesome', 'greeting' => 'hello'}\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format(data)\n    newline_conf, newline = data\n    d = create_driver({\"newline\" => newline_conf})\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(%Q!{\"message\"=>\"awesome\", \"greeting\"=>\"hello\"}#{newline}!, formatted.gsub(' => ', '=>').encode(Encoding::UTF_8))\n  end\n\n  def test_format_without_newline\n    d = create_driver('add_newline' => false)\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(%Q!{\"message\"=>\"awesome\", \"greeting\"=>\"hello\"}!, formatted.gsub(' => ', '=>').encode(Encoding::UTF_8))\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_formatter_json.rb",
    "content": "require_relative '../helper'\nrequire 'json'\nrequire 'fluent/test/driver/formatter'\nrequire 'fluent/plugin/formatter_json'\n\nclass JsonFormatterTest < ::Test::Unit::TestCase\n\n  def setup\n    @time = event_time\n    @default_newline = if Fluent.windows?\n                         \"\\r\\n\"\n                       else\n                         \"\\n\"\n                       end\n  end\n\n  def create_driver(conf = \"\")\n    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::JSONFormatter).configure(conf)\n  end\n\n  def tag\n    \"tag\"\n  end\n\n  def record\n    {'message' => 'awesome'}\n  end\n\n  def symbolic_record\n    {:message => :awesome}\n  end\n\n  data('oj with LF' => ['oj', \"lf\", \"\\n\"],\n       'oj with CRLF' => ['oj', \"crlf\", \"\\r\\n\"],\n       'yajl with LF' => ['yajl', \"lf\", \"\\n\"],\n       'yajl with CRLF' => ['yajl', \"crlf\", \"\\r\\n\"]\n      )\n  def test_format(data)\n    parser, newline_conf, newline = data\n    d = create_driver('json_parser' => parser, 'newline' => newline_conf)\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(\"#{JSON.generate(record)}#{newline}\", formatted)\n  end\n\n  data('oj' => 'oj', 'yajl' => 'yajl')\n  def test_format_without_nl(data)\n    d = create_driver('json_parser' => data, 'add_newline' => false)\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(JSON.generate(record), formatted)\n  end\n\n  data('oj' => 'oj', 'yajl' => 'yajl')\n  def test_format_with_symbolic_record(data)\n    d = create_driver('json_parser' => data)\n    formatted = d.instance.format(tag, @time, symbolic_record)\n\n    assert_equal(\"#{JSON.generate(record)}#{@default_newline}\", formatted)\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_formatter_ltsv.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/formatter'\nrequire 'fluent/plugin/formatter_ltsv'\n\nclass LabeledTSVFormatterTest < ::Test::Unit::TestCase\n  def setup\n    @time = event_time\n  end\n\n  def create_driver(conf = \"\")\n    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::LabeledTSVFormatter).configure(conf)\n  end\n\n  def tag\n    \"tag\"\n  end\n\n  def record\n    {'message' => 'awesome', 'greeting' => 'hello'}\n  end\n\n  def test_config_params\n    d = create_driver\n    assert_equal \"\\t\", d.instance.delimiter\n    assert_equal  \":\", d.instance.label_delimiter\n    assert_equal  true, d.instance.add_newline\n\n    d = create_driver(\n      'delimiter'       => ',',\n      'label_delimiter' => '=',\n      'add_newline' => false,\n    )\n\n    assert_equal \",\", d.instance.delimiter\n    assert_equal \"=\", d.instance.label_delimiter\n    assert_equal  false, d.instance.add_newline\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format(data)\n    newline_conf, newline = data\n    d = create_driver({\"newline\" => newline_conf})\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(\"message:awesome\\tgreeting:hello#{newline}\", formatted)\n  end\n\n  def test_format_without_newline\n    d = create_driver('add_newline' => false)\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(\"message:awesome\\tgreeting:hello\", formatted)\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format_with_customized_delimiters(data)\n    newline_conf, newline = data\n\n    d = create_driver(\n      'delimiter'       => ',',\n      'label_delimiter' => '=',\n      'newline' => newline_conf,\n    )\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(\"message=awesome,greeting=hello#{newline}\", formatted)\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_formatter_msgpack.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/formatter'\nrequire 'fluent/plugin/formatter_msgpack'\n\nclass MessagePackFormatterTest < ::Test::Unit::TestCase\n  def setup\n    @time = event_time\n  end\n\n  def create_driver(conf = \"\")\n    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::MessagePackFormatter).configure(conf)\n  end\n\n  def tag\n    \"tag\"\n  end\n\n  def record\n    {'message' => 'awesome'}\n  end\n\n  def test_format\n    d = create_driver({})\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(record.to_msgpack, formatted)\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_formatter_out_file.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/formatter'\nrequire 'fluent/plugin/formatter_out_file'\n\nclass OutFileFormatterTest < ::Test::Unit::TestCase\n  def setup\n    @time = event_time\n    @default_newline = if Fluent.windows?\n                         \"\\r\\n\"\n                       else\n                         \"\\n\"\n                       end\n  end\n\n  def create_driver(conf = {})\n    d = Fluent::Test::Driver::Formatter.new(Fluent::Plugin::OutFileFormatter)\n    case conf\n    when Fluent::Config::Element\n      d.configure(conf)\n    when Hash\n      d.configure({'utc' => true}.merge(conf))\n    else\n      d.configure(conf)\n    end\n  end\n\n  def tag\n    \"tag\"\n  end\n\n  def record\n    {'message' => 'awesome'}\n  end\n\n  data('both true' => 'true', 'both false' => 'false')\n  def test_configured_with_both_of_utc_and_localtime(value)\n    assert_raise(Fluent::ConfigError.new(\"both of utc and localtime are specified, use only one of them\")) do\n      create_driver({'utc' => value, 'localtime' => value})\n    end\n  end\n\n  time_i = Time.parse(\"2016-07-26 21:08:30 -0700\").to_i\n  data(\n    'configured for localtime by localtime' => ['localtime', 'true',  time_i, \"2016-07-26T21:08:30-07:00\"],\n    'configured for localtime by utc'       => ['utc',       'false', time_i, \"2016-07-26T21:08:30-07:00\"],\n    'configured for utc by localtime'       => ['localtime', 'false', time_i, \"2016-07-27T04:08:30Z\"],\n    'configured for utc by utc'             => ['utc',       'true',  time_i, \"2016-07-27T04:08:30Z\"],\n  )\n  def test_configured_with_utc_or_localtime(data)\n    key, value, time_i, expected = data\n    time = Time.at(time_i)\n    begin\n      oldtz, ENV['TZ'] = ENV['TZ'], \"UTC+07\"\n      d = create_driver(config_element('ROOT', '', {key => value}))\n      tag = 'test'\n      assert_equal \"#{expected}\\t#{tag}\\t#{JSON.generate(record)}#{@default_newline}\", d.instance.format(tag, time, record)\n    ensure\n      ENV['TZ'] = oldtz\n    end\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format(data)\n    newline_conf, newline = data\n    d = create_driver({\"newline\" => newline_conf})\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(\"#{time2str(@time)}\\t#{tag}\\t#{JSON.generate(record)}#{newline}\", formatted)\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format_without_time(data)\n    newline_conf, newline = data\n    d = create_driver('output_time' => 'false', 'newline' => newline_conf)\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(\"#{tag}\\t#{JSON.generate(record)}#{newline}\", formatted)\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format_without_tag(data)\n    newline_conf, newline = data\n    d = create_driver('output_tag' => 'false', 'newline' => newline_conf)\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(\"#{time2str(@time)}\\t#{JSON.generate(record)}#{newline}\", formatted)\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format_without_time_and_tag\n    newline_conf, newline = data\n    d = create_driver('output_tag' => 'false', 'output_time' => 'false', 'newline' => newline_conf)\n    formatted = d.instance.format('tag', @time, record)\n\n    assert_equal(\"#{JSON.generate(record)}#{newline}\", formatted)\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format_without_time_and_tag_against_string_literal_configure(data)\n    newline_conf, newline = data\n    d = create_driver(%[\n                        utc         true\n                        output_tag  false\n                        output_time false\n                        newline     #{newline_conf}\n                      ])\n    formatted = d.instance.format('tag', @time, record)\n\n    assert_equal(\"#{JSON.generate(record)}#{newline}\", formatted)\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_formatter_single_value.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/formatter'\nrequire 'fluent/plugin/formatter_single_value'\n\nclass SingleValueFormatterTest < ::Test::Unit::TestCase\n  def create_driver(conf = \"\")\n    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::SingleValueFormatter).configure(conf)\n  end\n\n  def test_config_params\n    d = create_driver\n    assert_equal \"message\", d.instance.message_key\n  end\n\n  def test_config_params_message_key\n    d = create_driver('message_key' => 'foobar')\n    assert_equal \"foobar\", d.instance.message_key\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format(data)\n    newline_conf, newline = data\n    d = create_driver('newline' => newline_conf)\n    formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})\n    assert_equal(\"awesome#{newline}\", formatted)\n  end\n\n  def test_format_without_newline\n    d = create_driver('add_newline' => 'false')\n    formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})\n    assert_equal(\"awesome\", formatted)\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format_with_message_key(data)\n    newline_conf, newline = data\n    d = create_driver('message_key' => 'foobar', 'newline' => newline_conf)\n    formatted = d.instance.format('tag', event_time, {'foobar' => 'foo'})\n\n    assert_equal(\"foo#{newline}\", formatted)\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_formatter_tsv.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/formatter'\nrequire 'fluent/plugin/formatter_tsv'\n\nclass TSVFormatterTest < ::Test::Unit::TestCase\n  def setup\n    @time = event_time\n  end\n\n  def create_driver(conf = \"\")\n    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::TSVFormatter).configure(conf)\n  end\n\n  def tag\n    \"tag\"\n  end\n\n  def record\n    {'message' => 'awesome', 'greeting' => 'hello'}\n  end\n\n  def test_config_params\n    d = create_driver(\n      'keys' => 'message,greeting',\n    )\n    assert_equal [\"message\", \"greeting\"], d.instance.keys\n    assert_equal \"\\t\", d.instance.delimiter\n    assert_equal true, d.instance.add_newline\n\n    d = create_driver(\n      'keys' => 'message,greeting',\n      'delimiter' => ',',\n      'add_newline' => false,\n    )\n    assert_equal [\"message\", \"greeting\"], d.instance.keys\n    assert_equal \",\", d.instance.delimiter\n    assert_equal false, d.instance.add_newline\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format(data)\n    newline_conf, newline = data\n    d = create_driver(\n      'keys' => 'message,greeting',\n      'newline' => newline_conf\n    )\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(\"awesome\\thello#{newline}\", formatted)\n  end\n\n  def test_format_without_newline\n    d = create_driver(\n      'keys' => 'message,greeting',\n      'add_newline' => false,\n    )\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(\"awesome\\thello\", formatted)\n  end\n\n  data(\"newline (LF)\" => [\"lf\", \"\\n\"],\n       \"newline (CRLF)\" => [\"crlf\", \"\\r\\n\"])\n  def test_format_with_customized_delimiters(data)\n    newline_conf, newline = data\n    d = create_driver(\n      'keys' => 'message,greeting',\n      'delimiter' => ',',\n      'newline' => newline_conf,\n    )\n    formatted = d.instance.format(tag, @time, record)\n\n    assert_equal(\"awesome,hello#{newline}\", formatted)\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_debug_agent.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_debug_agent'\nrequire 'fileutils'\n\nclass DebugAgentInputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    FileUtils.rm_rf(TMP_DIR)\n    FileUtils.mkdir_p(TMP_DIR)\n  end\n\n  TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/../tmp/in_debug_agent\")\n\n  def create_driver(conf = '')\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::DebugAgentInput).configure(conf)\n  end\n\n  def test_unix_path_writable\n    assert_nothing_raised do\n      create_driver %[unix_path #{TMP_DIR}/test_path]\n    end\n\n    assert_raise(Fluent::ConfigError) do\n      create_driver %[unix_path #{TMP_DIR}/does_not_exist/test_path]\n    end\n  end\n\n  def test_multi_worker_environment_with_port\n    Fluent::SystemConfig.overwrite_system_config('workers' => 4) do\n      d = Fluent::Test::Driver::Input.new(Fluent::Plugin::DebugAgentInput)\n      d.instance.instance_eval { @_fluentd_worker_id = 2 }\n      d.configure('port 24230')\n\n      assert_true d.instance.multi_workers_ready?\n      assert_equal(24232, d.instance.instance_variable_get(:@port))\n    end\n  end\n\n  def test_multi_worker_environment_with_unix_path\n    Fluent::SystemConfig.overwrite_system_config('workers' => 4) do\n      d = Fluent::Test::Driver::Input.new(Fluent::Plugin::DebugAgentInput)\n      d.instance.instance_eval { @_fluentd_worker_id = 2 }\n      d.configure(\"unix_path #{TMP_DIR}/test_path\")\n\n      assert_false d.instance.multi_workers_ready?\n    end\n  end\n\n  def test_default_configuration\n    assert_nothing_raised do\n      d = create_driver\n      assert_equal(['127.0.0.1', 24230, 'Fluent::Engine'],\n                   [d.instance.bind, d.instance.port, d.instance.object])\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_exec.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_exec'\nrequire 'timecop'\n\nclass ExecInputTest < Test::Unit::TestCase\n  SCRIPT_PATH = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))\n  TEST_TIME = \"2011-01-02 13:14:15\"\n  TEST_UNIX_TIME = Time.parse(TEST_TIME)\n\n  def setup\n    Fluent::Test.setup\n    @test_time = event_time()\n  end\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::ExecInput).configure(conf)\n  end\n\n  DEFAULT_CONFIG_ONLY_WITH_KEYS = %[\n    command ruby #{SCRIPT_PATH} \"#{TEST_TIME}\" 0\n    run_interval 1s\n    tag \"my.test.data\"\n    <parse>\n      keys [\"k1\", \"k2\", \"k3\"]\n    </parse>\n  ]\n\n  TSV_CONFIG = %[\n    command ruby #{SCRIPT_PATH} \"#{TEST_TIME}\" 0\n    run_interval 0.3\n    <parse>\n      @type tsv\n      keys time, tag, k1\n    </parse>\n    <extract>\n      tag_key tag\n      time_key time\n      time_type string\n      time_format %Y-%m-%d %H:%M:%S\n    </extract>\n  ]\n\n  JSON_CONFIG = %[\n    command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 1\n    run_interval 0.3\n    <parse>\n      @type json\n    </parse>\n    <extract>\n      tag_key tag\n      time_key time\n      time_type unixtime\n    </extract>\n  ]\n\n  MSGPACK_CONFIG = %[\n    command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 2\n    run_interval 0.3\n    <parse>\n      @type msgpack\n    </parse>\n    <extract>\n      tag_key tagger\n      time_key datetime\n      time_type unixtime\n    </extract>\n  ]\n\n  # here document for not de-quoting backslashes\n  REGEXP_CONFIG = %[\n    command ruby #{SCRIPT_PATH} \"#{TEST_TIME}\" 3\n    run_interval 0.3\n    tag regex_tag\n] + <<'EOC'\n    <parse>\n      @type regexp\n      expression \"(?<time>[^\\\\]]*) (?<message>[^ ]*)\"\n      time_key time\n      time_type string\n      time_format %Y-%m-%d %H:%M:%S\n    </parse>\nEOC\n\n  sub_test_case 'with configuration with sections' do\n    test 'configure with default tsv format without extract' do\n      d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS\n      assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }\n      assert_equal \"my.test.data\", d.instance.tag\n      assert_equal [\"k1\", \"k2\", \"k3\"], d.instance.parser.keys\n    end\n\n    test 'configure raises error if both of tag and extract.tag_key are missing' do\n      assert_raise Fluent::ConfigError.new(\"'tag' or 'tag_key' option is required on exec input\") do\n        create_driver %[\n          command ruby -e 'puts \"yay\"'\n          <parse>\n            keys y1\n          </parse>\n        ]\n      end\n    end\n\n    test 'configure for tsv' do\n      d = create_driver TSV_CONFIG\n      assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }\n      assert_equal [\"time\", \"tag\", \"k1\"], d.instance.parser.keys\n      assert_equal \"tag\", d.instance.extract_config.tag_key\n      assert_equal \"time\", d.instance.extract_config.time_key\n      assert_equal :string, d.instance.extract_config.time_type\n      assert_equal \"%Y-%m-%d %H:%M:%S\", d.instance.extract_config.time_format\n    end\n\n    test 'configure for json' do\n      d = create_driver JSON_CONFIG\n      assert{ d.instance.parser.is_a? Fluent::Plugin::JSONParser }\n      assert_equal \"tag\", d.instance.extract_config.tag_key\n      assert_equal \"time\", d.instance.extract_config.time_key\n      assert_equal :unixtime, d.instance.extract_config.time_type\n    end\n\n    test 'configure for msgpack' do\n      d = create_driver MSGPACK_CONFIG\n      assert{ d.instance.parser.is_a? Fluent::Plugin::MessagePackParser }\n      assert_equal \"tagger\", d.instance.extract_config.tag_key\n      assert_equal \"datetime\", d.instance.extract_config.time_key\n      assert_equal :unixtime, d.instance.extract_config.time_type\n    end\n\n    test 'configure for regexp' do\n      d = create_driver REGEXP_CONFIG\n      assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }\n      assert_equal \"regex_tag\", d.instance.tag\n      expression = /(?<time>[^\\]]*) (?<message>[^ ]*)/\n      assert_equal expression, d.instance.parser.expression\n      assert_nil d.instance.extract_config\n    end\n  end\n\n  TSV_CONFIG_COMPAT = %[\n      command ruby #{SCRIPT_PATH} \"#{TEST_TIME}\" 0\n      keys time,tag,k1\n      time_key time\n      tag_key tag\n      time_format %Y-%m-%d %H:%M:%S\n      run_interval 0.3\n  ]\n\n  JSON_CONFIG_COMPAT = %[\n      command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 1\n      format json\n      tag_key tag\n      time_key time\n      run_interval 0.3\n  ]\n\n  MSGPACK_CONFIG_COMPAT = %[\n      command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 2\n      format msgpack\n      tag_key tagger\n      time_key datetime\n      run_interval 0.3\n  ]\n\n  REGEXP_CONFIG_COMPAT = %[\n      command ruby #{SCRIPT_PATH} \"#{TEST_TIME}\" 3\n      format /(?<time>[^\\\\\\]]*) (?<message>[^ ]*)/\n      tag regex_tag\n      run_interval 0.3\n  ]\n\n  sub_test_case 'with traditional configuration' do\n    test 'configure' do\n      d = create_driver TSV_CONFIG_COMPAT\n      assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }\n      assert_equal [\"time\",\"tag\",\"k1\"], d.instance.parser.keys\n      assert_equal \"tag\", d.instance.extract_config.tag_key\n      assert_equal \"time\", d.instance.extract_config.time_key\n      assert_equal \"%Y-%m-%d %H:%M:%S\", d.instance.extract_config.time_format\n    end\n\n    test 'configure_with_json' do\n      d = create_driver JSON_CONFIG_COMPAT\n      assert{ d.instance.parser.is_a? Fluent::Plugin::JSONParser }\n    end\n\n    test 'configure_with_msgpack' do\n      d = create_driver MSGPACK_CONFIG_COMPAT\n      assert{ d.instance.parser.is_a? Fluent::Plugin::MessagePackParser }\n    end\n\n    test 'configure_with_regexp' do\n      d = create_driver REGEXP_CONFIG_COMPAT\n      assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }\n      assert_equal(/(?<time>[^\\]]*) (?<message>[^ ]*)/, d.instance.parser.expression)\n      assert_equal('regex_tag', d.instance.tag)\n    end\n  end\n\n  sub_test_case 'with default configuration' do\n    setup do\n      @current_event_time = event_time('2016-10-31 20:01:30.123 -0700')\n      Timecop.freeze(Time.at(@current_event_time))\n    end\n\n    teardown do\n      Timecop.return\n    end\n\n    test 'emits events with current timestamp if time key is not specified' do\n      d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS\n      d.run(expect_records: 2, timeout: 10)\n\n      assert{ d.events.length > 0 }\n      d.events.each do |event|\n        assert_equal [\"my.test.data\", @current_event_time, {\"k1\"=>\"2011-01-02 13:14:15\", \"k2\"=>\"tag1\", \"k3\"=>\"ok\"}], event\n      end\n    end\n  end\n\n  sub_test_case 'encoding' do\n    data(immediate: \"\")\n    data(run_interval: \"run_interval 1\")\n    test 'can handle non-ascii characters' do |additional_setting|\n      content = 'ひらがな漢字'\n\n      d = create_driver %[\n        command ruby -e \"puts '#{content}'\"\n        tag test\n        encoding utf-8\n        <parse>\n          @type none\n        </parse>\n        #{additional_setting}\n      ]\n\n      d.run(expect_records: 1, timeout: 10)\n\n      assert_equal 1, d.events.length\n      tag, time, record = d.events.first\n      assert_equal({\"message\" => content}, record)\n    end\n\n    test 'raise ConfigError for invalid encoding' do\n      assert_raise Fluent::ConfigError do\n        d = create_driver %[\n          command ruby -e \"puts foo\"\n          tag test\n          encoding invalid-encode\n          <parse>\n            @type none\n          </parse>\n        ]\n      end\n    end\n  end\n\n  data(\n    'default' => [TSV_CONFIG,     \"tag1\", event_time(\"2011-01-02 13:14:15\"), {\"k1\"=>\"ok\"}],\n    'json'    => [JSON_CONFIG,    \"tag1\", event_time(\"2011-01-02 13:14:15\"), {\"k1\"=>\"ok\"}],\n    'msgpack' => [MSGPACK_CONFIG, \"tag1\", event_time(\"2011-01-02 13:14:15\"), {\"k1\"=>\"ok\"}],\n    'regexp'  => [REGEXP_CONFIG,  \"regex_tag\", event_time(\"2011-01-02 13:14:15\"), {\"message\"=>\"hello\"}],\n    'default_c' => [TSV_CONFIG_COMPAT,     \"tag1\", event_time(\"2011-01-02 13:14:15\"), {\"k1\"=>\"ok\"}],\n    'json_c'    => [JSON_CONFIG_COMPAT,    \"tag1\", event_time(\"2011-01-02 13:14:15\"), {\"k1\"=>\"ok\"}],\n    'msgpack_c' => [MSGPACK_CONFIG_COMPAT, \"tag1\", event_time(\"2011-01-02 13:14:15\"), {\"k1\"=>\"ok\"}],\n    'regexp_c'  => [REGEXP_CONFIG_COMPAT,  \"regex_tag\", event_time(\"2011-01-02 13:14:15\"), {\"message\"=>\"hello\"}],\n  )\n  test 'emit with formats' do |data|\n    config, tag, time, record = data\n    d = create_driver(config)\n\n    d.run(expect_emits: 2, timeout: 10)\n\n    assert{ d.events.length > 0 }\n    d.events.each {|event|\n      assert_equal_event_time(time, event[1])\n      assert_equal [tag, time, record], event\n    }\n  end\n\n  test 'emit error message with read_with_stderr' do\n    d = create_driver %[\n      tag test\n      command ruby #{File.join(File.dirname(SCRIPT_PATH), 'foo_bar_baz_no_existence.rb')}\n      connect_mode read_with_stderr\n      <parse>\n        @type none\n      </parse>\n    ]\n    d.run(expect_records: 1, timeout: 10)\n\n    assert{ d.events.length > 0 }\n    d.events.each do |event|\n      assert_equal 'test', event[0]\n      assert_match(/LoadError/, event[2]['message'])\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_forward.rb",
    "content": "require_relative '../helper'\n\nrequire 'fluent/test/driver/input'\nrequire 'fluent/test/startup_shutdown'\nrequire 'base64'\n\nrequire 'fluent/env'\nrequire 'fluent/event'\nrequire 'fluent/plugin/in_forward'\nrequire 'fluent/plugin/compressable'\n\nrequire 'timecop'\n\nclass ForwardInputTest < Test::Unit::TestCase\n  include Fluent::Plugin::Compressable\n\n  def setup\n    Fluent::Test.setup\n    @responses = []  # for testing responses after sending data\n    @d = nil\n    # forward plugin uses TCP and UDP sockets on the same port number\n    @port = unused_port(protocol: :all)\n  end\n\n  def teardown\n    @d.instance_shutdown if @d\n    @port = nil\n  end\n\n  SHARED_KEY = 'foobar1'\n  USER_NAME = 'tagomoris'\n  USER_PASSWORD = 'fluentd'\n\n  def base_config\n    %[\n      port #{@port}\n      bind 127.0.0.1\n    ]\n  end\n  LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect(\"127.0.0.1\", 2048); sock.peeraddr[2] }\n  LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call\n  DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, \"127.0.0.1\", 0)\n\n  def config_auth\n    base_config + %[\n      <security>\n        self_hostname localhost\n        shared_key foobar1\n        user_auth true\n        <user>\n          username #{USER_NAME}\n          password #{USER_PASSWORD}\n        </user>\n        <client>\n          network 127.0.0.0/8\n          shared_key #{SHARED_KEY}\n          users [\"#{USER_NAME}\"]\n        </client>\n      </security>\n    ]\n  end\n\n  def create_driver(conf=base_config)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput).configure(conf)\n  end\n\n  sub_test_case '#configure' do\n    test 'simple' do\n      @d = d = create_driver\n      assert_equal @port, d.instance.port\n      assert_equal '127.0.0.1', d.instance.bind\n      assert_equal 0.5, d.instance.blocking_timeout\n      assert !d.instance.backlog\n    end\n\n    test 'auth' do\n      @d = d = create_driver(config_auth)\n      assert_equal @port, d.instance.port\n      assert_equal '127.0.0.1', d.instance.bind\n      assert !d.instance.backlog\n\n      assert d.instance.security\n      assert_equal 1, d.instance.security.users.size\n      assert_equal 1, d.instance.security.clients.size\n    end\n\n    data(tag: \"tag\",\n         add_tag_prefix: \"add_tag_prefix\")\n    test 'tag parameters' do |data|\n      assert_raise(Fluent::ConfigError.new(\"'#{data}' parameter must not be empty\")) {\n        create_driver(base_config + \"#{data} ''\")\n      }\n    end\n\n    test 'send_keepalive_packet is disabled by default' do\n      @d = d = create_driver(config_auth)\n      assert_false d.instance.send_keepalive_packet\n    end\n\n    test 'send_keepalive_packet can be enabled' do\n      @d = d = create_driver(config_auth + %[\n        send_keepalive_packet true\n      ])\n      assert_true d.instance.send_keepalive_packet\n    end\n\n    test 'both send_keepalive_packet and deny_keepalive cannot be enabled' do\n      assert_raise(Fluent::ConfigError.new(\"both 'send_keepalive_packet' and 'deny_keepalive' cannot be set to true\")) do\n        create_driver(config_auth + %[\n          send_keepalive_packet true\n          deny_keepalive true\n        ])\n      end\n    end\n  end\n\n  sub_test_case 'message' do\n    test 'time' do\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      begin\n        Timecop.freeze(Time.at(time))\n        @d = d = create_driver\n\n        records = [\n          [\"tag1\", time, {\"a\"=>1}],\n          [\"tag2\", time, {\"a\"=>2}],\n        ]\n\n        d.run(expect_records: records.length, timeout: 5) do\n          records.each {|tag, _time, record|\n            send_data packer.write([tag, 0, record]).to_s\n          }\n        end\n        assert_equal(records, d.events.sort_by {|a| a[0] })\n      ensure\n        Timecop.return\n      end\n    end\n\n    test 'plain' do\n      @d = d = create_driver\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      records = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag2\", time, {\"a\"=>2}],\n      ]\n\n      d.run(expect_records: records.length, timeout: 5) do\n        records.each {|tag, _time, record|\n          send_data packer.write([tag, _time, record]).to_s\n        }\n      end\n      assert_equal(records, d.events)\n    end\n\n    test 'time_as_integer' do\n      @d = d = create_driver\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\").to_i\n\n      records = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag2\", time_i, {\"a\"=>2}],\n      ]\n\n      d.run(expect_records: records.length, timeout: 5) do\n        records.each {|tag, _time, record|\n          send_data packer.write([tag, _time, record]).to_s\n        }\n      end\n\n      assert_equal(records, d.events)\n    end\n\n    test 'skip_invalid_event' do\n      @d = d = create_driver(base_config + \"skip_invalid_event true\")\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      records = [\n        [\"tag1\", time, {\"a\" => 1}],\n        [\"tag2\", time, {\"a\" => 2}],\n      ]\n\n      d.run(shutdown: false, expect_records: 2, timeout: 10) do\n        entries = []\n        # These entries are skipped\n        entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']\n        entries += records.map { |tag, _time, record| [tag, _time, record] }\n\n        entries.each {|tag, _time, record|\n          # Without ack, logs are sometimes not saved to logs during test.\n          send_data packer.write([tag, _time, record]).to_s #, try_to_receive_response: true\n        }\n      end\n\n      logs = d.instance.log.logs\n      assert_equal 2, logs.count { |line| line =~ /got invalid event and drop it/ }\n      assert_equal records[0], d.events[0]\n      assert_equal records[1], d.events[1]\n\n      d.instance_shutdown\n    end\n\n    test 'json_using_integer_time' do\n      @d = d = create_driver\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\").to_i\n\n      records = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag2\", time_i, {\"a\"=>2}],\n      ]\n\n      d.run(expect_records: records.length, timeout: 20) do\n        records.each {|tag, _time, record|\n          send_data [tag, _time, record].to_json\n        }\n      end\n\n      assert_equal(records, d.events.sort_by {|a| a[0] })\n    end\n\n    test 'json_with_newline' do\n      @d = d = create_driver\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\").to_i\n\n      records = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag2\", time_i, {\"a\"=>2}],\n      ]\n\n      d.run(expect_records: records.length, timeout: 20, shutdown: true) do\n        records.each {|tag, _time, record|\n          send_data [tag, _time, record].to_json + \"\\n\"\n        }\n      end\n\n      assert_equal(records, d.events.sort_by {|a| a[0] })\n    end\n  end\n\n  sub_test_case 'forward' do\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'plain' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      records = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag1\", time, {\"a\"=>2}]\n      ]\n\n      d.run(expect_records: records.length, timeout: 20) do\n        entries = []\n        records.each {|tag, _time, record|\n          entries << [_time, record]\n        }\n        send_data packer.write([\"tag1\", entries]).to_s, **options\n      end\n      assert_equal(records, d.events)\n    end\n\n    data(tag: {\n           param: \"tag new_tag\",\n           result: \"new_tag\"\n         },\n         add_tag_prefix: {\n           param: \"add_tag_prefix new_prefix\",\n           result: \"new_prefix.tag1\"\n         })\n    test 'tag parameters' do |data|\n      @d = create_driver(base_config + data[:param])\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      options = {auth: false}\n\n      records = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag1\", time, {\"a\"=>2}],\n      ]\n\n      @d.run(expect_records: records.length, timeout: 20) do\n        entries = []\n        records.each {|tag, _time, record|\n          entries << [_time, record]\n        }\n        send_data packer.write([\"tag1\", entries]).to_s, **options\n      end\n      assert_equal(data[:result], @d.events[0][0])\n    end\n\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'time_as_integer' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      records = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag1\", time_i, {\"a\"=>2}]\n      ]\n\n      d.run(expect_records: records.length, timeout: 20) do\n        entries = []\n        records.each {|tag, _time, record|\n          entries << [_time, record]\n        }\n        send_data packer.write([\"tag1\", entries]).to_s, **options\n      end\n\n      assert_equal(records, d.events)\n    end\n\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'skip_invalid_event' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config + \"skip_invalid_event true\")\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      records = [\n        [\"tag1\", time, {\"a\" => 1}],\n        [\"tag1\", time, {\"a\" => 2}],\n      ]\n\n      d.run(shutdown: false, expect_records: records.length, timeout: 20) do\n        entries = records.map { |tag, _time, record| [_time, record] }\n        # These entries are skipped\n        entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']\n\n        send_data packer.write([\"tag1\", entries]).to_s, **options\n      end\n\n      logs = d.instance.log.out.logs\n      assert{ logs.count{|line| line =~ /skip invalid event/ } == 2 }\n\n      d.instance_shutdown\n    end\n  end\n\n  sub_test_case 'packed forward' do\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'plain' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      records = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag1\", time, {\"a\"=>2}],\n      ]\n\n      d.run(expect_records: records.length, timeout: 20) do\n        entries = ''\n        records.each {|_tag, _time, record|\n          packer(entries).write([_time, record]).flush\n        }\n        send_data packer.write([\"tag1\", entries]).to_s, **options\n      end\n      assert_equal(records, d.events)\n    end\n\n    data(tag: {\n           param: \"tag new_tag\",\n           result: \"new_tag\"\n         },\n         add_tag_prefix: {\n           param: \"add_tag_prefix new_prefix\",\n           result: \"new_prefix.tag1\"\n         })\n    test 'tag parameters' do |data|\n      @d = create_driver(base_config + data[:param])\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      options = {auth: false}\n\n      records = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag1\", time, {\"a\"=>2}],\n      ]\n\n      @d.run(expect_records: records.length, timeout: 20) do\n        entries = ''\n        records.each {|_tag, _time, record|\n          packer(entries).write([_time, record]).flush\n        }\n        send_data packer.write([\"tag1\", entries]).to_s, **options\n      end\n      assert_equal(data[:result], @d.events[0][0])\n    end\n\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'time_as_integer' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\").to_i\n\n      records = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag1\", time_i, {\"a\"=>2}],\n      ]\n\n      d.run(expect_records: records.length, timeout: 20) do\n        entries = ''\n        records.each {|tag, _time, record|\n          packer(entries).write([_time, record]).flush\n        }\n        send_data packer.write([\"tag1\", entries]).to_s, **options\n      end\n      assert_equal(records, d.events)\n    end\n\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'skip_invalid_event' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config + \"skip_invalid_event true\")\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      records = [\n        [\"tag1\", time, {\"a\" => 1}],\n        [\"tag1\", time, {\"a\" => 2}],\n      ]\n\n      d.run(shutdown: false, expect_records: records.length, timeout: 20) do\n        entries = records.map { |tag, _time, record| [_time, record] }\n        # These entries are skipped\n        entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']\n\n        packed_entries = ''\n        entries.each { |_time, record|\n          packer(packed_entries).write([_time, record]).flush\n        }\n        send_data packer.write([\"tag1\", packed_entries]).to_s, **options\n      end\n\n      logs = d.instance.log.logs\n      assert_equal 2, logs.count { |line| line =~ /skip invalid event/ }\n\n      d.instance_shutdown\n    end\n  end\n\n  sub_test_case 'compressed packed forward' do\n    test 'set_compress_to_option_gzip' do\n      @d = d = create_driver\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\").to_i\n      events = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag1\", time_i, {\"a\"=>2}]\n      ]\n\n      # create compressed entries\n      entries = ''\n      events.each do |_tag, _time, record|\n        v = [_time, record].to_msgpack\n        entries << compress(v)\n      end\n      chunk = [\"tag1\", entries, { 'compressed' => 'gzip' }].to_msgpack\n\n      d.run do\n        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|\n          option = d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)\n          assert_equal 'gzip', option['compressed']\n        end\n      end\n\n      assert_equal events, d.events\n    end\n\n    test 'set_compress_to_option_zstd' do\n      @d = d = create_driver\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\").to_i\n      events = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag1\", time_i, {\"a\"=>2}]\n      ]\n\n      # create compressed entries\n      entries = ''\n      events.each do |_tag, _time, record|\n        v = [_time, record].to_msgpack\n        entries << compress(v, type: :zstd)\n      end\n      chunk = [\"tag1\", entries, { 'compressed' => 'zstd' }].to_msgpack\n\n      d.run do\n        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|\n          option = d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)\n          assert_equal 'zstd', option['compressed']\n        end\n      end\n\n      assert_equal events, d.events\n    end\n\n    test 'create_CompressedMessagePackEventStream_with_gzip_compress_option' do\n      @d = d = create_driver\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\").to_i\n      events = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag1\", time_i, {\"a\"=>2}]\n      ]\n\n      # create compressed entries\n      entries = ''\n      events.each do |_tag, _time, record|\n        v = [_time, record].to_msgpack\n        entries << compress(v)\n      end\n      chunk = [\"tag1\", entries, { 'compressed' => 'gzip' }].to_msgpack\n\n      d.run do\n        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|\n          option = d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)\n          assert_equal 'gzip', option['compressed']\n        end\n      end\n    end\n\n    test 'create_CompressedMessagePackEventStream_with_zstd_compress_option' do\n      @d = d = create_driver\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\").to_i\n      events = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag1\", time_i, {\"a\"=>2}]\n      ]\n\n      # create compressed entries\n      entries = ''\n      events.each do |_tag, _time, record|\n        v = [_time, record].to_msgpack\n        entries << compress(v, type: :zstd)\n      end\n      chunk = [\"tag1\", entries, { 'compressed' => 'zstd' }].to_msgpack\n\n      d.run do\n        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|\n          option = d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)\n          assert_equal 'zstd', option['compressed']\n        end\n      end\n    end\n  end\n\n  sub_test_case 'warning' do\n    test 'send_large_chunk_warning' do\n      @d = d = create_driver(base_config + %[\n        chunk_size_warn_limit 16M\n        chunk_size_limit 32M\n      ])\n\n      time = event_time(\"2014-04-25 13:14:15 UTC\")\n\n      # generate over 16M chunk\n      str = \"X\" * 1024 * 1024\n      chunk = [ \"test.tag\", (0...16).map{|i| [time + i, {\"data\" => str}] } ].to_msgpack\n      assert chunk.size > (16 * 1024 * 1024)\n      assert chunk.size < (32 * 1024 * 1024)\n\n      d.run(shutdown: false) do\n        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|\n          d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)\n        end\n      end\n\n      # check emitted data\n      emits = d.events\n      assert_equal 16, emits.size\n      assert emits.map(&:first).all?(\"test.tag\")\n      assert_equal (0...16).to_a, emits.map{|_tag, t, _record| t - time }\n\n      # check log\n      logs = d.instance.log.logs\n      assert_equal 1, logs.count{|line|\n        line =~ / \\[warn\\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&\n        line =~ / tag=\"test.tag\" host=\"#{LOCALHOST_HOSTNAME}\" limit=16777216 size=16777501/\n      }, \"large chunk warning is not logged\"\n\n      d.instance_shutdown\n    end\n\n    test 'send_large_chunk_only_warning' do\n      @d = d = create_driver(base_config + %[\n        chunk_size_warn_limit 16M\n      ])\n      time = event_time(\"2014-04-25 13:14:15 UTC\")\n\n      # generate over 16M chunk\n      str = \"X\" * 1024 * 1024\n      chunk = [ \"test.tag\", (0...16).map{|i| [time + i, {\"data\" => str}] } ].to_msgpack\n\n      d.run(shutdown: false) do\n        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|\n          d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)\n        end\n      end\n\n      # check log\n      logs = d.instance.log.logs\n      assert_equal 1, logs.count{ |line|\n        line =~ / \\[warn\\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&\n        line =~ / tag=\"test.tag\" host=\"#{LOCALHOST_HOSTNAME}\" limit=16777216 size=16777501/\n      }, \"large chunk warning is not logged\"\n\n      d.instance_shutdown\n    end\n\n    test 'send_large_chunk_limit' do\n      @d = d = create_driver(base_config + %[\n        chunk_size_warn_limit 16M\n        chunk_size_limit 32M\n      ])\n\n      time = event_time(\"2014-04-25 13:14:15 UTC\")\n\n      # generate over 32M chunk\n      str = \"X\" * 1024 * 1024\n      chunk = [ \"test.tag\", (0...32).map{|i| [time + i, {\"data\" => str}] } ].to_msgpack\n      assert chunk.size > (32 * 1024 * 1024)\n\n      # d.run => send_data\n      d.run(shutdown: false) do\n        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|\n          d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)\n        end\n      end\n\n      # check emitted data\n      emits = d.events\n      assert_equal 0, emits.size\n\n      # check log\n      logs = d.instance.log.logs\n      assert_equal 1, logs.count{|line|\n        line =~ / \\[warn\\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&\n        line =~ / tag=\"test.tag\" host=\"#{LOCALHOST_HOSTNAME}\" limit=33554432 size=33554989/\n      }, \"large chunk warning is not logged\"\n\n      d.instance_shutdown\n    end\n\n    data('string chunk' => 'broken string',\n         'integer chunk' => 10)\n    test 'send_broken_chunk' do |data|\n      @d = d = create_driver\n\n      # d.run => send_data\n      d.run(shutdown: false) do\n        d.instance.send(:on_message, data, 1000000000, DUMMY_SOCK)\n      end\n\n      # check emitted data\n      assert_equal 0, d.events.size\n\n      # check log\n      logs = d.instance.log.logs\n      assert_equal 1, logs.count{|line|\n        line =~ / \\[warn\\]: incoming chunk is broken: host=\"#{LOCALHOST_HOSTNAME}\" msg=#{data.inspect}/\n      }, \"should not accept broken chunk\"\n\n      d.instance_shutdown\n    end\n  end\n\n  sub_test_case 'respond to required ack' do\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'message' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      events = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag2\", time, {\"a\"=>2}]\n      ]\n\n      expected_acks = []\n\n      d.run(expect_records: events.size) do\n        events.each {|tag, _time, record|\n          op = { 'chunk' => Base64.encode64(record.object_id.to_s) }\n          expected_acks << op['chunk']\n          send_data([tag, _time, record, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)\n        }\n      end\n\n      assert_equal events, d.events\n      assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }\n    end\n\n    # FIX: response is not pushed into @responses because IO.select has been blocked until InputForward shutdowns\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'forward' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      events = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag1\", time, {\"a\"=>2}]\n      ]\n\n      expected_acks = []\n\n      d.run(expect_records: events.size) do\n        entries = []\n        events.each {|_tag, _time, record|\n          entries << [_time, record]\n        }\n        op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }\n        expected_acks << op['chunk']\n        send_data([\"tag1\", entries, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)\n      end\n\n      assert_equal events, d.events\n      assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }\n    end\n\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'packed_forward' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      events = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag1\", time, {\"a\"=>2}]\n      ]\n\n      expected_acks = []\n\n      d.run(expect_records: events.size) do\n        entries = ''\n        events.each {|_tag, _time,record|\n          [time, record].to_msgpack(entries)\n        }\n        op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }\n        expected_acks << op['chunk']\n        send_data([\"tag1\", entries, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)\n      end\n\n      assert_equal events, d.events\n      assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }\n    end\n\n    data(\n      tcp: {\n        options: {\n          auth: false\n        }\n      },\n      ### Auth is not supported with json\n      # auth: {\n      #   options: {\n      #     auth: true\n      #   }\n      # },\n    )\n    test 'message_json' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      events = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag2\", time_i, {\"a\"=>2}]\n      ]\n\n      expected_acks = []\n\n      d.run(expect_records: events.size, timeout: 20) do\n        events.each {|tag, _time, record|\n          op = { 'chunk' => Base64.encode64(record.object_id.to_s) }\n          expected_acks << op['chunk']\n          send_data([tag, _time, record, op].to_json, try_to_receive_response: true, response_timeout: 1, **options)\n        }\n      end\n\n      assert_equal events, d.events\n      assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }\n    end\n  end\n\n  sub_test_case 'not respond without required ack' do\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'message' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      events = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag2\", time, {\"a\"=>2}]\n      ]\n\n      d.run(expect_records: events.size, timeout: 20) do\n        events.each {|tag, _time, record|\n          send_data([tag, _time, record].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)\n        }\n      end\n\n      assert_equal events, d.events\n      assert_equal [nil, nil], @responses\n    end\n\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'forward' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      events = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag1\", time, {\"a\"=>2}]\n      ]\n\n      d.run(expect_records: events.size, timeout: 20) do\n        entries = []\n        events.each {|tag, _time, record|\n          entries << [_time, record]\n        }\n        send_data([\"tag1\", entries].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)\n      end\n\n      assert_equal events, d.events\n      assert_equal [nil], @responses\n    end\n\n    data(tcp: {\n           options: {\n             auth: false\n           }\n         },\n         auth: {\n           options: {\n             auth: true\n           }\n         })\n    test 'packed_forward' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      events = [\n        [\"tag1\", time, {\"a\"=>1}],\n        [\"tag1\", time, {\"a\"=>2}]\n      ]\n\n      d.run(expect_records: events.size, timeout: 20) do\n        entries = ''\n        events.each {|tag, _time, record|\n          [_time, record].to_msgpack(entries)\n        }\n        send_data([\"tag1\", entries].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)\n      end\n\n      assert_equal events, d.events\n      assert_equal [nil], @responses\n    end\n\n    data(\n      tcp: {\n        options: {\n          auth: false\n        }\n      },\n      ### Auth is not supported with json\n      # auth: {\n      #   config: config_auth,\n      #   options: {\n      #     auth: true\n      #   }\n      # },\n    )\n    test 'message_json' do |data|\n      options = data[:options]\n      config = options[:auth] ? config_auth : base_config\n      @d = d = create_driver(config)\n\n      time_i = event_time(\"2011-01-02 13:14:15 UTC\").to_i\n\n      events = [\n        [\"tag1\", time_i, {\"a\"=>1}],\n        [\"tag2\", time_i, {\"a\"=>2}]\n      ]\n\n      d.run(expect_records: events.size, timeout: 20) do\n        events.each {|tag, _time, record|\n          send_data([tag, _time, record].to_json, try_to_receive_response: true, response_timeout: 1, **options)\n        }\n      end\n\n      assert_equal events, d.events\n      assert_equal [nil, nil], @responses\n    end\n  end\n\n  def packer(*args)\n    Fluent::MessagePackFactory.msgpack_packer(*args)\n  end\n\n  def unpacker\n    Fluent::MessagePackFactory.msgpack_unpacker\n  end\n\n  # res\n  # '' : socket is disconnected without any data\n  # nil: socket read timeout\n  def read_data(io, timeout, &block)\n    res = ''\n    select_timeout = 0.5\n    clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC\n    timeout_at = Process.clock_gettime(clock_id) + timeout\n    begin\n      buf = ''\n      io_activated = false\n      while Process.clock_gettime(clock_id) < timeout_at\n        if IO.select([io], nil, nil, select_timeout)\n          io_activated = true\n          buf = io.readpartial(2048)\n          res ||= ''\n          res << buf\n\n          break if block.call(res)\n        end\n      end\n      res = nil unless io_activated # timeout without no data arrival\n    rescue Errno::EAGAIN\n      sleep 0.01\n      retry if res == ''\n      # if res is not empty, all data in socket buffer are read, so do not retry\n    rescue IOError, EOFError, Errno::ECONNRESET\n      # socket disconnected\n    end\n    res\n  end\n\n  def simulate_auth_sequence(io, shared_key=SHARED_KEY, username=USER_NAME, password=USER_PASSWORD)\n    auth_response_timeout = 30\n    shared_key_salt = 'salt'\n\n    # reading helo\n    helo_data = read_data(io, auth_response_timeout){|data| MessagePack.unpack(data) rescue nil }\n    raise \"Authentication packet timeout\" unless helo_data\n    raise \"Authentication connection closed\" if helo_data == ''\n    # ['HELO', options(hash)]\n    helo = MessagePack.unpack(helo_data)\n    raise \"Invalid HELO header\" unless helo[0] == 'HELO'\n    raise \"Invalid HELO option object\" unless helo[1].is_a?(Hash)\n    @options = helo[1]\n\n    # sending ping\n    ping = [\n      'PING',\n      'selfhostname',\n      shared_key_salt,\n      Digest::SHA512.new\n        .update(shared_key_salt)\n        .update('selfhostname')\n        .update(@options['nonce'])\n        .update(shared_key).hexdigest,\n    ]\n    if @options['auth'] # auth enabled -> value is auth salt\n      pass_digest = Digest::SHA512.new.update(@options['auth']).update(username).update(password).hexdigest\n      ping.push(username, pass_digest)\n    else\n      ping.push('', '')\n    end\n    io.write ping.to_msgpack\n    io.flush\n\n    # reading pong\n    pong_data = read_data(io, auth_response_timeout){|data| MessagePack.unpack(data) rescue nil }\n    raise \"PONG packet timeout\" unless pong_data\n    raise \"PONG connection closed\" if pong_data == ''\n    # ['PING', bool(auth_result), string(reason_if_failed), self_hostname, shared_key_digest]\n    pong = MessagePack.unpack(pong_data)\n    raise \"Invalid PONG header\" unless pong[0] == 'PONG'\n    raise \"Authentication Failure: #{pong[2]}\" unless pong[1]\n    clientside_calculated = Digest::SHA512.new\n      .update(shared_key_salt)\n      .update(pong[3])\n      .update(@options['nonce'])\n      .update(shared_key).hexdigest\n    raise \"Shared key digest mismatch\" unless clientside_calculated == pong[4]\n\n    # authentication success\n    true\n  end\n\n  def connect\n    TCPSocket.new('127.0.0.1', @port)\n  end\n\n  # Data ordering is not assured:\n  #  Records in different sockets are processed on different thread, so its scheduling make effect\n  #  on order of emitted records.\n  #  So, we MUST sort emitted records in different `send_data` before assertion.\n  def send_data(data, try_to_receive_response: false, response_timeout: 5, auth: false)\n    io = connect\n\n    if auth\n      simulate_auth_sequence(io)\n    end\n\n    io.write data\n    io.flush\n    if try_to_receive_response\n      @responses << read_data(io, response_timeout){|d| MessagePack.unpack(d) rescue nil }\n    end\n  ensure\n    io.close rescue nil # SSL socket requires any writes to close sockets\n  end\n\n  sub_test_case 'source_hostname_key and source_address_key features' do\n    data(\n      both: [:hostname, :address],\n      hostname: [:hostname],\n      address: [:address],\n    )\n    test 'message protocol' do |keys|\n      execute_test_with_source_hostname_key(*keys) { |events|\n        events.each { |tag, time, record|\n          send_data [tag, time, record].to_msgpack\n        }\n      }\n    end\n\n    data(\n      both: [:hostname, :address],\n      hostname: [:hostname],\n      address: [:address],\n    )\n    test 'forward protocol' do |keys|\n      execute_test_with_source_hostname_key(*keys) { |events|\n        entries = []\n        events.each {|tag,time,record|\n          entries << [time, record]\n        }\n        send_data ['tag1', entries].to_msgpack\n      }\n    end\n\n    data(\n      both: [:hostname, :address],\n      hostname: [:hostname],\n      address: [:address],\n    )\n    test 'packed forward protocol' do |keys|\n      execute_test_with_source_hostname_key(*keys) { |events|\n        entries = ''\n        events.each { |tag, time, record|\n          Fluent::MessagePackFactory.msgpack_packer(entries).write([time, record]).flush\n        }\n        send_data Fluent::MessagePackFactory.msgpack_packer.write([\"tag1\", entries]).to_s\n      }\n    end\n  end\n\n  def execute_test_with_source_hostname_key(*keys, &block)\n    conf = base_config.dup\n    if keys.include?(:hostname)\n      conf << <<EOL\nsource_hostname_key source_hostname\nEOL\n    end\n    if keys.include?(:address)\n      conf << <<EOL\nsource_address_key source_address\nEOL\n    end\n    @d = d = create_driver(conf)\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag1\", time, {\"a\"=>2}]\n    ]\n\n    d.run(expect_records: events.size) do\n      block.call(events)\n    end\n\n    d.events.each { |tag, _time, record|\n      if keys.include?(:hostname)\n        assert_true record.has_key?('source_hostname')\n        assert_equal DUMMY_SOCK.remote_host, record['source_hostname']\n      end\n      if keys.include?(:address)\n        assert_true record.has_key?('source_address')\n        assert_equal DUMMY_SOCK.remote_addr, record['source_address']\n      end\n    }\n  end\n\n  # TODO heartbeat\nend\n"
  },
  {
    "path": "test/plugin/test_in_gc_stat.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_gc_stat'\n\nclass GCStatInputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  CONFIG = %[\n    emit_interval 1\n    tag t1\n  ]\n\n  def create_driver(conf=CONFIG)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::GCStatInput).configure(conf)\n  end\n\n  def test_configure\n    d = create_driver\n    assert_equal(1, d.instance.emit_interval)\n    assert_equal(\"t1\", d.instance.tag)\n  end\n\n  def setup_gc_stat\n    stat = GC.stat\n    stub(GC).stat { stat }\n    stat\n  end\n\n  def test_emit\n    stat = setup_gc_stat\n\n    d = create_driver\n    d.run(expect_emits: 2)\n\n    events = d.events\n    assert(events.length > 0)\n    events.each_index {|i|\n      assert_equal(stat, events[i][2])\n      assert(events[i][1].is_a?(Fluent::EventTime))\n    }\n  end\n\n  def test_emit_with_use_symbol_keys_false\n    stat = setup_gc_stat\n    result = {}\n    stat.each_pair { |k, v|\n      result[k.to_s] = v\n    }\n\n    d = create_driver(CONFIG + \"use_symbol_keys false\")\n    d.run(expect_emits: 2)\n\n    events = d.events\n    assert(events.length > 0)\n    events.each_index {|i|\n      assert_equal(result, events[i][2])\n      assert(events[i][1].is_a?(Fluent::EventTime))\n    }\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_http.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_http'\nrequire 'net/http'\nrequire 'timecop'\n\nclass HttpInputTest < Test::Unit::TestCase\n  class << self\n    def startup\n      @server = ServerEngine::SocketManager::Server.open\n      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s\n    end\n\n    def shutdown\n      @server.close\n    end\n  end\n\n  def setup\n    Fluent::Test.setup\n    @port = unused_port(protocol: :tcp)\n  end\n\n  def teardown\n    Timecop.return\n    @port = nil\n  end\n\n  def config\n    %[\n      port #{@port}\n      bind \"127.0.0.1\"\n      body_size_limit 10m\n      keepalive_timeout 5\n      respond_with_empty_img true\n     use_204_response false\n    ]\n  end\n\n  def create_driver(conf=config)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::HttpInput).configure(conf)\n  end\n\n  def test_configure\n    d = create_driver\n    assert_equal @port, d.instance.port\n    assert_equal '127.0.0.1', d.instance.bind\n    assert_equal 10*1024*1024, d.instance.body_size_limit\n    assert_equal 5, d.instance.keepalive_timeout\n    assert_equal false, d.instance.add_http_headers\n    assert_equal false, d.instance.add_query_params\n  end\n\n  def test_time\n    d = create_driver\n    time = event_time(\"2011-01-02 13:14:15.123 UTC\")\n    Timecop.freeze(Time.at(time))\n\n    events = [\n      [\"tag1\", time, {\"a\" => 1}],\n      [\"tag2\", time, {\"a\" => 2}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, _time, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json})\n        res_codes << res.code\n      end\n    end\n\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_time_as_float\n    d = create_driver\n    time = event_time(\"2011-01-02 13:14:15.123 UTC\")\n    float_time = time.to_f\n\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 1) do\n      events.each do |tag, t, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json, \"time\"=>float_time.to_s})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n  end\n\n  def test_json\n    d = create_driver\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n\n    events = [\n      [\"tag1\", time_i, {\"a\"=>1}],\n      [\"tag2\", time_i, {\"a\"=>2}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, t, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json, \"time\"=>t.to_s})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  data('json' => ['json', :to_json],\n       'msgpack' => ['msgpack', :to_msgpack])\n  def test_default_with_time_format(data)\n    param, method_name = data\n    d = create_driver(config + %[\n      <parse>\n        keep_time_key\n        time_format %iso8601\n      </parse>\n    ])\n\n    time = event_time(\"2020-06-10T01:14:27+00:00\")\n    events = [\n      [\"tag1\", time, {\"a\" => 1, \"time\" => '2020-06-10T01:14:27+00:00'}],\n      [\"tag2\", time, {\"a\" => 2, \"time\" => '2020-06-10T01:14:27+00:00'}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, t, record|\n        res = post(\"/#{tag}\", {param => record.__send__(method_name)})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_multi_json\n    d = create_driver\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n\n    records = [{\"a\"=>1},{\"a\"=>2}]\n    events = [\n      [\"tag1\", time_i, records[0]],\n      [\"tag1\", time_i, records[1]],\n    ]\n    tag = \"tag1\"\n    res_codes = []\n    d.run(expect_records: 2, timeout: 5) do\n      res = post(\"/#{tag}\", {\"json\"=>records.to_json, \"time\"=>time_i.to_s})\n      res_codes << res.code\n    end\n    assert_equal [\"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_multi_json_with_time_field\n    d = create_driver\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    time_f = time.to_f\n\n    records = [{\"a\" => 1, 'time' => time_i},{\"a\" => 2, 'time' => time_f}]\n    events = [\n      [\"tag1\", time, {'a' => 1}],\n      [\"tag1\", time, {'a' => 2}],\n    ]\n    tag = \"tag1\"\n    res_codes = []\n    d.run(expect_records: 2, timeout: 5) do\n      res = post(\"/#{tag}\", {\"json\" => records.to_json})\n      res_codes << res.code\n    end\n    assert_equal [\"200\"], res_codes\n    assert_equal events, d.events\n    assert_instance_of Fluent::EventTime, d.events[0][1]\n    assert_instance_of Fluent::EventTime, d.events[1][1]\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  data('json' => ['json', :to_json],\n       'msgpack' => ['msgpack', :to_msgpack])\n  def test_default_multi_with_time_format(data)\n    param, method_name = data\n    d = create_driver(config + %[\n      <parse>\n        keep_time_key\n        time_format %iso8601\n      </parse>\n    ])\n    time = event_time(\"2020-06-10T01:14:27+00:00\")\n    events = [\n      [\"tag1\", time, {'a' => 1, 'time' => \"2020-06-10T01:14:27+00:00\"}],\n      [\"tag1\", time, {'a' => 2, 'time' => \"2020-06-10T01:14:27+00:00\"}],\n    ]\n    tag = \"tag1\"\n    res_codes = []\n    d.run(expect_records: 2, timeout: 5) do\n      res = post(\"/#{tag}\", {param => events.map { |e| e[2] }.__send__(method_name)})\n      res_codes << res.code\n    end\n    assert_equal [\"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_multi_json_with_nonexistent_time_key\n    d = create_driver(config + %[\n      <parse>\n        time_key missing\n      </parse>\n    ])\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    time_f = time.to_f\n\n    records = [{\"a\" => 1, 'time' => time_i},{\"a\" => 2, 'time' => time_f}]\n    tag = \"tag1\"\n    res_codes = []\n    d.run(expect_records: 2, timeout: 5) do\n      res = post(\"/#{tag}\", {\"json\" => records.to_json})\n      res_codes << res.code\n    end\n    assert_equal [\"200\"], res_codes\n    assert_equal 2, d.events.size\n    assert_not_equal time_i, d.events[0][1].sec # current time is used because \"missing\" field doesn't exist\n    assert_not_equal time_i, d.events[1][1].sec\n  end\n\n  def test_json_with_add_remote_addr\n    d = create_driver(config + \"add_remote_addr true\")\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n\n    events = [\n      [\"tag1\", time, {\"REMOTE_ADDR\"=>\"127.0.0.1\", \"a\"=>1}],\n      [\"tag2\", time, {\"REMOTE_ADDR\"=>\"127.0.0.1\", \"a\"=>2}],\n    ]\n    res_codes = []\n    d.run(expect_records: 2) do\n      events.each do |tag, _t, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json, \"time\"=>time_i.to_s})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_exact_match_for_expect\n    d = create_driver(config)\n    records = [{ \"a\" => 1}, { \"a\" => 2 }]\n    tag = \"tag1\"\n    res_codes = []\n\n    d.run(expect_records: 0, timeout: 5) do\n      res = post(\"/#{tag}\", { \"json\" => records.to_json }, { 'Expect' => 'something' })\n      res_codes << res.code\n    end\n    assert_equal [\"417\"], res_codes\n  end\n\n  def test_exact_match_for_expect_with_other_header\n    d = create_driver(config)\n\n    records = [{ \"a\" => 1}, { \"a\" => 2 }]\n    tag = \"tag1\"\n    res_codes = []\n\n    d.run(expect_records: 2, timeout: 5) do\n      res = post(\"/#{tag}\", { \"json\" => records.to_json, 'x-envoy-expected-rq-timeout-ms' => 4 })\n      res_codes << res.code\n    end\n    assert_equal [\"200\"], res_codes\n\n    assert_equal \"tag1\", d.events[0][0]\n    assert_equal 1, d.events[0][2][\"a\"]\n    assert_equal \"tag1\", d.events[1][0]\n    assert_equal 2, d.events[1][2][\"a\"]\n  end\n\n  def test_multi_json_with_add_remote_addr\n    d = create_driver(config + \"add_remote_addr true\")\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n\n    records = [{\"a\"=>1},{\"a\"=>2}]\n    tag = \"tag1\"\n    res_codes = []\n\n    d.run(expect_records: 2, timeout: 5) do\n      res = post(\"/#{tag}\", {\"json\"=>records.to_json, \"time\"=>time_i.to_s})\n      res_codes << res.code\n    end\n    assert_equal [\"200\"], res_codes\n\n    assert_equal \"tag1\", d.events[0][0]\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal 1, d.events[0][2][\"a\"]\n    assert{ d.events[0][2].has_key?(\"REMOTE_ADDR\") && d.events[0][2][\"REMOTE_ADDR\"] =~ /^\\d{1,4}(\\.\\d{1,4}){3}$/ }\n\n    assert_equal \"tag1\", d.events[1][0]\n    assert_equal_event_time time, d.events[1][1]\n    assert_equal 2, d.events[1][2][\"a\"]\n  end\n\n  def test_json_with_add_remote_addr_given_x_forwarded_for\n    d = create_driver(config + \"add_remote_addr true\")\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, _t, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json, \"time\"=>time_i.to_s}, {\"X-Forwarded-For\"=>\"129.78.138.66, 127.0.0.1\"})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n\n    assert_equal \"tag1\", d.events[0][0]\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal({\"REMOTE_ADDR\"=>\"129.78.138.66\", \"a\"=>1}, d.events[0][2])\n\n    assert_equal \"tag2\", d.events[1][0]\n    assert_equal_event_time time, d.events[1][1]\n    assert_equal({\"REMOTE_ADDR\"=>\"129.78.138.66\", \"a\"=>2}, d.events[1][2])\n  end\n\n  def test_multi_json_with_add_remote_addr_given_x_forwarded_for\n    d = create_driver(config + \"add_remote_addr true\")\n\n    tag = \"tag1\"\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    records = [{\"a\"=>1},{\"a\"=>2}]\n    events = [\n      [tag, time, {\"REMOTE_ADDR\"=>\"129.78.138.66\", \"a\"=>1}],\n      [tag, time, {\"REMOTE_ADDR\"=>\"129.78.138.66\", \"a\"=>2}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2, timeout: 5) do\n      res = post(\"/#{tag}\", {\"json\"=>records.to_json, \"time\"=>time_i.to_s}, {\"X-Forwarded-For\"=>\"129.78.138.66, 127.0.0.1\"})\n      res_codes << res.code\n    end\n    assert_equal [\"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_add_remote_addr_given_multi_x_forwarded_for\n    d = create_driver(config + \"add_remote_addr true\")\n\n    tag = \"tag1\"\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    record = {\"a\" => 1}\n    event = [\"tag1\", time, {\"REMOTE_ADDR\" => \"129.78.138.66\", \"a\" => 1}]\n    res_code = nil\n\n    d.run(expect_records: 1, timeout: 5) do\n      res = post(\"/#{tag}\", {\"json\" => record.to_json, \"time\" => time_i.to_s}) { |http, req|\n        # net/http can't send multiple headers so overwrite it.\n        def req.each_capitalized\n          block_given? or return enum_for(__method__) { @header.size }\n          @header.each do |k, vs|\n            vs.each { |v|\n              yield capitalize(k), v\n            }\n          end\n        end\n        req.add_field(\"X-Forwarded-For\", \"129.78.138.66, 127.0.0.1\")\n        req.add_field(\"X-Forwarded-For\", \"8.8.8.8\")\n      }\n      res_code = res.code\n    end\n    assert_equal \"200\", res_code\n    assert_equal event, d.events.first\n    assert_equal_event_time time, d.events.first[1]\n  end\n\n  def test_multi_json_with_add_http_headers\n    d = create_driver(config + \"add_http_headers true\")\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    records = [{\"a\"=>1},{\"a\"=>2}]\n    tag = \"tag1\"\n    res_codes = []\n\n    d.run(expect_records: 2, timeout: 5) do\n      res = post(\"/#{tag}\", {\"json\"=>records.to_json, \"time\"=>time_i.to_s})\n      res_codes << res.code\n    end\n    assert_equal [\"200\"], res_codes\n\n    assert_equal \"tag1\", d.events[0][0]\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal 1, d.events[0][2][\"a\"]\n\n    assert_equal \"tag1\", d.events[1][0]\n    assert_equal_event_time time, d.events[1][1]\n    assert_equal 2, d.events[1][2][\"a\"]\n\n    assert include_http_header?(d.events[0][2])\n    assert include_http_header?(d.events[1][2])\n  end\n\n  def test_json_with_add_http_headers\n    d = create_driver(config + \"add_http_headers true\")\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, t, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json, \"time\"=>time_i.to_s})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n\n    assert_equal \"tag1\", d.events[0][0]\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal 1, d.events[0][2][\"a\"]\n\n    assert_equal \"tag2\", d.events[1][0]\n    assert_equal_event_time time, d.events[1][1]\n    assert_equal 2, d.events[1][2][\"a\"]\n\n    assert include_http_header?(d.events[0][2])\n    assert include_http_header?(d.events[1][2])\n  end\n\n  def test_multi_json_with_custom_parser\n    d = create_driver(config + %[\n        <parse>\n          @type json\n          keep_time_key true\n          time_key foo\n          time_format %iso8601\n        </parse>\n    ])\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_s = Time.at(time).iso8601\n\n    records = [{\"foo\"=>time_s,\"bar\"=>\"test1\"},{\"foo\"=>time_s,\"bar\"=>\"test2\"}]\n    tag = \"tag1\"\n    res_codes = []\n\n    d.run(expect_records: 2, timeout: 5) do\n      res = post(\"/#{tag}\", records.to_json, {\"Content-Type\"=>\"application/octet-stream\"})\n      res_codes << res.code\n    end\n    assert_equal [\"200\"], res_codes\n\n    assert_equal \"tag1\", d.events[0][0]\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal d.events[0][2], records[0]\n\n    assert_equal \"tag1\", d.events[1][0]\n    assert_equal_event_time time, d.events[1][1]\n    assert_equal d.events[1][2], records[1]\n  end\n\n  def test_application_json\n    d = create_driver\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, t, record|\n        res = post(\"/#{tag}?time=#{time_i.to_s}\", record.to_json, {\"Content-Type\"=>\"application/json; charset=utf-8\"})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_csp_report\n    d = create_driver\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, t, record|\n        res = post(\"/#{tag}?time=#{time_i.to_s}\", record.to_json, {\"Content-Type\"=>\"application/csp-report; charset=utf-8\"})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_application_msgpack\n    d = create_driver\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, t, record|\n        res = post(\"/#{tag}?time=#{time_i.to_s}\", record.to_msgpack, {\"Content-Type\"=>\"application/msgpack\"})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_application_ndjson\n    d = create_driver\n    events = [\n        [\"tag1\", 1643935663, \"{\\\"a\\\":1}\\n{\\\"b\\\":2}\"],\n        [\"tag2\", 1643935664, \"{\\\"a\\\":3}\\r\\n{\\\"b\\\":4}\"]\n    ]\n\n    expected = [\n        [\"tag1\", 1643935663, {\"a\"=>1}],\n        [\"tag1\", 1643935663, {\"b\"=>2}],\n        [\"tag2\", 1643935664, {\"a\"=>3}],\n        [\"tag2\", 1643935664, {\"b\"=>4}]\n    ]\n\n    d.run(expect_records: 1) do\n      events.each do |tag, time, record|\n        res = post(\"/#{tag}?time=#{time}\", record, {\"Content-Type\"=>\"application/x-ndjson\"})\n        assert_equal(\"200\", res.code)\n      end\n    end\n    assert_equal(expected, d.events)\n  end\n\n  def test_multipart_formdata\n    tag = \"tag\"\n    time = event_time_without_nsec\n    event = [tag, time, {\"key\" => 0}]\n    body = \"--TESTBOUNDARY\\r\\n\" +\n           \"Content-Disposition: form-data; name=\\\"json\\\"\\r\\n\" +\n           \"\\r\\n\" +\n           %[{\"key\":0}\\r\\n] +\n           \"--TESTBOUNDARY\\r\\n\"\n\n    d = create_driver\n    res = nil\n    d.run(expect_records: 1) do\n      res = post(\"/#{tag}?time=#{time.to_s}\", body, {\"Content-Type\"=>\"multipart/form-data; boundary=TESTBOUNDARY\"})\n    end\n\n    assert_equal(\n      [\"200\", [event]],\n      [res.code, d.events],\n    )\n  end\n\n  def test_msgpack\n    d = create_driver\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, t, record|\n        res = post(\"/#{tag}\", {\"msgpack\"=>record.to_msgpack, \"time\"=>time_i.to_s})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_multi_msgpack\n    d = create_driver\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n\n    records = [{\"a\"=>1},{\"a\"=>2}]\n    events = [\n      [\"tag1\", time, records[0]],\n      [\"tag1\", time, records[1]],\n    ]\n    tag = \"tag1\"\n    res_codes = []\n    d.run(expect_records: 2) do\n      res = post(\"/#{tag}\", {\"msgpack\"=>records.to_msgpack, \"time\"=>time_i.to_s})\n      res_codes << res.code\n    end\n    assert_equal [\"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_with_regexp\n    d = create_driver(config + %[\n      format /^(?<field_1>\\\\d+):(?<field_2>\\\\w+)$/\n      types field_1:integer\n    ])\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"field_1\" => 1, \"field_2\" => 'str'}],\n      [\"tag2\", time, {\"field_1\" => 2, \"field_2\" => 'str'}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, t, record|\n        body = record.map { |k, v|\n          v.to_s\n        }.join(':')\n        res = post(\"/#{tag}?time=#{time_i.to_s}\", body, {'Content-Type' => 'application/octet-stream'})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_with_csv\n    require 'csv'\n\n    d = create_driver(config + %[\n      format csv\n      keys foo,bar\n    ])\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"foo\" => \"1\", \"bar\" => 'st\"r'}],\n      [\"tag2\", time, {\"foo\" => \"2\", \"bar\" => 'str'}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 2) do\n      events.each do |tag, t, record|\n        body = record.map { |k, v| v }.to_csv\n        res = post(\"/#{tag}?time=#{time_i.to_s}\", body, {'Content-Type' => 'text/comma-separated-values'})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_response_with_empty_img\n    d = create_driver(config)\n    assert_equal true, d.instance.respond_with_empty_img\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n    res_bodies = []\n\n    d.run do\n      events.each do |tag, _t, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json, \"time\"=>time_i.to_s})\n        res_codes << res.code\n        # Ruby returns ASCII-8 encoded string for GIF.\n        res_bodies << res.body.force_encoding(\"UTF-8\")\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal [Fluent::Plugin::HttpInput::EMPTY_GIF_IMAGE, Fluent::Plugin::HttpInput::EMPTY_GIF_IMAGE], res_bodies\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_response_without_empty_img\n    d = create_driver(config + \"respond_with_empty_img false\")\n    assert_equal false, d.instance.respond_with_empty_img\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n    res_bodies = []\n\n    d.run do\n      events.each do |tag, _t, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json, \"time\"=>time_i.to_s})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal [], res_bodies\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_response_use_204_response\n    d = create_driver(config + %[\n          respond_with_empty_img false\n          use_204_response true\n        ])\n    assert_equal true, d.instance.use_204_response\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n    res_bodies = []\n\n    d.run do\n      events.each do |tag, _t, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json, \"time\"=>time_i.to_s})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"204\", \"204\"], res_codes\n    assert_equal [], res_bodies\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_cors_allowed\n    d = create_driver(config + \"cors_allow_origins [\\\"http://foo.com\\\"]\")\n    assert_equal [\"http://foo.com\"], d.instance.cors_allow_origins\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n    res_headers = []\n\n    d.run do\n      events.each do |tag, _t, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json, \"time\"=>time_i.to_s}, {\"Origin\"=>\"http://foo.com\"})\n        res_codes << res.code\n        res_headers << res[\"Access-Control-Allow-Origin\"]\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal [\"http://foo.com\", \"http://foo.com\"], res_headers\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_cors_allowed_wildcard\n    d = create_driver(config + 'cors_allow_origins [\"*\"]')\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n    ]\n\n    d.run do\n      events.each do |tag, time, record|\n        headers = {\"Origin\" => \"http://foo.com\"}\n\n        res = post(\"/#{tag}\", {\"json\" => record.to_json, \"time\" => time.to_i}, headers)\n\n        assert_equal \"200\", res.code\n        assert_equal \"*\", res[\"Access-Control-Allow-Origin\"]\n      end\n    end\n  end\n\n  def test_get_request\n    d = create_driver(config)\n\n    d.run do\n      res = get(\"/cors.test\", {}, {})\n      assert_equal \"200\", res.code\n    end\n  end\n\n  def test_cors_preflight\n    d = create_driver(config + 'cors_allow_origins [\"*\"]')\n\n    d.run do\n      header = {\n        \"Origin\" => \"http://foo.com\",\n        \"Access-Control-Request-Method\" => \"POST\",\n        \"Access-Control-Request-Headers\" => \"Content-Type\",\n      }\n      res = options(\"/cors.test\", {}, header)\n\n      assert_equal \"200\", res.code\n      assert_equal \"*\", res[\"Access-Control-Allow-Origin\"]\n      assert_equal \"POST\", res[\"Access-Control-Allow-Methods\"]\n    end\n  end\n\n  def test_cors_allowed_wildcard_for_subdomain\n    d = create_driver(config + 'cors_allow_origins [\"http://*.foo.com\"]')\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n    ]\n\n    d.run do\n      events.each do |tag, time, record|\n        headers = {\"Origin\" => \"http://subdomain.foo.com\"}\n\n        res = post(\"/#{tag}\", {\"json\" => record.to_json, \"time\" => time.to_i}, headers)\n\n        assert_equal \"200\", res.code\n        assert_equal \"http://subdomain.foo.com\", res[\"Access-Control-Allow-Origin\"]\n      end\n    end\n  end\n\n  def test_cors_allowed_exclude_empty_string\n    d = create_driver(config + 'cors_allow_origins [\"\", \"http://*.foo.com\"]')\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n    ]\n\n    d.run do\n      events.each do |tag, time, record|\n        headers = {\"Origin\" => \"http://subdomain.foo.com\"}\n\n        res = post(\"/#{tag}\", {\"json\" => record.to_json, \"time\" => time.to_i}, headers)\n\n        assert_equal \"200\", res.code\n        assert_equal \"http://subdomain.foo.com\", res[\"Access-Control-Allow-Origin\"]\n      end\n    end\n  end\n\n  def test_cors_allowed_wildcard_preflight_for_subdomain\n    d = create_driver(config + 'cors_allow_origins [\"http://*.foo.com\"]')\n\n    d.run do\n      header = {\n        \"Origin\" => \"http://subdomain.foo.com\",\n        \"Access-Control-Request-Method\" => \"POST\",\n        \"Access-Control-Request-Headers\" => \"Content-Type\",\n      }\n      res = options(\"/cors.test\", {}, header)\n\n      assert_equal \"200\", res.code\n      assert_equal \"http://subdomain.foo.com\", res[\"Access-Control-Allow-Origin\"]\n      assert_equal \"POST\", res[\"Access-Control-Allow-Methods\"]\n    end\n  end\n\n  def test_cors_allow_credentials\n    d = create_driver(config + %[\n          cors_allow_origins [\"http://foo.com\"]\n          cors_allow_credentials\n        ])\n    assert_equal true, d.instance.cors_allow_credentials\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    event = [\"tag1\", time, {\"a\"=>1}]\n    res_code = nil\n    res_header = nil\n\n    d.run do\n      res = post(\"/#{event[0]}\", {\"json\"=>event[2].to_json, \"time\"=>time.to_i.to_s}, {\"Origin\"=>\"http://foo.com\"})\n      res_code = res.code\n      res_header = res[\"Access-Control-Allow-Credentials\"]\n    end\n    assert_equal(\n      {\n        response_code: \"200\",\n        allow_credentials_header: \"true\",\n        events: [event]\n      },\n      {\n        response_code: res_code,\n        allow_credentials_header: res_header,\n        events: d.events\n      }\n    )\n  end\n\n  def test_cors_allow_credentials_for_wildcard_origins\n    assert_raise(Fluent::ConfigError) do\n      create_driver(config + %[\n        cors_allow_origins [\"*\"]\n        cors_allow_credentials\n      ])\n    end\n  end\n\n  def test_cors_with_nil_origin\n    d = create_driver(config + %[\n      cors_allow_origins [\"http://foo.com\"]\n    ])\n    assert_equal [\"http://foo.com\"], d.instance.cors_allow_origins\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    event = [\"tag1\", time, {\"a\"=>1}]\n    res_code = nil\n\n    d.run do\n      res = post(\"/#{event[0]}\", {\"json\"=>event[2].to_json, \"time\"=>time.to_i.to_s})\n      res_code = res.code\n    end\n\n    assert_equal \"200\", res_code\n    assert_equal [event], d.events\n    assert_equal_event_time time, d.events[0][1]\n  end\n\n  def test_content_encoding_gzip\n    d = create_driver\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n\n    d.run do\n      events.each do |tag, time, record|\n        header = {'Content-Type'=>'application/json', 'Content-Encoding'=>'gzip'}\n        res = post(\"/#{tag}?time=#{time}\", compress_gzip(record.to_json), header)\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_content_encoding_deflate\n    d = create_driver\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n      [\"tag2\", time, {\"a\"=>2}],\n    ]\n    res_codes = []\n\n    d.run do\n      events.each do |tag, time, record|\n        header = {'Content-Type'=>'application/msgpack', 'Content-Encoding'=>'deflate'}\n        res = post(\"/#{tag}?time=#{time}\", Zlib.deflate(record.to_msgpack), header)\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal events, d.events\n    assert_equal_event_time time, d.events[0][1]\n    assert_equal_event_time time, d.events[1][1]\n  end\n\n  def test_cors_disallowed\n    d = create_driver(config + \"cors_allow_origins [\\\"http://foo.com\\\"]\")\n    assert_equal [\"http://foo.com\"], d.instance.cors_allow_origins\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    res_codes = []\n\n    d.end_if{ res_codes.size == 2 }\n    d.run do\n      res = post(\"/tag1\", {\"json\"=>{\"a\"=>1}.to_json, \"time\"=>time_i.to_s}, {\"Origin\"=>\"http://bar.com\"})\n      res_codes << res.code\n      res = post(\"/tag2\", {\"json\"=>{\"a\"=>1}.to_json, \"time\"=>time_i.to_s}, {\"Origin\"=>\"http://bar.com\"})\n      res_codes << res.code\n    end\n    assert_equal [\"403\", \"403\"], res_codes\n  end\n\n  def test_add_query_params\n    d = create_driver(config + \"add_query_params true\")\n    assert_equal true, d.instance.add_query_params\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    time_i = time.to_i\n    events = [\n      [\"tag1\", time, {\"a\"=>1, \"QUERY_A\"=>\"b\"}],\n      [\"tag2\", time, {\"a\"=>2, \"QUERY_A\"=>\"b\"}],\n    ]\n    res_codes = []\n    res_bodies = []\n\n    d.run do\n      events.each do |tag, _t, record|\n        res = post(\"/#{tag}?a=b\", {\"json\"=>record.to_json, \"time\"=>time_i.to_s})\n        res_codes << res.code\n      end\n    end\n    assert_equal [\"200\", \"200\"], res_codes\n    assert_equal [], res_bodies\n    assert_equal events, d.events\n  end\n\n  def test_add_tag_prefix\n    d = create_driver(config + \"add_tag_prefix test\")\n    assert_equal \"test\", d.instance.add_tag_prefix\n\n    time = event_time(\"2011-01-02 13:14:15.123 UTC\")\n    float_time = time.to_f\n\n    events = [\n      [\"tag1\", time, {\"a\"=>1}],\n    ]\n    res_codes = []\n\n    d.run(expect_records: 1) do\n      events.each do |tag, t, record|\n        res = post(\"/#{tag}\", {\"json\"=>record.to_json, \"time\"=>float_time.to_s})\n        res_codes << res.code\n      end\n    end\n\n    assert_equal [\"200\"], res_codes\n    assert_equal [[\"test.tag1\", time, {\"a\"=>1}]], d.events\n  end\n\n  $test_in_http_connection_object_ids = []\n  $test_in_http_content_types = []\n  $test_in_http_content_types_flag = false\n  module ContentTypeHook\n    def initialize(*args)\n      @io_handler = nil\n      super\n    end\n    def on_headers_complete(headers)\n      super\n      if $test_in_http_content_types_flag\n        $test_in_http_content_types << self.content_type\n      end\n    end\n\n    def on_message_begin\n      super\n      if $test_in_http_content_types_flag\n        $test_in_http_connection_object_ids << @io_handler.object_id\n      end\n    end\n  end\n\n  class Fluent::Plugin::HttpInput::Handler\n    prepend ContentTypeHook\n  end\n\n  def test_if_content_type_is_initialized_properly\n    # This test is to check if Fluent::HttpInput::Handler's @content_type is initialized properly.\n    # Especially when in Keep-Alive and the second request has no 'Content-Type'.\n\n    begin\n      d = create_driver\n\n      $test_in_http_content_types_flag = true\n      d.run do\n        # Send two requests the second one has no Content-Type in Keep-Alive\n        Net::HTTP.start(\"127.0.0.1\", @port) do |http|\n          req = Net::HTTP::Post.new(\"/foodb/bartbl\", {\"connection\" => \"keepalive\", \"Content-Type\" => \"application/json\"})\n          http.request(req)\n          req = Net::HTTP::Get.new(\"/foodb/bartbl\", {\"connection\" => \"keepalive\"})\n          http.request(req)\n        end\n\n      end\n    ensure\n      $test_in_http_content_types_flag = false\n    end\n    assert_equal(['application/json', ''], $test_in_http_content_types)\n    # Asserting keepalive\n    assert_equal $test_in_http_connection_object_ids[0], $test_in_http_connection_object_ids[1]\n  end\n\n  def get(path, params, header = {})\n    http = Net::HTTP.new(\"127.0.0.1\", @port)\n    req = Net::HTTP::Get.new(path, header)\n    http.request(req)\n  end\n\n  def options(path, params, header = {})\n    http = Net::HTTP.new(\"127.0.0.1\", @port)\n    req = Net::HTTP::Options.new(path, header)\n    http.request(req)\n  end\n\n  def post(path, params, header = {})\n    http = Net::HTTP.new(\"127.0.0.1\", @port)\n    req = Net::HTTP::Post.new(path, header)\n    yield http, req if block_given?\n    if params.is_a?(String)\n      unless header.has_key?('Content-Type')\n        header['Content-Type'] = 'application/octet-stream'\n      end\n      req.body = params\n    else\n      unless header.has_key?('Content-Type')\n        header['Content-Type'] = 'application/x-www-form-urlencoded'\n      end\n      req.set_form_data(params)\n    end\n    http.request(req)\n  end\n\n  def compress_gzip(data)\n    io = StringIO.new\n    io.binmode\n    Zlib::GzipWriter.wrap(io) { |gz| gz.write data }\n    return io.string\n  end\n\n  def include_http_header?(record)\n    record.keys.find { |header| header.start_with?('HTTP_') }\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_monitor_agent.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_monitor_agent'\nrequire 'fluent/engine'\nrequire 'fluent/config'\nrequire 'fluent/event_router'\nrequire 'fluent/supervisor'\nrequire 'fluent/version'\nrequire 'net/http'\nrequire 'json'\nrequire_relative '../test_plugin_classes'\n\nclass MonitorAgentInputTest < Test::Unit::TestCase\n  include FuzzyAssert\n\n  CONFIG_DIR = File.expand_path('../tmp/in_monitor_agent', __dir__)\n\n  def setup\n    Fluent::Test.setup\n  end\n\n  def create_driver(conf = '')\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::MonitorAgentInput).configure(conf)\n  end\n\n  def configure_ra(ra, conf_str)\n    conf = Fluent::Config.parse(conf_str, \"(test)\", \"(test_dir)\", true)\n    ra.configure(conf)\n    ra\n  end\n\n  def test_configure\n    d = create_driver\n    assert_equal(\"0.0.0.0\", d.instance.bind)\n    assert_equal(24220, d.instance.port)\n    assert_equal(nil, d.instance.tag)\n    assert_equal(60, d.instance.emit_interval)\n    assert_true d.instance.include_config\n  end\n\n  sub_test_case \"collect in_monitor_agent plugin statistics\" do\n    # Input Test Driver does not register metric callbacks.\n    # We should stub them here.\n    class TestEventMetricRouter < Fluent::Test::Driver::TestEventRouter\n      def initialize(driver)\n        super\n\n        raise ArgumentError, \"plugin does not respond metric_callback method\" unless @driver.instance.respond_to?(:metric_callback)\n      end\n\n      def emit(tag, time, record)\n        super\n        @driver.instance.metric_callback(OneEventStream.new(time, record))\n      end\n\n      def emit_array(tag, array)\n        super\n        @driver.instance.metric_callback(ArrayEventStream.new(array))\n      end\n\n      def emit_stream(tag, es)\n        super\n        @driver.instance.metric_callback(es)\n      end\n    end\n\n    class MetricInputDriver < Fluent::Test::Driver::Input\n      def configure(conf, syntax: :v1)\n        if conf.is_a?(Fluent::Config::Element)\n          @config = conf\n        else\n          @config = Fluent::Config.parse(conf, \"(test)\", \"(test_dir)\", syntax: syntax)\n        end\n\n        if @instance.respond_to?(:router=)\n          @event_streams = []\n          @error_events = []\n\n          driver = self\n          mojule =  Module.new do\n            define_method(:event_emitter_router) do |label_name|\n              TestEventMetricRouter.new(driver)\n            end\n          end\n          @instance.singleton_class.prepend mojule\n        end\n\n        @instance.configure(@config)\n        self\n      end\n    end\n\n    setup do\n      # check @type and type in one configuration\n      conf = <<-EOC\n<source>\n  @type test_in_gen\n  @id test_in_gen\n  num 10\n</source>\n<filter>\n  @type test_filter\n  @id test_filter\n</filter>\n<match **>\n  @type relabel\n  @id test_relabel\n  @label @test\n</match>\n<label @test>\n  <match **>\n    @type test_out\n    @id test_out\n  </match>\n</label>\n<label @copy>\n  <match **>\n    @type copy\n    <store>\n      @type test_out\n      @id copy_out_1\n    </store>\n    <store>\n      @type test_out\n      @id copy_out_2\n    </store>\n  </match>\n</label>\n<label @ERROR>\n  <match>\n    @type null\n    @id null\n  </match>\n</label>\nEOC\n      @ra = Fluent::RootAgent.new(log: $log)\n      stub(Fluent::Engine).root_agent { @ra }\n      @ra = configure_ra(@ra, conf)\n    end\n\n    data(:with_config_yes => true,\n         :with_config_no => false)\n    def test_enable_input_metrics(with_config)\n      monitor_agent_conf = <<-CONF\n        tag test.monitor\n        emit_interval 1\nCONF\n      @ra.inputs.first.context_router.emit(\"test.event\", Fluent::Engine.now, {\"message\":\"ok\"})\n      d = MetricInputDriver.new(Fluent::Plugin::MonitorAgentInput).configure(monitor_agent_conf)\n      d.run(expect_emits: 1, timeout: 3)\n\n      test_label = @ra.labels['@test']\n      error_label = @ra.labels['@ERROR']\n      input_info = {\n        \"output_plugin\"  => false,\n        \"plugin_category\"=> \"input\",\n        \"plugin_id\"      => \"test_in_gen\",\n        \"retry_count\"    => nil,\n        \"type\"           => \"test_in_gen\",\n        \"emit_records\"   => 0, # This field is not updated due to not to be assigned metric callback.\n        \"emit_size\"      => 0, # Ditto.\n      }\n      input_info[\"config\"] = {\"@id\" => \"test_in_gen\", \"@type\" => \"test_in_gen\", \"num\" => \"10\"} if with_config\n      filter_info = {\n        \"output_plugin\"   => false,\n        \"plugin_category\" => \"filter\",\n        \"plugin_id\"       => \"test_filter\",\n        \"retry_count\"     => nil,\n        \"type\"            => \"test_filter\",\n        \"emit_records\"   => Integer,\n        \"emit_size\"      => Integer,\n      }\n      filter_info[\"config\"] = {\"@id\" => \"test_filter\", \"@type\" => \"test_filter\"} if with_config\n      output_info = {\n        \"output_plugin\"   => true,\n        \"plugin_category\" => \"output\",\n        \"plugin_id\"       => \"test_out\",\n        \"retry_count\"     => 0,\n        \"type\"            => \"test_out\",\n        \"emit_count\"      => Integer,\n        \"emit_records\"    => Integer,\n        \"emit_size\"       => Integer,\n        \"write_count\"     => Integer,\n        \"write_secondary_count\" => Integer,\n        \"rollback_count\"  => Integer,\n        \"slow_flush_count\" => Integer,\n        \"flush_time_count\" => Integer,\n        \"drop_oldest_chunk_count\" => Integer,\n      }\n      output_info[\"config\"] = {\"@id\" => \"test_out\", \"@type\" => \"test_out\"} if with_config\n      error_label_info = {\n        \"buffer_queue_length\" => 0,\n        \"buffer_timekeys\" => [],\n        \"buffer_total_queued_size\" => 0,\n        \"output_plugin\"   => true,\n        \"plugin_category\" => \"output\",\n        \"plugin_id\"       => \"null\",\n        \"retry_count\"     => 0,\n        \"type\"            => \"null\",\n        \"buffer_available_buffer_space_ratios\" => Float,\n        \"buffer_queue_byte_size\" => Integer,\n        \"buffer_stage_byte_size\" => Integer,\n        \"buffer_stage_length\" => Integer,\n        \"emit_count\"      => Integer,\n        \"emit_records\"    => Integer,\n        \"emit_size\"       => Integer,\n        \"write_count\"     => Integer,\n        \"write_secondary_count\" => Integer,\n        \"rollback_count\"  => Integer,\n        \"slow_flush_count\" => Integer,\n        \"flush_time_count\" => Integer,\n        \"drop_oldest_chunk_count\" => Integer,\n      }\n      error_label_info[\"config\"] = {\"@id\"=>\"null\", \"@type\" => \"null\"} if with_config\n      opts = {with_config: with_config}\n      assert_equal(input_info, d.instance.get_monitor_info(@ra.inputs.first, opts))\n      assert_fuzzy_equal(filter_info, d.instance.get_monitor_info(@ra.filters.first, opts))\n      assert_fuzzy_equal(output_info, d.instance.get_monitor_info(test_label.outputs.first, opts))\n      assert_fuzzy_equal(error_label_info, d.instance.get_monitor_info(error_label.outputs.first, opts))\n      monitor_agent_emit_info = {\n        \"emit_records\" => Integer,\n        \"emit_size\" => Integer,\n      }\n      filter_statistics_info = {\n        \"emit_records\" => Integer,\n        \"emit_size\" => Integer,\n      }\n      assert_fuzzy_equal(monitor_agent_emit_info, d.instance.statistics[\"input\"])\n      assert_fuzzy_equal(filter_statistics_info, @ra.filters.first.statistics[\"filter\"])\n    end\n  end\n\n  sub_test_case \"collect plugin information\" do\n    setup do\n      # check @type and type in one configuration\n      conf = <<-EOC\n<source>\n  @type test_in\n  @id test_in\n</source>\n<filter>\n  @type test_filter\n  @id test_filter\n</filter>\n<match **>\n  @type relabel\n  @id test_relabel\n  @label @test\n</match>\n<label @test>\n  <match **>\n    @type test_out\n    @id test_out\n  </match>\n</label>\n<label @copy>\n  <match **>\n    @type copy\n    <store>\n      @type test_out\n      @id copy_out_1\n    </store>\n    <store>\n      @type test_out\n      @id copy_out_2\n    </store>\n  </match>\n</label>\n<label @ERROR>\n  <match>\n    @type null\n    @id null\n  </match>\n</label>\nEOC\n      @ra = Fluent::RootAgent.new(log: $log)\n      stub(Fluent::Engine).root_agent { @ra }\n      @ra = configure_ra(@ra, conf)\n    end\n\n    test \"plugin_category\" do\n      d = create_driver\n      test_label = @ra.labels['@test']\n      error_label = @ra.labels['@ERROR']\n      assert_equal(\"input\", d.instance.plugin_category(@ra.inputs.first))\n      assert_equal(\"filter\", d.instance.plugin_category(@ra.filters.first))\n      assert_equal(\"output\", d.instance.plugin_category(test_label.outputs.first))\n      assert_equal(\"output\", d.instance.plugin_category(error_label.outputs.first))\n    end\n\n    data(:with_config_yes => true,\n         :with_config_no => false)\n    test \"get_monitor_info\" do |with_config|\n      d = create_driver\n      test_label = @ra.labels['@test']\n      error_label = @ra.labels['@ERROR']\n      input_info = {\n        \"output_plugin\"  => false,\n        \"plugin_category\"=> \"input\",\n        \"plugin_id\"      => \"test_in\",\n        \"retry_count\"    => nil,\n        \"type\"           => \"test_in\",\n        \"emit_records\"   => 0,\n        \"emit_size\"      => 0,\n      }\n      input_info[\"config\"] = {\"@id\" => \"test_in\", \"@type\" => \"test_in\"} if with_config\n      filter_info = {\n        \"output_plugin\"   => false,\n        \"plugin_category\" => \"filter\",\n        \"plugin_id\"       => \"test_filter\",\n        \"retry_count\"     => nil,\n        \"type\"            => \"test_filter\",\n        \"emit_records\"   => 0,\n        \"emit_size\"      => 0,\n      }\n      filter_info[\"config\"] = {\"@id\" => \"test_filter\", \"@type\" => \"test_filter\"} if with_config\n      output_info = {\n        \"output_plugin\"   => true,\n        \"plugin_category\" => \"output\",\n        \"plugin_id\"       => \"test_out\",\n        \"retry_count\"     => 0,\n        \"type\"            => \"test_out\",\n        \"emit_count\"      => Integer,\n        \"emit_records\"    => Integer,\n        \"emit_size\"       => Integer,\n        \"write_count\"     => Integer,\n        \"write_secondary_count\" => Integer,\n        \"rollback_count\"  => Integer,\n        \"slow_flush_count\" => Integer,\n        \"flush_time_count\" => Integer,\n        \"drop_oldest_chunk_count\" => Integer,\n      }\n      output_info[\"config\"] = {\"@id\" => \"test_out\", \"@type\" => \"test_out\"} if with_config\n      error_label_info = {\n        \"buffer_queue_length\" => 0,\n        \"buffer_timekeys\" => [],\n        \"buffer_total_queued_size\" => 0,\n        \"output_plugin\"   => true,\n        \"plugin_category\" => \"output\",\n        \"plugin_id\"       => \"null\",\n        \"retry_count\"     => 0,\n        \"type\"            => \"null\",\n        \"buffer_available_buffer_space_ratios\" => Float,\n        \"buffer_queue_byte_size\" => Integer,\n        \"buffer_stage_byte_size\" => Integer,\n        \"buffer_stage_length\" => Integer,\n        \"emit_count\"      => Integer,\n        \"emit_records\"    => Integer,\n        \"emit_size\"       => Integer,\n        \"write_count\"     => Integer,\n        \"write_secondary_count\" => Integer,\n        \"rollback_count\"  => Integer,\n        \"slow_flush_count\" => Integer,\n        \"flush_time_count\" => Integer,\n        \"drop_oldest_chunk_count\" => Integer,\n      }\n      error_label_info[\"config\"] = {\"@id\"=>\"null\", \"@type\" => \"null\"} if with_config\n      opts = {with_config: with_config}\n      assert_equal(input_info, d.instance.get_monitor_info(@ra.inputs.first, opts))\n      assert_fuzzy_equal(filter_info, d.instance.get_monitor_info(@ra.filters.first, opts))\n      assert_fuzzy_equal(output_info, d.instance.get_monitor_info(test_label.outputs.first, opts))\n      assert_fuzzy_equal(error_label_info, d.instance.get_monitor_info(error_label.outputs.first, opts))\n    end\n\n    test \"fluentd opts\" do\n      d = create_driver\n\n      filepath = nil\n      begin\n        FileUtils.mkdir_p(CONFIG_DIR)\n        filepath = File.expand_path('fluentd.conf', CONFIG_DIR)\n        FileUtils.touch(filepath)\n        s = Fluent::Supervisor.new({config_path: filepath})\n        s.configure\n      ensure\n        FileUtils.rm_r(CONFIG_DIR) rescue _\n      end\n\n      expected_opts = {\n        \"config_path\" => filepath,\n        \"pid_file\"    => nil,\n        \"plugin_dirs\" => [\"/etc/fluent/plugin\"],\n        \"log_path\"    => nil,\n        \"root_dir\"    => nil,\n      }\n      assert_equal(expected_opts, d.instance.fluentd_opts)\n    end\n\n    test \"all_plugins\" do\n      d = create_driver\n      plugins = []\n      d.instance.all_plugins.each {|plugin| plugins << plugin.class }\n      assert_equal([FluentTest::FluentTestInput,\n                    Fluent::Plugin::RelabelOutput,\n                    FluentTest::FluentTestFilter,\n                    FluentTest::FluentTestOutput, # in label @test\n                    Fluent::Plugin::CopyOutput,\n                    FluentTest::FluentTestOutput, # in label @copy 1\n                    FluentTest::FluentTestOutput, # in label @copy 2\n                    Fluent::Plugin::NullOutput], plugins)\n    end\n\n    test \"emit\" do\n      port = unused_port(protocol: :tcp)\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{port}\n  tag monitor\n  emit_interval 1\n\")\n      d.instance.start\n      d.end_if do\n        d.events.size >= 5\n      end\n      d.run\n      expect_relabel_record = {\n        \"plugin_id\"       => \"test_relabel\",\n        \"plugin_category\" => \"output\",\n        \"type\"            => \"relabel\",\n        \"output_plugin\"   => true,\n        \"retry_count\"     => 0,\n        \"emit_count\"      => Integer,\n        \"emit_records\"    => Integer,\n        \"emit_size\"       => Integer,\n        \"write_count\"     => Integer,\n        \"write_secondary_count\" => Integer,\n        \"rollback_count\"  => Integer,\n        \"slow_flush_count\" => Integer,\n        \"flush_time_count\" => Integer,\n        \"drop_oldest_chunk_count\" => Integer,\n      }\n      expect_test_out_record = {\n        \"plugin_id\"       => \"test_out\",\n        \"plugin_category\" => \"output\",\n        \"type\"            => \"test_out\",\n        \"output_plugin\"   => true,\n        \"retry_count\"     => 0,\n        \"emit_count\"      => Integer,\n        \"emit_records\"    => Integer,\n        \"emit_size\"       => Integer,\n        \"write_count\"     => Integer,\n        \"write_secondary_count\" => Integer,\n        \"rollback_count\"  => Integer,\n        \"slow_flush_count\" => Integer,\n        \"flush_time_count\" => Integer,\n        \"drop_oldest_chunk_count\" => Integer,\n      }\n      assert_fuzzy_equal(expect_relabel_record, d.events[1][2])\n      assert_fuzzy_equal(expect_test_out_record, d.events[3][2])\n    end\n  end\n\n  def get(uri, header = {})\n    url = URI.parse(uri)\n    req = Net::HTTP::Get.new(url, header)\n    unless header.has_key?('Content-Type')\n      header['Content-Type'] = 'application/octet-stream'\n    end\n    Net::HTTP.start(url.host, url.port) {|http|\n      http.request(req)\n    }\n  end\n\n  sub_test_case \"servlets\" do\n    setup do\n      @port = unused_port(protocol: :tcp)\n      # check @type and type in one configuration\n      conf = <<-EOC\n<source>\n  @type test_in\n  @id test_in\n</source>\n<source>\n  @type monitor_agent\n  bind \"127.0.0.1\"\n  port #{@port}\n  tag monitor\n  @id monitor_agent\n</source>\n<filter>\n  @type test_filter\n  @id test_filter\n</filter>\n<match **>\n  @type relabel\n  @id test_relabel\n  @label @test\n</match>\n<label @test>\n  <match **>\n    @type test_out\n    @id test_out\n  </match>\n</label>\n<label @ERROR>\n  <match>\n    @type null\n    @id null\n  </match>\n</label>\nEOC\n\n\n      begin\n        @ra = Fluent::RootAgent.new(log: $log)\n        stub(Fluent::Engine).root_agent { @ra }\n        @ra = configure_ra(@ra, conf)\n        # store Supervisor instance to avoid collected by GC\n\n        FileUtils.mkdir_p(CONFIG_DIR)\n        @filepath = File.expand_path('fluentd.conf', CONFIG_DIR)\n        File.open(@filepath, 'w') do |v|\n          v.puts(conf)\n        end\n\n        @supervisor = Fluent::Supervisor.new({config_path: @filepath})\n        @supervisor.configure\n      ensure\n        FileUtils.rm_r(CONFIG_DIR) rescue _\n      end\n    end\n\n    test \"/api/plugins\" do\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{@port}\n  tag monitor\n\")\n      d.instance.start\n      expected_test_in_response = \"\\\nplugin_id:test_in\\tplugin_category:input\\ttype:test_in\\toutput_plugin:false\\tretry_count:\\temit_records:0\\temit_size:0\"\n      expected_test_filter_response = \"\\\nplugin_id:test_filter\\tplugin_category:filter\\ttype:test_filter\\toutput_plugin:false\\tretry_count:\\temit_records:0\\temit_size:0\"\n\n      response = get(\"http://127.0.0.1:#{@port}/api/plugins\").body\n      test_in = response.split(\"\\n\")[0]\n      test_filter = response.split(\"\\n\")[3]\n      assert_equal(expected_test_in_response, test_in)\n      assert_equal(expected_test_filter_response, test_filter)\n    end\n\n    data(:include_config_and_retry_yes => [true, true, \"include_config yes\", \"include_retry yes\"],\n         :include_config_and_retry_no => [false, false, \"include_config no\", \"include_retry no\"],)\n    test \"/api/plugins.json\" do |(with_config, with_retry, include_conf, retry_conf)|\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{@port}\n  tag monitor\n  #{include_conf}\n  #{retry_conf}\n\")\n      d.instance.start\n      expected_test_in_response = {\n        \"output_plugin\"   => false,\n        \"plugin_category\" => \"input\",\n        \"plugin_id\"       => \"test_in\",\n        \"retry_count\"     => nil,\n        \"type\"            => \"test_in\",\n        \"emit_records\"    => 0,\n        \"emit_size\"       => 0,\n      }\n      expected_test_in_response[\"config\"] = {\"@id\" => \"test_in\", \"@type\" => \"test_in\"} if with_config\n      expected_null_response = {\n        \"buffer_queue_length\" => 0,\n        \"buffer_timekeys\" => [],\n        \"buffer_total_queued_size\" => 0,\n        \"output_plugin\"   => true,\n        \"plugin_category\" => \"output\",\n        \"plugin_id\"       => \"null\",\n        \"retry_count\"     => 0,\n        \"type\"            => \"null\",\n        \"buffer_available_buffer_space_ratios\" => Float,\n        \"buffer_queue_byte_size\" => Integer,\n        \"buffer_stage_byte_size\" => Integer,\n        \"buffer_stage_length\" => Integer,\n        \"emit_count\"      => Integer,\n        \"emit_records\"    => Integer,\n        \"emit_size\"       => Integer,\n        \"write_count\"     => Integer,\n        \"write_secondary_count\" => Integer,\n        \"rollback_count\"  => Integer,\n        \"slow_flush_count\" => Integer,\n        \"flush_time_count\" => Integer,\n        \"drop_oldest_chunk_count\" => Integer,\n      }\n      expected_null_response[\"config\"] = {\"@id\" => \"null\", \"@type\" => \"null\"} if with_config\n      expected_null_response[\"retry\"] = {} if with_retry\n      response = JSON.parse(get(\"http://127.0.0.1:#{@port}/api/plugins.json\").body)\n      test_in_response = response[\"plugins\"][0]\n      null_response = response[\"plugins\"][5]\n      assert_equal(expected_test_in_response, test_in_response)\n      assert_fuzzy_equal(expected_null_response, null_response)\n    end\n\n    test \"/api/plugins.json/not_found\" do\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{@port}\n  tag monitor\n\")\n      d.instance.start\n      resp = get(\"http://127.0.0.1:#{@port}/api/plugins.json/not_found\")\n      assert_equal('404', resp.code)\n      body = JSON.parse(resp.body)\n      assert_equal(body['message'], 'Not found')\n    end\n\n    data(:with_config_and_retry_yes => [true, true, \"?with_config=yes&with_retry\"],\n         :with_config_and_retry_no => [false, false, \"?with_config=no&with_retry=no\"])\n    test \"/api/plugins.json with query parameter. query parameter is preferred than include_config\" do |(with_config, with_retry, query_param)|\n\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{@port}\n  tag monitor\n\")\n      d.instance.start\n      expected_test_in_response = {\n        \"output_plugin\"   => false,\n        \"plugin_category\" => \"input\",\n        \"plugin_id\"       => \"test_in\",\n        \"retry_count\"     => nil,\n        \"type\"            => \"test_in\",\n        \"emit_records\"   => 0,\n        \"emit_size\"      => 0,\n      }\n      expected_test_in_response[\"config\"] = {\"@id\" => \"test_in\", \"@type\" => \"test_in\"} if with_config\n      expected_null_response = {\n        \"buffer_queue_length\" => 0,\n        \"buffer_timekeys\" => [],\n        \"buffer_total_queued_size\" => 0,\n        \"output_plugin\"   => true,\n        \"plugin_category\" => \"output\",\n        \"plugin_id\"       => \"null\",\n        \"retry_count\"     => 0,\n        \"type\"            => \"null\",\n        \"buffer_available_buffer_space_ratios\" => Float,\n        \"buffer_queue_byte_size\" => Integer,\n        \"buffer_stage_byte_size\" => Integer,\n        \"buffer_stage_length\" => Integer,\n        \"emit_count\"      => Integer,\n        \"emit_records\"    => Integer,\n        \"emit_size\"       => Integer,\n        \"write_count\"     => Integer,\n        \"rollback_count\"  => Integer,\n        \"slow_flush_count\" => Integer,\n        \"flush_time_count\" => Integer,\n        \"drop_oldest_chunk_count\" => Integer,\n      }\n      expected_null_response[\"config\"] = {\"@id\" => \"null\", \"@type\" => \"null\"} if with_config\n      expected_null_response[\"retry\"] = {} if with_retry\n      response = JSON.parse(get(\"http://127.0.0.1:#{@port}/api/plugins.json#{query_param}\").body)\n      test_in_response = response[\"plugins\"][0]\n      null_response = response[\"plugins\"][5]\n      assert_equal(expected_test_in_response, test_in_response)\n      assert_fuzzy_include(expected_null_response, null_response)\n    end\n\n    test \"/api/plugins.json with 'with_ivars'. response contains specified instance variables of each plugin\" do\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{@port}\n  tag monitor\n\")\n      d.instance.start\n      expected_test_in_response = {\n        \"output_plugin\"   => false,\n        \"plugin_category\" => \"input\",\n        \"plugin_id\"       => \"test_in\",\n        \"retry_count\"     => nil,\n        \"type\"            => \"test_in\",\n        \"instance_variables\" => {\"id\" => \"test_in\"},\n        \"emit_records\"   => 0,\n        \"emit_size\"      => 0,\n      }\n      expected_null_response = {\n        \"buffer_queue_length\" => 0,\n        \"buffer_timekeys\" => [],\n        \"buffer_total_queued_size\" => 0,\n        \"output_plugin\"   => true,\n        \"plugin_category\" => \"output\",\n        \"plugin_id\"       => \"null\",\n        \"retry_count\"     => 0,\n        \"type\"            => \"null\",\n        \"instance_variables\" => {\"id\" => \"null\"},\n        \"buffer_available_buffer_space_ratios\" => Float,\n        \"buffer_queue_byte_size\" => Integer,\n        \"buffer_stage_byte_size\" => Integer,\n        \"buffer_stage_length\" => Integer,\n        \"emit_count\"      => Integer,\n        \"emit_records\"    => Integer,\n        \"emit_size\"       => Integer,\n        \"write_count\"     => Integer,\n        \"write_secondary_count\" => Integer,\n        \"rollback_count\"  => Integer,\n        \"slow_flush_count\" => Integer,\n        \"flush_time_count\" => Integer,\n        \"drop_oldest_chunk_count\" => Integer,\n      }\n      response = JSON.parse(get(\"http://127.0.0.1:#{@port}/api/plugins.json?with_config=no&with_retry=no&with_ivars=id,num_errors\").body)\n      test_in_response = response[\"plugins\"][0]\n      null_response = response[\"plugins\"][5]\n      assert_equal(expected_test_in_response, test_in_response)\n      assert_fuzzy_equal(expected_null_response, null_response)\n    end\n\n    test \"/api/config\" do\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{@port}\n  tag monitor\n\")\n      d.instance.start\n      expected_response_regex = %r{pid:\\d+\\tppid:\\d+\\tversion:#{Fluent::VERSION}\\tconfig_path:#{@filepath}\\tpid_file:\\tplugin_dirs:/etc/fluent/plugin\\tlog_path:}\n      assert_match(expected_response_regex,\n                   get(\"http://127.0.0.1:#{@port}/api/config\").body)\n    end\n\n    test \"/api/config.json\" do\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{@port}\n  tag monitor\n\")\n      d.instance.start\n      res = JSON.parse(get(\"http://127.0.0.1:#{@port}/api/config.json\").body)\n      assert_equal(@filepath, res[\"config_path\"])\n      assert_nil(res[\"pid_file\"])\n      assert_equal([\"/etc/fluent/plugin\"], res[\"plugin_dirs\"])\n      assert_nil(res[\"log_path\"])\n      assert_equal(Fluent::VERSION, res[\"version\"])\n    end\n\n    test \"/api/config.json?debug=1\" do\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{@port}\n  tag monitor\n\")\n      d.instance.start\n      # To check pretty print\n      assert_true !get(\"http://127.0.0.1:#{@port}/api/config.json\").body.include?(\"\\n\")\n      assert_true get(\"http://127.0.0.1:#{@port}/api/config.json?debug=1\").body.include?(\"\\n\")\n    end\n\n    test \"/api/config.json/not_found\" do\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{@port}\n  tag monitor\n\")\n      d.instance.start\n      resp = get(\"http://127.0.0.1:#{@port}/api/config.json/not_found\")\n      assert_equal('404', resp.code)\n      body = JSON.parse(resp.body)\n      assert_equal(body['message'], 'Not found')\n    end\n  end\n\n  sub_test_case \"check retry of buffered plugins\" do\n    class FluentTestFailWriteOutput < ::Fluent::Plugin::Output\n      ::Fluent::Plugin.register_output('test_out_fail_write', self)\n\n      def write(chunk)\n        raise \"chunk error!\"\n      end\n    end\n\n    setup do\n      @port = unused_port(protocol: :tcp)\n      # check @type and type in one configuration\n      conf = <<-EOC\n<source>\n  @type monitor_agent\n  @id monitor_agent\n  bind \"127.0.0.1\"\n  port #{@port}\n</source>\n<match **>\n  @type test_out_fail_write\n  @id test_out_fail_write\n  <buffer time>\n    timekey 1m\n    flush_mode immediate\n  </buffer>\n</match>\n      EOC\n      @ra = Fluent::RootAgent.new(log: $log)\n      stub(Fluent::Engine).root_agent { @ra }\n      @ra = configure_ra(@ra, conf)\n    end\n\n    test \"/api/plugins.json retry object should be filled if flush was failed\" do\n\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{@port}\n  include_config no\n\")\n      d.instance.start\n      output = @ra.outputs[0]\n      output.start\n      output.after_start\n      expected_test_out_fail_write_response = {\n          \"buffer_queue_length\" => 1,\n          \"buffer_timekeys\" => [output.calculate_timekey(event_time)],\n          \"buffer_total_queued_size\" => 40,\n          \"output_plugin\" => true,\n          \"plugin_category\" => \"output\",\n          \"plugin_id\" => \"test_out_fail_write\",\n          \"type\" => \"test_out_fail_write\",\n          \"buffer_newest_timekey\" => output.calculate_timekey(event_time),\n          \"buffer_oldest_timekey\" => output.calculate_timekey(event_time),\n          \"buffer_available_buffer_space_ratios\" => Float,\n          \"buffer_queue_byte_size\" => Integer,\n          \"buffer_stage_byte_size\" => Integer,\n          \"buffer_stage_length\" => Integer,\n          \"emit_count\" => Integer,\n          \"emit_records\" => Integer,\n          \"emit_size\" => Integer,\n          \"write_count\" => Integer,\n          \"write_secondary_count\" => Integer,\n          \"rollback_count\" => Integer,\n          'slow_flush_count' => Integer,\n          'flush_time_count' => Integer,\n          \"drop_oldest_chunk_count\" => Integer,\n      }\n      output.emit_events('test.tag', Fluent::ArrayEventStream.new([[event_time, {\"message\" => \"test failed flush 1\"}]]))\n      # flush few times to check steps\n      2.times do\n        output.force_flush\n        # output.force_flush calls #submit_flush_all, but #submit_flush_all skips to call #submit_flush_once when @retry exists.\n        # So that forced flush in retry state should be done by calling #submit_flush_once directly.\n        output.submit_flush_once\n        sleep 0.1 until output.buffer.queued?\n      end\n      response = JSON.parse(get(\"http://127.0.0.1:#{@port}/api/plugins.json\").body)\n      test_out_fail_write_response = response[\"plugins\"][1]\n      # remove dynamic keys\n      response_retry_count = test_out_fail_write_response.delete(\"retry_count\")\n      response_retry = test_out_fail_write_response.delete(\"retry\")\n      assert_fuzzy_equal(expected_test_out_fail_write_response, test_out_fail_write_response)\n      assert{ response_retry.has_key?(\"steps\") }\n      # it's very hard to check exact retry count (because retries are called by output flush thread scheduling)\n      assert{ response_retry_count >= 1 && response_retry[\"steps\"] >= 0 }\n      assert{ response_retry_count == response_retry[\"steps\"] + 1 }\n    end\n  end\n\n  sub_test_case \"check the port number of http server\" do\n    test \"on single worker environment\" do\n      port = unused_port(protocol: :tcp)\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{port}\n\")\n      d.instance.start\n      assert_equal(\"200\", get(\"http://127.0.0.1:#{port}/api/plugins\").code)\n    end\n\n    test \"worker_id = 2 on multi worker environment\" do\n      port = unused_port(protocol: :tcp)\n      Fluent::SystemConfig.overwrite_system_config('workers' => 4) do\n        d = Fluent::Test::Driver::Input.new(Fluent::Plugin::MonitorAgentInput)\n        d.instance.instance_eval{ @_fluentd_worker_id = 2 }\n        d.configure(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{port - 2}\n\")\n        d.instance.start\n      end\n      assert_equal(\"200\", get(\"http://127.0.0.1:#{port}/api/plugins\").code)\n    end\n  end\n\n  sub_test_case \"check NoMethodError does not happen\" do\n    class FluentTestBufferVariableOutput < ::Fluent::Plugin::Output\n      ::Fluent::Plugin.register_output('test_out_buffer_variable', self)\n      def configure(conf)\n        super\n        @buffer = []\n      end\n\n      def write(chunk)\n      end\n    end\n    class FluentTestBufferVariableFilter < ::Fluent::Plugin::Filter\n      ::Fluent::Plugin.register_filter(\"test_filter_buffer_variable\", self)\n      def initialize\n        super\n        @buffer = {}\n      end\n      def filter(tag, time, record)\n        record\n      end\n    end\n\n    setup do\n      conf = <<-EOC\n<match **>\n  @type test_out_buffer_variable\n  @id test_out_buffer_variable\n</match>\n<filter **>\n  @type test_filter_buffer_variable\n  @id test_filter_buffer_variable\n</filter>\nEOC\n      @ra = Fluent::RootAgent.new(log: $log)\n      stub(Fluent::Engine).root_agent { @ra }\n      @ra = configure_ra(@ra, conf)\n    end\n\n    test \"plugins have a variable named buffer does not throws NoMethodError\" do\n      port = unused_port(protocol: :tcp)\n      d = create_driver(\"\n  @type monitor_agent\n  bind '127.0.0.1'\n  port #{port}\n  include_config no\n\")\n      d.instance.start\n\n      assert_equal(\"200\", get(\"http://127.0.0.1:#{port}/api/plugins.json\").code)\n      assert{ d.logs.none?{|log| log.include?(\"NoMethodError\") } }\n      assert_equal(false, d.instance.instance_variable_get(:@first_warn))\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_object_space.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_object_space'\n\nrequire 'timeout'\n\nclass ObjectSpaceInputTest < Test::Unit::TestCase\n  def waiting(seconds, instance)\n    begin\n      Timeout.timeout(seconds) do\n        yield\n      end\n    rescue Timeout::Error\n      STDERR.print(*instance.log.out.logs)\n      raise\n    end\n  end\n\n  class FailObject\n  end\n\n  def setup\n    Fluent::Test.setup\n    # Overriding this behavior in the global scope will have an unexpected influence on other tests.\n    # So this should be overridden here and be removed in `teardown`.\n    def FailObject.class\n      raise \"FailObject error for tests in ObjectSpaceInputTest.\"\n    end\n  end\n\n  def teardown\n    FailObject.singleton_class.remove_method(:class)\n  end\n\n  TESTCONFIG = %[\n    emit_interval 0.2\n    tag t1\n    top 2\n  ]\n\n  def create_driver(conf=TESTCONFIG)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::ObjectSpaceInput).configure(conf)\n  end\n\n  def test_configure\n    d = create_driver\n    assert_equal 0.2, d.instance.emit_interval\n    assert_equal \"t1\", d.instance.tag\n    assert_equal 2, d.instance.top\n  end\n\n  def test_emit\n    # Force release garbaged objects due to avoid unexpected error by mock objects on `on_timer`\n    # https://github.com/fluent/fluentd/pull/5055\n    GC.start\n\n    d = create_driver\n\n    d.run(expect_emits: 3)\n\n    emits = d.events\n    assert{ emits.length > 0 }\n\n    emits.each { |tag, time, record|\n      assert_equal d.instance.tag, tag\n      assert_equal d.instance.top, record.keys.size\n      assert(time.is_a?(Fluent::EventTime))\n    }\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_sample.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_sample'\nrequire 'fileutils'\n\nclass SampleTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::SampleInput).configure(conf)\n  end\n\n  sub_test_case 'configure' do\n    test 'required parameters' do\n      assert_raise_message(\"'tag' parameter is required\") do\n        Fluent::Plugin::SampleInput.new.configure(config_element('ROOT',''))\n      end\n    end\n\n    test 'tag' do\n      d = create_driver(%[\n        tag sample\n      ])\n      assert_equal \"sample\", d.instance.tag\n    end\n\n    config = %[\n      tag sample\n    ]\n\n    test 'auto_increment_key' do\n      d = create_driver(config + %[\n        auto_increment_key id\n      ])\n      assert_equal \"id\", d.instance.auto_increment_key\n    end\n\n    test 'rate' do\n      d = create_driver(config + %[\n        rate 10\n      ])\n      assert_equal 10, d.instance.rate\n    end\n\n    test 'sample' do\n      # hash is okay\n      d = create_driver(config + %[sample {\"foo\":\"bar\"}])\n      assert_equal [{\"foo\"=>\"bar\"}], d.instance.sample\n\n      # array of hash is okay\n      d = create_driver(config + %[sample [{\"foo\":\"bar\"}]])\n      assert_equal [{\"foo\"=>\"bar\"}], d.instance.sample\n\n      assert_raise_message(/JSON::ParserError|got incomplete JSON/) do\n        create_driver(config + %[sample \"foo\"])\n      end\n\n      assert_raise_message(/is not a hash/) do\n        create_driver(config + %[sample [\"foo\"]])\n      end\n    end\n  end\n\n  sub_test_case \"emit\" do\n    config = %[\n      tag sample\n      rate 10\n      sample {\"foo\":\"bar\"}\n    ]\n\n    test 'simple' do\n      d = create_driver(config)\n      d.run(timeout: 0.5)\n\n      d.events.each do |tag, time, record|\n        assert_equal(\"sample\", tag)\n        assert_equal({\"foo\"=>\"bar\"}, record)\n        assert(time.is_a?(Fluent::EventTime))\n      end\n    end\n\n    test 'with auto_increment_key' do\n      d = create_driver(config + %[auto_increment_key id])\n      d.run(timeout: 0.5)\n\n      d.events.each_with_index do |(tag, _time, record), i|\n        assert_equal(\"sample\", tag)\n        assert_equal({\"foo\"=>\"bar\", \"id\"=>i}, record)\n      end\n    end\n  end\n\n  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_sample', 'store' )\n  FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH\n\n  sub_test_case 'when sample plugin has storage which is not specified the path'  do\n    config1 = {\n      'tag' => 'sample',\n      'rate' => '0',\n      'sample' => '[{\"x\": 1, \"y\": \"1\"}, {\"x\": 2, \"y\": \"2\"}, {\"x\": 3, \"y\": \"3\"}]',\n      'auto_increment_key' => 'id',\n    }\n    conf1 = config_element('ROOT', '', config1, [])\n    test \"value of auto increment key is not kept after stop-and-start\" do\n      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))\n\n      d1 = create_driver(conf1)\n      d1.run(timeout: 0.5) do\n        d1.instance.emit(2)\n      end\n\n      events = d1.events.sort{|a,b| a[2]['id'] <=> b[2]['id'] }\n\n      first_id1 = events.first[2]['id']\n      assert_equal 0, first_id1\n\n      last_id1 = events.last[2]['id']\n      assert { last_id1 > 0 }\n\n      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))\n\n      d2 = create_driver(conf1)\n      d2.run(timeout: 0.5) do\n        d2.instance.emit(2)\n      end\n\n      events = d2.events.sort{|a,b| a[2]['id'] <=> b[2]['id'] }\n\n      first_id2 = events.first[2]['id']\n      assert_equal 0, first_id2\n\n      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))\n    end\n  end\n\n  sub_test_case 'when sample plugin has storage which is specified the path'  do\n    setup do\n      FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)\n      FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))\n      FileUtils.chmod_R(0755, File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))\n    end\n\n    config2 = {\n      '@id' => 'test-02',\n      'tag' => 'sample',\n      'rate' => '0',\n      'sample' => '[{\"x\": 1, \"y\": \"1\"}, {\"x\": 2, \"y\": \"2\"}, {\"x\": 3, \"y\": \"3\"}]',\n      'auto_increment_key' => 'id',\n    }\n    conf2 = config_element('ROOT', '', config2, [\n              config_element(\n                'storage', '',\n                {'@type' => 'local',\n                 '@id' => 'test-02',\n                 'path' => File.join(TEST_PLUGIN_STORAGE_PATH,\n                                     'json', 'test-02.json'),\n                 'persistent' => true,\n                })\n            ])\n    test \"value of auto increment key is kept after stop-and-start\" do\n      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))\n\n      d1 = create_driver(conf2)\n      d1.run(timeout: 1) do\n        d1.instance.emit(2)\n      end\n\n      first_id1 = d1.events.first[2]['id']\n      assert_equal 0, first_id1\n\n      last_id1 = d1.events.last[2]['id']\n      assert { last_id1 > 0 }\n\n      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))\n\n      d2 = create_driver(conf2)\n      d2.run(timeout: 1) do\n        d2.instance.emit(2)\n      end\n      d2.events\n\n      first_id2 = d2.events.first[2]['id']\n      assert_equal last_id1 + 1, first_id2\n\n      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_syslog.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_syslog'\n\nclass SyslogInputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    @port = unused_port(protocol: :udp)\n  end\n\n  def teardown\n    @port = nil\n  end\n\n  def ipv4_config(port = @port)\n    %[\n      port #{port}\n      bind 127.0.0.1\n      tag syslog\n    ]\n  end\n\n  def ipv6_config(port = @port)\n    %[\n      port #{port}\n      bind ::1\n      tag syslog\n    ]\n  end\n\n  def create_driver(conf=ipv4_config)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::SyslogInput).configure(conf)\n  end\n\n  data(\n    ipv4: ['127.0.0.1', :ipv4, ::Socket::AF_INET],\n    ipv6: ['::1', :ipv6, ::Socket::AF_INET6],\n  )\n  def test_configure(data)\n    bind_addr, protocol, family = data\n    config = send(\"#{protocol}_config\")\n    omit \"IPv6 unavailable\" if family == ::Socket::AF_INET6 && !ipv6_enabled?\n\n    d = create_driver(config)\n    assert_equal @port, d.instance.port\n    assert_equal bind_addr, d.instance.bind\n  end\n\n  sub_test_case 'source_hostname_key and source_address_key features' do\n    test 'resolve_hostname must be true with source_hostname_key' do\n      assert_raise(Fluent::ConfigError) {\n        create_driver(ipv4_config + <<EOS)\nresolve_hostname false\nsource_hostname_key hostname\nEOS\n      }\n    end\n\n    data('resolve_hostname' => 'resolve_hostname true',\n         'source_hostname_key' => 'source_hostname_key source_host')\n    def test_configure_resolve_hostname(param)\n      d = create_driver([ipv4_config, param].join(\"\\n\"))\n      assert_true d.instance.resolve_hostname\n    end\n  end\n\n  data('Use protocol_type' => ['protocol_type tcp', :tcp, :udp],\n       'Use transport' => [\"<transport tcp>\\n </transport>\", nil, :tcp],\n       'Use transport and protocol' => [\"protocol_type udp\\n<transport tcp>\\n </transport>\", :udp, :tcp])\n  def test_configure_protocol(param)\n    conf, proto_type, transport_proto_type = *param\n    port = unused_port(protocol: proto_type ? proto_type : transport_proto_type)\n    d = create_driver([ipv4_config(port), conf].join(\"\\n\"))\n\n    assert_equal(d.instance.protocol_type, proto_type)\n    assert_equal(d.instance.transport_config.protocol, transport_proto_type)\n  end\n\n  # For backward compat\n  def test_respect_protocol_type_than_transport\n    d = create_driver([ipv4_config, \"<transport tcp> \\n</transport>\", \"protocol_type udp\"].join(\"\\n\"))\n    tests = create_test_case\n\n    d.run(expect_emits: 2) do\n      u = UDPSocket.new\n      u.connect('127.0.0.1', @port)\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests)\n  end\n\n\n  data(\n    ipv4: ['127.0.0.1', :ipv4, ::Socket::AF_INET],\n    ipv6: ['::1', :ipv6, ::Socket::AF_INET6],\n  )\n  def test_time_format(data)\n    bind_addr, protocol, family = data\n    config = send(\"#{protocol}_config\")\n    omit \"IPv6 unavailable\" if family == ::Socket::AF_INET6 && !ipv6_enabled?\n\n    d = create_driver(config)\n\n    tests = [\n      {'msg' => '<6>Dec 11 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec 11 00:00:00', '%b %d %H:%M:%S'))},\n      {'msg' => '<6>Dec  1 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec  1 00:00:00', '%b  %d %H:%M:%S'))},\n    ]\n    d.run(expect_emits: 2) do\n      u = UDPSocket.new(family)\n      u.connect(bind_addr, @port)\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    events = d.events\n    assert(events.size > 0)\n    events.each_index {|i|\n      assert_equal_event_time(tests[i]['expected'], events[i][1])\n    }\n  end\n\n  def test_msg_size\n    d = create_driver\n    tests = create_test_case\n\n    d.run(expect_emits: 2) do\n      u = UDPSocket.new\n      u.connect('127.0.0.1', @port)\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests)\n  end\n\n  def test_msg_size_udp_for_large_msg\n    d = create_driver(ipv4_config + %[\n      message_length_limit 5k\n    ])\n    tests = create_test_case(large_message: true)\n\n    d.run(expect_emits: 3) do\n      u = UDPSocket.new\n      u.connect('127.0.0.1', @port)\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests)\n  end\n\n  def test_msg_size_with_tcp\n    port = unused_port(protocol: :tcp)\n    d = create_driver([ipv4_config(port), \"<transport tcp> \\n</transport>\"].join(\"\\n\"))\n    tests = create_test_case\n\n    d.run(expect_emits: 2) do\n      tests.each {|test|\n        TCPSocket.open('127.0.0.1', port) do |s|\n          s.send(test['msg'], 0)\n        end\n      }\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests)\n  end\n\n  def test_emit_rfc5452\n    d = create_driver([ipv4_config, \"facility_key pri\\n<parse>\\n message_format rfc5424\\nwith_priority true\\n</parse>\"].join(\"\\n\"))\n    msg = '<1>1 2017-02-06T13:14:15.003Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00'\n\n    d.run(expect_emits: 1, timeout: 2) do\n      u = UDPSocket.new\n      u.connect('127.0.0.1', @port)\n      u.send(msg, 0)\n    end\n\n    tag, _, event = d.events[0]\n    assert_equal('syslog.kern.alert', tag)\n    assert_equal('kern', event['pri'])\n  end\n\n  def test_msg_size_with_same_tcp_connection\n    port = unused_port(protocol: :tcp)\n    d = create_driver([ipv4_config(port), \"<transport tcp> \\n</transport>\"].join(\"\\n\"))\n    tests = create_test_case\n\n    d.run(expect_emits: 2) do\n      TCPSocket.open('127.0.0.1', port) do |s|\n        tests.each {|test|\n          s.send(test['msg'], 0)\n        }\n      end\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests)\n  end\n\n  def test_msg_size_with_json_format\n    d = create_driver([ipv4_config, 'format json'].join(\"\\n\"))\n    time = Time.parse('2013-09-18 12:00:00 +0900').to_i\n    tests = ['Hello!', 'Syslog!'].map { |msg|\n      event = {'time' => time, 'message' => msg}\n      {'msg' => '<6>' + event.to_json + \"\\n\", 'expected' => msg}\n    }\n\n    d.run(expect_emits: 2) do\n      u = UDPSocket.new\n      u.connect('127.0.0.1', @port)\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests)\n  end\n\n  def test_msg_size_with_include_source_host\n    d = create_driver([ipv4_config, 'include_source_host true'].join(\"\\n\"))\n    tests = create_test_case\n\n    host = nil\n    d.run(expect_emits: 2) do\n      u = UDPSocket.new\n      u.connect('127.0.0.1', @port)\n      host = u.peeraddr[2]\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests, {host: host})\n  end\n\n  data(\n    severity_key: 'severity_key',\n    priority_key: 'priority_key',\n  )\n  def test_msg_size_with_severity_key(param_name)\n    d = create_driver([ipv4_config, \"#{param_name} severity\"].join(\"\\n\"))\n    tests = create_test_case\n\n    severity = 'info'\n    d.run(expect_emits: 2) do\n      u = UDPSocket.new\n      u.connect('127.0.0.1', @port)\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests, {severity: severity})\n  end\n\n  def test_msg_size_with_facility_key\n    d = create_driver([ipv4_config, 'facility_key facility'].join(\"\\n\"))\n    tests = create_test_case\n\n    facility = 'kern'\n    d.run(expect_emits: 2) do\n      u = UDPSocket.new\n      u.connect('127.0.0.1', @port)\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests, {facility: facility})\n  end\n\n  def test_msg_size_with_source_address_key\n    d = create_driver([ipv4_config, 'source_address_key source_address'].join(\"\\n\"))\n    tests = create_test_case\n\n    address = nil\n    d.run(expect_emits: 2) do\n      u = UDPSocket.new\n      u.connect('127.0.0.1', @port)\n      address = u.peeraddr[3]\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests, {address: address})\n  end\n\n  def test_msg_size_with_source_hostname_key\n    d = create_driver([ipv4_config, 'source_hostname_key source_hostname'].join(\"\\n\"))\n    tests = create_test_case\n\n    hostname = nil\n    d.run(expect_emits: 2) do\n      u = UDPSocket.new\n      u.do_not_reverse_lookup = false\n      u.connect('127.0.0.1', @port)\n      hostname = u.peeraddr[2]\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert(d.events.size > 0)\n    compare_test_result(d.events, tests, {hostname: hostname})\n  end\n\n  def create_test_case(large_message: false)\n    # actual syslog message has \"\\n\"\n    if large_message\n      [\n        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + \"\\n\", 'expected' => 'x' * 100},\n        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + \"\\n\", 'expected' => 'x' * 1024},\n        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 4096 + \"\\n\", 'expected' => 'x' * 4096},\n      ]\n    else\n      [\n        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + \"\\n\", 'expected' => 'x' * 100},\n        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + \"\\n\", 'expected' => 'x' * 1024},\n      ]\n    end\n  end\n\n  def compare_test_result(events, tests, options = {})\n    events.each_index { |i|\n      assert_equal('syslog.kern.info', events[i][0]) # <6> means kern.info\n      assert_equal(tests[i]['expected'], events[i][2]['message'])\n      assert_equal(options[:host], events[i][2]['source_host']) if options[:host]\n      assert_equal(options[:address], events[i][2]['source_address']) if options[:address]\n      assert_equal(options[:hostname], events[i][2]['source_hostname']) if options[:hostname]\n      assert_equal(options[:severity], events[i][2]['severity']) if options[:severity]\n      assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]\n    }\n  end\n\n  sub_test_case 'octet counting frame' do\n    def test_msg_size_with_tcp\n      port = unused_port(protocol: :tcp)\n      d = create_driver([ipv4_config(port), \"<transport tcp> \\n</transport>\", 'frame_type octet_count'].join(\"\\n\"))\n      tests = create_test_case\n\n      d.run(expect_emits: 2) do\n        tests.each {|test|\n          TCPSocket.open('127.0.0.1', port) do |s|\n            s.send(test['msg'], 0)\n          end\n        }\n      end\n\n      assert(d.events.size > 0)\n      compare_test_result(d.events, tests)\n    end\n\n    def test_msg_size_with_same_tcp_connection\n      port = unused_port(protocol: :tcp)\n      d = create_driver([ipv4_config(port), \"<transport tcp> \\n</transport>\", 'frame_type octet_count'].join(\"\\n\"))\n      tests = create_test_case\n\n      d.run(expect_emits: 2) do\n        TCPSocket.open('127.0.0.1', port) do |s|\n          tests.each {|test|\n            s.send(test['msg'], 0)\n          }\n        end\n      end\n\n      assert(d.events.size > 0)\n      compare_test_result(d.events, tests)\n    end\n\n    def create_test_case(large_message: false)\n      msgs = [\n        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100, 'expected' => 'x' * 100},\n        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024, 'expected' => 'x' * 1024},\n      ]\n      msgs.each { |msg|\n        m = msg['msg']\n        msg['msg'] = \"#{m.size} #{m}\"\n      }\n      msgs\n    end\n  end\n\n  def create_unmatched_lines_test_case\n    [\n      # valid message\n      {'msg' => '<6>Sep 10 00:00:00 localhost logger: xxx', 'expected' => {'host'=>'localhost', 'ident'=>'logger', 'message'=>'xxx'}},\n      # missing priority\n      {'msg' => 'hello world', 'expected' => {'unmatched_line' => 'hello world'}},\n      # timestamp parsing failure\n      {'msg' => '<6>ZZZ 99 99:99:99 localhost logger: xxx', 'expected' => {'unmatched_line' => '<6>ZZZ 99 99:99:99 localhost logger: xxx'}},\n    ]\n  end\n\n  def compare_unmatched_lines_test_result(events, tests, options = {})\n    events.each_index { |i|\n      tests[i]['expected'].each { |k,v|\n        assert_equal v, events[i][2][k], \"No key <#{k}> in response or value mismatch\"\n      }\n      assert_equal('syslog.unmatched', events[i][0], 'tag does not match syslog.unmatched') unless i==0\n      assert_equal(options[:address], events[i][2]['source_address'], 'response has no source_address or mismatch') if options[:address]\n      assert_equal(options[:hostname], events[i][2]['source_hostname'], 'response has no source_hostname or mismatch') if options[:hostname]\n    }\n  end\n\n  def test_emit_unmatched_lines\n    d = create_driver([ipv4_config, 'emit_unmatched_lines true'].join(\"\\n\"))\n    tests = create_unmatched_lines_test_case\n\n    d.run(expect_emits: 3) do\n      u = UDPSocket.new\n      u.do_not_reverse_lookup = false\n      u.connect('127.0.0.1', @port)\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert_equal tests.size, d.events.size\n    compare_unmatched_lines_test_result(d.events, tests)\n  end\n\n  def test_emit_unmatched_lines_with_hostname\n    d = create_driver([ipv4_config, 'emit_unmatched_lines true', 'source_hostname_key source_hostname'].join(\"\\n\"))\n    tests = create_unmatched_lines_test_case\n\n    hostname = nil\n    d.run(expect_emits: 3) do\n      u = UDPSocket.new\n      u.do_not_reverse_lookup = false\n      u.connect('127.0.0.1', @port)\n      hostname = u.peeraddr[2]\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert_equal tests.size, d.events.size\n    compare_unmatched_lines_test_result(d.events, tests, {hostname: hostname})\n  end\n\n  def test_emit_unmatched_lines_with_address\n    d = create_driver([ipv4_config, 'emit_unmatched_lines true', 'source_address_key source_address'].join(\"\\n\"))\n    tests = create_unmatched_lines_test_case\n\n    address = nil\n    d.run(expect_emits: 3) do\n      u = UDPSocket.new\n      u.do_not_reverse_lookup = false\n      u.connect('127.0.0.1', @port)\n      address = u.peeraddr[3]\n      tests.each {|test|\n        u.send(test['msg'], 0)\n      }\n    end\n\n    assert_equal tests.size, d.events.size\n    compare_unmatched_lines_test_result(d.events, tests, {address: address})\n  end\n\n  def test_send_keepalive_packet_is_disabled_by_default\n    port = unused_port(protocol: :tcp)\n    d = create_driver(ipv4_config(port) + %[\n      <transport tcp>\n      </transport>\n      protocol tcp\n    ])\n    assert_false d.instance.send_keepalive_packet\n  end\n\n  def test_send_keepalive_packet_can_be_enabled\n    addr = \"127.0.0.1\"\n    port = unused_port(protocol: :tcp)\n    d = create_driver(ipv4_config(port) + %[\n      <transport tcp>\n      </transport>\n      send_keepalive_packet true\n    ])\n    assert_true d.instance.send_keepalive_packet\n    mock.proxy(d.instance).server_create_connection(\n      :in_syslog_tcp_server, port,\n      bind: addr,\n      resolve_name: nil,\n      send_keepalive_packet: true)\n    d.run do\n      TCPSocket.open(addr, port)\n    end\n  end\n\n  def test_send_keepalive_packet_can_not_be_enabled_for_udp\n    assert_raise(Fluent::ConfigError) do\n      create_driver(ipv4_config + %[\n        send_keepalive_packet true\n      ])\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_tail.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_tail'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/system_config'\nrequire 'fluent/file_wrapper'\nrequire 'net/http'\nrequire 'flexmock/test_unit'\nrequire 'timecop'\nrequire 'tmpdir'\nrequire 'securerandom'\n\nclass TailInputTest < Test::Unit::TestCase\n  include FlexMock::TestCase\n\n  def tmp_dir\n    File.join(File.dirname(__FILE__), \"..\", \"tmp\", \"tail#{ENV['TEST_ENV_NUMBER']}\", SecureRandom.hex(10))\n  end\n\n  def setup\n    Fluent::Test.setup\n    @tmp_dir = tmp_dir\n    cleanup_directory(@tmp_dir)\n  end\n\n  def teardown\n    super\n    cleanup_directory(@tmp_dir)\n    Fluent::Engine.stop\n    Timecop.return\n  end\n\n  def cleanup_directory(path)\n    unless Dir.exist?(path)\n      FileUtils.mkdir_p(path)\n      return\n    end\n\n    FileUtils.remove_entry_secure(path, true)\n  end\n\n  def cleanup_file(path)\n    FileUtils.remove_entry_secure(path, true)\n  end\n\n  def create_target_info(path)\n    Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)\n  end\n\n  ROOT_CONFIG = config_element(\"ROOT\", \"\", {\n                            \"tag\" => \"t1\",\n                            \"rotate_wait\" => \"2s\",\n                            \"refresh_interval\" => \"1s\"\n                               })\n\n  def base_config\n    ROOT_CONFIG + config_element(\"\", \"\", { \"path\" => \"#{@tmp_dir}/tail.txt\" })\n  end\n\n  def common_config\n    base_config + config_element(\"\", \"\", { \"pos_file\" => \"#{@tmp_dir}/tail.pos\" })\n  end\n\n  def common_follow_inode_config\n    config_element(\"ROOT\", \"\", {\n                     \"path\" => \"#{@tmp_dir}/tail.txt*\",\n                     \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                     \"tag\" => \"t1\",\n                     \"refresh_interval\" => \"1s\",\n                     \"read_from_head\" => \"true\",\n                     \"format\" => \"none\",\n                     \"rotate_wait\" => \"1s\",\n                     \"follow_inodes\" => \"true\"\n                   })\n  end\n\n  CONFIG_READ_FROM_HEAD = config_element(\"\", \"\", { \"read_from_head\" => true })\n  CONFIG_DISABLE_WATCH_TIMER = config_element(\"\", \"\", { \"enable_watch_timer\" => false })\n  CONFIG_DISABLE_STAT_WATCHER = config_element(\"\", \"\", { \"enable_stat_watcher\" => false })\n  CONFIG_OPEN_ON_EVERY_UPDATE = config_element(\"\", \"\", { \"open_on_every_update\" => true })\n  SINGLE_LINE_CONFIG = config_element(\"\", \"\", { \"format\" => \"/(?<message>.*)/\" })\n  PARSE_SINGLE_LINE_CONFIG = config_element(\"\", \"\", {}, [config_element(\"parse\", \"\", { \"@type\" => \"/(?<message>.*)/\" })])\n  MULTILINE_CONFIG = config_element(\n    \"\", \"\", {\n      \"format\" => \"multiline\",\n      \"format1\" => \"/^s (?<message1>[^\\\\n]+)(\\\\nf (?<message2>[^\\\\n]+))?(\\\\nf (?<message3>.*))?/\",\n      \"format_firstline\" => \"/^[s]/\"\n    })\n  PARSE_MULTILINE_CONFIG = config_element(\n    \"\", \"\", {},\n    [config_element(\"parse\", \"\", {\n                      \"@type\" => \"multiline\",\n                      \"format1\" => \"/^s (?<message1>[^\\\\n]+)(\\\\nf (?<message2>[^\\\\n]+))?(\\\\nf (?<message3>.*))?/\",\n                      \"format_firstline\" => \"/^[s]/\"\n                    })\n    ])\n\n  MULTILINE_CONFIG_WITH_NEWLINE = config_element(\n    \"\", \"\", {\n      \"format\" => \"multiline\",\n      \"format1\" => \"/^s (?<message1>[^\\\\n]+)(\\\\nf (?<message2>[^\\\\n]+))?(\\\\nf (?<message3>.[^\\\\n]+))?/\",\n      \"format_firstline\" => \"/^[s]/\"\n    })\n  PARSE_MULTILINE_CONFIG_WITH_NEWLINE = config_element(\n    \"\", \"\", {},\n    [config_element(\"parse\", \"\", {\n                      \"@type\" => \"multiline\",\n                      \"format1\" => \"/^s (?<message1>[^\\\\n]+)(\\\\nf (?<message2>[^\\\\n]+))?(\\\\nf (?<message3>.[^\\\\n]+))?/\",\n                      \"format_firstline\" => \"/^[s]/\"\n                    })\n    ])\n\n  EX_ROTATE_WAIT = 0\n  EX_FOLLOW_INODES = false\n\n  def ex_config\n    config_element(\"\", \"\", {\n                     \"tag\" => \"tail\",\n                     \"path\" => \"test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log\",\n                     \"format\" => \"none\",\n                     \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                     \"read_from_head\" => true,\n                     \"refresh_interval\" => 30,\n                     \"rotate_wait\" => \"#{EX_ROTATE_WAIT}s\",\n                     \"follow_inodes\" => \"#{EX_FOLLOW_INODES}\",\n                   })\n  end\n\n  def tailing_group_pattern\n    \"/#{@tmp_dir}\\/(?<podname>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\/[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container>.+)-(?<docker_id>[a-z0-9]{6})\\.log$/\"\n  end\n\n  DEBUG_LOG_LEVEL = config_element(\"\", \"\", {\n    \"@log_level\" => \"debug\"\n  })\n\n  def create_group_directive(pattern, rate_period, *rules)\n    config_element(\"\", \"\", {}, [\n      config_element(\"group\", \"\", {\n        \"pattern\" => pattern,\n        \"rate_period\" => rate_period\n      }, rules)\n    ])\n  end\n\n  def create_rule_directive(match_named_captures, limit)\n    params = {\n      \"limit\" => limit,\n      \"match\" => match_named_captures,\n    }\n\n    config_element(\"rule\", \"\", params)\n  end\n\n  def create_path_element(path)\n    config_element(\"source\", \"\", { \"path\" => \"#{@tmp_dir}/#{path}\" })\n  end\n\n  def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)\n    config = use_common_conf ? common_config + conf : conf\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)\n  end\n\n  sub_test_case \"configure\" do\n    test \"plain single line\" do\n      d = create_driver\n      assert_equal([\"#{@tmp_dir}/tail.txt\"], d.instance.paths)\n      assert_equal(\"t1\", d.instance.tag)\n      assert_equal(2, d.instance.rotate_wait)\n      assert_equal(\"#{@tmp_dir}/tail.pos\", d.instance.pos_file)\n      assert_equal(1000, d.instance.read_lines_limit)\n      assert_equal(-1, d.instance.read_bytes_limit_per_second)\n      assert_equal(false, d.instance.ignore_repeated_permission_error)\n      assert_nothing_raised do\n        d.instance.have_read_capability?\n      end\n    end\n\n    data(\"empty\" => config_element,\n         \"w/o @type\" => config_element(\"\", \"\", {}, [config_element(\"parse\", \"\", {})]))\n    test \"w/o parse section\" do |conf|\n      assert_raise(Fluent::ConfigError) do\n        create_driver(conf)\n      end\n    end\n\n    test \"multi paths with path_delimiter\" do\n      c = config_element(\"ROOT\", \"\", { \"path\" => \"tail.txt|test2|tmp,dev\", \"tag\" => \"t1\", \"path_delimiter\" => \"|\" })\n      d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)\n      assert_equal([\"tail.txt\", \"test2\", \"tmp,dev\"], d.instance.paths)\n    end\n\n    test \"multi paths with same path configured twice\" do\n      c = config_element(\"ROOT\", \"\", { \"path\" => \"test1.txt,test2.txt,test1.txt\", \"tag\" => \"t1\", \"path_delimiter\" => \",\" })\n      d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)\n      assert_equal([\"test2.txt\",\"test1.txt\"].sort, d.instance.paths.sort)\n    end\n\n    test \"multi paths with invalid path_delimiter\" do\n      c = config_element(\"ROOT\", \"\", { \"path\" => \"tail.txt|test2|tmp,dev\", \"tag\" => \"t1\", \"path_delimiter\" => \"*\" })\n      assert_raise(Fluent::ConfigError) do\n        create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)\n      end\n    end\n\n    test \"follow_inodes w/o pos file\" do\n      assert_raise(Fluent::ConfigError) do\n        create_driver(base_config + config_element('', '', {'follow_inodes' => 'true'}))\n      end\n    end\n\n    sub_test_case \"log throttling per file\" do\n      test \"w/o watcher timer is invalid\" do\n        conf = CONFIG_DISABLE_WATCH_TIMER + config_element(\"ROOT\", \"\", {\"read_bytes_limit_per_second\" => \"8k\"})\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n\n      test \"valid\" do\n        conf = config_element(\"ROOT\", \"\", {\"read_bytes_limit_per_second\" => \"8k\"})\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n    end\n\n    test \"both enable_watch_timer and enable_stat_watcher are false\" do\n      assert_raise(Fluent::ConfigError) do\n        create_driver(CONFIG_DISABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)\n      end\n    end\n\n    sub_test_case \"encoding\" do\n      test \"valid\" do\n        conf = SINGLE_LINE_CONFIG + config_element(\"\", \"\", { \"encoding\" => \"utf-8\" })\n        d = create_driver(conf)\n        assert_equal(Encoding::UTF_8, d.instance.encoding)\n      end\n\n      test \"invalid\" do\n        conf = SINGLE_LINE_CONFIG + config_element(\"\", \"\", { \"encoding\" => \"no-such-encoding\" })\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n    end\n\n    sub_test_case \"from_encoding\" do\n      test \"only specified from_encoding raise ConfigError\" do\n        conf = SINGLE_LINE_CONFIG + config_element(\"\", \"\", { \"from_encoding\" => \"utf-8\" })\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n\n      test \"valid\" do\n        conf = SINGLE_LINE_CONFIG + config_element(\"\", \"\", {\n                                                     \"from_encoding\" => \"utf-8\",\n                                                     \"encoding\" => \"utf-8\"\n                                                   })\n        d = create_driver(conf)\n        assert_equal(Encoding::UTF_8, d.instance.from_encoding)\n      end\n\n      test \"invalid\" do\n        conf = SINGLE_LINE_CONFIG + config_element(\"\", \"\", {\n                                                     \"from_encoding\" => \"no-such-encoding\",\n                                                     \"encoding\" => \"utf-8\"\n                                                   })\n        assert_raise(Fluent::ConfigError) do\n          create_driver(conf)\n        end\n      end\n    end\n  end\n\n  sub_test_case \"configure group\" do\n    test \"<rule> required\" do\n      conf = create_group_directive('.', '1m') + SINGLE_LINE_CONFIG\n      assert_raise(Fluent::ConfigError) do\n        create_driver(conf)\n      end\n    end\n\n    test \"valid configuration\" do\n      rule1 = create_rule_directive({\n        \"namespace\"=> \"/namespace-a/\",\n        \"podname\"=> \"/podname-[b|c]/\"\n      }, 100)\n      rule2 = create_rule_directive({\n        \"namespace\"=> \"/namespace-[d|e]/\",\n        \"podname\"=> \"/podname-f/\",\n      }, 50)\n      rule3 = create_rule_directive({\n        \"podname\"=> \"/podname-g/\",\n      }, -1)\n      rule4 = create_rule_directive({\n        \"namespace\"=> \"/namespace-h/\",\n      }, 0)\n\n      conf = create_group_directive(tailing_group_pattern, '1m', rule1, rule2, rule3, rule4) + SINGLE_LINE_CONFIG\n      assert_nothing_raised do\n        create_driver(conf)\n      end\n    end\n\n    test \"limit should be greater than DEFAULT_LIMIT (-1)\" do\n      rule1 = create_rule_directive({\n        \"namespace\"=> \"/namespace-a/\",\n        \"podname\"=> \"/podname-[b|c]/\",\n      }, -100)\n      rule2 = create_rule_directive({\n        \"namespace\"=> \"/namespace-[d|e]/\",\n        \"podname\"=> \"/podname-f/\",\n      }, 50)\n      conf = create_group_directive(tailing_group_pattern, '1m', rule1, rule2) + SINGLE_LINE_CONFIG\n      assert_raise(RuntimeError) do\n        create_driver(conf)\n      end\n    end\n  end\n\n  sub_test_case \"group rules line limit resolution\" do\n    test \"valid\" do\n      rule1 = create_rule_directive({\n        \"namespace\"=> \"/namespace-a/\",\n        \"podname\"=> \"/podname-[b|c]/\",\n      }, 50)\n      rule2 = create_rule_directive({\n        \"podname\"=> \"/podname-[b|c]/\",\n      }, 400)\n      rule3 = create_rule_directive({\n        \"namespace\"=> \"/namespace-a/\",\n      }, 100)\n\n      conf = create_group_directive(tailing_group_pattern, '1m', rule3, rule1, rule2) + SINGLE_LINE_CONFIG\n      assert_nothing_raised do\n        d = create_driver(conf)\n        instance = d.instance\n\n        metadata = {\n          \"namespace\"=> \"namespace-a\",\n          \"podname\"=> \"podname-b\",\n        }\n        assert_equal(50, instance.find_group(metadata).limit)\n\n        metadata = {\n          \"namespace\" => \"namespace-a\",\n          \"podname\" => \"podname-c\",\n        }\n        assert_equal(50, instance.find_group(metadata).limit)\n\n        metadata = {\n          \"namespace\" => \"namespace-a\",\n          \"podname\" => \"podname-d\",\n        }\n        assert_equal(100, instance.find_group(metadata).limit)\n\n        metadata = {\n          \"namespace\" => \"namespace-f\",\n          \"podname\" => \"podname-b\",\n        }\n        assert_equal(400, instance.find_group(metadata).limit)\n\n        metadata = {\n          \"podname\" => \"podname-c\",\n        }\n        assert_equal(400, instance.find_group(metadata).limit)\n\n        assert_equal(-1, instance.find_group({}).limit)\n      end\n    end\n  end\n\n  sub_test_case \"files should be placed in groups\" do\n    test \"invalid regex pattern places files in default group\" do\n      rule1 = create_rule_directive({}, 100) ## limits default groups\n      conf = ROOT_CONFIG + DEBUG_LOG_LEVEL + create_group_directive(tailing_group_pattern, '1m', rule1) + create_path_element(\"test*.txt\") + SINGLE_LINE_CONFIG\n\n      d = create_driver(conf, false)\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/test1.txt\", 'w')\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/test2.txt\", 'w')\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/test3.txt\", 'w')\n\n      d.run do\n        ## checking default group_watcher's paths\n        instance = d.instance\n        key = instance.default_group_key\n\n        assert_equal(3, instance.log.logs.count{|a| a.match?(\"Cannot find group from metadata, Adding file in the default group\\n\")})\n        assert_equal(3, instance.group_watchers[key].size)\n        assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test1.txt'))\n        assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test2.txt'))\n        assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test3.txt'))\n      end\n    end\n\n    test \"valid regex pattern places file in their respective groups\" do\n      rule1 = create_rule_directive({\n        \"namespace\"=> \"/test-namespace1/\",\n        \"podname\"=> \"/test-podname1/\",\n      }, 100)\n      rule2 = create_rule_directive({\n        \"namespace\"=> \"/test-namespace1/\",\n      }, 200)\n      rule3 = create_rule_directive({\n        \"podname\"=> \"/test-podname2/\",\n      }, 300)\n      rule4 = create_rule_directive({}, 400)\n\n      path_element = create_path_element(\"test-podname*.log\")\n\n      conf = ROOT_CONFIG + create_group_directive(tailing_group_pattern, '1m', rule4, rule3, rule2, rule1) + path_element + SINGLE_LINE_CONFIG\n      d = create_driver(conf, false)\n\n      file1 = File.join(@tmp_dir, \"test-podname1_test-namespace1_test-container-15fabq.log\")\n      file2 = File.join(@tmp_dir, \"test-podname3_test-namespace1_test-container-15fabq.log\")\n      file3 = File.join(@tmp_dir, \"test-podname2_test-namespace2_test-container-15fabq.log\")\n      file4 = File.join(@tmp_dir, \"test-podname4_test-namespace3_test-container-15fabq.log\")\n\n      d.run do\n        Fluent::FileWrapper.open(file1, 'w')\n        Fluent::FileWrapper.open(file2, 'w')\n        Fluent::FileWrapper.open(file3, 'w')\n        Fluent::FileWrapper.open(file4, 'w')\n\n        instance = d.instance\n        assert_equal(100, instance.find_group_from_metadata(file1).limit)\n        assert_equal(200, instance.find_group_from_metadata(file2).limit)\n        assert_equal(300, instance.find_group_from_metadata(file3).limit)\n        assert_equal(400, instance.find_group_from_metadata(file4).limit)\n      end\n    end\n  end\n\n  sub_test_case \"singleline\" do\n    data(flat: SINGLE_LINE_CONFIG,\n         parse: PARSE_SINGLE_LINE_CONFIG)\n    def test_emit(data)\n      config = data\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d = create_driver(config)\n\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n          f.puts \"test3\\ntest4\"\n        }\n      end\n\n      events = d.events\n      assert_equal(true, events.length > 0)\n      assert_equal({\"message\" => \"test3\"}, events[0][2])\n      assert_equal({\"message\" => \"test4\"}, events[1][2])\n      assert(events[0][1].is_a?(Fluent::EventTime))\n      assert(events[1][1].is_a?(Fluent::EventTime))\n      assert_equal(1, d.emit_count)\n    end\n\n    def test_emit_with_emit_unmatched_lines_true\n      config = config_element(\"\", \"\", { \"format\" => \"/^(?<message>test.*)/\", \"emit_unmatched_lines\" => true })\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n\n      d = create_driver(config)\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n          f.puts \"test line 1\"\n          f.puts \"test line 2\"\n          f.puts \"bad line 1\"\n          f.puts \"test line 3\"\n        }\n      end\n\n      events = d.events\n      assert_equal(4, events.length)\n      assert_equal({\"message\" => \"test line 1\"}, events[0][2])\n      assert_equal({\"message\" => \"test line 2\"}, events[1][2])\n      assert_equal({\"unmatched_line\" => \"bad line 1\"}, events[2][2])\n      assert_equal({\"message\" => \"test line 3\"}, events[3][2])\n    end\n\n    data('flat 1' => [:flat, 1, 2],\n         'flat 10' => [:flat, 10, 1],\n         'parse 1' => [:parse, 1, 2],\n         'parse 10' => [:parse, 10, 1])\n    def test_emit_with_read_lines_limit(data)\n      config_style, limit, num_events = data\n      case config_style\n      when :flat\n        config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element(\"\", \"\", { \"read_lines_limit\" => limit })\n      when :parse\n        config = CONFIG_READ_FROM_HEAD + config_element(\"\", \"\", { \"read_lines_limit\" => limit }) + PARSE_SINGLE_LINE_CONFIG\n      end\n      d = create_driver(config)\n      msg = 'x' * 65000 # in_tail reads 65536 bytes at once.\n\n      d.run(expect_emits: num_events, timeout: 2) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n          f.puts msg\n          f.puts msg\n        }\n      end\n\n      events = d.events\n      assert_equal(true, events.length > 0)\n      assert_equal({\"message\" => msg}, events[0][2])\n      assert_equal({\"message\" => msg}, events[1][2])\n      assert num_events <= d.emit_count\n    end\n\n    sub_test_case \"log throttling per file\" do\n      teardown do\n        cleanup_file(\"#{@tmp_dir}/tail.txt\")\n      end\n\n      sub_test_case \"reads_bytes_per_second w/o throttled\" do\n        data(\"flat 65536 bytes, 2 events\"        => [:flat, 100, 65536, 2],\n             \"flat 65536 bytes, 2 events w/o stat watcher\" => [:flat_without_stat, 100, 65536, 2],\n             \"flat #{65536*10} bytes, 20 events\"  => [:flat, 100, (65536 * 10), 20],\n             \"flat #{65536*10} bytes, 20 events w/o stat watcher\"  => [:flat_without_stat, 100, (65536 * 10), 20],\n             \"parse #{65536*4} bytes, 8 events\"  => [:parse, 100, (65536 * 4), 8],\n             \"parse #{65536*4} bytes, 8 events w/o stat watcher\"  => [:parse_without_stat, 100, (65536 * 4), 8],\n             \"parse #{65536*10} bytes, 20 events\" => [:parse, 100, (65536 * 10), 20],\n             \"parse #{65536*10} bytes, 20 events w/o stat watcher\" => [:parse_without_stat, 100, (65536 * 10), 20],\n             \"flat 64k bytes with unit, 2 events\" => [:flat, 100, \"64k\", 2])\n        def test_emit_with_read_bytes_limit_per_second(data)\n          config_style, limit, limit_bytes, num_events = data\n          case config_style\n          when :flat\n            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element(\"\", \"\", { \"read_lines_limit\" => limit, \"read_bytes_limit_per_second\" => limit_bytes })\n          when :parse\n            config = CONFIG_READ_FROM_HEAD + config_element(\"\", \"\", { \"read_lines_limit\" => limit, \"read_bytes_limit_per_second\" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG\n          when :flat_without_stat\n            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + CONFIG_DISABLE_STAT_WATCHER + config_element(\"\", \"\", { \"read_lines_limit\" => limit, \"read_bytes_limit_per_second\" => limit_bytes })\n          when :parse_without_stat\n            config = CONFIG_READ_FROM_HEAD + CONFIG_DISABLE_STAT_WATCHER + config_element(\"\", \"\", { \"read_lines_limit\" => limit, \"read_bytes_limit_per_second\" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG\n          end\n\n          msg = 'x' * 65000 # in_tail reads 65536 bytes at once.\n          start_time = Fluent::Clock.now\n\n          d = create_driver(config)\n          d.run(expect_emits: 2) do\n            Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n              100.times do\n                f.puts msg\n              end\n            }\n          end\n\n          assert_true(Fluent::Clock.now - start_time > 1)\n          assert_equal(num_events, d.events.size)\n          assert_true(d.events.all? { |event| {\"message\" => msg} == event[2]})\n        end\n\n        def test_read_bytes_limit_precede_read_lines_limit\n          config = CONFIG_READ_FROM_HEAD +\n                   SINGLE_LINE_CONFIG +\n                   config_element(\"\", \"\", {\n                                    \"read_lines_limit\" => 1000,\n                                    \"read_bytes_limit_per_second\" => 8192\n                                  })\n          msg = 'abc' * 8\n          start_time = Fluent::Clock.now\n          d = create_driver(config)\n          d.run(expect_emits: 2) do\n            Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n              8000.times do\n                f.puts msg\n              end\n            }\n          end\n\n          assert_true(Fluent::Clock.now - start_time > 1)\n\n          # Line length: msg = 'abc' * 8 is 24 bytes, plus \\n (1 byte) = 25 bytes/line.\n          # Buffer size: BYTES_TO_READ is now 65,536 bytes.\n          # In this test environment, before the shutdown gracefully halts the read loop, in_tail manages to execute exactly two readpartial calls.\n          # Total bytes read = 65,536 * 2 = 131,072 bytes.\n          # 131,072 bytes / 25 bytes per line = 5242 lines (with a remainder of 22 bytes).\n          expected_read_lines = (65536 * 2) / ('abc' * 8 + \"\\n\").size\n          assert_equal(expected_read_lines, d.events.size)\n\n          assert_true(d.events.all? { |event| {\"message\" => msg} == event[2]})\n        end\n      end\n\n      sub_test_case \"reads_bytes_per_second w/ throttled already\" do\n        data(\"flat 65536 bytes\"  => [:flat, 100, 65536],\n             \"parse 65536 bytes\" => [:parse, 100, 65536])\n        def test_emit_with_read_bytes_limit_per_second(data)\n          config_style, limit, limit_bytes = data\n          case config_style\n          when :flat\n            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element(\"\", \"\", { \"read_lines_limit\" => limit, \"read_bytes_limit_per_second\" => limit_bytes })\n          when :parse\n            config = CONFIG_READ_FROM_HEAD + config_element(\"\", \"\", { \"read_lines_limit\" => limit, \"read_bytes_limit_per_second\" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG\n          end\n          d = create_driver(config)\n          msg = 'x' * 65000 # in_tail reads 65536 bytes at once.\n\n          mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|\n            require 'fluent/config/types'\n            limit_bytes_value = Fluent::Config.size_value(limit_bytes)\n            io_handler.instance_variable_set(:@number_bytes_read, limit_bytes_value)\n            if Fluent.linux?\n              mock.proxy(io_handler).handle_notify.at_least(5)\n            else\n              mock.proxy(io_handler).handle_notify.twice\n            end\n            io_handler\n          end\n\n          Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") do |f|\n            100.times do\n              f.puts msg\n            end\n          end\n\n          # We should not do shutdown here due to hard timeout.\n          d.run do\n            start_time = Fluent::Clock.now\n            while Fluent::Clock.now - start_time < 0.8 do\n              Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") do |f|\n                f.puts msg\n                f.flush\n              end\n              sleep 0.05\n            end\n          end\n\n          assert_equal([], d.events)\n        end\n      end\n\n      sub_test_case \"EOF with reads_bytes_per_second\" do\n        def test_longer_than_rotate_wait\n          limit_bytes = 8192\n          num_lines = 1024 * 3\n          msg = \"08bytes\"\n\n          Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") do |f|\n            f.write(\"#{msg}\\n\" * num_lines)\n          end\n\n          config = CONFIG_READ_FROM_HEAD +\n                   SINGLE_LINE_CONFIG +\n                   config_element(\"\", \"\", {\n                                    \"read_bytes_limit_per_second\" => limit_bytes,\n                                    \"rotate_wait\" => 0.1,\n                                    \"refresh_interval\" => 0.5,\n                                  })\n\n          rotated = false\n          d = create_driver(config)\n          d.run(timeout: 10) do\n            while d.events.size < num_lines do\n              if d.events.size > 0 && !rotated\n                cleanup_file(\"#{@tmp_dir}/tail.txt\")\n                FileUtils.touch(\"#{@tmp_dir}/tail.txt\")\n                rotated = true\n              end\n              sleep 0.3\n            end\n          end\n\n          assert_equal(num_lines,\n                       d.events.count do |event|\n                         event[2][\"message\"] == msg\n                       end)\n        end\n\n        def test_shorter_than_rotate_wait\n          limit_bytes = 8192\n          num_lines = 1024 * 2\n          msg = \"08bytes\"\n\n          Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") do |f|\n            f.write(\"#{msg}\\n\" * num_lines)\n          end\n\n          config = CONFIG_READ_FROM_HEAD +\n                   SINGLE_LINE_CONFIG +\n                   config_element(\"\", \"\", {\n                                    \"read_bytes_limit_per_second\" => limit_bytes,\n                                    \"rotate_wait\" => 2,\n                                    \"refresh_interval\" => 0.5,\n                                  })\n\n          start_time = Fluent::Clock.now\n          rotated = false\n          detached = false\n          d = create_driver(config)\n          mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|\n            mock.proxy(tw).detach(anything) do |v|\n              detached = true\n              v\n            end\n            tw\n          end.twice\n\n          d.run(timeout: 10) do\n            until detached do\n              if d.events.size > 0 && !rotated\n                cleanup_file(\"#{@tmp_dir}/tail.txt\")\n                FileUtils.touch(\"#{@tmp_dir}/tail.txt\")\n                rotated = true\n              end\n              sleep 0.3\n            end\n          end\n\n          assert_true(Fluent::Clock.now - start_time > 2)\n          assert_equal(num_lines,\n                       d.events.count do |event|\n                         event[2][\"message\"] == msg\n                       end)\n        end\n      end\n    end\n\n    data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,\n         parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)\n    def test_emit_with_read_from_head(data)\n      config = data\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d = create_driver(config)\n\n      d.run(expect_emits: 2) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n          f.puts \"test3\"\n          f.puts \"test4\"\n        }\n      end\n\n      events = d.events\n      assert(events.length > 0)\n      assert_equal({\"message\" => \"test1\"}, events[0][2])\n      assert_equal({\"message\" => \"test2\"}, events[1][2])\n      assert_equal({\"message\" => \"test3\"}, events[2][2])\n      assert_equal({\"message\" => \"test4\"}, events[3][2])\n    end\n\n    data(flat: CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,\n         parse: CONFIG_DISABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)\n    def test_emit_without_watch_timer(data)\n      config = data\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d = create_driver(config)\n\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n          f.puts \"test3\"\n          f.puts \"test4\"\n        }\n        # according to cool.io's stat_watcher.c, systems without inotify will use\n        # an \"automatic\" value, typically around 5 seconds\n      end\n\n      events = d.events\n      assert(events.length > 0)\n      assert_equal({\"message\" => \"test3\"}, events[0][2])\n      assert_equal({\"message\" => \"test4\"}, events[1][2])\n    end\n\n    # https://github.com/fluent/fluentd/pull/3541#discussion_r740197711\n    def test_watch_wildcard_path_without_watch_timer\n      omit \"need inotify\" unless Fluent.linux?\n\n      config = config_element(\"ROOT\", \"\", {\n                                \"path\" => \"#{@tmp_dir}/tail*.txt\",\n                                \"tag\" => \"t1\",\n                              })\n      config = config + CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d = create_driver(config, false)\n\n      d.run(expect_emits: 1, timeout: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n          f.puts \"test3\"\n          f.puts \"test4\"\n        }\n      end\n\n      assert_equal(\n        [\n          {\"message\" => \"test3\"},\n          {\"message\" => \"test4\"},\n        ],\n        d.events.collect { |event| event[2] })\n    end\n\n    data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG,\n         parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)\n    def test_emit_with_disable_stat_watcher(data)\n      config = data\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d = create_driver(config)\n\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n          f.puts \"test3\"\n          f.puts \"test4\"\n        }\n      end\n\n      events = d.events\n      assert(events.length > 0)\n      assert_equal({\"message\" => \"test3\"}, events[0][2])\n      assert_equal({\"message\" => \"test4\"}, events[1][2])\n    end\n\n    def test_always_read_from_head_on_detecting_a_new_file\n      d = create_driver(SINGLE_LINE_CONFIG)\n\n      d.run(expect_emits: 1, timeout: 3) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n          f.puts \"test1\\ntest2\\n\"\n        }\n      end\n\n      assert_equal(\n        [\n          {\"message\" => \"test1\"},\n          {\"message\" => \"test2\"},\n        ],\n        d.events.collect { |event| event[2] })\n    end\n  end\n\n  class TestWithSystem < self\n    include Fluent::SystemConfig::Mixin\n\n    OVERRIDE_FILE_PERMISSION = 0620\n    CONFIG_SYSTEM = %[\n      <system>\n        file_permission #{OVERRIDE_FILE_PERMISSION}\n      </system>\n    ]\n\n    def setup\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n      super\n      # Store default permission\n      @default_permission = system_config.instance_variable_get(:@file_permission)\n    end\n\n    def teardown\n      return if Fluent.windows?\n      super\n      # Restore default permission\n      system_config.instance_variable_set(:@file_permission, @default_permission)\n    end\n\n    def parse_system(text)\n      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')\n      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }\n    end\n\n    def test_emit_with_system\n      system_conf = parse_system(CONFIG_SYSTEM)\n      sc = Fluent::SystemConfig.new(system_conf)\n      Fluent::Engine.init(sc)\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d = create_driver\n\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n          f.puts \"test3\"\n          f.puts \"test4\"\n        }\n      end\n\n      events = d.events\n      assert_equal(true, events.length > 0)\n      assert_equal({\"message\" => \"test3\"}, events[0][2])\n      assert_equal({\"message\" => \"test4\"}, events[1][2])\n      assert(events[0][1].is_a?(Fluent::EventTime))\n      assert(events[1][1].is_a?(Fluent::EventTime))\n      assert_equal(1, d.emit_count)\n      pos = d.instance.instance_variable_get(:@pf_file)\n      mode = \"%o\" % File.stat(pos).mode\n      assert_equal OVERRIDE_FILE_PERMISSION, mode[-3, 3].to_i\n    end\n  end\n\n  sub_test_case \"rotate file\" do\n    def create_driver(conf = SINGLE_LINE_CONFIG)\n      config = common_config + conf\n      Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)\n    end\n\n    data(flat: SINGLE_LINE_CONFIG,\n         parse: PARSE_SINGLE_LINE_CONFIG)\n    def test_rotate_file(data)\n      config = data\n      events = sub_test_rotate_file(config, expect_emits: 2)\n      assert_equal(3.upto(6).collect { |i| {\"message\" => \"test#{i}\"} },\n                   events.collect { |event| event[2] })\n    end\n\n    data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,\n         parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)\n    def test_rotate_file_with_read_from_head(data)\n      config = data\n      events = sub_test_rotate_file(config, expect_records: 6)\n      assert_equal(1.upto(6).collect { |i| {\"message\" => \"test#{i}\"} },\n                   events.collect { |event| event[2] })\n    end\n\n    data(flat: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,\n         parse: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)\n    def test_rotate_file_with_open_on_every_update(data)\n      config = data\n      events = sub_test_rotate_file(config, expect_records: 6)\n      assert_equal(1.upto(6).collect { |i| {\"message\" => \"test#{i}\"} },\n                   events.collect { |event| event[2] })\n    end\n\n    data(flat: SINGLE_LINE_CONFIG,\n         parse: PARSE_SINGLE_LINE_CONFIG)\n    def test_rotate_file_with_write_old(data)\n      config = data\n      events = sub_test_rotate_file(config, expect_emits: 3) { |rotated_file|\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n        rotated_file.puts \"test7\"\n        rotated_file.puts \"test8\"\n        rotated_file.flush\n\n        sleep 1\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f|\n          f.puts \"test5\"\n          f.puts \"test6\"\n        }\n      }\n      # This test sometimes fails and it shows a potential bug of in_tail\n      # https://github.com/fluent/fluentd/issues/1434\n      assert_equal([3, 4, 7, 8, 5, 6].collect { |i| {\"message\" => \"test#{i}\"} },\n                   events.collect { |event| event[2] })\n    end\n\n    data(flat: SINGLE_LINE_CONFIG,\n         parse: PARSE_SINGLE_LINE_CONFIG)\n    def test_rotate_file_with_write_old_and_no_new_file(data)\n      config = data\n      events = sub_test_rotate_file(config, expect_emits: 2) { |rotated_file|\n        rotated_file.puts \"test7\"\n        rotated_file.puts \"test8\"\n        rotated_file.flush\n      }\n      assert_equal([3, 4, 7, 8].collect { |i| {\"message\" => \"test#{i}\"} },\n                   events.collect { |event| event[2] })\n    end\n\n    def sub_test_rotate_file(config = nil, expect_emits: nil, expect_records: nil, timeout: 5)\n      file = Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\")\n      file.puts \"test1\"\n      file.puts \"test2\"\n      file.flush\n\n      d = create_driver(config)\n      d.run(expect_emits: expect_emits, expect_records: expect_records, timeout: timeout) do\n        sleep(0.1) while d.instance.instance_variable_get(:@startup)\n        size = d.emit_count\n        file.puts \"test3\"\n        file.puts \"test4\"\n        file.flush\n        sleep(0.1) until d.emit_count >= size + 1\n        size = d.emit_count\n\n        if Fluent.windows?\n          file.close\n          FileUtils.mv(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail2.txt\", force: true)\n          file = Fluent::FileWrapper.open(\"#{@tmp_dir}/tail2.txt\", \"ab\")\n        else\n          FileUtils.mv(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail2.txt\")\n        end\n        if block_given?\n          yield file\n        else\n          Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n          sleep 1\n\n          Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f|\n            f.puts \"test5\"\n            f.puts \"test6\"\n          }\n        end\n      end\n\n      d.events\n    ensure\n      file.close if file && !file.closed?\n    end\n  end\n\n  def test_truncate_file\n    config = SINGLE_LINE_CONFIG\n    Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n      f.puts \"test1\"\n      f.puts \"test2\"\n      f.flush\n    }\n\n    d = create_driver(config)\n\n    d.run(expect_emits: 2) do\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n        f.puts \"test3\\ntest4\"\n        f.flush\n      }\n      waiting(2) { sleep 0.1 until d.events.length == 2 }\n      if Fluent.windows?\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f|\n          f.puts(\"test1\");\n        }\n      else\n        File.truncate(\"#{@tmp_dir}/tail.txt\", 6)\n      end\n    end\n\n    expected = {\n      emit_count: 2,\n      events: [\n        [Fluent::EventTime, {\"message\" => \"test3\"}],\n        [Fluent::EventTime, {\"message\" => \"test4\"}],\n        [Fluent::EventTime, {\"message\" => \"test1\"}],\n      ]\n    }\n    actual = {\n      emit_count: d.emit_count,\n      events: d.events.collect{|event| [event[1].class, event[2]]}\n    }\n    assert_equal(expected, actual)\n  end\n\n  def test_move_truncate_move_back\n    config = SINGLE_LINE_CONFIG\n    Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n      f.puts \"test1\"\n      f.puts \"test2\"\n    }\n\n    d = create_driver(config)\n\n    d.run(expect_emits: 1) do\n      if Fluent.windows?\n        FileUtils.mv(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail2.txt\", force: true)\n      else\n        FileUtils.mv(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail2.txt\")\n      end\n      sleep(1)\n      if Fluent.windows?\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail2.txt\", \"wb\") { |f|\n          f.puts(\"test1\");\n        }\n      else\n        File.truncate(\"#{@tmp_dir}/tail2.txt\", 6)\n      end\n      sleep(1)\n      if Fluent.windows?\n        FileUtils.mv(\"#{@tmp_dir}/tail2.txt\", \"#{@tmp_dir}/tail.txt\", force: true)\n      else\n        FileUtils.mv(\"#{@tmp_dir}/tail2.txt\", \"#{@tmp_dir}/tail.txt\")\n      end\n    end\n\n    events = d.events\n    assert_equal(1, events.length)\n    assert_equal({\"message\" => \"test1\"}, events[0][2])\n    assert(events[0][1].is_a?(Fluent::EventTime))\n    assert_equal(1, d.emit_count)\n  end\n\n  def test_lf\n    Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| }\n\n    d = create_driver\n\n    d.run(expect_emits: 1) do\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n        f.print \"test3\"\n      }\n      sleep 1\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n        f.puts \"test4\"\n      }\n    end\n\n    events = d.events\n    assert_equal(true, events.length > 0)\n    assert_equal({\"message\" => \"test3test4\"}, events[0][2])\n  end\n\n  def test_whitespace\n    Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| }\n\n    d = create_driver\n\n    d.run(expect_emits: 1) do\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n        f.puts \"    \"\t\t# 4 spaces\n        f.puts \"    4 spaces\"\n        f.puts \"4 spaces    \"\n        f.puts \"\t\"\t# tab\n        f.puts \"\ttab\"\n        f.puts \"tab\t\"\n      }\n    end\n\n    events = d.events\n    assert_equal(true, events.length > 0)\n    assert_equal({\"message\" => \"    \"}, events[0][2])\n    assert_equal({\"message\" => \"    4 spaces\"}, events[1][2])\n    assert_equal({\"message\" => \"4 spaces    \"}, events[2][2])\n    assert_equal({\"message\" => \"\t\"}, events[3][2])\n    assert_equal({\"message\" => \"\ttab\"}, events[4][2])\n    assert_equal({\"message\" => \"tab\t\"}, events[5][2])\n  end\n\n  data(\n    'flat default encoding' => [SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],\n    'flat explicit encoding config' => [SINGLE_LINE_CONFIG + config_element(\"\", \"\", { \"encoding\" => \"utf-8\" }), Encoding::UTF_8],\n    'parse default encoding' => [PARSE_SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],\n    'parse explicit encoding config' => [PARSE_SINGLE_LINE_CONFIG + config_element(\"\", \"\", { \"encoding\" => \"utf-8\" }), Encoding::UTF_8])\n  def test_encoding(data)\n    encoding_config, encoding = data\n\n    d = create_driver(CONFIG_READ_FROM_HEAD + encoding_config)\n\n    d.run(expect_emits: 1) do\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test\"\n      }\n    end\n\n    events = d.events\n    assert_equal(encoding, events[0][2]['message'].encoding)\n  end\n\n  def test_from_encoding\n    conf = config_element(\n      \"\", \"\", {\n        \"format\" => \"/(?<message>.*)/\",\n        \"read_from_head\" => \"true\",\n        \"from_encoding\" => \"cp932\",\n        \"encoding\" => \"utf-8\"\n      })\n    d = create_driver(conf)\n    cp932_message = \"\\x82\\xCD\\x82\\xEB\\x81\\x5B\\x82\\xED\\x81\\x5B\\x82\\xE9\\x82\\xC7\".force_encoding(Encoding::CP932)\n    utf8_message = cp932_message.encode(Encoding::UTF_8)\n\n    d.run(expect_emits: 1) do\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"w:cp932\") {|f|\n        f.puts cp932_message\n      }\n    end\n\n    events = d.events\n    assert_equal(utf8_message, events[0][2]['message'])\n    assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)\n  end\n\n  def test_from_encoding_utf16\n    conf = config_element(\n      \"\", \"\", {\n        \"format\" => \"/(?<message>.*)/\",\n        \"read_from_head\" => \"true\",\n        \"from_encoding\" => \"utf-16le\",\n        \"encoding\" => \"utf-8\"\n      })\n    d = create_driver(conf)\n    utf16_message = \"\\u306F\\u308D\\u30FC\\u308F\\u30FC\\u308B\\u3069\\n\".encode(Encoding::UTF_16LE)\n    utf8_message = utf16_message.encode(Encoding::UTF_8).strip\n\n    d.run(expect_emits: 1) do\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"w:utf-16le\") { |f|\n        f.write utf16_message\n      }\n    end\n\n    events = d.events\n    assert_equal(utf8_message, events[0][2]['message'])\n    assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)\n  end\n\n  def test_encoding_with_bad_character\n    conf = config_element(\n      \"\", \"\", {\n        \"format\" => \"/(?<message>.*)/\",\n        \"read_from_head\" => \"true\",\n        \"from_encoding\" => \"ASCII-8BIT\",\n        \"encoding\" => \"utf-8\"\n      })\n    d = create_driver(conf)\n\n    d.run(expect_emits: 1) do\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"w\") { |f|\n        f.write \"te\\x86st\\n\"\n      }\n    end\n\n    events = d.events\n    assert_equal(\"te\\uFFFDst\", events[0][2]['message'])\n    assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)\n  end\n\n  def test_encoding_for_regular_expression_parsing\n    conf = CONFIG_READ_FROM_HEAD +\n           config_element(\"\", \"\" , { \"encoding\" => \"utf-8\" },\n                          [config_element(\"parse\", \"\", { \"@type\" => \"/^あ(?<name>.*)お$/\" })])\n\n    d = create_driver(conf)\n    d.run(expect_emits: 1, timeout: 5) do\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f|\n        f.puts \"あいうえお\"\n      }\n    end\n    events = d.events\n    assert_equal(true, events.length > 0)\n    assert_equal({\"name\" => \"いうえ\"}, events[0][2])\n  end\n\n  sub_test_case \"multiline\" do\n    data(flat: MULTILINE_CONFIG,\n         parse: PARSE_MULTILINE_CONFIG)\n    def test_multiline(data)\n      config = data\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n\n      d = create_driver(config)\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f|\n          f.puts \"f test1\"\n          f.puts \"s test2\"\n          f.puts \"f test3\"\n          f.puts \"f test4\"\n          f.puts \"s test5\"\n          f.puts \"s test6\"\n          f.puts \"f test7\"\n          f.puts \"s test8\"\n        }\n      end\n\n      events = d.events\n      assert_equal(4, events.length)\n      assert_equal({\"message1\" => \"test2\", \"message2\" => \"test3\", \"message3\" => \"test4\"}, events[0][2])\n      assert_equal({\"message1\" => \"test5\"}, events[1][2])\n      assert_equal({\"message1\" => \"test6\", \"message2\" => \"test7\"}, events[2][2])\n      assert_equal({\"message1\" => \"test8\"}, events[3][2])\n    end\n\n    data(flat: MULTILINE_CONFIG,\n         parse: PARSE_MULTILINE_CONFIG)\n    def test_multiline_with_emit_unmatched_lines_true(data)\n      config = data + config_element(\"\", \"\", { \"emit_unmatched_lines\" => true })\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n\n      d = create_driver(config)\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f|\n          f.puts \"f test1\"\n          f.puts \"s test2\"\n          f.puts \"f test3\"\n          f.puts \"f test4\"\n          f.puts \"s test5\"\n          f.puts \"s test6\"\n          f.puts \"f test7\"\n          f.puts \"s test8\"\n        }\n      end\n\n      events = d.events\n      assert_equal(5, events.length)\n      assert_equal({\"unmatched_line\" => \"f test1\"}, events[0][2])\n      assert_equal({\"message1\" => \"test2\", \"message2\" => \"test3\", \"message3\" => \"test4\"}, events[1][2])\n      assert_equal({\"message1\" => \"test5\"}, events[2][2])\n      assert_equal({\"message1\" => \"test6\", \"message2\" => \"test7\"}, events[3][2])\n      assert_equal({\"message1\" => \"test8\"}, events[4][2])\n    end\n\n    data(\n      flat: MULTILINE_CONFIG_WITH_NEWLINE,\n      parse: PARSE_MULTILINE_CONFIG_WITH_NEWLINE)\n    def test_multiline_with_emit_unmatched_lines2(data)\n      config = data + config_element(\"\", \"\", { \"emit_unmatched_lines\" => true })\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n\n      d = create_driver(config)\n      d.run(expect_emits: 0, timeout: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f|\n          f.puts \"s test0\"\n          f.puts \"f test1\"\n          f.puts \"f test2\"\n\n          f.puts \"f test3\"\n\n          f.puts \"s test4\"\n          f.puts \"f test5\"\n          f.puts \"f test6\"\n        }\n      end\n\n      events = d.events\n      assert_equal({\"message1\" => \"test0\", \"message2\" => \"test1\", \"message3\" => \"test2\"}, events[0][2])\n      assert_equal({ 'unmatched_line' => \"f test3\" }, events[1][2])\n      assert_equal({\"message1\" => \"test4\", \"message2\" => \"test5\", \"message3\" => \"test6\"}, events[2][2])\n    end\n\n    data(flat: MULTILINE_CONFIG,\n         parse: PARSE_MULTILINE_CONFIG)\n    def test_multiline_with_flush_interval(data)\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n\n      config = data + config_element(\"\", \"\", { \"multiline_flush_interval\" => \"2s\" })\n      d = create_driver(config)\n\n      assert_equal(2, d.instance.multiline_flush_interval)\n\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f|\n          f.puts \"f test1\"\n          f.puts \"s test2\"\n          f.puts \"f test3\"\n          f.puts \"f test4\"\n          f.puts \"s test5\"\n          f.puts \"s test6\"\n          f.puts \"f test7\"\n          f.puts \"s test8\"\n        }\n      end\n\n      events = d.events\n      assert_equal(4, events.length)\n      assert_equal({\"message1\" => \"test2\", \"message2\" => \"test3\", \"message3\" => \"test4\"}, events[0][2])\n      assert_equal({\"message1\" => \"test5\"}, events[1][2])\n      assert_equal({\"message1\" => \"test6\", \"message2\" => \"test7\"}, events[2][2])\n      assert_equal({\"message1\" => \"test8\"}, events[3][2])\n    end\n\n    data(\n      'flat default encoding' => [MULTILINE_CONFIG, Encoding::ASCII_8BIT],\n      'flat explicit encoding config' => [MULTILINE_CONFIG + config_element(\"\", \"\", { \"encoding\" => \"utf-8\" }), Encoding::UTF_8],\n      'parse default encoding' => [PARSE_MULTILINE_CONFIG, Encoding::ASCII_8BIT],\n      'parse explicit encoding config' => [PARSE_MULTILINE_CONFIG + config_element(\"\", \"\", { \"encoding\" => \"utf-8\" }), Encoding::UTF_8])\n    def test_multiline_encoding_of_flushed_record(data)\n      encoding_config, encoding = data\n\n      config = config_element(\"\", \"\", {\n                                \"multiline_flush_interval\" => \"2s\",\n                                \"read_from_head\" => \"true\",\n                              })\n      d = create_driver(config + encoding_config)\n\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f|\n          f.puts \"s test\"\n        }\n      end\n      events = d.events\n      assert_equal(1, events.length)\n      assert_equal(encoding, events[0][2]['message1'].encoding)\n    end\n\n    def test_multiline_from_encoding_of_flushed_record\n      conf = MULTILINE_CONFIG + config_element(\n               \"\", \"\",\n               {\n                 \"multiline_flush_interval\" => \"1s\",\n                 \"read_from_head\" => \"true\",\n                 \"from_encoding\" => \"cp932\",\n                 \"encoding\" => \"utf-8\"\n               })\n      d = create_driver(conf)\n\n      cp932_message = \"s \\x82\\xCD\\x82\\xEB\\x81\\x5B\\x82\\xED\\x81\\x5B\\x82\\xE9\\x82\\xC7\".force_encoding(Encoding::CP932)\n      utf8_message = \"\\x82\\xCD\\x82\\xEB\\x81\\x5B\\x82\\xED\\x81\\x5B\\x82\\xE9\\x82\\xC7\".encode(Encoding::UTF_8, Encoding::CP932)\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"w:cp932\") { |f|\n          f.puts cp932_message\n        }\n      end\n\n      events = d.events\n      assert_equal(1, events.length)\n      assert_equal(utf8_message, events[0][2]['message1'])\n      assert_equal(Encoding::UTF_8, events[0][2]['message1'].encoding)\n    end\n\n    data(flat: config_element(\n           \"\", \"\", {\n             \"format\" => \"multiline\",\n             \"format1\" => \"/^s (?<message1>[^\\\\n]+)\\\\n?/\",\n             \"format2\" => \"/(f (?<message2>[^\\\\n]+)\\\\n?)?/\",\n             \"format3\" => \"/(f (?<message3>.*))?/\",\n             \"format_firstline\" => \"/^[s]/\"\n           }),\n         parse: config_element(\n           \"\", \"\", {},\n           [config_element(\"parse\", \"\", {\n                             \"@type\" => \"multiline\",\n                             \"format1\" => \"/^s (?<message1>[^\\\\n]+)\\\\n?/\",\n                             \"format2\" => \"/(f (?<message2>[^\\\\n]+)\\\\n?)?/\",\n                             \"format3\" => \"/(f (?<message3>.*))?/\",\n                             \"format_firstline\" => \"/^[s]/\"\n                           })\n           ])\n        )\n    def test_multiline_with_multiple_formats(data)\n      config = data\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n\n      d = create_driver(config)\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f|\n          f.puts \"f test1\"\n          f.puts \"s test2\"\n          f.puts \"f test3\"\n          f.puts \"f test4\"\n          f.puts \"s test5\"\n          f.puts \"s test6\"\n          f.puts \"f test7\"\n          f.puts \"s test8\"\n        }\n      end\n\n      events = d.events\n      assert(events.length > 0)\n      assert_equal({\"message1\" => \"test2\", \"message2\" => \"test3\", \"message3\" => \"test4\"}, events[0][2])\n      assert_equal({\"message1\" => \"test5\"}, events[1][2])\n      assert_equal({\"message1\" => \"test6\", \"message2\" => \"test7\"}, events[2][2])\n      assert_equal({\"message1\" => \"test8\"}, events[3][2])\n    end\n\n    data(flat: config_element(\n           \"\", \"\", {\n             \"format\" => \"multiline\",\n             \"format1\" => \"/^[s|f] (?<message>.*)/\",\n             \"format_firstline\" => \"/^[s]/\"\n           }),\n         parse: config_element(\n           \"\", \"\", {},\n           [config_element(\"parse\", \"\", {\n                             \"@type\" => \"multiline\",\n                             \"format1\" => \"/^[s|f] (?<message>.*)/\",\n                             \"format_firstline\" => \"/^[s]/\"\n                           })\n           ])\n        )\n    def test_multilinelog_with_multiple_paths(data)\n      files = [\"#{@tmp_dir}/tail1.txt\", \"#{@tmp_dir}/tail2.txt\"]\n      files.each { |file| Fluent::FileWrapper.open(file, \"wb\") { |f| } }\n\n      config = data + config_element(\"\", \"\", {\n                                       \"path\" => \"#{files[0]},#{files[1]}\",\n                                       \"tag\" => \"t1\",\n                                     })\n      d = create_driver(config, false)\n      d.run(expect_emits: 2) do\n        files.each do |file|\n          Fluent::FileWrapper.open(file, 'ab') { |f|\n            f.puts \"f #{file} line should be ignored\"\n            f.puts \"s test1\"\n            f.puts \"f test2\"\n            f.puts \"f test3\"\n            f.puts \"s test4\"\n          }\n        end\n      end\n\n      events = d.events\n      assert_equal({\"message\" => \"test1\\nf test2\\nf test3\"}, events[0][2])\n      assert_equal({\"message\" => \"test1\\nf test2\\nf test3\"}, events[1][2])\n      # \"test4\" events are here because these events are flushed at shutdown phase\n      assert_equal({\"message\" => \"test4\"}, events[2][2])\n      assert_equal({\"message\" => \"test4\"}, events[3][2])\n    end\n\n    data(flat: config_element(\"\", \"\", {\n                                \"format\" => \"multiline\",\n                                \"format1\" => \"/(?<var1>foo \\\\d)\\\\n/\",\n                                \"format2\" => \"/(?<var2>bar \\\\d)\\\\n/\",\n                                \"format3\" => \"/(?<var3>baz \\\\d)/\"\n                              }),\n         parse: config_element(\n           \"\", \"\", {},\n           [config_element(\"parse\", \"\", {\n                             \"@type\" => \"multiline\",\n                             \"format1\" => \"/(?<var1>foo \\\\d)\\\\n/\",\n                             \"format2\" => \"/(?<var2>bar \\\\d)\\\\n/\",\n                             \"format3\" => \"/(?<var3>baz \\\\d)/\"\n                           })\n           ])\n        )\n    def test_multiline_without_firstline(data)\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n\n      config = data\n      d = create_driver(config)\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f|\n          f.puts \"foo 1\"\n          f.puts \"bar 1\"\n          f.puts \"baz 1\"\n          f.puts \"foo 2\"\n          f.puts \"bar 2\"\n          f.puts \"baz 2\"\n        }\n      end\n\n      events = d.events\n      assert_equal(2, events.length)\n      assert_equal({\"var1\" => \"foo 1\", \"var2\" => \"bar 1\", \"var3\" => \"baz 1\"}, events[0][2])\n      assert_equal({\"var1\" => \"foo 2\", \"var2\" => \"bar 2\", \"var3\" => \"baz 2\"}, events[1][2])\n    end\n  end\n\n  sub_test_case \"path\" do\n    # * path test\n    # TODO: Clean up tests\n    def test_expand_paths\n      ex_paths = [\n        create_target_info('test/plugin/data/2010/01/20100102-030405.log'),\n        create_target_info('test/plugin/data/log/foo/bar.log'),\n        create_target_info('test/plugin/data/log/test.log')\n      ]\n      plugin = create_driver(ex_config, false).instance\n      flexstub(Time) do |timeclass|\n        timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))\n        assert_equal(ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })\n      end\n\n      # Test exclusion\n      exclude_config = ex_config + config_element(\"\", \"\", { \"exclude_path\" => %Q([\"#{ex_paths.last.path}\"]) })\n      plugin = create_driver(exclude_config, false).instance\n      assert_equal(ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })\n    end\n\n    sub_test_case \"expand_paths with glob\" do |data|\n      sub_test_case \"extended_glob\" do\n        data(\"curly braces\"          => [true, \"always\", \"test/plugin/data/log_numeric/{0,1}*.log\"],\n             \"square brackets\"       => [true, \"always\", \"test/plugin/data/log_numeric/[0-1][2-4].log\"],\n             \"asterisk\"              => [true, \"always\", \"test/plugin/data/log/*.log\"],\n             \"one character matcher\" => [true, \"always\", \"test/plugin/data/log/tes?.log\"],\n            )\n        def test_expand_paths_with_use_glob_p_and_almost_set_of_patterns\n          result, option, path = data\n          config = config_element(\"\", \"\", {\n                                    \"tag\" => \"tail\",\n                                    \"path\" => path,\n                                    \"format\" => \"none\",\n                                    \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                                    \"read_from_head\" => true,\n                                    \"refresh_interval\" => 30,\n                                    \"glob_policy\" => option,\n                                    \"path_delimiter\" => \"|\",\n                                    \"rotate_wait\" => \"#{EX_ROTATE_WAIT}s\",\n                                    \"follow_inodes\" => \"#{EX_FOLLOW_INODES}\",\n                                  })\n          plugin = create_driver(config, false).instance\n          assert_equal(result, !!plugin.use_glob?(path))\n        end\n\n        data(\"curly braces\"          => [true, false, \"extended\", \"test/plugin/data/log_numeric/{0,1}*.log\"],\n             \"square brackets\"       => [false, true, \"extended\", \"test/plugin/data/log_numeric/[0-1][2-4].log\"],\n             \"asterisk\"              => [false, true, \"extended\", \"test/plugin/data/log/*.log\"],\n             \"one character matcher\" => [false, true, \"extended\", \"test/plugin/data/log/tes?.log\"],\n            )\n        def test_expand_paths_with_use_glob_p\n          emit_exception_p, result, option, path = data\n          config = config_element(\"\", \"\", {\n                                    \"tag\" => \"tail\",\n                                    \"path\" => path,\n                                    \"format\" => \"none\",\n                                    \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                                    \"read_from_head\" => true,\n                                    \"refresh_interval\" => 30,\n                                    \"glob_policy\" => option,\n                                    \"rotate_wait\" => \"#{EX_ROTATE_WAIT}s\",\n                                    \"follow_inodes\" => \"#{EX_FOLLOW_INODES}\",\n                                  })\n          if emit_exception_p\n            assert_raise(Fluent::ConfigError) do\n              plugin = create_driver(config, false).instance\n            end\n          else\n            plugin = create_driver(config, false).instance\n            assert_equal(result, !!plugin.use_glob?(path))\n          end\n        end\n      end\n\n      sub_test_case \"only_use_backward_compatible\" do\n        data(\"square brackets\"       => [false, \"backward_compatible\", \"test/plugin/data/log_numeric/[0-1][2-4].log\"],\n             \"asterisk\"              => [true,  \"backward_compatible\", \"test/plugin/data/log/*.log\"],\n             \"one character matcher\" => [false, \"backward_compatible\", \"test/plugin/data/log/tes?.log\"],\n            )\n        def test_expand_paths_with_use_glob_p\n          result, option, path = data\n          config = config_element(\"\", \"\", {\n                                    \"tag\" => \"tail\",\n                                    \"path\" => path,\n                                    \"format\" => \"none\",\n                                    \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                                    \"read_from_head\" => true,\n                                    \"refresh_interval\" => 30,\n                                    \"glob_policy\" => option,\n                                    \"rotate_wait\" => \"#{EX_ROTATE_WAIT}s\",\n                                    \"follow_inodes\" => \"#{EX_FOLLOW_INODES}\",\n                                  })\n          plugin = create_driver(config, false).instance\n          assert_equal(result, !!plugin.use_glob?(path))\n        end\n      end\n    end\n\n    def ex_config_with_brackets\n      config_element(\"\", \"\", {\n                     \"tag\" => \"tail\",\n                     \"path\" => \"test/plugin/data/log_numeric/[0-1][2-4].log\",\n                     \"format\" => \"none\",\n                     \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                     \"read_from_head\" => true,\n                     \"refresh_interval\" => 30,\n                     \"glob_policy\" => \"extended\",\n                     \"rotate_wait\" => \"#{EX_ROTATE_WAIT}s\",\n                     \"follow_inodes\" => \"#{EX_FOLLOW_INODES}\",\n                   })\n    end\n\n    def test_config_with_always_with_default_delimiter\n      assert_raise(Fluent::ConfigError) do\n        config = config_element(\"\", \"\", {\n                         \"tag\" => \"tail\",\n                         \"path\" => \"test/plugin/data/log_numeric/[0-1][2-4].log\",\n                         \"format\" => \"none\",\n                         \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                         \"read_from_head\" => true,\n                         \"refresh_interval\" => 30,\n                         \"glob_policy\" => \"always\",\n                         \"rotate_wait\" => \"#{EX_ROTATE_WAIT}s\",\n                         \"follow_inodes\" => \"#{EX_FOLLOW_INODES}\",\n                       })\n\n        create_driver(config, false).instance\n      end\n    end\n\n    def test_config_with_always_with_custom_delimiter\n      assert_nothing_raised do\n        config = config_element(\"\", \"\", {\n                         \"tag\" => \"tail\",\n                         \"path\" => \"test/plugin/data/log_numeric/[0-1][2-4].log\",\n                         \"format\" => \"none\",\n                         \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                         \"read_from_head\" => true,\n                         \"refresh_interval\" => 30,\n                         \"glob_policy\" => \"always\",\n                         \"path_delimiter\" => \"|\",\n                         \"rotate_wait\" => \"#{EX_ROTATE_WAIT}s\",\n                         \"follow_inodes\" => \"#{EX_FOLLOW_INODES}\",\n                       })\n\n        create_driver(config, false).instance\n      end\n    end\n\n    def test_expand_paths_with_brackets\n      expanded_paths = [\n        create_target_info('test/plugin/data/log_numeric/01.log'),\n        create_target_info('test/plugin/data/log_numeric/02.log'),\n        create_target_info('test/plugin/data/log_numeric/12.log'),\n        create_target_info('test/plugin/data/log_numeric/14.log'),\n      ]\n\n      plugin = create_driver(ex_config_with_brackets, false).instance\n      assert_equal(expanded_paths - [expanded_paths.first], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })\n    end\n\n    def test_expand_paths_with_duplicate_configuration\n      expanded_paths = [\n        create_target_info('test/plugin/data/log/foo/bar.log'),\n        create_target_info('test/plugin/data/log/test.log')\n      ]\n      duplicate_config = ex_config.dup\n      duplicate_config[\"path\"]=\"test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log\"\n      plugin = create_driver(ex_config, false).instance\n      assert_equal(expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })\n    end\n\n    def test_expand_paths_with_timezone\n      ex_paths = [\n        create_target_info('test/plugin/data/2010/01/20100102-030405.log'),\n        create_target_info('test/plugin/data/log/foo/bar.log'),\n        create_target_info('test/plugin/data/log/test.log')\n      ]\n      ['Asia/Taipei', '+08'].each do |tz_type|\n        taipei_config = ex_config + config_element(\"\", \"\", {\"path_timezone\" => tz_type})\n        plugin = create_driver(taipei_config, false).instance\n\n        # Test exclude\n        exclude_config = taipei_config + config_element(\"\", \"\", { \"exclude_path\" => %Q([\"test/plugin/**/%Y%m%d-%H%M%S.log\"]) })\n        exclude_plugin = create_driver(exclude_config, false).instance\n\n        with_timezone('utc') do\n          flexstub(Time) do |timeclass|\n            # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)\n            timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))\n\n            assert_equal(ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })\n            assert_equal(ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })\n          end\n        end\n      end\n    end\n\n    def test_log_file_without_extension\n      expected_files = [\n        create_target_info('test/plugin/data/log/bar'),\n        create_target_info('test/plugin/data/log/foo/bar.log'),\n        create_target_info('test/plugin/data/log/foo/bar2'),\n        create_target_info('test/plugin/data/log/test.log')\n      ]\n\n      config = config_element(\"\", \"\", {\n        \"tag\" => \"tail\",\n        \"path\" => \"test/plugin/data/log/**/*\",\n        \"format\" => \"none\",\n        \"pos_file\" => \"#{@tmp_dir}/tail.pos\"\n      })\n\n      plugin = create_driver(config, false).instance\n      assert_equal(expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })\n    end\n\n    def test_unwatched_files_should_be_removed\n      config = config_element(\"\", \"\", {\n                                \"tag\" => \"tail\",\n                                \"path\" => \"#{@tmp_dir}/*.txt\",\n                                \"format\" => \"none\",\n                                \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                                \"read_from_head\" => true,\n                                \"refresh_interval\" => 1,\n                              })\n      d = create_driver(config, false)\n      d.end_if { d.instance.instance_variable_get(:@tails).keys.size >= 1 }\n      d.run(expect_emits: 1, shutdown: false) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f| f.puts \"test3\\n\" }\n      end\n\n      cleanup_file(\"#{@tmp_dir}/tail.txt\")\n      waiting(20) { sleep 0.1 until Dir.glob(\"#{@tmp_dir}/*.txt\").size == 0 } # Ensure file is deleted on Windows\n      waiting(5) { sleep 0.1 until d.logs.last.include?(\"detected rotation\") }\n      waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }\n\n      assert_equal(\n        {\n          files: [],\n          tails: []\n        },\n        {\n          files: Dir.glob(\"#{@tmp_dir}/*.txt\"),\n          tails: d.instance.instance_variable_get(:@tails).keys\n        }\n      )\n    ensure\n      d.instance_shutdown if d&.instance\n      cleanup_directory(@tmp_dir)\n    end\n\n    def count_timer_object\n      num = 0\n      ObjectSpace.each_object(Fluent::PluginHelper::Timer::TimerWatcher) { |obj|\n        num += 1\n      }\n      num\n    end\n  end\n\n  sub_test_case \"path w/ Linux capability\" do\n    def capability_enabled?\n      if Fluent.linux?\n        begin\n          require 'capng'\n          true\n        rescue LoadError\n          false\n        end\n      else\n        false\n      end\n    end\n\n    setup do\n      omit \"This environment is not enabled Linux capability handling feature\" unless capability_enabled?\n\n      @capng = CapNG.new(:current_process)\n      flexstub(Fluent::Capability) do |klass|\n        klass.should_receive(:new).with(:current_process).and_return(@capng)\n      end\n    end\n\n    data(\"dac_read_search\" => [:dac_read_search, true,  1],\n         \"dac_override\"    => [:dac_override,    true,  1],\n         \"chown\"           => [:chown,           false, 0],\n        )\n    test \"with partially elevated privileges\" do |data|\n      cap, result, readable_paths = data\n      @capng.update(:add, :effective, cap)\n\n      d = create_driver(\n        config_element(\"ROOT\", \"\", {\n                            \"path\" => \"/var/log/ker*.log\", # Use /var/log/kern.log\n                            \"tag\" => \"t1\",\n                            \"rotate_wait\" => \"2s\"\n                       }) + PARSE_SINGLE_LINE_CONFIG, false)\n\n      assert_equal(readable_paths, d.instance.expand_paths.length)\n      assert_equal(result, d.instance.have_read_capability?)\n    end\n  end\n\n  def test_pos_file_dir_creation\n    config = config_element(\"\", \"\", {\n      \"tag\" => \"tail\",\n      \"path\" => \"#{@tmp_dir}/*.txt\",\n      \"format\" => \"none\",\n      \"pos_file\" => \"#{@tmp_dir}/pos/tail.pos\",\n      \"read_from_head\" => true,\n      \"refresh_interval\" => 1\n    })\n\n    assert_path_not_exist(\"#{@tmp_dir}/pos\")\n    d = create_driver(config, false)\n    d.run\n    assert_path_exist(\"#{@tmp_dir}/pos\")\n    assert_equal('755', File.stat(\"#{@tmp_dir}/pos\").mode.to_s(8)[-3, 3])\n  ensure\n    cleanup_directory(@tmp_dir)\n  end\n\n  def test_pos_file_dir_creation_with_system_dir_permission\n    config = config_element(\"\", \"\", {\n      \"tag\" => \"tail\",\n      \"path\" => \"#{@tmp_dir}/*.txt\",\n      \"format\" => \"none\",\n      \"pos_file\" => \"#{@tmp_dir}/pos/tail.pos\",\n      \"read_from_head\" => true,\n      \"refresh_interval\" => 1\n    })\n\n    assert_path_not_exist(\"#{@tmp_dir}/pos\")\n\n    Fluent::SystemConfig.overwrite_system_config({ \"dir_permission\" => \"744\" }) do\n      d = create_driver(config, false)\n      d.run\n    end\n\n    assert_path_exist(\"#{@tmp_dir}/pos\")\n    if Fluent.windows?\n      assert_equal('755', File.stat(\"#{@tmp_dir}/pos\").mode.to_s(8)[-3, 3])\n    else\n      assert_equal('744', File.stat(\"#{@tmp_dir}/pos\").mode.to_s(8)[-3, 3])\n    end\n  ensure\n    cleanup_directory(@tmp_dir)\n  end\n\n  def test_z_refresh_watchers\n    ex_paths = [\n      create_target_info('test/plugin/data/2010/01/20100102-030405.log'),\n      create_target_info('test/plugin/data/log/foo/bar.log'),\n      create_target_info('test/plugin/data/log/test.log'),\n    ]\n    plugin = create_driver(ex_config, false).instance\n    sio = StringIO.new\n    plugin.instance_eval do\n      @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)\n      @loop = Coolio::Loop.new\n    end\n\n    Timecop.freeze(2010, 1, 2, 3, 4, 5) do\n      ex_paths.each do |target_info|\n        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once\n      end\n\n      plugin.refresh_watchers\n    end\n\n    path = 'test/plugin/data/2010/01/20100102-030405.log'\n    mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[path], Fluent::FileWrapper.stat(path).ino)\n\n    Timecop.freeze(2010, 1, 2, 3, 4, 6) do\n      path = \"test/plugin/data/2010/01/20100102-030406.log\"\n      inode = Fluent::FileWrapper.stat(path).ino\n      target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)\n      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once\n      plugin.refresh_watchers\n\n      flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|\n        watcherclass.should_receive(:new).never\n        plugin.refresh_watchers\n      end\n    end\n  end\n\n  sub_test_case \"refresh of pos file\" do\n    test 'type of pos_file_compaction_interval is time' do\n      tail = {\n        \"tag\" => \"tail\",\n        \"path\" => \"#{@tmp_dir}/*.txt\",\n        \"format\" => \"none\",\n        \"pos_file\" => \"#{@tmp_dir}/pos/tail.pos\",\n        \"refresh_interval\" => 1,\n        \"read_from_head\" => true,\n        'pos_file_compaction_interval' => '24h',\n      }\n      config = config_element(\"\", \"\", tail)\n      d = create_driver(config, false)\n      mock(d.instance).timer_execute(:in_tail_refresh_watchers, 1.0).once\n      mock(d.instance).timer_execute(:in_tail_refresh_compact_pos_file, 60 * 60 * 24).once\n      d.run                     # call start\n    end\n  end\n\n  sub_test_case \"receive_lines\" do\n    DummyWatcher = Struct.new(\"DummyWatcher\", :tag)\n\n    def test_tag\n      d = create_driver(ex_config, false)\n      d.run {}\n      plugin = d.instance\n      mock(plugin.router).emit_stream('tail', anything).once\n      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))\n    end\n\n    def test_tag_with_only_star\n      config = config_element(\"\", \"\", {\n                                \"tag\" => \"*\",\n                                \"path\" => \"test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log\",\n                                \"format\" => \"none\",\n                                \"read_from_head\" => true\n                              })\n      d = create_driver(config, false)\n      d.run {}\n      plugin = d.instance\n      mock(plugin.router).emit_stream('foo.bar.log', anything).once\n      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))\n    end\n\n    def test_tag_prefix\n      config = config_element(\"\", \"\", {\n                                \"tag\" => \"pre.*\",\n                                \"path\" => \"test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log\",\n                                \"format\" => \"none\",\n                                \"read_from_head\" => true\n                              })\n      d = create_driver(config, false)\n      d.run {}\n      plugin = d.instance\n      mock(plugin.router).emit_stream('pre.foo.bar.log', anything).once\n      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))\n    end\n\n    def test_tag_suffix\n      config = config_element(\"\", \"\", {\n                                \"tag\" => \"*.post\",\n                                \"path\" => \"test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log\",\n                                \"format\" => \"none\",\n                                \"read_from_head\" => true\n                              })\n      d = create_driver(config, false)\n      d.run {}\n      plugin = d.instance\n      mock(plugin.router).emit_stream('foo.bar.log.post', anything).once\n      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))\n    end\n\n    def test_tag_prefix_and_suffix\n      config = config_element(\"\", \"\", {\n                                \"tag\" => \"pre.*.post\",\n                                \"path\" => \"test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log\",\n                                \"format\" => \"none\",\n                                \"read_from_head\" => true\n                              })\n      d = create_driver(config, false)\n      d.run {}\n      plugin = d.instance\n      mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once\n      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))\n    end\n\n    def test_tag_prefix_and_suffix_ignore\n      config = config_element(\"\", \"\", {\n                                \"tag\" => \"pre.*.post*ignore\",\n                                \"path\" => \"test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log\",\n                                \"format\" => \"none\",\n                                \"read_from_head\" => true\n                              })\n      d = create_driver(config, false)\n      d.run {}\n      plugin = d.instance\n      mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once\n      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))\n    end\n  end\n\n  # Ensure that no fatal exception is raised when a file is missing and that\n  # files that do exist are still tailed as expected.\n  def test_missing_file\n    Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n      f.puts \"test1\"\n      f.puts \"test2\"\n    }\n\n    # Try two different configs - one with read_from_head and one without,\n    # since their interactions with the filesystem differ.\n    config1 = config_element(\"\", \"\", {\n                               \"tag\" => \"t1\",\n                               \"path\" => \"#{@tmp_dir}/non_existent_file.txt,#{@tmp_dir}/tail.txt\",\n                               \"format\" => \"none\",\n                               \"rotate_wait\" => \"2s\",\n                               \"pos_file\" => \"#{@tmp_dir}/tail.pos\"\n                             })\n    config2 = config1 + config_element(\"\", \"\", { \"read_from_head\" => true })\n    [config1, config2].each do |config|\n      d = create_driver(config, false)\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n          f.puts \"test3\"\n          f.puts \"test4\"\n        }\n      end\n      # This test sometimes fails and it shows a potential bug of in_tail\n      # https://github.com/fluent/fluentd/issues/1434\n      events = d.events\n      assert_equal(2, events.length)\n      assert_equal({\"message\" => \"test3\"}, events[0][2])\n      assert_equal({\"message\" => \"test4\"}, events[1][2])\n    end\n  end\n\n  sub_test_case 'inode_processing' do\n    def test_should_delete_file_pos_entry_for_non_existing_file_with_follow_inodes\n      config = common_follow_inode_config\n\n      path = \"#{@tmp_dir}/tail.txt\"\n      ino = 1\n      pos = 1234\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"wb\") {|f|\n        f.puts (\"%s\\t%016x\\t%016x\\n\" % [path, pos, ino])\n      }\n\n      d = create_driver(config, false)\n      d.run\n\n      pos_file = Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"r\")\n      pos_file.pos = 0\n\n      assert_raise(EOFError) do\n        pos_file.readline\n      end\n    end\n\n    def test_should_write_latest_offset_after_rotate_wait\n      config = common_follow_inode_config\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d = create_driver(config, false)\n      d.run(expect_emits: 2, shutdown: false) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"test3\\n\"}\n        FileUtils.move(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt\" + \"1\")\n        sleep 1\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\" + \"1\", \"ab\") {|f| f.puts \"test4\\n\"}\n      end\n\n      pos_file = Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"r\")\n      pos_file.pos = 0\n      line_parts = /^([^\\t]+)\\t([0-9a-fA-F]+)\\t([0-9a-fA-F]+)/.match(pos_file.readline)\n      waiting(5) {\n        while line_parts[2].to_i(16) != 24\n          sleep(0.1)\n          pos_file.pos = 0\n          line_parts = /^([^\\t]+)\\t([0-9a-fA-F]+)\\t([0-9a-fA-F]+)/.match(pos_file.readline)\n        end\n      }\n      assert_equal(24, line_parts[2].to_i(16))\n      d.instance_shutdown\n    end\n\n    def test_should_remove_deleted_file\n      config = config_element(\"\", \"\", {\"format\" => \"none\"})\n\n      path = \"#{@tmp_dir}/tail.txt\"\n      ino = 1\n      pos = 1234\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"wb\") {|f|\n        f.puts (\"%s\\t%016x\\t%016x\\n\" % [path, pos, ino])\n      }\n\n      d = create_driver(config)\n      d.run do\n        pos_file = Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"r\")\n        pos_file.pos = 0\n        assert_equal([], pos_file.readlines)\n      end\n    end\n\n    def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait\n      config = config_element(\"ROOT\", \"\", {\n          \"path\" => \"#{@tmp_dir}/tail.txt*\",\n          \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n          \"tag\" => \"t1\",\n          \"rotate_wait\" => \"1s\",\n          \"refresh_interval\" => \"1s\",\n          \"limit_recently_modified\" => \"1s\",\n          \"read_from_head\" => \"true\",\n          \"format\" => \"none\",\n          \"follow_inodes\" => \"true\",\n      })\n\n      d = create_driver(config, false)\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n      target_info = create_target_info(\"#{@tmp_dir}/tail.txt\")\n\n      d.run(expect_emits: 1, shutdown: false) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"test3\\n\"}\n      end\n\n\n      Timecop.travel(Time.now + 10) do\n        waiting(5) {\n          # @pos will be reset as 0 when UNWATCHED_POSITION is specified.\n          sleep 0.1 until d.instance.instance_variable_get(:@pf)[target_info].read_pos == 0\n        }\n      end\n\n      assert_equal(0, d.instance.instance_variable_get(:@pf)[target_info].read_pos)\n\n      d.instance_shutdown\n    end\n\n    def test_should_read_from_head_on_file_renaming_with_star_in_pattern\n      config = config_element(\"ROOT\", \"\", {\n          \"path\" => \"#{@tmp_dir}/tail.txt*\",\n          \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n          \"tag\" => \"t1\",\n          \"rotate_wait\" => \"10s\",\n          \"refresh_interval\" => \"1s\",\n          \"limit_recently_modified\" => \"60s\",\n          \"read_from_head\" => \"true\",\n          \"format\" => \"none\",\n          \"follow_inodes\" => \"true\"\n      })\n\n      d = create_driver(config, false)\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d.run(expect_emits: 2, shutdown: false) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"test3\\n\"}\n        FileUtils.move(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt1\")\n      end\n\n      events = d.events\n      assert_equal(3, events.length)\n      d.instance_shutdown\n    end\n\n    def test_should_not_read_from_head_on_rotation_when_watching_inodes\n      config = common_follow_inode_config\n\n      d = create_driver(config, false)\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d.run(expect_emits: 1, shutdown: false) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"test3\\n\"}\n      end\n\n      FileUtils.move(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt1\")\n      Timecop.travel(Time.now + 10) do\n        sleep 2\n        events = d.events\n        assert_equal(3, events.length)\n      end\n\n      d.instance_shutdown\n    end\n\n    def test_should_mark_file_unwatched_if_same_name_file_created_with_different_inode\n      config = common_follow_inode_config\n\n      d = create_driver(config, false)\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n      target_info = create_target_info(\"#{@tmp_dir}/tail.txt\")\n\n      d.run(expect_emits: 2, shutdown: false) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"test3\\n\"}\n        cleanup_file(\"#{@tmp_dir}/tail.txt\")\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| f.puts \"test4\\n\"}\n      end\n\n      new_target_info = create_target_info(\"#{@tmp_dir}/tail.txt\")\n\n      pos_file = d.instance.instance_variable_get(:@pf)\n\n      waiting(10) {\n        # @pos will be reset as 0 when UNWATCHED_POSITION is specified.\n        sleep 0.1 until pos_file[target_info].read_pos == 0\n      }\n      new_position = pos_file[new_target_info].read_pos\n      assert_equal(6, new_position)\n\n      d.instance_shutdown\n    end\n\n    def test_should_close_watcher_after_rotate_wait\n      now = Time.now\n      config = common_follow_inode_config + config_element('', '', {\"rotate_wait\" => \"1s\", \"limit_recently_modified\" => \"1s\"})\n\n      d = create_driver(config, false)\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n      target_info = create_target_info(\"#{@tmp_dir}/tail.txt\")\n      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once\n      d.run(shutdown: false)\n      assert d.instance.instance_variable_get(:@tails)[target_info.path]\n\n      Timecop.travel(now + 10) do\n        d.instance.instance_eval do\n          sleep 0.1 until @tails[target_info.path] == nil\n        end\n        assert_nil d.instance.instance_variable_get(:@tails)[target_info.path]\n      end\n      d.instance_shutdown\n    end\n\n    def test_should_create_new_watcher_for_new_file_with_same_name\n      now = Time.now\n      config = common_follow_inode_config + config_element('', '', {\"limit_recently_modified\" => \"2s\"})\n\n      d = create_driver(config, false)\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n      path_ino = create_target_info(\"#{@tmp_dir}/tail.txt\")\n\n      d.run(expect_emits: 1, shutdown: false) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"test3\\n\"}\n      end\n\n      cleanup_file(\"#{@tmp_dir}/tail.txt\")\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test3\"\n        f.puts \"test4\"\n      }\n      new_path_ino = create_target_info(\"#{@tmp_dir}/tail.txt\")\n\n      Timecop.travel(now + 10) do\n        sleep 3\n        d.instance.instance_eval do\n          @tails[path_ino.path] == nil\n          @tails[new_path_ino.path] != nil\n        end\n      end\n\n      events = d.events\n\n      assert_equal(5, events.length)\n\n      d.instance_shutdown\n    end\n\n    def test_truncate_file_with_follow_inodes\n      config = common_follow_inode_config\n\n      d = create_driver(config, false)\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d.run(expect_emits: 3, shutdown: false) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"test3\\n\"}\n        sleep 2\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"w+b\") {|f| f.puts \"test4\\n\"}\n      end\n\n      events = d.events\n      assert_equal(4, events.length)\n      assert_equal({\"message\" => \"test1\"}, events[0][2])\n      assert_equal({\"message\" => \"test2\"}, events[1][2])\n      assert_equal({\"message\" => \"test3\"}, events[2][2])\n      assert_equal({\"message\" => \"test4\"}, events[3][2])\n      d.instance_shutdown\n    end\n\n    # issue #3464\n    def test_should_replace_target_info\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\\n\"\n      }\n      target_info = create_target_info(\"#{@tmp_dir}/tail.txt\")\n      inodes = []\n\n      config = config_element(\"ROOT\", \"\", {\n                                \"path\" => \"#{@tmp_dir}/tail.txt*\",\n                                \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                                \"tag\" => \"t1\",\n                                \"refresh_interval\" => \"60s\",\n                                \"read_from_head\" => \"true\",\n                                \"format\" => \"none\",\n                                \"rotate_wait\" => \"1s\",\n                                \"follow_inodes\" => \"true\"\n                              })\n      d = create_driver(config, false)\n      d.run(timeout: 5) do\n        while d.events.size < 1 do\n          sleep 0.1\n        end\n        inodes = d.instance.instance_variable_get(:@tails).values.collect do |tw|\n          tw.ino\n        end\n        assert_equal([target_info.ino], inodes)\n\n        cleanup_file(\"#{@tmp_dir}/tail.txt\")\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| f.puts \"test2\\n\"}\n\n        while d.events.size < 2 do\n          sleep 0.1\n        end\n        inodes = d.instance.instance_variable_get(:@tails).values.collect do |tw|\n          tw.ino\n        end\n        new_target_info = create_target_info(\"#{@tmp_dir}/tail.txt\")\n        assert_not_equal(target_info.ino, new_target_info.ino)\n        assert_equal([new_target_info.ino], inodes)\n      end\n    end\n  end\n\n  sub_test_case \"tail_path\" do\n    def test_tail_path_with_singleline\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f|\n        f.puts \"test1\"\n        f.puts \"test2\"\n      }\n\n      d = create_driver(SINGLE_LINE_CONFIG + config_element(\"\", \"\", { \"path_key\" => \"path\" }))\n\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f|\n          f.puts \"test3\"\n          f.puts \"test4\"\n        }\n      end\n\n      events = d.events\n      assert_equal(true, events.length > 0)\n      events.each do |emit|\n        assert_equal(\"#{@tmp_dir}/tail.txt\", emit[2][\"path\"])\n      end\n    end\n\n    def test_tail_path_with_multiline_with_firstline\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n\n      config = config_element(\"\", \"\", {\n                                \"path_key\" => \"path\",\n                                \"format\" => \"multiline\",\n                                \"format1\" => \"/^s (?<message1>[^\\\\n]+)(\\\\nf (?<message2>[^\\\\n]+))?(\\\\nf (?<message3>.*))?/\",\n                                \"format_firstline\" => \"/^[s]/\"\n                              })\n      d = create_driver(config)\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f|\n          f.puts \"f test1\"\n          f.puts \"s test2\"\n          f.puts \"f test3\"\n          f.puts \"f test4\"\n          f.puts \"s test5\"\n          f.puts \"s test6\"\n          f.puts \"f test7\"\n          f.puts \"s test8\"\n        }\n      end\n\n      events = d.events\n      assert_equal(4, events.length)\n      events.each do |emit|\n        assert_equal(\"#{@tmp_dir}/tail.txt\", emit[2][\"path\"])\n      end\n    end\n\n    def test_tail_path_with_multiline_without_firstline\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") { |f| }\n\n      config = config_element(\"\", \"\", {\n                                \"path_key\" => \"path\",\n                                \"format\" => \"multiline\",\n                                \"format1\" => \"/(?<var1>foo \\\\d)\\\\n/\",\n                                \"format2\" => \"/(?<var2>bar \\\\d)\\\\n/\",\n                                \"format3\" => \"/(?<var3>baz \\\\d)/\",\n                              })\n      d = create_driver(config)\n      d.run(expect_emits: 1) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") { |f|\n          f.puts \"foo 1\"\n          f.puts \"bar 1\"\n          f.puts \"baz 1\"\n        }\n      end\n\n      events = d.events\n      assert(events.length > 0)\n      events.each do |emit|\n        assert_equal(\"#{@tmp_dir}/tail.txt\", emit[2][\"path\"])\n      end\n    end\n\n    def test_tail_path_with_multiline_with_multiple_paths\n      if ENV[\"APPVEYOR\"] && Fluent.windows?\n        omit \"This testcase is unstable on AppVeyor.\"\n      end\n      files = [\"#{@tmp_dir}/tail1.txt\", \"#{@tmp_dir}/tail2.txt\"]\n      files.each { |file| Fluent::FileWrapper.open(file, \"wb\") { |f| } }\n\n      config = config_element(\"\", \"\", {\n                                \"path\" => \"#{files[0]},#{files[1]}\",\n                                \"path_key\" => \"path\",\n                                \"tag\" => \"t1\",\n                                \"format\" => \"multiline\",\n                                \"format1\" => \"/^[s|f] (?<message>.*)/\",\n                                \"format_firstline\" => \"/^[s]/\"\n                              })\n      d = create_driver(config, false)\n      d.run(expect_emits: 2) do\n        files.each do |file|\n          Fluent::FileWrapper.open(file, 'ab') { |f|\n            f.puts \"f #{file} line should be ignored\"\n            f.puts \"s test1\"\n            f.puts \"f test2\"\n            f.puts \"f test3\"\n            f.puts \"s test4\"\n          }\n        end\n      end\n\n      events = d.events\n      assert_equal(4, events.length)\n      assert_equal(files, [events[0][2][\"path\"], events[1][2][\"path\"]].sort)\n      # \"test4\" events are here because these events are flushed at shutdown phase\n      assert_equal(files, [events[2][2][\"path\"], events[3][2][\"path\"]].sort)\n    end\n  end\n\n  def test_limit_recently_modified\n    now = Time.new(2010, 1, 2, 3, 4, 5)\n    FileUtils.touch(\"#{@tmp_dir}/tail_unwatch.txt\", mtime: (now - 3601))\n    FileUtils.touch(\"#{@tmp_dir}/tail_watch1.txt\", mtime: (now - 3600))\n    FileUtils.touch(\"#{@tmp_dir}/tail_watch2.txt\", mtime: now)\n\n    config = config_element('', '', {\n      'tag' => 'tail',\n      'path' => \"#{@tmp_dir}/*.txt\",\n      'format' => 'none',\n      'limit_recently_modified' => '3600s'\n    })\n\n    expected_files = [\n      create_target_info(\"#{@tmp_dir}/tail_watch1.txt\"),\n      create_target_info(\"#{@tmp_dir}/tail_watch2.txt\")\n    ]\n\n    Timecop.freeze(now) do\n      plugin = create_driver(config, false).instance\n      assert_equal(expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })\n    end\n  end\n\n  def test_skip_refresh_on_startup\n    FileUtils.touch(\"#{@tmp_dir}/tail.txt\")\n    config = config_element('', '', {\n                              'format' => 'none',\n                              'refresh_interval' => 1,\n                              'skip_refresh_on_startup' => true\n                            })\n    d = create_driver(config)\n    d.run(shutdown: false) {}\n    assert_equal(0, d.instance.instance_variable_get(:@tails).keys.size)\n    # detect a file at first execution of in_tail_refresh_watchers timer\n    waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }\n    d.instance_shutdown\n  end\n\n  def test_ENOENT_error_after_setup_watcher\n    path = \"#{@tmp_dir}/tail.txt\"\n    FileUtils.touch(path)\n    config = config_element('', '', {\n                              'format' => 'none',\n                            })\n    d = create_driver(config)\n    file_deleted = false\n    mock.proxy(d.instance).existence_path do |hash|\n      unless file_deleted\n        cleanup_file(path)\n        file_deleted = true\n      end\n      hash\n    end.twice\n    assert_nothing_raised do\n      d.run(shutdown: false) {}\n    end\n    assert(d.logs.any?{|log| log.include?(\"stat() for #{path} failed. Continuing without tailing it.\\n\") },\n           d.logs.join(\"\\n\"))\n  ensure\n    d.instance_shutdown if d&.instance\n  end\n\n  def test_EACCES_error_after_setup_watcher\n    omit \"Cannot test with root user\" if Process::UID.eid == 0\n    path = \"#{@tmp_dir}/noaccess/tail.txt\"\n    begin\n      FileUtils.mkdir_p(\"#{@tmp_dir}/noaccess\")\n      FileUtils.chmod(0755, \"#{@tmp_dir}/noaccess\")\n      FileUtils.touch(path)\n      config = config_element('', '', {\n                                'tag' => \"tail\",\n                                'path' => path,\n                                'format' => 'none',\n                              })\n      d = create_driver(config, false)\n      mock.proxy(d.instance).existence_path do |hash|\n        FileUtils.chmod(0000, \"#{@tmp_dir}/noaccess\")\n        hash\n      end.twice\n      assert_nothing_raised do\n        d.run(shutdown: false) {}\n      end\n      assert(d.logs.any?{|log| log.include?(\"stat() for #{path} failed. Continuing without tailing it.\\n\") },\n             d.logs.join(\"\\n\"))\n    end\n  ensure\n    d.instance_shutdown if d&.instance\n    if File.exist?(\"#{@tmp_dir}/noaccess\")\n      FileUtils.chmod(0755, \"#{@tmp_dir}/noaccess\")\n      FileUtils.rm_rf(\"#{@tmp_dir}/noaccess\")\n    end\n  end unless Fluent.windows?\n\n  def test_EACCES\n    path = \"#{@tmp_dir}/tail.txt\"\n    FileUtils.touch(path)\n    config = config_element('', '', {\n                              'format' => 'none',\n                            })\n    d = create_driver(config)\n    mock.proxy(Fluent::FileWrapper).stat(path) do |stat|\n      raise Errno::EACCES\n    end.at_least(1)\n    assert_nothing_raised do\n      d.run(shutdown: false) {}\n    end\n    assert(d.logs.any?{|log| log.include?(\"expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\\n\") })\n  ensure\n    d.instance_shutdown if d&.instance\n  end\n\n  def test_warn_without_directory_permission\n    omit \"Cannot test with root user\" if Process::UID.eid == 0\n    omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n    path = \"#{@tmp_dir}/noaccess/tail.txt\"\n    begin\n      FileUtils.mkdir_p(\"#{@tmp_dir}/noaccess\")\n      FileUtils.touch(path)\n      FileUtils.chmod(0400, path)\n      FileUtils.chmod(0600, \"#{@tmp_dir}/noaccess\")\n      config = config_element('', '', {\n                                'tag' => \"tail\",\n                                'path' => path,\n                                'format' => 'none',\n                                \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                              })\n      d = create_driver(config, false)\n      fname = File.expand_path(\"#{@tmp_dir}/noaccess\")\n      assert(d.logs.any?{|log| log.include?(\"Skip #{path} because '#{fname}' lacks execute permission.\\n\") })\n    end\n  end\n\n  def test_no_warn_with_directory_permission\n    omit \"Cannot test with root user\" if Process::UID.eid == 0\n    omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n    path = \"#{@tmp_dir}/noaccess/tail.txt\"\n    begin\n      FileUtils.mkdir_p(\"#{@tmp_dir}/noaccess\")\n      FileUtils.chmod(0100, \"#{@tmp_dir}/noaccess\")\n      config = config_element('', '', {\n                                'tag' => \"tail\",\n                                'path' => path,\n                                'format' => 'none',\n                                \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n                              })\n      d = create_driver(config, false)\n      fname = File.expand_path(\"#{@tmp_dir}/noaccess\")\n      assert(d.logs.all?{|log| !log.include?(\"Skip #{path} because '#{fname}' lacks execute permission.\\n\") })\n    end\n  end\n\n  def test_other_user_owned_files\n    # https://github.com/fluent/fluentd/issues/5141\n    omit \"Cannot test with root user\" if Process::UID.eid == 0\n    omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n    config = config_element('', '', {\n      'tag' => \"tail\",\n      'path' => \"/var/log/*.log\",\n      'format' => 'none',\n      \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n    })\n    assert_nothing_raised do\n      create_driver(config, false)\n    end\n  end\n\n  def test_shutdown_timeout\n    Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") do |f|\n      # Should be large enough to take too long time to consume\n      (1024 * 1024 * 5).times do\n        f.puts \"{\\\"test\\\":\\\"fizzbuzz\\\"}\"\n      end\n    end\n\n    config =\n      CONFIG_READ_FROM_HEAD +\n      config_element('', '', {\n                              'format' => 'json',\n                              'skip_refresh_on_startup' => true,\n                            })\n    shutdown_start_time = 0\n\n    d = create_driver(config)\n    mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|\n      mock.proxy(io_handler).ready_to_shutdown(anything) do\n        shutdown_start_time = Fluent::Clock.now\n      end\n      io_handler.shutdown_timeout = 0.5\n      io_handler\n    end\n\n    assert_nothing_raised do\n      d.run(expect_emits: 1)\n    end\n\n    elapsed = Fluent::Clock.now - shutdown_start_time\n    assert_true(elapsed > 0.5 && elapsed < 2.0,\n                \"elapsed time: #{elapsed}\")\n  end\n\n  sub_test_case \"throttling logs at in_tail level\" do\n    data(\"file test1.log no_limit 5120 text: msg\" => [\"test1.log\", 5120, \"msg\"],\n         \"file test2.log no_limit 1024 text: test\" => [\"test2.log\", 1024, \"test\"])\n    def test_lines_collected_with_no_throttling(data)\n      file, num_lines, msg = data\n\n      pattern = \"/^#{@tmp_dir}\\/(?<file>.+)\\.log$/\"\n      rule = create_rule_directive({\n        \"file\" => \"/test.*/\",\n      }, -1)\n      group = create_group_directive(pattern, \"1s\", rule)\n      path_element = create_path_element(file)\n\n      conf = ROOT_CONFIG + group + path_element + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/#{file}\", 'wb') do |f|\n        num_lines.times do\n          f.puts \"#{msg}\\n\"\n        end\n      end\n\n\n      d = create_driver(conf, false)\n      d.run(timeout: 3) do\n        start_time = Fluent::Clock.now\n\n        assert_equal(num_lines, d.record_count)\n        assert_equal({ \"message\" => msg }, d.events[0][2])\n\n        prev_count = d.record_count\n        sleep(0.1) while d.emit_count < 1\n        assert_true(Fluent::Clock.now - start_time < 2)\n        ## after waiting for 1 (+ jitter) secs, limit will reset\n        ## Plugin will start reading but it will encounter EOF Error\n        ## since no logs are left to be read\n        ## Hence, d.record_count = prev_count\n        tail_watcher_interval = 1.0 # hard coded value in in_tail\n        safety_ratio = 1.2\n        jitter = tail_watcher_interval * safety_ratio\n        sleep(1.0 + jitter)\n        assert_equal(0, d.record_count - prev_count)\n      end\n    end\n\n    test \"lines collected with throttling\" do\n      file = \"podname1_namespace12_container-123456.log\"\n      limit = 1000\n      rate_period = 2\n      num_lines = 3000\n      msg = \"a\" * 8190 # Total size = 8190 bytes + 2 (\\n) bytes\n\n      rule = create_rule_directive({\n        \"namespace\"=> \"/namespace.+/\",\n        \"podname\"=> \"/podname.+/\",\n      }, limit)\n      path_element = create_path_element(file)\n      conf = ROOT_CONFIG + create_group_directive(tailing_group_pattern, \"#{rate_period}s\", rule) + path_element + SINGLE_LINE_CONFIG + CONFIG_READ_FROM_HEAD\n\n      d = create_driver(conf, false)\n      file_path = \"#{@tmp_dir}/#{file}\"\n\n      Fluent::FileWrapper.open(file_path, 'wb') do |f|\n        num_lines.times do\n          f.puts msg\n        end\n      end\n\n      d.run(timeout: 15) do\n        sleep_interval = 0.1\n        tail_watcher_interval = 1.0 # hard coded value in in_tail\n        safety_ratio = 1.2\n        lower_jitter = sleep_interval * safety_ratio\n        upper_jitter = (tail_watcher_interval + sleep_interval) * safety_ratio\n        lower_interval = rate_period - lower_jitter\n        upper_interval = rate_period + upper_jitter\n\n        emit_count = 0\n        prev_count = 0\n\n        while emit_count < 3 do\n          start_time = Fluent::Clock.now\n          sleep(sleep_interval) while d.emit_count <= emit_count\n          elapsed_seconds = Fluent::Clock.now - start_time\n          if emit_count > 0\n            assert_true(elapsed_seconds > lower_interval && elapsed_seconds < upper_interval,\n                        \"elapsed_seconds #{elapsed_seconds} is out of allowed range:\\n\" +\n                        \"  lower: #{lower_interval} [sec]\\n\" +\n                        \"  upper: #{upper_interval} [sec]\")\n          end\n          assert_equal(limit, d.record_count - prev_count)\n          emit_count = d.emit_count\n          prev_count = d.record_count\n        end\n\n        ## When all the lines are read and rate_period seconds are over\n        ## limit will reset and since there are no more logs to be read,\n        ## number_lines_read will be 0\n        sleep upper_interval\n        gw = d.instance.find_group_from_metadata(file_path)\n        assert_equal(0, gw.current_paths[file_path].number_lines_read)\n      end\n    end\n  end\n\n  sub_test_case \"Update watchers for rotation with follow_inodes\" do\n    def test_updateTW_before_refreshTW_and_detach_before_refreshTW\n      config = config_element(\n        \"ROOT\",\n        \"\",\n        {\n          \"path\" => \"#{@tmp_dir}/tail.txt*\",\n          \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n          \"tag\" => \"t1\",\n          \"format\" => \"none\",\n          \"read_from_head\" => \"true\",\n          \"follow_inodes\" => \"true\",\n          # In order to detach the old watcher quickly.\n          \"rotate_wait\" => \"1s\",\n          # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not\n          # called by a timer.\n          \"refresh_interval\" => \"1h\",\n          # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,\n          # so disable it in order to reproduce the same condition stably.\n          \"enable_stat_watcher\" => \"false\",\n        }\n      )\n      d = create_driver(config, false)\n\n      tail_watchers = []\n      stub.proxy(d.instance).setup_watcher do |tw|\n        tail_watchers.append(tw)\n        tw\n      end\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| f.puts \"file1 log1\"}\n\n      d.run(expect_records: 4, timeout: 10) do\n        # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"file1 log2\"}\n        FileUtils.move(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt\" + \"1\")\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| f.puts \"file2 log1\"}\n\n        # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` updates the TailWatcher:\n        #     TailWatcher(path: \"tail.txt\", inode: inode_0) => TailWatcher(path: \"tail.txt\", inode: inode_1)\n        # The old TailWatcher is detached here since `rotate_wait` is just `1s`.\n        sleep 3\n\n        # This reproduces the following situation:\n        #     Rotation => update_watcher => refresh_watchers\n        # This adds a new TailWatcher: TailWatcher(path: \"tail.txt1\", inode: inode_0)\n        d.instance.refresh_watchers\n\n        # Append to the new current log file.\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"file2 log2\"}\n      end\n\n      inode_0 = tail_watchers[0].ino\n      inode_1 = tail_watchers[1].ino\n      record_values = d.events.collect { |event| event[2][\"message\"] }.sort\n      position_entries = []\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"r\") do |f|\n        f.readlines(chomp: true).each do |line|\n          values = line.split(\"\\t\")\n          position_entries.append([values[0], values[1], values[2].to_i(16)])\n        end\n      end\n\n      assert_equal(\n        {\n          record_values: [\"file1 log1\", \"file1 log2\", \"file2 log1\", \"file2 log2\"],\n          tail_watcher_paths: [\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt1\"],\n          tail_watcher_inodes: [inode_0, inode_1, inode_0],\n          tail_watcher_io_handler_opened_statuses: [false, false, false],\n          position_entries: [\n            [\"#{@tmp_dir}/tail.txt\", \"0000000000000016\", inode_0],\n            [\"#{@tmp_dir}/tail.txt\", \"0000000000000016\", inode_1],\n          ],\n        },\n        {\n          record_values: record_values,\n          tail_watcher_paths: tail_watchers.collect { |tw| tw.path },\n          tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },\n          tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },\n          position_entries: position_entries\n        },\n      )\n    end\n\n    def test_updateTW_before_refreshTW_and_detach_after_refreshTW\n      config = config_element(\n        \"ROOT\",\n        \"\",\n        {\n          \"path\" => \"#{@tmp_dir}/tail.txt*\",\n          \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n          \"tag\" => \"t1\",\n          \"format\" => \"none\",\n          \"read_from_head\" => \"true\",\n          \"follow_inodes\" => \"true\",\n          # In order to detach the old watcher after refresh_watchers.\n          \"rotate_wait\" => \"4s\",\n          # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not\n          # called by a timer.\n          \"refresh_interval\" => \"1h\",\n          # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,\n          # so disable it in order to reproduce the same condition stably.\n          \"enable_stat_watcher\" => \"false\",\n        }\n      )\n      d = create_driver(config, false)\n\n      tail_watchers = []\n      stub.proxy(d.instance).setup_watcher do |tw|\n        tail_watchers.append(tw)\n        tw\n      end\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| f.puts \"file1 log1\"}\n\n      d.run(expect_records: 4, timeout: 10) do\n        # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"file1 log2\"}\n        FileUtils.move(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt\" + \"1\")\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| f.puts \"file2 log1\"}\n\n        # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` updates the TailWatcher:\n        #     TailWatcher(path: \"tail.txt\", inode: inode_0) => TailWatcher(path: \"tail.txt\", inode: inode_1)\n        sleep 2\n\n        # This reproduces the following situation:\n        #     Rotation => update_watcher => refresh_watchers\n        # This adds a new TailWatcher: TailWatcher(path: \"tail.txt1\", inode: inode_0)\n        d.instance.refresh_watchers\n\n        # The old TailWatcher is detached here since `rotate_wait` is `4s`.\n        sleep 3\n\n        # Append to the new current log file.\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"file2 log2\"}\n      end\n\n      inode_0 = tail_watchers[0].ino\n      inode_1 = tail_watchers[1].ino\n      record_values = d.events.collect { |event| event[2][\"message\"] }.sort\n      position_entries = []\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"r\") do |f|\n        f.readlines(chomp: true).each do |line|\n          values = line.split(\"\\t\")\n          position_entries.append([values[0], values[1], values[2].to_i(16)])\n        end\n      end\n\n      assert_equal(\n        {\n          record_values: [\"file1 log1\", \"file1 log2\", \"file2 log1\", \"file2 log2\"],\n          tail_watcher_paths: [\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt1\"],\n          tail_watcher_inodes: [inode_0, inode_1, inode_0],\n          tail_watcher_io_handler_opened_statuses: [false, false, false],\n          position_entries: [\n            # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.\n            [\"#{@tmp_dir}/tail.txt\", \"0000000000000016\", inode_0],\n            [\"#{@tmp_dir}/tail.txt\", \"0000000000000016\", inode_1],\n          ],\n        },\n        {\n          record_values: record_values,\n          tail_watcher_paths: tail_watchers.collect { |tw| tw.path },\n          tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },\n          tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },\n          position_entries: position_entries\n        },\n      )\n    end\n\n    # The scenario where in_tail wrongly detaches TailWatcher.\n    # This is reported in https://github.com/fluent/fluentd/issues/4190.\n    def test_updateTW_after_refreshTW\n      config = config_element(\n        \"ROOT\",\n        \"\",\n        {\n          \"path\" => \"#{@tmp_dir}/tail.txt*\",\n          \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n          \"tag\" => \"t1\",\n          \"format\" => \"none\",\n          \"read_from_head\" => \"true\",\n          \"follow_inodes\" => \"true\",\n          # In order to detach the old watcher quickly.\n          \"rotate_wait\" => \"1s\",\n          # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not\n          # called by a timer.\n          \"refresh_interval\" => \"1h\",\n          # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,\n          # so disable it in order to reproduce the same condition stably.\n          \"enable_stat_watcher\" => \"false\",\n        }\n      )\n      d = create_driver(config, false)\n\n      tail_watchers = []\n      stub.proxy(d.instance).setup_watcher do |tw|\n        tail_watchers.append(tw)\n        tw\n      end\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| f.puts \"file1 log1\"}\n\n      d.run(expect_records: 4, timeout: 10) do\n        # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"file1 log2\"}\n        FileUtils.move(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt\" + \"1\")\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| f.puts \"file2 log1\"}\n\n        # This reproduces the following situation:\n        #     Rotation => refresh_watchers => update_watcher\n        # This add a new TailWatcher: TailWatcher(path: \"tail.txt\", inode: inode_1)\n        #     This overwrites `@tails[\"tail.txt\"]`\n        d.instance.refresh_watchers\n\n        # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher:\n        #     TailWatcher(path: \"tail.txt\", inode: inode_0) => TailWatcher(path: \"tail.txt\", inode: inode_1)\n        # However, it is already added in `refresh_watcher`, so `update_watcher` doesn't create the new TailWatcher.\n        # The old TailWatcher is detached here since `rotate_wait` is just `1s`.\n        sleep 3\n\n        # This adds a new TailWatcher: TailWatcher(path: \"tail.txt1\", inode: inode_0)\n        d.instance.refresh_watchers\n\n        # Append to the new current log file.\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"file2 log2\"}\n      end\n\n      inode_0 = tail_watchers[0].ino\n      inode_1 = tail_watchers[1].ino\n      record_values = d.events.collect { |event| event[2][\"message\"] }.sort\n      position_entries = []\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"r\") do |f|\n        f.readlines(chomp: true).each do |line|\n          values = line.split(\"\\t\")\n          position_entries.append([values[0], values[1], values[2].to_i(16)])\n        end\n      end\n\n      assert_equal(\n        {\n          record_values: [\"file1 log1\", \"file1 log2\", \"file2 log1\", \"file2 log2\"],\n          tail_watcher_paths: [\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt1\"],\n          tail_watcher_inodes: [inode_0, inode_1, inode_0],\n          tail_watcher_io_handler_opened_statuses: [false, false, false],\n          position_entries: [\n            # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.\n            [\"#{@tmp_dir}/tail.txt\", \"0000000000000016\", inode_0],\n            [\"#{@tmp_dir}/tail.txt\", \"0000000000000016\", inode_1],\n          ],\n        },\n        {\n          record_values: record_values,\n          tail_watcher_paths: tail_watchers.collect { |tw| tw.path },\n          tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },\n          tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },\n          position_entries: position_entries\n        },\n      )\n    end\n\n    def test_path_resurrection\n      config = config_element(\n        \"ROOT\",\n        \"\",\n        {\n          \"path\" => \"#{@tmp_dir}/tail.txt*\",\n          \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n          \"tag\" => \"t1\",\n          \"format\" => \"none\",\n          \"read_from_head\" => \"true\",\n          \"follow_inodes\" => \"true\",\n          # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not\n          # called by a timer.\n          \"refresh_interval\" => \"1h\",\n          # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632\n          # Because of this problem, log duplication can occur during `rotate_wait`.\n          # Need to set `rotate_wait 0` for a workaround.\n          \"rotate_wait\" => \"0s\",\n        }\n      )\n      d = create_driver(config, false)\n\n      tail_watchers = []\n      stub.proxy(d.instance).setup_watcher do |tw|\n        tail_watchers.append(tw)\n        tw\n      end\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| f.puts \"file1 log1\"}\n\n      d.run(expect_records: 5, timeout: 10) do\n        # Rotate\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"file1 log2\"}\n        FileUtils.move(\"#{@tmp_dir}/tail.txt\", \"#{@tmp_dir}/tail.txt\" + \"1\")\n        # TailWatcher(path: \"tail.txt\", inode: inode_0) detects `tail.txt` disappeared.\n        #     Call `update_watcher` to stop and discard self.\n        # If not discarding, then it will be a orphan and cause leak and log duplication.\n        #\n        # This reproduces the case where the notify to TailWatcher comes before the new file for the path\n        # is created during rotation.\n        # (stat_watcher notifies faster than a new file is created)\n        # Overall, this is a rotation operation, but from the TailWatcher, it appears as if the file\n        # was resurrected once it disappeared.\n        sleep 2 # On Windows and macOS, StatWatcher doesn't work, so need enough interval for TimeTrigger.\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"wb\") {|f| f.puts \"file2 log1\"}\n\n        # Add new TailWatchers\n        #     tail.txt: TailWatcher(path: \"tail.txt\", inode: inode_1)\n        #     tail.txt: TailWatcher(path: \"tail.txt1\", inode: inode_0)\n        # NOTE: If not discarding the first TailWatcher on notify, this makes it a orphan because\n        # this overwrites the `@tails[tail.txt]` by adding TailWatcher(path: \"tail.txt\", inode: inode_1)\n        d.instance.refresh_watchers\n\n        # This does nothing.\n        # NOTE: If not discarding the first TailWatcher on notify, this add\n        #     tail.txt1: TailWatcher(path: \"tail.txt1\", inode: inode_0)\n        # because the previous refresh_watcher overwrites `@tails[tail.txt]` and the inode_0 is lost.\n        # This would cause log duplication.\n        d.instance.refresh_watchers\n\n        # Append to the old file\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt1\", \"ab\") {|f| f.puts \"file1 log3\"}\n\n        # Append to the new current log file.\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt\", \"ab\") {|f| f.puts \"file2 log2\"}\n      end\n\n      inode_0 = Fluent::FileWrapper.stat(\"#{@tmp_dir}/tail.txt1\").ino\n      inode_1 = Fluent::FileWrapper.stat(\"#{@tmp_dir}/tail.txt\").ino\n      record_values = d.events.collect { |event| event[2][\"message\"] }.sort\n      position_entries = []\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"r\") do |f|\n        f.readlines(chomp: true).each do |line|\n          values = line.split(\"\\t\")\n          position_entries.append([values[0], values[1], values[2].to_i(16)])\n        end\n      end\n\n      assert_equal(\n        {\n          record_values: [\"file1 log1\", \"file1 log2\", \"file1 log3\", \"file2 log1\", \"file2 log2\"],\n          tail_watcher_set: Set[\n            {\n              path: \"#{@tmp_dir}/tail.txt\",\n              inode: inode_0,\n              io_handler_opened_status: false,\n            },\n            {\n              path: \"#{@tmp_dir}/tail.txt\",\n              inode: inode_1,\n              io_handler_opened_status: false,\n            },\n            {\n              path: \"#{@tmp_dir}/tail.txt1\",\n              inode: inode_0,\n              io_handler_opened_status: false,\n            },\n          ],\n          position_entries: [\n            [\"#{@tmp_dir}/tail.txt\", \"0000000000000021\", inode_0],\n            [\"#{@tmp_dir}/tail.txt\", \"0000000000000016\", inode_1],\n          ],\n        },\n        {\n          record_values: record_values,\n          tail_watcher_set: Set.new(tail_watchers.collect { |tw|\n            {\n              path: tw.path,\n              inode: tw.ino,\n              io_handler_opened_status: tw.instance_variable_get(:@io_handler)&.opened? || false,\n            }\n          }),\n          position_entries: position_entries,\n        },\n      )\n    end\n\n    def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait\n      config = config_element(\n        \"ROOT\",\n        \"\",\n        {\n          \"path\" => \"#{@tmp_dir}/tail.txt*\",\n          \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n          \"tag\" => \"t1\",\n          \"format\" => \"none\",\n          \"read_from_head\" => \"true\",\n          \"follow_inodes\" => \"true\",\n          \"rotate_wait\" => \"3s\",\n          \"refresh_interval\" => \"1h\",\n          # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,\n          # so disable it in order to reproduce the same condition stably.\n          \"enable_stat_watcher\" => \"false\",\n        }\n      )\n      d = create_driver(config, false)\n\n      tail_watchers = []\n      stub.proxy(d.instance).setup_watcher do |tw|\n        tail_watchers.append(tw)\n        mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.\n        tw\n      end\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"wb\") {|f| f.puts \"file1 log1\"}\n\n      d.run(expect_records: 6, timeout: 15) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"ab\") {|f| f.puts \"file1 log2\"}\n\n        sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)\n\n        FileUtils.move(\"#{@tmp_dir}/tail.txt0\", \"#{@tmp_dir}/tail.txt\" + \"1\")\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"wb\") {|f| f.puts \"file2 log1\"}\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"ab\") {|f| f.puts \"file2 log2\"}\n\n        sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)\n\n        # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)\n        [1, 0].each do |i|\n          FileUtils.move(\"#{@tmp_dir}/tail.txt#{i}\", \"#{@tmp_dir}/tail.txt#{i + 1}\")\n        end\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"wb\") {|f| f.puts \"file3 log1\"}\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"ab\") {|f| f.puts \"file3 log2\"}\n\n        # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.\n        # (Note: Currently, there is no harm in duplicate calls)\n        sleep 4\n      end\n\n      inode_0 = tail_watchers[0]&.ino\n      inode_1 = tail_watchers[1]&.ino\n      inode_2 = tail_watchers[2]&.ino\n      record_values = d.events.collect { |event| event[2][\"message\"] }.sort\n      position_entries = []\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"r\") do |f|\n        f.readlines(chomp: true).each do |line|\n          values = line.split(\"\\t\")\n          position_entries.append([values[0], values[1], values[2].to_i(16)])\n        end\n      end\n\n      assert_equal(\n        {\n          record_values: [\"file1 log1\", \"file1 log2\", \"file2 log1\", \"file2 log2\", \"file3 log1\", \"file3 log2\"],\n          tail_watcher_paths: [\"#{@tmp_dir}/tail.txt0\", \"#{@tmp_dir}/tail.txt0\", \"#{@tmp_dir}/tail.txt0\"],\n          tail_watcher_inodes: [inode_0, inode_1, inode_2],\n          tail_watcher_io_handler_opened_statuses: [false, false, false],\n          position_entries: [\n            [\"#{@tmp_dir}/tail.txt0\", \"0000000000000016\", inode_0],\n            [\"#{@tmp_dir}/tail.txt0\", \"0000000000000016\", inode_1],\n            [\"#{@tmp_dir}/tail.txt0\", \"0000000000000016\", inode_2],\n          ],\n        },\n        {\n          record_values: record_values,\n          tail_watcher_paths: tail_watchers.collect { |tw| tw.path },\n          tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },\n          tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },\n          position_entries: position_entries\n        },\n      )\n    end\n  end\n\n  sub_test_case \"Update watchers for rotation without follow_inodes\" do\n    # The scenario where in_tail wrongly unwatches the PositionEntry.\n    # This is reported in https://github.com/fluent/fluentd/issues/3614.\n    def test_refreshTW_during_rotation\n      config = config_element(\n        \"ROOT\",\n        \"\",\n        {\n          \"path\" => \"#{@tmp_dir}/tail.txt0\",\n          \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n          \"tag\" => \"t1\",\n          \"format\" => \"none\",\n          \"read_from_head\" => \"true\",\n          # In order to detach the old watcher quickly.\n          \"rotate_wait\" => \"3s\",\n          # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not\n          # called by a timer.\n          \"refresh_interval\" => \"1h\",\n          # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,\n          # so disable it in order to reproduce the same condition stably.\n          \"enable_stat_watcher\" => \"false\",\n        }\n      )\n      d = create_driver(config, false)\n\n      tail_watchers = []\n      stub.proxy(d.instance).setup_watcher do |tw|\n        tail_watchers.append(tw)\n        tw\n      end\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"wb\") {|f| f.puts \"file1 log1\"}\n\n      d.run(expect_records: 6, timeout: 15) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"ab\") {|f| f.puts \"file1 log2\"}\n        FileUtils.move(\"#{@tmp_dir}/tail.txt0\", \"#{@tmp_dir}/tail.txt\" + \"1\")\n\n        # This reproduces the following situation:\n        #     `refresh_watchers` is called during the rotation process and it detects the current file being lost.\n        #     Then it stops and unwatches the TailWatcher.\n        d.instance.refresh_watchers\n\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"wb\") {|f| f.puts \"file2 log1\"}\n\n        # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to add the new TailWatcher.\n        # After `rotate_wait` interval, the PositionEntry is unwatched.\n        # HOWEVER, the new TailWatcher is still using that PositionEntry, so this breaks the PositionFile!!\n        # That PositionEntry is removed from `PositionFile::map`, but it is still working and remaining in the real pos file.\n        sleep 5\n\n        # Append to the new current log file.\n        # The PositionEntry is updated although it does not exist in `PositionFile::map`.\n        #     `PositionFile::map`: empty\n        #     Real pos file: `.../tail.txt 0000000000000016 (inode)`\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"ab\") {|f| f.puts \"file2 log2\"}\n\n        # Rotate again\n        [1, 0].each do |i|\n          FileUtils.move(\"#{@tmp_dir}/tail.txt#{i}\", \"#{@tmp_dir}/tail.txt#{i + 1}\")\n        end\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"wb\") {|f| f.puts \"file3 log1\"}\n\n        # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher.\n        sleep 3\n\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"ab\") {|f| f.puts \"file3 log2\"}\n      end\n\n      pos_file_inode = tail_watchers[2].pe.read_inode\n      record_values = d.events.collect { |event| event[2][\"message\"] }.sort\n      position_entries = []\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"r\") do |f|\n        f.readlines(chomp: true).each do |line|\n          values = line.split(\"\\t\")\n          position_entries.append([values[0], values[1], values[2].to_i(16)])\n        end\n      end\n\n      assert_equal(\n        {\n          record_values: [\"file1 log1\", \"file1 log2\", \"file2 log1\", \"file2 log2\", \"file3 log1\", \"file3 log2\"],\n          tail_watcher_paths: [\"#{@tmp_dir}/tail.txt0\", \"#{@tmp_dir}/tail.txt0\", \"#{@tmp_dir}/tail.txt0\"],\n          tail_watcher_io_handler_opened_statuses: [false, false, false],\n          position_entries: [\n            # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.\n            [\"#{@tmp_dir}/tail.txt0\", \"0000000000000016\", pos_file_inode],\n          ],\n        },\n        {\n          record_values: record_values,\n          tail_watcher_paths: tail_watchers.collect { |tw| tw.path },\n          tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },\n          position_entries: position_entries\n        },\n      )\n    end\n\n    def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait\n      config = config_element(\n        \"ROOT\",\n        \"\",\n        {\n          \"path\" => \"#{@tmp_dir}/tail.txt0\",\n          \"pos_file\" => \"#{@tmp_dir}/tail.pos\",\n          \"tag\" => \"t1\",\n          \"format\" => \"none\",\n          \"read_from_head\" => \"true\",\n          \"rotate_wait\" => \"3s\",\n          \"refresh_interval\" => \"1h\",\n        }\n      )\n      d = create_driver(config, false)\n\n      tail_watchers = []\n      stub.proxy(d.instance).setup_watcher do |tw|\n        tail_watchers.append(tw)\n        mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.\n        tw\n      end\n\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"wb\") {|f| f.puts \"file1 log1\"}\n\n      d.run(expect_records: 6, timeout: 15) do\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"ab\") {|f| f.puts \"file1 log2\"}\n\n        sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)\n\n        FileUtils.move(\"#{@tmp_dir}/tail.txt0\", \"#{@tmp_dir}/tail.txt\" + \"1\")\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"wb\") {|f| f.puts \"file2 log1\"}\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"ab\") {|f| f.puts \"file2 log2\"}\n\n        sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)\n\n        # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)\n        [1, 0].each do |i|\n          FileUtils.move(\"#{@tmp_dir}/tail.txt#{i}\", \"#{@tmp_dir}/tail.txt#{i + 1}\")\n        end\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"wb\") {|f| f.puts \"file3 log1\"}\n        Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.txt0\", \"ab\") {|f| f.puts \"file3 log2\"}\n\n        # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.\n        # (Note: Currently, there is no harm in duplicate calls)\n        sleep 4\n      end\n\n      pos_file_inode = tail_watchers[2].pe.read_inode\n      record_values = d.events.collect { |event| event[2][\"message\"] }.sort\n      position_entries = []\n      Fluent::FileWrapper.open(\"#{@tmp_dir}/tail.pos\", \"r\") do |f|\n        f.readlines(chomp: true).each do |line|\n          values = line.split(\"\\t\")\n          position_entries.append([values[0], values[1], values[2].to_i(16)])\n        end\n      end\n\n      assert_equal(\n        {\n          record_values: [\"file1 log1\", \"file1 log2\", \"file2 log1\", \"file2 log2\", \"file3 log1\", \"file3 log2\"],\n          tail_watcher_paths: [\"#{@tmp_dir}/tail.txt0\", \"#{@tmp_dir}/tail.txt0\", \"#{@tmp_dir}/tail.txt0\"],\n          tail_watcher_io_handler_opened_statuses: [false, false, false],\n          position_entries: [\n            [\"#{@tmp_dir}/tail.txt0\", \"0000000000000016\", pos_file_inode],\n          ],\n        },\n        {\n          record_values: record_values,\n          tail_watcher_paths: tail_watchers.collect { |tw| tw.path },\n          tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },\n          position_entries: position_entries\n        },\n      )\n    end\n  end\n\n  data(\n    small: [\"128\", 128],\n    KiB: [\"1k\", 1024]\n  )\n  test 'max_line_size' do |(label, size)|\n    config = config_element(\"\", \"\", {\n                              \"tag\" => \"max_line_size\",\n                              \"path\" => \"#{@tmp_dir}/with_long_lines.txt\",\n                              \"format\" => \"none\",\n                              \"read_from_head\" => true,\n                              \"max_line_size\" => label,\n                              \"log_level\" => \"debug\"\n                            })\n    Fluent::FileWrapper.open(\"#{@tmp_dir}/with_long_lines.txt\", \"w+\") do |f|\n      f.puts \"foo\"\n      f.puts \"x\" * size # 'x' * size + \\n > @max_line_size\n      f.puts \"bar\"\n    end\n    d = create_driver(config, false)\n    timestamp = Time.parse(\"Mon Nov 29 11:22:33 UTC 2021\")\n    Timecop.freeze(timestamp)\n    d.run(expect_records: 2)\n    assert_equal([\n                    [{\"message\" => \"foo\"},{\"message\" => \"bar\"}],\n                    [\n                      \"2021-11-29 11:22:33 +0000 [warn]: received line length is longer than #{size}\\n\",\n                      \"2021-11-29 11:22:33 +0000 [debug]: skipped line: #{'x' * size}\\n\"\n                    ]\n                  ],\n                  [\n                    d.events.collect { |event| event.last },\n                    d.logs[-2..]\n                  ])\n  end\n\n  test 'statistics' do\n    config = config_element(\"\", \"\", {\n      \"tag\" => \"statistics\",\n      \"path\" => \"#{@tmp_dir}/statistics*.txt\",\n      \"format\" => \"none\",\n      \"read_from_head\" => true,\n    })\n    Fluent::FileWrapper.open(\"#{@tmp_dir}/statistics1.txt\", \"w+\") do |f|\n      f.puts \"foo\"\n    end\n\n    d = create_driver(config, false)\n    d.run(expect_records: 1, shutdown: false)\n\n    assert_equal({\n                   \"emit_records\" => 0,\n                   \"emit_size\" => 0,\n                   \"opened_file_count\" => 1,\n                   \"closed_file_count\" => 0,\n                   \"rotated_file_count\" => 0,\n                   \"throttled_log_count\" =>0,\n                   \"tracked_file_count\" => 1,\n                 },\n                 d.instance.statistics[\"input\"])\n\n    d.instance_shutdown\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_tcp.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_tcp'\n\nclass TcpInputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    @port = unused_port(protocol: :tcp)\n  end\n\n  def teardown\n    @port = nil\n  end\n\n  def base_config\n    %[\n      port #{@port}\n      tag tcp\n    ]\n  end\n\n  def ipv4_config\n    base_config + %[\n      bind 127.0.0.1\n      format none\n    ]\n  end\n\n  def ipv6_config\n    base_config + %[\n      bind ::1\n      format none\n    ]\n  end\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::TcpInput).configure(conf)\n  end\n\n  def create_tcp_socket(host, port, &block)\n    if block_given?\n      TCPSocket.open(host, port, &block)\n    else\n      TCPSocket.open(host, port)\n    end\n  end\n\n\n  data(\n    'ipv4' => ['127.0.0.1', :ipv4],\n    'ipv6' => ['::1', :ipv6],\n  )\n  test 'configure' do |data|\n    bind, protocol = data\n    conf = send(\"#{protocol}_config\")\n    omit \"IPv6 is not supported on this environment\" if protocol == :ipv6 && !ipv6_enabled?\n\n    d = create_driver(conf)\n    assert_equal @port, d.instance.port\n    assert_equal bind, d.instance.bind\n    assert_equal \"\\n\", d.instance.delimiter\n  end\n\n  test ' configure w/o parse section' do\n    assert_raise(Fluent::ConfigError.new(\"<parse> section is required.\")) {\n      create_driver(base_config)\n    }\n  end\n\n  test_case_data = {\n    'none' => {\n      'format' => 'none',\n      'payloads' => [ \"tcptest1\\n\", \"tcptest2\\n\" ],\n      'expecteds' => [\n        {'message' => 'tcptest1'},\n        {'message' => 'tcptest2'},\n      ],\n    },\n    'json' => {\n      'format' => 'json',\n      'payloads' => [\n        {'k' => 123, 'message' => 'tcptest1'}.to_json + \"\\n\",\n        {'k' => 'tcptest2', 'message' => 456}.to_json + \"\\n\",\n      ],\n      'expecteds' => [\n        {'k' => 123, 'message' => 'tcptest1'},\n        {'k' => 'tcptest2', 'message' => 456}\n      ],\n    },\n  }\n\n  data(test_case_data)\n  test 'test_msg_size' do |data|\n    format = data['format']\n    payloads = data['payloads']\n    expecteds = data['expecteds']\n\n    d = create_driver(base_config + \"format #{format}\")\n    d.run(expect_records: 2) do\n      payloads.each do |payload|\n        create_tcp_socket('127.0.0.1', @port) do |sock|\n          sock.send(payload, 0)\n        end\n      end\n    end\n\n    assert_equal 2, d.events.size\n    expecteds.each_with_index do |expected_record, i|\n      assert_equal \"tcp\", d.events[i][0]\n      assert d.events[i][1].is_a?(Fluent::EventTime)\n      assert_equal expected_record, d.events[i][2]\n    end\n  end\n\n  data(test_case_data)\n  test 'test data in a connection' do |data|\n    format = data['format']\n    payloads = data['payloads']\n    expecteds = data['expecteds']\n\n    d = create_driver(base_config + \"format #{format}\")\n    d.run(expect_records: 2) do\n      create_tcp_socket('127.0.0.1', @port) do |sock|\n        payloads.each do |payload|\n          sock.send(payload, 0)\n        end\n      end\n    end\n\n    assert_equal 2, d.events.size\n    expecteds.each_with_index do |expected_record, i|\n      assert_equal \"tcp\", d.events[i][0]\n      assert d.events[i][1].is_a?(Fluent::EventTime)\n      assert_equal expected_record, d.events[i][2]\n    end\n  end\n\n  test 'source_hostname_key' do\n    d = create_driver(base_config + %!\n      format none\n      source_hostname_key host\n    !)\n    hostname = nil\n    d.run(expect_records: 1) do\n      create_tcp_socket('127.0.0.1', @port) do |sock|\n        sock.do_not_reverse_lookup = false\n        hostname = sock.peeraddr[2]\n        sock.send(\"test\\n\", 0)\n      end\n    end\n\n    assert_equal 1, d.events.size\n    event = d.events[0]\n    assert_equal \"tcp\", event[0]\n    assert event[1].is_a?(Fluent::EventTime)\n    assert_equal hostname, event[2]['host']\n  end\n\n  test \"send_keepalive_packet_can_be_enabled\" do\n    d = create_driver(base_config + %!\n      format none\n      send_keepalive_packet true\n    !)\n    assert_true d.instance.send_keepalive_packet\n\n    d = create_driver(base_config + %!\n      format none\n    !)\n    assert_false d.instance.send_keepalive_packet\n  end\n\n  test 'source_address_key' do\n    d = create_driver(base_config + %!\n      format none\n      source_address_key addr\n    !)\n    address = nil\n    d.run(expect_records: 1) do\n      create_tcp_socket('127.0.0.1', @port) do |sock|\n        address = sock.peeraddr[3]\n        sock.send(\"test\\n\", 0)\n      end\n    end\n\n    assert_equal 1, d.events.size\n    event = d.events[0]\n    assert_equal \"tcp\", event[0]\n    assert event[1].is_a?(Fluent::EventTime)\n    assert_equal address, event[2]['addr']\n  end\n\n  sub_test_case '<security>' do\n    test 'accept from allowed client' do\n      d = create_driver(ipv4_config + %!\n        <security>\n          <client>\n            network 127.0.0.1\n          </client>\n        </security>\n      !)\n      d.run(expect_records: 1) do\n        create_tcp_socket('127.0.0.1', @port) do |sock|\n          sock.send(\"hello\\n\", 0)\n        end\n      end\n\n      assert_equal 1, d.events.size\n      event = d.events[0]\n      assert_equal 'tcp', event[0]\n      assert_equal 'hello', event[2]['message']\n    end\n\n    test 'deny from disallowed client' do\n      d = create_driver(ipv4_config + %!\n        <security>\n          <client>\n            network 200.0.0.0\n          </client>\n        </security>\n      !)\n      d.run(expect_records: 1, timeout: 2) do\n        create_tcp_socket('127.0.0.1', @port) do |sock|\n          sock.send(\"hello\\n\", 0)\n        end\n      end\n\n      assert_equal 1, d.logs.count { |l| l =~ /anonymous client/ }\n      assert_equal 0, d.events.size\n    end\n  end\n\n  sub_test_case '<extract>' do\n    test 'extract tag from record field' do\n      d = create_driver(base_config + %!\n        <parse>\n          @type json\n        </parse>\n        <extract>\n          tag_key tag\n        </extract>\n      !)\n      d.run(expect_records: 1) do\n        create_tcp_socket('127.0.0.1', @port) do |sock|\n          data = {'msg' => 'hello', 'tag' => 'helper_test'}\n          sock.send(\"#{data.to_json}\\n\", 0)\n        end\n      end\n\n      assert_equal 1, d.events.size\n      event = d.events[0]\n      assert_equal 'helper_test', event[0]\n      assert event[1].is_a?(Fluent::EventTime)\n      assert_equal 'hello', event[2]['msg']\n    end\n  end\n\n  sub_test_case \"message_length_limit\" do\n    data(\"batch_emit\", { extract: \"\" }, keep: true)\n    data(\"single_emit\", { extract: \"<extract>\\ntag_key tag\\n</extract>\\n\" }, keep: true)\n    test \"drop records exceeding limit\" do |data|\n      message_length_limit = 10\n      d = create_driver(base_config + %!\n        message_length_limit #{message_length_limit}\n        <parse>\n          @type none\n        </parse>\n        #{data[:extract]}\n      !)\n      d.run(expect_records: 2, timeout: 10) do\n        create_tcp_socket('127.0.0.1', @port) do |sock|\n          sock.send(\"a\" * message_length_limit + \"\\n\", 0)\n          sock.send(\"b\" * (message_length_limit + 1) + \"\\n\", 0)\n          sock.send(\"c\" * (message_length_limit - 1) + \"\\n\", 0)\n        end\n      end\n\n      expected_records = [\n        \"a\" * message_length_limit,\n        \"c\" * (message_length_limit - 1)\n      ]\n      actual_records = d.events.collect do |event|\n        event[2][\"message\"]\n      end\n\n      assert_equal expected_records, actual_records\n    end\n\n    test \"clear buffer and discard the subsequent data until the next delimiter\" do |data|\n      message_length_limit = 12\n      d = create_driver(base_config + %!\n        message_length_limit #{message_length_limit}\n        delimiter \";\"\n        <parse>\n          @type json\n        </parse>\n        #{data[:extract]}\n      !)\n      d.run(expect_records: 1, timeout: 10) do\n        create_tcp_socket('127.0.0.1', @port) do |sock|\n          sock.send('{\"message\":', 0)\n          sock.send('\"hello', 0)\n          sleep 1 # To make the server read data and clear the buffer here.\n          sock.send('world!\"};', 0) # This subsequent data must be discarded so that a parsing failure doesn't occur.\n          sock.send('{\"k\":\"v\"};', 0) # This will succeed to parse.\n        end\n      end\n\n      logs = d.logs.collect do |log|\n        log.gsub(/\\A\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2} [-+]\\d{4} /, \"\")\n      end\n      actual_records = d.events.collect do |event|\n        event[2]\n      end\n\n      assert_equal(\n        {\n          # Asserting that '[warn]: pattern not matched message=\"world!\\\"}\"' warning does not occur.\n          logs: ['[info]: The buffer size exceeds \\'message_length_limit\\', cleared: limit=12 size=17 head=\"{\\\"message\\\":\\\"hello\"' + \"\\n\"],\n          records: [{\"k\" => \"v\"}],\n        },\n        {\n          logs: logs[1..],\n          records: actual_records,\n        }\n      )\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_udp.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_udp'\n\nclass UdpInputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    @port = unused_port(protocol: :udp)\n  end\n\n  def teardown\n    @port = nil\n  end\n\n  def base_config\n    %[\n      port #{@port}\n      tag udp\n    ]\n  end\n\n  def ipv4_config\n    base_config + %!\n      bind 127.0.0.1\n      format /^\\\\[(?<time>[^\\\\]]*)\\\\] (?<message>.*)/\n    !\n  end\n\n  def ipv6_config\n    base_config + %!\n      bind ::1\n      format /^\\\\[(?<time>[^\\\\]]*)\\\\] (?<message>.*)/\n    !\n  end\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::UdpInput).configure(conf)\n  end\n\n  def create_udp_socket(host, port)\n    u = if IPAddr.new(IPSocket.getaddress(host)).ipv4?\n          UDPSocket.new(Socket::AF_INET)\n        else\n          UDPSocket.new(Socket::AF_INET6)\n        end\n    u.do_not_reverse_lookup = false\n    u.connect(host, port)\n    if block_given?\n      begin\n        yield u\n      ensure\n        u.close rescue nil\n      end\n    else\n      u\n    end\n  end\n\n  data(\n    'ipv4' => ['127.0.0.1', :ipv4],\n    'ipv6' => ['::1', :ipv6],\n  )\n  test 'configure' do |data|\n    bind, protocol = data\n    conf = send(\"#{protocol}_config\")\n    omit \"IPv6 is not supported on this environment\" if protocol == :ipv6 && !ipv6_enabled?\n\n    d = create_driver(conf)\n    assert_equal @port, d.instance.port\n    assert_equal bind, d.instance.bind\n    assert_equal 4096, d.instance.message_length_limit\n    assert_equal nil, d.instance.receive_buffer_size\n  end\n\n  test ' configure w/o parse section' do\n    assert_raise(Fluent::ConfigError.new(\"<parse> section is required.\")) {\n      create_driver(base_config)\n    }\n  end\n\n  data(\n    'ipv4' => ['127.0.0.1', :ipv4],\n    'ipv6' => ['::1', :ipv6],\n  )\n  test 'time_format' do |data|\n    bind, protocol = data\n    conf = send(\"#{protocol}_config\")\n    omit \"IPv6 is not supported on this environment\" if protocol == :ipv6 && !ipv6_enabled?\n\n    d = create_driver(conf)\n\n    tests = [\n      {'msg' => '[Sep 11 00:00:00] localhost logger: foo', 'expected' => event_time('Sep 11 00:00:00', format: '%b %d %H:%M:%S')},\n      {'msg' => '[Sep  1 00:00:00] localhost logger: foo', 'expected' => event_time('Sep  1 00:00:00', format: '%b  %d %H:%M:%S')},\n    ]\n\n    d.run(expect_records: 2) do\n      create_udp_socket(bind, @port) do |u|\n        tests.each do |test|\n          u.send(test['msg'], 0)\n        end\n      end\n    end\n\n    events = d.events\n    tests.each_with_index do |t, i|\n      assert_equal_event_time(t['expected'], events[i][1])\n    end\n  end\n\n  data(\n    'message_length_limit' => 'message_length_limit 2048',\n    'body_size_limit' => 'body_size_limit 2048'\n  )\n  test 'message_length_limit/body_size_limit compatibility' do |param|\n\n    d = create_driver(ipv4_config + param)\n    assert_equal 2048, d.instance.message_length_limit\n  end\n\n  data(\n    'none' => {\n      'format' => 'none',\n      'payloads' => [\"tcptest1\\n\", \"tcptest2\\n\"],\n      'expecteds' => [\n        {\"message\" => \"tcptest1\"},\n        {\"message\" => \"tcptest2\"},\n      ],\n    },\n    'json' => {\n      'format' => 'json',\n      'payloads' => [\n        {'k' => 123, 'message' => 'tcptest1'}.to_json + \"\\n\",\n        {'k' => 'tcptest2', 'message' => 456}.to_json + \"\\n\",\n      ],\n      'expecteds' => [\n        {'k' => 123, 'message' => 'tcptest1'},\n        {'k' => 'tcptest2', 'message' => 456},\n      ],\n    },\n    'regexp' => {\n      'format' => '/^\\\\[(?<time>[^\\\\]]*)\\\\] (?<message>.*)/',\n      'payloads' => [\n        '[Sep 10 00:00:00] localhost: ' + 'x' * 100 + \"\\n\",\n        '[Sep 10 00:00:00] localhost: ' + 'x' * 1024 + \"\\n\"\n      ],\n      'expecteds' => [\n        {\"message\" => 'localhost: ' + 'x' * 100},\n        {\"message\" => 'localhost: ' + 'x' * 1024},\n      ],\n    },\n  )\n  test 'message size with format' do |data|\n    format = data['format']\n    payloads = data['payloads']\n    expecteds = data['expecteds']\n\n    d = create_driver(base_config + \"format #{format}\")\n    d.run(expect_records: 2) do\n      create_udp_socket('127.0.0.1', @port) do |u|\n        payloads.each do |payload|\n          u.send(payload, 0)\n        end\n      end\n    end\n\n    assert_equal 2, d.events.size\n    expecteds.each_with_index do |expected_record, i|\n      assert_equal \"udp\", d.events[i][0]\n      assert d.events[i][1].is_a?(Fluent::EventTime)\n      assert_equal expected_record, d.events[i][2]\n    end\n  end\n\n  test 'remove_newline' do\n    d = create_driver(base_config + %!\n      format none\n      remove_newline false\n    !)\n    payloads = [\"test1\\n\", \"test2\\n\"]\n    d.run(expect_records: 2) do\n      create_udp_socket('127.0.0.1', @port) do |u|\n        payloads.each do |payload|\n          u.send(payload, 0)\n        end\n      end\n    end\n\n    expecteds = payloads.map { |payload| {'message' => payload} }\n    assert_equal 2, d.events.size\n    expecteds.each_with_index do |expected_record, i|\n      assert_equal \"udp\", d.events[i][0]\n      assert d.events[i][1].is_a?(Fluent::EventTime)\n      assert_equal expected_record, d.events[i][2]\n    end\n  end\n\n  test 'source_hostname_key' do\n    d = create_driver(base_config + %!\n      format none\n      source_hostname_key host\n    !)\n    hostname = nil\n    d.run(expect_records: 1) do\n      create_udp_socket('127.0.0.1', @port) do |u|\n        u.send(\"test\", 0)\n        hostname = u.peeraddr[2]\n      end\n    end\n\n    assert_equal 1, d.events.size\n    assert_equal \"udp\", d.events[0][0]\n    assert d.events[0][1].is_a?(Fluent::EventTime)\n    assert_equal hostname, d.events[0][2]['host']\n  end\n\n  test 'source_address_key' do\n    d = create_driver(base_config + %!\n      format none\n      source_address_key addr\n    !)\n    address = nil\n    d.run(expect_records: 1) do\n      create_udp_socket('127.0.0.1', @port) do |u|\n        u.send(\"test\", 0)\n        address = u.peeraddr[3]\n      end\n    end\n\n    assert_equal 1, d.events.size\n    assert_equal \"udp\", d.events[0][0]\n    assert d.events[0][1].is_a?(Fluent::EventTime)\n    assert_equal address, d.events[0][2]['addr']\n  end\n\n  test 'receive_buffer_size' do\n    # doesn't check exact value because it depends on platform and condition\n\n    # check if default socket and in_udp's one without receive_buffer_size have same size buffer\n    d0 = create_driver(base_config + %!\n      format none\n    !)\n    d0.run do\n      sock = d0.instance.instance_variable_get(:@_servers)[0].server.instance_variable_get(:@sock)\n      begin\n        default_sock = UDPSocket.new\n        assert_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)\n      ensure\n        default_sock.close\n      end\n    end\n\n    # check if default socket and in_udp's one with receive_buffer_size have different size buffer\n    d1 = create_driver(base_config + %!\n      format none\n      receive_buffer_size 1001\n    !)\n    d1.run do\n      sock = d1.instance.instance_variable_get(:@_servers)[0].server.instance_variable_get(:@sock)\n      begin\n        default_sock = UDPSocket.new\n        assert_not_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)\n      ensure\n        default_sock.close\n      end\n    end\n  end\n\n  test 'message_length_limit' do\n    message_length_limit = 32\n\n    if Fluent.windows?\n      expected_records = [\"0\" * 30, \"4\" * 30]\n    else\n      expected_records = 1.upto(3).collect do |i|\n        \"#{i}\" * message_length_limit\n      end\n      expected_records.prepend(\"0\" * 30)\n      expected_records.append(\"4\" * 30)\n    end\n\n    d = create_driver(base_config + %!\n      format none\n      message_length_limit #{message_length_limit}\n    !)\n    d.run(expect_records: expected_records.size, timeout: 5) do\n      create_udp_socket('127.0.0.1', @port) do |u|\n        u.send(\"0\" * 30 + \"\\n\", 0)\n        1.upto(3) do |i|\n          u.send(\"#{i}\" * 40 + \"\\n\", 0)\n        end\n        u.send(\"4\" * 30 + \"\\n\", 0)\n      end\n    end\n\n    actual_records = d.events.collect do |event|\n      event[2][\"message\"]\n    end\n\n    assert_equal expected_records, actual_records\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_in_unix.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_unix'\n\nclass UnixInputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    @d = nil\n  end\n\n  def teardown\n    @d.instance_shutdown if @d\n  end\n\n  TMP_DIR = File.dirname(__FILE__) + \"/../tmp/in_unix#{ENV['TEST_ENV_NUMBER']}\"\n  CONFIG = %[\n    path #{TMP_DIR}/unix\n    backlog 1000\n  ]\n\n  def create_driver(conf = CONFIG)\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::UnixInput).configure(conf)\n  end\n\n  def packer(*args)\n    Fluent::MessagePackFactory.msgpack_packer(*args)\n  end\n\n  def unpacker\n    Fluent::MessagePackFactory.msgpack_unpacker\n  end\n\n  def send_data(data)\n    io = UNIXSocket.new(\"#{TMP_DIR}/unix\")\n    begin\n      io.write data\n    ensure\n      io.close\n    end\n  end\n\n  def test_configure\n    @d = create_driver\n    assert_equal \"#{TMP_DIR}/unix\", @d.instance.path\n    assert_equal 1000, @d.instance.backlog\n  end\n\n  def test_time\n    @d = create_driver\n\n    time = Fluent::EventTime.now\n    records = [\n      [\"tag1\", 0, {\"a\" => 1}],\n      [\"tag2\", nil, {\"a\" => 2}],\n    ]\n\n    @d.run(expect_records: records.length, timeout: 5) do\n      records.each {|tag, _time, record|\n        send_data packer.write([tag, _time, record]).to_s\n      }\n    end\n\n    @d.events.each_with_index { |e, i|\n      orig = records[i]\n      assert_equal(orig[0], e[0])\n      assert_true(time <= e[1])\n      assert_equal(orig[2], e[2])\n    }\n  end\n\n  def test_message\n    @d = create_driver\n\n    time = Fluent::EventTime.now\n    records = [\n      [\"tag1\", time, {\"a\" => 1}],\n      [\"tag2\", time, {\"a\" => 2}],\n    ]\n\n    @d.run(expect_records: records.length, timeout: 5) do\n      records.each {|tag, _time, record|\n        send_data packer.write([tag, _time, record]).to_s\n      }\n    end\n\n    assert_equal(records, @d.events)\n  end\n\n  def test_forward\n    @d = create_driver\n\n    time = Fluent::EventTime.parse(\"2011-01-02 13:14:15 UTC\")\n    records = [\n      [\"tag1\", time, {\"a\" => 1}],\n      [\"tag1\", time, {\"a\" => 2}]\n    ]\n\n    @d.run(expect_records: records.length, timeout: 20) do\n      entries = []\n      records.each {|tag, _time, record|\n        entries << [_time, record]\n      }\n      send_data packer.write([\"tag1\", entries]).to_s\n    end\n    assert_equal(records, @d.events)\n  end\n\n  def test_packed_forward\n    @d = create_driver\n\n    time = Fluent::EventTime.now\n    records = [\n      [\"tag1\", time, {\"a\" => 1}],\n      [\"tag1\", time, {\"a\" => 2}],\n    ]\n\n    @d.run(expect_records: records.length, timeout: 20) do\n      entries = ''\n      records.each {|_tag, _time, record|\n        packer(entries).write([_time, record]).flush\n      }\n      send_data packer.write([\"tag1\", entries]).to_s\n    end\n    assert_equal(records, @d.events)\n  end\n\n  def test_message_json\n    @d = create_driver\n\n    time = Fluent::EventTime.now\n    records = [\n      [\"tag1\", time, {\"a\" => 1}],\n      [\"tag2\", time, {\"a\" => 2}],\n    ]\n\n    @d.run(expect_records: records.length, timeout: 5) do\n      tag, _time, record = records[0]\n      send_data [tag, _time.to_i, record].to_json\n      tag, _time, record = records[1]\n      send_data [tag, _time.to_f, record].to_json\n    end\n\n    assert_equal(records, @d.events)\n  end\n\n  def test_message_with_tag\n    @d = create_driver(CONFIG + \"tag new_tag\")\n\n    time = Fluent::EventTime.now\n    records = [\n      [\"tag1\", time, {\"a\" => 1}],\n      [\"tag2\", time, {\"a\" => 2}],\n    ]\n\n    @d.run(expect_records: records.length, timeout: 5) do\n      records.each {|tag, _time, record|\n        send_data packer.write([tag, _time, record]).to_s\n      }\n    end\n\n    @d.events.each { |event|\n      assert_equal(\"new_tag\", event[0])\n    }\n  end\n\n  data('string chunk' => 'broken string',\n       'integer chunk' => 10)\n  def test_broken_message(data)\n    @d = create_driver\n    @d.run(shutdown: false, timeout: 5) do\n      @d.instance.__send__(:on_message, data)\n    end\n\n    assert_equal 0, @d.events.size\n\n    logs = @d.instance.log.logs\n    assert_equal 1, logs.count { |line|\n      line =~ / \\[warn\\]: incoming data is broken: msg=#{data.inspect}/\n    }, \"should not accept broken chunk\"\n  end\nend unless Fluent.windows?\n"
  },
  {
    "path": "test/plugin/test_input.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/input'\nrequire 'flexmock/test_unit'\n\nmodule FluentPluginInputTest\n  class DummyPlugin < Fluent::Plugin::Input\n  end\nend\n\nclass InputTest < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n    @p = FluentPluginInputTest::DummyPlugin.new\n  end\n\n  test 'has healthy lifecycle' do\n    assert !@p.configured?\n    @p.configure(config_element())\n    assert @p.configured?\n\n    assert !@p.started?\n    @p.start\n    assert @p.start\n\n    assert !@p.stopped?\n    @p.stop\n    assert @p.stopped?\n\n    assert !@p.before_shutdown?\n    @p.before_shutdown\n    assert @p.before_shutdown?\n\n    assert !@p.shutdown?\n    @p.shutdown\n    assert @p.shutdown?\n\n    assert !@p.after_shutdown?\n    @p.after_shutdown\n    assert @p.after_shutdown?\n\n    assert !@p.closed?\n    @p.close\n    assert @p.closed?\n\n    assert !@p.terminated?\n    @p.terminate\n    assert @p.terminated?\n  end\n\n  test 'has plugin_id automatically generated' do\n    assert @p.respond_to?(:plugin_id_configured?)\n    assert @p.respond_to?(:plugin_id)\n\n    @p.configure(config_element())\n\n    assert !@p.plugin_id_configured?\n    assert @p.plugin_id\n    assert{ @p.plugin_id != 'mytest' }\n  end\n\n  test 'has plugin_id manually configured' do\n    @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))\n    assert @p.plugin_id_configured?\n    assert_equal 'mytest', @p.plugin_id\n  end\n\n  test 'has plugin logger' do\n    assert @p.respond_to?(:log)\n    assert @p.log\n\n    # default logger\n    original_logger = @p.log\n\n    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))\n\n    assert(@p.log.object_id != original_logger.object_id)\n    assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level\n  end\n\n  test 'can load plugin helpers' do\n    assert_nothing_raised do\n      class FluentPluginInputTest::DummyPlugin2 < Fluent::Plugin::Input\n        helpers :storage\n      end\n    end\n  end\n\n  test 'can use metrics plugins and fallback methods' do\n    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))\n\n    %w[emit_size_metrics emit_records_metrics].each do |metric_name|\n      assert_true @p.instance_variable_get(:\"@#{metric_name}\").is_a?(Fluent::Plugin::Metrics)\n    end\n\n    assert_equal 0, @p.emit_size\n    assert_equal 0, @p.emit_records\n  end\n\n  test 'are not available with multi workers configuration in default' do\n    assert_false @p.multi_workers_ready?\n  end\n\n  test 'has router and can emit into it' do\n    assert @p.has_router?\n\n    @p.configure(config_element())\n    assert @p.router\n\n    DummyRouter = Struct.new(:emits) do\n      def emit(tag, es)\n        self.emits << [tag, es]\n      end\n    end\n    @p.router = DummyRouter.new([])\n    @p.router.emit('mytag', [])\n    @p.router.emit('mytag.testing', ['it is not es, but no problem for tests'])\n\n    assert_equal ['mytag', []], @p.router.emits[0]\n    assert_equal ['mytag.testing', ['it is not es, but no problem for tests']], @p.router.emits[1]\n  end\n\n  test 'has router for specified label if configured' do\n    @p.configure(config_element())\n    original_router = @p.router\n\n    router_mock = flexmock('mytest')\n    router_mock.should_receive(:emit).once.with('mytag.testing', ['for mock'])\n    label_mock = flexmock('mylabel')\n    label_mock.should_receive(:event_router).once.and_return(router_mock)\n    Fluent::Engine.root_agent.labels['@mytest'] = label_mock\n\n    @p.configure(config_element('ROOT', '', {'@label' => '@mytest'}))\n    assert{ @p.router.object_id != original_router.object_id }\n\n    @p.router.emit('mytag.testing', ['for mock'])\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_metadata.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/buffer'\n\nclass BufferMetadataTest < Test::Unit::TestCase\n\n  def meta(timekey=nil, tag=nil, variables=nil)\n    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n  end\n\n  setup do\n    Fluent::Test.setup\n  end\n\n  sub_test_case 'about metadata' do\n    test 'comparison of variables should be stable' do\n      m = meta(nil, nil, nil)\n      # different sets of keys\n      assert_equal(-1, m.cmp_variables({}, {a: 1}))\n      assert_equal(1, m.cmp_variables({a: 1}, {}))\n      assert_equal(1, m.cmp_variables({c: 1}, {a: 1}))\n      assert_equal(-1, m.cmp_variables({a: 1}, {a: 1, b: 2}))\n      assert_equal(1, m.cmp_variables({a: 1, c: 1}, {a: 1, b: 2}))\n      assert_equal(1, m.cmp_variables({a: 1, b: 0, c: 1}, {a: 1, b: 2}))\n      # same set of keys\n      assert_equal(-1, m.cmp_variables({a: 1}, {a: 2}))\n      assert_equal(-1, m.cmp_variables({a: 1, b: 0}, {a: 1, b: 1}))\n      assert_equal(-1, m.cmp_variables({a: 1, b: 1, c: 100}, {a: 1, b: 1, c: 200}))\n      assert_equal(-1, m.cmp_variables({b: 1, c: 100, a: 1}, {a: 1, b: 1, c: 200})) # comparison sorts keys\n      assert_equal(-1, m.cmp_variables({a: nil}, {a: 1}))\n      assert_equal(-1, m.cmp_variables({a: 1, b: nil}, {a: 1, b: 1}))\n    end\n\n    test 'comparison of metadata should be stable' do\n      n = Time.now.to_i\n\n      assert_equal(0, meta(nil, nil, nil) <=> meta(nil, nil, nil))\n      assert_equal(0, meta(n, nil, nil) <=> meta(n, nil, nil))\n      assert_equal(0, meta(nil, \"t1\", nil) <=> meta(nil, \"t1\", nil))\n      assert_equal(0, meta(nil, nil, {}) <=> meta(nil, nil, {}))\n      assert_equal(0, meta(nil, nil, {a: \"1\"}) <=> meta(nil, nil, {a: \"1\"}))\n      assert_equal(0, meta(n, nil, {}) <=> meta(n, nil, {}))\n      assert_equal(0, meta(n, \"t1\", {}) <=> meta(n, \"t1\", {}))\n      assert_equal(0, meta(n, \"t1\", {a: \"x\", b: 10}) <=> meta(n, \"t1\", {a: \"x\", b: 10}))\n\n      # timekey is 1st comparison key\n      assert_equal(-1, meta(n - 300, nil, nil) <=> meta(n - 270, nil, nil))\n      assert_equal(1, meta(n + 1, \"a\", nil) <=> meta(n - 1, \"b\", nil))\n      assert_equal(-1, meta(n - 1, nil, {a: 100}) <=> meta(n + 1, nil, {}))\n\n      # tag is 2nd\n      assert_equal(-1, meta(nil, \"a\", {}) <=> meta(nil, \"b\", {}))\n      assert_equal(-1, meta(n, \"a\", {}) <=> meta(n, \"b\", {}))\n      assert_equal(1, meta(nil, \"x\", {a: 1}) <=> meta(nil, \"t\", {}))\n      assert_equal(1, meta(n, \"x\", {a: 1}) <=> meta(n, \"t\", {}))\n      assert_equal(1, meta(nil, \"x\", {a: 1}) <=> meta(nil, \"t\", {a: 1}))\n      assert_equal(1, meta(n, \"x\", {a: 1}) <=> meta(n, \"t\", {a: 2}))\n      assert_equal(1, meta(n, \"x\", {a: 1}) <=> meta(n, \"t\", {a: 10, b: 1}))\n\n      # variables is the last\n      assert_equal(-1, meta(nil, nil, {}) <=> meta(nil, nil, {a: 1}))\n      assert_equal(-1, meta(n, \"t\", {}) <=> meta(n, \"t\", {a: 1}))\n      assert_equal(1, meta(n, \"t\", {a: 1}) <=> meta(n, \"t\", {}))\n      assert_equal(-1, meta(n, \"t\", {a: 1}) <=> meta(n, \"t\", {a: 2}))\n      assert_equal(-1, meta(n, \"t\", {a: 1}) <=> meta(n, \"t\", {a: 1, b: 1}))\n      assert_equal(1, meta(nil, nil, {b: 1}) <=> meta(nil, nil, {a: 1}))\n      assert_equal(1, meta(n, \"t\", {b: 1}) <=> meta(n, \"t\", {a: 1}))\n    end\n\n    test 'metadata can be sorted' do\n      n = Time.now.to_i\n      m0 = meta(nil, nil, nil)\n      m1 = meta(n - 1, nil, nil)\n      m2 = meta(n - 1, \"a\", nil)\n      m3 = meta(n - 1, \"a\", {a: 1})\n      m4 = meta(n - 1, \"a\", {a: 100})\n      m5 = meta(n - 1, \"a\", {a: 100, b: 1})\n      m6 = meta(n - 1, \"aa\", nil)\n      m7 = meta(n - 1, \"aa\", {a: 1})\n      m8 = meta(n - 1, \"b\", nil)\n      m9 = meta(n, nil, nil)\n      m10 = meta(n + 1, nil, {a: 1})\n      expected = [m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10].freeze\n      ary = expected.dup\n      100.times do\n        assert_equal expected, ary.shuffle.sort\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_metrics.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/metrics'\nrequire 'fluent/plugin/base'\nrequire 'fluent/system_config'\n\nclass BareMetrics < Fluent::Plugin::Metrics\n  Fluent::Plugin.register_metrics('bare', self)\n\n  private\n\n  # Just override for tests.\n  def has_methods_for_counter?\n    false\n  end\nend\n\nclass BasicCounterMetrics < Fluent::Plugin::Metrics\n  Fluent::Plugin.register_metrics('example', self)\n\n  attr_reader :data\n\n  def initialize\n    super\n    @data = 0\n  end\n  def get\n    @data\n  end\n  def inc\n    @data +=1\n  end\n  def add(value)\n    @data += value\n  end\n  def set(value)\n    @data = value\n  end\n  def close\n    @data = 0\n    super\n  end\nend\n\nclass AliasedCounterMetrics < Fluent::Plugin::Metrics\n  Fluent::Plugin.register_metrics('example', self)\n\n  attr_reader :data\n\n  def initialize\n    super\n    @data = 0\n  end\n  def configure(conf)\n    super\n    class << self\n      alias_method :set, :set_counter\n    end\n  end\n  def get\n    @data\n  end\n  def inc\n    @data +=1\n  end\n  def add(value)\n    @data += value\n  end\n  def set_counter(value)\n    @data = value\n  end\n  def close\n    @data = 0\n    super\n  end\nend\n\nclass BasicGaugeMetrics < Fluent::Plugin::Metrics\n  Fluent::Plugin.register_metrics('example', self)\n\n  attr_reader :data\n\n  def initialize\n    super\n    @data = 0\n  end\n  def get\n    @data\n  end\n  def inc\n    @data +=1\n  end\n  def dec\n    @data -=1\n  end\n  def add(value)\n    @data += value\n  end\n  def sub(value)\n    @data -= value\n  end\n  def set(value)\n    @data = value\n  end\n  def close\n    @data = 0\n    super\n  end\nend\n\nclass AliasedGaugeMetrics < Fluent::Plugin::Metrics\n  Fluent::Plugin.register_metrics('example', self)\n\n  attr_reader :data\n\n  def initialize\n    super\n    @data = 0\n  end\n  def configure(conf)\n    super\n    class << self\n      alias_method :dec, :dec_gauge\n      alias_method :set, :set_gauge\n      alias_method :sub, :sub_gauge\n    end\n  end\n  def get\n    @data\n  end\n  def inc\n    @data +=1\n  end\n  def dec_gauge\n    @data -=1\n  end\n  def add(value)\n    @data += value\n  end\n  def sub_gauge(value)\n    @data -= value\n  end\n  def set_gauge(value)\n    @data = value\n  end\n  def close\n    @data = 0\n    super\n  end\nend\n\nclass StorageTest < Test::Unit::TestCase\n  sub_test_case 'BareMetrics' do\n    setup do\n      @m = BareMetrics.new\n      @m.configure(config_element())\n    end\n\n    test 'is configured with plugin information and system config' do\n      m = BareMetrics.new\n      m.configure(config_element('metrics', '', {}))\n\n      assert_false m.use_gauge_metric\n      assert_false m.has_methods_for_counter\n      assert_false m.has_methods_for_gauge\n    end\n\n    test 'all bare operations are not defined yet' do\n      assert_raise NotImplementedError do\n        @m.get\n      end\n      assert_raise NotImplementedError do\n        @m.inc\n      end\n      assert_raise NotImplementedError do\n        @m.dec\n      end\n      assert_raise NotImplementedError do\n        @m.add(10)\n      end\n      assert_raise NotImplementedError do\n        @m.sub(11)\n      end\n      assert_raise NotImplementedError do\n        @m.set(123)\n      end\n    end\n  end\n\n  sub_test_case 'BasicCounterMetric' do\n    setup do\n      @m = BasicCounterMetrics.new\n      @m.configure(config_element('metrics', '', {'@id' => '1'}))\n    end\n\n    test 'all basic counter operations work well' do\n      assert_true @m.has_methods_for_counter\n      assert_false @m.has_methods_for_gauge\n\n      assert_equal 0, @m.get\n      assert_equal 1, @m.inc\n\n      @m.add(20)\n      assert_equal 21, @m.get\n      assert_raise NotImplementedError do\n        @m.dec\n      end\n\n      @m.set(100)\n      assert_equal 100, @m.get\n      assert_raise NotImplementedError do\n        @m.sub(11)\n      end\n    end\n  end\n\n  sub_test_case 'AliasedCounterMetric' do\n    setup do\n      @m = AliasedCounterMetrics.new\n      @m.configure(config_element('metrics', '', {}))\n    end\n\n    test 'all aliased counter operations work well' do\n      assert_true @m.has_methods_for_counter\n      assert_false @m.has_methods_for_gauge\n\n      assert_equal 0, @m.get\n      assert_equal 1, @m.inc\n\n      @m.add(20)\n      assert_equal 21, @m.get\n      assert_raise NotImplementedError do\n        @m.dec\n      end\n\n      @m.set(100)\n      assert_equal 100, @m.get\n      assert_raise NotImplementedError do\n        @m.sub(11)\n      end\n    end\n  end\n\n  sub_test_case 'BasicGaugeMetric' do\n    setup do\n      @m = BasicGaugeMetrics.new\n      @m.use_gauge_metric = true\n      @m.configure(config_element('metrics', '', {}))\n    end\n\n    test 'all basic gauge operations work well' do\n      assert_false @m.has_methods_for_counter\n      assert_true @m.has_methods_for_gauge\n\n      assert_equal 0, @m.get\n      assert_equal 1, @m.inc\n\n      @m.add(20)\n      assert_equal 21, @m.get\n      @m.dec\n      assert_equal 20, @m.get\n\n      @m.set(100)\n      assert_equal 100, @m.get\n      @m.sub(11)\n      assert_equal 89, @m.get\n    end\n  end\n\n  sub_test_case 'AliasedGaugeMetric' do\n    setup do\n      @m = AliasedGaugeMetrics.new\n      @m.use_gauge_metric = true\n      @m.configure(config_element('metrics', '', {}))\n    end\n\n    test 'all aliased gauge operations work well' do\n      assert_false @m.has_methods_for_counter\n      assert_true @m.has_methods_for_gauge\n\n      assert_equal 0, @m.get\n      assert_equal 1, @m.inc\n\n      @m.add(20)\n      assert_equal 21, @m.get\n      @m.dec\n      assert_equal 20, @m.get\n\n      @m.set(100)\n      assert_equal 100, @m.get\n      @m.sub(11)\n      assert_equal 89, @m.get\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_metrics_local.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/metrics_local'\nrequire 'fluent/system_config'\n\nclass LocalMetricsTest < ::Test::Unit::TestCase\n  sub_test_case 'configure' do\n    test \"configured for counter mode\" do\n      m = Fluent::Plugin::LocalMetrics.new\n      m.configure(config_element('metrics', '', {\"labels\" => {test: \"test-unit\", language: \"Ruby\"}}))\n\n      assert_false m.use_gauge_metric\n      assert_equal({agent: \"Fluentd\", hostname: \"#{Socket.gethostname}\"}, m.default_labels)\n      assert_equal({test: \"test-unit\", language: \"Ruby\"}, m.labels)\n      assert_true m.has_methods_for_counter\n      assert_false m.has_methods_for_gauge\n    end\n\n    test \"configured for gauge mode\" do\n      m = Fluent::Plugin::LocalMetrics.new\n      m.use_gauge_metric = true\n      m.configure(config_element('metrics', '', {\"labels\" => {test: \"test-unit\", language: \"Ruby\"}}))\n\n      assert_true m.use_gauge_metric\n      assert_equal({agent: \"Fluentd\", hostname: \"#{Socket.gethostname}\"}, m.default_labels)\n      assert_equal({test: \"test-unit\", language: \"Ruby\"}, m.labels)\n      assert_false m.has_methods_for_counter\n      assert_true m.has_methods_for_gauge\n    end\n  end\n\n  sub_test_case 'LocalMetric' do\n    sub_test_case \"counter\" do\n      setup do\n        @m = Fluent::Plugin::LocalMetrics.new\n        @m.configure(config_element('metrics', '', {}))\n      end\n\n      test '#configure' do\n        assert_true @m.has_methods_for_counter\n        assert_false @m.has_methods_for_gauge\n      end\n\n      test 'all local counter operations work well' do\n        assert_equal 0, @m.get\n        assert_equal 1, @m.inc\n\n        @m.add(20)\n        assert_equal 21, @m.get\n        assert_raise NotImplementedError do\n          @m.dec\n        end\n\n        @m.set(100)\n        assert_equal 100, @m.get\n\n        @m.set(10)\n        assert_equal 100, @m.get # On counter, value should be overwritten bigger than stored one.\n        assert_raise NotImplementedError do\n          @m.sub(11)\n        end\n      end\n    end\n\n    sub_test_case \"gauge\" do\n      setup do\n        @m = Fluent::Plugin::LocalMetrics.new\n        @m.use_gauge_metric = true\n        @m.configure(config_element('metrics', '', {}))\n      end\n\n      test '#configure' do\n        assert_false @m.has_methods_for_counter\n        assert_true @m.has_methods_for_gauge\n      end\n\n      test 'all local gauge operations work well' do\n        assert_equal 0, @m.get\n        assert_equal 1, @m.inc\n\n        @m.add(20)\n        assert_equal 21, @m.get\n        @m.dec\n        assert_equal 20, @m.get\n\n        @m.set(100)\n        assert_equal 100, @m.get\n\n        @m.sub(11)\n        assert_equal 89, @m.get\n\n        @m.set(10)\n        assert_equal 10, @m.get # On gauge, value always should be overwritten.\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_multi_output.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/multi_output'\nrequire 'fluent/event'\n\nrequire 'json'\nrequire 'time'\nrequire 'timeout'\n\nmodule FluentPluginMultiOutputTest\n  class DummyMultiOutput < Fluent::Plugin::MultiOutput\n    attr_reader :events\n    def initialize\n      super\n      @events = []\n    end\n    def configure(conf)\n      super\n    end\n    def process(tag, es)\n      es.each do |time, record|\n        @events << [tag, time, record]\n      end\n    end\n  end\n  class DummyCompatMultiOutput < Fluent::Plugin::MultiOutput\n    def initialize\n      super\n      @compat = true\n    end\n    def configure(conf)\n      super\n    end\n    def process(tag, es)\n      # ...\n    end\n  end\n\n  class Dummy1Output < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('dummy_test_multi_output_1', self)\n    attr_reader :configured\n    def configure(conf)\n      super\n      @configured = true\n    end\n    def process(tag, es)\n    end\n  end\n  class Dummy2Output < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('dummy_test_multi_output_2', self)\n    attr_reader :configured\n    def configure(conf)\n      super\n      @configured = true\n    end\n    def process(tag, es)\n    end\n  end\n  class Dummy3Output < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('dummy_test_multi_output_3', self)\n    attr_reader :configured\n    def configure(conf)\n      super\n      @configured = true\n    end\n    def process(tag, es)\n    end\n  end\n  class Dummy4Output < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('dummy_test_multi_output_4', self)\n    attr_reader :configured\n    def configure(conf)\n      super\n      @configured = true\n    end\n    def process(tag, es)\n    end\n  end\nend\n\nclass MultiOutputTest < Test::Unit::TestCase\n  def create_output(type=:multi)\n    case type\n    when :compat_multi\n      FluentPluginMultiOutputTest::DummyCompatMultiOutput.new\n    else\n      FluentPluginMultiOutputTest::DummyMultiOutput.new\n    end\n  end\n\n  sub_test_case 'basic multi output plugin' do\n    setup do\n      Fluent::Test.setup\n      @i = create_output()\n    end\n\n    teardown do\n      @i.log.out.reset\n    end\n\n    test '#configure raises error if <store> sections are missing' do\n      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' }, [])\n      assert_raise Fluent::ConfigError do\n        @i.configure(conf)\n      end\n    end\n\n    test '#configure initialize child plugins and call these #configure' do\n      assert_equal [], @i.outputs\n\n      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },\n        [\n          config_element('store', '', { '@type' => 'dummy_test_multi_output_1' }),\n          config_element('store', '', { '@type' => 'dummy_test_multi_output_2' }),\n          config_element('store', '', { '@type' => 'dummy_test_multi_output_3' }),\n          config_element('store', '', { '@type' => 'dummy_test_multi_output_4' }),\n        ]\n      )\n      @i.configure(conf)\n\n      assert_equal 4, @i.outputs.size\n\n      assert @i.outputs[0].is_a? FluentPluginMultiOutputTest::Dummy1Output\n      assert @i.outputs[0].configured\n\n      assert @i.outputs[1].is_a? FluentPluginMultiOutputTest::Dummy2Output\n      assert @i.outputs[1].configured\n\n      assert @i.outputs[2].is_a? FluentPluginMultiOutputTest::Dummy3Output\n      assert @i.outputs[2].configured\n\n      assert @i.outputs[3].is_a? FluentPluginMultiOutputTest::Dummy4Output\n      assert @i.outputs[3].configured\n    end\n\n    test '#configure warns if \"type\" is used in <store> sections instead of \"@type\"' do\n      assert_equal [], @i.log.out.logs\n\n      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },\n        [\n          config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),\n          config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),\n          config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),\n          config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),\n        ]\n      )\n      @i.configure(conf)\n      assert_equal 4, @i.outputs.size\n\n      log_size_for_multi_output_itself = 4\n      log_size_for_metrics_plugin_helper = 4\n      expected_warn_log_size = log_size_for_multi_output_itself + log_size_for_metrics_plugin_helper\n      logs = @i.log.out.logs\n      assert{ logs.count{|log| log.include?('[warn]') && log.include?(\"'type' is deprecated parameter name. use '@type' instead.\") } == expected_warn_log_size }\n    end\n\n    test '#emit_events calls #process always' do\n      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },\n        [\n          config_element('store', '', { '@type' => 'dummy_test_multi_output_1' }),\n          config_element('store', '', { '@type' => 'dummy_test_multi_output_2' }),\n          config_element('store', '', { '@type' => 'dummy_test_multi_output_3' }),\n          config_element('store', '', { '@type' => 'dummy_test_multi_output_4' }),\n        ]\n      )\n      @i.configure(conf)\n      @i.start\n\n      assert @i.events.empty?\n\n      @i.emit_events(\n        'test.tag',\n        Fluent::ArrayEventStream.new(\n          [\n            [event_time(), {\"message\" => \"multi test 1\"}],\n            [event_time(), {\"message\" => \"multi test 1\"}],\n          ]\n        )\n      )\n\n      assert_equal 2, @i.events.size\n    end\n\n    test 'can use metrics plugins and fallback methods' do\n      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },\n        [\n          config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),\n          config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),\n          config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),\n          config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),\n        ]\n      )\n      @i.configure(conf)\n\n      %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|\n        assert_true @i.instance_variable_get(:\"@#{metric_name}\").is_a?(Fluent::Plugin::Metrics)\n      end\n\n      assert_equal 0, @i.num_errors\n      assert_equal 0, @i.emit_count\n      assert_equal 0, @i.emit_size\n      assert_equal 0, @i.emit_records\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_buffer.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/output'\nrequire 'fluent/plugin/out_buffer'\n\nclass BufferOutputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def create_driver(conf = \"\")\n    Fluent::Test::Driver::Output.new(Fluent::Plugin::BufferOutput).configure(conf)\n  end\n\n  test \"default setting\" do\n    d = create_driver(\n      config_element(\n        \"ROOT\", \"\", {},\n        [config_element(\"buffer\", \"\", {\"path\" => \"test\"})]\n      )\n    )\n\n    assert_equal(\n      [\n        \"file\",\n        [\"tag\"],\n        :interval,\n        10,\n      ],\n      [\n        d.instance.buffer_config[\"@type\"],\n        d.instance.buffer_config.chunk_keys,\n        d.instance.buffer_config.flush_mode,\n        d.instance.buffer_config.flush_interval,\n      ]\n    )\n  end\n\n  test \"#write\" do\n    d = create_driver(\n      config_element(\n        \"ROOT\", \"\", {},\n        [config_element(\"buffer\", \"\", {\"@type\" => \"memory\", \"flush_mode\" => \"immediate\"})]\n      )\n    )\n\n    time = event_time\n    record = {\"message\" => \"test\"}\n    d.run(default_tag: 'test') do\n      d.feed(time, record)\n    end\n\n    assert_equal [[\"test\", time, record]], d.events\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_copy.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/multi_output'\nrequire 'fluent/plugin/out_copy'\nrequire 'fluent/event'\nrequire 'flexmock/test_unit'\n\nclass CopyOutputTest < Test::Unit::TestCase\n  include FlexMock::TestCase\n\n  class << self\n    def startup\n      $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))\n      require 'fluent/plugin/out_test'\n      require 'fluent/plugin/out_test2'\n    end\n\n    def shutdown\n      $LOAD_PATH.shift\n    end\n  end\n\n  def setup\n    Fluent::Test.setup\n  end\n\n  CONFIG = %[\n    <store>\n      @type test\n      name c0\n    </store>\n    <store>\n      @type test2\n      name c1\n    </store>\n    <store>\n      @type test\n      name c2\n    </store>\n  ]\n\n  def create_driver(conf = CONFIG)\n    Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::CopyOutput).configure(conf)\n  end\n\n  def test_configure\n    d = create_driver\n\n    outputs = d.instance.outputs\n    assert_equal 3, outputs.size\n    assert_equal Fluent::Plugin::TestOutput, outputs[0].class\n    assert_equal Fluent::Plugin::Test2Output, outputs[1].class\n    assert_equal Fluent::Plugin::TestOutput, outputs[2].class\n    assert_equal \"c0\", outputs[0].name\n    assert_equal \"c1\", outputs[1].name\n    assert_equal \"c2\", outputs[2].name\n    assert_false d.instance.deep_copy\n    assert_equal :no_copy, d.instance.copy_mode\n  end\n\n  ERRORNEOUS_IGNORE_IF_PREV_SUCCESS_CONFIG = %[\n    <store ignore_if_prev_success ignore_error>\n      @type test\n      name c0\n    </store>\n    <store ignore_if_prev_success ignore_error>\n      @type test\n      name c1\n    </store>\n    <store ignore_if_prev_success>\n      @type test\n      name c2\n    </store>\n  ]\n  def test_configure_with_errorneus_ignore_if_prev_success\n    assert_raise(Fluent::ConfigError) do\n      create_driver(ERRORNEOUS_IGNORE_IF_PREV_SUCCESS_CONFIG)\n    end\n  end\n\n  ALL_IGNORE_ERROR_WITHOUT_IGNORE_IF_PREV_SUCCESS_CONFIG = %[\n    @log_level info\n    <store ignore_error>\n      @type test\n      name c0\n    </store>\n    <store ignore_error>\n      @type test\n      name c1\n    </store>\n    <store ignore_error>\n      @type test\n      name c2\n    </store>\n  ]\n  def test_configure_all_ignore_errors_without_ignore_if_prev_success\n    d = create_driver(ALL_IGNORE_ERROR_WITHOUT_IGNORE_IF_PREV_SUCCESS_CONFIG)\n    expected = /ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified./\n    matches = d.logs.grep(expected)\n    assert_equal(1, matches.length, \"Logs do not contain '#{expected}' '#{d.logs}'\")\n  end\n\n  def test_configure_with_deep_copy_and_use_shallow_copy_mode\n    d = create_driver(%[\n      deep_copy true\n      <store>\n        @type test\n        name c0\n      </store>\n    ])\n\n    outputs = d.instance.outputs\n    assert_equal 1, outputs.size\n    assert_equal Fluent::Plugin::TestOutput, outputs[0].class\n    assert_equal \"c0\", outputs[0].name\n    assert_true d.instance.deep_copy\n    assert_equal :shallow, d.instance.copy_mode\n  end\n\n  def test_feed_events\n    d = create_driver\n\n    assert !d.instance.outputs[0].has_router?\n    assert_not_nil d.instance.outputs[1].router\n    assert !d.instance.outputs[2].has_router?\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    d.run(default_tag: 'test') do\n      d.feed(time, {\"a\" => 1})\n      d.feed(time, {\"a\" => 2})\n    end\n\n    d.instance.outputs.each {|o|\n      assert_equal [ [time, {\"a\"=>1}], [time, {\"a\"=>2}] ], o.events\n    }\n  end\n\n  def test_msgpack_unpacker_cache_bug_for_msgpack_event_stream\n    d = create_driver\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    source = Fluent::ArrayEventStream.new([ [time, {\"a\" => 1}], [time, {\"a\" => 2}] ])\n    es = Fluent::MessagePackEventStream.new(source.to_msgpack_stream)\n\n    d.run(default_tag: 'test') do\n      d.feed(es)\n    end\n\n    d.instance.outputs.each { |o|\n      assert_equal [ [time, {\"a\"=>1}], [time, {\"a\"=>2}] ], o.events\n    }\n  end\n\n  def create_event_test_driver(copy_mode = 'no_copy')\n    config = %[\n      copy_mode #{copy_mode}\n      <store>\n        @type test\n        name output1\n      </store>\n      <store>\n        @type test\n        name output2\n      </store>\n    ]\n\n    d = Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::CopyOutput).configure(config)\n    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|\n      es.each do |time, record|\n        record['foo'] = 'bar'\n      end\n      super(tag, es)\n    end\n    d\n  end\n\n  time = event_time(\"2013-05-26 06:37:22 UTC\")\n  gen_multi_es = Proc.new {\n    es = Fluent::MultiEventStream.new\n    es.add(time, {\"a\" => 1, \"nest\" => {'k' => 'v'}})\n    es.add(time, {\"b\" => 1, \"nest\" => {'k' => 'v'}})\n    es\n  }\n\n  data(\n    \"OneEventStream without copy\" => ['no_copy', Fluent::OneEventStream.new(time, {\"a\" => 1, \"nest\" => {'k' => 'v'}})],\n    \"OneEventStream with shallow\" => ['shallow', Fluent::OneEventStream.new(time, {\"a\" => 1, \"nest\" => {'k' => 'v'}})],\n    \"OneEventStream with marshal\" => ['marshal', Fluent::OneEventStream.new(time, {\"a\" => 1, \"nest\" => {'k' => 'v'}})],\n    \"OneEventStream with deep\"    => ['deep',    Fluent::OneEventStream.new(time, {\"a\" => 1, \"nest\" => {'k' => 'v'}})],\n    \"ArrayEventStream without copy\" => ['no_copy', Fluent::ArrayEventStream.new([[time, {\"a\" => 1, \"nest\" => {'k' => 'v'}}], [time, {\"b\" => 2, \"nest\" => {'k' => 'v'}}]])],\n    \"ArrayEventStream with shallow\" => ['shallow', Fluent::ArrayEventStream.new([[time, {\"a\" => 1, \"nest\" => {'k' => 'v'}}], [time, {\"b\" => 2, \"nest\" => {'k' => 'v'}}]])],\n    \"ArrayEventStream with marshal\" => ['marshal', Fluent::ArrayEventStream.new([[time, {\"a\" => 1, \"nest\" => {'k' => 'v'}}], [time, {\"b\" => 2, \"nest\" => {'k' => 'v'}}]])],\n    \"ArrayEventStream with deep\"    => ['deep',    Fluent::ArrayEventStream.new([[time, {\"a\" => 1, \"nest\" => {'k' => 'v'}}], [time, {\"b\" => 2, \"nest\" => {'k' => 'v'}}]])],\n    \"MultiEventStream without copy\" => ['no_copy', gen_multi_es.call],\n    \"MultiEventStream with shallow\" => ['shallow', gen_multi_es.call],\n    \"MultiEventStream with marshal\" => ['marshal', gen_multi_es.call],\n    \"MultiEventStream with deep\"    => ['deep',    gen_multi_es.call],\n  )\n  def test_copy_mode_with_event_streams(data)\n    copy_mode, es = data\n\n    d = create_event_test_driver(copy_mode)\n    d.run(default_tag: 'test') do\n      d.feed(es)\n    end\n\n    events = d.instance.outputs.map(&:events)\n\n    if copy_mode != 'no_copy'\n      events[0].each_with_index do |entry0, i|\n        record0 = entry0.last\n        record1 = events[1][i].last\n\n        assert_not_equal record0.object_id, record1.object_id\n        assert_equal \"bar\", record0[\"foo\"]\n        assert !record1.has_key?(\"foo\")\n        if copy_mode == 'shallow'\n          assert_equal record0['nest'].object_id, record1['nest'].object_id\n        else\n          assert_not_equal record0['nest'].object_id, record1['nest'].object_id\n        end\n      end\n    else\n      events[0].each_with_index do |entry0, i|\n        record0 = entry0.last\n        record1 = events[1][i].last\n\n        assert_equal record0.object_id, record1.object_id\n        assert_equal \"bar\", record0[\"foo\"]\n        assert_equal \"bar\", record1[\"foo\"]\n        assert_equal record0['nest'].object_id, record1['nest'].object_id\n      end\n    end\n  end\n\n  IGNORE_ERROR_CONFIG = %[\n    <store ignore_error>\n      @type test\n      name c0\n    </store>\n    <store ignore_error>\n      @type test\n      name c1\n    </store>\n    <store>\n      @type test\n      name c2\n    </store>\n  ]\n\n  def test_ignore_error\n    d = create_driver(IGNORE_ERROR_CONFIG)\n\n    # override to raise an error\n    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|\n      raise ArgumentError, 'Failed'\n    end\n\n    time = Time.parse(\"2011-01-02 13:14:15 UTC\").to_i\n    assert_nothing_raised do\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n      end\n    end\n  end\n\n  IGNORE_IF_PREV_SUCCESS_CONFIG = %[\n    <store ignore_error>\n      @type test\n      name c0\n    </store>\n    <store ignore_if_prev_success ignore_error>\n      @type test\n      name c1\n    </store>\n    <store ignore_if_prev_success>\n      @type test\n      name c2\n    </store>\n  ]\n\n  def test_ignore_if_prev_success\n    d = create_driver(IGNORE_IF_PREV_SUCCESS_CONFIG)\n\n    # override to raise an error\n    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|\n      raise ArgumentError, 'Failed'\n    end\n\n    # check ingore_if_prev_success functionality:\n    # 1. output 2 is succeeded.\n    # 2. output 3 is not called.\n    flexstub(d.instance.outputs[1]) do |output|\n      output.should_receive(:process).once\n    end\n    flexstub(d.instance.outputs[2]) do |output|\n      output.should_receive(:process).never\n    end\n\n    time = Time.parse(\"2011-01-02 13:14:15 UTC\").to_i\n    assert_nothing_raised do\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n      end\n    end\n  end\n\nend\n\n"
  },
  {
    "path": "test/plugin/test_out_exec.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/output'\nrequire 'fluent/plugin/out_exec'\nrequire 'fileutils'\n\nclass ExecOutputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    FileUtils.rm_rf(TMP_DIR, secure: true)\n    if File.exist?(TMP_DIR)\n      # ensure files are closed for Windows, on which deleted files\n      # are still visible from filesystem\n      GC.start(full_mark: true, immediate_sweep: true)\n      FileUtils.remove_entry_secure(TMP_DIR)\n    end\n    FileUtils.mkdir_p(TMP_DIR)\n  end\n\n  TMP_DIR = File.dirname(__FILE__) + \"/../tmp/out_exec#{ENV['TEST_ENV_NUMBER']}\"\n\n  def create_driver(config)\n    Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecOutput).configure(config)\n  end\n\n  def create_test_data\n    time = event_time(\"2011-01-02 13:14:15.123\")\n    records = [{\"k1\"=>\"v1\",\"kx\"=>\"vx\"}, {\"k1\"=>\"v2\",\"kx\"=>\"vx\"}]\n    return time, records\n  end\n\n  DEFAULT_CONFIG_ONLY_WITH_KEYS = %[\n    command cat >#{TMP_DIR}/out\n    <format>\n      keys [\"k1\", \"kx\"]\n    </format>\n  ]\n\n  test 'configure in default' do\n    d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS\n    assert{ d.instance.formatter.is_a? Fluent::Plugin::TSVFormatter }\n    assert_equal [\"k1\", \"kx\"], d.instance.formatter.keys\n    assert_nil d.instance.inject_config\n  end\n\n  TSV_CONFIG = %[\n    command cat >#{TMP_DIR}/out\n    <inject>\n      tag_key tag\n      time_key time\n      time_format %Y-%m-%d %H:%M:%S\n      localtime yes\n    </inject>\n    <format>\n      @type tsv\n      keys time, tag, k1\n    </format>\n  ]\n  TSV_CONFIG_WITH_SUBSEC = %[\n    command cat >#{TMP_DIR}/out\n    <inject>\n      tag_key tag\n      time_key time\n      time_format %Y-%m-%d %H:%M:%S.%3N\n      localtime yes\n    </inject>\n    <format>\n      @type tsv\n      keys time, tag, k1\n    </format>\n  ]\n  TSV_CONFIG_WITH_BUFFER = TSV_CONFIG + %[\n    <buffer time>\n      @type memory\n      timekey 3600\n      flush_thread_count 5\n      chunk_limit_size 50m\n      total_limit_size #{50 * 1024 * 1024 * 128}\n      flush_at_shutdown yes\n    </buffer>\n  ]\n  JSON_CONFIG = %[\n    command cat >#{TMP_DIR}/out\n    <format>\n      @type json\n    </format>\n  ]\n  MSGPACK_CONFIG = %[\n    command cat >#{TMP_DIR}/out\n    <format>\n      @type msgpack\n    </format>\n  ]\n\n  CONFIG_COMPAT = %[\n    buffer_path #{TMP_DIR}/buffer\n    command cat >#{TMP_DIR}/out\n    localtime\n  ]\n  TSV_CONFIG_COMPAT = %[\n    keys \"time,tag,k1\"\n    tag_key \"tag\"\n    time_key \"time\"\n    time_format %Y-%m-%d %H:%M:%S\n  ]\n  BUFFER_CONFIG_COMPAT = %[\n    buffer_type memory\n    time_slice_format %Y%m%d%H\n    num_threads 5\n    buffer_chunk_limit 50m\n    buffer_queue_limit 128\n    flush_at_shutdown yes\n  ]\n  TSV_CONFIG_WITH_SUBSEC_COMPAT = %[\n    keys \"time,tag,k1\"\n    tag_key \"tag\"\n    time_key \"time\"\n    time_format %Y-%m-%d %H:%M:%S.%3N\n  ]\n\n  data(\n    'with sections' => TSV_CONFIG,\n    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,\n  )\n  test 'configure for tsv' do |conf|\n    d = create_driver(conf)\n\n    assert_equal [\"time\",\"tag\",\"k1\"], d.instance.formatter.keys\n    assert_equal \"tag\", d.instance.inject_config.tag_key\n    assert_equal \"time\", d.instance.inject_config.time_key\n    assert_equal \"%Y-%m-%d %H:%M:%S\", d.instance.inject_config.time_format\n    assert_equal true, d.instance.inject_config.localtime\n  end\n\n  data(\n    'with sections' => TSV_CONFIG_WITH_BUFFER,\n    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT + BUFFER_CONFIG_COMPAT,\n  )\n  test 'configure_with_compat_buffer_parameters' do |conf|\n    d = create_driver(conf)\n    assert_equal 3600, d.instance.buffer_config.timekey\n    assert_equal 5, d.instance.buffer_config.flush_thread_count\n    assert_equal 50*1024*1024, d.instance.buffer.chunk_limit_size\n    assert_equal 50*1024*1024*128, d.instance.buffer.total_limit_size\n    assert d.instance.buffer_config.flush_at_shutdown\n  end\n\n  data(\n    'with sections' => TSV_CONFIG,\n    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,\n  )\n  test 'format' do |conf|\n    d = create_driver(conf)\n    time, records = create_test_data\n\n    d.run(default_tag: 'test') do\n      d.feed(time, records[0])\n      d.feed(time, records[1])\n    end\n\n    assert_equal %[2011-01-02 13:14:15\\ttest\\tv1\\n], d.formatted[0]\n    assert_equal %[2011-01-02 13:14:15\\ttest\\tv2\\n], d.formatted[1]\n  end\n\n  data(\n    'with sections' => JSON_CONFIG,\n    'traditional' => CONFIG_COMPAT + \"format json\",\n  )\n  test 'format_json' do |conf|\n    d = create_driver(conf)\n    time, records = create_test_data\n\n    d.run(default_tag: 'test') do\n      d.feed(time, records[0])\n      d.feed(time, records[1])\n    end\n\n    assert_equal JSON.generate(records[0]) + \"\\n\", d.formatted[0]\n    assert_equal JSON.generate(records[1]) + \"\\n\", d.formatted[1]\n  end\n\n  data(\n    'with sections' => MSGPACK_CONFIG,\n    'traditional' => CONFIG_COMPAT + \"format msgpack\"\n  )\n  test 'format_msgpack' do |conf|\n    d = create_driver(conf)\n    time, records = create_test_data\n\n    d.run(default_tag: 'test') do\n      d.feed(time, records[0])\n      d.feed(time, records[1])\n    end\n\n    assert_equal records[0].to_msgpack, d.formatted[0]\n    assert_equal records[1].to_msgpack, d.formatted[1]\n  end\n\n  data(\n    'with sections' => TSV_CONFIG_WITH_SUBSEC,\n    'traditional' => CONFIG_COMPAT + TSV_CONFIG_WITH_SUBSEC_COMPAT,\n  )\n  test 'format subsecond time' do |conf|\n    d = create_driver(conf)\n    time, records = create_test_data\n\n    d.run(default_tag: 'test') do\n      d.feed(time, records[0])\n      d.feed(time, records[1])\n    end\n\n    assert_equal %[2011-01-02 13:14:15.123\\ttest\\tv1\\n], d.formatted[0]\n    assert_equal %[2011-01-02 13:14:15.123\\ttest\\tv2\\n], d.formatted[1]\n  end\n\n  data(\n    'with sections' => TSV_CONFIG,\n    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,\n  )\n  test 'write' do |conf|\n    d = create_driver(conf)\n    time, records = create_test_data\n\n    d.run(default_tag: 'test', flush: true) do\n      d.feed(time, records[0])\n      d.feed(time, records[1])\n    end\n\n    expect_path = \"#{TMP_DIR}/out\"\n\n    waiting(10, plugin: d.instance) do\n      sleep(0.1) until File.exist?(expect_path)\n    end\n\n    assert_equal true, File.exist?(expect_path)\n\n    data = File.read(expect_path)\n    expect_data =\n      %[2011-01-02 13:14:15\\ttest\\tv1\\n] +\n      %[2011-01-02 13:14:15\\ttest\\tv2\\n]\n    assert_equal expect_data, data\n  end\n\n  sub_test_case 'when executed process dies unexpectedly' do\n    setup do\n      @gen_config = ->(num){ <<EOC\n    command ruby -e \"ARGV.first.to_i == 0 ? open(ARGV[1]){|f| STDOUT.write(f.read); STDOUT.flush} : (sleep 1 ; exit ARGV.first.to_i)\" #{num} >#{TMP_DIR}/fail_out\n    <inject>\n      tag_key tag\n      time_key time\n      time_format %Y-%m-%d %H:%M:%S\n      localtime yes\n    </inject>\n    <format>\n      @type tsv\n      keys time, tag, k1\n    </format>\nEOC\n      }\n    end\n\n    test 'flushed chunk will be committed after child process successfully exits' do\n      d = create_driver(@gen_config.call(0))\n      time, records = create_test_data\n\n      expect_path = \"#{TMP_DIR}/fail_out\"\n\n      d.end_if{ File.exist?(expect_path) }\n      d.run(default_tag: 'test', flush: true, wait_flush_completion: true, shutdown: false) do\n        d.feed(time, records[0])\n        d.feed(time, records[1])\n      end\n\n      assert{ File.exist?(expect_path) }\n\n      data = File.read(expect_path)\n      expect_data =\n        %[2011-01-02 13:14:15\\ttest\\tv1\\n] +\n        %[2011-01-02 13:14:15\\ttest\\tv2\\n]\n      assert_equal expect_data, data\n\n      assert{ d.instance.buffer.queue.empty? }\n      assert{ d.instance.dequeued_chunks.empty? }\n\n    ensure\n      d.instance_shutdown if d&.instance\n    end\n\n    test 'flushed chunk will be taken back after child process unexpectedly exits' do\n      d = create_driver(@gen_config.call(3))\n      time, records = create_test_data\n\n      expect_path = \"#{TMP_DIR}/fail_out\"\n\n      d.end_if{ d.instance.log.out.logs.any?{|line| line.include?(\"command exits with error code\") } }\n      d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do\n        d.feed(time, records[0])\n        d.feed(time, records[1])\n      end\n\n      assert{ d.instance.dequeued_chunks.empty? } # because it's already taken back\n      assert{ d.instance.buffer.queue.size == 1 }\n\n      logs = d.instance.log.out.logs\n      assert{ logs.any?{|line| line.include?(\"command exits with error code\") && line.include?(\"status=3\") } }\n\n      assert{ File.exist?(expect_path) && File.size(expect_path) == 0 }\n\n    ensure\n      d.instance_shutdown if d&.instance\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_exec_filter.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/output'\nrequire 'fluent/plugin/out_exec_filter'\nrequire 'fileutils'\n\nclass ExecFilterOutputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  CONFIG = %[\n    command cat\n    num_children 3\n    <inject>\n      tag_key     tag\n      time_key    time_in\n      time_type   string\n      time_format %Y-%m-%d %H:%M:%S\n    </inject>\n    <format>\n      keys [\"time_in\", \"tag\", \"k1\"]\n    </format>\n    <parse>\n      keys [\"time_out\", \"tag\", \"k2\"]\n    </parse>\n    <extract>\n      tag_key     tag\n      time_key    time_out\n      time_type   string\n      time_format %Y-%m-%d %H:%M:%S\n    </extract>\n  ]\n\n  CONFIG_COMPAT = %[\n    command cat\n    in_keys time_in,tag,k1\n    out_keys time_out,tag,k2\n    tag_key tag\n    in_time_key time_in\n    out_time_key time_out\n    time_format %Y-%m-%d %H:%M:%S\n    localtime\n    num_children 3\n  ]\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecFilterOutput).configure(conf)\n  end\n\n  SED_SUPPORT_UNBUFFERED_OPTION = ->(){\n    system(\"echo xxx | sed --unbuffered -l -e 's/x/y/g' >#{IO::NULL} 2>&1\")\n    $?.success?\n  }.call\n  SED_UNBUFFERED_OPTION = SED_SUPPORT_UNBUFFERED_OPTION ? '--unbuffered' : ''\n\n  data(\n    'with sections' => CONFIG,\n    'traditional' => CONFIG_COMPAT,\n  )\n  test 'configure' do |conf|\n    d = create_driver(conf)\n\n    assert_false d.instance.parser.estimate_current_event\n\n    assert_equal [\"time_in\",\"tag\",\"k1\"], d.instance.formatter.keys\n    assert_equal [\"time_out\",\"tag\",\"k2\"], d.instance.parser.keys\n    assert_equal \"tag\", d.instance.inject_config.tag_key\n    assert_equal \"tag\", d.instance.extract_config.tag_key\n    assert_equal \"time_in\", d.instance.inject_config.time_key\n    assert_equal \"time_out\", d.instance.extract_config.time_key\n    assert_equal \"%Y-%m-%d %H:%M:%S\", d.instance.inject_config.time_format\n    assert_equal \"%Y-%m-%d %H:%M:%S\", d.instance.extract_config.time_format\n    assert_equal true, d.instance.inject_config.localtime\n    assert_equal 3, d.instance.num_children\n\n    d = create_driver %[\n      command sed -l -e s/foo/bar/\n      in_keys time,k1\n      out_keys time,k2\n      tag xxx\n      time_key time\n      num_children 3\n    ]\n    assert_equal \"sed -l -e s/foo/bar/\", d.instance.command\n\n    d = create_driver(conf + %[\n      remove_prefix before\n      add_prefix after\n    ])\n    assert_equal \"before\", d.instance.remove_prefix\n    assert_equal \"after\" , d.instance.add_prefix\n  end\n\n  data(\n    'with sections' => CONFIG,\n    'traditional' => CONFIG_COMPAT,\n  )\n  test 'emit events with TSV format' do |conf|\n    d = create_driver(conf)\n    time = event_time(\"2011-01-02 13:14:15\")\n\n    d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do\n      # sleep 0.1 until d.instance.children && !d.instance.children.empty? && d.instance.children.all?{|c| c.finished == false }\n      d.feed(time, {\"k1\"=>1})\n      d.feed(time, {\"k1\"=>2})\n    end\n\n    assert_equal \"2011-01-02 13:14:15\\ttest\\t1\\n\", d.formatted[0]\n    assert_equal \"2011-01-02 13:14:15\\ttest\\t2\\n\", d.formatted[1]\n\n    events = d.events\n    assert_equal 2, events.length\n    assert_equal_event_time time, events[0][1]\n    assert_equal [\"test\", time, {\"k2\"=>\"1\"}], events[0]\n    assert_equal_event_time time, events[1][1]\n    assert_equal [\"test\", time, {\"k2\"=>\"2\"}], events[1]\n  end\n\n  CONFIG_WITHOUT_TIME_FORMAT = %[\n    command cat\n    num_children 3\n    tag xxx\n    <inject>\n      time_key time\n      time_type unixtime\n    </inject>\n    <format>\n      keys time,k1\n    </format>\n    <parse>\n      keys time,k2\n      time_key time\n      time_type unixtime\n    </parse>\n  ]\n  CONFIG_WITHOUT_TIME_FORMAT_COMPAT = %[\n    command cat\n    in_keys time,k1\n    out_keys time,k2\n    tag xxx\n    time_key time\n    num_children 3\n  ]\n\n  data(\n    'with sections' => CONFIG_WITHOUT_TIME_FORMAT,\n    'traditional' => CONFIG_WITHOUT_TIME_FORMAT_COMPAT,\n  )\n  test 'emit events without time format configuration' do |conf|\n    d = create_driver(conf)\n    time = event_time(\"2011-01-02 13:14:15 +0900\")\n\n    d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do\n      d.feed(time, {\"k1\"=>1})\n      d.feed(time, {\"k1\"=>2})\n    end\n\n    assert_equal \"1293941655\\t1\\n\", d.formatted[0]\n    assert_equal \"1293941655\\t2\\n\", d.formatted[1]\n\n    events = d.events\n    assert_equal 2, events.length\n    assert_equal_event_time time, events[0][1]\n    assert_equal [\"xxx\", time, {\"k2\"=>\"1\"}], events[0]\n    assert_equal_event_time time, events[1][1]\n    assert_equal [\"xxx\", time, {\"k2\"=>\"2\"}], events[1]\n  end\n\n  CONFIG_TO_DO_GREP = %[\n    command grep --line-buffered -v poo\n    num_children 3\n    tag xxx\n    <inject>\n      time_key time\n      time_type unixtime\n    </inject>\n    <format>\n      keys time, val1\n    </format>\n    <parse>\n      keys time, val2\n      time_key time\n      time_type unixtime\n    </parse>\n  ]\n  CONFIG_TO_DO_GREP_COMPAT = %[\n    command grep --line-buffered -v poo\n    in_keys time,val1\n    out_keys time,val2\n    tag xxx\n    time_key time\n    num_children 3\n  ]\n\n  data(\n    'with sections' => CONFIG_TO_DO_GREP,\n    'traditional' => CONFIG_TO_DO_GREP_COMPAT,\n  )\n  test 'emit events through grep command' do |conf|\n    d = create_driver(conf)\n    time = event_time(\"2011-01-02 13:14:15 +0900\")\n\n    d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do\n      d.feed(time, {\"val1\"=>\"sed-ed value poo\"})\n      d.feed(time, {\"val1\"=>\"sed-ed value foo\"})\n    end\n\n    assert_equal \"1293941655\\tsed-ed value poo\\n\", d.formatted[0]\n    assert_equal \"1293941655\\tsed-ed value foo\\n\", d.formatted[1]\n\n    events = d.events\n    assert_equal 1, events.length\n    assert_equal_event_time time, events[0][1]\n    assert_equal [\"xxx\", time, {\"val2\"=>\"sed-ed value foo\"}], events[0]\n  end\n\n  CONFIG_TO_DO_SED = %[\n    command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/\n    num_children 3\n    tag xxx\n    <inject>\n      time_key time\n      time_type unixtime\n    </inject>\n    <format>\n      keys time, val1\n    </format>\n    <parse>\n      keys time, val2\n      time_key time\n      time_type unixtime\n    </parse>\n  ]\n  CONFIG_TO_DO_SED_COMPAT = %[\n    command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/\n    in_keys time,val1\n    out_keys time,val2\n    tag xxx\n    time_key time\n    num_children 3\n  ]\n\n  data(\n    'with sections' => CONFIG_TO_DO_SED,\n    'traditional' => CONFIG_TO_DO_SED_COMPAT,\n  )\n  test 'emit events through sed command' do |conf|\n    d = create_driver(conf)\n    time = event_time(\"2011-01-02 13:14:15 +0900\")\n\n    d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do\n      d.feed(time, {\"val1\"=>\"sed-ed value poo\"})\n      d.feed(time, {\"val1\"=>\"sed-ed value foo\"})\n    end\n\n    assert_equal \"1293941655\\tsed-ed value poo\\n\", d.formatted[0]\n    assert_equal \"1293941655\\tsed-ed value foo\\n\", d.formatted[1]\n\n    events = d.events\n    assert_equal 2, events.length\n    assert_equal_event_time time, events[0][1]\n    assert_equal [\"xxx\", time, {\"val2\"=>\"sed-ed value poo\"}], events[0]\n    assert_equal_event_time time, events[1][1]\n    assert_equal [\"xxx\", time, {\"val2\"=>\"sed-ed value bar\"}], events[1]\n  end\n\n  CONFIG_TO_DO_SED_WITH_TAG_MODIFY = %[\n    command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/\n    num_children 3\n    remove_prefix input\n    add_prefix output\n    <inject>\n      tag_key tag\n      time_key time\n    </inject>\n    <format>\n      keys tag, time, val1\n    </format>\n    <parse>\n      keys tag, time, val2\n    </parse>\n    <extract>\n      tag_key tag\n      time_key time\n    </extract>\n  ]\n  CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT = %[\n    command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/\n    in_keys tag,time,val1\n    remove_prefix input\n    out_keys tag,time,val2\n    add_prefix output\n    tag_key tag\n    time_key time\n    num_children 3\n  ]\n\n  data(\n    'with sections' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY,\n    'traditional' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT,\n  )\n  test 'emit events with add/remove tag prefix' do |conf|\n    d = create_driver(conf)\n\n    time = event_time(\"2011-01-02 13:14:15 +0900\")\n\n    d.run(default_tag: 'input.test', expect_emits: 2, timeout: 10) do\n      d.feed(time, {\"val1\"=>\"sed-ed value foo\"})\n      d.feed(time, {\"val1\"=>\"sed-ed value poo\"})\n    end\n\n    assert_equal \"test\\t1293941655\\tsed-ed value foo\\n\", d.formatted[0]\n    assert_equal \"test\\t1293941655\\tsed-ed value poo\\n\", d.formatted[1]\n\n    events = d.events\n    assert_equal 2, events.length\n    assert_equal_event_time time, events[0][1]\n    assert_equal [\"output.test\", time, {\"val2\"=>\"sed-ed value bar\"}], events[0]\n    assert_equal_event_time time, events[1][1]\n    assert_equal [\"output.test\", time, {\"val2\"=>\"sed-ed value poo\"}], events[1]\n  end\n\n  CONFIG_JSON = %[\n    command cat\n    <format>\n      @type tsv\n      keys message\n    </format>\n    <parse>\n      @type json\n      stream_buffer_size 1\n    </parse>\n    <extract>\n      tag_key tag\n      time_key time\n    </extract>\n  ]\n  CONFIG_JSON_COMPAT = %[\n    command cat\n    in_keys message\n    out_format json\n    out_stream_buffer_size 1\n    time_key time\n    tag_key tag\n  ]\n\n  data(\n    'with sections' => CONFIG_JSON,\n    'traditional' => CONFIG_JSON_COMPAT,\n  )\n  test 'using json format' do |conf|\n    d = create_driver(conf)\n    time = event_time(\"2011-01-02 13:14:15 +0900\")\n\n    d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do\n      i = d.instance\n      assert{ i.router }\n      d.feed(time, {\"message\"=>%[{\"time\":#{time},\"tag\":\"t1\",\"k1\":\"v1\"}]})\n    end\n\n    assert_equal '{\"time\":1293941655,\"tag\":\"t1\",\"k1\":\"v1\"}' + \"\\n\", d.formatted[0]\n\n    events = d.events\n    assert_equal 1, events.length\n    assert_equal_event_time time, events[0][1]\n    assert_equal [\"t1\", time, {\"k1\"=>\"v1\"}], events[0]\n  end\n\n  CONFIG_JSON_WITH_FLOAT_TIME = %[\n    command cat\n    <format>\n      @type tsv\n      keys message\n    </format>\n    <parse>\n      @type json\n      stream_buffer_size 1\n    </parse>\n    <extract>\n      tag_key tag\n      time_key time\n    </extract>\n  ]\n  CONFIG_JSON_WITH_FLOAT_TIME_COMPAT = %[\n    command cat\n    in_keys message\n    out_format json\n    out_stream_buffer_size 1\n    time_key time\n    tag_key tag\n  ]\n\n  data(\n    'with sections' => CONFIG_JSON_WITH_FLOAT_TIME,\n    'traditional' => CONFIG_JSON_WITH_FLOAT_TIME_COMPAT,\n  )\n  test 'using json format with float time' do |conf|\n    d = create_driver(conf)\n    time = event_time(\"2011-01-02 13:14:15.123 +0900\")\n\n    d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do\n      d.feed(time + 10, {\"message\"=>%[{\"time\":#{time.sec}.#{time.nsec},\"tag\":\"t1\",\"k1\":\"v1\"}]})\n    end\n\n    assert_equal '{\"time\":1293941655.123000000,\"tag\":\"t1\",\"k1\":\"v1\"}' + \"\\n\", d.formatted[0]\n\n    events = d.events\n    assert_equal 1, events.length\n    assert_equal_event_time time, events[0][1]\n    assert_equal [\"t1\", time, {\"k1\"=>\"v1\"}], events[0]\n  end\n\n  CONFIG_JSON_WITH_TIME_FORMAT = %[\n    command cat\n    <format>\n      @type tsv\n      keys message\n    </format>\n    <parse>\n      @type json\n      stream_buffer_size 1\n    </parse>\n    <extract>\n      tag_key tag\n      time_key time\n      time_type string\n      time_format %d/%b/%Y %H:%M:%S.%N %z\n    </extract>\n  ]\n  CONFIG_JSON_WITH_TIME_FORMAT_COMPAT = %[\n    command cat\n    in_keys message\n    out_format json\n    out_stream_buffer_size 1\n    time_key time\n    time_format %d/%b/%Y %H:%M:%S.%N %z\n    tag_key tag\n  ]\n\n  data(\n    'with sections' => CONFIG_JSON_WITH_TIME_FORMAT,\n    'traditional' => CONFIG_JSON_WITH_TIME_FORMAT_COMPAT,\n  )\n  test 'using json format with custom time format' do |conf|\n    d = create_driver(conf)\n    time_str = \"28/Feb/2013 12:00:00.123456789 +0900\"\n    time = event_time(time_str, format: \"%d/%b/%Y %H:%M:%S.%N %z\")\n\n    d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do\n      d.feed(time + 10, {\"message\"=>%[{\"time\":\"#{time_str}\",\"tag\":\"t1\",\"k1\":\"v1\"}]})\n    end\n\n    assert_equal '{\"time\":\"28/Feb/2013 12:00:00.123456789 +0900\",\"tag\":\"t1\",\"k1\":\"v1\"}' + \"\\n\", d.formatted[0]\n\n    events = d.events\n    assert_equal 1, events.length\n    assert_equal_event_time time, events[0][1]\n    assert_equal [\"t1\", time, {\"k1\"=>\"v1\"}], events[0]\n  end\n\n  CONFIG_ROUND_ROBIN = %[\n    command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + \"\\t\" + Process.pid.to_s }'\n    num_children 2\n    <inject>\n      tag_key     tag\n      time_key    time_in\n      time_type   string\n      time_format %Y-%m-%d %H:%M:%S\n    </inject>\n    <format>\n      keys [\"time_in\", \"tag\", \"k1\"]\n    </format>\n    <parse>\n      keys [\"time_out\", \"tag\", \"k2\", \"child_pid\"]\n    </parse>\n    <extract>\n      tag_key     tag\n      time_key    time_out\n      time_type   string\n      time_format %Y-%m-%d %H:%M:%S\n    </extract>\n  ]\n  CONFIG_ROUND_ROBIN_COMPAT = %[\n    command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + \"\\t\" + Process.pid.to_s }'\n    in_keys time_in,tag,k1\n    out_keys time_out,tag,k2,child_pid\n    tag_key tag\n    in_time_key time_in\n    out_time_key time_out\n    time_format %Y-%m-%d %H:%M:%S\n    localtime\n    num_children 2\n  ]\n\n  data(\n    'with sections' => CONFIG_ROUND_ROBIN,\n    'traditional' => CONFIG_ROUND_ROBIN_COMPAT,\n  )\n  test 'using child processes by round robin' do |conf|\n    d = create_driver(conf)\n    time = event_time('2011-01-02 13:14:15')\n\n    d.run(default_tag: 'test', expect_emits: 4) do\n      d.feed(time, {\"k1\" => 0})\n      d.flush\n      sleep 0.5\n      d.feed(time, {\"k1\" => 1})\n      d.flush\n      sleep 0.5\n      d.feed(time, {\"k1\" => 2})\n      d.flush\n      sleep 0.5\n      d.feed(time, {\"k1\" => 3})\n    end\n\n    assert_equal \"2011-01-02 13:14:15\\ttest\\t0\\n\", d.formatted[0]\n    assert_equal \"2011-01-02 13:14:15\\ttest\\t1\\n\", d.formatted[1]\n    assert_equal \"2011-01-02 13:14:15\\ttest\\t2\\n\", d.formatted[2]\n    assert_equal \"2011-01-02 13:14:15\\ttest\\t3\\n\", d.formatted[3]\n\n    events = d.events\n    assert_equal 4, events.length\n\n    pid_list = []\n    events.each do |event|\n      pid = event[2]['child_pid']\n      pid_list << pid unless pid_list.include?(pid)\n    end\n    assert_equal 2, pid_list.size, \"the number of pids should be same with number of child processes: #{pid_list.inspect}\"\n\n    assert_equal pid_list[0], events[0][2]['child_pid']\n    assert_equal pid_list[1], events[1][2]['child_pid']\n    assert_equal pid_list[0], events[2][2]['child_pid']\n    assert_equal pid_list[1], events[3][2]['child_pid']\n  end\n\n  # child process exits per 3 lines\n  CONFIG_RESPAWN = %[\n    command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + \"\\t\" + Process.pid.to_s}; proc.call; proc.call; proc.call'\n    num_children 2\n    child_respawn -1\n    <inject>\n      tag_key   tag\n      time_key  time_in\n      time_type unixtime\n    </inject>\n    <format>\n      keys [\"time_in\", \"tag\", \"k1\"]\n    </format>\n    <parse>\n      keys [\"time_out\", \"tag\", \"k2\", \"child_pid\"]\n    </parse>\n    <extract>\n      tag_key   tag\n      time_key  time_out\n      time_type unixtime\n    </extract>\n  ]\n\n  CONFIG_RESPAWN_COMPAT = %[\n    command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + \"\\t\" + Process.pid.to_s}; proc.call; proc.call; proc.call'\n    num_children 2\n    child_respawn -1\n    in_keys time_in,tag,k1\n    out_keys time_out,tag,k2,child_pid\n    tag_key tag\n    in_time_key time_in\n    out_time_key time_out\n#    time_format %Y-%m-%d %H:%M:%S\n#    localtime\n  ]\n\n  data(\n    'with sections' => CONFIG_RESPAWN,\n    'traditional' => CONFIG_RESPAWN_COMPAT,\n  )\n  test 'emit events via child processes which exits sometimes' do |conf|\n    d = create_driver(conf)\n    time = event_time(\"2011-01-02 13:14:15\")\n    countup = 0\n\n    d.run(start: true, shutdown: false)\n    assert_equal 2, d.instance.instance_eval{ @_child_process_processes.size }\n\n    2.times do\n      d.run(default_tag: 'test', expect_emits: 3, timeout: 3, force_flush_retry: true, start: false, shutdown: false) do\n        d.feed(time, { \"k1\" => countup }); countup += 1\n        d.feed(time, { \"k1\" => countup }); countup += 1\n        d.feed(time, { \"k1\" => countup }); countup += 1\n      end\n    end\n\n    events = d.events\n    assert_equal 6, events.length\n\n    pid_list = []\n    events.each do |event|\n      pid = event[2]['child_pid']\n      pid_list << pid unless pid_list.include?(pid)\n    end\n\n    # the number of pids should be same with number of child processes\n    assert_equal 2, pid_list.size\n    logs = d.instance.log.out.logs\n    assert_equal 2, logs.count { |l| l.include?('child process exits with error code') }\n    assert_equal 2, logs.count { |l| l.include?('respawning child process') }\n\n  ensure\n    d.run(start: false, shutdown: true)\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_file.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/output'\nrequire 'fluent/plugin/out_file'\nrequire 'fileutils'\nrequire 'time'\nrequire 'timecop'\nrequire 'zlib'\nrequire 'zstd-ruby'\nrequire 'fluent/file_wrapper'\n\nclass FileOutputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    FileUtils.rm_rf(TMP_DIR)\n    FileUtils.mkdir_p(TMP_DIR)\n    @default_newline = if Fluent.windows?\n                         \"\\r\\n\"\n                       else\n                         \"\\n\"\n                       end\n  end\n\n  TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/../tmp/out_file#{ENV['TEST_ENV_NUMBER']}\")\n\n  CONFIG = %[\n    path #{TMP_DIR}/out_file_test\n    compress gz\n    utc\n    <buffer>\n      timekey_use_utc true\n    </buffer>\n  ]\n\n  def create_driver(conf = CONFIG, opts = {})\n    Fluent::Test::Driver::Output.new(Fluent::Plugin::FileOutput, opts: opts).configure(conf)\n  end\n\n  sub_test_case 'configuration' do\n    test 'basic configuration' do\n      d = create_driver %[\n        path test_path\n        compress gz\n      ]\n      assert_equal 'test_path', d.instance.path\n      assert_equal :gz, d.instance.compress\n      assert_equal :gzip, d.instance.instance_eval{ @compress_method }\n    end\n\n    test 'using root_dir for buffer path' do\n      system_conf_opts = {'root_dir' => File.join(TMP_DIR, 'testrootdir')}\n      buf_conf = config_element('buffer', '', {'flush_interval' => '1s'})\n      conf = config_element('match', '**', {'@id' => 'myout', 'path' => 'test_path', 'append' => 'true'}, [buf_conf])\n      d = create_driver(conf, system_conf_opts)\n\n      assert_equal 'test_path', d.instance.path\n      assert d.instance.append\n\n      assert d.instance.buffer.respond_to?(:path) # file buffer\n      assert_equal 1, d.instance.buffer_config.flush_interval\n\n      assert_equal File.join(TMP_DIR, 'testrootdir', 'worker0', 'myout'), d.instance.plugin_root_dir\n\n      buffer_path_under_root_dir = File.join(TMP_DIR, 'testrootdir', 'worker0', 'myout', 'buffer', 'buffer.*.log')\n      assert_equal buffer_path_under_root_dir, d.instance.buffer.path\n    end\n\n    test 'path should be writable' do\n      assert_raise(Fluent::ConfigError.new(\"'path' parameter is required\")) do\n        create_driver \"\"\n      end\n\n      assert_nothing_raised do\n        create_driver %[path #{TMP_DIR}/test_path]\n      end\n\n      assert_nothing_raised do\n        FileUtils.mkdir_p(\"#{TMP_DIR}/test_dir\")\n        File.chmod(0777, \"#{TMP_DIR}/test_dir\")\n        create_driver %[path #{TMP_DIR}/test_dir/foo/bar/baz]\n      end\n\n      if Process.uid.nonzero?\n        assert_raise(Fluent::ConfigError) do\n          FileUtils.mkdir_p(\"#{TMP_DIR}/test_dir\")\n          File.chmod(0555, \"#{TMP_DIR}/test_dir\")\n          create_driver %[path #{TMP_DIR}/test_dir/foo/bar/baz]\n        end\n      end\n    end\n\n    test 'default timezone is localtime' do\n      d = create_driver(%[path #{TMP_DIR}/out_file_test])\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n      with_timezone(Fluent.windows? ? 'NST-8' : 'Asia/Taipei') do\n        d.run(default_tag: 'test') do\n          d.feed(time, {\"a\"=>1})\n        end\n      end\n      assert_equal 1, d.formatted.size\n      assert_equal %[2011-01-02T21:14:15+08:00\\ttest\\t{\"a\":1}#{@default_newline}], d.formatted[0]\n    end\n\n    test 'no configuration error raised for basic configuration using \"*\" (v0.12 style)' do\n      conf = config_element('match', '**', {\n          'path' => \"#{TMP_DIR}/test_out.*.log\",\n          'time_slice_format' => '%Y%m%d',\n      })\n      assert_nothing_raised do\n        create_driver(conf)\n      end\n    end\n\n    if Process.uid.nonzero?\n      test 'configuration error raised if specified directory via template is not writable' do\n        Timecop.freeze(Time.parse(\"2016-10-04 21:33:27 UTC\")) do\n          conf = config_element('match', '**', {\n            'path' => \"#{TMP_DIR}/prohibited/${tag}/file.%Y%m%d.log\",\n          }, [ config_element('buffer', 'time,tag', {'timekey' => 86400, 'timekey_zone' => '+0000'}) ])\n          FileUtils.mkdir_p(\"#{TMP_DIR}/prohibited\")\n          File.chmod(0555, \"#{TMP_DIR}/prohibited\")\n          assert_raise Fluent::ConfigError.new(\"out_file: `#{TMP_DIR}/prohibited/a/file.20161004.log_**.log` is not writable\") do\n            create_driver(conf)\n          end\n        end\n      end\n    end\n\n    test 'configuration using inject/format/buffer sections fully' do\n      conf = config_element('match', '**', {\n          'path' => \"#{TMP_DIR}/${tag}/${type}/conf_test.%Y%m%d.%H%M.log\",\n          'add_path_suffix' => 'false',\n          'append' => \"true\",\n          'symlink_path' => \"#{TMP_DIR}/${tag}/conf_test.current.log\",\n          'compress' => 'gzip',\n          'recompress' => 'true',\n        }, [\n          config_element('inject', '', {\n              'hostname_key' => 'hostname',\n              'hostname' => 'testing.local',\n              'tag_key' => 'tag',\n              'time_key' => 'time',\n              'time_type' => 'string',\n              'time_format' => '%Y/%m/%d %H:%M:%S %z',\n              'timezone' => '+0900',\n          }),\n          config_element('format', '', {\n              '@type' => 'out_file',\n              'include_tag' => 'true',\n              'include_time' => 'true',\n              'delimiter' => 'COMMA',\n              'time_type' => 'string',\n              'time_format' => '%Y-%m-%d %H:%M:%S %z',\n              'utc' => 'true',\n          }),\n          config_element('buffer', 'time,tag,type', {\n              '@type' => 'file',\n              'timekey' => '15m',\n              'timekey_wait' => '5s',\n              'timekey_zone' => '+0000',\n              'path' => \"#{TMP_DIR}/buf_conf_test\",\n              'chunk_limit_size' => '50m',\n              'total_limit_size' => '1g',\n              'compress' => 'gzip',\n          }),\n      ])\n      assert_nothing_raised do\n        create_driver(conf)\n      end\n    end\n\n    test 'configured as secondary with primary using chunk_key_tag and not using chunk_key_time' do\n      require 'fluent/plugin/out_null'\n      conf = config_element('match', '**', {\n        }, [\n          config_element('buffer', 'tag', {\n          }),\n          config_element('secondary', '', {\n              '@type' => 'file',\n              'path' => \"#{TMP_DIR}/testing_to_dump_by_out_file\",\n          }),\n      ])\n      assert_nothing_raised do\n        Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)\n      end\n    end\n\n    test 'warning for symlink_path not including correct placeholders corresponding to chunk keys' do\n      omit \"Windows doesn't support symlink\" if Fluent.windows?\n      conf = config_element('match', '**', {\n          'path' => \"#{TMP_DIR}/${tag}/${key1}/${key2}/conf_test.%Y%m%d.%H%M.log\",\n          'symlink_path' => \"#{TMP_DIR}/conf_test.current.log\",\n        }, [\n          config_element('buffer', 'time,tag,key1,key2', {\n              '@type' => 'file',\n              'timekey' => '1d',\n              'path' => \"#{TMP_DIR}/buf_conf_test\",\n          }),\n      ])\n      assert_nothing_raised do\n        d = create_driver(conf)\n        assert do\n          d.logs.count { |log| log.include?(\"multiple chunks are competing for a single symlink_path\") } == 2\n        end\n      end\n    end\n  end\n\n  sub_test_case 'fully configured output' do\n    setup do\n      Timecop.freeze(Time.parse(\"2016-10-03 23:58:00 UTC\"))\n      conf = config_element('match', '**', {\n          'path' => \"#{TMP_DIR}/${tag}/${type}/full.%Y%m%d.%H%M.log\",\n          'add_path_suffix' => 'false',\n          'append' => \"true\",\n          'symlink_path' => \"#{TMP_DIR}/full.current.log\",\n          'compress' => 'gzip',\n          'recompress' => 'true',\n        }, [\n          config_element('inject', '', {\n              'hostname_key' => 'hostname',\n              'hostname' => 'testing.local',\n              'tag_key' => 'tag',\n              'time_key' => 'time',\n              'time_type' => 'string',\n              'time_format' => '%Y/%m/%d %H:%M:%S %z',\n              'timezone' => '+0900',\n          }),\n          config_element('format', '', {\n              '@type' => 'out_file',\n              'include_tag' => 'true',\n              'include_time' => 'true',\n              'delimiter' => 'COMMA',\n              'time_type' => 'string',\n              'time_format' => '%Y-%m-%d %H:%M:%S %z',\n              'utc' => 'true',\n          }),\n          config_element('buffer', 'time,tag,type', {\n              '@type' => 'file',\n              'timekey' => '15m',\n              'timekey_wait' => '5s',\n              'timekey_zone' => '+0000',\n              'path' => \"#{TMP_DIR}/buf_full\",\n              'chunk_limit_size' => '50m',\n              'total_limit_size' => '1g',\n              'compress' => 'gzip',\n          }),\n      ])\n      @d = create_driver(conf)\n    end\n\n    teardown do\n      FileUtils.rm_rf(\"#{TMP_DIR}/buf_full\")\n      FileUtils.rm_rf(\"#{TMP_DIR}/my.data\")\n      FileUtils.rm_rf(\"#{TMP_DIR}/your.data\")\n      FileUtils.rm_rf(\"#{TMP_DIR}/full.current.log\")\n      Timecop.return\n    end\n\n    test 'can format/write data correctly' do\n      d = @d\n\n      assert_equal 50*1024*1024, d.instance.buffer.chunk_limit_size\n      assert_equal 1*1024*1024*1024, d.instance.buffer.total_limit_size\n\n      assert !(File.symlink?(\"#{TMP_DIR}/full.current.log\"))\n\n      t1 = event_time(\"2016-10-03 23:58:09 UTC\")\n      t2 = event_time(\"2016-10-03 23:59:33 UTC\")\n      t3 = event_time(\"2016-10-03 23:59:57 UTC\")\n      t4 = event_time(\"2016-10-04 00:00:17 UTC\")\n      t5 = event_time(\"2016-10-04 00:01:59 UTC\")\n\n      Timecop.freeze(Time.parse(\"2016-10-03 23:58:30 UTC\"))\n\n      d.run(start: true, flush: false, shutdown: false) do\n        d.feed('my.data', t1, {\"type\" => \"a\", \"message\" => \"data raw content\"})\n        d.feed('my.data', t2, {\"type\" => \"a\", \"message\" => \"data raw content\"})\n        d.feed('your.data', t3, {\"type\" => \"a\", \"message\" => \"data raw content\"})\n      end\n\n      assert_equal 3, d.formatted.size\n\n      assert Dir.exist?(\"#{TMP_DIR}/buf_full\")\n      assert !(Dir.exist?(\"#{TMP_DIR}/my.data/a\"))\n      assert !(Dir.exist?(\"#{TMP_DIR}/your.data/a\"))\n      buffer_files = Dir.entries(\"#{TMP_DIR}/buf_full\").reject{|e| e =~ /^\\.+$/ }\n      assert_equal 2, buffer_files.count{|n| n.end_with?('.meta') }\n      assert_equal 2, buffer_files.count{|n| !n.end_with?('.meta') }\n\n      m1 = d.instance.metadata('my.data', t1, {\"type\" => \"a\"})\n      m2 = d.instance.metadata('your.data', t3, {\"type\" => \"a\"})\n\n      assert_equal 2, d.instance.buffer.stage.size\n      b1_path = d.instance.buffer.stage[m1].path\n      b1_size = File.lstat(b1_path).size\n\n      unless Fluent.windows?\n        assert File.symlink?(\"#{TMP_DIR}/full.current.log\")\n        assert_equal d.instance.buffer.stage[m2].path, File.readlink(\"#{TMP_DIR}/full.current.log\")\n      end\n\n      Timecop.freeze(Time.parse(\"2016-10-04 00:00:06 UTC\"))\n\n      d.run(start: false, flush: true, shutdown: true) do\n        d.feed('my.data', t4, {\"type\" => \"a\", \"message\" => \"data raw content\"})\n        d.feed('your.data', t5, {\"type\" => \"a\", \"message\" => \"data raw content\"})\n      end\n\n      assert Dir.exist?(\"#{TMP_DIR}/buf_full\")\n      assert Dir.exist?(\"#{TMP_DIR}/my.data/a\")\n      assert Dir.exist?(\"#{TMP_DIR}/your.data/a\")\n\n      buffer_files = Dir.entries(\"#{TMP_DIR}/buf_full\").reject{|e| e =~ /^\\.+$/ }\n      assert_equal 0, buffer_files.size\n\n      assert File.exist?(\"#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz\")\n      assert File.exist?(\"#{TMP_DIR}/my.data/a/full.20161004.0000.log.gz\")\n      assert File.exist?(\"#{TMP_DIR}/your.data/a/full.20161003.2345.log.gz\")\n      assert File.exist?(\"#{TMP_DIR}/your.data/a/full.20161004.0000.log.gz\")\n\n      assert{ File.lstat(\"#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz\").size < b1_size } # recompress\n\n      assert_equal 5, d.formatted.size\n\n      r1 = %!2016-10-03 23:58:09 +0000,my.data,{\"type\":\"a\",\"message\":\"data raw content\",\"hostname\":\"testing.local\",\"tag\":\"my.data\",\"time\":\"2016/10/04 08:58:09 +0900\"}#{@default_newline}!\n      r2 = %!2016-10-03 23:59:33 +0000,my.data,{\"type\":\"a\",\"message\":\"data raw content\",\"hostname\":\"testing.local\",\"tag\":\"my.data\",\"time\":\"2016/10/04 08:59:33 +0900\"}#{@default_newline}!\n      r3 = %!2016-10-03 23:59:57 +0000,your.data,{\"type\":\"a\",\"message\":\"data raw content\",\"hostname\":\"testing.local\",\"tag\":\"your.data\",\"time\":\"2016/10/04 08:59:57 +0900\"}#{@default_newline}!\n      r4 = %!2016-10-04 00:00:17 +0000,my.data,{\"type\":\"a\",\"message\":\"data raw content\",\"hostname\":\"testing.local\",\"tag\":\"my.data\",\"time\":\"2016/10/04 09:00:17 +0900\"}#{@default_newline}!\n      r5 = %!2016-10-04 00:01:59 +0000,your.data,{\"type\":\"a\",\"message\":\"data raw content\",\"hostname\":\"testing.local\",\"tag\":\"your.data\",\"time\":\"2016/10/04 09:01:59 +0900\"}#{@default_newline}!\n      assert_equal r1, d.formatted[0]\n      assert_equal r2, d.formatted[1]\n      assert_equal r3, d.formatted[2]\n      assert_equal r4, d.formatted[3]\n      assert_equal r5, d.formatted[4]\n\n      read_gunzip = ->(path){\n        File.open(path, 'rb'){ |fio|\n          Zlib::GzipReader.new(StringIO.new(fio.read)).read\n        }\n      }\n      assert_equal r1 + r2, read_gunzip.call(\"#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz\")\n      assert_equal r3, read_gunzip.call(\"#{TMP_DIR}/your.data/a/full.20161003.2345.log.gz\")\n      assert_equal r4, read_gunzip.call(\"#{TMP_DIR}/my.data/a/full.20161004.0000.log.gz\")\n      assert_equal r5, read_gunzip.call(\"#{TMP_DIR}/your.data/a/full.20161004.0000.log.gz\")\n    end\n  end\n\n  sub_test_case 'format' do\n    test 'timezone UTC specified' do\n      d = create_driver\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n        d.feed(time, {\"a\"=>2})\n      end\n      assert_equal 2, d.formatted.size\n      assert_equal %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":1}#{@default_newline}], d.formatted[0]\n      assert_equal %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":2}#{@default_newline}], d.formatted[1]\n    end\n\n    test 'time formatted with specified timezone, using area name' do\n      d = create_driver %[\n        path #{TMP_DIR}/out_file_test\n        timezone Asia/Taipei\n      ]\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n      end\n      assert_equal 1, d.formatted.size\n      assert_equal %[2011-01-02T21:14:15+08:00\\ttest\\t{\"a\":1}#{@default_newline}], d.formatted[0]\n    end\n\n    test 'time formatted with specified timezone, using offset' do\n      d = create_driver %[\n        path #{TMP_DIR}/out_file_test\n        timezone -03:30\n      ]\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n      end\n      assert_equal 1, d.formatted.size\n      assert_equal %[2011-01-02T09:44:15-03:30\\ttest\\t{\"a\":1}#{@default_newline}], d.formatted[0]\n    end\n\n    test 'configuration error raised for invalid timezone' do\n      assert_raise(Fluent::ConfigError) do\n        create_driver %[\n          path #{TMP_DIR}/out_file_test\n          timezone Invalid/Invalid\n        ]\n      end\n    end\n  end\n\n  def check_zipped_result(path, expect, type: :gzip)\n    # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790\n    # Following code from https://www.ruby-forum.com/topic/971591#979520\n    result = ''\n    if type == :gzip || type == :gz\n      File.open(path, \"rb\") { |io|\n        loop do\n          gzr = Zlib::GzipReader.new(StringIO.new(io.read))\n          result << gzr.read\n          unused = gzr.unused\n          gzr.finish\n          break if unused.nil?\n          io.pos -= unused.length\n        end\n      }\n    elsif type == :zstd\n      File.open(path, \"rb\") { |io|\n        loop do\n          reader = Zstd::StreamReader.new(StringIO.new(io.read))\n          result << reader.read(1024)\n          break if io.eof?\n        end\n      }\n    else\n      raise \"Invalid compression type to check\"\n    end\n\n    assert_equal expect, result\n  end\n\n  def check_result(path, expect)\n    result = File.read(path, mode: \"rb\")\n    assert_equal expect, result\n  end\n\n  sub_test_case 'write' do\n    test 'basic case with gz' do\n      d = create_driver\n\n      assert_false File.exist?(\"#{TMP_DIR}/out_file_test.20110102_0.log.gz\")\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n        d.feed(time, {\"a\"=>2})\n      end\n\n      assert File.exist?(\"#{TMP_DIR}/out_file_test.20110102_0.log.gz\")\n      check_zipped_result(\"#{TMP_DIR}/out_file_test.20110102_0.log.gz\", %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":2}#{@default_newline}])\n    end\n\n    test 'write with zstd compression' do\n      d = create_driver %[\n        path #{TMP_DIR}/out_file_test\n        compress zstd\n        utc\n        <buffer>\n          timekey_use_utc true\n        </buffer>\n      ]\n\n      assert_false File.exist?(\"#{TMP_DIR}/out_file_test.20110102_0.log.zstd\")\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n        d.feed(time, {\"a\"=>2})\n      end\n\n      assert File.exist?(\"#{TMP_DIR}/out_file_test.20110102_0.log.zstd\")\n      check_zipped_result(\"#{TMP_DIR}/out_file_test.20110102_0.log.zstd\", %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":2}#{@default_newline}], type: :zstd)\n    end\n  end\n\n  sub_test_case 'file/directory permissions' do\n    TMP_DIR_WITH_SYSTEM = File.expand_path(File.dirname(__FILE__) + \"/../tmp/out_file_system#{ENV['TEST_ENV_NUMBER']}\")\n    # 0750 interprets as \"488\". \"488\".to_i(8) # => 4. So, it makes wrong permission. Umm....\n    OVERRIDE_DIR_PERMISSION = 750\n    OVERRIDE_FILE_PERMISSION = 0620\n    CONFIG_WITH_SYSTEM = %[\n      path #{TMP_DIR_WITH_SYSTEM}/out_file_test\n      compress gz\n      utc\n      <buffer>\n        timekey_use_utc true\n      </buffer>\n      <system>\n        file_permission #{OVERRIDE_FILE_PERMISSION}\n        dir_permission #{OVERRIDE_DIR_PERMISSION}\n      </system>\n    ]\n\n    setup do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n      FileUtils.rm_rf(TMP_DIR_WITH_SYSTEM)\n    end\n\n    def parse_system(text)\n      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')\n      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }\n    end\n\n    test 'write to file with permission specifications' do\n      system_conf = parse_system(CONFIG_WITH_SYSTEM)\n      sc = Fluent::SystemConfig.new(system_conf)\n      Fluent::Engine.init(sc)\n      d = create_driver CONFIG_WITH_SYSTEM\n\n      assert_false File.exist?(\"#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz\")\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n        d.feed(time, {\"a\"=>2})\n      end\n\n      assert File.exist?(\"#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz\")\n\n      check_zipped_result(\"#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz\", %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":1}\\n] + %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":2}\\n])\n      dir_mode = \"%o\" % File::stat(TMP_DIR_WITH_SYSTEM).mode\n      assert_equal(OVERRIDE_DIR_PERMISSION, dir_mode[-3, 3].to_i)\n      file_mode = \"%o\" % File::stat(\"#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz\").mode\n      assert_equal(OVERRIDE_FILE_PERMISSION, file_mode[-3, 3].to_i)\n    end\n  end\n\n  sub_test_case 'format specified' do\n    test 'json' do\n      d = create_driver [CONFIG, 'format json', 'include_time_key true', 'time_as_epoch'].join(\"\\n\")\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n        d.feed(time, {\"a\"=>2})\n      end\n\n      path = d.instance.last_written_path\n      check_zipped_result(path, %[#{JSON.generate({\"a\" => 1, 'time' => time.to_i})}#{@default_newline}] + %[#{JSON.generate({\"a\" => 2, 'time' => time.to_i})}#{@default_newline}])\n    end\n\n    test 'ltsv' do\n      d = create_driver [CONFIG, 'format ltsv', 'include_time_key true'].join(\"\\n\")\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n        d.feed(time, {\"a\"=>2})\n      end\n\n      path = d.instance.last_written_path\n      check_zipped_result(path, %[a:1\\ttime:2011-01-02T13:14:15Z#{@default_newline}] + %[a:2\\ttime:2011-01-02T13:14:15Z#{@default_newline}])\n    end\n\n    test 'single_value' do\n      d = create_driver [CONFIG, 'format single_value', 'message_key a'].join(\"\\n\")\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n        d.feed(time, {\"a\"=>2})\n      end\n\n      path = d.instance.last_written_path\n      check_zipped_result(path, %[1#{@default_newline}] + %[2#{@default_newline}])\n    end\n  end\n\n  test 'path with index number' do\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    formatted_lines = %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":2}#{@default_newline}]\n\n    write_once = ->(){\n      d = create_driver\n      d.run(default_tag: 'test'){\n        d.feed(time, {\"a\"=>1})\n        d.feed(time, {\"a\"=>2})\n      }\n      d.instance.last_written_path\n    }\n\n    assert !File.exist?(\"#{TMP_DIR}/out_file_test.20110102_0.log.gz\")\n\n    path = write_once.call\n    assert_equal \"#{TMP_DIR}/out_file_test.20110102_0.log.gz\", path\n    check_zipped_result(path, formatted_lines)\n    assert_equal 1, Dir.glob(\"#{TMP_DIR}/out_file_test.*\").size\n\n    path = write_once.call\n    assert_equal \"#{TMP_DIR}/out_file_test.20110102_1.log.gz\", path\n    check_zipped_result(path, formatted_lines)\n    assert_equal 2, Dir.glob(\"#{TMP_DIR}/out_file_test.*\").size\n\n    path = write_once.call\n    assert_equal \"#{TMP_DIR}/out_file_test.20110102_2.log.gz\", path\n    check_zipped_result(path, formatted_lines)\n    assert_equal 3, Dir.glob(\"#{TMP_DIR}/out_file_test.*\").size\n  end\n\n  data(\n    \"without compression\" => \"text\",\n    \"with gzip compression\" => \"gz\",\n    \"with zstd compression\" => \"zstd\"\n  )\n  test 'append' do |compression|\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    formatted_lines = %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\\ttest\\t{\"a\":2}#{@default_newline}]\n\n    write_once = ->(){\n      config = %[\n        path #{TMP_DIR}/out_file_test\n        utc\n        append true\n        <buffer>\n          timekey_use_utc true\n        </buffer>\n      ]\n      if compression != :text\n        config << \"        compress #{compression}\"\n      end\n      d = create_driver(config)\n      d.run(default_tag: 'test'){\n        d.feed(time, {\"a\"=>1})\n        d.feed(time, {\"a\"=>2})\n      }\n      d.instance.last_written_path\n    }\n\n    log_file_name = \"out_file_test.20110102.log\"\n    if compression != \"text\"\n      log_file_name << \".#{compression}\"\n    end\n\n    1.upto(3) do |i|\n      path = write_once.call\n      assert_equal \"#{TMP_DIR}/#{log_file_name}\", path\n      expect = formatted_lines * i\n      if compression != \"text\"\n        check_zipped_result(path, expect, type: compression.to_sym)\n      else\n        check_result(path, expect)\n      end\n    end\n  end\n\n  test 'append when JST' do\n    with_timezone(Fluent.windows? ? \"JST-9\" : \"Asia/Tokyo\") do\n      time = event_time(\"2011-01-02 03:14:15+09:00\")\n      formatted_lines = %[2011-01-02T03:14:15+09:00\\ttest\\t{\"a\":1}#{@default_newline}] + %[2011-01-02T03:14:15+09:00\\ttest\\t{\"a\":2}#{@default_newline}]\n\n      write_once = ->(){\n        d = create_driver %[\n          path #{TMP_DIR}/out_file_test\n          compress gz\n          append true\n          <buffer>\n            timekey_use_utc false\n            timekey_zone Asia/Tokyo\n          </buffer>\n        ]\n        d.run(default_tag: 'test'){\n          d.feed(time, {\"a\"=>1})\n          d.feed(time, {\"a\"=>2})\n        }\n        d.instance.last_written_path\n      }\n\n      path = write_once.call\n      assert_equal \"#{TMP_DIR}/out_file_test.20110102.log.gz\", path\n      check_zipped_result(path, formatted_lines)\n\n      path = write_once.call\n      assert_equal \"#{TMP_DIR}/out_file_test.20110102.log.gz\", path\n      check_zipped_result(path, formatted_lines * 2)\n\n      path = write_once.call\n      assert_equal \"#{TMP_DIR}/out_file_test.20110102.log.gz\", path\n      check_zipped_result(path, formatted_lines * 3)\n    end\n  end\n\n  test 'append when UTC-02 but timekey_zone is +0900' do\n    with_timezone(\"UTC-02\") do # +0200\n      time = event_time(\"2011-01-02 17:14:15+02:00\")\n      formatted_lines = %[2011-01-02T17:14:15+02:00\\ttest\\t{\"a\":1}#{@default_newline}] + %[2011-01-02T17:14:15+02:00\\ttest\\t{\"a\":2}#{@default_newline}]\n\n      write_once = ->(){\n        d = create_driver %[\n          path #{TMP_DIR}/out_file_test\n          compress gz\n          append true\n          <buffer>\n            timekey_use_utc false\n            timekey_zone +0900\n          </buffer>\n        ]\n        d.run(default_tag: 'test'){\n          d.feed(time, {\"a\"=>1})\n          d.feed(time, {\"a\"=>2})\n        }\n        d.instance.last_written_path\n      }\n\n      path = write_once.call\n      # Rotated at 2011-01-02 17:00:00+02:00\n      assert_equal \"#{TMP_DIR}/out_file_test.20110103.log.gz\", path\n      check_zipped_result(path, formatted_lines)\n\n      path = write_once.call\n      assert_equal \"#{TMP_DIR}/out_file_test.20110103.log.gz\", path\n      check_zipped_result(path, formatted_lines * 2)\n\n      path = write_once.call\n      assert_equal \"#{TMP_DIR}/out_file_test.20110103.log.gz\", path\n      check_zipped_result(path, formatted_lines * 3)\n    end\n  end\n\n  test '${chunk_id}' do\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n    write_once = ->(){\n      d = create_driver %[\n        path #{TMP_DIR}/out_file_chunk_id_${chunk_id}\n        utc\n        append true\n        <buffer>\n          timekey_use_utc true\n        </buffer>\n      ]\n      d.run(default_tag: 'test'){\n        d.feed(time, {\"a\"=>1})\n        d.feed(time, {\"a\"=>2})\n      }\n      d.instance.last_written_path\n    }\n\n    path = write_once.call\n    if File.basename(path) =~ /out_file_chunk_id_([-_.@a-zA-Z0-9].*).20110102.log/\n      unique_id = Fluent::UniqueId.hex(Fluent::UniqueId.generate)\n      assert_equal unique_id.size, $1.size, \"chunk_id size is mismatched\"\n    else\n      flunk \"chunk_id is not included in the path\"\n    end\n  end\n\n  SYMLINK_PATH = File.expand_path(\"#{TMP_DIR}/current\")\n\n  sub_test_case 'symlink' do\n    test 'static symlink' do\n      omit \"Windows doesn't support symlink\" if Fluent.windows?\n      conf = CONFIG + %[\n        symlink_path #{SYMLINK_PATH}\n      ]\n      symlink_path = \"#{SYMLINK_PATH}\"\n\n      d = create_driver(conf)\n      begin\n        run_and_check(d, symlink_path)\n      ensure\n        FileUtils.rm_rf(symlink_path)\n      end\n    end\n\n    test 'symlink with placeholders' do\n      omit \"Windows doesn't support symlink\" if Fluent.windows?\n      conf = %[\n        path #{TMP_DIR}/${tag}/out_file_test\n        symlink_path #{SYMLINK_PATH}/foo/${tag}\n        <buffer tag,time>\n        </buffer>\n      ]\n      symlink_path = \"#{SYMLINK_PATH}/foo/tag\"\n\n      d = create_driver(conf)\n      begin\n        run_and_check(d, symlink_path)\n      ensure\n        FileUtils.rm_rf(symlink_path)\n      end\n    end\n\n    test 'relative symlink' do\n      omit \"Windows doesn't support symlinks\" if Fluent.windows?\n\n      conf = CONFIG + %[\n        symlink_path #{SYMLINK_PATH}\n        symlink_path_use_relative true\n      ]\n      symlink_path = \"#{SYMLINK_PATH}\"\n\n      d = create_driver(conf)\n      begin\n        run_and_check(d, symlink_path, relative_symlink=true)\n      ensure\n        FileUtils.rm_rf(symlink_path)\n      end\n    end\n\n    def run_and_check(d, symlink_path, relative_symlink=false)\n      d.run(default_tag: 'tag') do\n        es = Fluent::OneEventStream.new(event_time(\"2011-01-02 13:14:15 UTC\"), {\"a\"=>1})\n        d.feed(es)\n\n        assert File.symlink?(symlink_path)\n        assert File.exist?(symlink_path) # This checks dest of symlink exists or not.\n\n        es = Fluent::OneEventStream.new(event_time(\"2011-01-03 14:15:16 UTC\"), {\"a\"=>2})\n        d.feed(es)\n\n        assert File.symlink?(symlink_path)\n        assert File.exist?(symlink_path)\n\n        meta = d.instance.metadata('tag', event_time(\"2011-01-03 14:15:16 UTC\"), {})\n        if relative_symlink\n          target_path = d.instance.buffer.instance_eval{ @stage[meta].path }\n          link_target = File.readlink(symlink_path)\n          expected_path = Pathname.new(target_path).relative_path_from(Pathname.new(File.dirname(symlink_path))).to_s\n          assert_equal expected_path, link_target\n        else\n          assert_equal d.instance.buffer.instance_eval{ @stage[meta].path }, File.readlink(symlink_path)\n        end\n      end\n    end\n  end\n\n  sub_test_case 'path' do\n    test 'normal' do\n      d = create_driver(%[\n        path #{TMP_DIR}/out_file_test\n        time_slice_format %Y-%m-%d-%H\n        utc true\n      ])\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n      end\n      path = d.instance.last_written_path\n      assert_equal \"#{TMP_DIR}/out_file_test.2011-01-02-13_0.log\", path\n    end\n\n    test 'normal with append' do\n      d = create_driver(%[\n        path #{TMP_DIR}/out_file_test\n        time_slice_format %Y-%m-%d-%H\n        utc true\n        append true\n      ])\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n      end\n      path = d.instance.last_written_path\n      assert_equal \"#{TMP_DIR}/out_file_test.2011-01-02-13.log\", path\n    end\n\n    test '*' do\n      d = create_driver(%[\n        path #{TMP_DIR}/out_file_test.*.txt\n        time_slice_format %Y-%m-%d-%H\n        utc true\n      ])\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n      end\n      path = d.instance.last_written_path\n      assert_equal \"#{TMP_DIR}/out_file_test.2011-01-02-13_0.txt\", path\n    end\n\n    test '* with append' do\n      d = create_driver(%[\n        path #{TMP_DIR}/out_file_test.*.txt\n        time_slice_format %Y-%m-%d-%H\n        utc true\n        append true\n      ])\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      d.run(default_tag: 'test') do\n        d.feed(time, {\"a\"=>1})\n      end\n      path = d.instance.last_written_path\n      assert_equal \"#{TMP_DIR}/out_file_test.2011-01-02-13.txt\", path\n    end\n  end\n\n  sub_test_case '#timekey_to_timeformat' do\n    setup do\n      @d = create_driver\n      @i = @d.instance\n    end\n\n    test 'returns empty string for nil' do\n      assert_equal '', @i.timekey_to_timeformat(nil)\n    end\n\n    test 'returns timestamp string with seconds for timekey smaller than 60' do\n      assert_equal '%Y%m%d%H%M%S', @i.timekey_to_timeformat(1)\n      assert_equal '%Y%m%d%H%M%S', @i.timekey_to_timeformat(30)\n      assert_equal '%Y%m%d%H%M%S', @i.timekey_to_timeformat(59)\n    end\n\n    test 'returns timestamp string with minutes for timekey smaller than 3600' do\n      assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(60)\n      assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(180)\n      assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(1800)\n      assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(3599)\n    end\n\n    test 'returns timestamp string with hours for timekey smaller than 86400 (1 day)' do\n      assert_equal '%Y%m%d%H', @i.timekey_to_timeformat(3600)\n      assert_equal '%Y%m%d%H', @i.timekey_to_timeformat(7200)\n      assert_equal '%Y%m%d%H', @i.timekey_to_timeformat(86399)\n    end\n\n    test 'returns timestamp string with days for timekey equal or greater than 86400' do\n      assert_equal '%Y%m%d', @i.timekey_to_timeformat(86400)\n      assert_equal '%Y%m%d', @i.timekey_to_timeformat(1000000)\n      assert_equal '%Y%m%d', @i.timekey_to_timeformat(1000000000)\n    end\n  end\n\n  sub_test_case '#compression_suffix' do\n    setup do\n      @i = create_driver.instance\n    end\n\n    test 'returns empty string for nil (no compression method specified)' do\n      assert_equal '', @i.compression_suffix(nil)\n    end\n\n    test 'returns .gz for gzip' do\n      assert_equal '.gz', @i.compression_suffix(:gzip)\n    end\n\n    test 'returns .zstd for zstd' do\n      assert_equal '.zstd', @i.compression_suffix(:zstd)\n    end\n  end\n\n  sub_test_case '#generate_path_template' do\n    setup do\n      @i = create_driver.instance\n    end\n\n    data(\n      'day' => [86400, '%Y%m%d', '%Y-%m-%d'],\n      'hour' => [3600, '%Y%m%d%H', '%Y-%m-%d_%H'],\n      'minute' => [60, '%Y%m%d%H%M', '%Y-%m-%d_%H%M'],\n    )\n    test 'generates path with timestamp placeholder for original path with tailing star with timekey' do |data|\n      timekey, placeholder, time_slice_format = data\n      # with index placeholder, without compression suffix when append disabled and compression disabled\n      assert_equal \"/path/to/file.#{placeholder}_**\",    @i.generate_path_template('/path/to/file.*', timekey, false, nil)\n      # with index placeholder, with .gz suffix when append disabled and gzip compression enabled\n      assert_equal \"/path/to/file.#{placeholder}_**.gz\", @i.generate_path_template('/path/to/file.*', timekey, false, :gzip)\n      # without index placeholder, without compression suffix when append enabled and compression disabled\n      assert_equal \"/path/to/file.#{placeholder}\",       @i.generate_path_template('/path/to/file.*', timekey, true, nil)\n      # without index placeholder, with .gz suffix when append disabled and gzip compression enabled\n      assert_equal \"/path/to/file.#{placeholder}.gz\",    @i.generate_path_template('/path/to/file.*', timekey, true, :gzip)\n\n      # time_slice_format will used instead of computed placeholder if specified\n      assert_equal \"/path/to/file.#{time_slice_format}_**\",    @i.generate_path_template('/path/to/file.*', timekey, false, nil, time_slice_format: time_slice_format)\n      assert_equal \"/path/to/file.#{time_slice_format}_**.gz\", @i.generate_path_template('/path/to/file.*', timekey, false, :gzip, time_slice_format: time_slice_format)\n      assert_equal \"/path/to/file.#{time_slice_format}\",       @i.generate_path_template('/path/to/file.*', timekey, true, nil, time_slice_format: time_slice_format)\n      assert_equal \"/path/to/file.#{time_slice_format}.gz\",    @i.generate_path_template('/path/to/file.*', timekey, true, :gzip, time_slice_format: time_slice_format)\n    end\n\n    data(\n      'day' => [86400 * 2, '%Y%m%d', '%Y-%m-%d'],\n      'hour' => [7200, '%Y%m%d%H', '%Y-%m-%d_%H'],\n      'minute' => [180, '%Y%m%d%H%M', '%Y-%m-%d_%H%M'],\n    )\n    test 'generates path with timestamp placeholder for original path with star and suffix with timekey' do |data|\n      timekey, placeholder, time_slice_format = data\n      # with index placeholder, without compression suffix when append disabled and compression disabled\n      assert_equal \"/path/to/file.#{placeholder}_**.data\",    @i.generate_path_template('/path/to/file.*.data', timekey, false, nil)\n      # with index placeholder, with .gz suffix when append disabled and gzip compression enabled\n      assert_equal \"/path/to/file.#{placeholder}_**.data.gz\", @i.generate_path_template('/path/to/file.*.data', timekey, false, :gzip)\n      # without index placeholder, without compression suffix when append enabled and compression disabled\n      assert_equal \"/path/to/file.#{placeholder}.data\",       @i.generate_path_template('/path/to/file.*.data', timekey, true, nil)\n      # without index placeholder, with .gz suffix when append disabled and gzip compression enabled\n      assert_equal \"/path/to/file.#{placeholder}.data.gz\",    @i.generate_path_template('/path/to/file.*.data', timekey, true, :gzip)\n\n      # time_slice_format will used instead of computed placeholder if specified\n      assert_equal \"/path/to/file.#{time_slice_format}_**.data\",    @i.generate_path_template('/path/to/file.*.data', timekey, false, nil, time_slice_format: time_slice_format)\n      assert_equal \"/path/to/file.#{time_slice_format}_**.data.gz\", @i.generate_path_template('/path/to/file.*.data', timekey, false, :gzip, time_slice_format: time_slice_format)\n      assert_equal \"/path/to/file.#{time_slice_format}.data\",       @i.generate_path_template('/path/to/file.*.data', timekey, true, nil, time_slice_format: time_slice_format)\n      assert_equal \"/path/to/file.#{time_slice_format}.data.gz\",    @i.generate_path_template('/path/to/file.*.data', timekey, true, :gzip, time_slice_format: time_slice_format)\n    end\n\n    test 'raise error to show it is a bug when path including * specified without timekey' do\n      assert_raise RuntimeError.new(\"BUG: configuration error must be raised for path including '*' without timekey\") do\n        @i.generate_path_template('/path/to/file.*.log', nil, false, nil)\n      end\n    end\n\n    data(\n      'day' => [86400 * 7, '%Y%m%d', '%Y-%m-%d'],\n      'hour' => [3600 * 6, '%Y%m%d%H', '%Y-%m-%d_%H'],\n      'minute' => [60 * 15, '%Y%m%d%H%M', '%Y-%m-%d_%H%M'],\n    )\n    test 'generates path with timestamp placeholder for original path without time placeholders & star with timekey, and path_suffix configured' do |data|\n      timekey, placeholder, time_slice_format = data\n      # with index placeholder, without compression suffix when append disabled and compression disabled\n      assert_equal \"/path/to/file.#{placeholder}_**.log\",    @i.generate_path_template('/path/to/file', timekey, false, nil, path_suffix: '.log')\n      # with index placeholder, with .gz suffix when append disabled and gzip compression enabled\n      assert_equal \"/path/to/file.#{placeholder}_**.log.gz\", @i.generate_path_template('/path/to/file', timekey, false, :gzip, path_suffix: '.log')\n      # without index placeholder, without compression suffix when append enabled and compression disabled\n      assert_equal \"/path/to/file.#{placeholder}.log\",       @i.generate_path_template('/path/to/file', timekey, true, nil, path_suffix: '.log')\n      # without index placeholder, with compression suffix when append enabled and gzip compression enabled\n      assert_equal \"/path/to/file.#{placeholder}.log.gz\",    @i.generate_path_template('/path/to/file', timekey, true, :gzip, path_suffix: '.log')\n\n      # time_slice_format will be appended always if it's specified\n      assert_equal \"/path/to/file.#{time_slice_format}_**.log\",    @i.generate_path_template('/path/to/file', timekey, false, nil, path_suffix: '.log', time_slice_format: time_slice_format)\n      assert_equal \"/path/to/file.#{time_slice_format}_**.log.gz\", @i.generate_path_template('/path/to/file', timekey, false, :gzip, path_suffix: '.log', time_slice_format: time_slice_format)\n      assert_equal \"/path/to/file.#{time_slice_format}.log\",       @i.generate_path_template('/path/to/file', timekey, true, nil, path_suffix: '.log', time_slice_format: time_slice_format)\n      assert_equal \"/path/to/file.#{time_slice_format}.log.gz\",    @i.generate_path_template('/path/to/file', timekey, true, :gzip, path_suffix: '.log', time_slice_format: time_slice_format)\n    end\n\n    data(\n      'day' => [86400, '%Y%m%d'],\n      'hour' => [3600, '%Y%m%d%H'],\n      'minute' => [60, '%Y%m%d%H%M'],\n    )\n    test 'generates path with timestamp placeholder for original path without star with timekey, and path_suffix not configured' do |data|\n      timekey, placeholder = data\n      # with index placeholder, without compression suffix when append disabled and compression disabled\n      assert_equal \"/path/to/file.#{placeholder}_**\",    @i.generate_path_template('/path/to/file', timekey, false, nil)\n      # with index placeholder, with .gz suffix when append disabled and gzip compression enabled\n      assert_equal \"/path/to/file.#{placeholder}_**.gz\", @i.generate_path_template('/path/to/file', timekey, false, :gzip)\n      # without index placeholder, without compression suffix when append enabled and compression disabled\n      assert_equal \"/path/to/file.#{placeholder}\",       @i.generate_path_template('/path/to/file', timekey, true, nil)\n      # without index placeholder, with compression suffix when append enabled and gzip compression enabled\n      assert_equal \"/path/to/file.#{placeholder}.gz\",    @i.generate_path_template('/path/to/file', timekey, true, :gzip)\n    end\n\n    test 'generates path without adding timestamp placeholder part if original path has enough placeholders for specified timekey' do\n      assert_equal \"/path/to/file.%Y%m%d\", @i.generate_path_template('/path/to/file.%Y%m%d', 86400, true, nil)\n      assert_equal \"/path/to/%Y%m%d/file\", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, nil)\n\n      assert_equal \"/path/to/%Y%m%d/file_**\", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, false, nil)\n\n      assert_raise Fluent::ConfigError.new(\"insufficient timestamp placeholders in path\") do\n        @i.generate_path_template('/path/to/%Y%m/file', 86400, true, nil)\n      end\n      assert_raise Fluent::ConfigError.new(\"insufficient timestamp placeholders in path\") do\n        @i.generate_path_template('/path/to/file.%Y%m%d.log', 3600, true, nil)\n      end\n\n      assert_equal \"/path/to/file.%Y%m%d_%H_**.log.gz\", @i.generate_path_template('/path/to/file.%Y%m%d_%H', 7200, false, :gzip, path_suffix: '.log')\n      assert_equal \"/path/to/${tag}/file.%Y%m%d_%H_**.log.gz\", @i.generate_path_template('/path/to/${tag}/file.%Y%m%d_%H', 7200, false, :gzip, path_suffix: '.log')\n    end\n\n    test 'generates path with specified time_slice_format appended even if path has sufficient timestamp placeholders' do\n      assert_equal \"/path/to/%Y%m%d/file.%Y-%m-%d_%H_**\", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, false, nil, time_slice_format: '%Y-%m-%d_%H')\n      assert_equal \"/path/to/%Y%m%d/file.%Y-%m-%d_%H\", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, nil, time_slice_format: '%Y-%m-%d_%H')\n      assert_equal \"/path/to/%Y%m%d/file.%Y-%m-%d_%H_**.log\", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, false, nil, time_slice_format: '%Y-%m-%d_%H', path_suffix: '.log')\n      assert_equal \"/path/to/%Y%m%d/file.%Y-%m-%d_%H.log\", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, nil, time_slice_format: '%Y-%m-%d_%H', path_suffix: '.log')\n      assert_equal \"/path/to/%Y%m%d/file.%Y-%m-%d_%H.log.gz\", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, :gzip, time_slice_format: '%Y-%m-%d_%H', path_suffix: '.log')\n    end\n\n    test 'generates path without timestamp placeholder when path does not include * and timekey not specified' do\n      assert_equal '/path/to/file.log', @i.generate_path_template('/path/to/file.log', nil, true, nil)\n      assert_equal '/path/to/file.log_**', @i.generate_path_template('/path/to/file.log', nil, false, nil)\n      assert_equal '/path/to/file.${tag}.log_**', @i.generate_path_template('/path/to/file.${tag}.log', nil, false, nil)\n      assert_equal '/path/to/file.${tag}_**.log', @i.generate_path_template('/path/to/file.${tag}', nil, false, nil, path_suffix: '.log')\n    end\n  end\n\n  sub_test_case '#find_filepath_available' do\n    setup do\n      @tmp = File.join(TMP_DIR, 'find_filepath_test')\n      FileUtils.mkdir_p @tmp\n      @i = create_driver.instance\n    end\n\n    teardown do\n      FileUtils.rm_rf @tmp\n    end\n\n    test 'raise error if argument path does not include index placeholder' do\n      assert_raise RuntimeError.new(\"BUG: index placeholder not found in path: #{@tmp}/myfile\") do\n        @i.find_filepath_available(\"#{@tmp}/myfile\") do |path|\n          # ...\n        end\n      end\n    end\n\n    data(\n      'without suffix' => ['myfile_0', 'myfile_**'],\n      'with timestamp' => ['myfile_20161003_0', 'myfile_20161003_**'],\n      'with base suffix' => ['myfile_0.log', 'myfile_**.log'],\n      'with compression suffix' => ['myfile_0.log.gz', 'myfile_**.log.gz'],\n    )\n    test 'returns filepath with _0 at first' do |data|\n      expected, argument = data\n      @i.find_filepath_available(File.join(@tmp, argument)) do |path|\n        assert_equal File.join(@tmp, expected), path\n      end\n    end\n\n    test 'returns filepath with index which does not exist yet' do\n      5.times do |i|\n        Fluent::FileWrapper.open(File.join(@tmp, \"exist_#{i}.log\"), 'a'){|f| } # open(create) and close\n      end\n      @i.find_filepath_available(File.join(@tmp, \"exist_**.log\")) do |path|\n        assert_equal File.join(@tmp, \"exist_5.log\"), path\n      end\n    end\n\n    test 'creates lock directory when with_lock is true to exclude operations of other worker process' do\n      5.times do |i|\n        Fluent::FileWrapper.open(File.join(@tmp, \"exist_#{i}.log\"), 'a')\n      end\n      Dir.mkdir(File.join(@tmp, \"exist_5.log.lock\"))\n      @i.find_filepath_available(File.join(@tmp, \"exist_**.log\"), with_lock: true) do |path|\n        assert Dir.exist?(File.join(@tmp, \"exist_6.log.lock\"))\n        assert_equal File.join(@tmp, \"exist_6.log\"), path\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_forward.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/output'\nrequire 'fluent/plugin/out_forward'\nrequire 'flexmock/test_unit'\n\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/in_forward'\n\nclass ForwardOutputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    FileUtils.rm_rf(TMP_DIR)\n    FileUtils.mkdir_p(TMP_DIR)\n    @d = nil\n    # forward plugin uses TCP and UDP sockets on the same port number\n    @target_port = unused_port(protocol: :all)\n  end\n\n  def teardown\n    @d.instance_shutdown if @d\n    @port = nil\n  end\n\n  TMP_DIR = File.join(__dir__, \"../tmp/out_forward#{ENV['TEST_ENV_NUMBER']}\")\n\n  TARGET_HOST = '127.0.0.1'\n\n  def config\n    %[\n      send_timeout 51\n      heartbeat_type udp\n      <server>\n        name test\n        host #{TARGET_HOST}\n        port #{@target_port}\n      </server>\n    ]\n  end\n\n  def target_config\n    %[\n      port #{@target_port}\n      bind #{TARGET_HOST}\n    ]\n  end\n\n  def create_driver(conf=config)\n    Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {\n      attr_reader :sent_chunk_ids, :ack_handler, :discovery_manager\n\n      def initialize\n        super\n        @sent_chunk_ids = []\n      end\n\n      def try_write(chunk)\n        retval = super\n        @sent_chunk_ids << chunk.unique_id\n        retval\n      end\n    }.configure(conf)\n  end\n\n  test 'configure' do\n    @d = d = create_driver(%[\n      self_hostname localhost\n      <server>\n        name test\n        host #{TARGET_HOST}\n        port #{@target_port}\n      </server>\n    ])\n    nodes = d.instance.nodes\n    assert_equal 60, d.instance.send_timeout\n    assert_equal :transport, d.instance.heartbeat_type\n    assert_equal 1, nodes.length\n    assert_nil d.instance.connect_timeout\n    node = nodes.first\n    assert_equal \"test\", node.name\n    assert_equal '127.0.0.1', node.host\n    assert_equal @target_port, node.port\n  end\n\n  test 'configure_traditional' do\n    @d = d = create_driver(<<EOL)\n      self_hostname localhost\n      <server>\n        name test\n        host #{TARGET_HOST}\n        port #{@target_port}\n      </server>\n      buffer_chunk_limit 10m\nEOL\n    instance = d.instance\n    assert instance.chunk_key_tag\n    assert !instance.chunk_key_time\n    assert_equal [], instance.chunk_keys\n    assert{ instance.buffer.is_a?(Fluent::Plugin::MemoryBuffer) }\n    assert_equal( 10*1024*1024, instance.buffer.chunk_limit_size )\n  end\n\n  test 'configure timeouts' do\n    @d = d = create_driver(%[\n      send_timeout 30\n      connect_timeout 10\n      hard_timeout 15\n      ack_response_timeout 20\n      <server>\n        host #{TARGET_HOST}\n        port #{@target_port}\n      </server>\n    ])\n    assert_equal 30, d.instance.send_timeout\n    assert_equal 10, d.instance.connect_timeout\n    assert_equal 15, d.instance.hard_timeout\n    assert_equal 20, d.instance.ack_response_timeout\n  end\n\n  test 'configure_udp_heartbeat' do\n    @d = d = create_driver(config + \"\\nheartbeat_type udp\")\n    assert_equal :udp, d.instance.heartbeat_type\n  end\n\n  test 'configure_none_heartbeat' do\n    @d = d = create_driver(config + \"\\nheartbeat_type none\")\n    assert_equal :none, d.instance.heartbeat_type\n  end\n\n  test 'configure_expire_dns_cache' do\n    @d = d = create_driver(config + \"\\nexpire_dns_cache 5\")\n    assert_equal 5, d.instance.expire_dns_cache\n  end\n\n  test 'configure_dns_round_robin udp' do\n    assert_raise(Fluent::ConfigError) do\n      create_driver(config + \"\\nheartbeat_type udp\\ndns_round_robin true\")\n    end\n  end\n\n  test 'configure_dns_round_robin transport' do\n    @d = d = create_driver(config + \"\\nheartbeat_type transport\\ndns_round_robin true\")\n    assert_equal true, d.instance.dns_round_robin\n  end\n\n  test 'configure_dns_round_robin none' do\n    @d = d = create_driver(config + \"\\nheartbeat_type none\\ndns_round_robin true\")\n    assert_equal true, d.instance.dns_round_robin\n  end\n\n  test 'configure_no_server' do\n    assert_raise(Fluent::ConfigError, 'forward output plugin requires at least one <server> is required') do\n      create_driver('')\n    end\n  end\n\n  test 'send_keepalive_packet is disabled by default' do\n    @d = d = create_driver(config)\n    assert_false d.instance.send_keepalive_packet\n  end\n\n  test 'send_keepalive_packet can be enabled' do\n    @d = d = create_driver(config + %[\n      keepalive true\n      send_keepalive_packet true\n    ])\n    assert_true d.instance.send_keepalive_packet\n  end\n\n  test 'send_keepalive_packet without keepalive raises error' do\n    assert_raise(Fluent::ConfigError.new(\"'send_keepalive_packet' is enabled but 'keepalive' is not. Enable 'keepalive' to use TCP keepalive.\")) do\n      create_driver(config + %[\n        send_keepalive_packet true\n      ])\n    end\n  end\n\n  test 'configure with ignore_network_errors_at_startup' do\n    normal_conf = config_element('match', '**', {}, [\n        config_element('server', '', {'name' => 'test', 'host' => 'unexisting.yaaaaaaaaaaaaaay.host.example.com'})\n      ])\n\n    if Socket.const_defined?(:ResolutionError) # as of Ruby 3.3\n      error_class = Socket::ResolutionError\n    else\n      error_class = SocketError\n    end\n\n    assert_raise error_class do\n      create_driver(normal_conf)\n    end\n\n    conf = config_element('match', '**', {'ignore_network_errors_at_startup' => 'true'}, [\n        config_element('server', '', {'name' => 'test', 'host' => 'unexisting.yaaaaaaaaaaaaaay.host.example.com'})\n      ])\n    @d = d = create_driver(conf)\n    expected_log = \"failed to resolve node name when configured\"\n    expected_detail = \"server=\\\"test\\\" error_class=#{error_class.name}\"\n    logs = d.logs\n    assert{ logs.any?{|log| log.include?(expected_log) && log.include?(expected_detail) } }\n  end\n\n  sub_test_case 'configure compress' do\n    data('default', ['', :text])\n    data('gzip', ['compress gzip', :gzip])\n    data('zstd', ['compress zstd', :zstd])\n    test 'should be applied' do |(option, expected)|\n      @d = d = create_driver(config + option)\n      node = d.instance.nodes.first\n\n      assert_equal(\n        [expected, expected],\n        [d.instance.compress, node.instance_variable_get(:@compress)]\n      )\n    end\n\n    data('default' => '')\n    data('gzip' => 'compress gzip')\n    data('zstd' => 'compress zstd')\n    test 'should log as experimental only for zstd' do |option|\n      @d = d = create_driver(config + option)\n\n      log_message = \"zstd compression feature is an experimental new feature\"\n      assert do\n        if d.instance.compress == :zstd\n          d.logs.any? { |log| log.include?(log_message) }\n        else\n          d.logs.none? { |log| log.include?(log_message) }\n        end\n      end\n    end\n\n    # TODO add tests that we cannot configure the different compress type between owner and buffer except for :text\n    data('gzip', ['compress gzip', :text, :gzip])\n    data('zstd', ['compress zstd', :text, :zstd])\n    test 'can configure buffer compress separately when owner uses :text' do |(buffer_option, expected_owner_compress, expected_buffer_compress)|\n      @d = d = create_driver(config + %[\n         <buffer>\n           type memory\n           #{buffer_option}\n         </buffer>\n       ])\n      node = d.instance.nodes.first\n\n      assert_equal(\n        [expected_owner_compress, expected_owner_compress, expected_buffer_compress],\n        [d.instance.compress, node.instance_variable_get(:@compress), d.instance.buffer.compress],\n      )\n\n      log_message = \"buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>\"\n      assert do\n        d.logs.any? { |log| log.include?(log_message) }\n      end\n    end\n  end\n\n  data('CA cert'     => 'tls_ca_cert_path',\n       'non CA cert' => 'tls_cert_path')\n  test 'configure tls_cert_path/tls_ca_cert_path' do |param|\n    dummy_cert_path = File.join(TMP_DIR, \"dummy_cert.pem\")\n    FileUtils.touch(dummy_cert_path)\n    conf = %[\n      send_timeout 5\n      transport tls\n      tls_insecure_mode true\n      #{param} #{dummy_cert_path}\n      <server>\n        host #{TARGET_HOST}\n        port #{@target_port}\n      </server>\n    ]\n\n    @d = d = create_driver(conf)\n    # In the plugin, tls_ca_cert_path is used for both cases\n    assert_equal([dummy_cert_path], d.instance.tls_ca_cert_path)\n  end\n\n  sub_test_case \"certstore loading parameters for Windows\" do\n    test 'certstore related config parameters' do\n      omit \"certstore related values raise error on not Windows\" if Fluent.windows?\n      conf = %[\n        send_timeout 5\n        transport tls\n        tls_cert_logical_store_name Root\n        tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b\n        <server>\n          host #{TARGET_HOST}\n          port #{@target_port}\n        </server>\n      ]\n\n      assert_raise(Fluent::ConfigError) do\n        create_driver(conf)\n      end\n    end\n\n    test 'cert_logical_store_name and tls_cert_thumbprint default values' do\n      conf = %[\n        send_timeout 5\n        transport tls\n        <server>\n          host #{TARGET_HOST}\n          port #{@target_port}\n        </server>\n      ]\n\n      @d = d = create_driver(conf)\n      assert_nil d.instance.tls_cert_logical_store_name\n      assert_nil d.instance.tls_cert_thumbprint\n    end\n\n    data('CA cert'     => 'tls_ca_cert_path',\n         'non CA cert' => 'tls_cert_path')\n    test 'specify tls_cert_logical_store_name and tls_cert_path should raise error' do |param|\n      omit \"Loading CertStore feature works only Windows\" unless Fluent.windows?\n      dummy_cert_path = File.join(TMP_DIR, \"dummy_cert.pem\")\n      FileUtils.touch(dummy_cert_path)\n      conf = %[\n        send_timeout 5\n        transport tls\n        #{param} #{dummy_cert_path}\n        tls_cert_logical_store_name Root\n        <server>\n          host #{TARGET_HOST}\n          port #{@target_port}\n        </server>\n      ]\n\n      assert_raise(Fluent::ConfigError) do\n        create_driver(conf)\n      end\n    end\n\n    test 'configure cert_logical_store_name and tls_cert_thumbprint' do\n      omit \"Loading CertStore feature works only Windows\" unless Fluent.windows?\n      conf = %[\n        send_timeout 5\n        transport tls\n        tls_cert_logical_store_name Root\n        tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b\n        <server>\n          host #{TARGET_HOST}\n          port #{@target_port}\n        </server>\n      ]\n\n      @d = d = create_driver(conf)\n      assert_equal \"Root\", d.instance.tls_cert_logical_store_name\n      assert_equal \"a909502dd82ae41433e6f83886b00d4277a32a7b\", d.instance.tls_cert_thumbprint\n    end\n  end\n\n  test 'server is an abbreviation of static type of service_discovery' do\n    @d = d = create_driver(%[\n<server>\n  host 127.0.0.1\n  port 1234\n</server>\n\n<service_discovery>\n  @type static\n\n  <service>\n    host 127.0.0.1\n    port 1235\n  </service>\n</service_discovery>\n    ])\n\n\n    assert_equal(\n      [\n        { host: '127.0.0.1', port: 1234 },\n        { host: '127.0.0.1', port: 1235 },\n      ],\n      d.instance.discovery_manager.services.collect do |service|\n        { host: service.host, port: service.port }\n      end\n    )\n  end\n\n  test 'pass username and password as empty string to HandshakeProtocol' do\n    config_path = File.join(TMP_DIR, \"sd_file.conf\")\n    File.open(config_path, 'w') do |file|\n      file.write(%[\n- 'host': 127.0.0.1\n  'port': 1234\n  'weight': 1\n])\n    end\n\n    mock(Fluent::Plugin::ForwardOutput::HandshakeProtocol).new(log: anything, hostname: nil, shared_key: anything, password: '', username: '')\n    @d = d = create_driver(%[\n<service_discovery>\n  @type file\n  path #{config_path}\n</service_discovery>\n    ])\n\n    assert_equal 1, d.instance.discovery_manager.services.size\n    assert_equal '127.0.0.1', d.instance.discovery_manager.services[0].host\n    assert_equal 1234, d.instance.discovery_manager.services[0].port\n  end\n\n  test 'phi_failure_detector disabled' do\n    @d = d = create_driver(config + %[phi_failure_detector false \\n phi_threshold 0])\n    node = d.instance.nodes.first\n    stub(node.failure).phi { raise 'Should not be called' }\n    node.tick\n    assert_true node.available?\n  end\n\n  test 'phi_failure_detector enabled' do\n    @d = d = create_driver(config + %[phi_failure_detector true \\n phi_threshold 0])\n    node = d.instance.nodes.first\n    node.tick\n    assert_false node.available?\n  end\n\n  test 'require_ack_response is disabled in default' do\n    @d = d = create_driver(config)\n    assert_equal false, d.instance.require_ack_response\n    assert_equal 190, d.instance.ack_response_timeout\n  end\n\n  test 'require_ack_response can be enabled' do\n    @d = d = create_driver(config + %[\n      require_ack_response true\n      ack_response_timeout 2s\n    ])\n    d.instance_start\n    assert d.instance.require_ack_response\n    assert_equal 2, d.instance.ack_response_timeout\n  end\n\n  test 'suspend_flush is disable before before_shutdown' do\n    @d = d = create_driver(config + %[\n      require_ack_response true\n      ack_response_timeout 2s\n    ])\n    d.instance_start\n    assert_false d.instance.instance_variable_get(:@suspend_flush)\n  end\n\n  test 'suspend_flush should be enabled and try_flush returns nil after before_shutdown' do\n    @d = d = create_driver(config + %[\n      require_ack_response true\n      ack_response_timeout 2s\n    ])\n    d.instance_start\n    d.instance.before_shutdown\n    assert_true d.instance.instance_variable_get(:@suspend_flush)\n    assert_nil d.instance.try_flush\n  end\n\n  test 'verify_connection_at_startup is disabled in default' do\n    @d = d = create_driver(config)\n    assert_false d.instance.verify_connection_at_startup\n  end\n\n  test 'verify_connection_at_startup can be enabled' do\n    @d = d = create_driver(config + %[\n      verify_connection_at_startup true\n    ])\n    assert_true d.instance.verify_connection_at_startup\n  end\n\n  test 'send tags in str (utf-8 strings)' do\n    target_input_driver = create_target_input_driver\n\n    @d = d = create_driver(config + %[flush_interval 1s])\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n    tag_in_utf8 = \"test.utf8\".encode(\"utf-8\")\n    tag_in_ascii = \"test.ascii\".encode(\"ascii-8bit\")\n\n    emit_events = [\n      [tag_in_utf8, time, {\"a\" => 1}],\n      [tag_in_ascii, time, {\"a\" => 2}],\n    ]\n\n    stub(d.instance.ack_handler).read_ack_from_sock(anything).never\n    assert_rr do\n      target_input_driver.run(expect_records: 2) do\n        d.run do\n          emit_events.each do |tag, t, record|\n            d.feed(tag, t, record)\n          end\n        end\n      end\n    end\n\n    events = target_input_driver.events\n    assert_equal_event_time(time, events[0][1])\n    assert_equal ['test.utf8', time, emit_events[0][2]], events[0]\n    assert_equal Encoding::UTF_8, events[0][0].encoding\n    assert_equal_event_time(time, events[1][1])\n    assert_equal ['test.ascii', time, emit_events[1][2]], events[1]\n    assert_equal Encoding::UTF_8, events[1][0].encoding\n  end\n\n  test 'send_with_time_as_integer' do\n    target_input_driver = create_target_input_driver\n\n    @d = d = create_driver(config + %[flush_interval 1s])\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n\n    stub(d.instance.ack_handler).read_ack_from_sock(anything).never\n    assert_rr do\n      target_input_driver.run(expect_records: 2) do\n        d.run(default_tag: 'test') do\n          records.each do |record|\n            d.feed(time, record)\n          end\n        end\n      end\n    end\n\n    events = target_input_driver.events\n    assert_equal_event_time(time, events[0][1])\n    assert_equal ['test', time, records[0]], events[0]\n    assert_equal_event_time(time, events[1][1])\n    assert_equal ['test', time, records[1]], events[1]\n  end\n\n  test 'send_without_time_as_integer' do\n    target_input_driver = create_target_input_driver\n\n    @d = d = create_driver(config + %[\n      flush_interval 1s\n      time_as_integer false\n    ])\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n    stub(d.instance.ack_handler).read_ack_from_sock(anything).never\n    assert_rr do\n      target_input_driver.run(expect_records: 2) do\n        d.run(default_tag: 'test') do\n          records.each do |record|\n            d.feed(time, record)\n          end\n        end\n      end\n    end\n\n    events = target_input_driver.events\n    assert_equal_event_time(time, events[0][1])\n    assert_equal ['test', time, records[0]], events[0]\n    assert_equal_event_time(time, events[1][1])\n    assert_equal ['test', time, records[1]], events[1]\n  end\n\n  test 'send_compressed_message_pack_stream_if_compress_is_gzip' do\n    target_input_driver = create_target_input_driver(conf: target_config + \"skip_invalid_event false\")\n\n    @d = d = create_driver(config + %[\n      flush_interval 1s\n      compress gzip\n    ])\n\n    time = event_time('2011-01-02 13:14:15 UTC')\n\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n    target_input_driver.run(expect_records: 2) do\n      d.run(default_tag: 'test') do\n        records.each do |record|\n          d.feed(time, record)\n        end\n      end\n    end\n\n    event_streams = target_input_driver.event_streams\n    assert_true event_streams[0][1].is_a?(Fluent::CompressedMessagePackEventStream)\n\n    events = target_input_driver.events\n    assert_equal ['test', time, records[0]], events[0]\n    assert_equal ['test', time, records[1]], events[1]\n  end\n\n  test 'send_compressed_message_pack_stream_if_compress_is_zstd' do\n    target_input_driver = create_target_input_driver(conf: target_config + \"skip_invalid_event false\")\n\n    @d = d = create_driver(config + %[\n      flush_interval 1s\n      compress zstd\n    ])\n\n    time = event_time('2011-01-02 13:14:15 UTC')\n\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n    target_input_driver.run(expect_records: 2) do\n      d.run(default_tag: 'test') do\n        records.each do |record|\n          d.feed(time, record)\n        end\n      end\n    end\n\n    event_streams = target_input_driver.event_streams\n    assert_true event_streams[0][1].is_a?(Fluent::CompressedMessagePackEventStream)\n\n    events = target_input_driver.events\n    assert_equal ['test', time, records[0]], events[0]\n    assert_equal ['test', time, records[1]], events[1]\n  end\n\n  test 'send_to_a_node_supporting_responses' do\n    target_input_driver = create_target_input_driver\n\n    @d = d = create_driver(config + %[flush_interval 1s])\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n    # not attempt to receive responses\n    stub(d.instance.ack_handler).read_ack_from_sock(anything).never\n    assert_rr do\n      target_input_driver.run(expect_records: 2) do\n        d.run(default_tag: 'test') do\n          records.each do |record|\n            d.feed(time, record)\n          end\n        end\n      end\n    end\n\n    events = target_input_driver.events\n    assert_equal ['test', time, records[0]], events[0]\n    assert_equal ['test', time, records[1]], events[1]\n  end\n\n  test 'send_to_a_node_not_supporting_responses' do\n    target_input_driver = create_target_input_driver\n\n    @d = d = create_driver(config + %[flush_interval 1s])\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n    # not attempt to receive responses\n    stub(d.instance.ack_handler).read_ack_from_sock(anything).never\n    assert_rr do\n      target_input_driver.run(expect_records: 2) do\n        d.run(default_tag: 'test') do\n          records.each do |record|\n            d.feed(time, record)\n          end\n        end\n      end\n    end\n\n    events = target_input_driver.events\n    assert_equal ['test', time, records[0]], events[0]\n    assert_equal ['test', time, records[1]], events[1]\n  end\n\n  test 'a node supporting responses' do\n    target_input_driver = create_target_input_driver\n\n    @d = d = create_driver(config + %[\n      require_ack_response true\n      <buffer tag>\n        flush_mode immediate\n        retry_type periodic\n        retry_wait 30s\n        flush_at_shutdown true\n      </buffer>\n    ])\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n    acked_chunk_ids = []\n    nacked = false\n    mock.proxy(d.instance.ack_handler).read_ack_from_sock(anything) do |info, success|\n      if success\n        acked_chunk_ids << info.chunk_id\n      else\n        nacked = true\n      end\n      [info, success]\n    end\n\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n    target_input_driver.run(expect_records: 2, timeout: 5) do\n      d.end_if { acked_chunk_ids.size > 0 || nacked }\n      d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do\n        d.feed([[time, records[0]], [time,records[1]]])\n      end\n    end\n\n    assert(!nacked, d.instance.log.logs.join)\n\n    events = target_input_driver.events\n    assert_equal ['test', time, records[0]], events[0]\n    assert_equal ['test', time, records[1]], events[1]\n\n    assert_equal 1, acked_chunk_ids.size\n    assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first\n  end\n\n  test 'a node supporting responses after stop' do\n    target_input_driver = create_target_input_driver\n\n    @d = d = create_driver(config + %[\n      require_ack_response true\n      <buffer tag>\n        flush_mode immediate\n        retry_type periodic\n        retry_wait 30s\n        flush_at_shutdown true\n      </buffer>\n    ])\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n    acked_chunk_ids = []\n    nacked = false\n    mock.proxy(d.instance.ack_handler).read_ack_from_sock(anything) do |info, success|\n      if success\n        acked_chunk_ids << info.chunk_id\n      else\n        nacked = true\n      end\n      [info, success]\n    end\n\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n    target_input_driver.run(expect_records: 2, timeout: 5) do\n      d.end_if { acked_chunk_ids.size > 0 || nacked }\n      d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do\n        d.instance.stop\n        d.feed([[time, records[0]], [time,records[1]]])\n        d.instance.before_shutdown\n        d.instance.shutdown\n        d.instance.after_shutdown\n      end\n    end\n\n    assert(!nacked, d.instance.log.logs.join)\n\n    events = target_input_driver.events\n    assert_equal ['test', time, records[0]], events[0]\n    assert_equal ['test', time, records[1]], events[1]\n\n    assert_equal 1, acked_chunk_ids.size\n    assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first\n  end\n\n  data('ack true' => true,\n       'ack false' => false)\n  test 'TLS transport and ack parameter combination' do |ack|\n    omit \"TLS and 'ack false' always fails on AppVeyor. Need to debug\" if Fluent.windows? && !ack\n\n    input_conf = target_config + %[\n                   <transport tls>\n                     insecure true\n                   </transport>\n                 ]\n    target_input_driver = create_target_input_driver(conf: input_conf)\n\n    output_conf = %[\n      send_timeout 5\n      require_ack_response #{ack}\n      transport tls\n      tls_insecure_mode true\n      <server>\n        host #{TARGET_HOST}\n        port #{@target_port}\n      </server>\n      <buffer>\n        #flush_mode immediate\n        flush_interval 0s\n        flush_at_shutdown false # suppress errors in d.instance_shutdown\n      </buffer>\n    ]\n    @d = d = create_driver(output_conf)\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    records = [{\"a\" => 1}, {\"a\" => 2}]\n    target_input_driver.run(expect_records: 2, timeout: 3) do\n      d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do\n        records.each do |record|\n          d.feed(time, record)\n        end\n      end\n    end\n\n    events = target_input_driver.events\n    assert{ events != [] }\n    assert_equal(['test', time, records[0]], events[0])\n    assert_equal(['test', time, records[1]], events[1])\n  end\n\n  test 'a destination node not supporting responses by just ignoring' do\n    target_input_driver = create_target_input_driver(response_stub: ->(_option) { nil }, disconnect: false)\n\n    @d = d = create_driver(config + %[\n      require_ack_response true\n      ack_response_timeout 1s\n      <buffer tag>\n        flush_mode immediate\n        retry_type periodic\n        retry_wait 30s\n        flush_at_shutdown false # suppress errors in d.instance_shutdown\n        flush_thread_interval 30s\n      </buffer>\n    ])\n\n    node = d.instance.nodes.first\n    delayed_commit_timeout_value = nil\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n    target_input_driver.end_if{ d.instance.rollback_count > 0 }\n    target_input_driver.end_if{ !node.available? }\n    target_input_driver.run(expect_records: 2, timeout: 25) do\n      d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do\n        delayed_commit_timeout_value = d.instance.delayed_commit_timeout\n        d.feed([[time, records[0]], [time,records[1]]])\n      end\n    end\n\n    assert_equal (1 + 2), delayed_commit_timeout_value\n\n    events = target_input_driver.events\n    assert_equal ['test', time, records[0]], events[0]\n    assert_equal ['test', time, records[1]], events[1]\n\n    assert{ d.instance.rollback_count > 0 }\n\n    logs = d.instance.log.logs\n    assert{ logs.any?{|log| log.include?(\"no response from node. regard it as unavailable.\") } }\n  end\n\n  test 'a destination node not supporting responses by disconnection' do\n    target_input_driver = create_target_input_driver(response_stub: ->(_option) { nil }, disconnect: true)\n\n    @d = d = create_driver(config + %[\n      require_ack_response true\n      ack_response_timeout 1s\n      <buffer tag>\n        flush_mode immediate\n        retry_type periodic\n        retry_wait 30s\n        flush_at_shutdown false # suppress errors in d.instance_shutdown\n        flush_thread_interval 30s\n      </buffer>\n    ])\n\n    node = d.instance.nodes.first\n    delayed_commit_timeout_value = nil\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n    target_input_driver.end_if{ d.instance.rollback_count > 0 }\n    target_input_driver.end_if{ !node.available? }\n    target_input_driver.run(expect_records: 2, timeout: 25) do\n      d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do\n        delayed_commit_timeout_value = d.instance.delayed_commit_timeout\n        d.feed([[time, records[0]], [time,records[1]]])\n      end\n    end\n\n    assert_equal (1 + 2), delayed_commit_timeout_value\n\n    events = target_input_driver.events\n    assert_equal ['test', time, records[0]], events[0]\n    assert_equal ['test', time, records[1]], events[1]\n\n    assert{ d.instance.rollback_count > 0 }\n\n    logs = d.instance.log.logs\n    assert{ logs.any?{|log| log.include?(\"no response from node. regard it as unavailable.\") } }\n  end\n\n  test 'authentication_with_shared_key' do\n    input_conf = target_config + %[\n                   <security>\n                     self_hostname in.localhost\n                     shared_key fluentd-sharedkey\n                     <client>\n                       host 127.0.0.1\n                     </client>\n                   </security>\n                 ]\n    target_input_driver = create_target_input_driver(conf: input_conf)\n\n    output_conf = %[\n      send_timeout 51\n      <security>\n        self_hostname localhost\n        shared_key fluentd-sharedkey\n      </security>\n      <server>\n        name test\n        host #{TARGET_HOST}\n        port #{@target_port}\n        shared_key fluentd-sharedkey\n      </server>\n    ]\n    @d = d = create_driver(output_conf)\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n\n    target_input_driver.run(expect_records: 2, timeout: 15) do\n      d.run(default_tag: 'test') do\n        records.each do |record|\n          d.feed(time, record)\n        end\n      end\n    end\n\n    events = target_input_driver.events\n    assert{ events != [] }\n    assert_equal(['test', time, records[0]], events[0])\n    assert_equal(['test', time, records[1]], events[1])\n  end\n\n  test 'keepalive + shared_key' do\n    input_conf = target_config + %[\n                   <security>\n                     self_hostname in.localhost\n                     shared_key fluentd-sharedkey\n                   </security>\n                 ]\n    target_input_driver = create_target_input_driver(conf: input_conf)\n\n    output_conf = %[\n      send_timeout 51\n      keepalive true\n      <security>\n        self_hostname localhost\n        shared_key fluentd-sharedkey\n      </security>\n      <server>\n        name test\n        host #{TARGET_HOST}\n        port #{@target_port}\n      </server>\n    ]\n    @d = d = create_driver(output_conf)\n\n    time = event_time('2011-01-02 13:14:15 UTC')\n    records = [{ 'a' => 1 }, { 'a' => 2 }]\n    records2 = [{ 'b' => 1}, { 'b' => 2}]\n    target_input_driver.run(expect_records: 4, timeout: 15) do\n      d.run(default_tag: 'test') do\n        records.each do |record|\n          d.feed(time, record)\n        end\n\n        d.flush # emit buffer to reuse same socket later\n        records2.each do |record|\n          d.feed(time, record)\n        end\n      end\n    end\n\n    events = target_input_driver.events\n    assert{ events != [] }\n    assert_equal(['test', time, records[0]], events[0])\n    assert_equal(['test', time, records[1]], events[1])\n    assert_equal(['test', time, records2[0]], events[2])\n    assert_equal(['test', time, records2[1]], events[3])\n  end\n\n   test 'authentication_with_user_auth' do\n    input_conf = target_config + %[\n                   <security>\n                     self_hostname in.localhost\n                     shared_key fluentd-sharedkey\n                     user_auth true\n                     <user>\n                       username fluentd\n                       password fluentd\n                     </user>\n                     <client>\n                       host 127.0.0.1\n                     </client>\n                   </security>\n                 ]\n    target_input_driver = create_target_input_driver(conf: input_conf)\n\n    output_conf = %[\n      send_timeout 51\n      <security>\n        self_hostname localhost\n        shared_key fluentd-sharedkey\n      </security>\n      <server>\n        name test\n        host #{TARGET_HOST}\n        port #{@target_port}\n        shared_key fluentd-sharedkey\n        username fluentd\n        password fluentd\n      </server>\n    ]\n    @d = d = create_driver(output_conf)\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    records = [\n      {\"a\" => 1},\n      {\"a\" => 2}\n    ]\n\n    target_input_driver.run(expect_records: 2, timeout: 15) do\n      d.run(default_tag: 'test') do\n        records.each do |record|\n          d.feed(time, record)\n        end\n      end\n    end\n\n    events = target_input_driver.events\n    assert{ events != [] }\n    assert_equal(['test', time, records[0]], events[0])\n    assert_equal(['test', time, records[1]], events[1])\n  end\n\n  # This test is not 100% but test failed with previous Node implementation which has race condition\n  test 'Node with security is thread-safe on multi threads' do\n    input_conf = target_config + %[\n                   <security>\n                     self_hostname in.localhost\n                     shared_key fluentd-sharedkey\n                     <client>\n                       host 127.0.0.1\n                     </client>\n                   </security>\n                 ]\n    target_input_driver = create_target_input_driver(conf: input_conf)\n    output_conf = %[\n      send_timeout 51\n      <security>\n        self_hostname localhost\n        shared_key fluentd-sharedkey\n      </security>\n      <server>\n        name test\n        host #{TARGET_HOST}\n        port #{@target_port}\n        shared_key fluentd-sharedkey\n      </server>\n    ]\n    @d = d = create_driver(output_conf)\n\n    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))\n    target_input_driver.run(timeout: 15) do\n      d.run(shutdown: false) do\n        node = d.instance.nodes.first\n        arr = []\n        4.times {\n          arr << Thread.new {\n            node.send_data('test', chunk) rescue nil\n          }\n        }\n        arr.each { |a| a.join }\n      end\n    end\n\n    logs = d.logs\n    assert_false(logs.any? { |log|\n                   log.include?(\"invalid format for PONG message\") || log.include?(\"shared key mismatch\")\n                 }, \"Actual log:\\n#{logs.join}\")\n  end\n\n  def create_target_input_driver(response_stub: nil, disconnect: false, conf: target_config)\n    require 'fluent/plugin/in_forward'\n\n    # TODO: Support actual TCP heartbeat test\n    Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput) {\n      if response_stub.nil?\n        # do nothing because in_forward responds for ack option in default\n      else\n        define_method(:response) do |options|\n          return response_stub.(options)\n        end\n      end\n    }.configure(conf)\n  end\n\n  test 'heartbeat_type_none' do\n    @d = d = create_driver(config + \"\\nheartbeat_type none\")\n    node = d.instance.nodes.first\n    assert_equal Fluent::Plugin::ForwardOutput::NoneHeartbeatNode, node.class\n\n    d.instance_start\n    assert_nil d.instance.instance_variable_get(:@loop)   # no HeartbeatHandler, or HeartbeatRequestTimer\n    assert_nil d.instance.instance_variable_get(:@thread) # no HeartbeatHandler, or HeartbeatRequestTimer\n\n    stub(node.failure).phi { raise 'Should not be called' }\n    node.tick\n    assert_true node.available?\n  end\n\n  test 'heartbeat_type_udp' do\n    @d = d = create_driver(config + \"\\nheartbeat_type udp\")\n\n    d.instance_start\n    usock = d.instance.instance_variable_get(:@usock)\n    servers = d.instance.instance_variable_get(:@_servers)\n    timers = d.instance.instance_variable_get(:@_timers)\n    assert_equal Fluent::PluginHelper::Socket::WrappedSocket::UDP, usock.class\n    assert_kind_of UDPSocket, usock\n    assert servers.find{|s| s.title == :out_forward_heartbeat_receiver }\n    assert timers.include?(:out_forward_heartbeat_request)\n\n    mock(usock).send(\"\\0\", 0, Socket.pack_sockaddr_in(@target_port, '127.0.0.1')).once\n    d.instance.send(:on_heartbeat_timer)\n  end\n\n  test 'acts_as_secondary' do\n    i = Fluent::Plugin::ForwardOutput.new\n    conf = config_element(\n      'match',\n      'primary.**',\n      {'@type' => 'forward'},\n      [\n        config_element('server', '', {'host' => '127.0.0.1'}),\n        config_element('secondary', '', {}, [\n            config_element('server', '', {'host' => '192.168.1.2'}),\n            config_element('server', '', {'host' => '192.168.1.3'})\n          ]),\n      ]\n    )\n    assert_nothing_raised do\n      i.configure(conf)\n    end\n  end\n\n  test 'when out_forward has @id' do\n    # cancel https://github.com/fluent/fluentd/blob/077508ac817b7637307434d0c978d7cdc3d1c534/lib/fluent/plugin_id.rb#L43-L53\n    # it always return true in test\n    mock.proxy(Fluent::Plugin).new_sd('static', parent: anything) { |v|\n      stub(v).plugin_id_for_test? { false }\n    }.once\n\n    output = Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {\n      def plugin_id_for_test?\n        false\n      end\n    }\n\n    assert_nothing_raised do\n      output.configure(config + %[\n        @id unique_out_forward\n      ])\n    end\n  end\n\n  sub_test_case 'verify_connection_at_startup' do\n    test 'nodes are not available' do\n      @d = d = create_driver(config + %[\n        verify_connection_at_startup true\n      ])\n      e = assert_raise Fluent::UnrecoverableError do\n        d.instance_start\n      end\n      if Fluent.windows?\n        assert_match(/No connection could be made because the target machine actively refused it/, e.message)\n      else\n        assert_match(/Connection refused/, e.message)\n      end\n\n      d.instance_shutdown\n    end\n\n    test 'nodes_shared_key_miss_match' do\n      input_conf = target_config + %[\n                   <security>\n                     self_hostname in.localhost\n                     shared_key fluentd-sharedkey\n                   </security>\n                 ]\n      target_input_driver = create_target_input_driver(conf: input_conf)\n      output_conf = %[\n        transport tcp\n        verify_connection_at_startup true\n        <security>\n          self_hostname localhost\n          shared_key key_miss_match\n        </security>\n\n        <server>\n          host #{TARGET_HOST}\n          port #{@target_port}\n        </server>\n      ]\n      @d = d = create_driver(output_conf)\n\n      target_input_driver.run(expect_records: 1, timeout: 1) do\n        e = assert_raise Fluent::UnrecoverableError do\n          d.instance_start\n        end\n        assert_match(/failed to establish connection/, e.message)\n      end\n    end\n\n    test 'nodes_shared_key_miss_match with TLS' do\n      input_conf = target_config + %[\n                   <security>\n                     self_hostname in.localhost\n                     shared_key fluentd-sharedkey\n                   </security>\n                   <transport tls>\n                     insecure true\n                   </transport>\n                 ]\n      target_input_driver = create_target_input_driver(conf: input_conf)\n      output_conf = %[\n        transport tls\n        tls_insecure_mode true\n        verify_connection_at_startup true\n        <security>\n          self_hostname localhost\n          shared_key key_miss_match\n        </security>\n\n        <server>\n          host #{TARGET_HOST}\n          port #{@target_port}\n        </server>\n      ]\n      @d = d = create_driver(output_conf)\n\n      target_input_driver.run(expect_records: 1, timeout: 1) do\n        e = assert_raise Fluent::UnrecoverableError do\n          d.instance_start\n        end\n\n        assert_match(/failed to establish connection/, e.message)\n      end\n    end\n\n    test 'nodes_shared_key_match' do\n      input_conf = target_config + %[\n                       <security>\n                         self_hostname in.localhost\n                         shared_key fluentd-sharedkey\n                       </security>\n                     ]\n      target_input_driver = create_target_input_driver(conf: input_conf)\n      output_conf = %[\n          verify_connection_at_startup true\n          <security>\n            self_hostname localhost\n            shared_key fluentd-sharedkey\n          </security>\n          <server>\n            name test\n            host #{TARGET_HOST}\n            port #{@target_port}\n          </server>\n      ]\n      @d = d = create_driver(output_conf)\n\n      time = event_time(\"2011-01-02 13:14:15 UTC\")\n      records = [{ \"a\" => 1 }, { \"a\" => 2 }]\n\n      target_input_driver.run(expect_records: 2, timeout: 3) do\n        d.run(default_tag: 'test') do\n          records.each do |record|\n            d.feed(time, record)\n          end\n        end\n      end\n\n      events = target_input_driver.events\n      assert_false events.empty?\n      assert_equal(['test', time, records[0]], events[0])\n      assert_equal(['test', time, records[1]], events[1])\n    end\n  end\n\n  test 'Create new connection per send_data' do\n    target_input_driver = create_target_input_driver(conf: target_config)\n    output_conf = config\n    d = create_driver(output_conf)\n\n    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))\n    mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,\n                                             linger_timeout: anything,\n                                             send_timeout: anything,\n                                             recv_timeout: anything,\n                                             connect_timeout: anything,\n                                             send_keepalive_packet: anything\n                                            ) { |sock| mock(sock).close.once; sock }.twice\n\n    target_input_driver.run(timeout: 15) do\n      d.run do\n        node = d.instance.nodes.first\n        2.times do\n          node.send_data('test', chunk) rescue nil\n        end\n      end\n    end\n  end\n\n  test 'if no available node' do\n    # do not create output driver\n    d = create_driver(%[\n    <server>\n      name test\n      standby\n      host #{TARGET_HOST}\n      port #{@target_port}\n    </server>\n    ])\n    assert_nothing_raised { d.run }\n  end\n\n  sub_test_case 'keepalive' do\n    test 'Do not create connection per send_data' do\n      target_input_driver = create_target_input_driver(conf: target_config)\n      output_conf = config + %[\n        keepalive true\n        keepalive_timeout 2\n      ]\n      d = create_driver(output_conf)\n\n      chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))\n      mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,\n                                               linger_timeout: anything,\n                                               send_timeout: anything,\n                                               recv_timeout: anything,\n                                               connect_timeout: anything,\n                                               send_keepalive_packet: anything\n                                              ) { |sock| mock(sock).close.once; sock }.once\n\n      target_input_driver.run(timeout: 15) do\n        d.run do\n          node = d.instance.nodes.first\n          2.times do\n            node.send_data('test', chunk) rescue nil\n          end\n        end\n      end\n    end\n\n    test 'create timer of purging obsolete sockets' do\n      output_conf = config + %[keepalive true]\n      @d = d = create_driver(output_conf)\n\n      mock(d.instance).timer_execute(:out_forward_heartbeat_request, 1).once\n      mock(d.instance).timer_execute(:out_forward_keep_alived_socket_watcher, 5).once\n      d.instance_start\n    end\n\n    sub_test_case 'with require_ack_response' do\n      test 'Create connection per send_data' do\n        target_input_driver = create_target_input_driver(conf: target_config)\n        output_conf = config + %[\n          require_ack_response true\n          keepalive true\n          keepalive_timeout 2\n        ]\n        d = create_driver(output_conf)\n\n        chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))\n        mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,\n                                                 linger_timeout: anything,\n                                                 send_timeout: anything,\n                                                 recv_timeout: anything,\n                                                 connect_timeout: anything,\n                                                 send_keepalive_packet: anything) { |sock|\n          mock(sock).close.once; sock\n        }.twice\n\n        target_input_driver.run(timeout: 15) do\n          d.run do\n            node = d.instance.nodes.first\n            2.times do\n              node.send_data('test', chunk) rescue nil\n            end\n          end\n        end\n      end\n    end\n  end\n\n  test 'can use metrics plugins and fallback methods' do\n    @d = create_driver\n\n    %w[healthy_nodes_count_metrics registered_nodes_count_metrics].each do |metric_name|\n      assert_true @d.instance.instance_variable_get(:\"@#{metric_name}\").is_a?(Fluent::Plugin::Metrics)\n    end\n\n    assert_equal 0, @d.instance.healthy_nodes_count\n    assert_equal 0, @d.instance.registered_nodes_count\n  end\n\n  test 'establish_connection_timeout' do\n    @d = d = create_driver(%[\n      hard_timeout 1\n      <server>\n        host #{TARGET_HOST}\n        port #{@target_port}\n      </server>\n    ])\n\n    node = d.instance.nodes.first\n    mock_sock = flexmock('socket')\n    mock_sock.should_receive(:read_nonblock).with(512).and_return('').at_least.once\n\n    ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)\n\n    assert_true node.available?\n    node.establish_connection(mock_sock, ri)\n    assert_false node.available?\n\n    logs = d.logs\n    assert{ logs.any?{|log| log.include?('handshake timeout after 1.0s') } }\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_http.rb",
    "content": "require_relative \"../helper\"\nrequire 'fluent/test/driver/output'\nrequire 'fluent/plugin/out_http'\n\nrequire 'webrick'\nrequire 'webrick/https'\nrequire 'net/http'\nrequire 'uri'\nrequire 'json'\nrequire 'aws-sdk-core'\n\n# WEBrick's ProcHandler doesn't handle PUT by default\nmodule WEBrick::HTTPServlet\n  class ProcHandler < AbstractServlet\n    alias do_PUT do_GET\n  end\nend\n\nclass HTTPOutputTest < Test::Unit::TestCase\n  include Fluent::Test::Helpers\n\n  TMP_DIR = File.join(__dir__, \"../tmp/out_http#{ENV['TEST_ENV_NUMBER']}\")\n  DEFAULT_LOGGER = ::WEBrick::Log.new(::STDOUT, ::WEBrick::BasicLog::FATAL)\n\n  class << self\n    # Use class variable to reduce server start/shutdown time\n    def startup\n      @@result = nil\n      @@auth_handler = nil\n      @@http_server_thread = nil\n    end\n\n    def shutdown\n      @@http_server_thread.kill\n      @@http_server_thread.join\n    rescue\n    end\n  end\n\n  def server_port\n    19880\n  end\n\n  def base_endpoint\n    \"http://127.0.0.1:#{server_port}\"\n  end\n\n  def server_config\n    config = {BindAddress: '127.0.0.1', Port: server_port}\n    # Suppress webrick logs\n    config[:Logger] = DEFAULT_LOGGER\n    config[:AccessLog] = []\n    config\n  end\n\n  def http_client(**opts, &block)\n    opts = opts.merge(open_timeout: 1, read_timeout: 1)\n    if block_given?\n      Net::HTTP.start('127.0.0.1', server_port, **opts, &block)\n    else\n      Net::HTTP.start('127.0.0.1', server_port, **opts)\n    end\n  end\n\n  def run_http_server\n    server = ::WEBrick::HTTPServer.new(server_config)\n    server.mount_proc('/test') { |req, res|\n      if @@auth_handler\n        @@auth_handler.call(req, res)\n      end\n\n      @@result.method = req.request_method\n      @@result.content_type = req.content_type\n      req.each do |key, value|\n        @@result.headers[key] = value\n      end\n\n      body = \"\"\n      data = []\n\n      case req['content-encoding']\n      when 'gzip'\n        body = Zlib::GzipReader.new(StringIO.new(req.body)).read\n      else\n        body = req.body\n      end\n\n      case req.content_type\n      when 'application/x-ndjson'\n        body.each_line { |l|\n          data << JSON.parse(l)\n        }\n      when 'application/json'\n        data = JSON.parse(body)\n      when 'text/plain'\n        # Use single_value in this test\n        body.each_line { |line|\n          data << line.chomp\n        }\n      else\n        data << body\n      end\n      @@result.data = data\n\n      res.status = 200\n      res.body = \"success\"\n    }\n    server.mount_proc('/503') { |_, res|\n      res.status = 503\n      res.body = 'Service Unavailable'\n    }\n    server.mount_proc('/404') { |_, res|\n      res.status = 404\n      res.body = 'Not Found'\n    }\n    # For start check\n    server.mount_proc('/') { |_, res|\n      res.status = 200\n      res.body = 'Hello Fluentd!'\n    }\n    server.start\n  ensure\n    server.shutdown rescue nil\n  end\n\n  Result = Struct.new(\"Result\", :method, :content_type, :headers, :data)\n\n  setup do\n    Fluent::Test.setup\n    FileUtils.rm_rf(TMP_DIR)\n\n    @@result = Result.new(nil, nil, {}, nil)\n    @@http_server_thread ||= Thread.new do\n      run_http_server\n    end\n\n    now = Time.now\n    started = false\n    until started\n      raise \"Server not started\" if (Time.now - now > 10.0)\n      begin\n        http_client { |c| c.request_get('/') }\n        started = true\n      rescue\n        sleep 0.5\n      end\n    end\n  end\n\n  teardown do\n    @@result = nil\n    @@auth_handler = nil\n  end\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Output.new(Fluent::Plugin::HTTPOutput).configure(conf)\n  end\n\n  def config\n    %[\n      endpoint #{base_endpoint}/test\n    ]\n  end\n\n  def test_events\n    [\n      {\"message\" => \"hello\", \"num\" => 10, \"bool\" => true},\n      {\"message\" => \"hello\", \"num\" => 11, \"bool\" => false}\n    ]\n  end\n\n  def test_configure\n    d = create_driver(config)\n    assert_equal \"http://127.0.0.1:#{server_port}/test\", d.instance.endpoint\n    assert_equal :post, d.instance.http_method\n    assert_equal 'application/x-ndjson', d.instance.content_type\n    assert_equal [503], d.instance.retryable_response_codes\n    assert_true d.instance.error_response_as_unrecoverable\n    assert_nil d.instance.proxy\n    assert_nil d.instance.headers\n  end\n\n  def test_configure_with_warn\n    d = create_driver(config)\n    assert_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join)\n  end\n\n  def test_configure_without_warn\n    d = create_driver(<<~CONFIG)\n      endpoint #{base_endpoint}/test\n      retryable_response_codes [503]\n    CONFIG\n    assert_not_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join)\n  end\n\n  # Check if an exception is raised on not JSON format use\n  data('not_json' => 'msgpack')\n  def test_configure_with_json_array_err(format_type)\n    assert_raise(Fluent::ConfigError) do\n      create_driver(config + %[\n        json_array true\n        <format>\n          @type #{format_type}\n        </format>\n      ])\n    end\n  end\n\n  data('json' => ['json', 'application/x-ndjson'],\n       'ltsv' => ['ltsv', 'text/tab-separated-values'],\n       'msgpack' => ['msgpack', 'application/x-msgpack'],\n       'single_value' => ['single_value', 'text/plain'])\n  def test_configure_content_type(types)\n    format_type, content_type = types\n    d = create_driver(config + %[\n      <format>\n        @type #{format_type}\n      </format>\n    ])\n    assert_equal content_type, d.instance.content_type\n  end\n\n  # Check that json_array setting sets content_type = application/json\n  data('json' => 'application/json')\n  def test_configure_content_type_json_array(content_type)\n    d = create_driver(config + \"json_array true\")\n\n    assert_equal content_type, d.instance.content_type\n  end\n\n  data('PUT' => 'put', 'POST' => 'post')\n  def test_write_with_method(method)\n    d = create_driver(config + \"http_method #{method}\")\n    d.run(default_tag: 'test.http') do\n      test_events.each { |event|\n        d.feed(event)\n      }\n    end\n\n    result = @@result\n    assert_equal method.upcase, result.method\n    assert_equal 'application/x-ndjson', result.content_type\n    assert_equal test_events, result.data\n    assert_not_empty result.headers\n  end\n\n  # Check that JSON at HTTP request body is valid\n  def test_write_with_json_array_setting\n    d = create_driver(config + \"json_array true\")\n    d.run(default_tag: 'test.http') do\n      test_events.each { |event|\n        d.feed(event)\n      }\n    end\n\n    result = @@result\n    assert_equal 'application/json', result.content_type\n    assert_equal test_events, result.data\n    assert_not_empty result.headers\n  end\n\n  def test_write_with_single_value_format\n    d = create_driver(config + %[\n      <format>\n        @type single_value\n      </format>\n    ])\n    d.run(default_tag: 'test.http') do\n      test_events.each { |event|\n        d.feed(event)\n      }\n    end\n\n    result = @@result\n    assert_equal 'text/plain', result.content_type\n    assert_equal (test_events.map { |e| e['message'] }), result.data\n    assert_not_empty result.headers\n  end\n\n  def test_write_with_headers\n    d = create_driver(config + 'headers {\"test_header\":\"fluentd!\"}')\n    d.run(default_tag: 'test.http') do\n      test_events.each { |event|\n        d.feed(event)\n      }\n    end\n\n    result = @@result\n    assert_true result.headers.has_key?('test_header')\n    assert_equal \"fluentd!\", result.headers['test_header']\n  end\n\n  def test_write_with_headers_from_placeholders\n    d = create_driver(config + %[\n      headers_from_placeholders {\"x-test\":\"${$.foo.bar}-test\",\"x-tag\":\"${tag}\"}\n      <buffer tag,$.foo.bar>\n      </buffer>\n    ])\n    d.run(default_tag: 'test.http') do\n      test_events.each { |event|\n        ev = event.dup\n        ev['foo'] = {'bar' => 'abcd'}\n        d.feed(ev)\n      }\n    end\n\n    result = @@result\n    assert_equal \"abcd-test\", result.headers['x-test']\n    assert_equal \"test.http\", result.headers['x-tag']\n  end\n\n  def test_write_with_retryable_response\n    old_report_on_exception = Thread.report_on_exception\n    Thread.report_on_exception = false # thread finished as invalid state since RetryableResponse raises.\n\n    d = create_driver(\"endpoint #{base_endpoint}/503\")\n    assert_raise(Fluent::Plugin::HTTPOutput::RetryableResponse) do\n      d.run(default_tag: 'test.http', shutdown: false) do\n        test_events.each { |event|\n          d.feed(event)\n        }\n      end\n    end\n\n    d.instance_shutdown(log: $log)\n  ensure\n    Thread.report_on_exception = old_report_on_exception\n  end\n\n  def test_write_with_disabled_unrecoverable\n    d = create_driver(%[\n      endpoint #{base_endpoint}/404\n      error_response_as_unrecoverable false\n    ])\n    d.run(default_tag: 'test.http', shutdown: false) do\n      test_events.each { |event|\n        d.feed(event)\n      }\n    end\n    assert_match(/got error response from.*404 Not Found Not Found/, d.instance.log.out.logs.join)\n    d.instance_shutdown\n  end\n\n  sub_test_case 'basic auth' do\n    setup do\n      FileUtils.mkdir_p(TMP_DIR)\n      htpd = WEBrick::HTTPAuth::Htpasswd.new(File.join(TMP_DIR, 'dot.htpasswd'))\n      htpd.set_passwd(nil, 'test', 'hey')\n      authenticator = WEBrick::HTTPAuth::BasicAuth.new(:UserDB => htpd, :Realm => 'test', :Logger => DEFAULT_LOGGER)\n      @@auth_handler = Proc.new { |req, res| authenticator.authenticate(req, res) }\n    end\n\n    teardown do\n      FileUtils.rm_rf(TMP_DIR)\n    end\n\n    def server_port\n      19881\n    end\n\n    def test_basic_auth\n      d = create_driver(config + %[\n        <auth>\n          method basic\n          username test\n          password hey\n        </auth>\n      ])\n      d.run(default_tag: 'test.http') do\n        test_events.each { |event|\n          d.feed(event)\n        }\n      end\n\n      result = @@result\n      assert_equal 'POST', result.method\n      assert_equal 'application/x-ndjson', result.content_type\n      assert_equal test_events, result.data\n      assert_not_empty result.headers\n    end\n\n    # This test includes `error_response_as_unrecoverable true` behaviour check\n    def test_basic_auth_with_invalid_auth\n      d = create_driver(config + %[\n        <auth>\n          method basic\n          username ayaya\n          password hello?\n        </auth>\n      ])\n      d.instance.system_config_override(root_dir: TMP_DIR) # Backup files are generated in TMP_DIR.\n      d.run(default_tag: 'test.http', shutdown: false) do\n        test_events.each { |event|\n          d.feed(event)\n        }\n      end\n      assert_match(/got unrecoverable error/, d.instance.log.out.logs.join)\n\n      d.instance_shutdown\n    end\n  end\n\n\n  sub_test_case 'aws sigv4 auth' do\n    setup do\n      @@fake_aws_credentials = Aws::Credentials.new(\n        'fakeaccess',\n        'fakesecret',\n        'fake session token'\n      )\n    end\n\n    def server_port\n      19883\n    end\n\n    def test_aws_sigv4_sts_role_arn\n      stub(Aws::AssumeRoleCredentials).new do |credentials_provider|\n        stub(credentials_provider).credentials {\n          @@fake_aws_credentials\n        }\n        credentials_provider\n      end\n\n      d = create_driver(config + %[\n          <auth>\n            method aws_sigv4\n            aws_service someservice\n            aws_region my-region-1\n            aws_role_arn arn:aws:iam::123456789012:role/MyRole\n          </auth>\n        ])\n      d.run(default_tag: 'test.http') do\n        test_events.each { |event|\n          d.feed(event)\n        }\n      end\n\n      result = @@result\n      assert_equal 'POST', result.method\n      assert_equal 'application/x-ndjson', result.content_type\n      assert_equal test_events, result.data\n      assert_not_empty result.headers\n      assert_not_nil result.headers['authorization']\n      assert_match(/AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\\/\\d+\\/my-region-1\\/someservice\\/aws4_request/, result.headers['authorization'])\n      assert_match(/SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization'])\n      assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']\n      assert_not_nil result.headers['x-amz-content-sha256']\n      assert_not_empty result.headers['x-amz-content-sha256']\n      assert_not_nil result.headers['x-amz-security-token']\n      assert_not_empty result.headers['x-amz-security-token']\n      assert_not_nil result.headers['x-amz-date']\n      assert_not_empty result.headers['x-amz-date']\n    end\n\n    def test_aws_sigv4_no_role\n      stub(Aws::CredentialProviderChain).new do |provider_chain|\n        stub(provider_chain).resolve {\n          @@fake_aws_credentials\n        }\n        provider_chain\n      end\n      d = create_driver(config + %[\n          <auth>\n            method aws_sigv4\n            aws_service someservice\n            aws_region my-region-1\n          </auth>\n        ])\n      d.run(default_tag: 'test.http') do\n        test_events.each { |event|\n          d.feed(event)\n        }\n      end\n\n      result = @@result\n      assert_equal 'POST', result.method\n      assert_equal 'application/x-ndjson', result.content_type\n      assert_equal test_events, result.data\n      assert_not_empty result.headers\n      assert_not_nil result.headers['authorization']\n      assert_match(/AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\\/\\d+\\/my-region-1\\/someservice\\/aws4_request/, result.headers['authorization'])\n      assert_match(/SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization'])\n      assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']\n      assert_not_nil result.headers['x-amz-content-sha256']\n      assert_not_empty result.headers['x-amz-content-sha256']\n      assert_not_nil result.headers['x-amz-security-token']\n      assert_not_empty result.headers['x-amz-security-token']\n      assert_not_nil result.headers['x-amz-date']\n      assert_not_empty result.headers['x-amz-date']\n    end\n  end\n\n  sub_test_case 'HTTPS' do\n    def server_port\n      19882\n    end\n\n    def server_config\n      config = super\n      # WEBrick supports self-generated self-signed certificate\n      config[:SSLEnable] = true\n      config[:SSLCertName] = [[\"CN\", WEBrick::Utils::getservername]]\n      config[:SSLMaxVersion] = OpenSSL::SSL::TLS1_3_VERSION\n      config\n    end\n\n    def http_client(&block)\n      super(use_ssl: true, verify_mode: OpenSSL::SSL::VERIFY_NONE, &block)\n    end\n\n    def test_write_with_https\n      d = create_driver(%[\n        endpoint https://127.0.0.1:#{server_port}/test\n        tls_verify_mode none\n        tls_version TLSv1_3\n        ssl_timeout 2s\n      ])\n      d.run(default_tag: 'test.http') do\n        test_events.each { |event|\n          d.feed(event)\n        }\n      end\n\n      result = @@result\n      assert_equal 'POST', result.method\n      assert_equal 'application/x-ndjson', result.content_type\n      assert_equal test_events, result.data\n      assert_not_empty result.headers\n    end\n  end\n\n  sub_test_case 'GZIP' do\n    def server_port\n      19882\n    end\n\n    data(:json_array, [false, true])\n    data(:buffer_compress, [\"text\", \"gzip\"])\n    def test_write_with_gzip\n      d = create_driver(%[\n        endpoint http://127.0.0.1:#{server_port}/test\n        compress gzip\n        json_array #{data[:json_array]}\n        <buffer>\n          @type memory\n          compress #{data[:buffer_compress]}\n        </buffer>\n      ])\n      d.run(default_tag: 'test.http') do\n        test_events.each { |event|\n          d.feed(event)\n        }\n      end\n\n      result = @@result\n      assert_equal 'POST', result.method\n      assert_equal(\n        data[:json_array] ? 'application/json' : 'application/x-ndjson',\n        result.content_type\n      )\n      assert_equal 'gzip', result.headers['content-encoding']\n      assert_equal test_events,  result.data\n      assert_not_empty result.headers\n    end\n  end\n\n  sub_test_case 'connection_reuse' do\n    def server_port\n      19883\n    end\n\n    def test_connection_recreation\n      d = create_driver(%[\n        endpoint http://127.0.0.1:#{server_port}/test\n        reuse_connections true\n      ])\n\n      d.run(default_tag: 'test.http', shutdown: false) do\n        d.feed(test_events[0])\n      end\n\n      data = @@result.data\n\n      # Restart server to simulate connection loss\n      @@http_server_thread.kill\n      @@http_server_thread.join\n      @@http_server_thread = Thread.new do\n        run_http_server\n      end\n\n      d.run(default_tag: 'test.http') do\n        d.feed(test_events[1])\n      end\n\n      result = @@result\n      assert_equal 'POST', result.method\n      assert_equal 'application/x-ndjson', result.content_type\n      assert_equal test_events, data.concat(result.data)\n      assert_not_empty result.headers\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_null.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/output'\nrequire 'fluent/plugin/out_null'\n\nclass NullOutputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def create_driver(conf = \"\")\n    Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)\n  end\n\n  sub_test_case 'non-buffered' do\n    test 'configure' do\n      assert_nothing_raised do\n        create_driver\n      end\n    end\n\n    test 'process' do\n      d = create_driver\n      assert_nothing_raised do\n        d.run do\n          d.feed(\"test\", Fluent::EventTime.now, {\"test\" => \"null\"})\n        end\n      end\n    assert_equal([], d.events(tag: \"test\"))\n    end\n  end\n\n  sub_test_case 'buffered' do\n    test 'default chunk limit size is 100' do\n      d = create_driver(config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\")]))\n      assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size\n      assert d.instance.buffer_config.flush_at_shutdown\n      assert_equal ['tag'], d.instance.buffer_config.chunk_keys\n      assert d.instance.chunk_key_tag\n      assert !d.instance.chunk_key_time\n      assert_equal [], d.instance.chunk_keys\n    end\n\n    test 'writes standard formatted chunks' do\n      d = create_driver(config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\")]))\n      t = event_time(\"2016-05-23 00:22:13 -0800\")\n      d.run(default_tag: 'test', flush: true) do\n        d.feed(t, {\"message\" => \"null null null\"})\n        d.feed(t, {\"message\" => \"null null\"})\n        d.feed(t, {\"message\" => \"null\"})\n      end\n\n      assert_equal 3, d.instance.emit_count\n      assert_equal 3, d.instance.emit_records\n    end\n\n    test 'check for chunk passed to #write' do\n      d = create_driver(config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\")]))\n      data = []\n      d.instance.feed_proc = ->(chunk){ data << [chunk.unique_id, chunk.metadata.tag, chunk.read] }\n\n      t = event_time(\"2016-05-23 00:22:13 -0800\")\n      d.run(default_tag: 'test', flush: true) do\n        d.feed(t, {\"message\" => \"null null null\"})\n        d.feed(t, {\"message\" => \"null null\"})\n        d.feed(t, {\"message\" => \"null\"})\n      end\n\n      assert_equal 1, data.size\n      _, tag, binary = data.first\n      events = []\n      Fluent::MessagePackFactory.unpacker.feed_each(binary){|obj| events << obj }\n      assert_equal 'test', tag\n      assert_equal [ [t, {\"message\" => \"null null null\"}], [t, {\"message\" => \"null null\"}], [t, {\"message\" => \"null\"}] ], events\n    end\n\n    test 'check for chunk passed to #try_write' do\n      d = create_driver(config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\")]))\n      data = []\n      d.instance.feed_proc = ->(chunk){ data << [chunk.unique_id, chunk.metadata.tag, chunk.read] }\n      d.instance.delayed = true\n\n      t = event_time(\"2016-05-23 00:22:13 -0800\")\n      d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do\n        d.feed(t, {\"message\" => \"null null null\"})\n        d.feed(t, {\"message\" => \"null null\"})\n        d.feed(t, {\"message\" => \"null\"})\n      end\n\n      assert_equal 1, data.size\n      chunk_id, tag, binary = data.first\n      events = []\n      Fluent::MessagePackFactory.unpacker.feed_each(binary){|obj| events << obj }\n      assert_equal 'test', tag\n      assert_equal [ [t, {\"message\" => \"null null null\"}], [t, {\"message\" => \"null null\"}], [t, {\"message\" => \"null\"}] ], events\n\n      assert_equal [chunk_id], d.instance.buffer.dequeued.keys\n\n      d.instance.commit_write(chunk_id)\n\n      assert_equal [], d.instance.buffer.dequeued.keys\n\n      d.instance_shutdown\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_relabel.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/output'\nrequire 'fluent/plugin/out_relabel'\n\nclass RelabelOutputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def default_config\n    config_element('ROOT', '', {\"@type\"=>\"relabel\", \"@label\"=>\"@RELABELED\"})\n  end\n\n  def create_driver(conf = default_config)\n    Fluent::Test::Driver::Output.new(Fluent::Plugin::RelabelOutput).configure(conf)\n  end\n\n  def test_process\n    d = create_driver\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    d.run(default_tag: 'test') do\n      d.feed(time, {\"a\"=>1})\n      d.feed(time, {\"a\"=>2})\n    end\n    assert_equal [[\"test\", time, {\"a\"=>1}], [\"test\", time, {\"a\"=>2}]], d.events\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_roundrobin.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/multi_output'\nrequire 'fluent/plugin/out_roundrobin'\n\nclass RoundRobinOutputTest < Test::Unit::TestCase\n  class << self\n    def startup\n      $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))\n      require 'fluent/plugin/out_test'\n      require 'fluent/plugin/out_test2'\n    end\n\n    def shutdown\n      $LOAD_PATH.shift\n    end\n  end\n\n  def setup\n    Fluent::Test.setup\n  end\n\n  CONFIG = %[\n    <store>\n      @type test\n      name c0\n    </store>\n    <store>\n      @type test2\n      name c1\n    </store>\n    <store>\n      @type test\n      name c2\n    </store>\n  ]\n  CONFIG_WITH_WEIGHT = %[\n    <store>\n      @type test\n      name c0\n      weight 3\n    </store>\n    <store>\n      @type test2\n      name c1\n      weight 3\n    </store>\n    <store>\n      @type test\n      name c2\n    </store>\n  ]\n\n  def create_driver(conf = CONFIG)\n    Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::RoundRobinOutput).configure(conf)\n  end\n\n  def test_configure\n    d = create_driver\n\n    outputs = d.instance.outputs\n    assert_equal 3, outputs.size\n\n    assert_equal Fluent::Plugin::TestOutput, outputs[0].class\n    assert_equal Fluent::Plugin::Test2Output, outputs[1].class\n    assert_equal Fluent::Plugin::TestOutput, outputs[2].class\n\n    assert !outputs[0].has_router?\n    assert outputs[1].has_router?\n    assert outputs[1].router\n    assert !outputs[2].has_router?\n\n    assert_equal \"c0\", outputs[0].name\n    assert_equal \"c1\", outputs[1].name\n    assert_equal \"c2\", outputs[2].name\n\n    weights = d.instance.weights\n    assert_equal 3, weights.size\n    assert_equal 1, weights[0]\n    assert_equal 1, weights[1]\n    assert_equal 1, weights[2]\n\n    d = create_driver(CONFIG_WITH_WEIGHT)\n\n    outputs = d.instance.outputs\n    assert_equal 3, outputs.size\n\n    assert_equal Fluent::Plugin::TestOutput, outputs[0].class\n    assert_equal Fluent::Plugin::Test2Output, outputs[1].class\n    assert_equal Fluent::Plugin::TestOutput, outputs[2].class\n\n    assert_equal \"c0\", outputs[0].name\n    assert_equal \"c1\", outputs[1].name\n    assert_equal \"c2\", outputs[2].name\n\n    weights = d.instance.weights\n    assert_equal 3, weights.size\n    assert_equal 3, weights[0]\n    assert_equal 3, weights[1]\n    assert_equal 1, weights[2]\n  end\n\n  def test_events_feeded_to_plugins_by_roundrobin\n    d = create_driver\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    d.run(default_tag: 'test') do\n      d.feed(time, {\"a\" => 1})\n      d.feed(time, {\"a\" => 2})\n      d.feed(time, {\"a\" => 3})\n      d.feed(time, {\"a\" => 4})\n    end\n\n    os = d.instance.outputs\n\n    assert_equal [\n        [time, {\"a\"=>1}],\n        [time, {\"a\"=>4}],\n      ], os[0].events\n\n    assert_equal [\n        [time, {\"a\"=>2}],\n      ], os[1].events\n\n    assert_equal [\n        [time, {\"a\"=>3}],\n      ], os[2].events\n  end\n\n  def test_events_feeded_with_specified_weights\n    d = create_driver(CONFIG_WITH_WEIGHT)\n\n    time = event_time(\"2011-01-02 13:14:15 UTC\")\n    d.run(default_tag: 'test') do\n      14.times do |i|\n        d.feed(time, {\"a\" => i})\n      end\n    end\n\n    os = d.instance.outputs\n\n    assert_equal 6, os[0].events.size  # weight=3\n    assert_equal 6, os[1].events.size  # weight=3\n    assert_equal 2, os[2].events.size  # weight=1\n  end\nend\n\n"
  },
  {
    "path": "test/plugin/test_out_secondary_file.rb",
    "content": "require_relative '../helper'\nrequire 'time'\nrequire 'fileutils'\nrequire 'fluent/event'\nrequire 'fluent/unique_id'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin/out_secondary_file'\nrequire 'fluent/plugin/buffer/memory_chunk'\nrequire 'fluent/test/driver/output'\n\nclass FileOutputSecondaryTest < Test::Unit::TestCase\n  include Fluent::UniqueId::Mixin\n\n  def setup\n    Fluent::Test.setup\n    FileUtils.rm_rf(TMP_DIR)\n    FileUtils.mkdir_p(TMP_DIR)\n  end\n\n  TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/../tmp/out_secondary_file#{ENV['TEST_ENV_NUMBER']}\")\n\n  CONFIG = %[\n    directory #{TMP_DIR}\n    basename out_file_test\n    compress gzip\n  ]\n\n  class DummyOutput < Fluent::Plugin::Output\n    def write(chunk); end\n  end\n\n  def create_primary(buffer_config = config_element('buffer'))\n    DummyOutput.new.configure(config_element('ROOT','',{}, [buffer_config]))\n  end\n\n  def create_driver(conf = CONFIG, primary = create_primary)\n    c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)\n    c.instance.acts_as_secondary(primary)\n    c.configure(conf)\n  end\n\n  sub_test_case 'configure' do\n    test 'default configuration' do\n      d = create_driver %[directory #{TMP_DIR}]\n      assert_equal 'dump.bin', d.instance.basename\n      assert_equal TMP_DIR, d.instance.directory\n      assert_equal :text, d.instance.compress\n      assert_equal false, d.instance.append\n    end\n\n    test 'should be configurable' do\n      d = create_driver %[\n         directory #{TMP_DIR}\n         basename out_file_test\n         compress gzip\n         append true\n      ]\n      assert_equal 'out_file_test', d.instance.basename\n      assert_equal TMP_DIR, d.instance.directory\n      assert_equal :gzip, d.instance.compress\n      assert_equal true, d.instance.append\n    end\n\n    test 'should only use in secondary' do\n      c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)\n      assert_raise Fluent::ConfigError.new(\"This plugin can only be used in the <secondary> section\") do\n        c.configure(CONFIG)\n      end\n    end\n\n    test 'basename should not include `/`' do\n      assert_raise Fluent::ConfigError.new(\"basename should not include `/`\") do\n        create_driver %[\n          directory #{TMP_DIR}\n          basename out/file\n        ]\n      end\n    end\n\n    test 'directory should be writable' do\n      assert_nothing_raised do\n        create_driver %[directory #{TMP_DIR}/test_dir/foo/bar/]\n      end\n\n      assert_nothing_raised do\n        FileUtils.mkdir_p(\"#{TMP_DIR}/test_dir\")\n        File.chmod(0777, \"#{TMP_DIR}/test_dir\")\n        create_driver %[directory #{TMP_DIR}/test_dir/foo/bar/]\n      end\n\n      if Process.uid.nonzero?\n        assert_raise Fluent::ConfigError.new(\"out_secondary_file: `#{TMP_DIR}/test_dir/foo/bar/` should be writable\") do\n          FileUtils.mkdir_p(\"#{TMP_DIR}/test_dir\")\n          File.chmod(0555, \"#{TMP_DIR}/test_dir\")\n          create_driver %[directory #{TMP_DIR}/test_dir/foo/bar/]\n        end\n      end\n    end\n\n    test 'should be passed directory' do\n      assert_raise Fluent::ConfigError do\n        i = Fluent::Plugin::SecondaryFileOutput.new\n        i.acts_as_secondary(create_primary)\n        i.configure(config_element())\n      end\n\n      assert_nothing_raised do\n        create_driver %[directory #{TMP_DIR}/test_dir/foo/bar/]\n      end\n    end\n  end\n\n  def check_gzipped_result(path, expect)\n    # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790\n    # Following code from https://www.ruby-forum.com/topic/971591#979520\n    result = \"\"\n    waiting(10) do\n      # we can expect that GzipReader#read can wait unflushed raw data of `io` on disk\n      File.open(path, \"rb\") { |io|\n        loop do\n          gzr = Zlib::GzipReader.new(io)\n          result << gzr.read\n          unused = gzr.unused\n          gzr.finish\n          break if unused.nil?\n          io.pos -= unused.length\n        end\n      }\n    end\n\n    assert_equal expect, result\n  end\n\n  def create_chunk(primary, metadata, es)\n    primary.buffer.generate_chunk(metadata).tap do |c|\n      c.concat(es.to_msgpack_stream, es.size) # to_msgpack_stream is standard_format\n      c.commit\n    end\n  end\n\n  sub_test_case 'write' do\n    setup do\n      @record = { 'key' => 'value' }\n      @time = event_time\n      @es = Fluent::OneEventStream.new(@time, @record)\n      @primary = create_primary\n      metadata = @primary.buffer.new_metadata\n      @chunk = create_chunk(@primary, metadata, @es)\n    end\n\n    test 'should output compressed file when compress option is gzip' do\n      d = create_driver(CONFIG, @primary)\n      path = d.instance.write(@chunk)\n\n      assert_equal \"#{TMP_DIR}/out_file_test.0.gz\", path\n      check_gzipped_result(path, @es.to_msgpack_stream.force_encoding('ASCII-8BIT'))\n    end\n\n    test 'should output plain text when compress option is default(text)' do\n      d = create_driver(%[\n        directory #{TMP_DIR}/\n        basename out_file_test\n      ], @primary)\n\n      msgpack_binary = @es.to_msgpack_stream.force_encoding('ASCII-8BIT')\n\n      path = d.instance.write(@chunk)\n      assert_equal \"#{TMP_DIR}/out_file_test.0\", path\n      waiting(5) do\n        sleep 0.1 until File.stat(path).size == msgpack_binary.size\n      end\n\n      assert_equal msgpack_binary, File.binread(path)\n    end\n\n    test 'path should be incremental when append option is false' do\n      d = create_driver(CONFIG, @primary)\n      packed_value = @es.to_msgpack_stream.force_encoding('ASCII-8BIT')\n\n      5.times do |i|\n        path = d.instance.write(@chunk)\n        assert_equal \"#{TMP_DIR}/out_file_test.#{i}.gz\", path\n        check_gzipped_result(path, packed_value)\n      end\n    end\n\n    test 'path should be unchanged when append option is true' do\n      d = create_driver(CONFIG + %[append true], @primary)\n      packed_value = @es.to_msgpack_stream.force_encoding('ASCII-8BIT')\n\n      [*1..5].each do |i|\n        path = d.instance.write(@chunk)\n        assert_equal \"#{TMP_DIR}/out_file_test.gz\", path\n        check_gzipped_result(path, packed_value * i)\n      end\n    end\n  end\n\n  sub_test_case 'Syntax of placeholders' do\n    data(\n      tag: '${tag}',\n      tag_index: '${tag[0]}',\n      tag_index1: '${tag[10]}',\n      variable: '${key1}',\n      variable2: '${key@value}',\n      variable3: '${key_value}',\n      variable4: '${key.value}',\n      variable5: '${key-value}',\n      variable6: '${KEYVALUE}',\n      variable7: '${tags}',\n      variable8: '${tag${key}', # matched ${key}\n    )\n    test 'matches with a valid placeholder' do |path|\n      assert Fluent::Plugin::SecondaryFileOutput::PLACEHOLDER_REGEX.match(path)\n    end\n\n    data(\n      invalid_tag: 'tag',\n      invalid_tag2: '{tag}',\n      invalid_tag3: '${tag',\n      invalid_tag4: '${tag0]}',\n      invalid_tag5: '${tag[]]}',\n      invalid_variable: '${key[0]}',\n      invalid_variable2: '${key{key2}}',\n    )\n    test \"doesn't match with an invalid placeholder\" do |path|\n      assert !Fluent::Plugin::SecondaryFileOutput::PLACEHOLDER_REGEX.match(path)\n    end\n  end\n\n  sub_test_case 'path' do\n    setup do\n      @record = { 'key' => 'value' }\n      @time = event_time\n      @es = Fluent::OneEventStream.new(@time, @record)\n      primary = create_primary\n      m = primary.buffer.new_metadata\n      @c = create_chunk(primary, m, @es)\n    end\n\n    test 'normal path when compress option is gzip' do\n      d = create_driver\n      path = d.instance.write(@c)\n      assert_equal \"#{TMP_DIR}/out_file_test.0.gz\", path\n    end\n\n    test 'normal path when compress option is default' do\n      d = create_driver %[\n        directory #{TMP_DIR}\n        basename out_file_test\n      ]\n      path = d.instance.write(@c)\n      assert_equal \"#{TMP_DIR}/out_file_test.0\", path\n    end\n\n    test 'normal path when append option is true' do\n      d = create_driver %[\n        directory #{TMP_DIR}\n        append true\n      ]\n      path = d.instance.write(@c)\n      assert_equal \"#{TMP_DIR}/dump.bin\", path\n    end\n\n    test 'path with ${chunk_id}' do\n      d = create_driver %[\n        directory #{TMP_DIR}\n        basename out_file_chunk_id_${chunk_id}\n      ]\n      path = d.instance.write(@c)\n      if File.basename(path) =~ /out_file_chunk_id_([-_.@a-zA-Z0-9].*).0/\n        unique_id = Fluent::UniqueId.hex(Fluent::UniqueId.generate)\n        assert_equal unique_id.size, $1.size, \"chunk_id size is mismatched\"\n      else\n        flunk \"chunk_id is not included in the path\"\n      end\n    end\n\n    data(\n      invalid_tag: [/tag/, '${tag}'],\n      invalid_tag0: [/tag\\[0\\]/, '${tag[0]}'],\n      invalid_variable: [/dummy/, '${dummy}'],\n      invalid_timeformat: [/time/, '%Y%m%d'],\n    )\n    test 'raise an error when basename includes incompatible placeholder' do |(expected_message, invalid_basename)|\n      c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)\n      c.instance.acts_as_secondary(DummyOutput.new)\n\n      assert_raise_message(expected_message) do\n        c.configure %[\n          directory #{TMP_DIR}/\n          basename #{invalid_basename}\n          compress gzip\n        ]\n      end\n    end\n\n    data(\n      invalid_tag: [/tag/, '${tag}'],\n      invalid_tag0: [/tag\\[0\\]/, '${tag[0]}'],\n      invalid_variable: [/dummy/, '${dummy}'],\n      invalid_timeformat: [/time/, '%Y%m%d'],\n    )\n    test 'raise an error when directory includes incompatible placeholder' do |(expected_message, invalid_directory)|\n      c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)\n      c.instance.acts_as_secondary(DummyOutput.new)\n\n      assert_raise_message(expected_message) do\n        c.configure %[\n          directory #{invalid_directory}/\n          compress gzip\n        ]\n      end\n    end\n\n    test 'basename includes tag' do\n      primary = create_primary(config_element('buffer', 'tag'))\n\n      d = create_driver(%[\n        directory #{TMP_DIR}/\n        basename cool_${tag}\n        compress gzip\n      ], primary)\n\n      m = primary.buffer.new_metadata(tag: 'test.dummy')\n      c = create_chunk(primary, m, @es)\n\n      path = d.instance.write(c)\n      assert_equal \"#{TMP_DIR}/cool_test.dummy.0.gz\", path\n    end\n\n    test 'basename includes /tag[\\d+]/' do\n      primary = create_primary(config_element('buffer', 'tag'))\n\n      d = create_driver(%[\n        directory #{TMP_DIR}/\n        basename cool_${tag[0]}_${tag[1]}\n        compress gzip\n      ], primary)\n\n      m = primary.buffer.new_metadata(tag: 'test.dummy')\n      c = create_chunk(primary, m, @es)\n\n      path = d.instance.write(c)\n      assert_equal \"#{TMP_DIR}/cool_test_dummy.0.gz\", path\n    end\n\n    test 'basename includes time format' do\n      primary = create_primary(\n        config_element('buffer', 'time', { 'timekey_zone' => '+0900', 'timekey' => 1 })\n      )\n\n      d = create_driver(%[\n        directory #{TMP_DIR}/\n        basename cool_%Y%m%d%H\n        compress gzip\n      ], primary)\n\n      m = primary.buffer.new_metadata(timekey: event_time(\"2011-01-02 13:14:15 UTC\"))\n      c = create_chunk(primary, m, @es)\n\n      path = d.instance.write(c)\n      assert_equal \"#{TMP_DIR}/cool_2011010222.0.gz\", path\n    end\n\n    test 'basename includes time format with timekey_use_utc option' do\n      primary = create_primary(\n        config_element('buffer', 'time', { 'timekey_zone' => '+0900', 'timekey' => 1, 'timekey_use_utc' => true })\n      )\n\n      d = create_driver(%[\n        directory #{TMP_DIR}/\n        basename cool_%Y%m%d%H\n        compress gzip\n      ], primary)\n\n      m = primary.buffer.new_metadata(timekey: event_time(\"2011-01-02 13:14:15 UTC\"))\n      c = create_chunk(primary, m, @es)\n\n      path = d.instance.write(c)\n      assert_equal \"#{TMP_DIR}/cool_2011010213.0.gz\", path\n    end\n\n    test 'basename includes variable' do\n      primary = create_primary(config_element('buffer', 'test1'))\n\n      d = create_driver(%[\n        directory #{TMP_DIR}/\n        basename cool_${test1}\n        compress gzip\n      ], primary)\n\n      m = primary.buffer.new_metadata(variables: { \"test1\".to_sym => \"dummy\" })\n      c = create_chunk(primary, m, @es)\n\n      path = d.instance.write(c)\n      assert_equal \"#{TMP_DIR}/cool_dummy.0.gz\", path\n    end\n\n    test 'basename includes unnecessary variable' do\n      primary = create_primary(config_element('buffer', 'test1'))\n      c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)\n      c.instance.acts_as_secondary(primary)\n\n      assert_raise_message(/test2/) do\n        c.configure %[\n          directory #{TMP_DIR}/\n          basename ${test1}_${test2}\n          compress gzip\n        ]\n      end\n    end\n\n    test 'basename includes tag, time format, and variables' do\n      primary = create_primary(\n        config_element('buffer', 'time,tag,test1', { 'timekey_zone' => '+0000', 'timekey' => 1 })\n      )\n\n      d = create_driver(%[\n        directory #{TMP_DIR}/\n        basename cool_%Y%m%d%H_${tag}_${test1}\n        compress gzip\n      ], primary)\n\n      m = primary.buffer.new_metadata(\n        timekey: event_time(\"2011-01-02 13:14:15 UTC\"),\n        tag: 'test.tag',\n        variables: { \"test1\".to_sym => \"dummy\" }\n      )\n\n      c = create_chunk(primary, m, @es)\n\n      path = d.instance.write(c)\n      assert_equal \"#{TMP_DIR}/cool_2011010213_test.tag_dummy.0.gz\", path\n    end\n\n    test 'directory includes tag, time format, and variables' do\n      primary = create_primary(\n        config_element('buffer', 'time,tag,test1', { 'timekey_zone' => '+0000', 'timekey' => 1 })\n      )\n\n      d = create_driver(%[\n        directory #{TMP_DIR}/%Y%m%d%H/${tag}/${test1}\n        compress gzip\n      ], primary)\n\n      m = primary.buffer.new_metadata(\n        timekey: event_time(\"2011-01-02 13:14:15 UTC\"),\n        tag: 'test.tag',\n        variables: { \"test1\".to_sym => \"dummy\" }\n      )\n      c = create_chunk(primary, m, @es)\n\n      path = d.instance.write(c)\n      assert_equal \"#{TMP_DIR}/2011010213/test.tag/dummy/dump.bin.0.gz\", path\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_stdout.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/output'\nrequire 'fluent/plugin/out_stdout'\n\nclass StdoutOutputTest < Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  CONFIG = %[\n  ]\n  TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'\n\n  def create_driver(conf = CONFIG)\n    Fluent::Test::Driver::Output.new(Fluent::Plugin::StdoutOutput).configure(conf)\n  end\n\n  sub_test_case 'non-buffered' do\n    test 'configure' do\n      d = create_driver\n      assert_equal 1, d.instance.formatter_configs.size # init: true\n      assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter\n      assert_equal 'json', d.instance.formatter.output_type\n    end\n\n    test 'configure output_type' do\n      d = create_driver(CONFIG + \"\\noutput_type json\")\n      assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter\n      assert_equal 'json', d.instance.formatter.output_type\n\n      d = create_driver(CONFIG + \"\\noutput_type hash\")\n      assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter\n      assert_equal 'hash', d.instance.formatter.output_type\n\n      assert_raise(Fluent::NotFoundPluginError) do\n        d = create_driver(CONFIG + \"\\noutput_type foo\")\n      end\n    end\n\n    test 'configure with time_format' do\n      d = create_driver(CONFIG + <<-CONF)\n        <format>\n          @type stdout\n          time_format %Y-%m-%dT%H:%M:%S.%L%z\n        </format>\n      CONF\n\n      time = event_time\n      out = capture_log do\n        d.run(default_tag: 'test') do\n          d.feed(time, {'test' => 'test'})\n        end\n      end\n\n      t = Time.at(time).localtime.strftime(\"%Y-%m-%dT%H:%M:%S.%L%z\")\n      assert_equal \"#{t} test: {\\\"test\\\":\\\"test\\\"}\\n\", out\n    end\n\n    test 'emit with default configuration' do\n      d = create_driver\n      time = event_time()\n      out = capture_log do\n        d.run(default_tag: 'test') do\n          d.feed(time, {'test' => 'test1'})\n        end\n      end\n      assert_equal \"#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\\\"test\\\":\\\"test1\\\"}\\n\", out\n    end\n\n    data('oj' => 'oj', 'yajl' => 'yajl')\n    test 'emit in json format' do |data|\n      d = create_driver(CONFIG + \"\\noutput_type json\\njson_parser #{data}\")\n      time = event_time()\n      out = capture_log do\n        d.run(default_tag: 'test') do\n          d.feed(time, {'test' => 'test1'})\n        end\n      end\n      assert_equal \"#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\\\"test\\\":\\\"test1\\\"}\\n\", out\n\n      if data == 'yajl'\n        # NOTE: Float::NAN is not jsonable\n        assert_raise(JSON::GeneratorError) { d.feed('test', time, {'test' => Float::NAN}) }\n      else\n        out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }\n        assert_equal \"#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\\\"test\\\":NaN}\\n\", out\n      end\n    end\n\n    test 'emit in hash format' do\n      d = create_driver(CONFIG + \"\\noutput_type hash\")\n      time = event_time()\n      out = capture_log do\n        d.run(default_tag: 'test') do\n          d.feed(time, {'test' => 'test2'})\n        end\n      end\n      assert_equal \"#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\\\"test\\\"=>\\\"test2\\\"}\\n\", out.gsub(' => ', '=>')\n\n      # NOTE: Float::NAN is not jsonable, but hash string can output it.\n      out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }\n      assert_equal \"#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\\\"test\\\"=>NaN}\\n\", out.gsub(' => ', '=>')\n    end\n  end\n\n  sub_test_case 'buffered' do\n    test 'configure' do\n      d = create_driver(config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\")]))\n      assert_equal 1, d.instance.formatter_configs.size\n      assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter\n      assert_equal 'json', d.instance.formatter.output_type\n      assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size\n      assert d.instance.buffer_config.flush_at_shutdown\n      assert_equal ['tag'], d.instance.buffer_config.chunk_keys\n      assert d.instance.chunk_key_tag\n      assert !d.instance.chunk_key_time\n      assert_equal [], d.instance.chunk_keys\n    end\n\n    test 'configure with output_type' do\n      d = create_driver(config_element(\"ROOT\", \"\", {\"output_type\" => \"json\"}, [config_element(\"buffer\")]))\n      assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter\n      assert_equal 'json', d.instance.formatter.output_type\n\n      d = create_driver(config_element(\"ROOT\", \"\", {\"output_type\" => \"hash\"}, [config_element(\"buffer\")]))\n      assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter\n      assert_equal 'hash', d.instance.formatter.output_type\n\n      assert_raise(Fluent::NotFoundPluginError) do\n        create_driver(config_element(\"ROOT\", \"\", {\"output_type\" => \"foo\"}, [config_element(\"buffer\")]))\n      end\n    end\n\n    sub_test_case \"emit with default config\" do\n      test '#write(synchronous)' do\n        d = create_driver(config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\")]))\n        time = event_time()\n\n        out = capture_log do\n          d.run(default_tag: 'test', flush: true) do\n            d.feed(time, {'test' => 'test'})\n          end\n        end\n        assert_equal \"#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\\\"test\\\":\\\"test\\\"}\\n\", out\n      end\n    end\n\n    sub_test_case \"emit json\" do\n      data('oj' => 'oj', 'yajl' => 'yajl')\n      test '#write(synchronous)' do |data|\n        d = create_driver(config_element(\"ROOT\", \"\", {\"output_type\" => \"json\", \"json_parser\" => data}, [config_element(\"buffer\")]))\n        time = event_time()\n\n        out = capture_log do\n          d.run(default_tag: 'test', flush: true) do\n            d.feed(time, {'test' => 'test'})\n          end\n        end\n        assert_equal \"#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\\\"test\\\":\\\"test\\\"}\\n\", out\n      end\n    end\n\n    sub_test_case 'emit hash' do\n      test '#write(synchronous)' do\n        d = create_driver(config_element(\"ROOT\", \"\", {\"output_type\" => \"hash\"}, [config_element(\"buffer\")]))\n        time = event_time()\n\n        out = capture_log do\n          d.run(default_tag: 'test', flush: true) do\n            d.feed(time, {'test' => 'test'})\n          end\n        end\n\n        assert_equal \"#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\\\"test\\\"=>\\\"test\\\"}\\n\", out.gsub(' => ', '=>')\n      end\n    end\n  end\n\n  data(\n    'utc and !localtime' => \"utc true\\nlocaltime false\",\n    '!utc and localtime' => \"utc false\\nlocaltime true\")\n  test 'success when configure with localtime and utc' do |c|\n    assert_nothing_raised do\n      create_driver(CONFIG + c)\n    end\n  end\n\n  data('utc and localtime' => \"utc true\\nlocaltime true\",\n       '!utc and !localtime' => \"utc false\\nlocaltime false\")\n  test 'raise an error when configure with localtime and utc' do |c|\n    assert_raise(Fluent::ConfigError.new('both of utc and localtime are specified, use only one of them')) do\n      create_driver(CONFIG + c)\n    end\n  end\n\n  test 'use_logger false' do\n    d = create_driver(<<~EOC)\n      use_logger false\n    EOC\n    time = event_time\n\n    out = capture_stdout do\n      d.run(default_tag: 'test', flush: true) do\n        d.feed(time, {'test' => 'test'})\n      end\n    end\n\n    assert_equal \"#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\\\"test\\\":\\\"test\\\"}\\n\", out\n  end\n\n  def capture_log\n    tmp = $log\n    $log = StringIO.new\n    yield\n    return $log.string\n  ensure\n    $log = tmp\n  end\n\n  def capture_stdout\n    tmp = $stdout\n    $stdout = StringIO.new\n    yield\n    return $stdout.string\n  ensure\n    $stdout = tmp\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_out_stream.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test'\nrequire 'fluent/plugin/out_stream'\n\nmodule StreamOutputTest\n  def setup\n    Fluent::Test.setup\n  end\n\n  def test_write\n    d = create_driver\n\n    time = Time.parse(\"2011-01-02 13:14:15 UTC\").to_i\n    d.emit({\"a\"=>1}, time)\n    d.emit({\"a\"=>2}, time)\n\n    expect = [\"test\",\n        [time,{\"a\"=>1}].to_msgpack +\n        [time,{\"a\"=>2}].to_msgpack\n      ].to_msgpack\n\n    result = d.run\n    assert_equal(expect, result)\n  end\n\n  def test_write_event_time\n    d = create_driver\n\n    time = Fluent::EventTime.parse(\"2011-01-02 13:14:15 UTC\")\n    d.emit({\"a\"=>1}, time)\n    d.emit({\"a\"=>2}, time)\n\n    expect = [\"test\",\n        Fluent::MessagePackFactory.msgpack_packer.write([time,{\"a\"=>1}]).to_s +\n        Fluent::MessagePackFactory.msgpack_packer.write([time,{\"a\"=>2}]).to_s\n      ]\n    expect = Fluent::MessagePackFactory.msgpack_packer.write(expect).to_s\n\n    result = d.run\n    assert_equal(expect, result)\n  end\n\n  def create_driver(klass, conf)\n    Fluent::Test::BufferedOutputTestDriver.new(klass) do\n      def write(chunk)\n        chunk.read\n      end\n    end.configure(conf)\n  end\nend\n\nclass TcpOutputTest < Test::Unit::TestCase\n  include StreamOutputTest\n\n  def setup\n    super\n    @port = unused_port(protocol: :tcp)\n  end\n\n  def teardown\n    @port = nil\n  end\n\n  def config\n    %[\n      port #{@port}\n      host 127.0.0.1\n      send_timeout 51\n    ]\n  end\n\n  def create_driver(conf=config)\n    super(Fluent::TcpOutput, conf)\n  end\n\n  def test_configure\n    d = create_driver\n    assert_equal @port, d.instance.port\n    assert_equal '127.0.0.1', d.instance.host\n    assert_equal 51, d.instance.send_timeout\n  end\nend\n\nclass UnixOutputTest < Test::Unit::TestCase\n  include StreamOutputTest\n\n  TMP_DIR = File.dirname(__FILE__) + \"/../tmp/out_unix#{ENV['TEST_ENV_NUMBER']}\"\n  CONFIG = %[\n    path #{TMP_DIR}/unix\n    send_timeout 52\n  ]\n\n  def create_driver(conf=CONFIG)\n    super(Fluent::UnixOutput, conf)\n  end\n\n  def test_configure\n    d = create_driver\n    assert_equal \"#{TMP_DIR}/unix\", d.instance.path\n    assert_equal 52, d.instance.send_timeout\n  end\nend\n\n"
  },
  {
    "path": "test/plugin/test_output.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/event'\n\nrequire 'json'\nrequire 'time'\nrequire 'timeout'\n\nmodule FluentPluginOutputTest\n  class DummyBareOutput < Fluent::Plugin::Output\n    def register(name, &block)\n      instance_variable_set(\"@#{name}\", block)\n    end\n  end\n  class DummySyncOutput < DummyBareOutput\n    def initialize\n      super\n      @process = nil\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n  end\n  class DummyAsyncOutput < DummyBareOutput\n    def initialize\n      super\n      @format = nil\n      @write = nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n  end\n  class DummyAsyncStandardOutput < DummyBareOutput\n    def initialize\n      super\n      @write = nil\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n  end\n  class DummyDelayedOutput < DummyBareOutput\n    def initialize\n      super\n      @format = nil\n      @try_write = nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n  class DummyDelayedStandardOutput < DummyBareOutput\n    def initialize\n      super\n      @try_write = nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n  class DummyFullFeatureOutput < DummyBareOutput\n    def initialize\n      super\n      @prefer_buffered_processing = nil\n      @prefer_delayed_commit = nil\n      @process = nil\n      @format = nil\n      @write = nil\n      @try_write = nil\n    end\n    def prefer_buffered_processing\n      @prefer_buffered_processing ? @prefer_buffered_processing.call : false\n    end\n    def prefer_delayed_commit\n      @prefer_delayed_commit ? @prefer_delayed_commit.call : false\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\nend\n\nclass OutputTest < Test::Unit::TestCase\n  class << self\n    def startup\n      $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '../scripts'))\n      require 'fluent/plugin/out_test'\n    end\n\n    def shutdown\n      $LOAD_PATH.shift\n    end\n  end\n\n  def create_output(type=:full)\n    case type\n    when :bare     then FluentPluginOutputTest::DummyBareOutput.new\n    when :sync     then FluentPluginOutputTest::DummySyncOutput.new\n    when :buffered then FluentPluginOutputTest::DummyAsyncOutput.new\n    when :standard then FluentPluginOutputTest::DummyAsyncStandardOutput.new\n    when :delayed  then FluentPluginOutputTest::DummyDelayedOutput.new\n    when :sdelayed then FluentPluginOutputTest::DummyDelayedStandardOutput.new\n    when :full     then FluentPluginOutputTest::DummyFullFeatureOutput.new\n    else\n      raise ArgumentError, \"unknown type: #{type}\"\n    end\n  end\n  def create_metadata(timekey: nil, tag: nil, variables: nil)\n    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n  end\n  def create_chunk(timekey: nil, tag: nil, variables: nil)\n    m = Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n    Fluent::Plugin::Buffer::MemoryChunk.new(m)\n  end\n  def waiting(seconds)\n    begin\n      Timeout.timeout(seconds) do\n        yield\n      end\n    rescue Timeout::Error\n      STDERR.print(*@i.log.out.logs)\n      raise\n    end\n  end\n\n  sub_test_case 'basic output feature' do\n    setup do\n      @i = create_output(:full)\n    end\n\n    test 'are not available with multi workers configuration in default' do\n      assert_false @i.multi_workers_ready?\n    end\n\n    test '#implement? can return features for plugin instances' do\n      i1 = FluentPluginOutputTest::DummyBareOutput.new\n      assert !i1.implement?(:synchronous)\n      assert !i1.implement?(:buffered)\n      assert !i1.implement?(:delayed_commit)\n      assert !i1.implement?(:custom_format)\n\n      i2 = FluentPluginOutputTest::DummySyncOutput.new\n      assert i2.implement?(:synchronous)\n      assert !i2.implement?(:buffered)\n      assert !i2.implement?(:delayed_commit)\n      assert !i2.implement?(:custom_format)\n\n      i3 = FluentPluginOutputTest::DummyAsyncOutput.new\n      assert !i3.implement?(:synchronous)\n      assert i3.implement?(:buffered)\n      assert !i3.implement?(:delayed_commit)\n      assert i3.implement?(:custom_format)\n\n      i4 = FluentPluginOutputTest::DummyAsyncStandardOutput.new\n      assert !i4.implement?(:synchronous)\n      assert i4.implement?(:buffered)\n      assert !i4.implement?(:delayed_commit)\n      assert !i4.implement?(:custom_format)\n\n      i5 = FluentPluginOutputTest::DummyDelayedOutput.new\n      assert !i5.implement?(:synchronous)\n      assert !i5.implement?(:buffered)\n      assert i5.implement?(:delayed_commit)\n      assert i5.implement?(:custom_format)\n\n      i6 = FluentPluginOutputTest::DummyDelayedStandardOutput.new\n      assert !i6.implement?(:synchronous)\n      assert !i6.implement?(:buffered)\n      assert i6.implement?(:delayed_commit)\n      assert !i6.implement?(:custom_format)\n\n      i6 = FluentPluginOutputTest::DummyFullFeatureOutput.new\n      assert i6.implement?(:synchronous)\n      assert i6.implement?(:buffered)\n      assert i6.implement?(:delayed_commit)\n      assert i6.implement?(:custom_format)\n    end\n\n    test 'plugin lifecycle for configure/start/stop/before_shutdown/shutdown/after_shutdown/close/terminate' do\n      assert !@i.configured?\n      @i.configure(config_element())\n      assert @i.configured?\n      assert !@i.started?\n      @i.start\n      assert @i.started?\n      assert !@i.after_started?\n      @i.after_start\n      assert @i.after_started?\n      assert !@i.stopped?\n      @i.stop\n      assert @i.stopped?\n      assert !@i.before_shutdown?\n      @i.before_shutdown\n      assert @i.before_shutdown?\n      assert !@i.shutdown?\n      @i.shutdown\n      assert @i.shutdown?\n      assert !@i.after_shutdown?\n      @i.after_shutdown\n      assert @i.after_shutdown?\n      assert !@i.closed?\n      @i.close\n      assert @i.closed?\n      assert !@i.terminated?\n      @i.terminate\n      assert @i.terminated?\n    end\n\n    test 'can use metrics plugins and fallback methods' do\n      @i.configure(config_element())\n\n      %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics write_count_metrics\n         write_secondary_count_metrics rollback_count_metrics flush_time_count_metrics slow_flush_count_metrics\n         drop_oldest_chunk_count_metrics].each do |metric_name|\n        assert_true @i.instance_variable_get(:\"@#{metric_name}\").is_a?(Fluent::Plugin::Metrics)\n      end\n\n      assert_equal 0, @i.num_errors\n      assert_equal 0, @i.emit_count\n      assert_equal 0, @i.emit_records\n      assert_equal 0, @i.emit_size\n      assert_equal 0, @i.emit_records\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.write_secondary_count\n      assert_equal 0, @i.rollback_count\n      assert_equal 0, @i.flush_time_count\n      assert_equal 0, @i.slow_flush_count\n      assert_equal 0, @i.drop_oldest_chunk_count\n    end\n\n    data(:new_api => :chunk,\n         :old_api => :metadata)\n    test '#extract_placeholders does nothing if chunk key is not specified' do |api|\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n      assert !@i.chunk_key_time\n      assert !@i.chunk_key_tag\n      assert_equal [], @i.chunk_keys\n      tmpl = \"/mypath/%Y/%m/%d/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {key1: \"value1\", key2: \"value2\"}\n      c = if api == :chunk\n            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)\n          else\n            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n          end\n      assert_equal tmpl, @i.extract_placeholders(tmpl, c)\n    end\n\n    data(:new_api => :chunk,\n         :old_api => :metadata)\n    test '#extract_placeholders can extract time if time key and range are configured' do |api|\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'timekey_zone' => \"+0900\"})]))\n      assert @i.chunk_key_time\n      assert !@i.chunk_key_tag\n      assert_equal [], @i.chunk_keys\n      tmpl = \"/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {key1: \"value1\", key2: \"value2\"}\n      c = if api == :chunk\n            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)\n          else\n            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n          end\n      assert_equal \"/mypath/2016/04/11/20-30/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail\", @i.extract_placeholders(tmpl, c)\n    end\n\n    data(:new_api => :chunk,\n         :old_api => :metadata)\n    test '#extract_placeholders can extract tag and parts of tag if tag is configured' do |api|\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag', {})]))\n      assert !@i.chunk_key_time\n      assert @i.chunk_key_tag\n      assert_equal [], @i.chunk_keys\n      tmpl = \"/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {key1: \"value1\", key2: \"value2\"}\n      c = if api == :chunk\n            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)\n          else\n            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n          end\n      assert_equal \"/mypath/%Y/%m/%d/%H-%M/fluentd.test.output/test/output/${key1}/${key2}/tail\", @i.extract_placeholders(tmpl, c)\n    end\n\n    data(:new_api => :chunk,\n         :old_api => :metadata)\n    test '#extract_placeholders can extract variables if variables are configured' do |api|\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1,key2', {})]))\n      assert !@i.chunk_key_time\n      assert !@i.chunk_key_tag\n      assert_equal ['key1','key2'], @i.chunk_keys\n      tmpl = \"/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {key1: \"value1\", key2: \"value2\"}\n      c = if api == :chunk\n            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)\n          else\n            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n          end\n      assert_equal \"/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail\", @i.extract_placeholders(tmpl, c)\n    end\n\n    data(:new_api => :chunk,\n         :old_api => :metadata)\n    test '#extract_placeholders can extract nested variables if variables are configured with dot notation' do |api|\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key,$.nest.key', {})]))\n      assert !@i.chunk_key_time\n      assert !@i.chunk_key_tag\n      assert_equal ['key','$.nest.key'], @i.chunk_keys\n      tmpl = \"/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key}/${$.nest.key}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {:key => \"value1\", :\"$.nest.key\" => \"value2\"}\n      c = if api == :chunk\n            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)\n          else\n            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n          end\n      assert_equal \"/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail\", @i.extract_placeholders(tmpl, c)\n    end\n\n    data(:new_api => :chunk,\n         :old_api => :metadata)\n    test '#extract_placeholders can extract all chunk keys if configured' do |api|\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => \"+0900\"})]))\n      assert @i.chunk_key_time\n      assert @i.chunk_key_tag\n      assert_equal ['key1','key2'], @i.chunk_keys\n      tmpl = \"/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {key1: \"value1\", key2: \"value2\"}\n      c = if api == :chunk\n            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)\n          else\n            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n          end\n      assert_equal \"/mypath/2016/04/11/20-30/fluentd.test.output/test/output/value1/value2/tail\", @i.extract_placeholders(tmpl, c)\n    end\n\n    data(:new_api => :chunk,\n         :old_api => :metadata)\n    test '#extract_placeholders can extract negative index with tag' do |api|\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => \"+0900\"})]))\n      assert @i.chunk_key_time\n      assert @i.chunk_key_tag\n      assert_equal ['key1','key2'], @i.chunk_keys\n      tmpl = \"/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[-1]}/${tag[-2]}/${key1}/${key2}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {key1: \"value1\", key2: \"value2\"}\n      c = if api == :chunk\n            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)\n          else\n            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n          end\n      assert_equal \"/mypath/2016/04/11/20-30/fluentd.test.output/output/test/value1/value2/tail\", @i.extract_placeholders(tmpl, c)\n    end\n\n    data(:new_api => :chunk,\n         :old_api => :metadata)\n    test '#extract_placeholders removes out-of-range tag part and unknown variable placeholders' do |api|\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => \"+0900\"})]))\n      assert @i.chunk_key_time\n      assert @i.chunk_key_tag\n      assert_equal ['key1','key2'], @i.chunk_keys\n      tmpl = \"/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[-4]}/${key3}/${key4}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {key1: \"value1\", key2: \"value2\"}\n      c = if api == :chunk\n            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)\n          else\n            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n          end\n      assert_equal \"/mypath/2016/04/11/20-30/fluentd.test.output/////tail\", @i.extract_placeholders(tmpl, c)\n    end\n\n    test '#extract_placeholders logs warn message if metadata is passed for ${chunk_id} placeholder' do\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n      tmpl = \"/mypath/${chunk_id}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {key1: \"value1\", key2: \"value2\"}\n      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n      @i.extract_placeholders(tmpl, m)\n      logs = @i.log.out.logs\n      assert { logs.any? { |log| log.include?(\"${chunk_id} is not allowed in this plugin\") } }\n    end\n\n    test '#extract_placeholders does not log for ${chunk_id} placeholder' do\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n      tmpl = \"/mypath/${chunk_id}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {key1: \"value1\", key2: \"value2\"}\n      c = create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)\n      @i.log.out.logs.clear\n      @i.extract_placeholders(tmpl, c)\n      logs = @i.log.out.logs\n      assert { logs.none? { |log| log.include?(\"${chunk_id}\") } }\n    end\n\n    test '#extract_placeholders does not log for ${chunk_id} placeholder (with @chunk_keys)' do\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1')]))\n      tmpl = \"/mypath/${chunk_id}/${key1}/tail\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = {key1: \"value1\", key2: \"value2\"}\n      c = create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)\n      @i.log.out.logs.clear\n      @i.extract_placeholders(tmpl, c)\n      logs = @i.log.out.logs\n      assert { logs.none? { |log| log.include?(\"${chunk_id}\") } }\n    end\n\n    test '#extract_placeholders logs warn message with not replaced key' do\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n      tmpl = \"/mypath/${key1}/test\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = { key1: \"value1\" }\n      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n      @i.extract_placeholders(tmpl, m)\n      logs = @i.log.out.logs\n\n      assert { logs.any? { |log| log.include?(\"chunk key placeholder 'key1' not replaced. template:#{tmpl}\") } }\n    end\n\n    test '#extract_placeholders logs warn message with not replaced key if variables exist and chunk_key is not empty' do\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1')]))\n      tmpl = \"/mypath/${key1}/${key2}/test\"\n      t = event_time('2016-04-11 20:30:00 +0900')\n      v = { key1: \"value1\" }\n      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)\n      @i.extract_placeholders(tmpl, m)\n      logs = @i.log.out.logs\n\n      assert { logs.any? { |log| log.include?(\"chunk key placeholder 'key2' not replaced. template:#{tmpl}\") } }\n    end\n\n    sub_test_case '#placeholder_validators' do\n      test 'returns validators for time, tag and keys when a template has placeholders even if plugin is not configured with these keys' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n        validators = @i.placeholder_validators(:path, \"/my/path/${tag}/${username}/file.%Y%m%d_%H%M.log\")\n        assert_equal 3, validators.size\n        assert_equal 1, validators.count(&:time?)\n        assert_equal 1, validators.count(&:tag?)\n        assert_equal 1, validators.count(&:keys?)\n      end\n\n      test 'returns validators for time, tag and keys when a plugin is configured with these keys even if a template does not have placeholders' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,username', {'timekey' => 60})]))\n        validators = @i.placeholder_validators(:path, \"/my/path/file.log\")\n        assert_equal 3, validators.size\n        assert_equal 1, validators.count(&:time?)\n        assert_equal 1, validators.count(&:tag?)\n        assert_equal 1, validators.count(&:keys?)\n      end\n\n      test 'returns a validator for time if a template has timestamp placeholders' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n        validators = @i.placeholder_validators(:path, \"/my/path/file.%Y-%m-%d.log\")\n        assert_equal 1, validators.size\n        assert_equal 1, validators.count(&:time?)\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/file.%Y-%m-%d.log' has timestamp placeholders, but chunk key 'time' is not configured\") do\n          validators.first.validate!\n        end\n      end\n\n      test 'returns a validator for time if a plugin is configured with time key' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => '30'})]))\n        validators = @i.placeholder_validators(:path, \"/my/path/to/file.log\")\n        assert_equal 1, validators.size\n        assert_equal 1, validators.count(&:time?)\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/to/file.log' doesn't have timestamp placeholders for timekey 30\") do\n          validators.first.validate!\n        end\n      end\n\n      test 'returns a validator for tag if a template has tag placeholders' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n        validators = @i.placeholder_validators(:path, \"/my/path/${tag}/file.log\")\n        assert_equal 1, validators.size\n        assert_equal 1, validators.count(&:tag?)\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/${tag}/file.log' has tag placeholders, but chunk key 'tag' is not configured\") do\n          validators.first.validate!\n        end\n      end\n\n      test 'returns a validator for tag if a plugin is configured with tag key' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))\n        validators = @i.placeholder_validators(:path, \"/my/path/file.log\")\n        assert_equal 1, validators.size\n        assert_equal 1, validators.count(&:tag?)\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/file.log' doesn't have tag placeholder\") do\n          validators.first.validate!\n        end\n      end\n\n      test 'returns a validator for variable keys if a template has variable placeholders' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n        validators = @i.placeholder_validators(:path, \"/my/path/${username}/file.${group}.log\")\n        assert_equal 1, validators.size\n        assert_equal 1, validators.count(&:keys?)\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/${username}/file.${group}.log' has placeholders, but chunk keys doesn't have keys group,username\") do\n          validators.first.validate!\n        end\n      end\n\n      test 'returns a validator for variable keys if a plugin is configured with variable keys' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,group')]))\n        validators = @i.placeholder_validators(:path, \"/my/path/file.log\")\n        assert_equal 1, validators.size\n        assert_equal 1, validators.count(&:keys?)\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys group,username\") do\n          validators.first.validate!\n        end\n      end\n    end\n\n    sub_test_case '#placeholder_validate!' do\n      test 'raises configuration error for a template when timestamp placeholders exist but time key is missing' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /path/without/timestamp/file.%Y%m%d-%H%M.log' has timestamp placeholders, but chunk key 'time' is not configured\") do\n          @i.placeholder_validate!(:path, \"/path/without/timestamp/file.%Y%m%d-%H%M.log\")\n        end\n      end\n\n      test 'raises configuration error for a template without timestamp placeholders when timekey is configured' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {\"timekey\" => 180})]))\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/file.log' doesn't have timestamp placeholders for timekey 180\") do\n          @i.placeholder_validate!(:path, \"/my/path/file.log\")\n        end\n        assert_nothing_raised do\n          @i.placeholder_validate!(:path, \"/my/path/%Y%m%d/file.%H%M.log\")\n        end\n      end\n\n      test 'raises configuration error for a template with timestamp placeholders when plugin is configured more fine timekey' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {\"timekey\" => 180})]))\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/file.%Y%m%d_%H.log' doesn't have timestamp placeholder for hour('%H') for timekey 180\") do\n          @i.placeholder_validate!(:path, \"/my/path/file.%Y%m%d_%H.log\")\n        end\n        assert_nothing_raised do\n          @i.placeholder_validate!(:path, \"/my/path/file.%Y%m%d_%H%M.log\")\n        end\n      end\n\n      test 'raises configuration error for a template when tag placeholders exist but tag key is missing' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/${tag}/file.${tag[2]}.log' has tag placeholders, but chunk key 'tag' is not configured\") do\n          @i.placeholder_validate!(:path, \"/my/path/${tag}/file.${tag[2]}.log\")\n        end\n      end\n\n      test 'raises configuration error for a template without tag placeholders when tagkey is configured' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/file.log' doesn't have tag placeholder\") do\n          @i.placeholder_validate!(:path, \"/my/path/file.log\")\n        end\n        assert_nothing_raised do\n          @i.placeholder_validate!(:path, \"/my/path/${tag}/file.${tag[2]}.log\")\n        end\n        assert_nothing_raised do\n          @i.placeholder_validate!(:path, \"/my/path/${tag}/file.${tag[-1]}.log\")\n        end\n      end\n\n      test 'raises configuration error for a template when variable key placeholders exist but chunk keys are missing' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/${service}/file.${username}.log' has placeholders, but chunk keys doesn't have keys service,username\") do\n          @i.placeholder_validate!(:path, \"/my/path/${service}/file.${username}.log\")\n        end\n      end\n\n      test 'raises configuration error for a template without variable key placeholders when chunk keys are configured' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys service,username\") do\n          @i.placeholder_validate!(:path, \"/my/path/file.log\")\n        end\n        assert_nothing_raised do\n          @i.placeholder_validate!(:path, \"/my/path/${service}/file.${username}.log\")\n        end\n      end\n\n      test 'raise configuration error for a template and configuration with keys mismatch' do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/file.${username}.log' doesn't have enough placeholders for keys service\") do\n          @i.placeholder_validate!(:path, \"/my/path/file.${username}.log\")\n        end\n        assert_raise Fluent::ConfigError.new(\"Parameter 'path: /my/path/${service}/file.log' doesn't have enough placeholders for keys username\") do\n          @i.placeholder_validate!(:path, \"/my/path/${service}/file.log\")\n        end\n        assert_nothing_raised do\n          @i.placeholder_validate!(:path, \"/my/path/${service}/file.${username}.log\")\n        end\n      end\n    end\n\n    test '#get_placeholders_time returns seconds,title and example placeholder for a template' do\n      s, t, e = @i.get_placeholders_time(\"/path/to/dir/yay\")\n      assert_nil s\n      assert_nil t\n      assert_nil e\n\n      s, t, e = @i.get_placeholders_time(\"/path/to/%Y%m%d/yay\")\n      assert_equal 86400, s\n      assert_equal :day, t\n      assert_equal '%d', e\n      s, t, e = @i.get_placeholders_time(\"my birthday! at %F\")\n      assert_equal 86400, s\n      assert_equal :day, t\n      assert_equal '%d', e\n\n      s, t, e = @i.get_placeholders_time(\"myfile.%Y-%m-%d_%H.log\")\n      assert_equal 3600, s\n      assert_equal :hour, t\n      assert_equal '%H', e\n\n      s, t, e = @i.get_placeholders_time(\"part-%Y%m%d-%H%M.ts\")\n      assert_equal 60, s\n      assert_equal :minute, t\n      assert_equal '%M', e\n\n      s, t, e = @i.get_placeholders_time(\"my first data at %F %T %z\")\n      assert_equal 1, s\n      assert_equal :second, t\n      assert_equal '%S', e\n    end\n\n    test '#get_placeholders_tag returns a list of tag part position for a template' do\n      assert_equal [], @i.get_placeholders_tag(\"db.table\")\n      assert_equal [], @i.get_placeholders_tag(\"db.table_${non_tag}\")\n      assert_equal [-1], @i.get_placeholders_tag(\"table_${tag}\")\n      assert_equal [0, 1], @i.get_placeholders_tag(\"db_${tag[0]}.table_${tag[1]}\")\n      assert_equal [-1, 0], @i.get_placeholders_tag(\"/treedir/${tag[0]}/${tag}\")\n    end\n\n    test '#get_placeholders_keys returns a list of keys for a template' do\n      assert_equal [], @i.get_placeholders_keys(\"/path/to/my/data/file.log\")\n      assert_equal [], @i.get_placeholders_keys(\"/path/to/my/${tag}/file.log\")\n      assert_equal ['key1', 'key2'], @i.get_placeholders_keys(\"/path/to/${key2}/${tag}/file.${key1}.log\")\n      assert_equal ['.hidden', '0001', '@timestamp', 'a_key', 'my-domain'], @i.get_placeholders_keys(\"http://${my-domain}/${.hidden}/${0001}/${a_key}?timestamp=${@timestamp}\")\n    end\n\n    data('include space' => 'ke y',\n         'bracket notation' => \"$['key']\",\n         'invalid notation' => \"$.ke y\")\n    test 'configure checks invalid chunk keys' do |chunk_keys|\n      i = create_output(:buffered)\n      assert_raise Fluent::ConfigError do\n        i.configure(config_element('ROOT' , '', {}, [config_element('buffer', chunk_keys)]))\n      end\n    end\n\n    test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do\n      tag = 'test.output'\n      time = event_time('2016-04-12 15:31:23 -0700')\n      timekey = event_time('2016-04-12 15:00:00 -0700')\n      record = {\"key1\" => \"value1\", \"num1\" => 1, \"message\" => \"my message\", \"nest\" => {\"key\" => \"nested value\"}}\n\n      i1 = create_output(:buffered)\n      i1.configure(config_element('ROOT','',{},[config_element('buffer', '')]))\n      assert_equal create_metadata(), i1.metadata(tag, time, record)\n\n      i2 = create_output(:buffered)\n      i2.configure(config_element('ROOT','',{},[config_element('buffer', 'tag')]))\n      assert_equal create_metadata(tag: tag), i2.metadata(tag, time, record)\n\n      i3 = create_output(:buffered)\n      i3.configure(config_element('ROOT','',{},[config_element('buffer', 'time', {\"timekey\" => 3600, \"timekey_zone\" => \"-0700\"})]))\n      assert_equal create_metadata(timekey: timekey), i3.metadata(tag, time, record)\n\n      i4 = create_output(:buffered)\n      i4.configure(config_element('ROOT','',{},[config_element('buffer', 'key1', {})]))\n      assert_equal create_metadata(variables: {key1: \"value1\"}), i4.metadata(tag, time, record)\n\n      i5 = create_output(:buffered)\n      i5.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,num1', {})]))\n      assert_equal create_metadata(variables: {key1: \"value1\", num1: 1}), i5.metadata(tag, time, record)\n\n      i6 = create_output(:buffered)\n      i6.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,time', {\"timekey\" => 3600, \"timekey_zone\" => \"-0700\"})]))\n      assert_equal create_metadata(timekey: timekey, tag: tag), i6.metadata(tag, time, record)\n\n      i7 = create_output(:buffered)\n      i7.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,num1', {\"timekey\" => 3600, \"timekey_zone\" => \"-0700\"})]))\n      assert_equal create_metadata(tag: tag, variables: {num1: 1}), i7.metadata(tag, time, record)\n\n      i8 = create_output(:buffered)\n      i8.configure(config_element('ROOT','',{},[config_element('buffer', 'time,tag,key1', {\"timekey\" => 3600, \"timekey_zone\" => \"-0700\"})]))\n      assert_equal create_metadata(timekey: timekey, tag: tag, variables: {key1: \"value1\"}), i8.metadata(tag, time, record)\n\n      i9 = create_output(:buffered)\n      i9.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,$.nest.key', {})]))\n      assert_equal create_metadata(variables: {:key1 => \"value1\", :\"$.nest.key\" => 'nested value'}), i9.metadata(tag, time, record)\n    end\n\n    test '#emit calls #process via #emit_sync for non-buffered output' do\n      i = create_output(:sync)\n      process_called = false\n      i.register(:process){|tag, es| process_called = true }\n      i.configure(config_element())\n      i.start\n      i.after_start\n\n      t = event_time()\n      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ]))\n\n      assert process_called\n\n      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate\n    end\n\n    test '#emit calls #format for buffered output' do\n      i = create_output(:buffered)\n      format_called_times = 0\n      i.register(:format){|tag, time, record| format_called_times += 1; '' }\n      i.configure(config_element())\n      i.start\n      i.after_start\n\n      t = event_time()\n      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ]))\n\n      assert_equal 2, format_called_times\n\n      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate\n    end\n\n    test '#prefer_buffered_processing (returns false) decides non-buffered without <buffer> section' do\n      i = create_output(:full)\n\n      process_called = false\n      format_called_times = 0\n      i.register(:process){|tag, es| process_called = true }\n      i.register(:format){|tag, time, record| format_called_times += 1; '' }\n\n      i.configure(config_element())\n      i.register(:prefer_buffered_processing){ false } # delayed decision is possible to change after (output's) configure\n      i.start\n      i.after_start\n\n      assert !i.prefer_buffered_processing\n\n      t = event_time()\n      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ]))\n\n      waiting(4){ Thread.pass until process_called }\n\n      assert process_called\n      assert_equal 0, format_called_times\n\n      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate\n    end\n\n    test '#prefer_buffered_processing (returns true) decides buffered without <buffer> section' do\n      i = create_output(:full)\n\n      process_called = false\n      format_called_times = 0\n      i.register(:process){|tag, es| process_called = true }\n      i.register(:format){|tag, time, record| format_called_times += 1; '' }\n\n      i.configure(config_element())\n      i.register(:prefer_buffered_processing){ true } # delayed decision is possible to change after (output's) configure\n      i.start\n      i.after_start\n\n      assert i.prefer_buffered_processing\n\n      t = event_time()\n      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ]))\n\n      assert !process_called\n      assert_equal 2, format_called_times\n\n      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate\n    end\n\n    test 'output plugin will call #write for normal buffered plugin to flush buffer chunks' do\n      i = create_output(:buffered)\n      write_called = false\n      i.register(:write){ |chunk| write_called = true }\n\n      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {\"flush_mode\" => \"immediate\"})]))\n      i.start\n      i.after_start\n\n      t = event_time()\n      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ]))\n      i.force_flush\n\n      waiting(4){ Thread.pass until write_called }\n\n      assert write_called\n\n      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate\n    end\n\n    test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do\n      tmp_dir = File.join(__dir__, '../tmp/test_output')\n\n      i = create_output(:delayed)\n      i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.\n      try_write_called = false\n      i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }\n\n      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {\"flush_mode\" => \"immediate\"})]))\n      i.start\n      i.after_start\n\n      t = event_time()\n      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ]))\n      i.force_flush\n\n      waiting(4){ Thread.pass until try_write_called }\n\n      assert try_write_called\n\n      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate\n    ensure\n      FileUtils.rm_rf(tmp_dir)\n    end\n\n    test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do\n      i = create_output(:full)\n      write_called = false\n      try_write_called = false\n      i.register(:write){ |chunk| write_called = true }\n      i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }\n\n      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {\"flush_mode\" => \"immediate\"})]))\n      i.register(:prefer_delayed_commit){ false } # delayed decision is possible to change after (output's) configure\n      i.start\n      i.after_start\n\n      assert !i.prefer_delayed_commit\n\n      t = event_time()\n      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ]))\n      i.force_flush\n\n      waiting(4){ Thread.pass until write_called || try_write_called }\n\n      assert write_called\n      assert !try_write_called\n\n      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate\n    end\n\n    test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do\n      tmp_dir = File.join(__dir__, '../tmp/test_output')\n\n      i = create_output(:full)\n      i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.\n      write_called = false\n      try_write_called = false\n      i.register(:write){ |chunk| write_called = true }\n      i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }\n\n      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {\"flush_mode\" => \"immediate\"})]))\n      i.register(:prefer_delayed_commit){ true } # delayed decision is possible to change after (output's) configure\n      i.start\n      i.after_start\n\n      assert i.prefer_delayed_commit\n\n      t = event_time()\n      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ]))\n      i.force_flush\n\n      waiting(4){ Thread.pass until write_called || try_write_called }\n\n      assert !write_called\n      assert try_write_called\n\n      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate\n    ensure\n      FileUtils.rm_rf(tmp_dir)\n    end\n\n    test 'flush_interval is ignored when flush_mode is not interval' do\n      mock(@i.log).warn(\"'flush_interval' is ignored because default 'flush_mode' is not 'interval': 'lazy'\")\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'flush_interval' => 10})]))\n    end\n\n    data(:lazy => 'lazy', :immediate => 'immediate')\n    test 'flush_interval and non-interval flush_mode is exclusive ' do |mode|\n      assert_raise Fluent::ConfigError.new(\"'flush_interval' can't be specified when 'flush_mode' is not 'interval' explicitly: '#{mode}'\") do\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'flush_mode' => mode, 'flush_interval' => 10})]))\n      end\n    end\n\n    test 'flush_mode is set to interval when flush_interval with v0.12 configuration is given' do\n      mock(@i.log).info(\"'flush_interval' is configured at out side of <buffer>. 'flush_mode' is set to 'interval' to keep existing behaviour\")\n      @i.configure(config_element('ROOT', '', {'flush_interval' => 60}, []))\n      assert_equal :interval, @i.instance_variable_get(:@flush_mode)\n    end\n\n    sub_test_case 'configure secondary' do\n      test \"Warn if primary type is different from secondary type and either primary or secondary has custom_format\" do\n        o = create_output(:buffered)\n        mock(o.log).warn(\"Use different plugin for secondary. Check the plugin works with primary like secondary_file\",\n                         primary: o.class.to_s, secondary: \"Fluent::Plugin::TestOutput\")\n\n        o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => \"cool\"})]))\n        assert_not_nil o.instance_variable_get(:@secondary)\n      end\n\n      test \"don't warn if primary type is the same as secondary type\" do\n        o = Fluent::Plugin::TestOutput.new\n        mock(o.log).warn(\"Use different plugin for secondary. Check the plugin works with primary like secondary_file\",\n                         primary: o.class.to_s, secondary: \"Fluent::Plugin::TestOutput\" ).never\n\n        o.configure(config_element('ROOT','',{'name' => \"cool2\"},\n                                   [config_element('secondary','',{'@type'=>'test', 'name' => \"cool\"}),\n                                    config_element('buffer','',{'@type'=>'memory'})]\n                                  ))\n        assert_not_nil o.instance_variable_get(:@secondary)\n      end\n\n      test \"don't warn if primary type is different from secondary type and both don't have custom_format\" do\n        o = create_output(:standard)\n        mock(o.log).warn(\"Use different plugin for secondary. Check the plugin works with primary like secondary_file\",\n                         primary: o.class.to_s, secondary: \"Fluent::Plugin::TestOutput\").never\n\n        o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => \"cool\"})]))\n        assert_not_nil o.instance_variable_get(:@secondary)\n      end\n\n      test \"raise configuration error if secondary type specifies non buffered output\" do\n        o = create_output(:standard)\n        assert_raise Fluent::ConfigError do\n          o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'copy'})]))\n        end\n      end\n    end\n  end\n\n  test 'raises an error if timekey is less than equal 0' do\n    i = create_output(:delayed)\n    assert_raise Fluent::ConfigError.new(\"<buffer ...> argument includes 'time', but timekey is not configured\") do\n      i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { \"timekey\" => nil })]))\n    end\n\n    i = create_output(:delayed)\n    assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: 0.0') do\n      i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { \"timekey\" => 0 })]))\n    end\n\n    i = create_output(:delayed)\n    assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: -1.0') do\n      i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { \"timekey\" => -1 })]))\n    end\n  end\n\n  sub_test_case 'sync output feature' do\n    setup do\n      @i = create_output(:sync)\n    end\n\n    test 'raises configuration error if <buffer> section is specified' do\n      assert_raise Fluent::ConfigError do\n        @i.configure(config_element('ROOT','',{},[config_element('buffer', '')]))\n      end\n    end\n\n    test 'raises configuration error if <secondary> section is specified' do\n      assert_raise Fluent::ConfigError do\n        @i.configure(config_element('ROOT','',{},[config_element('secondary','')]))\n      end\n    end\n\n    test '#process is called for each event streams' do\n      ary = []\n      @i.register(:process){|tag, es| ary << [tag, es] }\n      @i.configure(config_element())\n      @i.start\n      @i.after_start\n\n      t = event_time()\n      es = Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ])\n      5.times do\n        @i.emit_events('tag', es)\n      end\n      assert_equal 5, ary.size\n\n      @i.stop; @i.before_shutdown; @i.shutdown; @i.after_shutdown; @i.close; @i.terminate\n    end\n  end\n\n  sub_test_case '#generate_format_proc' do\n    test \"when output doesn't have <buffer>\" do\n      i = create_output(:sync)\n      i.configure(config_element('ROOT', '', {}, []))\n      assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc\n    end\n\n    test \"when output doesn't have <buffer> and time_as_integer is true\" do\n      i = create_output(:sync)\n      i.configure(config_element('ROOT', '', {'time_as_integer' => true}))\n      assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, i.generate_format_proc\n    end\n\n    test 'when output has <buffer> and compress is gzip' do\n      i = create_output(:buffered)\n      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'gzip'})]))\n      assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM_GZIP, i.generate_format_proc\n    end\n\n    test 'when output has <buffer> and compress is gzip and time_as_integer is true' do\n      i = create_output(:buffered)\n      i.configure(config_element('ROOT', '', {'time_as_integer' => true}, [config_element('buffer', '', {'compress' => 'gzip'})]))\n      assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_GZIP, i.generate_format_proc\n    end\n\n    test 'when output has <buffer> and compress is zstd' do\n      i = create_output(:buffered)\n      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'zstd'})]))\n      assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM_ZSTD, i.generate_format_proc\n    end\n\n    test 'when output has <buffer> and compress is zstd and time_as_integer is true' do\n      i = create_output(:buffered)\n      i.configure(config_element('ROOT', '', {'time_as_integer' => true}, [config_element('buffer', '', {'compress' => 'zstd'})]))\n      assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_ZSTD, i.generate_format_proc\n    end\n\n    test 'when output has <buffer> and compress is text' do\n      i = create_output(:buffered)\n      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'text'})]))\n      assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc\n    end\n  end\n\n  sub_test_case 'slow_flush_log_threshold' do\n    def invoke_slow_flush_log_threshold_test(i)\n      i.configure(config_element('ROOT', '', {'slow_flush_log_threshold' => 0.5},\n                                 [config_element('buffer', '', {\"flush_mode\" => \"immediate\", \"flush_thread_interval\" => 30})]))\n      i.start\n      i.after_start\n\n      t = event_time()\n      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ]))\n      i.force_flush\n\n      waiting(4) { Thread.pass until i.test_finished? }\n\n      yield\n    ensure\n      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate\n    end\n\n    test '#write flush took longer time than slow_flush_log_threshold' do\n      i = create_output(:buffered)\n      write_called = false\n      i.register(:write) { |chunk| sleep 3 }\n      i.define_singleton_method(:test_finished?) { write_called }\n      i.define_singleton_method(:try_flush) { super(); write_called = true }\n\n      invoke_slow_flush_log_threshold_test(i) {\n        assert write_called\n        logs = i.log.out.logs\n        assert{ logs.any?{|log| log.include?(\"buffer flush took longer time than slow_flush_log_threshold: elapsed_time\") } }\n      }\n    end\n\n    test '#try_write flush took longer time than slow_flush_log_threshold' do\n      i = create_output(:delayed)\n      try_write_called = false\n      i.register(:try_write){ |chunk| sleep 3 }\n      i.define_singleton_method(:test_finished?) { try_write_called }\n      i.define_singleton_method(:try_flush) { super(); try_write_called = true }\n\n      invoke_slow_flush_log_threshold_test(i) {\n        assert try_write_called\n        logs = i.log.out.logs\n        assert{ logs.any?{|log| log.include?(\"buffer flush took longer time than slow_flush_log_threshold: elapsed_time\") } }\n      }\n    end\n  end\n\n  sub_test_case \"actual_flush_thread_count\" do\n    data(\n      \"Not buffered\",\n      {\n        output_type: :sync,\n        config: config_element(),\n        expected: 0,\n      }\n    )\n    data(\n      \"Buffered with singile thread\",\n      {\n        output_type: :full,\n        config: config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\", \"\", {})]),\n        expected: 1,\n      }\n    )\n    data(\n      \"Buffered with multiple threads\",\n      {\n        output_type: :full,\n        config: config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\", \"\", {\"flush_thread_count\" => 8})]),\n        expected: 8,\n      }\n    )\n    test \"actual_flush_thread_count\" do |data|\n      o = create_output(data[:output_type])\n      o.configure(data[:config])\n      assert_equal data[:expected], o.actual_flush_thread_count\n    end\n\n    data(\n      \"Buffered with single thread\",\n      {\n        output_type: :full,\n        config: config_element(\n          \"ROOT\", \"\", {},\n          [\n            config_element(\"buffer\", \"\", {}),\n            config_element(\"secondary\", \"\", {\"@type\" => \"test\", \"name\" => \"test\"}),\n          ]\n        ),\n        expected: 1,\n      }\n    )\n    data(\n      \"Buffered with multiple threads\",\n      {\n        output_type: :full,\n        config: config_element(\n          \"ROOT\", \"\", {},\n          [\n            config_element(\"buffer\", \"\", {\"flush_thread_count\" => 8}),\n            config_element(\"secondary\", \"\", {\"@type\" => \"test\", \"name\" => \"test\"}),\n          ]\n        ),\n        expected: 8,\n      }\n    )\n    test \"actual_flush_thread_count for secondary\" do |data|\n      primary = create_output(data[:output_type])\n      primary.configure(data[:config])\n      assert_equal data[:expected], primary.secondary.actual_flush_thread_count\n    end\n  end\n\n  sub_test_case \"synchronize_path\" do\n    def setup\n      Dir.mktmpdir do |lock_dir|\n        ENV['FLUENTD_LOCK_DIR'] = lock_dir\n        yield\n      end\n    end\n\n    def assert_worker_lock(lock_path, expect_locked)\n      # With LOCK_NB set, flock() returns:\n      #   * `false` when the file is already locked.\n      #   * `0` when the file is not locked.\n      File.open(lock_path, \"w\") do |f|\n        if expect_locked\n          assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)\n        else\n          assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)\n        end\n      end\n    end\n\n    def assert_thread_lock(output_plugin, expect_locked)\n      t = Thread.new do\n        output_plugin.synchronize_path(\"test\") do\n        end\n      end\n      if expect_locked\n        assert_nil t.join(3)\n      else\n        assert_not_nil t.join(3)\n      end\n    end\n\n    data(\n      \"Not buffered with single worker\",\n      {\n        output_type: :sync,\n        config: config_element(),\n        workers: 1,\n        expect_worker_lock: false,\n        expect_thread_lock: false,\n      }\n    )\n    data(\n      \"Not buffered with multiple workers\",\n      {\n        output_type: :sync,\n        config: config_element(),\n        workers: 4,\n        expect_worker_lock: true,\n        expect_thread_lock: false,\n      }\n    )\n    data(\n      \"Buffered with single thread and single worker\",\n      {\n        output_type: :full,\n        config: config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\", \"\", {})]),\n        workers: 1,\n        expect_worker_lock: false,\n        expect_thread_lock: false,\n      }\n    )\n    data(\n      \"Buffered with multiple threads and single worker\",\n      {\n        output_type: :full,\n        config: config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\", \"\", {\"flush_thread_count\" => 8})]),\n        workers: 1,\n        expect_worker_lock: false,\n        expect_thread_lock: true,\n      }\n    )\n    data(\n      \"Buffered with single thread and multiple workers\",\n      {\n        output_type: :full,\n        config: config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\", \"\", {})]),\n        workers: 4,\n        expect_worker_lock: true,\n        expect_thread_lock: false,\n      }\n    )\n    data(\n      \"Buffered with multiple threads and multiple workers\",\n      {\n        output_type: :full,\n        config: config_element(\"ROOT\", \"\", {}, [config_element(\"buffer\", \"\", {\"flush_thread_count\" => 8})]),\n        workers: 4,\n        expect_worker_lock: true,\n        expect_thread_lock: true,\n      }\n    )\n    test \"synchronize_path\" do |data|\n      o = create_output(data[:output_type])\n      o.configure(data[:config])\n      o.system_config_override(workers: data[:workers])\n\n      test_lock_name = \"test_lock_name\"\n      lock_path = o.get_lock_path(test_lock_name)\n\n      o.synchronize_path(test_lock_name) do\n        assert_worker_lock(lock_path, data[:expect_worker_lock])\n        assert_thread_lock(o, data[:expect_thread_lock])\n      end\n\n      assert_worker_lock(lock_path, false)\n      assert_thread_lock(o, false)\n    end\n\n    data(\n      \"Buffered with single thread and single worker\",\n      {\n        output_type: :full,\n        config: config_element(\n          \"ROOT\", \"\", {},\n          [\n            config_element(\"buffer\", \"\", {}),\n            config_element(\"secondary\", \"\", {\"@type\" => \"test\", \"name\" => \"test\"}),\n          ]\n        ),\n        workers: 1,\n        expect_worker_lock: false,\n        expect_thread_lock: false,\n      }\n    )\n    data(\n      \"Buffered with multiple threads and single worker\",\n      {\n        output_type: :full,\n        config: config_element(\n          \"ROOT\", \"\", {},\n          [\n            config_element(\"buffer\", \"\", {\"flush_thread_count\" => 8}),\n            config_element(\"secondary\", \"\", {\"@type\" => \"test\", \"name\" => \"test\"}),\n          ]\n        ),\n        workers: 1,\n        expect_worker_lock: false,\n        expect_thread_lock: true,\n      }\n    )\n    data(\n      \"Buffered with single thread and multiple workers\",\n      {\n        output_type: :full,\n        config: config_element(\n          \"ROOT\", \"\", {},\n          [\n            config_element(\"buffer\", \"\", {}),\n            config_element(\"secondary\", \"\", {\"@type\" => \"test\", \"name\" => \"test\"}),\n          ]\n        ),\n        workers: 4,\n        expect_worker_lock: true,\n        expect_thread_lock: false,\n      }\n    )\n    data(\n      \"Buffered with multiple threads and multiple workers\",\n      {\n        output_type: :full,\n        config: config_element(\n          \"ROOT\", \"\", {},\n          [\n            config_element(\"buffer\", \"\", {\"flush_thread_count\" => 8}),\n            config_element(\"secondary\", \"\", {\"@type\" => \"test\", \"name\" => \"test\"}),\n          ]\n        ),\n        workers: 4,\n        expect_worker_lock: true,\n        expect_thread_lock: true,\n      }\n    )\n    test \"synchronize_path for secondary\" do |data|\n      primary = create_output(data[:output_type])\n      primary.configure(data[:config])\n      secondary = primary.secondary\n      secondary.system_config_override(workers: data[:workers])\n\n      test_lock_name = \"test_lock_name\"\n      lock_path = secondary.get_lock_path(test_lock_name)\n\n      secondary.synchronize_path(test_lock_name) do\n        assert_worker_lock(lock_path, data[:expect_worker_lock])\n        assert_thread_lock(secondary, data[:expect_thread_lock])\n      end\n\n      assert_worker_lock(lock_path, false)\n      assert_thread_lock(secondary, false)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_output_as_buffered.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/output'\nrequire 'fluent/event'\n\nrequire 'json'\nrequire 'time'\nrequire 'timeout'\nrequire 'timecop'\n\nmodule FluentPluginOutputAsBufferedTest\n  class DummyBareOutput < Fluent::Plugin::Output\n    def register(name, &block)\n      instance_variable_set(\"@#{name}\", block)\n    end\n  end\n  class DummySyncOutput < DummyBareOutput\n    def initialize\n      super\n      @process = nil\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n  end\n  class DummyAsyncOutput < DummyBareOutput\n    def initialize\n      super\n      @format = nil\n      @write = nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n  end\n  class DummyDelayedOutput < DummyBareOutput\n    def initialize\n      super\n      @format = nil\n      @try_write = nil\n      @shutdown_hook = nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n    def shutdown\n      if @shutdown_hook\n        @shutdown_hook.call\n      end\n      super\n    end\n  end\n  class DummyStandardBufferedOutput < DummyBareOutput\n    def initialize\n      super\n      @prefer_delayed_commit = nil\n      @write = nil\n      @try_write = nil\n    end\n    def prefer_delayed_commit\n      @prefer_delayed_commit ? @prefer_delayed_commit.call : false\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n  class DummyCustomFormatBufferedOutput < DummyBareOutput\n    def initialize\n      super\n      @format_type_is_msgpack = nil\n      @prefer_delayed_commit = nil\n      @write = nil\n      @try_write = nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def formatted_to_msgpack_binary?\n      @format_type_is_msgpack ? @format_type_is_msgpack.call : false\n    end\n    def prefer_delayed_commit\n      @prefer_delayed_commit ? @prefer_delayed_commit.call : false\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n  # check for formatted_to_msgpack_binary compatibility\n  class DummyOldCustomFormatBufferedOutput < DummyBareOutput\n    def initialize\n      super\n      @format_type_is_msgpack = nil\n      @prefer_delayed_commit = nil\n      @write = nil\n      @try_write = nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def formatted_to_msgpack_binary\n      @format_type_is_msgpack ? @format_type_is_msgpack.call : false\n    end\n    def prefer_delayed_commit\n      @prefer_delayed_commit ? @prefer_delayed_commit.call : false\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n  class DummyFullFeatureOutput < DummyBareOutput\n    def initialize\n      super\n      @prefer_buffered_processing = nil\n      @prefer_delayed_commit = nil\n      @process = nil\n      @format = nil\n      @write = nil\n      @try_write = nil\n    end\n    def prefer_buffered_processing\n      @prefer_buffered_processing ? @prefer_buffered_processing.call : false\n    end\n    def prefer_delayed_commit\n      @prefer_delayed_commit ? @prefer_delayed_commit.call : false\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n  module OldPluginMethodMixin\n    def initialize\n      super\n      @format = nil\n      @write = nil\n    end\n    def register(name, &block)\n      instance_variable_set(\"@#{name}\", block)\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n  end\n  class DummyOldBufferedOutput < Fluent::BufferedOutput\n    include OldPluginMethodMixin\n  end\n  class DummyOldObjectBufferedOutput < Fluent::ObjectBufferedOutput\n    include OldPluginMethodMixin\n  end\nend\n\nclass BufferedOutputTest < Test::Unit::TestCase\n  def create_output(type=:full)\n    case type\n    when :bare     then FluentPluginOutputAsBufferedTest::DummyBareOutput.new\n    when :sync     then FluentPluginOutputAsBufferedTest::DummySyncOutput.new\n    when :buffered then FluentPluginOutputAsBufferedTest::DummyAsyncOutput.new\n    when :delayed  then FluentPluginOutputAsBufferedTest::DummyDelayedOutput.new\n    when :standard then FluentPluginOutputAsBufferedTest::DummyStandardBufferedOutput.new\n    when :custom   then FluentPluginOutputAsBufferedTest::DummyCustomFormatBufferedOutput.new\n    when :full     then FluentPluginOutputAsBufferedTest::DummyFullFeatureOutput.new\n    when :old_buf  then FluentPluginOutputAsBufferedTest::DummyOldBufferedOutput.new\n    when :old_obj  then FluentPluginOutputAsBufferedTest::DummyOldObjectBufferedOutput.new\n    when :old_custom then FluentPluginOutputAsBufferedTest::DummyOldCustomFormatBufferedOutput.new\n    else\n      raise ArgumentError, \"unknown type: #{type}\"\n    end\n  end\n  def create_metadata(timekey: nil, tag: nil, variables: nil)\n    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n  end\n  def waiting(seconds)\n    begin\n      Timeout.timeout(seconds) do\n        yield\n      end\n    rescue Timeout::Error\n      STDERR.print(*@i.log.out.logs)\n      raise\n    end\n  end\n\n  setup do\n    @i = nil\n  end\n\n  teardown do\n    if @i\n      @i.stop unless @i.stopped?\n      @i.before_shutdown unless @i.before_shutdown?\n      @i.shutdown unless @i.shutdown?\n      @i.after_shutdown unless @i.after_shutdown?\n      @i.close unless @i.closed?\n      @i.terminate unless @i.terminated?\n    end\n    Timecop.return\n  end\n\n  test 'queued_chunks_limit_size is same as flush_thread_count by default' do\n    hash = {'flush_thread_count' => 4}\n    i = create_output\n    i.register(:prefer_buffered_processing) { true }\n    i.configure(config_element('ROOT', '', {}, [config_element('buffer','tag',hash)]))\n\n    assert_equal 4, i.buffer.queued_chunks_limit_size\n  end\n\n  test 'prefer queued_chunks_limit_size parameter than flush_thread_count' do\n    hash = {'flush_thread_count' => 4, 'queued_chunks_limit_size' => 2}\n    i = create_output\n    i.register(:prefer_buffered_processing) { true }\n    i.configure(config_element('ROOT', '', {}, [config_element('buffer','tag',hash)]))\n\n    assert_equal 2, i.buffer.queued_chunks_limit_size\n  end\n\n  sub_test_case 'chunk feature in #write for output plugins' do\n    setup do\n      @stored_global_logger = $log\n      $log = Fluent::Test::TestLogger.new\n      @hash = {\n        'flush_mode' => 'immediate',\n        'flush_thread_interval' => '0.01',\n        'flush_thread_burst_interval' => '0.01',\n      }\n    end\n\n    teardown do\n      $log = @stored_global_logger\n    end\n\n    test 'plugin using standard format can iterate chunk for time, record in #write' do\n      events_from_chunk = []\n      @i = create_output(:standard)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))\n      @i.register(:prefer_delayed_commit){ false }\n      @i.register(:write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:write, e] }\n      @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:try_write, e] }\n      @i.start\n      @i.after_start\n\n      events = [\n        [event_time('2016-10-05 16:16:16 -0700'), {\"message\" => \"yaaaaaaaaay!\"}],\n        [event_time('2016-10-05 16:16:17 -0700'), {\"message\" => \"yoooooooooy!\"}],\n      ]\n\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n\n      waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }\n\n      assert_equal 2, events_from_chunk.size\n      2.times.each do |i|\n        assert_equal :write, events_from_chunk[i][0]\n        assert_equal events, events_from_chunk[i][1]\n      end\n    end\n\n    test 'plugin using standard format can iterate chunk for time, record in #try_write' do\n      events_from_chunk = []\n      @i = create_output(:standard)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))\n      @i.register(:prefer_delayed_commit){ true }\n      @i.register(:write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:write, e] }\n      @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:try_write, e] }\n      @i.start\n      @i.after_start\n\n      events = [\n        [event_time('2016-10-05 16:16:16 -0700'), {\"message\" => \"yaaaaaaaaay!\"}],\n        [event_time('2016-10-05 16:16:17 -0700'), {\"message\" => \"yoooooooooy!\"}],\n      ]\n\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n\n      waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }\n\n      assert_equal 2, events_from_chunk.size\n      2.times.each do |i|\n        assert_equal :try_write, events_from_chunk[i][0]\n        assert_equal events, events_from_chunk[i][1]\n      end\n    end\n\n    test 'plugin using custom format cannot iterate chunk in #write' do\n      events_from_chunk = []\n      @i = create_output(:custom)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))\n      @i.register(:prefer_delayed_commit){ false }\n      @i.register(:format){ |tag, time, record| [tag,time,record].to_json }\n      @i.register(:format_type_is_msgpack){ false }\n      @i.register(:write){ |chunk| assert !(chunk.respond_to?(:each)) }\n      @i.register(:try_write){ |chunk| assert !(chunk.respond_to?(:each)) }\n      @i.start\n      @i.after_start\n\n      events = [\n        [event_time('2016-10-05 16:16:16 -0700'), {\"message\" => \"yaaaaaaaaay!\"}],\n        [event_time('2016-10-05 16:16:17 -0700'), {\"message\" => \"yoooooooooy!\"}],\n      ]\n\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n\n      assert_equal 0, events_from_chunk.size\n    end\n\n    test 'plugin using custom format cannot iterate chunk in #try_write' do\n      events_from_chunk = []\n      @i = create_output(:custom)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))\n      @i.register(:prefer_delayed_commit){ true }\n      @i.register(:format){ |tag, time, record| [tag,time,record].to_json }\n      @i.register(:format_type_is_msgpack){ false }\n      @i.register(:write){ |chunk| assert !(chunk.respond_to?(:each)) }\n      @i.register(:try_write){ |chunk| assert !(chunk.respond_to?(:each)) }\n      @i.start\n      @i.after_start\n\n      events = [\n        [event_time('2016-10-05 16:16:16 -0700'), {\"message\" => \"yaaaaaaaaay!\"}],\n        [event_time('2016-10-05 16:16:17 -0700'), {\"message\" => \"yoooooooooy!\"}],\n      ]\n\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n\n      assert_equal 0, events_from_chunk.size\n    end\n\n    data('formatted_to_msgpack_binary?' => :custom,\n         'formatted_to_msgpack_binary' => :old_custom)\n    test 'plugin using custom format can iterate chunk in #write if #format returns msgpack' do |out_type|\n      events_from_chunk = []\n      @i = create_output(out_type)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))\n      @i.register(:prefer_delayed_commit){ false }\n      @i.register(:format){ |tag, time, record| [tag,time,record].to_msgpack }\n      @i.register(:format_type_is_msgpack){ true }\n      @i.register(:write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:write, e] }\n      @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:try_write, e] }\n      @i.start\n      @i.after_start\n\n      events = [\n        [event_time('2016-10-05 16:16:16 -0700'), {\"message\" => \"yaaaaaaaaay!\"}],\n        [event_time('2016-10-05 16:16:17 -0700'), {\"message\" => \"yoooooooooy!\"}],\n      ]\n\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n\n      waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }\n\n      assert_equal 2, events_from_chunk.size\n      2.times.each do |i|\n        assert_equal :write, events_from_chunk[i][0]\n        each_pushed = events_from_chunk[i][1]\n        assert_equal 2, each_pushed.size\n        assert_equal 'test.tag', each_pushed[0][0]\n        assert_equal 'test.tag', each_pushed[1][0]\n        assert_equal events, each_pushed.map{|tag,time,record| [time,record]}\n      end\n    end\n\n    data(:handle_stream_simple => '',\n         :handle_stream_with_custom_format => 'tag,message')\n    test 'plugin using custom format can skip record chunk when format return nil' do |chunk_keys|\n      events_from_chunk = []\n      @i = create_output(:custom)\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', chunk_keys, @hash)]))\n      @i.register(:prefer_delayed_commit) { false }\n      @i.register(:format) { |tag, time, record|\n        if record['message'] == 'test1'\n          nil\n        else\n          [tag,time,record].to_msgpack\n        end\n      }\n      @i.register(:format_type_is_msgpack) { true }\n      @i.register(:write){ |chunk| e = []; chunk.each { |ta, t, r| e << [ta, t, r] }; events_from_chunk << [:write, e] }\n      @i.start\n      @i.after_start\n\n      events = [\n        [event_time('2016-10-05 16:16:16 -0700'), {\"message\" => \"test1\"}],\n        [event_time('2016-10-05 16:16:17 -0700'), {\"message\" => \"test2\"}],\n      ]\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n\n      waiting(5) { sleep 0.1 until events_from_chunk.size == 1 }\n\n      assert_equal 1, events_from_chunk.size\n      assert_equal :write, events_from_chunk[0][0]\n      each_pushed = events_from_chunk[0][1]\n      assert_equal 1, each_pushed.size\n      assert_equal 'test.tag', each_pushed[0][0]\n      assert_equal \"test2\", each_pushed[0][2]['message']\n    end\n\n    test 'plugin using custom format can iterate chunk in #try_write if #format returns msgpack' do\n      events_from_chunk = []\n      @i = create_output(:custom)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))\n      @i.register(:prefer_delayed_commit){ true }\n      @i.register(:format){ |tag, time, record| [tag,time,record].to_msgpack }\n      @i.register(:format_type_is_msgpack){ true }\n      @i.register(:write){ |chunk| events_from_chunk = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:write, e] }\n      @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:try_write, e] }\n      @i.start\n      @i.after_start\n\n      events = [\n        [event_time('2016-10-05 16:16:16 -0700'), {\"message\" => \"yaaaaaaaaay!\"}],\n        [event_time('2016-10-05 16:16:17 -0700'), {\"message\" => \"yoooooooooy!\"}],\n      ]\n\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n\n      waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }\n\n      assert_equal 2, events_from_chunk.size\n      2.times.each do |i|\n        assert_equal :try_write, events_from_chunk[i][0]\n        each_pushed = events_from_chunk[i][1]\n        assert_equal 2, each_pushed.size\n        assert_equal 'test.tag', each_pushed[0][0]\n        assert_equal 'test.tag', each_pushed[1][0]\n        assert_equal events, each_pushed.map{|tag,time,record| [time,record]}\n      end\n    end\n\n    data(:BufferedOutput => :old_buf,\n         :ObjectBufferedOutput => :old_obj)\n    test 'old plugin types can iterate chunk by msgpack_each in #write' do |plugin_type|\n      events_from_chunk = []\n      # event_emitter helper requires Engine.root_agent for routing\n      ra = Fluent::RootAgent.new(log: $log)\n      stub(Fluent::Engine).root_agent { ra }\n      @i = create_output(plugin_type)\n      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', @hash)]))\n      @i.register(:format) { |tag, time, record| [time, record].to_msgpack }\n      @i.register(:write) { |chunk| e = []; chunk.msgpack_each { |t, r| e << [t, r] }; events_from_chunk << [:write, e]; }\n      @i.start\n      @i.after_start\n\n      events = [\n        [event_time('2016-10-05 16:16:16 -0700'), {\"message\" => \"yaaaaaaaaay!\"}],\n        [event_time('2016-10-05 16:16:17 -0700'), {\"message\" => \"yoooooooooy!\"}],\n      ]\n\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n\n      waiting(5) { sleep 0.1 until events_from_chunk.size == 2 }\n\n      assert_equal 2, events_from_chunk.size\n      2.times.each do |i|\n        assert_equal :write, events_from_chunk[i][0]\n        assert_equal events, events_from_chunk[i][1]\n      end\n    end\n  end\n\n  sub_test_case 'buffered output configured with many chunk keys' do\n    setup do\n      @stored_global_logger = $log\n      $log = Fluent::Test::TestLogger.new\n      @hash = {\n        'flush_mode' => 'interval',\n        'flush_thread_burst_interval' => 0.01,\n        'chunk_limit_size' => 1024,\n        'timekey' => 60,\n      }\n      @i = create_output(:buffered)\n    end\n    teardown do\n      $log = @stored_global_logger\n    end\n    test 'nothing are warned with less chunk keys' do\n      chunk_keys = 'time,key1,key2,key3'\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))\n      logs = @i.log.out.logs.dup\n      @i.start\n      @i.after_start\n      assert{ logs.count{|log| log.include?('[warn]') } == 0 }\n    end\n\n    test 'a warning reported with 4 chunk keys' do\n      chunk_keys = 'key1,key2,key3,key4'\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))\n      logs = @i.log.out.logs.dup\n\n      @i.start # this calls `log.reset`... capturing logs about configure must be done before this line\n      @i.after_start\n      assert_equal ['key1', 'key2', 'key3', 'key4'], @i.chunk_keys\n\n      assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }\n    end\n\n    test 'a warning reported with 4 chunk keys including \"tag\"' do\n      chunk_keys = 'tag,key1,key2,key3'\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))\n      logs = @i.log.out.logs.dup\n      @i.start # this calls `log.reset`... capturing logs about configure must be done before this line\n      @i.after_start\n      assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }\n    end\n\n    test 'time key is not included for warned chunk keys' do\n      chunk_keys = 'time,key1,key2,key3'\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))\n      logs = @i.log.out.logs.dup\n      @i.start\n      @i.after_start\n      assert{ logs.count{|log| log.include?('[warn]') } == 0 }\n    end\n  end\n\n  sub_test_case 'buffered output feature without any buffer key, flush_mode: lazy' do\n    setup do\n      hash = {\n        'flush_mode' => 'lazy',\n        'flush_thread_burst_interval' => 0.01,\n        'flush_thread_count' => 2,\n        'chunk_limit_size' => 1024,\n      }\n      @i = create_output(:buffered)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))\n      @i.start\n      @i.after_start\n    end\n\n    test '#start does not create enqueue thread, but creates flush threads' do\n      @i.thread_wait_until_start\n\n      assert @i.thread_exist?(:flush_thread_0)\n      assert @i.thread_exist?(:flush_thread_1)\n      assert !@i.thread_exist?(:enqueue_thread)\n    end\n\n    test '#format is called for each events' do\n      ary = []\n      @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }\n\n      t = event_time()\n      es = Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ])\n\n      4.times do\n        @i.emit_events('tag.test', es)\n      end\n\n      assert_equal 8, ary.size\n      4.times do |i|\n        assert_equal [\"tag.test\", t, {\"key\" => \"value1\"}], ary[i*2]\n        assert_equal [\"tag.test\", t, {\"key\" => \"value2\"}], ary[i*2+1]\n      end\n    end\n\n    test '#write is called only when chunk bytes limit exceeded, and buffer chunk is purged' do\n      ary = []\n      @i.register(:write){|chunk| ary << chunk.read }\n\n      tag = \"test.tag\"\n      t = event_time()\n      r = {}\n      (0...10).each do |i|\n        r[\"key#{i}\"] = \"value #{i}\"\n      end\n      event_size = [tag, t, r].to_json.size # 195\n\n      (1024 * 0.9 / event_size).to_i.times do |i|\n        @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new([ [t, r] ]))\n      end\n      assert{ @i.buffer.queue.size == 0 && ary.size == 0 }\n\n      staged_chunk = @i.buffer.stage[@i.buffer.stage.keys.first]\n      assert{ staged_chunk.size != 0 }\n\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new([ [t, r] ]))\n\n      assert{ @i.buffer.queue.size > 0 || @i.buffer.dequeued.size > 0 || ary.size > 0 }\n\n      waiting(10) do\n        Thread.pass until @i.buffer.queue.size == 0 && @i.buffer.dequeued.size == 0\n        Thread.pass until staged_chunk.size == 0\n      end\n\n      assert_equal 1, ary.size\n      assert_equal [tag,t,r].to_json * (1024 / event_size), ary.first\n    end\n\n    test 'flush_at_shutdown work well when plugin is shutdown' do\n      ary = []\n      @i.register(:write){|chunk| ary << chunk.read }\n\n      tag = \"test.tag\"\n      t = event_time()\n      r = {}\n      (0...10).each do |i|\n        r[\"key#{i}\"] = \"value #{i}\"\n      end\n      event_size = [tag, t, r].to_json.size # 195\n\n      (1024 * 0.9 / event_size).to_i.times do |i|\n        @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new([ [t, r] ]))\n      end\n      assert{ @i.buffer.queue.size == 0 && ary.size == 0 }\n\n      @i.stop\n      @i.before_shutdown\n      @i.shutdown\n      @i.after_shutdown\n\n      waiting(10) do\n        Thread.pass until ary.size == 1\n      end\n      assert_equal [tag,t,r].to_json * (1024 * 0.9 / event_size), ary.first\n    end\n  end\n\n  sub_test_case 'buffered output feature without any buffer key, flush_mode: interval' do\n    setup do\n      hash = {\n        'flush_mode' => 'interval',\n        'flush_interval' => 1,\n        'flush_thread_count' => 1,\n        'flush_thread_burst_interval' => 0.01,\n        'chunk_limit_size' => 1024,\n      }\n      @i = create_output(:buffered)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))\n      @i.start\n      @i.after_start\n    end\n\n    test '#start creates enqueue thread and flush threads' do\n      @i.thread_wait_until_start\n\n      assert @i.thread_exist?(:flush_thread_0)\n      assert @i.thread_exist?(:enqueue_thread)\n    end\n\n    test '#format is called for each event streams' do\n      ary = []\n      @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }\n\n      t = event_time()\n      es = Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ])\n\n      4.times do\n        @i.emit_events('tag.test', es)\n      end\n\n      assert_equal 8, ary.size\n      4.times do |i|\n        assert_equal [\"tag.test\", t, {\"key\" => \"value1\"}], ary[i*2]\n        assert_equal [\"tag.test\", t, {\"key\" => \"value2\"}], ary[i*2+1]\n      end\n    end\n\n    test '#write is called per flush_interval, and buffer chunk is purged' do\n      @i.thread_wait_until_start\n\n      ary = []\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| chunk.read.split(\"\\n\").reject{|l| l.empty? }.each{|data| ary << data } }\n\n      t = event_time()\n      r = {}\n      (0...10).each do |i|\n        r[\"key#{i}\"] = \"value #{i}\"\n      end\n\n      2.times do |i|\n        rand_records = rand(1..4)\n        es = Fluent::ArrayEventStream.new([ [t, r] ] * rand_records)\n        assert_equal rand_records, es.size\n\n        @i.interrupt_flushes\n\n        assert{ @i.buffer.queue.size == 0 }\n\n        @i.emit_events(\"test.tag\", es)\n\n        assert{ @i.buffer.queue.size == 0 }\n        assert{ @i.buffer.stage.size == 1 }\n\n        staged_chunk = @i.instance_eval{ @buffer.stage[@buffer.stage.keys.first] }\n        assert{ staged_chunk.size != 0 }\n\n        @i.enqueue_thread_wait\n\n        waiting(10) do\n          Thread.pass until @i.buffer.queue.size == 0 && @i.buffer.dequeued.size == 0\n          Thread.pass until staged_chunk.size == 0\n        end\n\n        assert_equal rand_records, ary.size\n        ary.reject!{|e| true }\n      end\n    end\n  end\n\n  sub_test_case 'with much longer flush_interval' do\n    setup do\n      hash = {\n        'flush_mode' => 'interval',\n        'flush_interval' => 3000,\n        'flush_thread_count' => 1,\n        'flush_thread_burst_interval' => 0.01,\n        'chunk_limit_size' => 1024,\n      }\n      @i = create_output(:buffered)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))\n      @i.start\n      @i.after_start\n    end\n\n    test 'flush_at_shutdown work well when plugin is shutdown' do\n      ary = []\n      @i.register(:write){|chunk| ary << chunk.read }\n\n      tag = \"test.tag\"\n      t = event_time()\n      r = {}\n      (0...10).each do |i|\n        r[\"key#{i}\"] = \"value #{i}\"\n      end\n      event_size = [tag, t, r].to_json.size # 195\n\n      (1024 * 0.9 / event_size).to_i.times do |i|\n        @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new([ [t, r] ]))\n      end\n      queue_size = @i.buffer.queue.size\n      assert{ queue_size == 0 && ary.size == 0 }\n\n      @i.stop\n      @i.before_shutdown\n      @i.shutdown\n      @i.after_shutdown\n\n      waiting(10){ sleep 0.1 until ary.size == 1 }\n      assert_equal [tag,t,r].to_json * (1024 * 0.9 / event_size), ary.first\n    end\n  end\n\n  sub_test_case 'buffered output feature without any buffer key, flush_mode: immediate' do\n    setup do\n      hash = {\n        'flush_mode' => 'immediate',\n        'flush_thread_count' => 1,\n        'flush_thread_burst_interval' => 0.01,\n        'chunk_limit_size' => 1024,\n      }\n      @i = create_output(:buffered)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))\n      @i.start\n      @i.after_start\n    end\n\n    test '#start does not create enqueue thread, but creates flush threads' do\n      @i.thread_wait_until_start\n\n      assert @i.thread_exist?(:flush_thread_0)\n      assert !@i.thread_exist?(:enqueue_thread)\n    end\n\n    test '#format is called for each event streams' do\n      ary = []\n      @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }\n\n      t = event_time()\n      es = Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ])\n\n      4.times do\n        @i.emit_events('tag.test', es)\n      end\n\n      assert_equal 8, ary.size\n      4.times do |i|\n        assert_equal [\"tag.test\", t, {\"key\" => \"value1\"}], ary[i*2]\n        assert_equal [\"tag.test\", t, {\"key\" => \"value2\"}], ary[i*2+1]\n      end\n    end\n\n    test '#write is called every time for each emits, and buffer chunk is purged' do\n      @i.thread_wait_until_start\n\n      ary = []\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| chunk.read.split(\"\\n\").reject{|l| l.empty? }.each{|data| ary << data } }\n\n      t = event_time()\n      r = {}\n      (0...10).each do |i|\n        r[\"key#{i}\"] = \"value #{i}\"\n      end\n\n      3.times do |i|\n        rand_records = rand(1..5)\n        es = Fluent::ArrayEventStream.new([ [t, r] ] * rand_records)\n        assert_equal rand_records, es.size\n        @i.emit_events(\"test.tag\", es)\n\n        waiting(10){ sleep 0.1 until @i.buffer.stage.size == 0 } # make sure that the emitted es is enqueued by \"flush_mode immediate\"\n        waiting(10){ sleep 0.1 until @i.buffer.queue.size == 0 && @i.buffer.dequeued.size == 0 }\n        waiting(10){ sleep 0.1 until ary.size == rand_records }\n\n        assert_equal rand_records, ary.size\n        ary.reject!{|e| true }\n      end\n    end\n\n    test 'flush_at_shutdown work well when plugin is shutdown' do\n      ary = []\n      @i.register(:write){|chunk| ary << chunk.read }\n\n      tag = \"test.tag\"\n      t = event_time()\n      r = {}\n      (0...10).each do |i|\n        r[\"key#{i}\"] = \"value #{i}\"\n      end\n      @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new([ [t, r] ]))\n\n      @i.stop\n      @i.before_shutdown\n      @i.shutdown\n      @i.after_shutdown\n\n      waiting(10) do\n        Thread.pass until ary.size == 1\n      end\n      assert_equal [tag,t,r].to_json, ary.first\n    end\n  end\n\n  sub_test_case 'buffered output feature with timekey and range' do\n    setup do\n      chunk_key = 'time'\n      hash = {\n        'timekey' => 30, # per 30seconds\n        'timekey_wait' => 5, # 5 second delay for flush\n        'flush_thread_count' => 1,\n        'flush_thread_burst_interval' => 0.01,\n      }\n      @i = create_output(:buffered)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.start\n      @i.after_start\n    end\n\n    test '#configure raises config error if timekey is not specified' do\n      i = create_output(:buffered)\n      assert_raise Fluent::ConfigError do\n        i.configure(config_element('ROOT','',{},[config_element('buffer','time',)]))\n      end\n    end\n\n    test 'default flush_mode is set to :lazy' do\n      assert_equal :lazy, @i.instance_eval{ @flush_mode }\n    end\n\n    test '#start creates enqueue thread and flush threads' do\n      @i.thread_wait_until_start\n\n      assert @i.thread_exist?(:flush_thread_0)\n      assert @i.thread_exist?(:enqueue_thread)\n    end\n\n    test '#format is called for each event streams' do\n      ary = []\n      @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }\n\n      t = event_time()\n      es = Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ])\n\n      5.times do\n        @i.emit_events('tag.test', es)\n      end\n\n      assert_equal 10, ary.size\n      5.times do |i|\n        assert_equal [\"tag.test\", t, {\"key\" => \"value1\"}], ary[i*2]\n        assert_equal [\"tag.test\", t, {\"key\" => \"value2\"}], ary[i*2+1]\n      end\n    end\n\n    test '#write is called per time ranges after timekey_wait, and buffer chunk is purged' do\n      Timecop.freeze( Time.parse('2016-04-13 14:04:00 +0900') )\n\n      @i.thread_wait_until_start\n\n      ary = []\n      metachecks = []\n\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| chunk.read.split(\"\\n\").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (chunk.metadata.timekey.to_i <= e[1].to_i && e[1].to_i < chunk.metadata.timekey.to_i + 30) } }\n\n      r = {}\n      (0...10).each do |i|\n        r[\"key#{i}\"] = \"value #{i}\"\n      end\n      ts = [\n        Fluent::EventTime.parse('2016-04-13 14:03:21 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:23 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:29 +0900'),\n        Fluent::EventTime.parse('2016-04-13 14:03:30 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:33 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:38 +0900'),\n        Fluent::EventTime.parse('2016-04-13 14:03:43 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:49 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:51 +0900'),\n        Fluent::EventTime.parse('2016-04-13 14:04:00 +0900'), Fluent::EventTime.parse('2016-04-13 14:04:01 +0900'),\n      ]\n      events = [\n        [\"test.tag.1\", ts[0], r], # range 14:03:00 - 03:29\n        [\"test.tag.2\", ts[1], r],\n        [\"test.tag.1\", ts[2], r],\n        [\"test.tag.1\", ts[3], r], # range 14:03:30 - 04:00\n        [\"test.tag.1\", ts[4], r],\n        [\"test.tag.1\", ts[5], r],\n        [\"test.tag.1\", ts[6], r],\n        [\"test.tag.1\", ts[7], r],\n        [\"test.tag.2\", ts[8], r],\n        [\"test.tag.1\", ts[9], r], # range 14:04:00 - 04:29\n        [\"test.tag.2\", ts[10], r],\n      ]\n\n      assert_equal 0, @i.write_count\n\n      @i.interrupt_flushes\n\n      events.shuffle.each do |tag, time, record|\n        @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))\n      end\n      assert{ @i.buffer.stage.size == 3 }\n      assert{ @i.write_count == 0 }\n\n      @i.enqueue_thread_wait\n\n      waiting(4){ sleep 0.1 until @i.write_count > 0 }\n\n      assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 }\n\n      waiting(4){ sleep 0.1 until ary.size == 3 }\n\n      assert_equal 3, ary.size\n      assert_equal 2, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 1, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:04 +0900') )\n\n      @i.enqueue_thread_wait\n\n      assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:06 +0900') )\n\n      @i.enqueue_thread_wait\n      waiting(4){ sleep 0.1 until @i.write_count > 1 }\n\n      assert{ @i.buffer.stage.size == 1 && @i.write_count == 2 }\n\n      assert_equal 9, ary.size\n      assert_equal 7, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 2, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      assert metachecks.all?{|e| e }\n    end\n\n    test 'flush_at_shutdown work well when plugin is shutdown' do\n      Timecop.freeze( Time.parse('2016-04-13 14:04:00 +0900') )\n\n      @i.thread_wait_until_start\n\n      ary = []\n      metachecks = []\n\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk|\n        chunk.read.split(\"\\n\").reject{|l| l.empty? }.each{|data|\n          e = JSON.parse(data)\n          ary << e\n          metachecks << (chunk.metadata.timekey.to_i <= e[1].to_i && e[1].to_i < chunk.metadata.timekey.to_i + 30)\n        }\n      }\n\n      r = {}\n      (0...10).each do |i|\n        r[\"key#{i}\"] = \"value #{i}\"\n      end\n      ts = [\n        Fluent::EventTime.parse('2016-04-13 14:03:21 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:23 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:29 +0900'),\n        Fluent::EventTime.parse('2016-04-13 14:03:30 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:33 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:38 +0900'),\n        Fluent::EventTime.parse('2016-04-13 14:03:43 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:49 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:51 +0900'),\n        Fluent::EventTime.parse('2016-04-13 14:04:00 +0900'), Fluent::EventTime.parse('2016-04-13 14:04:01 +0900'),\n      ]\n      events = [\n        [\"test.tag.1\", ts[0], r], # range 14:03:00 - 03:29\n        [\"test.tag.2\", ts[1], r],\n        [\"test.tag.1\", ts[2], r],\n        [\"test.tag.1\", ts[3], r], # range 14:03:30 - 04:00\n        [\"test.tag.1\", ts[4], r],\n        [\"test.tag.1\", ts[5], r],\n        [\"test.tag.1\", ts[6], r],\n        [\"test.tag.1\", ts[7], r],\n        [\"test.tag.2\", ts[8], r],\n        [\"test.tag.1\", ts[9], r], # range 14:04:00 - 04:29\n        [\"test.tag.2\", ts[10], r],\n      ]\n\n      assert_equal 0, @i.write_count\n\n      @i.interrupt_flushes\n\n      events.shuffle.each do |tag, time, record|\n        @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))\n      end\n      assert{ @i.buffer.stage.size == 3 }\n      assert{ @i.write_count == 0 }\n\n      @i.enqueue_thread_wait\n\n      waiting(4){ sleep 0.1 until @i.write_count > 0 }\n\n      assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:04 +0900') )\n\n      @i.enqueue_thread_wait\n\n      assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:06 +0900') )\n\n      @i.enqueue_thread_wait\n      waiting(4){ sleep 0.1 until @i.write_count > 1 }\n\n      assert{ @i.buffer.stage.size == 1 && @i.write_count == 2 }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') )\n\n      waiting(4){ sleep 0.1 until ary.size == 9 }\n      assert_equal 9, ary.size\n\n      @i.stop\n      @i.before_shutdown\n      @i.shutdown\n      @i.after_shutdown\n\n      waiting(4){ sleep 0.1 until @i.write_count > 2 && ary.size == 11 }\n\n      assert_equal 11, ary.size\n      assert metachecks.all?{|e| e }\n    end\n  end\n\n  sub_test_case 'buffered output with large timekey and small timekey_wait' do\n    test 'writes event in proper interval' do\n      chunk_key = 'time'\n      hash = {\n        'timekey_zone' => '+0900',\n        'timekey' => 86400, # per 1 day\n        'timekey_wait' => 10, # 10 seconds delay for flush\n        'flush_thread_count' => 1,\n        'flush_thread_burst_interval' => 0.01,\n      }\n\n      with_timezone(\"UTC-9\") do\n        Timecop.freeze(Time.parse('2019-02-08 00:01:00 +0900'))\n        @i = create_output(:buffered)\n        # timezone is set\n        @i.configure(config_element('ROOT', '', {}, [config_element('buffer',chunk_key,hash)]))\n        @i.start\n        @i.after_start\n        @i.thread_wait_until_start\n        assert_equal(0, @i.write_count)\n        @i.interrupt_flushes\n\n        events = [\n          [event_time('2019-02-08 00:02:00 +0900'), { \"message\" => \"foobar\" }]\n        ]\n        @i.emit_events(\"test.tag\", Fluent::ArrayEventStream.new(events))\n        @i.enqueue_thread_wait\n        assert_equal(0, @i.write_count)\n\n        Timecop.freeze(Time.parse('2019-02-09 00:00:08 +0900'))\n        @i.enqueue_thread_wait\n        assert_equal(0, @i.write_count)\n\n        Timecop.freeze(Time.parse('2019-02-09 00:00:12 +0900'))\n        # write should be called in few seconds since\n        # running interval of enqueue thread is timekey_wait / 11.0.\n        waiting(5){ sleep 0.1 until @i.write_count == 1 }\n      end\n    end\n  end\n\n  sub_test_case 'buffered output feature with tag key' do\n    setup do\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 10,\n        'flush_thread_count' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'chunk_limit_size' => 1024,\n        'queued_chunks_limit_size' => 100\n      }\n      @i = create_output(:buffered)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.start\n      @i.after_start\n    end\n\n    test 'default flush_mode is set to :interval' do\n      assert_equal :interval, @i.instance_eval{ @flush_mode }\n    end\n\n    test '#start creates enqueue thread and flush threads' do\n      @i.thread_wait_until_start\n\n      assert @i.thread_exist?(:flush_thread_0)\n      assert @i.thread_exist?(:enqueue_thread)\n    end\n\n    test '#format is called for each event streams' do\n      ary = []\n      @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }\n\n      t = event_time()\n      es = Fluent::ArrayEventStream.new([ [t, {\"key\" => \"value1\"}], [t, {\"key\" => \"value2\"}] ])\n\n      5.times do\n        @i.emit_events('tag.test', es)\n      end\n\n      assert_equal 10, ary.size\n      5.times do |i|\n        assert_equal [\"tag.test\", t, {\"key\" => \"value1\"}], ary[i*2]\n        assert_equal [\"tag.test\", t, {\"key\" => \"value2\"}], ary[i*2+1]\n      end\n    end\n\n    test '#write is called per tags, per flush_interval & chunk sizes, and buffer chunk is purged' do\n      Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )\n\n      ary = []\n      metachecks = []\n\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| chunk.read.split(\"\\n\").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (chunk.metadata.tag == e[0]) } }\n\n      @i.thread_wait_until_start\n\n      r = {}\n      (0...10).each do |i|\n        r[\"key#{i}\"] = \"value #{i}\"\n      end\n      ts = [\n        event_time('2016-04-13 14:03:21 +0900'), event_time('2016-04-13 14:03:23 +0900'), event_time('2016-04-13 14:03:29 +0900'),\n        event_time('2016-04-13 14:03:30 +0900'), event_time('2016-04-13 14:03:33 +0900'), event_time('2016-04-13 14:03:38 +0900'),\n        event_time('2016-04-13 14:03:43 +0900'), event_time('2016-04-13 14:03:49 +0900'), event_time('2016-04-13 14:03:51 +0900'),\n        event_time('2016-04-13 14:04:00 +0900'), event_time('2016-04-13 14:04:01 +0900'),\n      ]\n      # size of a event is 197\n      events = [\n        [\"test.tag.1\", ts[0], r],\n        [\"test.tag.2\", ts[1], r],\n        [\"test.tag.1\", ts[2], r],\n        [\"test.tag.1\", ts[3], r],\n        [\"test.tag.1\", ts[4], r],\n        [\"test.tag.1\", ts[5], r],\n        [\"test.tag.1\", ts[6], r],\n        [\"test.tag.1\", ts[7], r],\n        [\"test.tag.2\", ts[8], r],\n        [\"test.tag.1\", ts[9], r],\n        [\"test.tag.2\", ts[10], r],\n      ]\n\n      assert_equal 0, @i.write_count\n\n      @i.interrupt_flushes\n\n      events.shuffle.each do |tag, time, record|\n        @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))\n      end\n      assert{ @i.buffer.stage.size == 2 } # test.tag.1 x1, test.tag.2 x1\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 0\n      end\n\n      assert{ @i.buffer.stage.size == 2 }\n      assert{ @i.write_count == 1 }\n      assert{ @i.buffer.queue.size == 0 }\n\n      # events fulfills a chunk (and queued immediately)\n      assert_equal 5, ary.size\n      assert_equal 5, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 0, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )\n\n      @i.enqueue_thread_wait\n\n      assert{ @i.buffer.stage.size == 2 }\n\n      # to trigger try_flush with flush_thread_burst_interval\n      Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:15 +0900') )\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      assert{ @i.buffer.stage.size == 0 }\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 2\n      end\n\n      assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 }\n\n      assert_equal 11, ary.size\n      assert_equal 8, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 3, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      assert metachecks.all?{|e| e }\n    end\n\n    test 'flush_at_shutdown work well when plugin is shutdown' do\n      Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )\n\n      ary = []\n      metachecks = []\n\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| chunk.read.split(\"\\n\").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (chunk.metadata.tag == e[0]) } }\n\n      @i.thread_wait_until_start\n\n      r = {}\n      (0...10).each do |i|\n        r[\"key#{i}\"] = \"value #{i}\"\n      end\n      ts = [\n        event_time('2016-04-13 14:03:21 +0900'), event_time('2016-04-13 14:03:23 +0900'), event_time('2016-04-13 14:03:29 +0900'),\n        event_time('2016-04-13 14:03:30 +0900'), event_time('2016-04-13 14:03:33 +0900'), event_time('2016-04-13 14:03:38 +0900'),\n        event_time('2016-04-13 14:03:43 +0900'), event_time('2016-04-13 14:03:49 +0900'), event_time('2016-04-13 14:03:51 +0900'),\n        event_time('2016-04-13 14:04:00 +0900'), event_time('2016-04-13 14:04:01 +0900'),\n      ]\n      # size of a event is 197\n      events = [\n        [\"test.tag.1\", ts[0], r],\n        [\"test.tag.2\", ts[1], r],\n        [\"test.tag.1\", ts[2], r],\n        [\"test.tag.1\", ts[3], r],\n        [\"test.tag.1\", ts[4], r],\n        [\"test.tag.1\", ts[5], r],\n        [\"test.tag.1\", ts[6], r],\n        [\"test.tag.1\", ts[7], r],\n        [\"test.tag.2\", ts[8], r],\n        [\"test.tag.1\", ts[9], r],\n        [\"test.tag.2\", ts[10], r],\n      ]\n\n      assert_equal 0, @i.write_count\n\n      @i.interrupt_flushes\n\n      events.shuffle.each do |tag, time, record|\n        @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))\n      end\n      assert{ @i.buffer.stage.size == 2 } # test.tag.1 x1, test.tag.2 x1\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 0\n      end\n\n      assert{ @i.buffer.stage.size == 2 }\n      assert{ @i.write_count == 1 }\n      assert{ @i.buffer.queue.size == 0 }\n\n      # events fulfills a chunk (and queued immediately)\n      assert_equal 5, ary.size\n      assert_equal 5, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 0, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      @i.stop\n      @i.before_shutdown\n      @i.shutdown\n      @i.after_shutdown\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 1\n      end\n\n      assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 3 }\n\n      assert_equal 11, ary.size\n      assert_equal 8, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 3, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      assert metachecks.all?{|e| e }\n    end\n  end\n\n  sub_test_case 'buffered output feature with variables' do\n    setup do\n      chunk_key = 'name,service'\n      hash = {\n        'flush_interval' => 10,\n        'flush_thread_count' => 1,\n        'flush_thread_interval' => 0.1,\n        'flush_thread_burst_interval' => 0.1,\n        'chunk_limit_size' => 1024,\n      }\n      @i = create_output(:buffered)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.start\n      @i.after_start\n    end\n\n    test 'default flush_mode is set to :interval' do\n      assert_equal :interval, @i.instance_eval{ @flush_mode }\n    end\n\n    test '#start creates enqueue thread and flush threads' do\n      @i.thread_wait_until_start\n\n      assert @i.thread_exist?(:flush_thread_0)\n      assert @i.thread_exist?(:enqueue_thread)\n    end\n\n    test '#format is called for each event streams' do\n      ary = []\n      @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }\n\n      t = event_time()\n      es = Fluent::ArrayEventStream.new([\n        [t, {\"key\" => \"value1\", \"name\" => \"moris\", \"service\" => \"a\"}],\n        [t, {\"key\" => \"value2\", \"name\" => \"moris\", \"service\" => \"b\"}],\n      ])\n\n      5.times do\n        @i.emit_events('tag.test', es)\n      end\n\n      assert_equal 10, ary.size\n      5.times do |i|\n        assert_equal [\"tag.test\", t, {\"key\" => \"value1\", \"name\" => \"moris\", \"service\" => \"a\"}], ary[i*2]\n        assert_equal [\"tag.test\", t, {\"key\" => \"value2\", \"name\" => \"moris\", \"service\" => \"b\"}], ary[i*2+1]\n      end\n    end\n\n    test '#write is called per value combination of variables, per flush_interval & chunk sizes, and buffer chunk is purged' do\n      Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )\n\n      ary = []\n      metachecks = []\n\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| chunk.read.split(\"\\n\").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (e[2][\"name\"] == chunk.metadata.variables[:name] && e[2][\"service\"] == chunk.metadata.variables[:service]) } }\n\n      @i.thread_wait_until_start\n\n      # size of a event is 195\n      dummy_data = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"\n      events = [\n        [\"test.tag.1\", event_time('2016-04-13 14:03:21 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1) xxx-a (6 events)\n        [\"test.tag.2\", event_time('2016-04-13 14:03:23 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}], #(2) yyy-a (3 events)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:29 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:30 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:33 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:38 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"b\"}], #(3) xxx-b (2 events)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:43 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:49 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"b\"}], #(3)\n        [\"test.tag.2\", event_time('2016-04-13 14:03:51 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}], #(2)\n        [\"test.tag.1\", event_time('2016-04-13 14:04:00 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1)\n        [\"test.tag.2\", event_time('2016-04-13 14:04:01 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}], #(2)\n      ]\n\n      assert_equal 0, @i.write_count\n\n      @i.interrupt_flushes\n\n      events.shuffle.each do |tag, time, record|\n        @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))\n      end\n      assert{ @i.buffer.stage.size == 3 }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 0\n      end\n\n      assert{ @i.buffer.stage.size == 3 }\n      assert{ @i.write_count == 1 }\n      assert{ @i.buffer.queue.size == 0 }\n\n      # events fulfills a chunk (and queued immediately)\n      assert_equal 5, ary.size\n      assert_equal 5, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 0, ary.count{|e| e[0] == \"test.tag.2\" }\n      assert ary[0...5].all?{|e| e[2][\"name\"] == \"xxx\" && e[2][\"service\"] == \"a\" }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )\n\n      @i.enqueue_thread_wait\n\n      assert{ @i.buffer.stage.size == 3 }\n\n      # to trigger try_flush with flush_thread_interval\n      Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )\n      @i.enqueue_thread_wait\n\n      assert{ @i.buffer.stage.size == 0 }\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 1\n      end\n\n      assert{ @i.buffer.stage.size == 0 && @i.write_count == 4 }\n\n      assert_equal 11, ary.size\n      assert_equal 8, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 3, ary.count{|e| e[0] == \"test.tag.2\" }\n      assert_equal 6, ary.count{|e| e[2][\"name\"] == \"xxx\" && e[2][\"service\"] == \"a\" }\n      assert_equal 3, ary.count{|e| e[2][\"name\"] == \"yyy\" && e[2][\"service\"] == \"a\" }\n      assert_equal 2, ary.count{|e| e[2][\"name\"] == \"xxx\" && e[2][\"service\"] == \"b\" }\n\n      assert metachecks.all?{|e| e }\n    end\n\n    test 'flush_at_shutdown work well when plugin is shutdown' do\n      Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )\n\n      ary = []\n      metachecks = []\n\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| chunk.read.split(\"\\n\").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (e[2][\"name\"] == chunk.metadata.variables[:name] && e[2][\"service\"] == chunk.metadata.variables[:service]) } }\n\n      @i.thread_wait_until_start\n\n      # size of a event is 195\n      dummy_data = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"\n      events = [\n        [\"test.tag.1\", event_time('2016-04-13 14:03:21 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1) xxx-a (6 events)\n        [\"test.tag.2\", event_time('2016-04-13 14:03:23 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}], #(2) yyy-a (3 events)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:29 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:30 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:33 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:38 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"b\"}], #(3) xxx-b (2 events)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:43 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1)\n        [\"test.tag.1\", event_time('2016-04-13 14:03:49 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"b\"}], #(3)\n        [\"test.tag.2\", event_time('2016-04-13 14:03:51 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}], #(2)\n        [\"test.tag.1\", event_time('2016-04-13 14:04:00 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}], #(1)\n        [\"test.tag.2\", event_time('2016-04-13 14:04:01 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}], #(2)\n      ]\n\n      assert_equal 0, @i.write_count\n\n      @i.interrupt_flushes\n\n      events.shuffle.each do |tag, time, record|\n        @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))\n      end\n      assert{ @i.buffer.stage.size == 3 }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 0\n      end\n\n      assert{ @i.buffer.stage.size == 3 }\n      assert{ @i.write_count == 1 }\n      assert{ @i.buffer.queue.size == 0 }\n\n      # events fulfills a chunk (and queued immediately)\n      assert_equal 5, ary.size\n      assert_equal 5, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 0, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      @i.stop\n      @i.before_shutdown\n      @i.shutdown\n      @i.after_shutdown\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 1\n      end\n\n      assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 4 }\n\n      assert_equal 11, ary.size\n      assert_equal 8, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 3, ary.count{|e| e[0] == \"test.tag.2\" }\n      assert_equal 6, ary.count{|e| e[2][\"name\"] == \"xxx\" && e[2][\"service\"] == \"a\" }\n      assert_equal 3, ary.count{|e| e[2][\"name\"] == \"yyy\" && e[2][\"service\"] == \"a\" }\n      assert_equal 2, ary.count{|e| e[2][\"name\"] == \"xxx\" && e[2][\"service\"] == \"b\" }\n\n      assert metachecks.all?{|e| e }\n    end\n  end\n\n  sub_test_case 'buffered output feature with many keys' do\n    test 'default flush mode is set to :interval if keys does not include time' do\n      chunk_key = 'name,service,tag'\n      hash = {\n        'flush_interval' => 10,\n        'flush_thread_count' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'chunk_limit_size' => 1024,\n      }\n      @i = create_output(:buffered)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.start\n      @i.after_start\n\n      assert_equal :interval, @i.instance_eval{ @flush_mode }\n    end\n\n    test 'default flush mode is set to :lazy if keys includes time' do\n      chunk_key = 'name,service,tag,time'\n      hash = {\n        'timekey' => 60,\n        'flush_interval' => 10,\n        'flush_thread_count' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'chunk_limit_size' => 1024,\n      }\n      @i = create_output(:buffered)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.start\n      @i.after_start\n\n      assert_equal :lazy, @i.instance_eval{ @flush_mode }\n    end\n  end\n\n  sub_test_case 'buffered output feature with delayed commit' do\n    setup do\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 10,\n        'flush_thread_count' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'delayed_commit_timeout' => 30,\n        'chunk_limit_size' => 1024,\n      }\n      @i = create_output(:delayed)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.start\n      @i.after_start\n      @i.log = Fluent::Test::TestLogger.new\n    end\n\n    test '#format is called for each event streams' do\n      ary = []\n      @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }\n\n      t = event_time()\n      es = Fluent::ArrayEventStream.new([\n        [t, {\"key\" => \"value1\", \"name\" => \"moris\", \"service\" => \"a\"}],\n        [t, {\"key\" => \"value2\", \"name\" => \"moris\", \"service\" => \"b\"}],\n      ])\n\n      5.times do\n        @i.emit_events('tag.test', es)\n      end\n\n      assert_equal 10, ary.size\n      5.times do |i|\n        assert_equal [\"tag.test\", t, {\"key\" => \"value1\", \"name\" => \"moris\", \"service\" => \"a\"}], ary[i*2]\n        assert_equal [\"tag.test\", t, {\"key\" => \"value2\", \"name\" => \"moris\", \"service\" => \"b\"}], ary[i*2+1]\n      end\n    end\n\n    test '#try_write is called per flush, buffer chunk is not purged until #commit_write is called' do\n      Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )\n\n      ary = []\n      metachecks = []\n      chunks = []\n\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:try_write) do |chunk|\n        chunks << chunk\n        chunk.read.split(\"\\n\").reject{|l| l.empty? }.each do |data|\n          e = JSON.parse(data)\n          ary << e\n          metachecks << (e[0] == chunk.metadata.tag)\n        end\n      end\n\n      @i.thread_wait_until_start\n\n      # size of a event is 195\n      dummy_data = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"\n      events = [\n        [\"test.tag.1\", event_time('2016-04-13 14:03:21 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.2\", event_time('2016-04-13 14:03:23 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:29 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:30 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:33 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:38 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"b\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:43 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:49 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"b\"}],\n        [\"test.tag.2\", event_time('2016-04-13 14:03:51 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:04:00 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.2\", event_time('2016-04-13 14:04:01 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}],\n      ]\n\n      assert_equal 0, @i.write_count\n\n      @i.interrupt_flushes\n\n      events.shuffle.each do |tag, time, record|\n        @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))\n      end\n      assert{ @i.buffer.stage.size == 2 }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 0\n      end\n\n      assert{ @i.buffer.stage.size == 2 }\n      assert{ @i.write_count == 1 }\n      assert{ @i.buffer.queue.size == 0 }\n      assert{ @i.buffer.dequeued.size == 1 }\n\n      # events fulfills a chunk (and queued immediately)\n      assert_equal 5, ary.size\n      assert_equal 5, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 0, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      assert_equal 1, chunks.size\n      assert !chunks.first.empty?\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )\n\n      @i.enqueue_thread_wait\n\n      assert{ @i.buffer.stage.size == 2 }\n\n      # to trigger try_flush with flush_thread_burst_interval\n      Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      assert{ @i.buffer.stage.size == 0 }\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 1\n      end\n\n      assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 }\n      assert{ @i.buffer.dequeued.size == 3 }\n\n      assert_equal 11, ary.size\n      assert_equal 8, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 3, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      assert_equal 3, chunks.size\n      assert chunks.all?{|c| !c.empty? }\n\n      assert metachecks.all?{|e| e }\n\n      @i.commit_write(chunks[0].unique_id)\n      assert{ @i.buffer.dequeued.size == 2 }\n      assert chunks[0].empty?\n\n      @i.commit_write(chunks[1].unique_id)\n      assert{ @i.buffer.dequeued.size == 1 }\n      assert chunks[1].empty?\n\n      @i.commit_write(chunks[2].unique_id)\n      assert{ @i.buffer.dequeued.size == 0 }\n      assert chunks[2].empty?\n\n      # no problem to commit chunks already committed\n      assert_nothing_raised do\n        @i.commit_write(chunks[2].unique_id)\n      end\n    end\n\n    test '#rollback_write and #try_rollback_write can rollback buffer chunks for delayed commit after timeout, and then be able to write it again' do\n      Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )\n\n      ary = []\n      metachecks = []\n      chunks = []\n\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:try_write) do |chunk|\n        chunks << chunk\n        chunk.read.split(\"\\n\").reject{|l| l.empty? }.each do |data|\n          e = JSON.parse(data)\n          ary << e\n          metachecks << (e[0] == chunk.metadata.tag)\n        end\n      end\n\n      @i.thread_wait_until_start\n\n      # size of a event is 195\n      dummy_data = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"\n      events = [\n        [\"test.tag.1\", event_time('2016-04-13 14:03:21 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.2\", event_time('2016-04-13 14:03:23 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:29 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:30 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:33 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:38 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"b\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:43 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:49 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"b\"}],\n        [\"test.tag.2\", event_time('2016-04-13 14:03:51 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:04:00 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.2\", event_time('2016-04-13 14:04:01 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}],\n      ]\n\n      assert_equal 0, @i.write_count\n\n      @i.interrupt_flushes\n\n      events.shuffle.each do |tag, time, record|\n        @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))\n      end\n      assert{ @i.buffer.stage.size == 2 }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 0\n      end\n\n      assert{ @i.buffer.stage.size == 2 }\n      assert{ @i.write_count == 1 }\n      assert{ @i.buffer.queue.size == 0 }\n      assert{ @i.buffer.dequeued.size == 1 }\n\n      # events fulfills a chunk (and queued immediately)\n      assert_equal 5, ary.size\n      assert_equal 5, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 0, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      assert_equal 1, chunks.size\n      assert !chunks.first.empty?\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )\n\n      @i.enqueue_thread_wait\n\n      assert{ @i.buffer.stage.size == 2 }\n\n      # to trigger try_flush with flush_thread_burst_interval\n      Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      assert{ @i.buffer.stage.size == 0 }\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 2\n      end\n\n      assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 }\n      assert{ @i.buffer.dequeued.size == 3 }\n\n      assert_equal 11, ary.size\n      assert_equal 8, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 3, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      assert_equal 3, chunks.size\n      assert chunks.all?{|c| !c.empty? }\n\n      assert metachecks.all?{|e| e }\n\n      @i.interrupt_flushes\n\n      @i.rollback_write(chunks[2].unique_id)\n\n      assert{ @i.buffer.dequeued.size == 2 }\n      assert{ @i.buffer.queue.size == 1 && @i.buffer.queue.first.unique_id == chunks[2].unique_id }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:15 +0900') )\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 3\n      end\n\n      assert{ @i.write_count == 4 }\n      assert{ @i.rollback_count == 1 }\n      assert{ @i.instance_eval{ @dequeued_chunks.size } == 3 }\n      assert{ @i.buffer.dequeued.size == 3 }\n      assert{ @i.buffer.queue.size == 0 }\n\n      assert_equal 4, chunks.size\n      assert chunks[2].unique_id == chunks[3].unique_id\n\n      ary.reject!{|e| true }\n      chunks.reject!{|e| true }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:46 +0900') )\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4) do\n        Thread.pass until @i.rollback_count == 4\n      end\n\n      assert{ chunks[0...3].all?{|c| !c.empty? } }\n\n      # rollback is in progress, but some may be flushed again in retry state, after rollback\n      # retry.next_time is 14:04:49\n      Timecop.freeze( Time.parse('2016-04-13 14:04:51 +0900') )\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4) do\n        Thread.pass until @i.write_count == 7\n      end\n\n      assert{ @i.write_count == 7 }\n      assert_equal 11, ary.size\n      assert_equal 8, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 3, ary.count{|e| e[0] == \"test.tag.2\" }\n      assert{ chunks.size == 3 }\n      assert{ chunks.all?{|c| !c.empty? } }\n\n      chunks.each{|c| @i.commit_write(c.unique_id) }\n      assert{ chunks.all?{|c| c.empty? } }\n\n      assert{ @i.buffer.dequeued.size == 0 }\n    end\n\n    test '#try_rollback_all will be called for all waiting chunks after shutdown' do\n      Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )\n\n      ary = []\n      metachecks = []\n      chunks = []\n\n      @i.register(:format){|tag,time,record| [tag,time,record].to_json + \"\\n\" }\n      @i.register(:try_write) do |chunk|\n        chunks << chunk\n        chunk.read.split(\"\\n\").reject{|l| l.empty? }.each do |data|\n          e = JSON.parse(data)\n          ary << e\n          metachecks << (e[0] == chunk.metadata.tag)\n        end\n      end\n\n      @i.thread_wait_until_start\n\n      # size of a event is 195\n      dummy_data = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"\n      events = [\n        [\"test.tag.1\", event_time('2016-04-13 14:03:21 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.2\", event_time('2016-04-13 14:03:23 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:29 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:30 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:33 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:38 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"b\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:43 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:03:49 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"b\"}],\n        [\"test.tag.2\", event_time('2016-04-13 14:03:51 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}],\n        [\"test.tag.1\", event_time('2016-04-13 14:04:00 +0900'), {\"data\" => dummy_data, \"name\" => \"xxx\", \"service\" => \"a\"}],\n        [\"test.tag.2\", event_time('2016-04-13 14:04:01 +0900'), {\"data\" => dummy_data, \"name\" => \"yyy\", \"service\" => \"a\"}],\n      ]\n\n      assert_equal 0, @i.write_count\n\n      @i.interrupt_flushes\n\n      events.shuffle.each do |tag, time, record|\n        @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))\n      end\n      assert{ @i.buffer.stage.size == 2 }\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 0\n      end\n\n      assert{ @i.buffer.stage.size == 2 }\n      assert{ @i.write_count == 1 }\n      assert{ @i.buffer.queue.size == 0 }\n      assert{ @i.buffer.dequeued.size == 1 }\n\n      # events fulfills a chunk (and queued immediately)\n      assert_equal 5, ary.size\n      assert_equal 5, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 0, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      assert_equal 1, chunks.size\n      assert !chunks.first.empty?\n\n      Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )\n\n      @i.enqueue_thread_wait\n\n      assert{ @i.buffer.stage.size == 2 }\n\n      # to trigger try_flush with flush_thread_burst_interval\n      Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') )\n      @i.enqueue_thread_wait\n      Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      assert{ @i.buffer.stage.size == 0 }\n\n      waiting(4) do\n        Thread.pass until @i.write_count > 2\n      end\n\n      assert{ @i.buffer.stage.size == 0 }\n      assert{ @i.buffer.queue.size == 0 }\n      assert{ @i.buffer.dequeued.size == 3 }\n      assert{ @i.write_count == 3 }\n      assert{ @i.rollback_count == 0 }\n\n      assert_equal 11, ary.size\n      assert_equal 8, ary.count{|e| e[0] == \"test.tag.1\" }\n      assert_equal 3, ary.count{|e| e[0] == \"test.tag.2\" }\n\n      assert{ chunks.size == 3 }\n      assert{ chunks.all?{|c| !c.empty? } }\n\n      @i.register(:shutdown_hook){ @i.commit_write(chunks[1].unique_id) }\n\n      @i.stop\n      @i.before_shutdown\n      @i.shutdown\n\n      assert{ @i.buffer.dequeued.size == 2 }\n      assert{ !chunks[0].empty? }\n      assert{ chunks[1].empty? }\n      assert{ !chunks[2].empty? }\n\n      @i.after_shutdown\n\n      assert{ @i.rollback_count == 2 }\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_output_as_buffered_backup.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/event'\nrequire 'fluent/error'\n\nrequire 'json'\nrequire 'time'\nrequire 'timeout'\nrequire 'timecop'\n\n\nclass BufferedOutputBackupTest < Test::Unit::TestCase\n  class BareOutput < Fluent::Plugin::Output\n    def register(name, &block)\n      instance_variable_set(\"@#{name}\", block)\n    end\n  end\n  class DummyOutput < BareOutput\n    def initialize\n      super\n      @process = nil\n      @format = nil\n      @write = nil\n      @try_write = nil\n    end\n    def prefer_buffered_processing\n      true\n    end\n    def prefer_delayed_commit\n      false\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n    def format(tag, time, record)\n      [tag, time.to_i, record].to_json + \"\\n\"\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n  class DummyOutputForSecondary < BareOutput\n    def initialize\n      super\n      @process = nil\n      @format = nil\n      @write = nil\n      @try_write = nil\n    end\n    def prefer_buffered_processing\n      true\n    end\n    def prefer_delayed_commit\n      false\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n    def format(tag, time, record)\n      [tag, time.to_i, record].to_json + \"\\n\"\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n  class DummyAsyncOutputForSecondary < BareOutput\n    def initialize\n      super\n      @process = nil\n      @format = nil\n      @write = nil\n      @try_write = nil\n    end\n    def prefer_buffered_processing\n      true\n    end\n    def prefer_delayed_commit\n      true\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n    def format(tag, time, record)\n      [tag, time.to_i, record].to_json + \"\\n\"\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n\n  TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/../tmp/bu#{ENV['TEST_ENV_NUMBER']}\")\n\n  def create_output\n    DummyOutput.new\n  end\n  def create_metadata(timekey: nil, tag: nil, variables: nil)\n    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n  end\n  def waiting(seconds)\n    begin\n      Timeout.timeout(seconds) do\n        yield\n      end\n    rescue Timeout::Error\n      STDERR.print(*@i.log.out.logs)\n      raise\n    end\n  end\n\n  def dummy_event_stream\n    Fluent::ArrayEventStream.new([\n      [ event_time('2016-04-13 18:33:00'), {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data1\"} ],\n      [ event_time('2016-04-13 18:33:13'), {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data2\"} ],\n      [ event_time('2016-04-13 18:33:32'), {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data3\"} ],\n    ])\n  end\n\n  setup do\n    @i = create_output\n    FileUtils.rm_rf(TMP_DIR)\n    FileUtils.mkdir_p(TMP_DIR)\n\n    Fluent::Plugin.register_output('backup_output', DummyOutput)\n    Fluent::Plugin.register_output('backup_output2', DummyOutputForSecondary)\n    Fluent::Plugin.register_output('backup_async_output', DummyAsyncOutputForSecondary)\n  end\n\n  teardown do\n    if @i\n      @i.stop unless @i.stopped?\n      @i.before_shutdown unless @i.before_shutdown?\n      @i.shutdown unless @i.shutdown?\n      @i.after_shutdown unless @i.after_shutdown?\n      @i.close unless @i.closed?\n      @i.terminate unless @i.terminated?\n    end\n    Timecop.return\n  end\n\n  sub_test_case 'buffered output for broken chunks' do\n    def flush_chunks\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze(now)\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n      now = Time.parse('2016-04-13 18:33:32 -0700')\n      Timecop.freeze(now)\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4) { Thread.pass until @i.write_count > 0 }\n\n      assert { @i.write_count > 0 }\n      Timecop.freeze(now)\n      @i.flush_thread_wakeup\n    end\n\n    def wait_flush(target_file)\n      waiting(5) {\n        target_dir = File.join(File.dirname(target_file), \"*\")\n        while Dir.glob(target_dir).size.zero?\n          # Avoid to lose globbed entries on Windows in busy loop\n          sleep 0.1 if Fluent.windows?\n        end\n      }\n    end\n\n    data('unrecoverable error' => Fluent::UnrecoverableError,\n         'type error' => TypeError,\n         'argument error' => ArgumentError,\n         'no method error' => NoMethodError,\n         'msgpack unpack error' => MessagePack::UnpackError,\n         'encoding error' => EncodingError)\n    test 'backup chunk without secondary' do |error_class|\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do\n        id = 'backup_test'\n        hash = {\n          'flush_interval' => 1,\n          'flush_thread_burst_interval' => 0.1,\n        }\n        chunk_id = nil\n        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))\n        @i.register(:write) { |chunk|\n          chunk_id = chunk.unique_id\n          raise error_class, \"yay, your #write must fail\"\n        }\n\n        flush_chunks\n\n        target = \"#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log\"\n        wait_flush(target)\n        assert_true File.exist?(target)\n        logs = @i.log.out.logs\n        assert { logs.any? { |l| l.include?(\"got unrecoverable error in primary and no secondary\") } }\n      end\n    end\n\n    test 'backup chunk with same type secondary' do\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do\n        id = 'backup_test_with_same_secondary'\n        hash = {\n          'flush_interval' => 1,\n          'flush_thread_burst_interval' => 0.1,\n        }\n        chunk_id = nil\n        secconf = config_element('secondary','',{'@type' => 'backup_output'})\n        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))\n        @i.register(:write) { |chunk|\n          chunk_id = chunk.unique_id\n          raise Fluent::UnrecoverableError, \"yay, your #write must fail\"\n        }\n\n        flush_chunks\n\n        target = \"#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log\"\n        wait_flush(target)\n        assert_true File.exist?(target)\n        logs = @i.log.out.logs\n        assert { logs.any? { |l| l.include?(\"got unrecoverable error in primary and secondary type is same as primary\") } }\n      end\n    end\n\n    test 'create directory' do\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do\n        id = 'backup_test_with_same_secondary'\n        hash = { 'flush_interval' => 1, 'flush_thread_burst_interval' => 0.1 }\n        chunk_id = nil\n        secconf = config_element('secondary', '', { '@type' => 'backup_output' })\n        @i.configure(config_element('ROOT', '', { '@id' => id }, [config_element('buffer', 'tag', hash), secconf]))\n        @i.register(:write) { |chunk|\n          chunk_id = chunk.unique_id\n          raise Fluent::UnrecoverableError, \"yay, your #write must fail\"\n        }\n\n        flush_chunks\n\n        target = \"#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log\"\n        target_dir = File.dirname(target)\n        wait_flush(target)\n\n        assert_path_exist(target_dir)\n        assert_equal '755', File.stat(target_dir).mode.to_s(8)[-3, 3]\n      end\n    end\n\n    test 'create directory with specific mode' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR, 'dir_permission' => '744') do\n        id = 'backup_test_with_same_secondary'\n        hash = { 'flush_interval' => 1, 'flush_thread_burst_interval' => 0.1 }\n        chunk_id = nil\n        secconf = config_element('secondary', '', { '@type' => 'backup_output' })\n        @i.configure(config_element('ROOT', '', { '@id' => id }, [config_element('buffer', 'tag', hash), secconf]))\n        @i.register(:write) { |chunk|\n          chunk_id = chunk.unique_id\n          raise Fluent::UnrecoverableError, \"yay, your #write must fail\"\n        }\n\n        flush_chunks\n\n        target = \"#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log\"\n        target_dir = File.dirname(target)\n        wait_flush(target)\n\n        assert_path_exist(target_dir)\n        assert_equal '744', File.stat(target_dir).mode.to_s(8)[-3, 3]\n      end\n    end\n\n    test 'backup chunk with different type secondary' do\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do\n        id = 'backup_test_with_diff_secondary'\n        hash = {\n          'flush_interval' => 1,\n          'flush_thread_burst_interval' => 0.1,\n        }\n        chunk_id = nil\n        secconf = config_element('secondary','',{'@type' => 'backup_output2'})\n        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))\n        @i.register(:write) { |chunk|\n          chunk_id = chunk.unique_id\n          raise Fluent::UnrecoverableError, \"yay, your #write must fail\"\n        }\n        @i.secondary.register(:write) { |chunk|\n          raise Fluent::UnrecoverableError, \"yay, your secondary #write must fail\"\n        }\n\n        flush_chunks\n\n        target = \"#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log\"\n        wait_flush(target)\n        assert_true File.exist?(target)\n        logs = @i.log.out.logs\n        assert { logs.any? { |l| l.include?(\"got unrecoverable error in primary. Skip retry and flush chunk to secondary\") } }\n        assert { logs.any? { |l| l.include?(\"got an error in secondary for unrecoverable error\") } }\n      end\n    end\n\n    test 'backup chunk with async secondary' do\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do\n        id = 'backup_test_with_diff_secondary'\n        hash = {\n          'flush_interval' => 1,\n          'flush_thread_burst_interval' => 0.1,\n        }\n        chunk_id = nil\n        secconf = config_element('secondary','',{'@type' => 'backup_async_output'})\n        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))\n        @i.register(:write) { |chunk|\n          chunk_id = chunk.unique_id\n          raise Fluent::UnrecoverableError, \"yay, your #write must fail\"\n        }\n\n        flush_chunks\n\n        target = \"#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log\"\n        wait_flush(target)\n        assert_true File.exist?(target)\n        logs = @i.log.out.logs\n        assert { logs.any? { |l| l.include?(\"got unrecoverable error in primary and secondary is async output\") } }\n      end\n    end\n\n    test 'chunk is thrown away when disable_chunk_backup is true' do\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do\n        id = 'backup_test'\n        hash = {\n          'flush_interval' => 1,\n          'flush_thread_burst_interval' => 0.1,\n          'disable_chunk_backup' => true\n        }\n        chunk_id = nil\n        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))\n        @i.register(:write) { |chunk|\n          chunk_id = chunk.unique_id\n          raise Fluent::UnrecoverableError, \"yay, your #write must fail\"\n        }\n\n        flush_chunks\n\n        target = \"#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log\"\n        assert_false File.exist?(target)\n        logs = @i.log.out.logs\n        assert { logs.any? { |l| l.include?(\"disable_chunk_backup is true\") } }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_output_as_buffered_compress.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin/compressable'\nrequire 'fluent/event'\n\nrequire 'timeout'\n\nmodule FluentPluginOutputAsBufferedCompressTest\n  class DummyBareOutput < Fluent::Plugin::Output\n    def register(name, &block)\n      instance_variable_set(\"@#{name}\", block)\n    end\n  end\n\n  class DummyAsyncOutput < DummyBareOutput\n    def initialize\n      super\n      @format = @write = nil\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n  end\n\n  class DummyAsyncOutputWithFormat < DummyBareOutput\n    def initialize\n      super\n      @format = nil\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n  end\n\n  def self.dummy_event_stream\n    Fluent::ArrayEventStream.new(\n      [\n        [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],\n        [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],\n        [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],\n      ]\n    )\n  end\nend\n\nclass BufferedOutputCompressTest < Test::Unit::TestCase\n  include Fluent::Plugin::Compressable\n\n  def create_output(type=:async)\n    case type\n    when :async then FluentPluginOutputAsBufferedCompressTest::DummyAsyncOutput.new\n    when :async_with_format then FluentPluginOutputAsBufferedCompressTest::DummyAsyncOutputWithFormat.new\n    else\n      raise ArgumentError, \"unknown type: #{type}\"\n    end\n  end\n\n  def waiting(seconds)\n    begin\n      Timeout.timeout(seconds) do\n        yield\n      end\n    rescue Timeout::Error\n      STDERR.print(*@i.log.out.logs)\n      raise\n    end\n  end\n\n  TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)\n\n  setup do\n    FileUtils.rm_r TMP_DIR rescue nil\n    FileUtils.mkdir_p TMP_DIR\n  end\n\n  teardown do\n    if @i\n      @i.stop unless @i.stopped?\n      @i.before_shutdown unless @i.before_shutdown?\n      @i.shutdown unless @i.shutdown?\n      @i.after_shutdown unless @i.after_shutdown?\n      @i.close unless @i.closed?\n      @i.terminate unless @i.terminated?\n    end\n  end\n\n  data(\n    :buffer_config,\n    [\n      config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),\n      config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),\n      config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),\n      config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),\n    ],\n  )\n  data(\n    :input_es,\n    [\n      FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,\n      # If already compressed data is incoming, it must be written as is (i.e. without decompressed).\n      # https://github.com/fluent/fluentd/issues/4146\n      Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),\n    ],\n  )\n  test 'call a standard format when output plugin adds data to chunk' do |data|\n    buffer_config = data[:buffer_config]\n    es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.\n\n    @i = create_output(:async)\n    @i.configure(config_element('ROOT','', {}, [buffer_config]))\n    @i.start\n    @i.after_start\n\n    io = StringIO.new\n    expected = es.dup.map { |t, r| [t, r] }\n    compressed_data = ''\n\n    assert_equal :gzip, @i.buffer.compress\n\n    @i.register(:write) do |c|\n      compressed_data = c.read(compressed: :gzip)\n      c.write_to(io)\n    end\n\n    @i.emit_events('tag', es)\n    @i.enqueue_thread_wait\n    @i.flush_thread_wakeup\n    waiting(4) { Thread.pass until io.size > 0 }\n\n    assert_equal expected, Fluent::MessagePackEventStream.new(decompress(compressed_data)).map { |t, r| [t, r] }\n    assert_equal expected, Fluent::MessagePackEventStream.new(io.string).map { |t, r| [t, r] }\n  end\n\n  data(\n    handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),\n    handle_stream_with_custom_format:  config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),\n    handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),\n    handle_stream_with_custom_format_and_file_chunk:  config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),\n  )\n  test 'call a custom format when output plugin adds data to chunk' do |buffer_config|\n    @i = create_output(:async_with_format)\n    @i.configure(config_element('ROOT','', {}, [buffer_config]))\n    @i.start\n    @i.after_start\n\n    io = StringIO.new\n    es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream\n    expected = es.map { |e| \"#{e[1]}\\n\" }.join # e[1] is record\n    compressed_data = ''\n\n    assert_equal :gzip, @i.buffer.compress\n\n    @i.register(:format) { |tag, time, record| \"#{record}\\n\" }\n    @i.register(:write) { |c|\n      compressed_data = c.read(compressed: :gzip)\n      c.write_to(io)\n    }\n\n    @i.emit_events('tag', es)\n    @i.enqueue_thread_wait\n    @i.flush_thread_wakeup\n    waiting(4) { sleep 0.1 until io.size > 0 }\n\n    assert_equal expected, decompress(compressed_data)\n    assert_equal expected, io.string\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_output_as_buffered_overflow.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/event'\n\nrequire 'json'\nrequire 'time'\nrequire 'timeout'\nrequire 'timecop'\n\nmodule FluentPluginOutputAsBufferedOverflowTest\n  class DummyBareOutput < Fluent::Plugin::Output\n    def register(name, &block)\n      instance_variable_set(\"@#{name}\", block)\n    end\n  end\n  class DummyAsyncOutput < DummyBareOutput\n    def initialize\n      super\n      @format = @write = nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n  end\nend\n\nclass BufferedOutputOverflowTest < Test::Unit::TestCase\n  def create_output\n    FluentPluginOutputAsBufferedOverflowTest::DummyAsyncOutput.new\n  end\n  def create_metadata(timekey: nil, tag: nil, variables: nil)\n    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n  end\n  def waiting(seconds)\n    begin\n      Timeout.timeout(seconds) do\n        yield\n      end\n    rescue Timeout::Error\n      logs = @i.log.out.logs\n      STDERR.print(*logs)\n      raise\n    end\n  end\n\n  teardown do\n    if @i\n      @i.stop unless @i.stopped?\n      @i.before_shutdown unless @i.before_shutdown?\n      @i.shutdown unless @i.shutdown?\n      @i.after_shutdown unless @i.after_shutdown?\n      @i.close unless @i.closed?\n      @i.terminate unless @i.terminated?\n    end\n    Timecop.return\n  end\n\n  sub_test_case 'buffered output with default configuration (throws exception for buffer overflow)' do\n    setup do\n      hash = {\n        'flush_mode' => 'lazy',\n        'flush_thread_burst_interval' => 0.01,\n        'chunk_limit_size' => 1024,\n        'total_limit_size' => 4096,\n      }\n      @i = create_output()\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','tag',hash)]))\n      @i.start\n      @i.after_start\n    end\n\n    test '#emit_events raises error when buffer is full' do\n      @i.register(:format){|tag, time, record| \"x\" * 128 } # 128bytes per record (x4 -> 512bytes)\n\n      es = Fluent::ArrayEventStream.new([\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n      ])\n\n      8.times do |i|\n        @i.emit_events(\"tag#{i}\", es)\n      end\n\n      assert !@i.buffer.storable?\n\n      assert_raise(Fluent::Plugin::Buffer::BufferOverflowError) do\n        @i.emit_events(\"tag9\", es)\n      end\n      logs = @i.log.out.logs\n      assert{ logs.any?{|line| line.include?(\"failed to write data into buffer by buffer overflow\") } }\n    end\n  end\n\n  sub_test_case 'buffered output configured with \"overflow_action block\"' do\n    setup do\n      hash = {\n        'flush_mode' => 'lazy',\n        'flush_thread_burst_interval' => 0.01,\n        'chunk_limit_size' => 1024,\n        'total_limit_size' => 4096,\n        'overflow_action' => \"block\",\n      }\n      @i = create_output()\n      @i.configure(config_element('ROOT','',{'log_level' => 'debug'},[config_element('buffer','tag',hash)]))\n      @i.start\n      @i.after_start\n    end\n\n    test '#emit_events blocks until any queues are flushed' do\n      failing = true\n      flushed_chunks = []\n      @i.register(:format){|tag, time, record| \"x\" * 128 } # 128bytes per record (x4 -> 512bytes)\n      @i.register(:write) do |chunk|\n        if failing\n          raise \"blocking\"\n        end\n        flushed_chunks << chunk\n      end\n\n      es = Fluent::ArrayEventStream.new([\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n      ])\n\n      4.times do |i|\n        @i.emit_events(\"tag#{i}\", es)\n      end\n\n      assert !@i.buffer.storable?\n\n      Thread.new do\n        sleep 1\n        failing = false\n      end\n\n      assert_nothing_raised do\n        @i.emit_events(\"tag9\", es)\n      end\n\n      assert !failing\n      assert{ flushed_chunks.size > 0 }\n\n      logs = @i.log.out.logs\n      assert{ logs.any?{|line| line.include?(\"failed to write data into buffer by buffer overflow\") } }\n      assert{ logs.any?{|line| line.include?(\"buffer.write is now blocking\") } }\n      assert{ logs.any?{|line| line.include?(\"retrying buffer.write after blocked operation\") } }\n    end\n  end\n\n  sub_test_case 'buffered output configured with \"overflow_action drop_oldest_chunk\"' do\n    setup do\n      hash = {\n        'flush_mode' => 'lazy',\n        'flush_thread_burst_interval' => 0.01,\n        'chunk_limit_size' => 1024,\n        'total_limit_size' => 4096,\n        'overflow_action' => \"drop_oldest_chunk\",\n      }\n      @i = create_output()\n      @i.configure(config_element('ROOT','',{'log_level' => 'debug'},[config_element('buffer','tag',hash)]))\n      @i.start\n      @i.after_start\n    end\n\n    test '#emit_events will success by dropping oldest chunk' do\n      failing = true\n      flushed_chunks = []\n      @i.register(:format){|tag, time, record| \"x\" * 128 } # 128bytes per record (x4 -> 512bytes)\n      @i.register(:write) do |chunk|\n        if failing\n          raise \"blocking\"\n        end\n        flushed_chunks << chunk\n      end\n\n      es = Fluent::ArrayEventStream.new([\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n      ])\n\n      4.times do |i|\n        @i.emit_events(\"tag#{i}\", es)\n      end\n\n      assert !@i.buffer.storable?\n\n      assert{ @i.buffer.queue[0].metadata.tag == \"tag0\" }\n      assert{ @i.buffer.queue[1].metadata.tag == \"tag1\" }\n\n      assert_nothing_raised do\n        @i.emit_events(\"tag9\", es)\n      end\n\n      assert failing\n      assert{ flushed_chunks.size == 0 }\n\n      assert{ @i.buffer.queue[0].metadata.tag == \"tag1\" }\n\n      logs = @i.log.out.logs\n      assert{ logs.any?{|line| line.include?(\"failed to write data into buffer by buffer overflow\") } }\n      assert{ logs.any?{|line| line.include?(\"dropping oldest chunk to make space after buffer overflow\") } }\n      assert{ @i.drop_oldest_chunk_count > 0 }\n    end\n\n    test '#emit_events raises OverflowError if all buffer spaces are used by staged chunks' do\n      @i.register(:format){|tag, time, record| \"x\" * 128 } # 128bytes per record (x4 -> 512bytes)\n\n      es = Fluent::ArrayEventStream.new([\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n        [event_time(), {\"message\" => \"test\"}],\n      ])\n\n      8.times do |i|\n        @i.emit_events(\"tag#{i}\", es)\n      end\n\n      assert !@i.buffer.storable?\n\n      assert{ @i.buffer.queue.size == 0 }\n      assert{ @i.buffer.stage.size == 8 }\n\n      assert_raise Fluent::Plugin::Buffer::BufferOverflowError do\n        @i.emit_events(\"tag9\", es)\n      end\n\n      logs = @i.log.out.logs\n      assert{ logs.any?{|line| line.include?(\"failed to write data into buffer by buffer overflow\") } }\n      assert{ logs.any?{|line| line.include?(\"no queued chunks to be dropped for drop_oldest_chunk\") } }\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_output_as_buffered_retries.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/event'\n\nrequire 'json'\nrequire 'time'\nrequire 'timeout'\nrequire 'timecop'\n\nmodule FluentPluginOutputAsBufferedRetryTest\n  class DummyBareOutput < Fluent::Plugin::Output\n    def register(name, &block)\n      instance_variable_set(\"@#{name}\", block)\n    end\n  end\n  class DummySyncOutput < DummyBareOutput\n    def initialize\n      super\n      @process = nil\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n  end\n  class DummyFullFeatureOutput < DummyBareOutput\n    def initialize\n      super\n      @prefer_buffered_processing = nil\n      @prefer_delayed_commit = nil\n      @process = nil\n      @format = nil\n      @write = nil\n      @try_write = nil\n    end\n    def prefer_buffered_processing\n      @prefer_buffered_processing ? @prefer_buffered_processing.call : false\n    end\n    def prefer_delayed_commit\n      @prefer_delayed_commit ? @prefer_delayed_commit.call : false\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n  class DummyFullFeatureOutput2 < DummyFullFeatureOutput\n    def prefer_buffered_processing; true; end\n    def prefer_delayed_commit; super; end\n    def format(tag, time, record); super; end\n    def write(chunk); super; end\n    def try_write(chunk); super; end\n  end\nend\n\nclass BufferedOutputRetryTest < Test::Unit::TestCase\n  def create_output(type=:full)\n    case type\n    when :bare then FluentPluginOutputAsBufferedRetryTest::DummyBareOutput.new\n    when :sync then FluentPluginOutputAsBufferedRetryTest::DummySyncOutput.new\n    when :full then FluentPluginOutputAsBufferedRetryTest::DummyFullFeatureOutput.new\n    else\n      raise ArgumentError, \"unknown type: #{type}\"\n    end\n  end\n  def create_metadata(timekey: nil, tag: nil, variables: nil)\n    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n  end\n  def waiting(seconds)\n    begin\n      Timeout.timeout(seconds) do\n        yield\n      end\n    rescue Timeout::Error\n      STDERR.print(*@i.log.out.logs)\n      raise\n    end\n  end\n  def dummy_event_stream\n    Fluent::ArrayEventStream.new([\n      [ event_time('2016-04-13 18:33:00'), {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data1\"} ],\n      [ event_time('2016-04-13 18:33:13'), {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data2\"} ],\n      [ event_time('2016-04-13 18:33:32'), {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data3\"} ],\n    ])\n  end\n  def get_log_time(msg, logs)\n    log_time = nil\n    log = logs.find{|l| l.include?(msg) }\n    if log && /^(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2} [-+]\\d{4}) \\[error\\]/ =~ log\n      log_time = Time.parse($1)\n    end\n    log_time\n  end\n\n  setup do\n    @i = create_output\n  end\n\n  teardown do\n    if @i\n      @i.stop unless @i.stopped?\n      @i.before_shutdown unless @i.before_shutdown?\n      @i.shutdown unless @i.shutdown?\n      @i.after_shutdown unless @i.after_shutdown?\n      @i.close unless @i.closed?\n      @i.terminate unless @i.terminated?\n    end\n    Timecop.return\n  end\n\n  sub_test_case 'buffered output for retries with exponential backoff' do\n    test 'exponential backoff is default strategy for retries' do\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_randomize' => false,\n        'queued_chunks_limit_size' => 100\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.start\n      @i.after_start\n\n      assert_equal :exponential_backoff, @i.buffer_config.retry_type\n      assert_equal 1, @i.buffer_config.retry_wait\n      assert_equal 2.0, @i.buffer_config.retry_exponential_backoff_base\n      assert !@i.buffer_config.retry_randomize\n\n      now = Time.parse('2016-04-13 18:17:00 -0700')\n      Timecop.freeze( now )\n\n      retry_state = @i.retry_state( @i.buffer_config.retry_randomize )\n      retry_state.step\n      assert_equal (1 * (2 ** 1)), (retry_state.next_time - now)\n      retry_state.step\n      assert_equal (1 * (2 ** 2)), (retry_state.next_time - now)\n      retry_state.step\n      assert_equal (1 * (2 ** 3)), (retry_state.next_time - now)\n      retry_state.step\n      assert_equal (1 * (2 ** 4)), (retry_state.next_time - now)\n    end\n\n    test 'does retries correctly when #write fails' do\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_randomize' => false,\n        'retry_max_interval' => 60 * 60,\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:32 -0700')\n      Timecop.freeze( now )\n\n      @i.enqueue_thread_wait\n\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 0 }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      now = @i.next_flush_time\n      Timecop.freeze( now )\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 1 }\n\n      assert{ @i.write_count > 1 }\n      assert{ @i.num_errors > 1 }\n    end\n\n    test 'max retry interval is limited by retry_max_interval' do\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_randomize' => false,\n        'retry_max_interval' => 60,\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:32 -0700')\n      Timecop.freeze( now )\n\n      @i.enqueue_thread_wait\n\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      10.times do\n        now = @i.next_flush_time\n        Timecop.freeze( now )\n        @i.flush_thread_wakeup\n        waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }\n\n        assert{ @i.write_count > prev_write_count }\n        assert{ @i.num_errors > prev_num_errors }\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n      end\n      # exponential backoff interval: 1 * 2 ** 10 == 1024\n      # but it should be limited by retry_max_interval=60\n      assert_equal 60, (@i.next_flush_time - now)\n    end\n\n    test 'output plugin give retries up by retry_timeout, and clear queue in buffer' do\n      written_tags = []\n\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_randomize' => false,\n        'retry_timeout' => 3600,\n        'queued_chunks_limit_size' => 100\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      15.times do |i| # large enough\n        now = @i.next_flush_time\n        # p({i: i, now: now, diff: (now - Time.now)})\n        # * if loop count is 12:\n        # {:i=>0, :now=>2016-04-13 18:33:32 -0700, :diff=>1.0}\n        # {:i=>1, :now=>2016-04-13 18:33:34 -0700, :diff=>2.0}\n        # {:i=>2, :now=>2016-04-13 18:33:38 -0700, :diff=>4.0}\n        # {:i=>3, :now=>2016-04-13 18:33:46 -0700, :diff=>8.0}\n        # {:i=>4, :now=>2016-04-13 18:34:02 -0700, :diff=>16.0}\n        # {:i=>5, :now=>2016-04-13 18:34:34 -0700, :diff=>32.0}\n        # {:i=>6, :now=>2016-04-13 18:35:38 -0700, :diff=>64.0}\n        # {:i=>7, :now=>2016-04-13 18:37:46 -0700, :diff=>128.0}\n        # {:i=>8, :now=>2016-04-13 18:42:02 -0700, :diff=>256.0}\n        # {:i=>9, :now=>2016-04-13 18:50:34 -0700, :diff=>512.0}\n        # {:i=>10, :now=>2016-04-13 19:07:38 -0700, :diff=>1024.0}\n        # {:i=>11, :now=>2016-04-13 19:33:31 -0700, :diff=>1553.0} # clear_queue!\n\n        Timecop.freeze( now )\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }\n\n        assert{ @i.write_count > prev_write_count }\n        assert{ @i.num_errors > prev_num_errors }\n\n        break if @i.buffer.queue.size == 0\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n\n        assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n      end\n      assert{ now >= first_failure + 3600 }\n\n      assert{ @i.buffer.stage.size == 0 }\n      assert{ written_tags.all?('test.tag.1') }\n\n      @i.emit_events(\"test.tag.3\", dummy_event_stream())\n\n      logs = @i.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"[error]: Hit limit for retries. dropping all chunks in the buffer queue.\") } }\n    end\n\n    test 'output plugin give retries up by retry_max_times, and clear queue in buffer' do\n      written_tags = []\n\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_randomize' => false,\n        'retry_max_times' => 10,\n        'queued_chunks_limit_size' => 100\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      _first_failure = @i.retry.start\n\n      chunks = @i.buffer.queue.dup\n\n      20.times do |i| # large times enough\n        now = @i.next_flush_time\n\n        Timecop.freeze( now )\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }\n\n        assert{ @i.write_count > prev_write_count }\n        assert{ @i.num_errors > prev_num_errors }\n\n        break if @i.buffer.queue.size == 0\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n\n        assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n      end\n      assert{ @i.buffer.stage.size == 0 }\n      assert{ written_tags.all?('test.tag.1') }\n\n      @i.emit_events(\"test.tag.3\", dummy_event_stream())\n\n      logs = @i.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"[error]: Hit limit for retries. dropping all chunks in the buffer queue.\") && l.include?(\"retry_times=10\") } }\n\n      assert{ @i.buffer.queue.size == 0 }\n      assert{ @i.buffer.stage.size == 1 }\n      assert{ chunks.all?{|c| c.empty? } }\n    end\n\n    test 'output plugin limits queued chunks via queued_chunks_limit_size' do\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_randomize' => false,\n        'retry_max_times' => 7,\n        'queued_chunks_limit_size' => 2,\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing) { true }\n      @i.register(:format) { |tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write) { |chunk| raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze(now)\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze(now)\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4) { Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert { @i.buffer.queue.size > 0 }\n      assert { @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert { @i.write_count > 0 }\n      assert { @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      20.times do |i| # large times enough\n        now = @i.next_flush_time\n\n        Timecop.freeze(now)\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4) { Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }\n\n        @i.emit_events(\"test.tag.1\", dummy_event_stream())\n        assert { @i.buffer.queue.size <= 2 }\n        assert { @i.buffer.stage.size == 1 } # all new data is stored into staged chunk\n\n        break if @i.buffer.queue.size == 0\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n      end\n    end\n  end\n\n  sub_test_case 'buffered output for retries with periodical retry' do\n    test 'periodical retries should retry to write in failing status per retry_wait' do\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_type' => :periodic,\n        'retry_wait' => 3,\n        'retry_randomize' => false,\n        'queued_chunks_limit_size' => 100\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:32 -0700')\n      Timecop.freeze( now )\n\n      @i.enqueue_thread_wait\n\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 0 }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      now = @i.next_flush_time\n      Timecop.freeze( now )\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 1 }\n\n      assert{ @i.write_count > 1 }\n      assert{ @i.num_errors > 1 }\n    end\n\n    test 'output plugin give retries up by retry_timeout, and clear queue in buffer' do\n      written_tags = []\n\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_type' => :periodic,\n        'retry_wait' => 30,\n        'retry_randomize' => false,\n        'retry_timeout' => 120,\n        'queued_chunks_limit_size' => 100\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      3.times do |i|\n        now = @i.next_flush_time\n\n        Timecop.freeze( now )\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }\n\n        assert{ @i.write_count > prev_write_count }\n        assert{ @i.num_errors > prev_num_errors }\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n      end\n\n      assert{ @i.next_flush_time >= first_failure + 120 }\n\n      assert{ @i.buffer.queue.size == 2 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n      assert{ @i.buffer.stage.size == 0 }\n\n      assert{ written_tags.all?('test.tag.1') }\n\n      chunks = @i.buffer.queue.dup\n\n      @i.emit_events(\"test.tag.3\", dummy_event_stream())\n\n      now = @i.next_flush_time\n      Timecop.freeze( now )\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }\n\n      logs = @i.log.out.logs\n\n      target_time = Time.parse(\"2016-04-13 18:35:31 -0700\")\n      target_msg = \"[error]: Hit limit for retries. dropping all chunks in the buffer queue.\"\n      assert{ logs.any?{|l| l.include?(target_msg) } }\n\n      log_time = get_log_time(target_msg, logs)\n      assert_equal target_time.localtime, log_time.localtime\n\n      assert{ @i.buffer.queue.size == 0 }\n      assert{ @i.buffer.stage.size == 1 }\n      assert{ chunks.all?{|c| c.empty? } }\n    end\n\n    test 'retry_max_times can limit maximum times for retries' do\n      written_tags = []\n\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_type' => :periodic,\n        'retry_wait' => 3,\n        'retry_randomize' => false,\n        'retry_max_times' => 10,\n        'queued_chunks_limit_size' => 100\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      _first_failure = @i.retry.start\n\n      chunks = @i.buffer.queue.dup\n\n      20.times do |i|\n        now = @i.next_flush_time\n\n        Timecop.freeze( now )\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }\n\n        assert{ @i.write_count > prev_write_count }\n        assert{ @i.num_errors > prev_num_errors }\n\n        break if @i.buffer.queue.size == 0\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n\n        assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n      end\n      assert{ @i.buffer.stage.size == 0 }\n      assert{ written_tags.all?('test.tag.1') }\n\n\n      @i.emit_events(\"test.tag.3\", dummy_event_stream())\n\n      logs = @i.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"[error]: Hit limit for retries. dropping all chunks in the buffer queue.\") && l.include?(\"retry_times=10\") } }\n\n      assert{ @i.buffer.queue.size == 0 }\n      assert{ @i.buffer.stage.size == 1 }\n      assert{ chunks.all?{|c| c.empty? } }\n    end\n\n    test 'Do not retry when retry_max_times is 0' do\n      written_tags = []\n\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_type' => :periodic,\n        'retry_wait' => 1,\n        'retry_randomize' => false,\n        'retry_max_times' => 0,\n        'queued_chunks_limit_size' => 100\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal(0, @i.write_count)\n      assert_equal(0, @i.num_errors)\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(2){ Thread.pass until @i.write_count == 1 && @i.num_errors == 1 }\n\n      assert(@i.write_count == 1)\n      assert(@i.num_errors == 1)\n      assert(@i.log.out.logs.any?{|l| l.include?(\"[error]: Hit limit for retries. dropping all chunks in the buffer queue.\") && l.include?(\"retry_times=0\") })\n      assert(@i.buffer.queue.size == 0)\n      assert(@i.buffer.stage.size == 1)\n      assert(@i.buffer.queue.all?{|c| c.empty? })\n    end\n  end\n\n  sub_test_case 'buffered output configured as retry_forever' do\n    setup do\n      Fluent::Plugin.register_output('output_retries_secondary_test', FluentPluginOutputAsBufferedRetryTest::DummyFullFeatureOutput2)\n    end\n\n    test 'warning logs are generated if secondary section is configured' do\n      chunk_key = 'tag'\n      hash = {\n        'retry_forever' => true,\n        'retry_randomize' => false,\n      }\n      i = create_output()\n      i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash),config_element('secondary','', {'@type' => 'output_retries_secondary_test'})]))\n      logs = i.log.out.logs\n      assert { logs.any? { |l| l.include?(\"<secondary> with 'retry_forever', only unrecoverable errors are moved to secondary\") } }\n    end\n\n    test 'retry_timeout and retry_max_times will be ignored if retry_forever is true for exponential backoff' do\n      written_tags = []\n\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_type' => :exponential_backoff,\n        'retry_forever' => true,\n        'retry_randomize' => false,\n        'retry_timeout' => 3600,\n        'retry_max_times' => 10,\n        'queued_chunks_limit_size' => 100\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      15.times do |i|\n        now = @i.next_flush_time\n\n        Timecop.freeze( now + 1 )\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ sleep 0.1 until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }\n\n        assert{ @i.write_count > prev_write_count }\n        assert{ @i.num_errors > prev_num_errors }\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n      end\n\n      assert{ @i.buffer.queue.size == 2 }\n      assert{ @i.retry.steps > 10 }\n      assert{ now > first_failure + 3600 }\n    end\n\n    test 'retry_timeout and retry_max_times will be ignored if retry_forever is true for periodical retries' do\n      written_tags = []\n\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_type' => :periodic,\n        'retry_forever' => true,\n        'retry_randomize' => false,\n        'retry_wait' => 30,\n        'retry_timeout' => 360,\n        'retry_max_times' => 10,\n        'queued_chunks_limit_size' => 100\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      15.times do |i|\n        now = @i.next_flush_time\n\n        Timecop.freeze( now + 1 )\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ sleep 0.1 until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }\n\n        assert{ @i.write_count > prev_write_count }\n        assert{ @i.num_errors > prev_num_errors }\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n      end\n\n      assert{ @i.buffer.queue.size == 2 }\n      assert{ @i.retry.steps > 10 }\n      assert{ now > first_failure + 360 }\n    end\n  end\n\n  sub_test_case 'buffered output with delayed commit' do\n    test 'does retries correctly when #try_write fails' do\n      chunk_key = 'tag'\n      hash = {\n        'flush_interval' => 1,\n        'flush_thread_burst_interval' => 0.1,\n        'retry_randomize' => false,\n        'retry_max_interval' => 60 * 60,\n      }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:prefer_delayed_commit){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:try_write){|chunk| raise \"yay, your #write must fail\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:32 -0700')\n      Timecop.freeze( now )\n\n      @i.enqueue_thread_wait\n\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }\n      waiting(4) do\n        state = @i.instance_variable_get(:@output_flush_threads).first\n        state.thread.status == 'sleep'\n      end\n\n      assert(@i.write_count > 0)\n      assert(@i.num_errors > 0)\n\n      now = @i.next_flush_time\n      Timecop.freeze( now )\n      @i.flush_thread_wakeup\n      waiting(4){ Thread.pass until @i.write_count > 1 && @i.num_errors > 1 }\n      waiting(4) do\n        state = @i.instance_variable_get(:@output_flush_threads).first\n        state.thread.status == 'sleep'\n      end\n\n      assert(@i.write_count > 1)\n      assert(@i.num_errors > 1)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_output_as_buffered_secondary.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/event'\n\nrequire 'json'\nrequire 'time'\nrequire 'timeout'\nrequire 'timecop'\n\nmodule FluentPluginOutputAsBufferedSecondaryTest\n  class DummyBareOutput < Fluent::Plugin::Output\n    def register(name, &block)\n      instance_variable_set(\"@#{name}\", block)\n    end\n  end\n  class DummySyncOutput < DummyBareOutput\n    def initialize\n      super\n      @process = nil\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n  end\n  class DummyFullFeatureOutput < DummyBareOutput\n    def initialize\n      super\n      @prefer_buffered_processing = nil\n      @prefer_delayed_commit = nil\n      @process = nil\n      @format = nil\n      @write = nil\n      @try_write = nil\n    end\n    def prefer_buffered_processing\n      @prefer_buffered_processing ? @prefer_buffered_processing.call : false\n    end\n    def prefer_delayed_commit\n      @prefer_delayed_commit ? @prefer_delayed_commit.call : false\n    end\n    def process(tag, es)\n      @process ? @process.call(tag, es) : nil\n    end\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n    def try_write(chunk)\n      @try_write ? @try_write.call(chunk) : nil\n    end\n  end\n  class DummyFullFeatureOutput2 < DummyFullFeatureOutput\n    def prefer_buffered_processing; true; end\n    def prefer_delayed_commit; super; end\n    def format(tag, time, record); super; end\n    def write(chunk); super; end\n    def try_write(chunk); super; end\n  end\nend\n\nclass BufferedOutputSecondaryTest < Test::Unit::TestCase\n  def create_output(type=:full)\n    case type\n    when :bare then FluentPluginOutputAsBufferedSecondaryTest::DummyBareOutput.new\n    when :sync then FluentPluginOutputAsBufferedSecondaryTest::DummySyncOutput.new\n    when :full then FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput.new\n    else\n      raise ArgumentError, \"unknown type: #{type}\"\n    end\n  end\n  def create_metadata(timekey: nil, tag: nil, variables: nil)\n    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n  end\n  def waiting(seconds)\n    begin\n      Timeout.timeout(seconds) do\n        yield\n      end\n    rescue Timeout::Error\n      STDERR.print(*@i.log.out.logs)\n      raise\n    end\n  end\n  def dummy_event_stream\n    Fluent::ArrayEventStream.new([\n      [ event_time('2016-04-13 18:33:00'), {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data1\"} ],\n      [ event_time('2016-04-13 18:33:13'), {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data2\"} ],\n      [ event_time('2016-04-13 18:33:32'), {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data3\"} ],\n    ])\n  end\n\n  setup do\n    @i = create_output\n  end\n\n  teardown do\n    if @i\n      @i.stop unless @i.stopped?\n      @i.before_shutdown unless @i.before_shutdown?\n      @i.shutdown unless @i.shutdown?\n      @i.after_shutdown unless @i.after_shutdown?\n      @i.close unless @i.closed?\n      @i.terminate unless @i.terminated?\n    end\n    Timecop.return\n  end\n\n  sub_test_case 'secondary plugin feature for buffered output with periodical retry' do\n    setup do\n      Fluent::Plugin.register_output('output_secondary_test', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput)\n      Fluent::Plugin.register_output('output_secondary_test2', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput2)\n    end\n\n    test 'raises configuration error if primary does not support buffering' do\n      i = create_output(:sync)\n      assert_raise Fluent::ConfigError do\n        i.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'output_secondary_test'})]))\n      end\n    end\n\n    test 'raises configuration error if <buffer>/<secondary> section is specified in <secondary> section' do\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})\n      secconf1 = config_element('secondary','',{'@type' => 'output_secondary_test'},[config_element('buffer', 'time')])\n      secconf2 = config_element('secondary','',{'@type' => 'output_secondary_test'},[config_element('secondary', '')])\n      i = create_output()\n      assert_raise Fluent::ConfigError do\n        i.configure(config_element('ROOT','',{},[priconf,secconf1]))\n      end\n      assert_raise Fluent::ConfigError do\n        i.configure(config_element('ROOT','',{},[priconf,secconf2]))\n      end\n    end\n\n    test 'uses same plugin type with primary if @type is missing in secondary' do\n      bufconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})\n      secconf = config_element('secondary','',{})\n      priconf = config_element('ROOT', '', {'@type' => 'output_secondary_test'}, [bufconf, secconf])\n      i = create_output()\n      assert_nothing_raised do\n        i.configure(priconf)\n      end\n      logs = i.log.out.logs\n      assert{ logs.empty? }\n      assert{ i.secondary.is_a? FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput }\n    end\n\n    test 'warns if secondary plugin is different type from primary one' do\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})\n      secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})\n      i = create_output()\n      i.configure(config_element('ROOT','',{},[priconf,secconf]))\n      logs = i.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"Use different plugin for secondary. Check the plugin works with primary like secondary_file\") } }\n    end\n\n    test 'secondary plugin lifecycle is kicked by primary' do\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})\n      secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})\n      i = create_output()\n      i.configure(config_element('ROOT','',{},[priconf,secconf]))\n      logs = i.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"Use different plugin for secondary. Check the plugin works with primary like secondary_file\") } }\n\n      assert i.secondary.configured?\n\n      assert !i.secondary.started?\n      i.start\n      assert i.secondary.started?\n\n      assert !i.secondary.after_started?\n      i.after_start\n      assert i.secondary.after_started?\n\n      assert !i.secondary.stopped?\n      i.stop\n      assert i.secondary.stopped?\n\n      assert !i.secondary.before_shutdown?\n      i.before_shutdown\n      assert i.secondary.before_shutdown?\n\n      assert !i.secondary.shutdown?\n      i.shutdown\n      assert i.secondary.shutdown?\n\n      assert !i.secondary.after_shutdown?\n      i.after_shutdown\n      assert i.secondary.after_shutdown?\n\n      assert !i.secondary.closed?\n      i.close\n      assert i.secondary.closed?\n\n      assert !i.secondary.terminated?\n      i.terminate\n      assert i.secondary.terminated?\n    end\n\n    test 'primary plugin will emit event streams to secondary after retries for time of retry_timeout * retry_secondary_threshold' do\n      written = []\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})\n      secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})\n      @i.configure(config_element('ROOT','',{},[priconf,secconf]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:prefer_delayed_commit){ false }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| raise \"yay, your #write must fail\" }\n      @i.secondary.register(:prefer_delayed_commit){ false }\n      @i.secondary.register(:write){|chunk| chunk.read.split(\"\\n\").each{|line| written << JSON.parse(line) } }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      # retry_timeout == 60(sec), retry_secondary_threshold == 0.8\n      now = first_failure + 60 * 0.8 + 1 # to step from primary to secondary\n      Timecop.freeze( now )\n\n      unless @i.retry.secondary?\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n\n        # next step is on secondary\n        now = first_failure + 60 * 0.8 + 10\n        Timecop.freeze( now )\n      end\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n      current_write_count = @i.write_count\n      current_num_errors = @i.num_errors\n      assert{ current_write_count > prev_write_count }\n      assert{ current_num_errors == prev_num_errors }\n\n      assert_nil @i.retry\n\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data1\"} ], written[0]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data2\"} ], written[1]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data3\"} ], written[2]\n\n      logs = @i.log.out.logs\n      waiting(4){ sleep 0.1 until logs.any?{|l| l.include?(\"[warn]: retry succeeded by secondary.\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: retry succeeded by secondary.\") } }\n    end\n\n    test 'secondary can do non-delayed commit even if primary do delayed commit' do\n      written = []\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})\n      secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})\n      @i.configure(config_element('ROOT','',{},[priconf,secconf]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:prefer_delayed_commit){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:try_write){|chunk| raise \"yay, your #write must fail\" }\n      @i.secondary.register(:prefer_delayed_commit){ false }\n      @i.secondary.register(:write){|chunk| chunk.read.split(\"\\n\").each{|line| written << JSON.parse(line) } }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n      assert_equal 0, @i.write_secondary_count\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      # retry_timeout == 60(sec), retry_secondary_threshold == 0.8\n      now = first_failure + 60 * 0.8 + 1 # to step from primary to secondary\n      Timecop.freeze( now )\n\n      unless @i.retry.secondary?\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n\n        # next step is on secondary\n        now = first_failure + 60 * 0.8 + 10\n        Timecop.freeze( now )\n      end\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n      assert{ @i.write_count > prev_write_count }\n      assert{ @i.num_errors == prev_num_errors }\n\n      assert{ @i.write_secondary_count > 0 }\n\n      assert_nil @i.retry\n\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data1\"} ], written[0]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data2\"} ], written[1]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data3\"} ], written[2]\n\n      logs = @i.log.out.logs\n      waiting(4){ sleep 0.1 until logs.any?{|l| l.include?(\"[warn]: retry succeeded by secondary.\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: retry succeeded by secondary.\") } }\n    end\n\n    test 'secondary plugin can do delayed commit if primary do it' do\n      written = []\n      chunks = []\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})\n      secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})\n      @i.configure(config_element('ROOT','',{},[priconf,secconf]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:prefer_delayed_commit){ true }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:try_write){|chunk| raise \"yay, your #write must fail\" }\n      @i.secondary.register(:prefer_delayed_commit){ true }\n      @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split(\"\\n\").each{|line| written << JSON.parse(line) } }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      # retry_timeout == 60(sec), retry_secondary_threshold == 0.8\n      now = first_failure + 60 * 0.8 + 1 # to step from primary to secondary\n      Timecop.freeze( now )\n\n      unless @i.retry.secondary?\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n\n        # next step is on secondary\n        now = first_failure + 60 * 0.8 + 10\n        Timecop.freeze( now )\n      end\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n      assert{ @i.write_count > prev_write_count }\n      assert{ @i.num_errors == prev_num_errors }\n\n      assert @i.retry\n\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data1\"} ], written[0]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data2\"} ], written[1]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data3\"} ], written[2]\n\n      assert{ @i.buffer.dequeued.size > 0 }\n      assert{ chunks.size > 0 }\n      assert{ !chunks.first.empty? }\n\n      @i.secondary.commit_write(chunks[0].unique_id)\n\n      assert{ @i.buffer.dequeued[chunks[0].unique_id].nil? }\n      assert{ chunks.first.empty? }\n\n      assert{ @i.write_secondary_count > 0 }\n\n      assert_nil @i.retry\n\n      logs = @i.log.out.logs\n      waiting(4){ sleep 0.1 until logs.any?{|l| l.include?(\"[warn]: retry succeeded by secondary.\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: retry succeeded by secondary.\") } }\n    end\n\n    test 'secondary plugin can do delayed commit even if primary does not do it' do\n      written = []\n      chunks = []\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})\n      secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})\n      @i.configure(config_element('ROOT','',{},[priconf,secconf]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:prefer_delayed_commit){ false }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| raise \"yay, your #write must fail\" }\n      @i.secondary.register(:prefer_delayed_commit){ true }\n      @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split(\"\\n\").each{|line| written << JSON.parse(line) } }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      # retry_timeout == 60(sec), retry_secondary_threshold == 0.8\n      now = first_failure + 60 * 0.8 + 1 # to step from primary to secondary\n      Timecop.freeze( now )\n\n      unless @i.retry.secondary?\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n\n        # next step is on secondary\n        now = first_failure + 60 * 0.8 + 10\n        Timecop.freeze( now )\n      end\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n      assert{ @i.write_count > prev_write_count }\n      assert{ @i.num_errors == prev_num_errors }\n\n      assert @i.retry\n\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data1\"} ], written[0]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data2\"} ], written[1]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data3\"} ], written[2]\n\n      assert{ @i.buffer.dequeued.size > 0 }\n      assert{ chunks.size > 0 }\n      assert{ !chunks.first.empty? }\n\n      @i.secondary.commit_write(chunks[0].unique_id)\n\n      assert{ @i.buffer.dequeued[chunks[0].unique_id].nil? }\n      assert{ chunks.first.empty? }\n\n      assert_nil @i.retry\n\n      logs = @i.log.out.logs\n      waiting(4){ sleep 0.1 until logs.any?{|l| l.include?(\"[warn]: retry succeeded by secondary.\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: retry succeeded by secondary.\") } }\n    end\n\n    test 'secondary plugin can do delayed commit even if primary does not do it, and non-committed chunks will be rollbacked by primary' do\n      written = []\n      chunks = []\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false, 'queued_chunks_limit_size' => 10})\n      secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})\n      @i.configure(config_element('ROOT','',{},[priconf,secconf]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:prefer_delayed_commit){ false }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| raise \"yay, your #write must fail\" }\n      @i.secondary.register(:prefer_delayed_commit){ true }\n      @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split(\"\\n\").each{|line| written << JSON.parse(line) } }\n      @i.secondary.register(:write){|chunk| raise \"don't use this\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size == 2 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      # retry_timeout == 60(sec), retry_secondary_threshold == 0.8\n\n      now = first_failure + 60 * 0.8 + 1\n      Timecop.freeze( now )\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      now = first_failure + 60 * 0.8 + 2\n      Timecop.freeze( now )\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n\n      waiting(4){ sleep 0.1 until chunks.size == 2 }\n\n      assert{ @i.write_count > prev_write_count }\n      assert{ @i.num_errors == prev_num_errors }\n\n      assert @i.retry\n\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data1\"} ], written[0]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data2\"} ], written[1]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data3\"} ], written[2]\n      assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:00').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data1\"} ], written[3]\n      assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:13').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data2\"} ], written[4]\n      assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:32').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data3\"} ], written[5]\n\n      assert{ @i.buffer.dequeued.size == 2 }\n      assert{ chunks.size == 2 }\n      assert{ !chunks[0].empty? }\n      assert{ !chunks[1].empty? }\n\n      30.times do |i| # large enough\n        # In https://github.com/fluent/fluentd/blob/c90c024576b3d35f356a55fd33d1232947114a9a/lib/fluent/plugin_helper/retry_state.rb\n        # @timeout_at is 2016-04-13 18:34:31, @next_time must be less than 2016-04-13 18:34:30\n        #\n        # first_failure + 60 * 0.8 + 2 # => 2016-04-13 18:34:21\n        # @next_time is not added by 1, but by randomize(@retry_wait) https://github.com/fluent/fluentd/blob/c90c024576b3d35f356a55fd33d1232947114a9a/lib/fluent/plugin_helper/retry_state.rb#L196\n        # current_time(=Time.now) + randomize(@retry_wait) < @timeout_at\n        # (2016-04-13 18:34:21 + 6) + 3 < 2016-04-13 18:34:31\n        # So, current_time must be at most 6\n        now = first_failure + 60 * 0.8 + 2 + [i, 6].min\n        Timecop.freeze( now )\n        @i.flush_thread_wakeup\n\n        break if @i.buffer.dequeued.size == 0\n      end\n\n      assert @i.retry\n      logs = @i.log.out.logs\n      waiting(4){ sleep 0.1 until logs.count{|l| l.include?(\"[warn]: failed to flush the buffer chunk, timeout to commit.\") } == 2 }\n      assert{ logs.count{|l| l.include?(\"[warn]: failed to flush the buffer chunk, timeout to commit.\") } == 2 }\n    end\n\n    test 'retry_wait for secondary is same with one for primary' do\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})\n      secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})\n      @i.configure(config_element('ROOT','',{},[priconf,secconf]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:prefer_delayed_commit){ false }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| raise \"yay, your #write must fail\" }\n      @i.secondary.register(:prefer_delayed_commit){ false }\n      @i.secondary.register(:write){|chunk| raise \"your secondary is also useless.\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      # retry_timeout == 60(sec), retry_secondary_threshold == 0.8\n\n      now = first_failure + 60 * 0.8 + 1\n\n      Timecop.freeze( now )\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n      assert{ @i.write_count > prev_write_count }\n      assert{ @i.num_errors > prev_num_errors }\n\n      assert @i.retry\n\n      assert_equal 3, (@i.next_flush_time - Time.now)\n\n      logs = @i.log.out.logs\n      waiting(4){ sleep 0.1 until logs.any?{|l| l.include?(\"[warn]: failed to flush the buffer with secondary output.\") } }\n      assert{ logs.any?{|l| l.include?(\"[warn]: failed to flush the buffer with secondary output.\") } }\n    end\n  end\n\n  sub_test_case 'secondary plugin feature for buffered output with exponential backoff' do\n    setup do\n      Fluent::Plugin.register_output('output_secondary_test', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput)\n      Fluent::Plugin.register_output('output_secondary_test2', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput2)\n    end\n\n    test 'primary plugin will emit event streams to secondary after retries for time of retry_timeout * retry_secondary_threshold' do\n      written = []\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :exponential_backoff, 'retry_wait' => 1, 'retry_timeout' => 60, 'retry_randomize' => false})\n      secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})\n      @i.configure(config_element('ROOT','',{},[priconf,secconf]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:prefer_delayed_commit){ false }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| raise \"yay, your #write must fail\" }\n      @i.secondary.register(:prefer_delayed_commit){ false }\n      @i.secondary.register(:write){|chunk| chunk.read.split(\"\\n\").each{|line| written << JSON.parse(line) } }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n      assert_equal 0, @i.write_secondary_count\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      first_failure = @i.retry.start\n\n      20.times do |i| # large enough\n        now = @i.next_flush_time\n        Timecop.freeze( now )\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n        assert{ @i.write_count > prev_write_count }\n\n        break if @i.buffer.queue.size == 0\n\n        prev_write_count = @i.write_count\n      end\n\n      assert{ @i.write_secondary_count > 0 }\n\n      # retry_timeout == 60(sec), retry_secondary_threshold == 0.8\n\n      assert{ now >= first_failure + 60 * 0.8 }\n\n      assert_nil @i.retry\n\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data1\"} ], written[0]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data2\"} ], written[1]\n      assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {\"name\" => \"moris\", \"age\" => 36, \"message\" => \"data3\"} ], written[2]\n\n      assert(@i.log.out.logs.any?{|l| l.include?(\"[warn]: retry succeeded by secondary.\") })\n    end\n\n    test 'exponential backoff interval will be initialized when switched to secondary' do\n      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :exponential_backoff, 'retry_wait' => 1, 'retry_timeout' => 60, 'retry_randomize' => false})\n      secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})\n      @i.configure(config_element('ROOT','',{},[priconf,secconf]))\n      @i.register(:prefer_buffered_processing){ true }\n      @i.register(:prefer_delayed_commit){ false }\n      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + \"\\n\" }\n      @i.register(:write){|chunk| raise \"yay, your #write must fail\" }\n      @i.secondary.register(:prefer_delayed_commit){ false }\n      @i.secondary.register(:write){|chunk| raise \"your secondary is also useless.\" }\n      @i.start\n      @i.after_start\n\n      @i.interrupt_flushes\n\n      now = Time.parse('2016-04-13 18:33:30 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.1\", dummy_event_stream())\n\n      now = Time.parse('2016-04-13 18:33:31 -0700')\n      Timecop.freeze( now )\n\n      @i.emit_events(\"test.tag.2\", dummy_event_stream())\n\n      assert_equal 0, @i.write_count\n      assert_equal 0, @i.num_errors\n\n      @i.enqueue_thread_wait\n      @i.flush_thread_wakeup\n      waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }\n\n      assert{ @i.buffer.queue.size > 0 }\n      assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n\n      assert{ @i.write_count > 0 }\n      assert{ @i.num_errors > 0 }\n\n      prev_write_count = @i.write_count\n      prev_num_errors = @i.num_errors\n\n      first_failure = @i.retry.start\n\n      20.times do |i| # large enough\n        now = @i.next_flush_time\n        # p({i: i, now: now, diff: (now - Time.now)})\n        # {:i=>0, :now=>2016-04-13 18:33:32 -0700, :diff=>1.0}\n        # {:i=>1, :now=>2016-04-13 18:33:34 -0700, :diff=>2.0}\n        # {:i=>2, :now=>2016-04-13 18:33:38 -0700, :diff=>4.0}\n        # {:i=>3, :now=>2016-04-13 18:33:46 -0700, :diff=>8.0}\n        # {:i=>4, :now=>2016-04-13 18:34:02 -0700, :diff=>16.0}\n        # {:i=>5, :now=>2016-04-13 18:34:19 -0700, :diff=>17.0}\n        Timecop.freeze( now )\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n        assert{ @i.write_count > prev_write_count }\n        assert{ @i.num_errors > prev_num_errors }\n\n        prev_write_count = @i.write_count\n        prev_num_errors = @i.num_errors\n\n        break if @i.retry.secondary?\n\n        assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }\n      end\n\n      # retry_timeout == 60(sec), retry_secondary_threshold == 0.8\n\n      assert{ now >= first_failure + 60 * 0.8 }\n      assert @i.retry\n      logs = @i.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"[warn]: failed to flush the buffer with secondary output.\") } }\n\n      assert{ (@i.next_flush_time - Time.now) <= 2 } # <= retry_wait (1s) * base (2) ** 1\n\n      20.times do |i| # large enough again\n        now = @i.next_flush_time\n        # p({i: i, now: now, diff: (now - Time.now)})\n        # {:i=>0, :now=>2016-04-13 18:34:20 -0700, :diff=>1.0}\n        # {:i=>1, :now=>2016-04-13 18:34:24 -0700, :diff=>4.0}\n        # {:i=>2, :now=>2016-04-13 18:34:31 -0700, :diff=>7.0}\n\n        Timecop.freeze( now )\n        @i.enqueue_thread_wait\n        @i.flush_thread_wakeup\n        waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }\n\n        assert{ @i.write_count > prev_write_count }\n        assert{ @i.num_errors > prev_num_errors }\n\n        break if @i.buffer.queue.size == 0\n      end\n\n      logs = @i.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"[error]: Hit limit for retries. dropping all chunks in the buffer queue.\") } }\n\n      assert{ now >= first_failure + 60 }\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_output_as_standard.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/msgpack_factory'\nrequire 'fluent/event'\n\nrequire 'json'\nrequire 'time'\nrequire 'timeout'\n\nrequire 'flexmock/test_unit'\n\nmodule FluentPluginStandardBufferedOutputTest\n  class DummyBareOutput < Fluent::Plugin::Output\n    def register(name, &block)\n      instance_variable_set(\"@#{name}\", block)\n    end\n  end\n  class DummyAsyncOutput < DummyBareOutput\n    def format(tag, time, record)\n      @format ? @format.call(tag, time, record) : [tag, time, record].to_json\n    end\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n  end\n  class DummyAsyncStandardOutput < DummyBareOutput\n    def write(chunk)\n      @write ? @write.call(chunk) : nil\n    end\n  end\nend\n\nclass StandardBufferedOutputTest < Test::Unit::TestCase\n  def create_output(type=:full)\n    case type\n    when :bare     then FluentPluginStandardBufferedOutputTest::DummyBareOutput.new\n    when :buffered then FluentPluginStandardBufferedOutputTest::DummyAsyncOutput.new\n    when :standard then FluentPluginStandardBufferedOutputTest::DummyAsyncStandardOutput.new\n    else\n      raise ArgumentError, \"unknown type: #{type}\"\n    end\n  end\n  def create_metadata(timekey: nil, tag: nil, variables: nil)\n    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)\n  end\n  def waiting(seconds)\n    begin\n      Timeout.timeout(seconds) do\n        yield\n      end\n    rescue Timeout::Error\n      STDERR.print(*@i.log.out.logs)\n      raise\n    end\n  end\n  def test_event_stream\n    es = Fluent::MultiEventStream.new\n    es.add(event_time('2016-04-21 17:19:00 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n    es.add(event_time('2016-04-21 17:19:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris2\", \"message\" => \"hello!\"})\n    es.add(event_time('2016-04-21 17:19:25 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n    es.add(event_time('2016-04-21 17:20:01 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n    es.add(event_time('2016-04-21 17:20:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n    es.add(event_time('2016-04-21 17:21:32 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n    es\n  end\n\n  setup do\n    @i = nil\n  end\n\n  teardown do\n    if @i\n      @i.stop unless @i.stopped?\n      @i.before_shutdown unless @i.before_shutdown?\n      @i.shutdown unless @i.shutdown?\n      @i.after_shutdown unless @i.after_shutdown?\n      @i.close unless @i.closed?\n      @i.terminate unless @i.terminated?\n    end\n  end\n\n  sub_test_case 'standard buffered without any chunk keys' do\n    test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format' do\n      @i = create_output(:standard)\n      @i.configure(config_element())\n      @i.start\n      @i.after_start\n\n      m = create_metadata()\n      es = test_event_stream\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false)\n\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n\n    test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format, but time will be int if time_as_integer specified' do\n      @i = create_output(:standard)\n      @i.configure(config_element('ROOT','',{\"time_as_integer\"=>\"true\"}))\n      @i.start\n      @i.after_start\n\n      m = create_metadata()\n      es = test_event_stream\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false)\n\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n  end\n\n  sub_test_case 'standard buffered with tag chunk key' do\n    test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format' do\n      @i = create_output(:standard)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))\n      @i.start\n      @i.after_start\n\n      m = create_metadata(tag: \"mytag.test\")\n      es = test_event_stream\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false)\n\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n\n    test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format, but time will be int if time_as_integer specified' do\n      @i = create_output(:standard)\n      @i.configure(config_element('ROOT','',{\"time_as_integer\"=>\"true\"},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))\n      @i.start\n      @i.after_start\n\n      m = create_metadata(tag: \"mytag.test\")\n      es = test_event_stream\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false)\n\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n  end\n\n  sub_test_case 'standard buffered with time chunk key' do\n    test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format' do\n      @i = create_output(:standard)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','time',{\"timekey\" => \"60\",'flush_thread_burst_interval' => 0.01})]))\n      @i.start\n      @i.after_start\n\n      m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)\n      m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)\n      m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)\n\n      es1 = Fluent::MultiEventStream.new\n      es1.add(event_time('2016-04-21 17:19:00 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:19:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris2\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:19:25 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      es2 = Fluent::MultiEventStream.new\n      es2.add(event_time('2016-04-21 17:20:01 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es2.add(event_time('2016-04-21 17:20:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      es3 = Fluent::MultiEventStream.new\n      es3.add(event_time('2016-04-21 17:21:32 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).once.with({\n          m1 => es1,\n          m2 => es2,\n          m3 => es3,\n        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false)\n\n      es = test_event_stream\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n\n    test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format, but time will be int if time_as_integer specified' do\n      @i = create_output(:standard)\n      @i.configure(config_element('ROOT','',{\"time_as_integer\" => \"true\"},[config_element('buffer','time',{\"timekey\" => \"60\",'flush_thread_burst_interval' => 0.01})]))\n      @i.start\n      @i.after_start\n\n      m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)\n      m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)\n      m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)\n\n      es1 = Fluent::MultiEventStream.new\n      es1.add(event_time('2016-04-21 17:19:00 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:19:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris2\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:19:25 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      es2 = Fluent::MultiEventStream.new\n      es2.add(event_time('2016-04-21 17:20:01 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es2.add(event_time('2016-04-21 17:20:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      es3 = Fluent::MultiEventStream.new\n      es3.add(event_time('2016-04-21 17:21:32 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).with({\n          m1 => es1,\n          m2 => es2,\n          m3 => es3,\n        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false)\n\n      es = test_event_stream\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n  end\n\n  sub_test_case 'standard buffered with variable chunk keys' do\n    test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format' do\n      @i = create_output(:standard)\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))\n      @i.start\n      @i.after_start\n\n      m1 = create_metadata(variables: {key: \"my value\", name: \"moris1\"})\n      es1 = Fluent::MultiEventStream.new\n      es1.add(event_time('2016-04-21 17:19:00 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:19:25 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:20:01 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:20:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:21:32 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      m2 = create_metadata(variables: {key: \"my value\", name: \"moris2\"})\n      es2 = Fluent::MultiEventStream.new\n      es2.add(event_time('2016-04-21 17:19:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris2\", \"message\" => \"hello!\"})\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).with({\n          m1 => es1,\n          m2 => es2,\n        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false).once\n\n      es = test_event_stream\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n\n    test '#execute_chunking calls @buffer.write(bulk: true) in times of # of variable variations with predefined msgpack format, but time will be int if time_as_integer specified' do\n      @i = create_output(:standard)\n      @i.configure(config_element('ROOT','',{\"time_as_integer\" => \"true\"},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))\n      @i.start\n      @i.after_start\n\n      m1 = create_metadata(variables: {key: \"my value\", name: \"moris1\"})\n      es1 = Fluent::MultiEventStream.new\n      es1.add(event_time('2016-04-21 17:19:00 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:19:25 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:20:01 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:20:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:21:32 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      m2 = create_metadata(variables: {key: \"my value\", name: \"moris2\"})\n      es2 = Fluent::MultiEventStream.new\n      es2.add(event_time('2016-04-21 17:19:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris2\", \"message\" => \"hello!\"})\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).with({\n          m1 => es1,\n          m2 => es2,\n        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false).once\n\n      es = test_event_stream\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n  end\n\n  sub_test_case 'custom format buffered without any chunk keys' do\n    test '#execute_chunking calls @buffer.write(bulk: true) just once with customized format' do\n      @i = create_output(:buffered)\n      @i.register(:format){|tag, time, record| [time, record].to_json }\n      @i.configure(config_element())\n      @i.start\n      @i.after_start\n\n      m = create_metadata()\n      es = test_event_stream\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).once.with({m => es.map{|t,r| [t,r].to_json }}, format: nil, enqueue: false)\n\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n  end\n\n  sub_test_case 'custom format buffered with tag chunk key' do\n    test '#execute_chunking calls @buffer.write(bulk: true) just once with customized format' do\n      @i = create_output(:buffered)\n      @i.register(:format){|tag, time, record| [time, record].to_json }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))\n      @i.start\n      @i.after_start\n\n      m = create_metadata(tag: \"mytag.test\")\n      es = test_event_stream\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).once.with({m => es.map{|t,r| [t,r].to_json}}, format: nil, enqueue: false)\n\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n  end\n  sub_test_case 'custom format buffered with time chunk key' do\n    test '#execute_chunking calls @buffer.write with customized format' do\n      @i = create_output(:buffered)\n      @i.register(:format){|tag, time, record| [time, record].to_json }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','time',{\"timekey\" => \"60\",'flush_thread_burst_interval' => 0.01})]))\n      @i.start\n      @i.after_start\n\n      m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)\n      m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)\n      m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)\n\n      es1 = Fluent::MultiEventStream.new\n      es1.add(event_time('2016-04-21 17:19:00 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:19:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris2\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:19:25 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      es2 = Fluent::MultiEventStream.new\n      es2.add(event_time('2016-04-21 17:20:01 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es2.add(event_time('2016-04-21 17:20:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      es3 = Fluent::MultiEventStream.new\n      es3.add(event_time('2016-04-21 17:21:32 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).with({\n          m1 => es1.map{|t,r| [t,r].to_json },\n          m2 => es2.map{|t,r| [t,r].to_json },\n          m3 => es3.map{|t,r| [t,r].to_json },\n        }, enqueue: false).once\n\n      es = test_event_stream\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n  end\n\n  sub_test_case 'custom format buffered with variable chunk keys' do\n    test '#execute_chunking calls @buffer.write in times of # of variable variations with customized format' do\n      @i = create_output(:buffered)\n      @i.register(:format){|tag, time, record| [time, record].to_json }\n      @i.configure(config_element('ROOT','',{},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))\n      @i.start\n      @i.after_start\n\n      m1 = create_metadata(variables: {key: \"my value\", name: \"moris1\"})\n      es1 = Fluent::MultiEventStream.new\n      es1.add(event_time('2016-04-21 17:19:00 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:19:25 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:20:01 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:20:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n      es1.add(event_time('2016-04-21 17:21:32 -0700'), {\"key\" => \"my value\", \"name\" => \"moris1\", \"message\" => \"hello!\"})\n\n      m2 = create_metadata(variables: {key: \"my value\", name: \"moris2\"})\n      es2 = Fluent::MultiEventStream.new\n      es2.add(event_time('2016-04-21 17:19:13 -0700'), {\"key\" => \"my value\", \"name\" => \"moris2\", \"message\" => \"hello!\"})\n\n      buffer_mock = flexmock(@i.buffer)\n      buffer_mock.should_receive(:write).with({\n          m1 => es1.map{|t,r| [t,r].to_json },\n          m2 => es2.map{|t,r| [t,r].to_json },\n        }, enqueue: false).once\n\n      es = test_event_stream\n      @i.execute_chunking(\"mytag.test\", es)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_owned_by.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/base'\nrequire 'fluent/plugin/input'\nrequire 'fluent/plugin/owned_by_mixin'\n\nmodule OwnedByMixinTestEnv\n  class DummyParent < Fluent::Plugin::Input\n    Fluent::Plugin.register_input('dummy_parent', self)\n  end\n  class DummyChild < Fluent::Plugin::Base\n    include Fluent::Plugin::OwnedByMixin\n    Fluent::Plugin.register_parser('dummy_child', self)\n  end\nend\n\nclass OwnedByMixinTest < Test::Unit::TestCase\n  sub_test_case 'Owned plugins' do\n    setup do\n      Fluent::Test.setup\n    end\n\n    test 'inherits plugin id and logger from parent' do\n      parent = Fluent::Plugin.new_input('dummy_parent')\n      parent.configure(config_element('ROOT', '', {'@id' => 'my_parent_id', '@log_level' => 'trace'}))\n      child = Fluent::Plugin.new_parser('dummy_child', parent: parent)\n\n      assert_equal parent.object_id, child.owner.object_id\n\n      assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }\n\n      assert_equal Fluent::Log::LEVEL_TRACE, child.log.level\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser'\nrequire 'json'\nrequire 'timecop'\n\nclass ParserTest < ::Test::Unit::TestCase\n  class ExampleParser < Fluent::Plugin::Parser\n    def parse(data)\n      r = JSON.parse(data)\n      yield convert_values(parse_time(r), r)\n    end\n  end\n\n  def create_driver(conf={})\n    Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser).configure(conf)\n  end\n\n  def setup\n    Fluent::Test.setup\n  end\n\n  sub_test_case 'base class works as plugin' do\n    def test_init\n      i = Fluent::Plugin::Parser.new\n      assert_nil i.types\n      assert_nil i.null_value_pattern\n      assert !i.null_empty_string\n      assert i.estimate_current_event\n      assert !i.keep_time_key\n    end\n\n    def test_configure_against_string_literal\n      d = create_driver('keep_time_key true')\n      assert_true d.instance.keep_time_key\n    end\n\n    def test_parse\n      d = create_driver\n      assert_raise NotImplementedError do\n        d.instance.parse('')\n      end\n    end\n  end\n\n  sub_test_case '#string_like_null' do\n    setup do\n      @i = ExampleParser.new\n    end\n\n    test 'returns false if null_empty_string is false and null_value_regexp is nil' do\n      assert ! @i.string_like_null('a', false, nil)\n      assert ! @i.string_like_null('', false, nil)\n    end\n\n    test 'returns true if null_empty_string is true and string value is empty' do\n      assert ! @i.string_like_null('a', true, nil)\n      assert @i.string_like_null('', true, nil)\n    end\n\n    test 'returns true if null_value_regexp has regexp and it matches string value' do\n      assert ! @i.string_like_null('a', false, /null/i)\n      assert @i.string_like_null('NULL', false, /null/i)\n      assert @i.string_like_null('empty', false, /null|empty/i)\n    end\n  end\n\n  sub_test_case '#build_type_converters converters' do\n    setup do\n      @i = ExampleParser.new\n      types_config = {\n        \"s\" => \"string\",\n        \"i\" => \"integer\",\n        \"f\" => \"float\",\n        \"b\" => \"bool\",\n        \"t1\" => \"time\",\n        \"t2\" => \"time:%Y-%m-%d %H:%M:%S.%N\",\n        \"t3\" => \"time:+0100:%Y-%m-%d %H:%M:%S.%N\",\n        \"t4\" => \"time:unixtime\",\n        \"t5\" => \"time:float\",\n        \"a1\" => \"array\",\n        \"a2\" => \"array:|\",\n      }\n      @hash = {\n        'types' => JSON.dump(types_config),\n      }\n    end\n\n    test 'to do #to_s by \"string\" type' do\n      @i.configure(config_element('parse', '', @hash))\n      c = @i.type_converters[\"s\"]\n      assert_equal \"\", c.call(\"\")\n      assert_equal \"a\", c.call(\"a\")\n      assert_equal \"1\", c.call(1)\n      assert_equal \"1.01\", c.call(1.01)\n      assert_equal \"true\", c.call(true)\n      assert_equal \"false\", c.call(false)\n    end\n\n    test 'to do #to_i by \"integer\" type' do\n      @i.configure(config_element('parse', '', @hash))\n      c = @i.type_converters[\"i\"]\n      assert_equal 0, c.call(\"\")\n      assert_equal 0, c.call(\"0\")\n      assert_equal 0, c.call(\"a\")\n      assert_equal(-1000, c.call(\"-1000\"))\n      assert_equal 1, c.call(1)\n      assert_equal 1, c.call(1.01)\n      assert_equal 0, c.call(true)\n      assert_equal 0, c.call(false)\n    end\n\n    test 'to do #to_f by \"float\" type' do\n      @i.configure(config_element('parse', '', @hash))\n      c = @i.type_converters[\"f\"]\n      assert_equal 0.0, c.call(\"\")\n      assert_equal 0.0, c.call(\"0\")\n      assert_equal 0.0, c.call(\"a\")\n      assert_equal(-1000.0, c.call(\"-1000\"))\n      assert_equal 1.0, c.call(1)\n      assert_equal 1.01, c.call(1.01)\n      assert_equal 0.0, c.call(true)\n      assert_equal 0.0, c.call(false)\n    end\n\n    test 'to return true/false, which returns true only for true/yes/1 (C & perl style), by \"bool\"' do\n      @i.configure(config_element('parse', '', @hash))\n      c = @i.type_converters[\"b\"]\n      assert_false c.call(\"\")\n      assert_false c.call(\"0\")\n      assert_false c.call(\"a\")\n      assert_true c.call(\"1\")\n      assert_true c.call(\"true\")\n      assert_true c.call(\"True\")\n      assert_true c.call(\"YES\")\n      assert_true c.call(true)\n      assert_false c.call(false)\n      assert_false c.call(\"1.0\")\n    end\n\n    test 'to parse time string by ruby default time parser without any options' do\n      # \"t1\" => \"time\",\n      with_timezone(\"UTC+02\") do # -0200\n        @i.configure(config_element('parse', '', @hash))\n        c = @i.type_converters[\"t1\"]\n        assert_nil c.call(\"\")\n        assert_equal_event_time event_time(\"2016-10-21 01:54:30 -0200\"), c.call(\"2016-10-21 01:54:30\")\n        assert_equal_event_time event_time(\"2016-10-21 03:54:30 -0200\"), c.call(\"2016-10-21 01:54:30 -0400\")\n        assert_equal_event_time event_time(\"2016-10-21 01:55:24 -0200\"), c.call(\"2016-10-21T01:55:24-02:00\")\n        assert_equal_event_time event_time(\"2016-10-21 01:55:24 -0200\"), c.call(\"2016-10-21T03:55:24Z\")\n      end\n    end\n\n    test 'to parse time string with specified time format' do\n      # \"t2\" => \"time:%Y-%m-%d %H:%M:%S.%N\",\n      with_timezone(\"UTC+02\") do # -0200\n        @i.configure(config_element('parse', '', @hash))\n        c = @i.type_converters[\"t2\"]\n        assert_nil c.call(\"\")\n        assert_equal_event_time event_time(\"2016-10-21 01:54:30.123000000 -0200\"), c.call(\"2016-10-21 01:54:30.123\")\n        assert_equal_event_time event_time(\"2016-10-21 01:54:30.012345678 -0200\"), c.call(\"2016-10-21 01:54:30.012345678\")\n        assert_nil c.call(\"2016/10/21 015430\")\n      end\n    end\n\n    test 'to parse time string with specified time format and timezone' do\n      # \"t3\" => \"time:+0100:%Y-%m-%d %H:%M:%S.%N\",\n      with_timezone(\"UTC+02\") do # -0200\n        @i.configure(config_element('parse', '', @hash))\n        c = @i.type_converters[\"t3\"]\n        assert_nil c.call(\"\")\n        assert_equal_event_time event_time(\"2016-10-21 01:54:30.123000000 +0100\"), c.call(\"2016-10-21 01:54:30.123\")\n        assert_equal_event_time event_time(\"2016-10-21 01:54:30.012345678 +0100\"), c.call(\"2016-10-21 01:54:30.012345678\")\n      end\n    end\n\n    test 'to parse time string in unix timestamp' do\n      # \"t4\" => \"time:unixtime\",\n      with_timezone(\"UTC+02\") do # -0200\n        @i.configure(config_element('parse', '', @hash))\n        c = @i.type_converters[\"t4\"]\n        assert_equal_event_time event_time(\"1970-01-01 00:00:00.0 +0000\"), c.call(\"\")\n        assert_equal_event_time event_time(\"2016-10-21 01:54:30.0 -0200\"), c.call(\"1477022070\")\n        assert_equal_event_time event_time(\"2016-10-21 01:54:30.0 -0200\"), c.call(\"1477022070.01\")\n      end\n    end\n\n    test 'to parse time string in floating point value' do\n      # \"t5\" => \"time:float\",\n      with_timezone(\"UTC+02\") do # -0200\n        @i.configure(config_element('parse', '', @hash))\n        c = @i.type_converters[\"t5\"]\n        assert_equal_event_time event_time(\"1970-01-01 00:00:00.0 +0000\"), c.call(\"\")\n        assert_equal_event_time event_time(\"2016-10-21 01:54:30.012 -0200\"), c.call(\"1477022070.012\")\n        assert_equal_event_time event_time(\"2016-10-21 01:54:30.123456789 -0200\"), c.call(\"1477022070.123456789\")\n      end\n    end\n\n    test 'to return array of string' do\n      @i.configure(config_element('parse', '', @hash))\n      c = @i.type_converters[\"a1\"]\n      assert_equal [], c.call(\"\")\n      assert_equal [\"0\"], c.call(\"0\")\n      assert_equal [\"0\"], c.call(0)\n      assert_equal [\"0\", \"1\"], c.call(\"0,1\")\n      assert_equal [\"0|1\", \"2\"], c.call(\"0|1,2\")\n      assert_equal [\"true\"], c.call(true)\n    end\n\n    test 'to return array of string using specified delimiter' do\n      @i.configure(config_element('parse', '', @hash))\n      c = @i.type_converters[\"a2\"]\n      assert_equal [], c.call(\"\")\n      assert_equal [\"0\"], c.call(\"0\")\n      assert_equal [\"0\"], c.call(0)\n      assert_equal [\"0,1\"], c.call(\"0,1\")\n      assert_equal [\"0\", \"1,2\"], c.call(\"0|1,2\")\n      assert_equal [\"true\"], c.call(true)\n    end\n  end\n\n  sub_test_case 'example parser without any configurations' do\n    setup do\n      @current_time = Time.parse(\"2016-10-21 14:22:01.0 +1000\")\n      @current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)\n      # @current_time.to_i #=> 1477023721\n      Timecop.freeze(@current_time)\n      @i = ExampleParser.new\n      @i.configure(config_element('parse', '', {}))\n    end\n\n    teardown do\n      Timecop.return\n    end\n\n    test 'parser returns parsed JSON object, leaving empty/NULL strings, with current time' do\n      json = '{\"t1\":\"1477023720.101\",\"s1\":\"\",\"s2\":\"NULL\",\"s3\":\"null\",\"k1\":1,\"k2\":\"13.1\",\"k3\":\"1\",\"k4\":\"yes\"}'\n      @i.parse(json) do |time, record|\n        assert_equal_event_time @current_event_time, time\n        assert_equal \"1477023720.101\", record[\"t1\"]\n        assert_equal \"\", record[\"s1\"]\n        assert_equal \"NULL\", record[\"s2\"]\n        assert_equal \"null\", record[\"s3\"]\n        assert_equal 1, record[\"k1\"]\n        assert_equal \"13.1\", record[\"k2\"]\n        assert_equal \"1\", record[\"k3\"]\n        assert_equal \"yes\", record[\"k4\"]\n      end\n    end\n  end\n\n  sub_test_case 'example parser fully configured' do\n    setup do\n      @current_time = Time.parse(\"2016-10-21 14:22:01.0 +1000\")\n      @current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)\n      # @current_time.to_i #=> 1477023721\n      Timecop.freeze(@current_time)\n      @i = ExampleParser.new\n      hash = {\n        'keep_time_key' => \"no\",\n        'estimate_current_event' => \"yes\",\n        'time_key' => \"t1\",\n        'time_type' => \"float\",\n        'null_empty_string' => 'yes',\n        'null_value_pattern' => 'NULL|null',\n        'types' => \"k1:string, k2:integer, k3:float, k4:bool\",\n      }\n      @i.configure(config_element('parse', '', hash))\n    end\n\n    teardown do\n      Timecop.return\n    end\n\n    test 'parser returns parsed JSON object, leaving empty/NULL strings, with current time' do\n      json = '{\"t1\":\"1477023720.101\",\"s1\":\"\",\"s2\":\"NULL\",\"s3\":\"null\",\"k1\":1,\"k2\":\"13.1\",\"k3\":\"1\",\"k4\":\"yes\"}'\n      @i.parse(json) do |time, record|\n        assert_equal_event_time Fluent::EventTime.new(1477023720, 101_000_000), time\n        assert !record.has_key?(\"t1\")\n        assert{ record.has_key?(\"s1\") && record[\"s1\"].nil? }\n        assert{ record.has_key?(\"s2\") && record[\"s2\"].nil? }\n        assert{ record.has_key?(\"s3\") && record[\"s3\"].nil? }\n        assert_equal \"1\", record[\"k1\"]\n        assert_equal 13, record[\"k2\"]\n        assert_equal 1.0, record[\"k3\"]\n        assert_equal true, record[\"k4\"]\n      end\n    end\n\n    test 'parser returns current time if a field is missing specified by time_key' do\n      json = '{\"s1\":\"\",\"s2\":\"NULL\",\"s3\":\"null\",\"k1\":1,\"k2\":\"13.1\",\"k3\":\"1\",\"k4\":\"yes\"}'\n      @i.parse(json) do |time, record|\n        assert_equal_event_time @current_event_time, time\n        assert !record.has_key?(\"t1\")\n        assert{ record.has_key?(\"s1\") && record[\"s1\"].nil? }\n        assert{ record.has_key?(\"s2\") && record[\"s2\"].nil? }\n        assert{ record.has_key?(\"s3\") && record[\"s3\"].nil? }\n        assert_equal \"1\", record[\"k1\"]\n        assert_equal 13, record[\"k2\"]\n        assert_equal 1.0, record[\"k3\"]\n        assert_equal true, record[\"k4\"]\n      end\n    end\n  end\n\n  sub_test_case 'example parser configured not to estimate current time, and to keep time key' do\n    setup do\n      @current_time = Time.parse(\"2016-10-21 14:22:01.0 +1000\")\n      @current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)\n      # @current_time.to_i #=> 1477023721\n      Timecop.freeze(@current_time)\n      @i = ExampleParser.new\n      hash = {\n        'keep_time_key' => \"yes\",\n        'estimate_current_event' => \"no\",\n        'time_key' => \"t1\",\n        'time_type' => \"float\",\n        'null_empty_string' => 'yes',\n        'null_value_pattern' => 'NULL|null',\n        'types' => \"k1:string, k2:integer, k3:float, k4:bool\",\n      }\n      @i.configure(config_element('parse', '', hash))\n    end\n\n    teardown do\n      Timecop.return\n    end\n\n    test 'parser returns parsed time with original field and value if the field of time exists' do\n      json = '{\"t1\":\"1477023720.101\",\"s1\":\"\",\"s2\":\"NULL\",\"s3\":\"null\",\"k1\":1,\"k2\":\"13.1\",\"k3\":\"1\",\"k4\":\"yes\"}'\n      @i.parse(json) do |time, record|\n        assert_equal_event_time Fluent::EventTime.new(1477023720, 101_000_000), time\n        assert_equal \"1477023720.101\", record[\"t1\"]\n        assert{ record.has_key?(\"s1\") && record[\"s1\"].nil? }\n        assert{ record.has_key?(\"s2\") && record[\"s2\"].nil? }\n        assert{ record.has_key?(\"s3\") && record[\"s3\"].nil? }\n        assert_equal \"1\", record[\"k1\"]\n        assert_equal 13, record[\"k2\"]\n        assert_equal 1.0, record[\"k3\"]\n        assert_equal true, record[\"k4\"]\n      end\n    end\n\n    test 'parser returns nil as time if the field of time is missing' do\n      json = '{\"s1\":\"\",\"s2\":\"NULL\",\"s3\":\"null\",\"k1\":1,\"k2\":\"13.1\",\"k3\":\"1\",\"k4\":\"yes\"}'\n      @i.parse(json) do |time, record|\n        assert_nil time\n        assert !record.has_key?(\"t1\")\n        assert{ record.has_key?(\"s1\") && record[\"s1\"].nil? }\n        assert{ record.has_key?(\"s2\") && record[\"s2\"].nil? }\n        assert{ record.has_key?(\"s3\") && record[\"s3\"].nil? }\n        assert_equal \"1\", record[\"k1\"]\n        assert_equal 13, record[\"k2\"]\n        assert_equal 1.0, record[\"k3\"]\n        assert_equal true, record[\"k4\"]\n      end\n    end\n  end\n\n  sub_test_case 'timeout' do\n    class SleepParser < Fluent::Plugin::Parser\n      attr :test_value\n\n      def configure(conf)\n        super\n\n        @test_value = nil\n      end\n\n      def parse(data)\n        sleep 10\n        @test_value = :passed\n        yield JSON.parse(data), Fluent::EventTime.now\n      end\n    end\n\n    setup do\n      @i = SleepParser.new\n      @i.instance_variable_set(:@log, Fluent::Test::TestLogger.new)\n      @i.configure(config_element('parse', '', {'timeout' => '1.0'}))\n      @i.start\n    end\n\n    teardown do\n      @i.stop\n    end\n\n    test 'stop longer processing and return nil' do\n      waiting(10) {\n        @i.parse('{\"k\":\"v\"}') do |time, record|\n          assert_nil @i.test_value\n          assert_nil time\n          assert_nil record\n        end\n        assert_true @i.log.out.logs.first.include?('parsing timed out')\n      }\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_apache.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser_apache'\n\nclass ApacheParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def create_driver(conf = {})\n    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheParser.new).configure(conf)\n  end\n\n  data('parse' => :parse, 'call' => :call)\n  def test_call(method_name)\n    d = create_driver\n    m = d.instance.method(method_name)\n    m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] \"GET / HTTP/1.1\" 200 777') { |time, record|\n      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)\n      assert_equal({\n                     'user'    => '-',\n                     'method'  => 'GET',\n                     'code'    => '200',\n                     'size'    => '777',\n                     'host'    => '192.168.0.1',\n                     'path'    => '/'\n                   }, record)\n    }\n  end\n\n  def test_parse_with_keep_time_key\n    conf = {\n      'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n      'keep_time_key' => 'true',\n    }\n    d = create_driver(conf)\n    text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] \"GET / HTTP/1.1\" 200 777'\n    d.instance.parse(text) do |_time, record|\n      assert_equal \"28/Feb/2013:12:00:00 +0900\", record['time']\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_apache2.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser'\n\nclass Apache2ParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Apache2Parser)\n    @expected = {\n      'user'    => nil,\n      'method'  => 'GET',\n      'code'    => 200,\n      'size'    => 777,\n      'host'    => '192.168.0.1',\n      'path'    => '/',\n      'referer' => nil,\n      'agent'   => 'Opera/12.0'\n    }\n    @parser.configure({})\n  end\n\n  def test_parse\n    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] \"GET / HTTP/1.1\" 200 777 \"-\" \"Opera/12.0\"') { |time, record|\n      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)\n      assert_equal(@expected, record)\n    }\n    assert_equal(Fluent::Plugin::Apache2Parser::REGEXP,\n                 @parser.instance.patterns['format'])\n    assert_equal(Fluent::Plugin::Apache2Parser::TIME_FORMAT,\n                 @parser.instance.patterns['time_format'])\n  end\n\n  def test_parse_without_http_version\n    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] \"GET /\" 200 777 \"-\" \"Opera/12.0\"') { |time, record|\n      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)\n      assert_equal(@expected, record)\n    }\n  end\n\n  def test_parse_with_escape_sequence\n    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] \"GET /\\\" HTTP/1.1\" 200 777 \"referer \\\\\\ \\\"\" \"user agent \\\\\\ \\\"\"') { |_, record|\n      assert_equal('/\\\"', record['path'])\n      assert_equal('referer \\\\\\ \\\"', record['referer'])\n      assert_equal('user agent \\\\\\ \\\"', record['agent'])\n    }\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_apache_error.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser_apache_error'\n\nclass ApacheErrorParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    @expected = {\n      'level' => 'error',\n      'client' => '127.0.0.1',\n      'message' => 'client denied by server configuration'\n    }\n  end\n\n  def create_driver\n    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheErrorParser.new).configure({})\n  end\n\n  def test_parse\n    d = create_driver\n    d.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|\n      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)\n      assert_equal(@expected, record)\n    }\n  end\n\n  def test_parse_with_pid\n    d = create_driver\n    d.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|\n      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)\n      assert_equal(@expected.merge('pid' => '1000'), record)\n    }\n  end\n\n  def test_parse_without_client\n    d = create_driver\n    d.instance.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|\n      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)\n      assert_equal({\n                     'level' => 'notice',\n                     'message' => 'Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations'\n                   }, record)\n    }\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_csv.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser_csv'\n\nclass CSVParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def create_driver(conf={})\n    Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser).configure(conf)\n  end\n\n  data('array param' => '[\"time\",\"c\",\"d\"]', 'string param' => 'time,c,d')\n  def test_parse(param)\n    d = create_driver('keys' => param, 'time_key' => 'time')\n    d.instance.parse(\"2013/02/28 12:00:00,192.168.0.1,111\") { |time, record|\n      assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)\n      assert_equal({\n                     'c' => '192.168.0.1',\n                     'd' => '111',\n                   }, record)\n    }\n  end\n\n  data('array param' => '[\"c\",\"d\"]', 'string param' => 'c,d')\n  def test_parse_without_time(param)\n    time_at_start = Time.now.to_i\n\n    d = create_driver('keys' => param)\n    d.instance.parse(\"192.168.0.1,111\") { |time, record|\n      assert time && time >= time_at_start, \"parser puts current time without time input\"\n      assert_equal({\n                     'c' => '192.168.0.1',\n                     'd' => '111',\n                   }, record)\n    }\n\n    d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)\n    d.configure('keys' => param, 'estimate_current_event' => 'no')\n    d.instance.parse(\"192.168.0.1,111\") { |time, record|\n      assert_equal({\n                     'c' => '192.168.0.1',\n                     'd' => '111',\n                   }, record)\n      assert_nil time, \"parser returns nil w/o time and if configured so\"\n    }\n  end\n\n  def test_parse_with_keep_time_key\n    d = create_driver(\n                     'keys'=>'time',\n                     'time_key'=>'time',\n                     'time_format'=>\"%d/%b/%Y:%H:%M:%S %z\",\n                     'keep_time_key'=>'true',\n                     )\n    text = '28/Feb/2013:12:00:00 +0900'\n    d.instance.parse(text) do |time, record|\n      assert_equal text, record['time']\n    end\n  end\n\n  data('array param' => '[\"a\",\"b\",\"c\",\"d\",\"e\",\"f\"]', 'string param' => 'a,b,c,d,e,f')\n  def test_parse_with_null_value_pattern(param)\n    d = create_driver(\n                     'keys'=>param,\n                     'null_value_pattern'=>'^(-|null|NULL)$'\n                     )\n    d.instance.parse(\"-,null,NULL,,--,nuLL\") do |time, record|\n      assert_nil record['a']\n      assert_nil record['b']\n      assert_nil record['c']\n      assert_nil record['d']\n      assert_equal record['e'], '--'\n      assert_equal record['f'], 'nuLL'\n    end\n  end\n\n  data('array param' => '[\"a\",\"b\"]', 'string param' => 'a,b')\n  def test_parse_with_null_empty_string(param)\n    d = create_driver(\n                     'keys'=>param,\n                     'null_empty_string'=>true\n                     )\n    d.instance.parse(\", \") do |time, record|\n      assert_nil record['a']\n      assert_equal record['b'], ' '\n    end\n  end\n\n  data('array param' => '[\"a\",\"b\",\"c\"]', 'string param' => 'a,b,c')\n  def test_parse_with_option_delimiter(param)\n    d = create_driver(\n                     'keys'=>param,\n                     'delimiter'=>' ',\n                     )\n    d.instance.parse(\"123 456 789\") do |time, record|\n      assert_equal record['a'], '123'\n      assert_equal record['b'], '456'\n      assert_equal record['c'], '789'\n    end\n  end\n\n  sub_test_case 'parser' do\n    data('normal' => 'normal',\n         'fast' => 'fast')\n    def test_compatibility_between_normal_and_fast_parser(param)\n      d = create_driver(\n        'keys' => 'time,key1,key2,key3,key4,key5',\n        'time_key' => 'time',\n        'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n        'keep_time_key' => 'false',\n        'parser_type' => param\n      )\n\n      # non quoted\n      text = '28/Feb/2013:12:00:00 +0900,value1,value2,value3,value4,value5'\n      expected = {'key1' => 'value1', 'key2' => 'value2', 'key3' => \"value3\",\n                  'key4' => 'value4', 'key5' => \"value5\"}\n      d.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"28/Feb/2013:12:00:00 +0900\", format: '%d/%b/%Y:%H:%M:%S %z'), time)\n        assert_equal expected, record\n      end\n\n      # quoted\n      text = '28/Feb/2013:12:00:00 +0900,\"value1\",\"val,ue2\",\"va,lu,e3\",\"val ue4\",\"\"'\n      expected = {'key1' => 'value1', 'key2' => 'val,ue2', 'key3' => \"va,lu,e3\",\n                  'key4' => 'val ue4', 'key5' => \"\"}\n      d.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"28/Feb/2013:12:00:00 +0900\", format: '%d/%b/%Y:%H:%M:%S %z'), time)\n        assert_equal expected, record\n      end\n\n      # mixed\n      text = '28/Feb/2013:12:00:00 +0900,message,\"mes,sage\",\"me,ssa,ge\",mess age,\"\"'\n      expected = {'key1' => 'message', 'key2' => 'mes,sage', 'key3' => \"me,ssa,ge\",\n                  'key4' => 'mess age', 'key5' => \"\"}\n      d.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"28/Feb/2013:12:00:00 +0900\", format: '%d/%b/%Y:%H:%M:%S %z'), time)\n        assert_equal expected, record\n      end\n\n      # escaped\n      text = '28/Feb/2013:12:00:00 +0900,\"message\",\"mes\"\"sage\",\"\"\"message\"\"\",,\"\"\"\"\"\"'\n      expected = {'key1' => 'message', 'key2' => 'mes\"sage', 'key3' => '\"message\"',\n                  'key4' => nil, 'key5' => '\"\"'}\n      d.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"28/Feb/2013:12:00:00 +0900\", format: '%d/%b/%Y:%H:%M:%S %z'), time)\n        assert_equal expected, record\n      end\n    end\n\n    def test_incompatibility_between_normal_and_fast_parser\n      normal = create_driver(\n        'keys' => 'key1,key2',\n        'parser_type' => 'normal'\n      )\n      fast = create_driver(\n        'keys' => 'key1,key2',\n        'parser_type' => 'fast'\n      )\n\n      # unexpected quote position\n      text = 'a\"b,\"a\"\"\"c\"'\n      assert_raise(CSV::MalformedCSVError) {\n        normal.instance.parse(text) { |t, r| }\n      }\n      assert_nothing_raised {\n        # generate broken record\n        fast.instance.parse(text) { |t, r| }\n      }\n\n      # incorrect the number of column\n      text = 'a,b,c'\n      expected = {\"key1\" => 'a', \"key2\" => 'b'}\n      normal.instance.parse(text) { |t, r|\n        assert_equal expected, r\n      }\n      fast.instance.parse(text) { |t, r|\n        assert_not_equal expected, r\n      }\n\n      # And more...\n    end\n  end\n\n  # \"parser_type\" config shouldn't hide Fluent::Plugin::Parser#plugin_type\n  # https://github.com/fluent/fluentd/issues/3296\n  data('normal' => :normal, 'fast' => :fast)\n  def test_parser_type_method(engine)\n    d = create_driver('keys' => '[\"time\"]','time_key' => 'time', 'parser_type' => engine.to_s)\n    assert_equal(:text_per_line, d.instance.parser_type)\n  end\n\n  data('normal' => :normal, 'fast' => :fast)\n  def test_parser_engine(engine)\n    d = create_driver('keys' => '[\"time\"]', 'time_key' => 'time', 'parser_engine' => engine.to_s)\n    assert_equal(engine, d.instance.parser_engine)\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_json.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser'\n\nclass JsonParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)\n  end\n\n  sub_test_case \"configure_json_parser\" do\n    data(\"oj\", [:oj, [Oj.method(:load), Oj::ParseError]])\n    data(\"json\", [:json, [JSON.method(:parse), JSON::ParserError]])\n    data(\"yajl\", [:yajl, [Yajl.method(:load), Yajl::ParseError]])\n    def test_return_each_loader((input, expected_return))\n      result = @parser.instance.configure_json_parser(input)\n      assert_equal expected_return, result\n    end\n\n    def test_raise_exception_for_unknown_input\n      assert_raise RuntimeError do\n        @parser.instance.configure_json_parser(:unknown)\n      end\n    end\n\n    def test_fall_back_oj_to_json_if_oj_not_available\n      stub(Fluent::OjOptions).available? { false }\n\n      result = @parser.instance.configure_json_parser(:oj)\n\n      assert_equal [JSON.method(:parse), JSON::ParserError], result\n      logs = @parser.logs.collect do |log|\n        log.gsub(/\\A\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2} [-+]\\d{4} /, \"\")\n      end\n      assert_equal(\n        [\"[info]: Oj is not installed, and failing back to JSON for json parser\\n\"],\n        logs\n      )\n    end\n  end\n\n  data('oj' => 'oj', 'yajl' => 'yajl')\n  def test_parse(data)\n    @parser.configure('json_parser' => data)\n    @parser.instance.parse('{\"time\":1362020400,\"host\":\"192.168.0.1\",\"size\":777,\"method\":\"PUT\"}') { |time, record|\n      assert_equal(event_time('2013-02-28 12:00:00 +0900').to_i, time)\n      assert_equal({\n                     'host'   => '192.168.0.1',\n                     'size'   => 777,\n                     'method' => 'PUT',\n                   }, record)\n    }\n  end\n\n  data('oj' => 'oj', 'yajl' => 'yajl')\n  def test_parse_with_large_float(data)\n    @parser.configure('json_parser' => data)\n    @parser.instance.parse('{\"num\":999999999999999999999999999999.99999}') { |time, record|\n      assert_equal(Float, record['num'].class)\n    }\n  end\n\n  data('oj' => 'oj', 'yajl' => 'yajl')\n  def test_parse_without_time(data)\n    time_at_start = Time.now.to_i\n\n    @parser.configure('json_parser' => data)\n    @parser.instance.parse('{\"host\":\"192.168.0.1\",\"size\":777,\"method\":\"PUT\"}') { |time, record|\n      assert time && time >= time_at_start, \"parser puts current time without time input\"\n      assert_equal({\n                     'host'   => '192.168.0.1',\n                     'size'   => 777,\n                     'method' => 'PUT',\n                   }, record)\n    }\n\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)\n    parser.configure('json_parser' => data, 'estimate_current_event' => 'false')\n    parser.instance.parse('{\"host\":\"192.168.0.1\",\"size\":777,\"method\":\"PUT\"}') { |time, record|\n      assert_equal({\n                     'host'   => '192.168.0.1',\n                     'size'   => 777,\n                     'method' => 'PUT',\n                   }, record)\n      assert_nil time, \"parser return nil w/o time and if specified so\"\n    }\n  end\n\n  data('oj' => 'oj', 'yajl' => 'yajl')\n  def test_parse_with_colon_string(data)\n    @parser.configure('json_parser' => data)\n    @parser.instance.parse('{\"time\":1362020400,\"log\":\":message\"}') { |time, record|\n      assert_equal(record['log'], ':message')\n    }\n  end\n\n  data('oj' => 'oj', 'yajl' => 'yajl')\n  def test_parse_with_invalid_time(data)\n    @parser.configure('json_parser' => data)\n    assert_raise Fluent::ParserError do\n      @parser.instance.parse('{\"time\":[],\"k\":\"v\"}') { |time, record| }\n    end\n  end\n\n  data('oj' => 'oj', 'yajl' => 'yajl')\n  def test_parse_float_time(data)\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)\n    parser.configure('json_parser' => data)\n    text = \"100.1\"\n    parser.instance.parse(\"{\\\"time\\\":\\\"#{text}\\\"}\") do |time, record|\n      assert_equal 100, time.sec\n      assert_equal 100_000_000, time.nsec\n    end\n  end\n\n  data('oj' => 'oj', 'yajl' => 'yajl')\n  def test_parse_with_keep_time_key(data)\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)\n    format = \"%d/%b/%Y:%H:%M:%S %z\"\n    parser.configure(\n                     'time_format' => format,\n                     'keep_time_key' => 'true',\n                     'json_parser' => data\n                     )\n    text = \"28/Feb/2013:12:00:00 +0900\"\n    parser.instance.parse(\"{\\\"time\\\":\\\"#{text}\\\"}\") do |time, record|\n      assert_equal Time.strptime(text, format).to_i, time.sec\n      assert_equal text, record['time']\n    end\n  end\n\n  data('oj' => 'oj', 'yajl' => 'yajl')\n  def test_parse_with_keep_time_key_without_time_format(data)\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)\n    parser.configure(\n                     'keep_time_key' => 'true',\n                     'json_parser' => data\n                     )\n    text = \"100\"\n    parser.instance.parse(\"{\\\"time\\\":\\\"#{text}\\\"}\") do |time, record|\n      assert_equal text.to_i, time.sec\n      assert_equal text, record['time']\n    end\n  end\n\n  def test_yajl_parse_io_with_buffer_smaller_than_input\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)\n    parser.configure(\n                     'keep_time_key' => 'true',\n                     'json_parser' => 'yajl',\n                     'stream_buffer_size' => 1,\n                     )\n    text = \"100\"\n\n    waiting(5) do\n      rd, wr = IO.pipe\n      wr.write \"{\\\"time\\\":\\\"#{text}\\\"}\"\n\n      parser.instance.parse_io(rd) do |time, record|\n        assert_equal text.to_i, time.sec\n        assert_equal text, record['time']\n\n        # Once a record has been received the 'write' end of the pipe must be\n        # closed, otherwise the test will block waiting for more input.\n        wr.close\n      end\n    end\n  end\n\n  sub_test_case \"various record pattern\" do\n    data(\"Only string\", { record: '\"message\"', expected: [nil] }, keep: true)\n    data(\"Only string without quotation\", { record: \"message\", expected: [nil] }, keep: true)\n    data(\"Only number\", { record: \"0\", expected: [nil] }, keep: true)\n    data(\n      \"Array of Hash\",\n      {\n        record: '[{\"k1\": 1}, {\"k2\": 2}]',\n        expected: [{\"k1\" => 1}, {\"k2\" => 2}]\n      },\n      keep: true,\n    )\n    data(\n      \"Array of both Hash and invalid\",\n      {\n        record: '[{\"k1\": 1}, \"string\", {\"k2\": 2}, 0]',\n        expected: [{\"k1\" => 1}, nil, {\"k2\" => 2}, nil]\n      },\n      keep: true,\n    )\n    data(\n      \"Array of all invalid\",\n      {\n        record: '[\"string\", 0, [{\"k\": 0}]]',\n        expected: [nil, nil, nil]\n      },\n      keep: true,\n    )\n\n    def test_oj(data)\n      parsed_records = []\n      @parser.configure(\"json_parser\" => \"oj\")\n      @parser.instance.parse(data[:record]) { |time, record|\n        parsed_records.append(record)\n      }\n      assert_equal(data[:expected], parsed_records)\n    end\n\n    def test_yajl(data)\n      parsed_records = []\n      @parser.configure(\"json_parser\" => \"yajl\")\n      @parser.instance.parse(data[:record]) { |time, record|\n        parsed_records.append(record)\n      }\n      assert_equal(data[:expected], parsed_records)\n    end\n\n    def test_json(json)\n      parsed_records = []\n      @parser.configure(\"json_parser\" => \"json\")\n      @parser.instance.parse(data[:record]) { |time, record|\n        parsed_records.append(record)\n      }\n      assert_equal(data[:expected], parsed_records)\n    end\n  end\n\n  # This becomes NoMethodError if a non-Hash object is passed to convert_values.\n  # https://github.com/fluent/fluentd/issues/4100\n  sub_test_case \"execute_convert_values with null_empty_string\" do\n    data(\"Only string\", { record: '\"message\"', expected: [nil] }, keep: true)\n    data(\n      \"Hash\",\n      {\n        record: '{\"k1\": 1, \"k2\": \"\"}',\n        expected: [{\"k1\" => 1, \"k2\" => nil}]\n      },\n      keep: true,\n    )\n    data(\n      \"Array of Hash\",\n      {\n        record: '[{\"k1\": 1}, {\"k2\": \"\"}]',\n        expected: [{\"k1\" => 1}, {\"k2\" => nil}]\n      },\n      keep: true,\n    )\n\n    def test_oj(data)\n      parsed_records = []\n      @parser.configure(\"json_parser\" => \"oj\", \"null_empty_string\" => true)\n      @parser.instance.parse(data[:record]) { |time, record|\n        parsed_records.append(record)\n      }\n      assert_equal(data[:expected], parsed_records)\n    end\n\n    def test_yajl(data)\n      parsed_records = []\n      @parser.configure(\"json_parser\" => \"yajl\", \"null_empty_string\" => true)\n      @parser.instance.parse(data[:record]) { |time, record|\n        parsed_records.append(record)\n      }\n      assert_equal(data[:expected], parsed_records)\n    end\n\n    def test_json(json)\n      parsed_records = []\n      @parser.configure(\"json_parser\" => \"json\", \"null_empty_string\" => true)\n      @parser.instance.parse(data[:record]) { |time, record|\n        parsed_records.append(record)\n      }\n      assert_equal(data[:expected], parsed_records)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_labeled_tsv.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser'\n\nclass LabeledTSVParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def test_config_params\n    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)\n\n    assert_equal \"\\t\", parser.instance.delimiter\n    assert_equal  \":\", parser.instance.label_delimiter\n\n    parser.configure(\n                     'delimiter'       => ',',\n                     'label_delimiter' => '=',\n                     )\n\n    assert_equal \",\", parser.instance.delimiter\n    assert_equal \"=\", parser.instance.label_delimiter\n  end\n\n  def test_parse\n    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)\n    parser.configure({})\n    parser.instance.parse(\"time:2013/02/28 12:00:00\\thost:192.168.0.1\\treq_id:111\") { |time, record|\n      assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)\n      assert_equal({\n                     'host'   => '192.168.0.1',\n                     'req_id' => '111',\n                   }, record)\n    }\n  end\n\n  def test_parse_with_customized_delimiter\n    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)\n    parser.configure(\n                     'delimiter'       => ',',\n                     'label_delimiter' => '=',\n                     )\n    parser.instance.parse('time=2013/02/28 12:00:00,host=192.168.0.1,req_id=111') { |time, record|\n      assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)\n      assert_equal({\n                     'host'   => '192.168.0.1',\n                     'req_id' => '111',\n                   }, record)\n    }\n  end\n\n  def test_parse_with_customized_time_format\n    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)\n    parser.configure(\n                     'time_key'    => 'mytime',\n                     'time_format' => '%d/%b/%Y:%H:%M:%S %z',\n                     )\n    parser.instance.parse(\"mytime:28/Feb/2013:12:00:00 +0900\\thost:192.168.0.1\\treq_id:111\") { |time, record|\n      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)\n      assert_equal({\n                     'host'   => '192.168.0.1',\n                     'req_id' => '111',\n                   }, record)\n    }\n  end\n\n  def test_parse_without_time\n    time_at_start = Time.now.to_i\n\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)\n    parser.configure({})\n    parser.instance.parse(\"host:192.168.0.1\\treq_id:111\") { |time, record|\n      assert time && time >= time_at_start, \"parser puts current time without time input\"\n      assert_equal({\n                     'host'   => '192.168.0.1',\n                     'req_id' => '111',\n                   }, record)\n    }\n\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)\n    parser.configure({'estimate_current_event' => 'no'})\n    parser.instance.parse(\"host:192.168.0.1\\treq_id:111\") { |time, record|\n      assert_equal({\n                     'host'   => '192.168.0.1',\n                     'req_id' => '111',\n                   }, record)\n      assert_nil time, \"parser returns nil w/o time and if configured so\"\n    }\n  end\n\n  def test_parse_with_keep_time_key\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)\n    parser.configure(\n                     'time_format'=>\"%d/%b/%Y:%H:%M:%S %z\",\n                     'keep_time_key'=>'true',\n                     )\n    text = '28/Feb/2013:12:00:00 +0900'\n    parser.instance.parse(\"time:#{text}\") do |time, record|\n      assert_equal text, record['time']\n    end\n  end\n\n  def test_parse_and_reject_invalid_kv_pairs\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)\n    parser.configure(\n                     'delimiter'       => ' ',\n                     'label_delimiter' => '=',\n                     )\n    text = 'A leading portion that is not LTSV    :  foo=bar baz=derp and a trailing portion'\n\n    expected = {'foo' => 'bar', 'baz' => 'derp'}\n    parser.instance.parse(text) do |time, record|\n      assert_equal expected, record\n    end\n\n  end\n\n  def test_parse_with_null_value_pattern\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)\n    parser.configure(\n                     'null_value_pattern'=>'^(-|null|NULL)$'\n                     )\n    parser.instance.parse(\"a:-\\tb:null\\tc:NULL\\td:\\te:--\\tf:nuLL\") do |time, record|\n      assert_nil record['a']\n      assert_nil record['b']\n      assert_nil record['c']\n      assert_equal record['d'], ''\n      assert_equal record['e'], '--'\n      assert_equal record['f'], 'nuLL'\n    end\n  end\n\n  def test_parse_with_null_empty_string\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)\n    parser.configure(\n                     'null_empty_string'=>true\n                     )\n    parser.instance.parse(\"a:\\tb: \") do |time, record|\n      assert_nil record['a']\n      assert_equal record['b'], ' '\n    end\n  end\n\n  data(\"single space\" => [\"k1=v1 k2=v2\", { \"k1\" => \"v1\", \"k2\" => \"v2\" }],\n       \"multiple space\" => [\"k1=v1    k2=v2\", { \"k1\" => \"v1\", \"k2\" => \"v2\" }],\n       \"reverse\" => [\"k2=v2 k1=v1\", { \"k1\" => \"v1\", \"k2\" => \"v2\" }],\n       \"tab\" => [\"k2=v2\\tk1=v1\", { \"k1\" => \"v1\", \"k2\" => \"v2\" }],\n       \"tab and space\" => [\"k2=v2\\t k1=v1\", { \"k1\" => \"v1\", \"k2\" => \"v2\" }])\n  def test_parse_with_delimiter_pattern(data)\n    text, expected = data\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)\n    parser.configure(\n                     'delimiter_pattern' => '/\\s+/',\n                     'label_delimiter' => '='\n                    )\n    parser.instance.parse(text) do |_time, record|\n      assert_equal(expected, record)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_msgpack.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser_msgpack'\n\nclass MessagePackParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Parser.new(Fluent::Plugin::MessagePackParser).configure(conf)\n  end\n\n  sub_test_case \"simple setting\" do\n    data(\n      \"Normal Hash\",\n      {\n        input: \"\\x82\\xA7message\\xADHello msgpack\\xA3numd\",\n        expected: [{\"message\" => \"Hello msgpack\", \"num\" => 100}]\n      },\n      keep: true\n    )\n    data(\n      \"Array of multiple Hash\",\n      {\n        input: \"\\x92\\x81\\xA7message\\xA3foo\\x81\\xA7message\\xA3bar\",\n        expected: [{\"message\"=>\"foo\"}, {\"message\"=>\"bar\"}]\n      },\n      keep: true\n    )\n    data(\n      \"String\",\n      {\n        # \"Hello msgpack\".to_msgpack\n        input: \"\\xADHello msgpack\",\n        expected: [nil]\n      },\n      keep: true\n    )\n    data(\n      \"Array of String\",\n      {\n        # [\"foo\", \"bar\"].to_msgpack\n        input: \"\\x92\\xA3foo\\xA3bar\",\n        expected: [nil, nil]\n      },\n      keep: true\n    )\n    data(\n      \"Array of String and Hash\",\n      {\n        # [\"foo\", {message: \"bar\"}].to_msgpack\n        input: \"\\x92\\xA3foo\\x81\\xA7message\\xA3bar\",\n        expected: [nil, {\"message\"=>\"bar\"}]\n      },\n      keep: true\n    )\n\n    def test_parse(data)\n      parsed_records = []\n      create_driver(\"\").instance.parse(data[:input]) do |time, record|\n        parsed_records.append(record)\n      end\n      assert_equal(data[:expected], parsed_records)\n    end\n\n    def test_parse_io(data)\n      parsed_records = []\n      StringIO.open(data[:input]) do |io|\n        create_driver(\"\").instance.parse_io(io) do |time, record|\n          parsed_records.append(record)\n        end\n      end\n      assert_equal(data[:expected], parsed_records)\n    end\n  end\n\n  # This becomes NoMethodError if a non-Hash object is passed to convert_values.\n  # https://github.com/fluent/fluentd/issues/4100\n  sub_test_case \"execute_convert_values with null_empty_string\" do\n    data(\n      \"Normal hash\",\n      {\n        # {message: \"foo\", empty: \"\"}.to_msgpack\n        input: \"\\x82\\xA7message\\xA3foo\\xA5empty\\xA0\",\n        expected: [{\"message\" => \"foo\", \"empty\" => nil}]\n      },\n      keep: true\n    )\n    data(\n      \"Array of multiple Hash\",\n      {\n        # [{message: \"foo\", empty: \"\"}, {message: \"bar\", empty: \"\"}].to_msgpack\n        input: \"\\x92\\x82\\xA7message\\xA3foo\\xA5empty\\xA0\\x82\\xA7message\\xA3bar\\xA5empty\\xA0\",\n        expected: [{\"message\"=>\"foo\", \"empty\" => nil}, {\"message\"=>\"bar\", \"empty\" => nil}]\n      },\n      keep: true\n    )\n    data(\n      \"String\",\n      {\n        # \"Hello msgpack\".to_msgpack\n        input: \"\\xADHello msgpack\",\n        expected: [nil]\n      },\n      keep: true\n    )\n\n    def test_parse(data)\n      parsed_records = []\n      create_driver(\"null_empty_string\").instance.parse(data[:input]) do |time, record|\n        parsed_records.append(record)\n      end\n      assert_equal(data[:expected], parsed_records)\n    end\n\n    def test_parse_io(data)\n      parsed_records = []\n      StringIO.open(data[:input]) do |io|\n        create_driver(\"null_empty_string\").instance.parse_io(io) do |time, record|\n          parsed_records.append(record)\n        end\n      end\n      assert_equal(data[:expected], parsed_records)\n    end\n  end\nend"
  },
  {
    "path": "test/plugin/test_parser_multiline.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser'\n\nclass MultilineParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def create_parser(conf)\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultilineParser).configure(conf)\n    parser\n  end\n\n  def test_configure_with_invalid_params\n    [{'format100' => '/(?<msg>.*)/'}, {'format1' => '/(?<msg>.*)/', 'format3' => '/(?<msg>.*)/'}, 'format1' => '/(?<msg>.*)'].each { |config|\n      assert_raise(Fluent::ConfigError) {\n        create_parser(config)\n      }\n    }\n  end\n\n  def test_parse\n    parser = create_parser('format1' => '/^(?<time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2}) \\[(?<thread>.*)\\] (?<level>[^\\s]+)(?<message>.*)/')\n    parser.instance.parse(<<EOS.chomp) { |time, record|\n2013-3-03 14:27:33 [main] ERROR Main - Exception\njavax.management.RuntimeErrorException: null\n\\tat Main.main(Main.java:16) ~[bin/:na]\nEOS\n\n      assert_equal(event_time('2013-3-03 14:27:33').to_i, time)\n      assert_equal({\n                     \"thread\"  => \"main\",\n                     \"level\"   => \"ERROR\",\n                     \"message\" => \" Main - Exception\\njavax.management.RuntimeErrorException: null\\n\\tat Main.main(Main.java:16) ~[bin/:na]\"\n                   }, record)\n    }\n  end\n\n  def test_parse_with_firstline\n    parser = create_parser('format_firstline' => '/----/', 'format1' => '/time=(?<time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2}).*message=(?<message>.*)/')\n    parser.instance.parse(<<EOS.chomp) { |time, record|\n----\ntime=2013-3-03 14:27:33\nmessage=test1\nEOS\n\n      assert(parser.instance.firstline?('----'))\n      assert_equal(event_time('2013-3-03 14:27:33').to_i, time)\n      assert_equal({\"message\" => \"test1\"}, record)\n    }\n  end\n\n  def test_parse_with_multiple_formats\n    parser = create_parser('format_firstline' => '/^Started/',\n                           'format1' => '/Started (?<method>[^ ]+) \"(?<path>[^\"]+)\" for (?<host>[^ ]+) at (?<time>[^ ]+ [^ ]+ [^ ]+)\\n/',\n                           'format2' => '/Processing by (?<controller>[^\\u0023]+)\\u0023(?<controller_method>[^ ]+) as (?<format>[^ ]+?)\\n/',\n                           'format3' => '/(  Parameters: (?<parameters>[^ ]+)\\n)?/',\n                           'format4' => '/  Rendered (?<template>[^ ]+) within (?<layout>.+) \\([\\d\\.]+ms\\)\\n/',\n                           'format5' => '/Completed (?<code>[^ ]+) [^ ]+ in (?<runtime>[\\d\\.]+)ms \\(Views: (?<view_runtime>[\\d\\.]+)ms \\| ActiveRecord: (?<ar_runtime>[\\d\\.]+)ms\\)/'\n                           )\n    parser.instance.parse(<<EOS.chomp) { |time, record|\nStarted GET \"/users/123/\" for 127.0.0.1 at 2013-06-14 12:00:11 +0900\nProcessing by UsersController#show as HTML\n  Parameters: {\"user_id\"=>\"123\"}\n  Rendered users/show.html.erb within layouts/application (0.3ms)\nCompleted 200 OK in 4ms (Views: 3.2ms | ActiveRecord: 0.0ms)\nEOS\n\n      assert(parser.instance.firstline?('Started GET \"/users/123/\" for 127.0.0.1...'))\n      assert_equal(event_time('2013-06-14 12:00:11 +0900').to_i, time)\n      assert_equal({\n                     \"method\" => \"GET\",\n                     \"path\" => \"/users/123/\",\n                     \"host\" => \"127.0.0.1\",\n                     \"controller\" => \"UsersController\",\n                     \"controller_method\" => \"show\",\n                     \"format\" => \"HTML\",\n                     \"parameters\" => \"{\\\"user_id\\\"=>\\\"123\\\"}\",\n                     \"template\" => \"users/show.html.erb\",\n                     \"layout\" => \"layouts/application\",\n                     \"code\" => \"200\",\n                     \"runtime\" => \"4\",\n                     \"view_runtime\" => \"3.2\",\n                     \"ar_runtime\" => \"0.0\"\n                   }, record)\n    }\n  end\n\n  def test_parse_with_keep_time_key\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultilineParser).configure(\n      'format1' => '/^(?<time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2})/',\n      'keep_time_key' => 'true'\n    )\n    text = '2013-3-03 14:27:33'\n    parser.instance.parse(text) { |time, record|\n      assert_equal text, record['time']\n    }\n  end\n\n  def test_parse_unmatched_lines\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultilineParser).configure(\n      'format1' => '/^message (?<message_id>\\d)/',\n      'unmatched_lines' => true,\n    )\n    text = \"message 1\\nmessage a\"\n    r = []\n    parser.instance.parse(text) { |_, record| r << record }\n    assert_equal [{ 'message_id' => '1' }, { 'unmatched_line' => 'message a'}], r\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_nginx.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser_nginx'\n\nclass NginxParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    @expected = {\n      'remote'  => '127.0.0.1',\n      'host'    => '192.168.0.1',\n      'user'    => '-',\n      'method'  => 'GET',\n      'path'    => '/',\n      'code'    => '200',\n      'size'    => '777',\n      'referer' => '-',\n      'agent'   => 'Opera/12.0'\n    }\n    @expected_extended = {\n      'remote'               => '127.0.0.1',\n      'host'                 => '192.168.0.1',\n      'user'                 => '-',\n      'method'               => 'GET',\n      'path'                 => '/',\n      'code'                 => '200',\n      'size'                 => '777',\n      'referer'              => '-',\n      'agent'                => 'Opera/12.0',\n      'http_x_forwarded_for' => '-'\n    }\n    @expected_extended_multiple_ip = {\n      'remote'               => '127.0.0.1',\n      'host'                 => '192.168.0.1',\n      'user'                 => '-',\n      'method'               => 'GET',\n      'path'                 => '/',\n      'code'                 => '200',\n      'size'                 => '777',\n      'referer'              => '-',\n      'agent'                => 'Opera/12.0',\n      'http_x_forwarded_for' => '127.0.0.1, 192.168.0.1'\n    }\n  end\n\n  def create_driver\n    Fluent::Test::Driver::Parser.new(Fluent::Plugin::NginxParser.new).configure({})\n  end\n\n  def test_parse\n    d = create_driver\n    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] \"GET / HTTP/1.1\" 200 777 \"-\" \"Opera/12.0\"') { |time, record|\n      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)\n      assert_equal(@expected, record)\n    }\n  end\n\n  def test_parse_with_empty_included_path\n    d = create_driver\n    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] \"GET /a[ ]b HTTP/1.1\" 200 777 \"-\" \"Opera/12.0\"') { |time, record|\n      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)\n      assert_equal(@expected.merge('path' => '/a[ ]b'), record)\n    }\n  end\n\n  def test_parse_without_http_version\n    d = create_driver\n    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] \"GET /\" 200 777 \"-\" \"Opera/12.0\"') { |time, record|\n      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)\n      assert_equal(@expected, record)\n    }\n  end\n\n  def test_parse_with_http_x_forwarded_for\n    d = create_driver\n    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] \"GET / HTTP/1.1\" 200 777 \"-\" \"Opera/12.0\" -') { |time, record|\n      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)\n      assert_equal(@expected_extended, record)\n    }\n  end\n\n  def test_parse_with_http_x_forwarded_for_multiple_ip\n    d = create_driver\n    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] \"GET / HTTP/1.1\" 200 777 \"-\" \"Opera/12.0\" \"127.0.0.1, 192.168.0.1\"') { |time, record|\n      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)\n      assert_equal(@expected_extended_multiple_ip, record)\n    }\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_none.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser'\n\nclass NoneParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def test_config_params\n    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::NoneParser)\n    parser.configure({})\n    assert_equal \"message\", parser.instance.message_key\n\n    parser.configure('message_key' => 'foobar')\n    assert_equal \"foobar\", parser.instance.message_key\n  end\n\n  def test_parse\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('none'))\n    parser.configure({})\n    parser.instance.parse('log message!') { |time, record|\n      assert_equal({'message' => 'log message!'}, record)\n    }\n  end\n\n  def test_parse_with_message_key\n    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::NoneParser)\n    parser.configure('message_key' => 'foobar')\n    parser.instance.parse('log message!') { |time, record|\n      assert_equal({'foobar' => 'log message!'}, record)\n    }\n  end\n\n  def test_parse_without_default_time\n    time_at_start = Time.now.to_i\n\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('none'))\n    parser.configure({})\n    parser.instance.parse('log message!') { |time, record|\n      assert time && time >= time_at_start, \"parser puts current time without time input\"\n      assert_equal({'message' => 'log message!'}, record)\n    }\n\n    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('none'))\n    parser.configure({'estimate_current_event' => 'false'})\n    parser.instance.parse('log message!') { |time, record|\n      assert_equal({'message' => 'log message!'}, record)\n      assert_nil time, \"parser returns nil w/o time if configured so\"\n    }\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_regexp.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser'\n\nclass RegexpParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def internal_test_case(parser)\n    text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] [14/Feb/2013:12:00:00 +0900] \"true /,/user HTTP/1.1\" 200 777'\n    parser.parse(text) { |time, record|\n      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)\n      assert_equal({\n                     'user' => '-',\n                     'flag' => true,\n                     'code' => 200.0,\n                     'size' => 777,\n                     'date' => event_time('14/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'),\n                     'host' => '192.168.0.1',\n                     'path' => ['/', '/user']\n                   }, record)\n    }\n  end\n\n  sub_test_case \"Fluent::Compat::TextParser::RegexpParser\" do\n    def create_driver(regexp, conf = {}, initialize_conf: false)\n      if initialize_conf\n        Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))\n      else\n        Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)\n      end\n    end\n\n    def test_parse_with_typed\n      # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0\n      regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \\[(?<date>[^\\]]*)\\] \"(?<flag>\\S+)(?: +(?<path>[^ ]*) +\\S*)?\" (?<code>[^ ]*) (?<size>[^ ]*)$!)\n      conf = {\n        'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n        'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'\n      }\n      d = create_driver(regexp, conf, initialize_conf: true)\n      internal_test_case(d.instance)\n    end\n\n    def test_parse_with_configure\n      # Specify conf by configure method instead of initializer\n      regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \\[(?<date>[^\\]]*)\\] \"(?<flag>\\S+)(?: +(?<path>[^ ]*) +\\S*)?\" (?<code>[^ ]*) (?<size>[^ ]*)$!)\n      conf = {\n        'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n        'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'\n      }\n      d = create_driver(regexp, conf)\n      internal_test_case(d.instance)\n      assert_equal(regexp, d.instance.patterns['format'])\n      assert_equal(\"%d/%b/%Y:%H:%M:%S %z\", d.instance.patterns['time_format'])\n    end\n\n    def test_parse_with_typed_and_name_separator\n      regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \\[(?<date>[^\\]]*)\\] \"(?<flag>\\S+)(?: +(?<path>[^ ]*) +\\S*)?\" (?<code>[^ ]*) (?<size>[^ ]*)$!)\n      conf = {\n        'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n        'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',\n        'types_label_delimiter' => '|'\n      }\n      d = create_driver(regexp, conf)\n      internal_test_case(d.instance)\n    end\n\n    def test_parse_with_time_key\n      conf = {\n        'time_format' => \"%Y-%m-%d %H:%M:%S %z\",\n        'time_key' => 'logtime'\n      }\n      d = create_driver(/(?<logtime>[^\\]]*)/, conf)\n      text = '2013-02-28 12:00:00 +0900'\n      d.instance.parse(text) do |time, _record|\n        assert_equal Fluent::EventTime.parse(text), time\n      end\n    end\n\n    def test_parse_without_time\n      time_at_start = Time.now.to_i\n      text = \"tagomori_satoshi tagomoris 34\\n\"\n\n      regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\\d*)$!)\n      conf = {\n        'types' => 'name:string,user:string,age:integer'\n      }\n      d = create_driver(regexp, conf)\n\n      d.instance.parse(text) { |time, record|\n        assert time && time >= time_at_start, \"parser puts current time without time input\"\n        assert_equal \"tagomori_satoshi\", record[\"name\"]\n        assert_equal \"tagomoris\", record[\"user\"]\n        assert_equal 34, record[\"age\"]\n      }\n    end\n\n    def test_parse_without_time_estimate_current_event_false\n      text = \"tagomori_satoshi tagomoris 34\\n\"\n      regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\\d*)$!)\n      conf = {\n        'types' => 'name:string,user:string,age:integer'\n      }\n      d = create_driver(regexp, conf)\n      d.instance.estimate_current_event = false\n      d.instance.parse(text) { |time, record|\n        assert_equal \"tagomori_satoshi\", record[\"name\"]\n        assert_equal \"tagomoris\", record[\"user\"]\n        assert_equal 34, record[\"age\"]\n\n        assert_nil time, \"parser returns nil if configured so\"\n      }\n    end\n\n    def test_parse_with_keep_time_key\n      regexp = Regexp.new(%q!(?<time>.*)!)\n      conf = {\n        'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n        'keep_time_key' => 'true',\n      }\n      d = create_driver(regexp, conf)\n      text = '28/Feb/2013:12:00:00 +0900'\n      d.instance.parse(text) do |_time, record|\n        assert_equal text, record['time']\n      end\n    end\n\n    def test_parse_with_keep_time_key_with_typecast\n      regexp = Regexp.new(%q!(?<time>.*)!)\n      conf = {\n        'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n        'keep_time_key' => 'true',\n        'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',\n      }\n      d = create_driver(regexp, conf)\n      text = '28/Feb/2013:12:00:00 +0900'\n      d.instance.parse(text) do |_time, record|\n        assert_equal 1362020400, record['time']\n      end\n    end\n  end\n\n  sub_test_case \"Fluent::Plugin::RegexpParser\" do\n    def create_driver(conf)\n      Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new).configure(conf)\n    end\n\n    sub_test_case \"configure\" do\n      def test_bad_expression\n        conf = {\n          'expression' => %q!/.*/!,\n        }\n        assert_raise Fluent::ConfigError do\n          create_driver(conf)\n        end\n      end\n\n      def test_default_options\n        conf = {\n          'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \\[(?<date>[^\\]]*)\\] \"(?<flag>\\S+)(?: +(?<path>[^ ]*) +\\S*)?\" (?<code>[^ ]*) (?<size>[^ ]*)$/!,\n          'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n          'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'\n        }\n        d = create_driver(conf)\n        regexp = d.instance.expression\n        assert_equal(0, regexp.options)\n      end\n\n      data(\n        ignorecase: [\"i\", Regexp::IGNORECASE],\n        multiline: [\"m\", Regexp::MULTILINE],\n        ignorecase_multiline: [\"im\", Regexp::IGNORECASE | Regexp::MULTILINE],\n      )\n      def test_options(data)\n        regexp_option, expected = data\n        conf = {\n          'expression' => %Q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \\[(?<date>[^\\]]*)\\] \"(?<flag>\\S+)(?: +(?<path>[^ ]*) +\\S*)?\" (?<code>[^ ]*) (?<size>[^ ]*)$/#{regexp_option}!,\n          'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n          'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'\n        }\n        d = create_driver(conf)\n        regexp = d.instance.expression\n        assert_equal(expected, regexp.options)\n      end\n    end\n\n    def test_parse_with_typed\n      conf = {\n        'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \\[(?<date>[^\\]]*)\\] \"(?<flag>\\S+)(?: +(?<path>[^ ]*) +\\S*)?\" (?<code>[^ ]*) (?<size>[^ ]*)$/!,\n        'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n        'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'\n      }\n      d = create_driver(conf)\n      internal_test_case(d.instance)\n    end\n\n    def test_parse_with_typed_by_json_hash\n      conf = {\n        'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \\[(?<date>[^\\]]*)\\] \"(?<flag>\\S+)(?: +(?<path>[^ ]*) +\\S*)?\" (?<code>[^ ]*) (?<size>[^ ]*)$/!,\n        'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n        'types' => '{\"user\":\"string\",\"date\":\"time:%d/%b/%Y:%H:%M:%S %z\",\"flag\":\"bool\",\"path\":\"array\",\"code\":\"float\",\"size\":\"integer\"}',\n      }\n      d = create_driver(conf)\n      internal_test_case(d.instance)\n    end\n\n    def test_parse_with_time_key\n      conf = {\n        'expression' => %q!/(?<logtime>[^\\]]*)/!,\n        'time_format' => \"%Y-%m-%d %H:%M:%S %z\",\n        'time_key' => 'logtime'\n      }\n      d = create_driver(conf)\n      text = '2013-02-28 12:00:00 +0900'\n      d.instance.parse(text) do |time, _record|\n        assert_equal Fluent::EventTime.parse(text), time\n      end\n    end\n\n    def test_parse_without_time\n      time_at_start = Time.now.to_i\n      text = \"tagomori_satoshi tagomoris 34\\n\"\n\n      conf = {\n        'expression' => %q!/^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\\d*)$/!,\n        'types' => 'name:string,user:string,age:integer'\n      }\n      d = create_driver(conf)\n\n      d.instance.parse(text) { |time, record|\n        assert time && time >= time_at_start, \"parser puts current time without time input\"\n        assert_equal \"tagomori_satoshi\", record[\"name\"]\n        assert_equal \"tagomoris\", record[\"user\"]\n        assert_equal 34, record[\"age\"]\n      }\n    end\n\n    def test_parse_without_time_estimate_current_event_false\n      text = \"tagomori_satoshi tagomoris 34\\n\"\n      conf = {\n        'expression' => %q!/^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\\d*)$/!,\n        'types' => 'name:string,user:string,age:integer'\n      }\n      d = create_driver(conf)\n      d.instance.estimate_current_event = false\n      d.instance.parse(text) { |time, record|\n        assert_equal \"tagomori_satoshi\", record[\"name\"]\n        assert_equal \"tagomoris\", record[\"user\"]\n        assert_equal 34, record[\"age\"]\n\n        assert_nil time, \"parser returns nil if configured so\"\n      }\n    end\n\n    def test_parse_with_keep_time_key\n      conf = {\n        'expression' => %q!/(?<time>.*)/!,\n        'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n        'keep_time_key' => 'true',\n      }\n      d = create_driver(conf)\n      text = '28/Feb/2013:12:00:00 +0900'\n      d.instance.parse(text) do |_time, record|\n        assert_equal text, record['time']\n      end\n    end\n\n    def test_parse_with_keep_time_key_with_typecast\n      conf = {\n        'expression' => %q!/(?<time>.*)/!,\n        'time_format' => \"%d/%b/%Y:%H:%M:%S %z\",\n        'keep_time_key' => 'true',\n        'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',\n      }\n      d = create_driver(conf)\n      text = '28/Feb/2013:12:00:00 +0900'\n      d.instance.parse(text) do |_time, record|\n        assert_equal 1362020400, record['time']\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_syslog.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser'\n\nclass SyslogParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::SyslogParser)\n    @expected = {\n      'host'    => '192.168.0.1',\n      'ident'   => 'fluentd',\n      'pid'     => '11111',\n      'message' => '[error] Syslog test'\n    }\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse(param)\n    @parser.configure({'parser_type' => param})\n    @parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|\n      assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)\n      assert_equal(@expected, record)\n    }\n    assert_equal(Fluent::Plugin::SyslogParser::RFC3164_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])\n    assert_equal(\"%b %d %H:%M:%S\", @parser.instance.patterns['time_format'])\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse_with_time_format(param)\n    @parser.configure('time_format' => '%b %d %M:%S:%H', 'parser_type' => param)\n    @parser.instance.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|\n      assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)\n      assert_equal(@expected, record)\n    }\n    assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse_with_time_format2(param)\n    @parser.configure('time_format' => '%Y-%m-%dT%H:%M:%SZ', 'parser_type' => param)\n    @parser.instance.parse(\"#{Time.now.year}-03-03T10:14:29Z 192.168.0.1 fluentd[11111]: [error] Syslog test\") { |time, record|\n      assert_equal(event_time('Mar 03 10:14:29', format: '%b %d %H:%M:%S'), time)\n      assert_equal(@expected, record)\n    }\n    assert_equal('%Y-%m-%dT%H:%M:%SZ', @parser.instance.patterns['time_format'])\n  end\n\n  def test_parse_with_time_format_rfc5424\n    @parser.configure('time_format' => '%Y-%m-%dT%H:%M:%SZ', 'message_format' => 'rfc5424')\n    @parser.instance.parse(\"#{Time.now.year}-03-03T10:14:29Z 192.168.0.1 fluentd 11111 - - [error] Syslog test\") { |time, record|\n      assert_equal(event_time('Mar 03 10:14:29', format: '%b %d %H:%M:%S'), time)\n      assert_equal(@expected.merge('host' => '192.168.0.1', 'msgid' => '-', 'extradata' => '-'), record)\n    }\n    assert_equal('%Y-%m-%dT%H:%M:%SZ', @parser.instance.patterns['time_format'])\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse_with_subsecond_time(param)\n    @parser.configure('time_format' => '%b %d %H:%M:%S.%N', 'parser_type' => param)\n    @parser.instance.parse('Feb 28 12:00:00.456 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|\n      assert_equal(event_time('Feb 28 12:00:00.456', format: '%b %d %H:%M:%S.%N'), time)\n      assert_equal(@expected, record)\n    }\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse_with_priority(param)\n    @parser.configure('with_priority' => true, 'parser_type' => param)\n    @parser.instance.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|\n      assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)\n      assert_equal(@expected.merge('pri' => 6), record)\n    }\n    assert_equal(Fluent::Plugin::SyslogParser::RFC3164_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])\n    assert_equal(\"%b %d %H:%M:%S\", @parser.instance.patterns['time_format'])\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse_rfc5452_with_priority(param)\n    @parser.configure('with_priority' => true, 'parser_type' => param, 'message_format' => 'rfc5424')\n    @parser.instance.parse('<30>1 2020-03-31T20:32:54Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00') do |time, record|\n      assert_equal(event_time('2020-03-31T20:32:54Z', format: '%Y-%m-%dT%H:%M:%S%z'), time)\n      expected = { 'extradata' => '-', 'host' => 'myhostname', 'ident' => '02abaf0687f5', 'message' => 'method=POST db=0.00', 'msgid' => '02abaf0687f5', 'pid' => '10339', 'pri' => 30 }\n      assert_equal(expected, record)\n    end\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse_with_empty_priority(param)\n    @parser.configure('with_priority' => true, 'parser_type' => param)\n    @parser.instance.parse('<>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|\n      assert_nil time\n      assert_nil record\n    }\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse_without_colon(param)\n    @parser.configure({'parser_type' => param})\n    @parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|\n      assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)\n      assert_equal(@expected, record)\n    }\n    assert_equal(Fluent::Plugin::SyslogParser::RFC3164_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])\n    assert_equal(\"%b %d %H:%M:%S\", @parser.instance.patterns['time_format'])\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse_with_keep_time_key(param)\n    @parser.configure(\n                      'time_format' => '%b %d %M:%S:%H',\n                      'keep_time_key'=>'true',\n                      'parser_type' => param\n                      )\n    text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'\n    @parser.instance.parse(text) do |time, record|\n      assert_equal \"Feb 28 00:00:12\", record['time']\n    end\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse_various_characters_for_tag(param)\n    ident = '~!@#$%^&*()_+=-`]{};\"\\'/?\\\\,.<>'\n    @parser.configure({'parser_type' => param})\n    @parser.instance.parse(\"Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test\") { |time, record|\n      assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)\n      assert_equal(@expected.merge('ident' => ident), record)\n    }\n  end\n\n  data('regexp' => 'regexp', 'string' => 'string')\n  def test_parse_various_characters_for_tag_with_priority(param)\n    ident = '~!@#$%^&*()_+=-`]{};\"\\'/?\\\\,.<>'\n    @parser.configure('with_priority' => true, 'parser_type' => param)\n    @parser.instance.parse(\"<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test\") { |time, record|\n      assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)\n      assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)\n    }\n  end\n\n  sub_test_case 'Check the difference of regexp and string parser' do\n    # examples from rfc3164\n    data('regexp' => 'regexp', 'string' => 'string')\n    test 'wrong result with no ident message by default' do |param|\n      @parser.configure('parser_type' => param)\n      @parser.instance.parse('Feb  5 17:32:18 10.0.0.99 Use the BFG!') { |time, record|\n        assert_equal({'host' => '10.0.0.99', 'ident' => 'Use', 'message' => 'the BFG!'}, record)\n      }\n    end\n\n    test \"proper result with no ident message by 'support_colonless_ident false'\" do\n      @parser.configure('parser_type' => 'string', 'support_colonless_ident' => false)\n      @parser.instance.parse('Feb  5 17:32:18 10.0.0.99 Use the BFG!') { |time, record|\n        assert_equal({'host' => '10.0.0.99', 'message' => 'Use the BFG!'}, record)\n      }\n    end\n\n    test \"string parsers can't parse broken syslog message and generate wrong record\" do\n      @parser.configure('parser_type' => 'string')\n      @parser.instance.parse(\"1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.32 sched[0]: That's All Folks!\") { |time, record|\n        expected = {'host' => 'scapegoat.dmz.example.org', 'ident' => 'sched', 'pid' => '0', 'message' => \"That's All Folks!\"}\n        assert_not_equal(expected, record)\n      }\n    end\n\n    test \"regexp parsers can't parse broken syslog message and raises an error\" do\n      @parser.configure('parser_type' => 'regexp')\n      assert_raise(Fluent::TimeParser::TimeParseError) {\n        @parser.instance.parse(\"1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.32 sched[0]: That's All Folks!\") { |time, record| }\n      }\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    test \"':' included message breaks regexp parser\" do |param|\n      @parser.configure('parser_type' => param)\n      @parser.instance.parse('Aug 10 12:00:00 127.0.0.1 test foo:bar') { |time, record|\n        expected = {'host' => '127.0.0.1', 'ident' => 'test', 'message' => 'foo:bar'}\n        if param == 'string'\n          assert_equal(expected, record)\n        else\n          assert_not_equal(expected, record)\n        end\n      }\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    test \"Only no whitespace content in MSG causes different result\" do |param|\n      @parser.configure('parser_type' => param)\n      @parser.instance.parse('Aug 10 12:00:00 127.0.0.1 value1,value2,value3,value4') { |time, record|\n        # 'message' is correct but regexp set it as 'ident'\n        if param == 'string'\n          expected = {'host' => '127.0.0.1', 'message' => 'value1,value2,value3,value4'}\n          assert_equal(expected, record)\n        else\n          expected = {'host' => '127.0.0.1', 'ident' => 'value1,value2,value3,value4', 'message' => ''}\n          assert_equal(expected, record)\n        end\n      }\n    end\n  end\n\n  class TestRFC5424Regexp < self\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_message(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n      assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_message_trailing_eol(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = \"<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!\\n\"\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n      assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_multiline_message(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = \"<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi,\\nfrom\\nFluentd!\"\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi,\\nfrom\\nFluentd!\", record[\"message\"]\n      end\n      assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_message_and_without_priority(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'rfc5424',\n                        'parser_type' => param\n                        )\n      text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n      assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_empty_message_and_without_priority(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'rfc5424',\n                        'parser_type' => param\n                        )\n      text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - -'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_nil record[\"message\"]\n      end\n      assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_message_without_time_format(param)\n      @parser.configure(\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_message_with_priority_and_pid(param)\n      @parser.configure(\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<28>1 2018-09-26T15:54:26.620412+09:00 machine minissdpd 1298 - -  peer 192.168.0.5:50123 is not from a LAN'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2018-09-26T15:54:26.620412+0900\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"1298\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \" peer 192.168.0.5:50123 is not from a LAN\", record[\"message\"]\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_structured_message(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"] [Hi] from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"11111\", record[\"pid\"]\n        assert_equal \"ID24224\", record[\"msgid\"]\n        assert_equal \"[exampleSDID@20224 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"11211\\\"]\",\n                     record[\"extradata\"]\n        assert_equal \"[Hi] from Fluentd!\", record[\"message\"]\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_multiple_structured_message(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"][exampleSDID@20224 class=\"high\"] Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"11111\", record[\"pid\"]\n        assert_equal \"ID24224\", record[\"msgid\"]\n        assert_equal \"[exampleSDID@20224 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"11211\\\"][exampleSDID@20224 class=\\\"high\\\"]\",\n                     record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_message_includes_right_bracket(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"] [Hi] from Fluentd]!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"11111\", record[\"pid\"]\n        assert_equal \"ID24224\", record[\"msgid\"]\n        assert_equal \"[exampleSDID@20224 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"11211\\\"]\",\n                     record[\"extradata\"]\n        assert_equal \"[Hi] from Fluentd]!\", record[\"message\"]\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_empty_message(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"11111\", record[\"pid\"]\n        assert_equal \"ID24224\", record[\"msgid\"]\n        assert_equal \"[exampleSDID@20224 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"11211\\\"]\",\n                     record[\"extradata\"]\n        assert_nil record[\"message\"]\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_space_empty_message(param)\n      @parser.configure(\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"] '\n      @parser.instance.parse(text) do |time, record|\n        if param == 'string'\n          assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n          assert_equal \"11111\", record[\"pid\"]\n          assert_equal \"ID24224\", record[\"msgid\"]\n          assert_equal \"[exampleSDID@20224 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"11211\\\"]\",\n                       record[\"extradata\"]\n          assert_equal '', record[\"message\"]\n        else\n          assert_nil time\n          assert_nil record\n        end\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_message_without_subseconds(param)\n      @parser.configure(\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15Z\", format: '%Y-%m-%dT%H:%M:%S%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_message_both_timestamp(param)\n      @parser.configure(\n                        'message_format' => 'rfc5424',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15Z\", format: '%Y-%m-%dT%H:%M:%S%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd with subseconds!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd with subseconds!\", record[\"message\"]\n      end\n    end\n  end\n\n  class TestAutoRegexp < self\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_auto_with_legacy_syslog_message(param)\n      @parser.configure(\n                        'time_format' => '%b %d %M:%S:%H',\n                        'message_format' => 'auto',\n                        'parser_type' => param\n                        )\n      text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"Feb 28 00:00:12\", format: '%b %d %M:%S:%H'), time)\n        assert_equal(@expected, record)\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_auto_with_legacy_syslog_priority_message(param)\n      @parser.configure(\n                        'time_format' => '%b %d %M:%S:%H',\n                        'with_priority' => true,\n                        'message_format' => 'auto',\n                        'parser_type' => param\n                        )\n      text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"Feb 28 12:00:00\", format: '%b %d %M:%S:%H'), time)\n        assert_equal(@expected.merge('pri' => 6), record)\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_message(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'auto',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal 16, record[\"pri\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_rfc5424_structured_message(param)\n      @parser.configure(\n                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'message_format' => 'auto',\n                        'with_priority' => true,\n                        'parser_type' => param\n                        )\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"] Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"11111\", record[\"pid\"]\n        assert_equal \"ID24224\", record[\"msgid\"]\n        assert_equal \"[exampleSDID@20224 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"11211\\\"]\",\n                     record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_both_message_type(param)\n      @parser.configure(\n        'time_format' => '%b %d %M:%S:%H',\n        'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n        'message_format' => 'auto',\n        'with_priority' => true,\n        'parser_type' => param\n      )\n      text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"Feb 28 12:00:00\", format: '%b %d %M:%S:%H'), time)\n        assert_equal(@expected.merge('pri' => 1), record)\n      end\n\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"] Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"11111\", record[\"pid\"]\n        assert_equal \"ID24224\", record[\"msgid\"]\n        assert_equal \"[exampleSDID@20224 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"11211\\\"]\",\n                     record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n\n      text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test 2>1'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"Feb 28 12:00:02\", format: '%b %d %M:%S:%H'), time)\n        assert_equal(@expected.merge('pri' => 1, 'message'=> '[error] Syslog test 2>1'), record)\n      end\n\n      text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"Feb 28 12:00:02\", format: '%b %d %M:%S:%H'), time)\n        assert_equal(@expected.merge('pri' => 1), record)\n      end\n\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n    end\n\n    data('regexp' => 'regexp', 'string' => 'string')\n    def test_parse_with_both_message_type_and_priority(param)\n      @parser.configure(\n                        'time_format' => '%b %d %M:%S:%H',\n                        'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',\n                        'with_priority' => true,\n                        'message_format' => 'auto',\n                        'parser_type' => param\n                        )\n      text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"Feb 28 12:00:00\", format: '%b %d %M:%S:%H'), time)\n        assert_equal(@expected.merge('pri' => 6), record)\n      end\n\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"] Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"11111\", record[\"pid\"]\n        assert_equal \"ID24224\", record[\"msgid\"]\n        assert_equal \"[exampleSDID@20224 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"11211\\\"]\",\n                     record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n\n      text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"Feb 28 12:00:02\", format: '%b %d %M:%S:%H'), time)\n        assert_equal(@expected.merge('pri' => 16), record)\n      end\n\n      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15.003Z\", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd!\", record[\"message\"]\n      end\n\n      text = '<16>1 2017-02-06T13:14:15Z 192.168.0.1 fluentd - - - Hi, from Fluentd without subseconds!'\n      @parser.instance.parse(text) do |time, record|\n        assert_equal(event_time(\"2017-02-06T13:14:15Z\", format: '%Y-%m-%dT%H:%M:%S%z'), time)\n        assert_equal \"-\", record[\"pid\"]\n        assert_equal \"-\", record[\"msgid\"]\n        assert_equal \"-\", record[\"extradata\"]\n        assert_equal \"Hi, from Fluentd without subseconds!\", record[\"message\"]\n      end\n    end\n  end\n\n  # \"parser_type\" config shouldn't hide Fluent::Plugin::Parser#plugin_type\n  # https://github.com/fluent/fluentd/issues/3296\n  data('regexp' => :regexp, 'fast' => :string)\n  def test_parser_type_method(engine)\n    @parser.configure({'parser_type' => engine.to_s})\n    assert_equal(:text_per_line, @parser.instance.parser_type)\n  end\n\n  data('regexp' => :regexp, 'string' => :string)\n  def test_parser_engine(engine)\n    @parser.configure({'parser_engine' => engine.to_s})\n    assert_equal(engine, @parser.instance.parser_engine)\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_parser_tsv.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/parser_tsv'\n\nclass TSVParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def create_driver(conf={})\n    Fluent::Test::Driver::Parser.new(Fluent::TextParser::TSVParser).configure(conf)\n  end\n\n  data('array param' => '[\"a\",\"b\"]', 'string param' => 'a,b')\n  def test_config_params(param)\n    d = create_driver(\n      'keys' => param,\n    )\n\n    assert_equal \"\\t\", d.instance.delimiter\n\n    d = create_driver(\n      'keys' => param,\n      'delimiter' => ',',\n    )\n\n    assert_equal ['a', 'b'], d.instance.keys\n    assert_equal \",\", d.instance.delimiter\n  end\n\n  data('array param' => '[\"time\",\"a\",\"b\"]', 'string param' => 'time,a,b')\n  def test_parse(param)\n    d = create_driver('keys' => param, 'time_key' => 'time')\n    d.instance.parse(\"2013/02/28 12:00:00\\t192.168.0.1\\t111\") { |time, record|\n      assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)\n      assert_equal({\n          'a' => '192.168.0.1',\n          'b' => '111',\n        }, record)\n    }\n  end\n\n  def test_parse_with_time\n    time_at_start = Time.now.to_i\n\n    d = create_driver('keys' => 'a,b')\n    d.instance.parse(\"192.168.0.1\\t111\") { |time, record|\n      assert time && time >= time_at_start, \"parser puts current time without time input\"\n      assert_equal({\n          'a' => '192.168.0.1',\n          'b' => '111',\n        }, record)\n    }\n\n    d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)\n    d.configure('keys' => 'a,b', 'time_key' => 'time', 'estimate_current_event' => 'no')\n    d.instance.parse(\"192.168.0.1\\t111\") { |time, record|\n      assert_equal({\n                     'a' => '192.168.0.1',\n                     'b' => '111',\n                   }, record)\n      assert_nil time, \"parser returns nil w/o time and if configured so\"\n    }\n  end\n\n  data(\n    'left blank column' => [\"\\t@\\t@\", {\"1\" => \"\",\"2\" => \"@\",\"3\" => \"@\"}],\n    'center blank column' => [\"@\\t\\t@\", {\"1\" => \"@\",\"2\" => \"\",\"3\" => \"@\"}],\n    'right blank column' => [\"@\\t@\\t\", {\"1\" => \"@\",\"2\" => \"@\",\"3\" => \"\"}],\n    '2 right blank columns' => [\"@\\t\\t\", {\"1\" => \"@\",\"2\" => \"\",\"3\" => \"\"}],\n    'left blank columns' => [\"\\t\\t@\", {\"1\" => \"\",\"2\" => \"\",\"3\" => \"@\"}],\n    'all blank columns' => [\"\\t\\t\", {\"1\" => \"\",\"2\" => \"\",\"3\" => \"\"}])\n  def test_black_column(data)\n    line, expected = data\n\n    d = create_driver('keys' => '1,2,3')\n    d.instance.parse(line) { |time, record|\n      assert_equal(expected, record)\n    }\n  end\n\n  def test_parse_with_keep_time_key\n    d = create_driver(\n      'keys'=>'time',\n      'time_key'=>'time',\n      'time_format'=>\"%d/%b/%Y:%H:%M:%S %z\",\n      'keep_time_key'=>'true',\n    )\n    text = '28/Feb/2013:12:00:00 +0900'\n    d.instance.parse(text) do |time, record|\n      assert_equal text, record['time']\n    end\n  end\n\n  data('array param' => '[\"a\",\"b\",\"c\",\"d\",\"e\",\"f\"]', 'string param' => 'a,b,c,d,e,f')\n  def test_parse_with_null_value_pattern(param)\n    d = create_driver(\n      'keys'=>param,\n      'null_value_pattern'=>'^(-|null|NULL)$'\n    )\n    d.instance.parse(\"-\\tnull\\tNULL\\t\\t--\\tnuLL\") do |time, record|\n      assert_nil record['a']\n      assert_nil record['b']\n      assert_nil record['c']\n      assert_equal record['d'], ''\n      assert_equal record['e'], '--'\n      assert_equal record['f'], 'nuLL'\n    end\n  end\n\n  data('array param' => '[\"a\",\"b\"]', 'string param' => 'a,b')\n  def test_parse_with_null_empty_string(param)\n    d = create_driver(\n      'keys'=>param,\n      'null_empty_string'=>true\n    )\n    d.instance.parse(\"\\t \") do |time, record|\n      assert_nil record['a']\n      assert_equal record['b'], ' '\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_sd_file.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/sd_file'\nrequire 'fileutils'\nrequire 'json'\n\nclass FileServiceDiscoveryTest < ::Test::Unit::TestCase\n  setup do\n    @dir = File.expand_path('data/sd_file', __dir__)\n    FileUtils.mkdir_p(File.join(@dir, 'tmp'))\n  end\n\n  teardown do\n    FileUtils.rm_r(File.join(@dir, 'tmp'))\n  end\n\n  sub_test_case 'configure' do\n    test 'load yml' do\n      sdf = Fluent::Plugin::FileServiceDiscovery.new\n      sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yml') }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24224, 'test1', 1, false, 'user1', 'pass1', 'key1'), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24225, nil, 1), sdf.services[1]\n    end\n\n    test 'load yaml' do\n      sdf = Fluent::Plugin::FileServiceDiscovery.new\n      sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yaml') }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24224, 'test1', 1, false, 'user1', 'pass1', 'key1'), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24225, nil, 1), sdf.services[1]\n    end\n\n    test 'load json' do\n      sdf = Fluent::Plugin::FileServiceDiscovery.new\n      sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.json') }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24224, 'test1', 1, false, 'user1', 'pass1', 'key1'), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24225, nil, 1), sdf.services[1]\n    end\n\n    test 'regard as yaml if ext is not given' do\n      sdf = Fluent::Plugin::FileServiceDiscovery.new\n      sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config') }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24224, 'test1', 1, false, 'user1', 'pass1', 'key1'), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24225, nil, 1), sdf.services[1]\n    end\n\n    test 'raise an error if config has error' do\n      sdf = Fluent::Plugin::FileServiceDiscovery.new\n      e = assert_raise Fluent::ConfigError do\n        sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'invalid_config.yaml') }))\n      end\n      assert_match(/path=/, e.message)\n    end\n\n    test 'raise an error if config file does not exist' do\n      sdf = Fluent::Plugin::FileServiceDiscovery.new\n      e = assert_raise Fluent::ConfigError do\n        sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'invalid_not_found.json') }))\n      end\n      assert_match(/not found/, e.message)\n    end\n  end\n\n  sub_test_case '#start' do\n    module TestStatEventHelperWrapper\n      # easy to control statsevent\n      def event_loop_attach(watcher)\n        unless watcher.is_a?(Fluent::Plugin::FileServiceDiscovery::StatWatcher)\n          super\n          return\n        end\n\n        @test_stat_event_helper_wrapper_watchers ||= []\n        @test_stat_event_helper_wrapper_watchers << watcher\n\n        @test_stat_event_helper_wrapper_context = Fiber.new do\n          loop do\n            @test_stat_event_helper_wrapper_watchers.each do |w|\n              w.on_change('old', 'new')\n            end\n\n            if Fiber.yield == :finish\n              break\n            end\n          end\n        end\n        resume\n      end\n\n      def resume\n        @test_stat_event_helper_wrapper_context.resume(:resume)\n      end\n\n      def shutdown\n        super\n\n        if @test_stat_event_helper_wrapper_context\n          @test_stat_event_helper_wrapper_context.resume(:finish)\n        end\n      end\n    end\n\n    def create_tmp_config(path, body)\n      File.write(File.join(@dir, 'tmp', path), body)\n    end\n\n    setup do\n      sdf = Fluent::Plugin::FileServiceDiscovery.new\n      @sd_file = sdf\n    end\n\n    teardown do\n      if @sd_file\n        @sd_file.stop unless @sd_file.stopped?\n        @sd_file.before_shutdown unless @sd_file.before_shutdown?\n        @sd_file.shutdown unless @sd_file.shutdown?\n        @sd_file.after_shutdown unless @sd_file.after_shutdown?\n        @sd_file.close unless @sd_file.closed?\n        @sd_file.terminate unless @sd_file.terminated?\n      end\n    end\n\n    test 'Skip if file is not updated' do\n      @sd_file.extend(TestStatEventHelperWrapper)\n\n      create_tmp_config('config.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))\n      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yml') }))\n      queue = []\n      mock.proxy(@sd_file).refresh_file(queue).twice\n\n      @sd_file.start(queue)\n      assert_empty queue\n\n      @sd_file.resume\n      assert_empty queue\n    end\n\n    test 'Skip if file is invalid contents' do\n      @sd_file.extend(TestStatEventHelperWrapper)\n\n      create_tmp_config('config.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))\n      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yml') }))\n\n      queue = []\n      @sd_file.start(queue)\n\n      mock.proxy(@sd_file).refresh_file(queue).once\n      create_tmp_config('test.json', 'invalid contents')\n      @sd_file.resume\n\n      assert_empty queue\n    end\n\n    test 'Skip if error is occurred' do\n      @sd_file.extend(TestStatEventHelperWrapper)\n\n      create_tmp_config('config.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))\n      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yml') }))\n      queue = []\n\n      FileUtils.rm_r(File.join(@dir, 'tmp', 'config.json'))\n      mock.proxy(@sd_file).refresh_file(queue).twice\n\n      @sd_file.start(queue)\n      assert_empty queue\n\n      @sd_file.resume\n      assert_empty queue\n    end\n\n    test 'if service is updated, service_in and service_out event happen' do\n      @sd_file.extend(TestStatEventHelperWrapper)\n\n      create_tmp_config('test.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))\n      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'tmp/test.json') }))\n\n      queue = []\n      @sd_file.start(queue)\n      create_tmp_config('test.json', JSON.generate([{ port: 1234, host: '127.0.0.1' }]))\n      @sd_file.resume\n\n      assert_equal 2, queue.size\n      join = queue.shift\n      drain = queue.shift\n      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_IN, join.type\n      assert_equal 1234, join.service.port\n      assert_equal '127.0.0.1', join.service.host\n\n      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_OUT, drain.type\n      assert_equal 1233, drain.service.port\n      assert_equal '127.0.0.1', drain.service.host\n    end\n\n    test 'if service is deleted, service_out event happens' do\n      @sd_file.extend(TestStatEventHelperWrapper)\n\n      create_tmp_config('test.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }, { port: 1234, host: '127.0.0.2' }]))\n      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'tmp/test.json') }))\n\n      queue = []\n      @sd_file.start(queue)\n      create_tmp_config('test.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))\n      @sd_file.resume\n\n      assert_equal 1, queue.size\n      drain = queue.shift\n      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_OUT, drain.type\n      assert_equal 1234, drain.service.port\n      assert_equal '127.0.0.2', drain.service.host\n    end\n\n    test 'if new service is added, service_in event happens' do\n      @sd_file.extend(TestStatEventHelperWrapper)\n\n      create_tmp_config('test.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))\n      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'tmp/test.json') }))\n\n      queue = []\n      @sd_file.start(queue)\n      create_tmp_config('test.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }, { port: 1234, host: '127.0.0.2' }]))\n      @sd_file.resume\n\n      assert_equal 1, queue.size\n      join = queue.shift\n      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_IN, join.type\n      assert_equal 1234, join.service.port\n      assert_equal '127.0.0.2', join.service.host\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_sd_srv.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/sd_srv'\nrequire 'fileutils'\nrequire 'flexmock/test_unit'\nrequire 'json'\n\nclass SrvServiceDiscoveryTest < ::Test::Unit::TestCase\n  SRV_RECORD1 = Resolv::DNS::Resource::IN::SRV.new(1, 10, 8081, 'service1.example.com')\n  SRV_RECORD2 = Resolv::DNS::Resource::IN::SRV.new(2, 20, 8082, 'service2.example.com')\n\n  sub_test_case 'configure' do\n    test 'set services ordered by priority' do\n      sdf = Fluent::Plugin::SrvServiceDiscovery.new\n      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD2, SRV_RECORD1], getaddress: '127.0.0.1') }\n\n      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]\n    end\n\n    test 'return host name without resolving name when dns_lookup is false' do\n      sdf = Fluent::Plugin::SrvServiceDiscovery.new\n      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD1, SRV_RECORD2], getaddress: '127.0.0.1') }\n\n      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, 'service1.example.com', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, 'service2.example.com', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]\n    end\n\n    test 'pass a value as :nameserver to Resolve::DNS when dns_server_host is given' do\n      sdf = Fluent::Plugin::SrvServiceDiscovery.new\n      mock(Resolv::DNS).new(nameserver: '8.8.8.8') { flexmock('dns_resolver', getresources: [SRV_RECORD1, SRV_RECORD2], getaddress: '127.0.0.1') }\n\n      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_server_host' => '8.8.8.8' }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]\n    end\n\n    test 'pass a value as :nameserver_port to Resolve::DNS when dns_server_host has port' do\n      sdf = Fluent::Plugin::SrvServiceDiscovery.new\n      mock(Resolv::DNS).new(nameserver_port: [['8.8.8.8', 8080]]) { flexmock('dns_resolver', getresources: [SRV_RECORD1, SRV_RECORD2], getaddress: '127.0.0.1') }\n\n      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_server_host' => '8.8.8.8:8080' }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]\n    end\n\n    test 'target follows RFC2782' do\n      sdf = Fluent::Plugin::SrvServiceDiscovery.new\n      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')\n               .should_receive(:getresources).with(\"_service1._tcp.example.com\", Resolv::DNS::Resource::IN::SRV)\n               .and_return([SRV_RECORD1, SRV_RECORD2])\n               .mock\n\n      mock(Resolv::DNS).new { mock }\n      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]\n    end\n\n    test 'can change protocol' do\n      sdf = Fluent::Plugin::SrvServiceDiscovery.new\n      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')\n               .should_receive(:getresources).with(\"_service1._udp.example.com\", Resolv::DNS::Resource::IN::SRV)\n               .and_return([SRV_RECORD1, SRV_RECORD2])\n               .mock\n\n      mock(Resolv::DNS).new { mock }\n      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'proto' => 'udp' }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]\n    end\n\n    test 'can set password, username, password' do\n      sdf = Fluent::Plugin::SrvServiceDiscovery.new\n      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD2, SRV_RECORD1], getaddress: '127.0.0.1') }\n\n      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'shared_key' => 'key', 'username' => 'user', 'password' => 'pass' }))\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, 'user', 'pass', 'key'), sdf.services[0]\n      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, 'user', 'pass', 'key'), sdf.services[1]\n    end\n  end\n\n  sub_test_case '#start' do\n    module TestTimerEventHelperWrapper\n      # easy to control statsevent\n      def timer_execute(_name, _interval, &block)\n        @test_timer_event_helper_wrapper_context = Fiber.new do\n          loop do\n            block.call\n\n            if Fiber.yield == :finish\n              break\n            end\n          end\n        end\n\n        resume\n      end\n\n      def resume\n        @test_timer_event_helper_wrapper_context.resume(:resume)\n      end\n\n      def shutdown\n        super\n\n        if @test_timer_event_helper_wrapper_context\n          @test_timer_event_helper_wrapper_context.resume(:finish)\n        end\n\n      end\n    end\n\n    setup do\n      sds = Fluent::Plugin::SrvServiceDiscovery.new\n      @sd_srv = sds\n    end\n\n    teardown do\n      if @sd_srv\n        @sd_srv.stop unless @sd_srv.stopped?\n        @sd_srv.before_shutdown unless @sd_srv.before_shutdown?\n        @sd_srv.shutdown unless @sd_srv.shutdown?\n        @sd_srv.after_shutdown unless @sd_srv.after_shutdown?\n        @sd_srv.close unless @sd_srv.closed?\n        @sd_srv.terminate unless @sd_srv.terminated?\n      end\n    end\n\n    test 'Skip if srv record is not updated' do\n      @sd_srv.extend(TestTimerEventHelperWrapper)\n      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD2, SRV_RECORD1], getaddress: '127.0.0.1') }\n      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))\n      queue = []\n\n      @sd_srv.start(queue)\n      assert_empty queue\n\n      @sd_srv.resume\n      assert_empty queue\n    end\n\n    test 'Skip if DNS resolver raise an error' do\n      @sd_srv.extend(TestTimerEventHelperWrapper)\n      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')\n               .should_receive(:getresources)\n               .and_return([SRV_RECORD1, SRV_RECORD2])\n               .and_return { raise 'some error' } # for start\n               .and_return { raise 'some error' } # for resume\n               .mock\n\n      mock(Resolv::DNS).new { mock }\n      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))\n      queue = []\n\n      @sd_srv.start(queue)\n      assert_empty queue\n\n      @sd_srv.resume\n      assert_empty queue\n    end\n\n    test 'if service is updated, service_in and service_out event happen' do\n      @sd_srv.extend(TestTimerEventHelperWrapper)\n      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')\n               .should_receive(:getresources)\n               .and_return([SRV_RECORD1])\n               .and_return([SRV_RECORD2])\n               .mock\n\n      mock(Resolv::DNS).new { mock }\n      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))\n      queue = []\n\n      @sd_srv.start(queue)\n      join = queue.shift\n      drain = queue.shift\n      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_IN, join.type\n      assert_equal 8082, join.service.port\n      assert_equal 'service2.example.com', join.service.host\n\n      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_OUT, drain.type\n      assert_equal 8081, drain.service.port\n      assert_equal 'service1.example.com', drain.service.host\n    end\n\n    test 'if service is deleted, service_out event happens' do\n      @sd_srv.extend(TestTimerEventHelperWrapper)\n      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')\n               .should_receive(:getresources)\n               .and_return([SRV_RECORD1, SRV_RECORD2])\n               .and_return([SRV_RECORD2])\n               .mock\n\n      mock(Resolv::DNS).new { mock }\n      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))\n      queue = []\n\n      @sd_srv.start(queue)\n\n      assert_equal 1, queue.size\n      drain = queue.shift\n      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_OUT, drain.type\n      assert_equal 8081, drain.service.port\n      assert_equal 'service1.example.com', drain.service.host\n    end\n\n    test 'if new service is added, service_in event happens' do\n      @sd_srv.extend(TestTimerEventHelperWrapper)\n      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')\n               .should_receive(:getresources)\n               .and_return([SRV_RECORD2])\n               .and_return([SRV_RECORD1, SRV_RECORD2])\n               .mock\n\n      mock(Resolv::DNS).new { mock }\n      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))\n      queue = []\n\n      @sd_srv.start(queue)\n\n      assert_equal 1, queue.size\n      join = queue.shift\n      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_IN, join.type\n      assert_equal 8081, join.service.port\n      assert_equal 'service1.example.com', join.service.host\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_storage.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/storage'\nrequire 'fluent/plugin/base'\n\nclass DummyPlugin < Fluent::Plugin::TestBase\nend\n\nclass BareStorage < Fluent::Plugin::Storage\n  Fluent::Plugin.register_storage('bare', self)\nend\n\nclass BasicStorage < Fluent::Plugin::Storage\n  Fluent::Plugin.register_storage('example', self)\n\n  attr_reader :data, :saved\n\n  def initialize\n    super\n    @data = @saved = nil\n  end\n  def load\n    @data = {}\n  end\n  def save\n    @saved = @data.dup\n  end\n  def get(key)\n    @data[key]\n  end\n  def fetch(key, defval)\n    @data.fetch(key, defval)\n  end\n  def put(key, value)\n    @data[key] = value\n  end\n  def delete(key)\n    @data.delete(key)\n  end\n  def update(key, &block)\n    @data[key] = block.call(@data[key])\n  end\n  def close\n    @data = {}\n    super\n  end\n  def terminate\n    @saved = {}\n    super\n  end\nend\n\nclass StorageTest < Test::Unit::TestCase\n  sub_test_case 'BareStorage' do\n    setup do\n      plugin = DummyPlugin.new\n      @s = BareStorage.new\n      @s.configure(config_element())\n      @s.owner = plugin\n    end\n\n    test 'is configured with plugin information and system config' do\n      plugin = DummyPlugin.new\n      plugin.system_config_override({'process_name' => 'mytest'})\n      plugin.configure(config_element('ROOT', '', {'@id' => '1'}))\n      s = BareStorage.new\n      s.configure(config_element())\n      s.owner = plugin\n\n      assert_equal 'mytest', s.owner.system_config.process_name\n      assert_equal '1', s.instance_eval{ @_plugin_id }\n    end\n\n    test 'does NOT have features for high-performance/high-consistent storages' do\n      assert_equal false, @s.persistent_always?\n      assert_equal false, @s.synchronized?\n    end\n\n    test 'does have default values which is conservative for almost all users' do\n      assert_equal false, @s.persistent\n      assert_equal true, @s.autosave\n      assert_equal 10,   @s.autosave_interval\n      assert_equal true, @s.save_at_shutdown\n    end\n\n    test 'load/save does NOT anything: just as memory storage' do\n      assert_nothing_raised{ @s.load }\n      assert_nothing_raised{ @s.save }\n    end\n\n    test 'all operations are not defined yet' do\n      assert_raise NotImplementedError do\n        @s.get('key')\n      end\n      assert_raise NotImplementedError do\n        @s.fetch('key', 'value')\n      end\n      assert_raise NotImplementedError do\n        @s.put('key', 'value')\n      end\n      assert_raise NotImplementedError do\n        @s.delete('key')\n      end\n      assert_raise NotImplementedError do\n        @s.update('key'){ |v| v + '2' }\n      end\n    end\n  end\n\n  sub_test_case 'ExampleStorage' do\n    setup do\n      plugin = DummyPlugin.new\n      plugin.configure(config_element('ROOT', '', {'@id' => '1'}))\n      @s = BasicStorage.new\n      @s.configure(config_element())\n      @s.owner = plugin\n    end\n\n    test 'load/save works well as plugin internal state operations' do\n      plugin = DummyPlugin.new\n      plugin.configure(config_element('ROOT', '', {'@id' => '0'}))\n      s = BasicStorage.new\n      s.owner = plugin\n\n      assert_nothing_raised{ s.load }\n      assert s.data\n      assert_nil s.saved\n\n      assert_nothing_raised{ s.save }\n      assert s.saved\n      assert{ s.data == s.saved }\n      assert{ s.data.object_id != s.saved.object_id }\n    end\n\n    test 'all operations work well' do\n      @s.load\n\n      assert_nil @s.get('key')\n      assert_equal 'value', @s.fetch('key', 'value')\n      assert_nil @s.get('key')\n\n      assert_equal 'value', @s.put('key', 'value')\n      assert_equal 'value', @s.get('key')\n\n      assert_equal 'valuevalue', @s.update('key'){|v| v * 2 }\n\n      assert_equal 'valuevalue', @s.delete('key')\n    end\n\n    test 'close and terminate work to operate internal states' do\n      @s.load\n      @s.put('k1', 'v1')\n      @s.put('k2', 'v2')\n      assert_equal 2, @s.data.size\n      @s.save\n      assert_equal @s.data.size, @s.saved.size\n\n      assert_nothing_raised{ @s.close }\n      assert @s.data.empty?\n      assert !@s.saved.empty?\n\n      assert_nothing_raised{ @s.terminate }\n      assert @s.data.empty?\n      assert @s.saved.empty?\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_storage_local.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/storage_local'\nrequire 'fluent/plugin/input'\nrequire 'fluent/system_config'\nrequire 'fileutils'\n\nclass LocalStorageTest < Test::Unit::TestCase\n  TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/tmp/storage_local#{ENV['TEST_ENV_NUMBER']}\")\n\n  class MyInput < Fluent::Plugin::Input\n    helpers :storage\n    config_section :storage do\n      config_set_default :@type, 'local'\n    end\n  end\n\n  setup do\n    FileUtils.rm_rf(TMP_DIR)\n    FileUtils.mkdir_p(TMP_DIR)\n    Fluent::Test.setup\n    @d = MyInput.new\n  end\n\n  teardown do\n    @d.stop unless @d.stopped?\n    @d.before_shutdown unless @d.before_shutdown?\n    @d.shutdown unless @d.shutdown?\n    @d.after_shutdown unless @d.after_shutdown?\n    @d.close unless @d.closed?\n    @d.terminate unless @d.terminated?\n  end\n\n  sub_test_case 'without any configuration' do\n    test 'works as on-memory storage' do\n      conf = config_element()\n\n      @d.configure(conf)\n      @d.start\n      @p = @d.storage_create()\n\n      assert_nil @p.path\n      assert @p.store.empty?\n\n      assert_nil @p.get('key1')\n      assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')\n\n      @p.put('key1', '1')\n      assert_equal '1', @p.get('key1')\n\n      @p.update('key1') do |v|\n        (v.to_i * 2).to_s\n      end\n      assert_equal '2', @p.get('key1')\n\n      @p.save # on-memory storage does nothing...\n\n      @d.stop; @d.before_shutdown; @d.shutdown; @d.after_shutdown; @d.close; @d.terminate\n\n      # re-create to reload storage contents\n      @d = MyInput.new\n      @d.configure(conf)\n      @d.start\n      @p = @d.storage_create()\n\n      assert @p.store.empty?\n    end\n\n    test 'warns about on-memory storage if autosave is true' do\n      @d.configure(config_element())\n      @d.start\n      @p = @d.storage_create()\n\n      logs = @d.log.out.logs\n      assert{ logs.any?{|log| log.include?(\"[warn]: both of Plugin @id and path for <storage> are not specified. Using on-memory store.\") } }\n    end\n\n    test 'show info log about on-memory storage if autosave is false' do\n      @d.configure(config_element('ROOT', '', {}, [config_element('storage', '', {'autosave' => 'false'})]))\n      @d.start\n      @p = @d.storage_create()\n\n      logs = @d.log.out.logs\n      assert{ logs.any?{|log| log.include?(\"[info]: both of Plugin @id and path for <storage> are not specified. Using on-memory store.\") } }\n    end\n  end\n\n  sub_test_case 'configured with file path' do\n    test 'works as storage which stores data on disk' do\n      storage_path = File.join(TMP_DIR, 'my_store.json')\n      conf = config_element('ROOT', '', {}, [config_element('storage', '', {'path' => storage_path})])\n      @d.configure(conf)\n      @d.start\n      @p = @d.storage_create()\n\n      assert_equal storage_path, @p.path\n      assert @p.store.empty?\n\n      assert_nil @p.get('key1')\n      assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')\n\n      @p.put('key1', '1')\n      assert_equal '1', @p.get('key1')\n\n      @p.update('key1') do |v|\n        (v.to_i * 2).to_s\n      end\n      assert_equal '2', @p.get('key1')\n\n      @p.save # stores all data into file\n\n      assert File.exist?(storage_path)\n\n      @p.put('key2', 4)\n\n      @d.stop; @d.before_shutdown; @d.shutdown; @d.after_shutdown; @d.close; @d.terminate\n\n      assert_equal({'key1' => '2', 'key2' => 4}, File.open(storage_path){|f| JSON.parse(f.read) })\n\n      # re-create to reload storage contents\n      @d = MyInput.new\n      @d.configure(conf)\n      @d.start\n      @p = @d.storage_create()\n\n      assert_false @p.store.empty?\n\n      assert_equal '2', @p.get('key1')\n      assert_equal 4, @p.get('key2')\n    end\n\n    test 'raise configuration error if a file specified with multi worker configuration' do\n      storage_path = File.join(TMP_DIR, 'my_store.json')\n      conf = config_element('ROOT', '', {}, [config_element('storage', '', {'path' => storage_path})])\n      @d.system_config_override('workers' => 3)\n      assert_raise Fluent::ConfigError.new(\"Plugin 'local' does not support multi workers configuration (Fluent::Plugin::LocalStorage)\") do\n        @d.configure(conf)\n      end\n    end\n  end\n\n  sub_test_case 'configured with root-dir and plugin id' do\n    test 'works as storage which stores data under root dir' do\n      root_dir = File.join(TMP_DIR, 'root')\n      expected_storage_path = File.join(root_dir, 'worker0', 'local_storage_test', 'storage.json')\n      conf = config_element('ROOT', '', {'@id' => 'local_storage_test'})\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do\n        @d.configure(conf)\n      end\n      @d.start\n      @p = @d.storage_create()\n\n      assert_equal expected_storage_path, @p.path\n      assert @p.store.empty?\n\n      assert_nil @p.get('key1')\n      assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')\n\n      @p.put('key1', '1')\n      assert_equal '1', @p.get('key1')\n\n      @p.update('key1') do |v|\n        (v.to_i * 2).to_s\n      end\n      assert_equal '2', @p.get('key1')\n\n      @p.save # stores all data into file\n\n      assert File.exist?(expected_storage_path)\n\n      @p.put('key2', 4)\n\n      @d.stop; @d.before_shutdown; @d.shutdown; @d.after_shutdown; @d.close; @d.terminate\n\n      assert_equal({'key1' => '2', 'key2' => 4}, File.open(expected_storage_path){|f| JSON.parse(f.read) })\n\n      # re-create to reload storage contents\n      @d = MyInput.new\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do\n        @d.configure(conf)\n      end\n      @d.start\n      @p = @d.storage_create()\n\n      assert_false @p.store.empty?\n\n      assert_equal '2', @p.get('key1')\n      assert_equal 4, @p.get('key2')\n    end\n\n    test 'works with customized path by specified usage' do\n      root_dir = File.join(TMP_DIR, 'root')\n      expected_storage_path = File.join(root_dir, 'worker0', 'local_storage_test', 'storage.usage.json')\n      conf = config_element('ROOT', 'usage', {'@id' => 'local_storage_test'})\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do\n        @d.configure(conf)\n      end\n      @d.start\n      @p = @d.storage_create(usage: 'usage', type: 'local')\n\n      assert_equal expected_storage_path, @p.path\n      assert @p.store.empty?\n    end\n  end\n\n  sub_test_case 'configured with root-dir and plugin id, and multi workers' do\n    test 'works as storage which stores data under root dir, also in workers' do\n      root_dir = File.join(TMP_DIR, 'root')\n      expected_storage_path = File.join(root_dir, 'worker1', 'local_storage_test', 'storage.json')\n      conf = config_element('ROOT', '', {'@id' => 'local_storage_test'})\n      with_worker_config(root_dir: root_dir, workers: 2, worker_id: 1) do\n        @d.configure(conf)\n      end\n      @d.start\n      @p = @d.storage_create()\n\n      assert_equal expected_storage_path, @p.path\n      assert @p.store.empty?\n    end\n  end\n\n  sub_test_case 'persistent specified' do\n    test 'works well with path' do\n      omit \"It's hard to test on Windows\" if Fluent.windows?\n\n      storage_path = File.join(TMP_DIR, 'my_store.json')\n      conf = config_element('ROOT', '', {}, [config_element('storage', '', {'path' => storage_path, 'persistent' => 'true'})])\n      @d.configure(conf)\n      @d.start\n      @p = @d.storage_create()\n\n      assert_equal storage_path, @p.path\n      assert @p.store.empty?\n\n      assert_nil @p.get('key1')\n      assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')\n\n      @p.put('key1', '1')\n      assert_equal({'key1' => '1'}, File.open(storage_path){|f| JSON.parse(f.read) })\n\n      @p.update('key1') do |v|\n        (v.to_i * 2).to_s\n      end\n      assert_equal({'key1' => '2'}, File.open(storage_path){|f| JSON.parse(f.read) })\n    end\n\n    test 'works well with root-dir and plugin id' do\n      omit \"It's hard to test on Windows\" if Fluent.windows?\n\n      root_dir = File.join(TMP_DIR, 'root')\n      expected_storage_path = File.join(root_dir, 'worker0', 'local_storage_test', 'storage.json')\n      conf = config_element('ROOT', '', {'@id' => 'local_storage_test'}, [config_element('storage', '', {'persistent' => 'true'})])\n      Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do\n        @d.configure(conf)\n      end\n      @d.start\n      @p = @d.storage_create()\n\n      assert_equal expected_storage_path, @p.path\n      assert @p.store.empty?\n\n      assert_nil @p.get('key1')\n      assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')\n\n      @p.put('key1', '1')\n      assert_equal({'key1' => '1'}, File.open(expected_storage_path){|f| JSON.parse(f.read) })\n\n      @p.update('key1') do |v|\n        (v.to_i * 2).to_s\n      end\n      assert_equal({'key1' => '2'}, File.open(expected_storage_path){|f| JSON.parse(f.read) })\n    end\n\n    test 'raises error if it is configured to use on-memory storage' do\n      assert_raise Fluent::ConfigError.new(\"Plugin @id or path for <storage> required when 'persistent' is true\") do\n        @d.configure(config_element('ROOT', '', {}, [config_element('storage', '', {'persistent' => 'true'})]))\n      end\n    end\n  end\n\n  sub_test_case 'with various configurations' do\n    test 'mode and dir_mode controls permissions of stored data' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      storage_path = File.join(TMP_DIR, 'dir', 'my_store.json')\n      storage_conf = {\n        'path' => storage_path,\n        'mode' => '0600',\n        'dir_mode' => '0777',\n      }\n      conf = config_element('ROOT', '', {}, [config_element('storage', '', storage_conf)])\n      @d.configure(conf)\n      @d.start\n      @p = @d.storage_create()\n\n      assert_equal storage_path, @p.path\n      assert @p.store.empty?\n\n      @p.put('key1', '1')\n      assert_equal '1', @p.get('key1')\n\n      @p.save # stores all data into file\n\n      assert File.exist?(storage_path)\n      assert_equal 0600, (File.stat(storage_path).mode % 01000)\n      assert_equal 0777, (File.stat(File.dirname(storage_path)).mode % 01000)\n    end\n\n    test 'pretty_print controls to write data in files as human-easy-to-read' do\n      storage_path = File.join(TMP_DIR, 'dir', 'my_store.json')\n      storage_conf = {\n        'path' => storage_path,\n        'pretty_print' => 'true',\n      }\n      conf = config_element('ROOT', '', {}, [config_element('storage', '', storage_conf)])\n      @d.configure(conf)\n      @d.start\n      @p = @d.storage_create()\n\n      assert_equal storage_path, @p.path\n      assert @p.store.empty?\n\n      @p.put('key1', '1')\n      assert_equal '1', @p.get('key1')\n\n      @p.save # stores all data into file\n\n      expected_pp_json = <<JSON.chomp\n{\n  \"key1\": \"1\"\n}\nJSON\n      assert_equal expected_pp_json, File.read(storage_path)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin/test_string_util.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin/string_util'\n\nclass StringUtilTest < Test::Unit::TestCase\n  def setup\n    @null_value_pattern = Regexp.new(\"^(-|null|NULL)$\")\n  end\n\n  sub_test_case 'valid string' do\n    test 'null string' do\n      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, \"null\").to_s, \"null\"\n      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, \"NULL\").to_s, \"NULL\"\n      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, \"-\").to_s, \"-\"\n    end\n\n    test 'normal string' do\n      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, \"fluentd\"), nil\n    end\n  end\n\n  sub_test_case 'invalid string' do\n    test 'normal string' do\n      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, \"\\xff\"), nil\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/data/cert/cert-key.pem",
    "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAKCAQEArbUTk5n5RruIQGhK1T8g/emKonlfWNMSj/J/f/U9NJ14ugIx\nyHBMqx4WaTyA4zjT2VJO5tRBe385zlIUf8i+x7Ovt/MgsjiwXyKv7qdsE5KHLq+V\nXJfA+s5vAAyzBHY/BA7xxh/QqCI8a/a1OyHyaQ9pFRFXtQBlTH7Fc1qSw5Yg0EXo\nfa6YIBQuDjfqa7FRPj+bEWDO5PUqOMzH5XKBUPS9GLHOqia0CnzF2a51TArC0Dl1\noNFa7myVmjBuNtkG88Fkd7YNzGa+sNBJPmuGvFXuU3XPEnrtARO/SG4g9/MQUvfM\nI3jFFOJAMmEKd8QXxO5FgIuEnaNOhVRVH/e9wwIDAQABAoIBAQCcYYAWSW/Y9ctb\n8IguIn8ZF77vNkVYOe1kGfQexjErxPiCvKcEw9TB2HxkkUuXQ/m0lBzkQgIRpB/u\nVoaN47OdJW70M7UjvVVK0HeCascpd6irpvbYPOZG5MGZUdV4Ftv0U1/l9Y5rTycG\nKzxM1+rcGjuH8+z5zjlj/FzV1Jx75pgH2vwE7tBC9hMIefKJVxsnj+1I2BWwT3Ra\ntFayaKW80UYvGQeQjMHyWzEAHAAco5EVTNU9q4xkeKubZx/dod8b05S0Oral0SHY\nAFzXJCp6k29w0m3saEtVQBTImwNut8scJ4KoryGRp9fj9a/dBb1RHe/6EXZ8kDRe\nzhB2/XWBAoGBANpaAPdcIXFyke4DRB07JGin4NU+yzIZy+RwsHme1OIwu9jUYQ1j\nSzcrnJ9adkjgHl4z2psXyeSz/lSUGj7ZjcMVaDu49JSYMPP3heXGkHdtlVyB7KFR\nOBt1rgNdXYfOLOBg6j7mf5G5fXUIATM8fNw5nDYIivD7U3cl9TdUe3OJAoGBAMuo\ne/pmRMEaaDmXuNkDGv/ZoOC4j9tO1fGheSdna9gZWoOyY+MUTFNBQuz/8aw6VsKS\nc7RXY0QI6NsgdLD1OGVgH9AXhDqvgfOBXDWdr/DNJa0wDFNaWRrMFeNQLm7Xhxuf\nOBQycq6vUiWJ6XTqcdk+xErcWRbsQPsGozCEcHfrAoGBAI+sl2QsMClI/PLDHWeq\nict/Y3aNigCebsYSzFxKgcOP05raLD417sEPplBIovpS1kigECDrJ0KgmSIoDZj7\nZ8dM57gcfHrmJZfycB2AaYDcD2K+bui625Nd/LFkFu4MYQUHYDshSiGmDwIZ0UhY\nbcQu+DWpq1rj1Mjaph7RLSYZAoGBAMAJNiZt6qT/ZonwA/A1mm7oYmekAJo2I0y+\nCwTZ43oYph4Kx19nU3Up05aw3MyfEhJUkF/cCxErKY7+cqouAKiu1DCfDCNePTZE\no5M3BqEUgFNnPCgYyG6ZRIZco/wGpTKplvKMg1EuplIZUhbZvtKv+d3xGtyfl8Jc\nhuKH75lTAoGAHQ3Sy56zgQEJ3QFrIw4puztf/KXpSgLA+aSDh6Ti4IY7L+CwZmuW\nJyVbgF33CwSIKEhacmALs1ZZsDc1ZXov1HWwePxe4/m7smoqZDtrRXQsbRB5COzA\nU30iEMW/rwI3WmHhiIWZYje85+VJZqBc5QS8guv4/XrS/FE7C8Suksc=\n-----END RSA PRIVATE KEY-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/cert-with-CRLF.pem",
    "content": "-----BEGIN CERTIFICATE-----\r\nMIIDLDCCAhSgAwIBAgIIEJHFsHrKBGYwDQYJKoZIhvcNAQELBQAwJjEQMA4GA1UE\r\nChMHRmx1ZW50ZDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTE5MDYxOTA1MTM0NVoX\r\nDTE5MDkxODExMTg0NVowJjEQMA4GA1UEChMHRmx1ZW50ZDESMBAGA1UEAxMJbG9j\r\nYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArbUTk5n5RruI\r\nQGhK1T8g/emKonlfWNMSj/J/f/U9NJ14ugIxyHBMqx4WaTyA4zjT2VJO5tRBe385\r\nzlIUf8i+x7Ovt/MgsjiwXyKv7qdsE5KHLq+VXJfA+s5vAAyzBHY/BA7xxh/QqCI8\r\na/a1OyHyaQ9pFRFXtQBlTH7Fc1qSw5Yg0EXofa6YIBQuDjfqa7FRPj+bEWDO5PUq\r\nOMzH5XKBUPS9GLHOqia0CnzF2a51TArC0Dl1oNFa7myVmjBuNtkG88Fkd7YNzGa+\r\nsNBJPmuGvFXuU3XPEnrtARO/SG4g9/MQUvfMI3jFFOJAMmEKd8QXxO5FgIuEnaNO\r\nhVRVH/e9wwIDAQABo14wXDAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYB\r\nBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFPLdOVnVWuuB\r\n7Pnvpgte4BHitzFYMA0GCSqGSIb3DQEBCwUAA4IBAQBp8LAzjWIJapwTBnnivwZk\r\nD6Lr028mZIacbBZKsmmPi0VvDFqCvUAbHN8ytPlRBWnvvkihDkZs1TwcDCXGsWYs\r\ndNNwsYGpk3mQxsHQ9atvy0mQGLDlfaSs/329bfVCw1cPFo9n+MeivSBoE6asdIbH\r\ntOW3kk1XtJZ2qQJJRvexFImZc0z8c2cG0+eR5hQxQd9bLnAczi/8mZ8VzaU/O3UU\r\nOJoVuyp0AA8f2f0f1QDaeH9stWZtJQj3ZX1DWHRE3OmVkoBdlt8EHYGggtvQaLIF\r\nXbHigLHzYztMjmDt4fmRczu/Fu6M4xNro8jLgjiIjqlLBjDZiKrSbOwgyebwFDlv\r\n-----END CERTIFICATE-----\r\n"
  },
  {
    "path": "test/plugin_helper/data/cert/cert-with-no-newline.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDLDCCAhSgAwIBAgIIEJHFsHrKBGYwDQYJKoZIhvcNAQELBQAwJjEQMA4GA1UE\nChMHRmx1ZW50ZDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTE5MDYxOTA1MTM0NVoX\nDTE5MDkxODExMTg0NVowJjEQMA4GA1UEChMHRmx1ZW50ZDESMBAGA1UEAxMJbG9j\nYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArbUTk5n5RruI\nQGhK1T8g/emKonlfWNMSj/J/f/U9NJ14ugIxyHBMqx4WaTyA4zjT2VJO5tRBe385\nzlIUf8i+x7Ovt/MgsjiwXyKv7qdsE5KHLq+VXJfA+s5vAAyzBHY/BA7xxh/QqCI8\na/a1OyHyaQ9pFRFXtQBlTH7Fc1qSw5Yg0EXofa6YIBQuDjfqa7FRPj+bEWDO5PUq\nOMzH5XKBUPS9GLHOqia0CnzF2a51TArC0Dl1oNFa7myVmjBuNtkG88Fkd7YNzGa+\nsNBJPmuGvFXuU3XPEnrtARO/SG4g9/MQUvfMI3jFFOJAMmEKd8QXxO5FgIuEnaNO\nhVRVH/e9wwIDAQABo14wXDAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYB\nBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFPLdOVnVWuuB\n7Pnvpgte4BHitzFYMA0GCSqGSIb3DQEBCwUAA4IBAQBp8LAzjWIJapwTBnnivwZk\nD6Lr028mZIacbBZKsmmPi0VvDFqCvUAbHN8ytPlRBWnvvkihDkZs1TwcDCXGsWYs\ndNNwsYGpk3mQxsHQ9atvy0mQGLDlfaSs/329bfVCw1cPFo9n+MeivSBoE6asdIbH\ntOW3kk1XtJZ2qQJJRvexFImZc0z8c2cG0+eR5hQxQd9bLnAczi/8mZ8VzaU/O3UU\nOJoVuyp0AA8f2f0f1QDaeH9stWZtJQj3ZX1DWHRE3OmVkoBdlt8EHYGggtvQaLIF\nXbHigLHzYztMjmDt4fmRczu/Fu6M4xNro8jLgjiIjqlLBjDZiKrSbOwgyebwFDlv\n-----END CERTIFICATE-----"
  },
  {
    "path": "test/plugin_helper/data/cert/cert.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDLDCCAhSgAwIBAgIIEJHFsHrKBGYwDQYJKoZIhvcNAQELBQAwJjEQMA4GA1UE\nChMHRmx1ZW50ZDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTE5MDYxOTA1MTM0NVoX\nDTE5MDkxODExMTg0NVowJjEQMA4GA1UEChMHRmx1ZW50ZDESMBAGA1UEAxMJbG9j\nYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArbUTk5n5RruI\nQGhK1T8g/emKonlfWNMSj/J/f/U9NJ14ugIxyHBMqx4WaTyA4zjT2VJO5tRBe385\nzlIUf8i+x7Ovt/MgsjiwXyKv7qdsE5KHLq+VXJfA+s5vAAyzBHY/BA7xxh/QqCI8\na/a1OyHyaQ9pFRFXtQBlTH7Fc1qSw5Yg0EXofa6YIBQuDjfqa7FRPj+bEWDO5PUq\nOMzH5XKBUPS9GLHOqia0CnzF2a51TArC0Dl1oNFa7myVmjBuNtkG88Fkd7YNzGa+\nsNBJPmuGvFXuU3XPEnrtARO/SG4g9/MQUvfMI3jFFOJAMmEKd8QXxO5FgIuEnaNO\nhVRVH/e9wwIDAQABo14wXDAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYB\nBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFPLdOVnVWuuB\n7Pnvpgte4BHitzFYMA0GCSqGSIb3DQEBCwUAA4IBAQBp8LAzjWIJapwTBnnivwZk\nD6Lr028mZIacbBZKsmmPi0VvDFqCvUAbHN8ytPlRBWnvvkihDkZs1TwcDCXGsWYs\ndNNwsYGpk3mQxsHQ9atvy0mQGLDlfaSs/329bfVCw1cPFo9n+MeivSBoE6asdIbH\ntOW3kk1XtJZ2qQJJRvexFImZc0z8c2cG0+eR5hQxQd9bLnAczi/8mZ8VzaU/O3UU\nOJoVuyp0AA8f2f0f1QDaeH9stWZtJQj3ZX1DWHRE3OmVkoBdlt8EHYGggtvQaLIF\nXbHigLHzYztMjmDt4fmRczu/Fu6M4xNro8jLgjiIjqlLBjDZiKrSbOwgyebwFDlv\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem",
    "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA21in8qRqBrfYlRHr1N6eGq3d4CnKe+lTt0sGbab1670s0+h1\nlGtjWny6gLSnt5PBY36CUoFB8MtRciVG2zUhB+dAZe4Mst5dzJO6cONT/l/USbCV\nrQ/x8m3YzeJNj89fCZ7/7po8CpEfn16BMAd6TMXZZ8O+DCgv3SliEyxpi9CLg2PK\nwAV4lwktbGenYadWLFWjv5a+QElFGH/tu5jUDhggoisoQGyZCls2Vmbr6RH+uiy8\njs3gxy+5YgbuwkrQtjbfwxe5yEsd6RgzO8oLqlowCwCrn4CNaXFwaZa/vqcWzY4L\nqiEWLfmFpFcSZdoWjr1WHsJS/PM3AyJea8nAnQIDAQABAoIBAGkMpaqsmWbMR7rl\nEVgqoffPCzMfcK01ivV+xf5f9ulG+aAndaB2aefdUojvfF+MMRNQdGPFKeqDxWbw\neWXkpQQe+ZWXk5darfubSLBl/0UVahs8qgJvX4WmnC3GUzUrsK1v68y/K0A4TrfJ\nz/9LpYP9QWjTs0IpQPsfpavfGlFt1TJvPxmomn7D+n0N6zzD3kFlGdeCSYwtPURR\ndJ7ifX2vU4E8UtSm7gLrXlmAl1O7bJjFvL3+QlzKDM4M1WJpA9MkmM2icKxdYpew\nbiRmIiUHdV4soad2Aq58v4q5UgdMqGI8f+fWPO9qgXN7AKbT9NkXr1w/pziOt5rq\n/77lbKECgYEA9bmOSA1LfRRclJCaoWWu4PiaXdvMHvsNC9r8NUGuwmiosEMKHFEJ\nz8wvzVXprziXAwDTwqkPJmKseNPC8WCq2GfNu7lqJ1V1wHxpzHKhiAWhs7rE7skW\nmF2xn2Tocba/L3Nb+K0tXhOZ7daduz94YXFhmRtsutZ9JvFv3G+QmHkCgYEA5IS4\n+cOFyha486U9jXyLcPmZXZY9ql4ILuKuQ5pHA7CBO/zU2behuwcainW6P0WzbhKs\nCP8xR5W/otqcaHZxy6hg9/nUFVVZIR/0jgKyxCqJA1W49jt23gykmU6q0vj7haZJ\nQMhcAbleHBC7YvYoMTpTMe7tZP4YsFNysBn76EUCgYBqtFAn07YjM7NcREsRqSE+\nylXmSisijOxGaKq6ybIE9APEvufmEf7LwKRFa3hVwaI6CKLsVhOhHJo+wd5WiR7H\naJQ7X7HMMN04YA5lXKXudluYu5MHCkWIlq8qQ1x4/N2a0mJu42ze/G4MjPTjuhUh\nY2X5YaJepAOm5JMpyzykKQKBgQDj9eaU+dBgLdSo8UD7ALAVrlio/HRdnNoq82SF\n+cQ30P7KucgXvFDxQv/d+d0mu0BoYOYPP4uIbsEyE0SODQIt+LVrCmTgNzjni3op\npFVyzT/K/Nu7fsxwbEpSySAtv8UhqSVQI89sxN81vhdAfHDR0u4lVMSqx7QXSdeS\nBwm9xQKBgQDSeoHoXbmiqRBss9Em69JPwWRZOc9eLns+fFJ1x/WkhY4bnnVXTYkZ\nLzRN4ICq18xHy8egbl93uxDcjkMxi3Wj64QZcygI9gWmDBW2hdMpEArR/grsM/FH\nDYu9KUMhm5T1KtXtV3izSVCprthdGjbKSbmR6hGw3n9Z1uP6V1rnXQ==\n-----END RSA PRIVATE KEY-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/cert_chains/ca-cert.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDPTCCAiWgAwIBAgILAJJ76Kpa6DYtsncwDQYJKoZIhvcNAQELBQAwUzELMAkG\nA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1Nb3VudGFpbiBWaWV3MR8w\nHQYDVQQDDBZjYS50ZXN0aW5nLmZsdWVudGQub3JnMCAXDTcwMDEwMTAwMDAwMFoY\nDzIxMTgxMDI3MDgwNjU3WjBTMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExFjAU\nBgNVBAcMDU1vdW50YWluIFZpZXcxHzAdBgNVBAMMFmNhLnRlc3RpbmcuZmx1ZW50\nZC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDbWKfypGoGt9iV\nEevU3p4ard3gKcp76VO3SwZtpvXrvSzT6HWUa2NafLqAtKe3k8FjfoJSgUHwy1Fy\nJUbbNSEH50Bl7gyy3l3Mk7pw41P+X9RJsJWtD/HybdjN4k2Pz18Jnv/umjwKkR+f\nXoEwB3pMxdlnw74MKC/dKWITLGmL0IuDY8rABXiXCS1sZ6dhp1YsVaO/lr5ASUUY\nf+27mNQOGCCiKyhAbJkKWzZWZuvpEf66LLyOzeDHL7liBu7CStC2Nt/DF7nISx3p\nGDM7yguqWjALAKufgI1pcXBplr++pxbNjguqIRYt+YWkVxJl2haOvVYewlL88zcD\nIl5rycCdAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEB\nAGPhEAc+vb9PHyet9t5gKJTZF7S4x207hKZ0zLqCQCLM7VjrTVBh/TUFT+saiPsv\n4k2TYSZKInYXh6tkSzdGlRyzk166hF1eOTOae/92Da5PEQMT0AC+SDRDHXrNTyD8\nZxP2xOl0j+FNeKQDdNL+PCU56PYehY40dYZ/EjRyu4VjKaSCMHcVx7eMTyxU3wtB\nHbOa8UJdUXEjn4h7icz4aca22aqLedXNaDr6YX+mo25t1jsTO7cEMfr4Ce7GvxdA\nmsey/b5jnsKcpL7/2uYz5+owD39JZQBRQt+1YAzIU+BNiz8yzdXJO4aQpGWTflIB\njO31o7x4loEvaBTQbX3iP2g=\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/cert_chains/cert-key.pem",
    "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAzQtGZgClcBIDKC0U1sQpznVV9HzAZANDVazrDGA8Euf8Kx4X\ndnVoE+HzwAviL4VLDPzNBLyoZ2wf3wg6D+1UAhA0DqPRCihhfJqBX0WgewakSl5a\nwrc1XIXtiMxcWE/5+XtodjO71gECBgmd/l0UtdFY7mMt/gArH8PQf29ABGiB1zzn\nd0MNgA0r2OlXICe4kDzTHqDBv7nFgQFro2vkI/QJD5fABbb62HRWgUfQ13DUShSG\nH5FPkMxTkCs5GShyjYCC5IhI+AKx5RWvb4kuMRsxWC7JHc/XKPdbeLkNWRCFzTnO\nOR09Qdp45tBbXtX0IjInxsnNOBzW0zTSg0xUtwIDAQABAoIBAG4N5zNIlYOZp2gh\nClZb47SU9hXL/9euiK2rql1yKcxcB9V8yUsjqUFCvfoOZtDq0mWeKsyoFhusxU6I\ns+FomPaii85vzvuMwQaIR3hDfueJoRTpn/1zKIkIuX37cnVUN+/YdTE8g01SLSvg\nbZThkQQl4X3SbhUvMfZSu84qgEncdtJ08zx81CB1Fpn0HIjpmpRM1F9jjDwtkVUw\n1gVQm9g7fwbgRfqSUyKkEqKvCVYnLQx3X5inuAEps4zIzAXF8RtxFymyHfExzZqr\nPj5uYjTbf/wkaU9nWQzXeCbRTz3ZrjvFnisFt1HI2uijd1DjJ7CB7Ls752RWekXS\nYc9FFwECgYEA/gZ3Hx4dIb6NW6UmPKjiQex/tVqwCRaxGtpA7qensf187N2nn4FD\njdQJ69XmoDRKr2X+e7tsgKpETGufdDQ8MNBHz2krfqXVSlsEIOjeLfRRqhh+8Bcr\nflri0sumVbk0X9bnXzyCc4AGcRhtT9U69HCMTlly2z0pjIlVvmFXARECgYEAzqNV\nFSHHbEU+5zToJK0jyTaE/F+u9n5YKDcRN9RrKrEYcoUzKbIYJrUulbRGfsqAxY5L\nKtIDRI9NmPDBYqBwqSFnm9LcLn8CgQH/rRSFJv1Iw+sPHd/Nrk1KWjpO0OYRgfiB\noxcgGSzHp0OZP+e7Trtq1TKW6VyadEsPZ7oKeUcCgYB9TiMkrm4gXybLtkOOWKCD\ndG3qv7lmQlNKs66kCv+lxS0CirRM8i6on5flRbZmAGV28BEAaAu1zEe0isI1SC8I\nxTUnEvHpn1P/QbZfpX8zm/lMtpinRkamJZ8N7Hc4ggtb2152lBqlbtm+oBYL81sJ\niRss6uLFUv5T3Mr3Bn0sgQKBgGr4xQf+h6V2J307t12dQCRfE/MueX3jpDGVaFV1\notDkAxrt97GDH9uR+f7H56KlpIohAqq1M7nfUbV2FTbAhfIYd/GD9DYhzCMK7Ngm\nAlRP1MaPvjCh9nFgU7hn7PtZzwBwrHPIefZuZyEg7onVpfK5NTIPUW6XYOIJJX12\nIwvrAoGAGwg2Cq6Vs/KkiCDxoy42HVZLS/DS/iHfUTAXtekF88qwT1zHFqF1ImvO\n+FEUx8IbxY1oifSQc3Jw9HYnmYqmFW1PThNG3CwyaOGgtlEFcvhHsNjLRGJSQbN2\ncUYBe2hHXii7OL8T8kIvLlCAKGA8IIfPGhsu3uUtrvMBo5c0WmQ=\n-----END RSA PRIVATE KEY-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/cert_chains/cert.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDQjCCAiqgAwIBAgILANkmymLgb5e04tcwDQYJKoZIhvcNAQELBQAwVDELMAkG\nA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1Nb3VudGFpbiBWaWV3MSAw\nHgYDVQQDDBdjYTIudGVzdGluZy5mbHVlbnRkLm9yZzAgFw03MDAxMDEwMDAwMDBa\nGA8yMTE4MTAyNzA4MDY1N1owVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYw\nFAYDVQQHDA1Nb3VudGFpbiBWaWV3MSMwIQYDVQQDDBpzZXJ2ZXIudGVzdGluZy5m\nbHVlbnRkLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM0LRmYA\npXASAygtFNbEKc51VfR8wGQDQ1Ws6wxgPBLn/CseF3Z1aBPh88AL4i+FSwz8zQS8\nqGdsH98IOg/tVAIQNA6j0QooYXyagV9FoHsGpEpeWsK3NVyF7YjMXFhP+fl7aHYz\nu9YBAgYJnf5dFLXRWO5jLf4AKx/D0H9vQARogdc853dDDYANK9jpVyAnuJA80x6g\nwb+5xYEBa6Nr5CP0CQ+XwAW2+th0VoFH0Ndw1EoUhh+RT5DMU5ArORkoco2AguSI\nSPgCseUVr2+JLjEbMVguyR3P1yj3W3i5DVkQhc05zjkdPUHaeObQW17V9CIyJ8bJ\nzTgc1tM00oNMVLcCAwEAAaMQMA4wDAYDVR0TBAUwAwEBADANBgkqhkiG9w0BAQsF\nAAOCAQEAMzoxZJqhu8zyKI20Hz5SAqva+jcHbZbR9AXlt4LWQEttsXy4S52XLZBB\n4Es/Fah4lyou9R1xdUfF6iGBV6HgDMIuh+QYfqA/jJ4HXNMKLtDXakS8xYp2A0Wn\ng0EnjGx44DNweBkKGB9asldgwOM7MYMoQvS3Nkyu3oFqpfLpuWpWB/3Vw3PMGEdx\nr/c8Ev5XoVtl2+dXszsEp2jwUOFo8+DZzFdBZdA2F7RmcP+zxbAHIw/jHf/ptDGg\nNtzHmLhxiLblGlJHw/4a0HhXa453jzLsD1+hQpqBRUNbXBp1qpjqEr5SF1iVNGpZ\npeWFawp53MkIBXRls66ViJsl//fPBg==\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIDPTCCAiWgAwIBAgIKRbRr9ZfmrY72TjANBgkqhkiG9w0BAQsFADBTMQswCQYD\nVQQGEwJVUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDU1vdW50YWluIFZpZXcxHzAd\nBgNVBAMMFmNhLnRlc3RpbmcuZmx1ZW50ZC5vcmcwIBcNNzAwMTAxMDAwMDAwWhgP\nMjExODEwMjcwODA2NTdaMFQxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQG\nA1UEBwwNTW91bnRhaW4gVmlldzEgMB4GA1UEAwwXY2EyLnRlc3RpbmcuZmx1ZW50\nZC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6gnGxD4RQuAA1\nfQNMl9mctiIvcJ0Dz5f3JtCJlKKqBDKJ17py7tjFZ2MZ5vytIl1DUU2oxHew8F/u\nU8s+8ljmL3seiXx2kMCOxCvSW3JYUQrzbTHKszna7Ffy94SzpRDh/4/2RGatnSqi\nn4fRpalrchhH6Ds6v486o7YcwEKPR2dlD+Ltd/V3lMAOAmoRoe2aQlLudOUQjHJU\nYWGim8XS2dNAEy4WR5DvjYEA+CwKEX7f9p0Dihs8FKBjB2TvBwUkdfILLEnwA1rs\nr0coUt3kwYt9TYNYv6/SZYvFtaYjM3BmLVHI3mtEchaA+Tij4V4gRZ573Ai1+Fb8\nrxn+oTsVAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEB\nAHID7VD2PS7svmCXRg4qe7MkhA0veIadZRE6WZgJo3Z64p9jUWu/Y3ZrlZuDg0qI\nyV65UgWrgUOiGN6RZmI5A850DH3ryjvQElPHWjRavm0GtrOxFhHYpX8N7DRFW7t0\n7mld7YvbKRj0BrovU3avPC1P9w8FpegMKh48HsoRCO3NhHvYwtAf2B767UuAaqnI\nDW9rMPRTZv+Pv+G+kJN+updOs6HiHFxeS8Bwb8GlADcwWNGMqyJJyKmaefYVzDKt\nxfBiKiwev6Hm8RVTk1H3rKVZox/ZkDiZfZhmBJqO4mdDR4LLFIRIPqjUPxENsKFJ\nMyrDcwH+0Aq9ZSiRaVyfCkQ=\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/empty.pem",
    "content": ""
  },
  {
    "path": "test/plugin_helper/data/cert/generate_cert.rb",
    "content": "require 'fluent/plugin_helper/cert_option'\nrequire 'fileutils'\n\nmodule CertUtil\n  extend Fluent::PluginHelper::CertOption\nend\n\nWITHOUT_CA_DIR = './without_ca'.freeze\nWITH_CA_DIR = './with_ca'.freeze\nWITH_CERT_CHAIN_DIR = './cert_chains'.freeze\n\nCA_OPTION = {\n  private_key_length: 2048,\n  country: 'US',\n  state: 'CA',\n  locality: 'Mountain View',\n  common_name: 'ca.testing.fluentd.org',\n  expiration: 30 * 86400 * 12 * 100,\n  digest: :sha256,\n}\n\nSERVER_OPTION = {\n  private_key_length: 2048,\n  country: 'US',\n  state: 'CA',\n  locality: 'Mountain View',\n  common_name: 'server.testing.fluentd.org',\n  expiration: 30 * 86400 * 12 * 100,\n  digest: :sha256,\n}\n\ndef write_cert_and_key(cert_path, cert, key_path, key, passphrase)\n  File.open(cert_path, 'w') { |f| f.write(cert.to_pem) }\n\n  # Write the secret key (raw or encrypted by AES256) in PEM format\n  key_str = passphrase ? key.export(OpenSSL::Cipher.new('AES-256-CBC'), passphrase) : key.export\n  File.open(key_path, 'w') { |f| f.write(key_str) }\n  File.chmod(0o600, cert_path, key_path)\nend\n\ndef create_server_pair_signed_by_self(cert_path, private_key_path, passphrase)\n  cert, key, _ = CertUtil.cert_option_generate_server_pair_self_signed(SERVER_OPTION)\n  write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)\n  cert\nend\n\ndef create_ca_pair_signed_by_self(cert_path, private_key_path, passphrase)\n  cert, key, _ = CertUtil.cert_option_generate_ca_pair_self_signed(CA_OPTION)\n  write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)\n  cert\nend\n\ndef create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)\n  cert, key, _ = CertUtil.cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, SERVER_OPTION)\n  write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)\n  cert\nend\n\ndef create_without_ca\n  FileUtils.mkdir_p(WITHOUT_CA_DIR)\n  cert_path = File.join(WITHOUT_CA_DIR, 'cert.pem')\n  cert_key_path = File.join(WITHOUT_CA_DIR, 'cert-key.pem')\n  cert_pass_path = File.join(WITHOUT_CA_DIR, 'cert-pass.pem')\n  cert_key_pass_path = File.join(WITHOUT_CA_DIR, 'cert-key-pass.pem')\n\n  create_server_pair_signed_by_self(cert_path, cert_key_path, nil)\n  create_server_pair_signed_by_self(cert_pass_path, cert_key_pass_path, 'apple') # with passphrase\nend\n\ndef create_with_ca\n  FileUtils.mkdir_p(WITH_CA_DIR)\n  cert_path = File.join(WITH_CA_DIR, 'cert.pem')\n  cert_key_path = File.join(WITH_CA_DIR, 'cert-key.pem')\n  ca_cert_path = File.join(WITH_CA_DIR, 'ca-cert.pem')\n  ca_key_path = File.join(WITH_CA_DIR, 'ca-cert-key.pem')\n  create_ca_pair_signed_by_self(ca_cert_path, ca_key_path, nil)\n  create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, nil, cert_path, cert_key_path, nil)\n\n  cert_pass_path = File.join(WITH_CA_DIR, 'cert-pass.pem')\n  cert_key_pass_path = File.join(WITH_CA_DIR, 'cert-key-pass.pem')\n  ca_cert_pass_path = File.join(WITH_CA_DIR, 'ca-cert-pass.pem')\n  ca_key_pass_path = File.join(WITH_CA_DIR, 'ca-cert-key-pass.pem')\n  create_ca_pair_signed_by_self(ca_cert_pass_path, ca_key_pass_path, 'orange') # with passphrase\n  create_server_pair_signed_by_ca(ca_cert_pass_path, ca_key_pass_path, 'orange', cert_pass_path, cert_key_pass_path, 'apple')\nend\n\ndef create_cert_pair_chained_with_root_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)\n  root_cert, root_key, _ = CertUtil.cert_option_generate_ca_pair_self_signed(CA_OPTION)\n  write_cert_and_key(ca_cert_path, root_cert, ca_key_path, root_key, ca_key_passphrase)\n\n  intermediate_ca_options = CA_OPTION.dup\n  intermediate_ca_options[:common_name] = 'ca2.testing.fluentd.org'\n  chain_cert, chain_key = CertUtil.cert_option_generate_pair(intermediate_ca_options, root_cert.subject)\n  chain_cert.add_extension(OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)])))\n  chain_cert.sign(root_key, 'sha256')\n\n  cert, server_key, _ = CertUtil.cert_option_generate_pair(SERVER_OPTION, chain_cert.subject)\n  cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))\n  cert.sign(chain_key, 'sha256')\n\n  # write chained cert\n  File.open(cert_path, 'w') do |f|\n    f.write(cert.to_pem)\n    f.write(chain_cert.to_pem)\n  end\n\n  key_str = passphrase ? server_key.export(OpenSSL::Cipher.new(\"AES-256-CBC\"), passphrase) : server_key.export\n  File.open(private_key_path, \"w\") { |f| f.write(key_str) }\n  File.chmod(0600, cert_path, private_key_path)\nend\n\ndef create_cert_chain\n  FileUtils.mkdir_p(WITH_CERT_CHAIN_DIR)\n  ca_cert_path = File.join(WITH_CERT_CHAIN_DIR, 'ca-cert.pem')\n  ca_key_path = File.join(WITH_CERT_CHAIN_DIR, 'ca-cert-key.pem')\n\n  cert_path = File.join(WITH_CERT_CHAIN_DIR, 'cert.pem')\n  private_key_path = File.join(WITH_CERT_CHAIN_DIR, 'cert-key.pem')\n\n  create_server_pair_chained_with_root_ca(ca_cert_path, ca_key_path, nil, cert_path, private_key_path, nil)\nend\n\ncreate_without_ca\ncreate_with_ca\ncreate_cert_chain\n"
  },
  {
    "path": "test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem",
    "content": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,A73EC2C28AEE020735AE48A88ED5D3C2\n\nTpi5VwaaGpF7rrxjCtj+0Y3jZviTIBZSBYG43BPzRM+mhQ5AXBuLsttJkYURssoU\nyn+1zoilnehx+RUWGEBFyPeyDHUaAeUEGf8d6/FHv3D/9hlrzil/pkvyKN9KXzG9\nHYLJobLG1JPR0XgINMUP9W77n9bitR2hxm0ib3TAuTmEn7Ms3LoLmVIGYE91OExC\nKXcIpt+0Ue6qEzKv+Q4/L+7a/i3pNUUvMc15c6cVGdA+wiCpBCkwKvUGapjeoYrB\n+CVdKGte9Ug85fE68tIuSD7UG/1y1MXC0EwPAgO+U4S280aFiBfWgzfzEd5T72kN\n1O2vad/wCAN8Ruk2XXi7LlM1jFGY3R/CMuaiyVoFQZj5AY32BMxRIilsL8dnl7jG\nNmkpa0UOi0xhNpnXQ81BeN/nHFZY9xWX/uJwClw71IBJDk3+KKXEk7Ym0AhfAuao\nmX6C7KDyAjG6O5VoQ0YwoROzC2sl6ui1zRkey53552025p4OvaDGbuMkPApp3b75\nsVsA03evcFWQKF7roPN28+obW+SAgBNsg9EfR8Ebbmp4bPpZO9j1bitRxCtJEBLD\n9fHKfGRscRetWNBInkijM8+/vWMAf0U2gKwAJLXzelfIyqp2Ay03GgAEubcSWRjS\niU2AItmBsPHOaZ7dSh9MkU/H4DxqLZFxrj48sbDbm5i0zSgAi/9qd3LYXvL6oYZ9\n3ZHW2+hL6htemXf3DmMeNSnhU5W9Qqoths3CTJcIWeMuaKlkdiw4Gr3Qu0FCRcDE\ncrjpeR8ND5Bm2N/EkZKEMm/UDcobMW38F54Do/KVkyctelpW1XuYYcVqt4UhWatT\nEn64RbiSqG7wHa8jt4jHX10Zjc1LZAevXk5sVGJRN+y1IL6HFe4CcglLAOigvvPO\n33D0ft1faUgZlMQPAsXwnQyRpmeUrkyK3aec/6ML1PeJhDHl9OfQpkl2+8SL125o\nrVIMzlRefAG6SqywxqGk7DiSzp4vqJ2dkV9J9NI8mJYpPGpbuICtuOOlaNdiO4na\neESgCxkOk0yh74n2ioweXeT7Z+mnI8KhMMBqFmM4zyH4s3W/DbojQxZ4+W9s798d\nonoWoj6Knj9IgPlMOJGR7jtsPeTGNxWtvVMWD6uIy4Nml4hvSXn12MU3PJe0Iw+G\nmAlaQySxXBJiaRUGE52HLPF39Td8lB7RT11alQRRFjnulh4ayQlXDPiuj0s2sbop\nyFXCHYgPm16RCLk8qB/Ai2L0H6+A+KR7ewMPSjl6/xQQXf5UwjhU0RXoVpNjLnPA\nrhep+L+YUuQlf5nJ/6jBuSxSO3pG5RkrmoTpq4z0QvlL7ooK4p8NfZZV0Bs+ud8D\n6OWCda18t1C5k+tr9rR5rQbLsTy9X0B4udvO0m1b7WWVRAfOX8MK7p+VFb0WmKmQ\nDdzZ9Kmsi/j/VSlzqaqPq5XUgtMhnlLtf/9HK/QkDhIMITiyu9oGRASbOmRscVFH\nQGhtXY1XBOHUtOH4d6Dy8/AZSW1u2rHsWJDnyYdFzyOPlrdBG4qOtTG9GitxtA64\nBGDopav+kIdh0NsSG+6I9vxAP9MfGVtaXzVrdvBd3yXsZlRMWemj9SINq4GcMO3C\n-----END RSA PRIVATE KEY-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/with_ca/ca-cert-key.pem",
    "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAvyCuBhvNR2ZUukny+Qly3e0CTlQSee1nEULxxmg9N6aZc8Sr\nBW2RhCObexPYPUCVgyB0MJ2AYXXiXHy8xUXg6Cya6RqzPPLsvXcuoV2UgaKKG6HZ\nUBgzE9o9k2upbDPsQzzz2EEQxfkjHqmZNzkw7BbO+g8OF10owo+fFxlVxLh02GQf\ngkYns0rRb8YJcrEH/XdZrIJZkimQ1oX1RhpOvD5M63s8Yrg/WwX3e52W6IYmAzyi\nl6HKSZ1fKzQAYNj63CkUt5gXCxQ5vN9jYq7dPdVXr+YLO4QlC9Yr+R98iWYgcahW\nAISPWHC/9Nd2wQTLWubt9o8g8yaHqyIMcCCobwIDAQABAoIBAQCA7xfWsRyrZQIV\nZduOWuxtf/zFQHvyWs/j2ihmM13Q1AfDDFJe6Ap5fNffkISY1Z/HBLa9MRMxLChX\nQiLKZhiSJ70dMdDmde505hjD7PBI/5BxkZ6bJKgYZWqSr9dsUh65QT4yjbo2A5rI\njZr5oRA6xCw+jrdv1X2wutiU6Zdti+N9ouq/7ne6EY8pHh5VQ+bXtMbVrm/ejWV2\nPrsQwLqy1QyuNo9kUN126rlQIOy0s7KTBory3UGCSiZLeRlQPipIAuhQU1o+3B3b\n91ULwkzRorzqDoWsDOMeK/K76sQ+2HfAQYqjOIo3LcFA+xDxarIludVX4Aot3xkA\nBbieE0IZAoGBAN1eDbwLGG3S7NN+qXuMnNRSN2F9SWuHBuxbNfE+bg2ydah2R6um\nLL/jgzQhMiLMnkRqhgkvTDgWVlH0nH7JK6pt+vdrncygTrNmlLsaa62bm5zEQtzA\ns6IEncKkK5T7JCeICQ1W6L3S971ScHVfb6D4B8IAGzlmsATCvRsJf6UFAoGBAN0H\nf+XU8Qhuj4oJeF7KoA0hVgQoCO+LFCqYddwu7t1HGBhzKue7fs/45AX6gwEOPm3Q\nIfWfktlQpIjSyrBdj8ewUTDd9jX/tGjwTFSpkX+BzT/Vtl9gYMXymylmkfD8F4ll\nh5KdAHzxQuzhnsju9S3Py5x+zCN08ks1h7ACzBHjAoGBALBqlV2IJibYEv8WEXGy\nBQY8o4AhPdLg4EabBilFTKXD1Hq2ELYeD8m5QkXYMsGC+pqhnkJRnFeSjaZw4As2\nvYLsdTabYD3EgBP/K4bVLn1D5scPrg2J0V7MeQG2njjz0MNkaXplCcTGDcOmoJMZ\nxLevE1eriAa97IdOvzB96GkVAoGAe+3ajnlI9FtZbCJHNY8TEomgexAfgoBOvFtj\nqiM+lx9nqT3ZzrjYL6/z2k5N6eecpWHPyLLCWcrXlkfqkdzD40k/HNE3XauT8krZ\n4ZZ6GC4lcSdY7D1TxWl0ClCSf7Y5VDZzP4d8YJG93qfaqfshyZ/7IDIQL8lAmV7Y\nR54p4dUCgYBeJzyD7SETML67K0VvJn9jLrcE+hZTqaXIMDMVWRp6ekNuFDUaT1/A\nGYV9oPbo4QSg+FydWVuOwYvtf8YTfEt/FFNg2rmSXljHghRS6TbIAkS1kh3Xz6Vl\nYOD0ekf1FqxCO+h3PuaPHpC7uOh6HoB2ppH7jCvb4z4iWATujnYiEg==\n-----END RSA PRIVATE KEY-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDPDCCAiSgAwIBAgIKDYpkw43U3AatXzANBgkqhkiG9w0BAQsFADBTMQswCQYD\nVQQGEwJVUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDU1vdW50YWluIFZpZXcxHzAd\nBgNVBAMMFmNhLnRlc3RpbmcuZmx1ZW50ZC5vcmcwIBcNNzAwMTAxMDAwMDAwWhgP\nMjExODA5MTkwNjA0NTZaMFMxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQG\nA1UEBwwNTW91bnRhaW4gVmlldzEfMB0GA1UEAwwWY2EudGVzdGluZy5mbHVlbnRk\nLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKs8jpSiSUhphToD\nYV+8I27mVBr48Wo+R876RRVLAkdKb934xSYRvsy8QD9k+ZaA2EspchQ7ky3sdztu\nv4x5WqOHE7VE/HkbRcRfBKU9ujFou9l8ruSyLl8jTWoq1BPyVii7+ojcjNsodNF9\np0tsJOsnLgSA+yt3dI56V+PXsQkIwil5QGWDMCDQ80a1ncIA65uLjLwnWP6+XvgQ\njA/D2SrlORc18qGYC4J4ZA08VPdPtnzThjSV2UxkgBz+rT3FB97b/IzwImpG3GKI\nYeI1wfZAVEp3sA+kC9ufGKyGTHbMltB3c1E6z93gWpRSEk7TT802ixcYvl9V3L/A\nSbOseMkCAwEAAaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEA\nAGQkwrLOnOyf4EZjfz1Sn6IdiU7zU0jDnl1+pU5xtU4qZRa7MvMsfUPRFk3sLKls\nkeVu4DOWjBaC2ivgUt+j8odRqii2UdyltXAAoO2o1MMKoUKRZHYl5RPorzjuKwiP\nq6uxSkt/3c5Z6jAeS5JtDmX4EQgl9IHiV1zmWxV/rYJQ0kKBlsBhFX0CMJpNKJi5\nbxYH9kg7jMAL8Q/10tzWFT45O91UJlxTxEJy2YDiAMwiiOwA51h8ro23I4ag6XGE\nNYEd0C0GclxLgP9YCddHYXQe1ZwfeZIBNPboQurNhWes6igrhuPlJTDC9p8rF65b\nzxWqJGLs6eM+ctNibLCQfg==\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/with_ca/ca-cert.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDPDCCAiSgAwIBAgIKRRRc9wNmgVG33zANBgkqhkiG9w0BAQsFADBTMQswCQYD\nVQQGEwJVUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDU1vdW50YWluIFZpZXcxHzAd\nBgNVBAMMFmNhLnRlc3RpbmcuZmx1ZW50ZC5vcmcwIBcNNzAwMTAxMDAwMDAwWhgP\nMjExODA5MTkwNjA0NTZaMFMxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQG\nA1UEBwwNTW91bnRhaW4gVmlldzEfMB0GA1UEAwwWY2EudGVzdGluZy5mbHVlbnRk\nLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL8grgYbzUdmVLpJ\n8vkJct3tAk5UEnntZxFC8cZoPTemmXPEqwVtkYQjm3sT2D1AlYMgdDCdgGF14lx8\nvMVF4Ogsmukaszzy7L13LqFdlIGiihuh2VAYMxPaPZNrqWwz7EM889hBEMX5Ix6p\nmTc5MOwWzvoPDhddKMKPnxcZVcS4dNhkH4JGJ7NK0W/GCXKxB/13WayCWZIpkNaF\n9UYaTrw+TOt7PGK4P1sF93udluiGJgM8opehykmdXys0AGDY+twpFLeYFwsUObzf\nY2Ku3T3VV6/mCzuEJQvWK/kffIlmIHGoVgCEj1hwv/TXdsEEy1rm7faPIPMmh6si\nDHAgqG8CAwEAAaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEA\nIDGochw4cgYtK2RKgHQhsZ2aXUBo4qtM3L4HGFluagkDns4oHp0i3lwterYDYmf6\nnqVea3SJtk5XmVuGweOsilwIEmjUNk1kLwnjKyZYR1/vcfhriefGXmAzppRFeYh2\nK1OrmFZXvs8tRtMSt9YBbBeGbDdqpc9O4VkZb+hrysaaZiCE58cMGbxQEkvbyoGx\nEhRhIA+NbnWQ2RRoHXk76iMe2kbe3H5Bj06wJqYI2jH8yxEtDzMA96iVJuuz8l2p\n1Dv1f+4aFqUHOL0GgFuzwmmSQ6DS6/7WFglL3jeL9TqCSTiyY3PQtrzBI2hCDQpn\nYN6ELSQUASXSVmEQ4V5I3Q==\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/with_ca/cert-key-pass.pem",
    "content": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,F639769C79C6B97B06C4FD86B1ABC8B0\n\n++PSSn+v3H7i0DEZbM0Ypa718aw5CfkOleFFPn4FQNSZJckuIWblPQ6RLPdtkmlh\nKil/opC4y9dLoukze8CH+cGsqfPilT2ePs23t2ejdYu6N+3/aLbd0BIWCp2UtBqh\nOaHu9cbyUCGV95xx404zljc+cgcnzgBPtkhoa3zRdYb1NOOJK9l7WItCugktGfEJ\nMymOsRl2SspF9nni0pwz+VFUIRkCVii8u4sY5aGyasNzefIIuk2ItLIgmO+t9sz+\no0cfCxQndwrhEWKRTyOaP4tc6vHIToAlikjvf49K5YztNf+aMWQ5+FvwLQk5SxBx\n8Mba3UHZmFFG5R2pH/EiZKul2eXXFuEfamX7Sa2QoLQHlziev7QHQHNeOrpTIUYg\ngV0vSgpMDqU2mAYFyLWxvmiyJLVozVlJuZWisT+rcgRkSnv4M6ybJCxvWBEkQDvF\nAYh5OqZ8P/POxhDDWFpJCfzIdtUGT6ED/TyKtqbqFocEQYKUlp4EQjDmt0asbcqN\nwcwliPeU5fcgiwZyxK6bIr+AV9emF5acDT7mKeAYZI83OnRzkBg+Pfgw0X0HCMJo\nZzQy0oYFhxVWe1yEGvyy3XbA2+N1TowzgCk5aMA0RmB30hyeBjZx7wFZFOx9ZzDj\nUdC32L7Y1Tl+preRloQw5aYFbfKII88GiG8OqY9yusM3QHZUptjEXMv+nBfCsGg2\nA1Ki3dbwYer9ML/E8ndFpLjEW9fAoBUW4Qa7tKTF77wkNN58pdgs+tGddAT2wuV5\nF1YRVRENDXwaEoQirU7yn2cJtI1cX/Mx7pbkh7hYSTSekSZlHdRH/OFnitZ34BqA\nyC2Du3ea5djMdCrtKkmpeR8GhHkihSfT8ANdZwA2dw2uSMbqT60Dr1zK78h/P4c7\nscITpS1FB+tDBegZ2T6iFBYpReEXDpvpwVl6UHj7ds51se+752SHaW8Y54AEpkdH\n/EvAEKtBc0zvOWT7BqUcgNfCaDD/4B11twc7weiQlK5i829U9VL+8DaLuGtSOuyM\nA4+wLym9aeQIAMcdetaO3q6Ks66YQRHvuL205cYDik59SPQt6QjN3/fcc9enqsXg\npbQYduyDGW4po6dBaDsc20Jba61PNtKpU3qL8NhYysPPWlmw4CgZHm6m43NUkYxw\noWa4CRJuPR6c8XDX5GqPLDkd/dxmTHDIT64u/p2a5JNkT6SCMXHVQhI0DKcR+F7n\nmkEzf/qptbClEL3IdqcFCHXhmFZ9x6Ed6zykgKb3i+qWKcXBRJ+PEQeANS/2HnyV\nwtRN70UaLag9RBq8bFD9Fyh7NBKzlcf3CoqYKxGFnegHvhtwI+X2B9bNxJam/V49\n2TCHhx3GAtqL8f29PHUATudvdvTQXTsmm5CB5QD/hvwYO1f1DMP9qMQAr7FDz8t9\n2+sSWL8GyG7PI4aQ8pr7a6q2dXK2AWfXF/avAc1gXzmU8pk/m5j4qLRNA9OIxoec\nrj5FCGkwP/mR9u3eB2AGTPU1OzWwkoQpy2H3kTVdQ7UWqz72pA3v3t+gEqAfw9RB\nBf/hugMNWXFdy8S0P6mzRq9gcjMrLfDI/0Q2tYXhzoMrBBgsSq3uM0N4WS0dj+ck\n-----END RSA PRIVATE KEY-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/with_ca/cert-key.pem",
    "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAyINyAOHXX8VRh8IC+hs+jB78tOqM6dvJt40L1sAjKc78zt+Y\nWJI/DdU7c8Rs+gv0DvVwjEFDXk6O3DsREInmqXUsh6YyC2M4BdLrlUkuKJnTg3FS\nBG5eXNA/eBOME0A94YQAkREEqHREfUZv6T8EwNPz17dtWD+wmezKA4ZtGugAajOK\n3txRJsGA4GAiAP1JL+yr3yAnI4gqkkekFrs0ChhfRXKMppGaSgBWOFOLtoUzWmXP\nmI5etpyIUGpp7l7YLlc/HiwI7bwtVcpEDUqGj+iBrfAbsg8yrsNY5VDJXrABX2kB\n7rHuRga5FfZQZFzP1XiDAINdnI8zZ8/x6peAkwIDAQABAoIBAQDBa+LmLPUoglwi\nfCmJAAz299FH59YkpQrRz/JypweEhhHisVfxYKoIewANrZZ+Ile8slBuS2pkJ52x\nnhzY+l2Sa+CXvd8akwxwucTdTimDqteZPRAkDB9kotIGz7xYuai3yTEOvv7iTL3g\npri/pDW+mPboyR1mepGt+ffpLJxEzYAW6IbX7BYsDLVZgyluj0JguAZG+6IGATwm\nAsvxWu32UclJIPvkZMmlmTiAXozqy4giZMkjujGq04srz6bk3vtfcnKqt7uTd/3J\n4RETTUTpTKfXmdiFeWacb8dJ9qpmq3fBsl6+AhuzIE7Uwfpf8s2ip0J7JnaMqldH\nCgs85xaJAoGBAO7v4+PbXkmMZZSyyKVMpj+KHnG0S0LqnuaxPGAHJhH3VUBFyQ99\nUCZdn2CiR1m57LA5ZqI6RtpqN/iSCnyr6SBod03ngZQ8vqdlSleUYKzOn906yDNA\nrO08vTSi4dgypgIE96TyPN0E6j0yIF+pz5xXr4HUPS61WAZ+VbziVRr3AoGBANbV\nHeXypaL7h4JEzxwxGo5NdTV8dL8Gncy0KynNyLvHrc+EYGOG3ggKk2X/4xKBUUyr\nKcVXPG+dKV8HGlnWqS1F6fnqQXxV+jfEGTPPM3npQNSavHPDAh0OpEOq9guY3yYd\nFvEqNlXE/hr7sAJJizbtbZBud1QA34F9mYINZ6RFAoGARVysakwXXgB8yKLx7lKG\n/HELD3UIWB+mC1AuN647lgGX5PWdPXriVpDdIH9beSepFytCqRuD0An/v5AdrT12\niqro31uyWScdLZkQ8MWpHuFItT9IOOCwkpoMTwhtyYZCfNmWTnLnyj2QBj3T+hFg\nrIEj+Ot062dHa3MnTpLho9cCgYEAkBldN5s9fQgISdnLrhZFwMO15c14JZXTjLYg\n6aZtiX8TV8lsOfYBwhUN73HnJsd0998rYoH0OM2LLYF/LFlvGr31hjuZ90NYVcWE\nPhsyZQiPAsxTsbZRGZpzliv7Y6YDr/X9KIBu97Lt2r2NvssDxTEWt7VQ9xiq0pmj\nuPcy2J0CgYBNuetHnG0NBFHtidyjV/kvvJ2E5vt9dBxcC4dBssi6XWyiuO/KlRIM\nT0tBXHS4PlkyO4O/KTugEZmLiGvVowCGx1CjhU0rTKwZNSAfyFRrngQvkHCqAaoK\nCM7zlXxnRedmLKD83vNlP1jnpnuLPSJNZrQy1Avy5xKSmiO1NYvm0Q==\n-----END RSA PRIVATE KEY-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/with_ca/cert-pass.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDczCCAlugAwIBAgILALvoPZFcaSh8StIwDQYJKoZIhvcNAQELBQAwUzELMAkG\nA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1Nb3VudGFpbiBWaWV3MR8w\nHQYDVQQDDBZjYS50ZXN0aW5nLmZsdWVudGQub3JnMCAXDTcwMDEwMTAwMDAwMFoY\nDzIxMTgwOTE5MDYwNDU2WjBXMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExFjAU\nBgNVBAcMDU1vdW50YWluIFZpZXcxIzAhBgNVBAMMGnNlcnZlci50ZXN0aW5nLmZs\ndWVudGQub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5LPQ9WhJ\nLE91y+ttRTgBhkDZo8lRBtxX5NDMR5dak2WCKwsM5ZtN5LaYNNbG73i7HOSu3zvu\nNOUEop7q5ci0tcx8jw9DidwOYjqsC7IvcwCFzc4JuQuVZgofVUf8Ia0LWPtOMBqM\n/LlXW0s+am9am0oNaOYV6VLTXr+2YzYhSoQEVu+FeoWN2wKfPuI01+6tWRwIDnbU\navRVDeFaHlSJVHaeNavMX7x+tWHpOy3JU0J5u29G7eplLXv8s8cI3p+CY9UpODFw\nJ9Ii7/SH4yR8iaOPhZPE/clf7AWioPOstctnPh2E+x6oM2Rkzp8HNX1s0gBpispW\nft+DJoEB28JCsQIDAQABo0IwQDAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIG\nQDALBgNVHQ8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQEL\nBQADggEBAAAKJcp/ZBYyClafm4JXGaiolgH9z8BfOktSxOCWmJrfKbflkPOu2UqZ\nUKJAUtDJVWU+U+oUDfnMpq0/iBm1HioxXfV8lSXKP380o7wQuar8G2jSjbGPWCxr\nv0PMx3buD0saS2A5sDgH+V5gMIPP/ZiuEu3Mro0+mHRj194vs/1Iedfd5Khg0o15\nHs15qcqY16E2TvuirONBwwMu+Zuemnu0NbwxyRx0VoHCDTJ4JMhVTGNLa/xWvwD4\nYoqYnWm3DIVr1hmWivCUD6p+wU6aywGMu8uGBA7pZtP748l/tsMOzOhYNE0h514F\nFxFbU/Iyvo2KW60v9HnaJsCk1LjXIHU=\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/with_ca/cert.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDcjCCAlqgAwIBAgIKUV9t8OkUnCm2mDANBgkqhkiG9w0BAQsFADBTMQswCQYD\nVQQGEwJVUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDU1vdW50YWluIFZpZXcxHzAd\nBgNVBAMMFmNhLnRlc3RpbmcuZmx1ZW50ZC5vcmcwIBcNNzAwMTAxMDAwMDAwWhgP\nMjExODA5MTkwNjA0NTZaMFcxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQG\nA1UEBwwNTW91bnRhaW4gVmlldzEjMCEGA1UEAwwac2VydmVyLnRlc3RpbmcuZmx1\nZW50ZC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDIg3IA4ddf\nxVGHwgL6Gz6MHvy06ozp28m3jQvWwCMpzvzO35hYkj8N1TtzxGz6C/QO9XCMQUNe\nTo7cOxEQieapdSyHpjILYzgF0uuVSS4omdODcVIEbl5c0D94E4wTQD3hhACREQSo\ndER9Rm/pPwTA0/PXt21YP7CZ7MoDhm0a6ABqM4re3FEmwYDgYCIA/Ukv7KvfICcj\niCqSR6QWuzQKGF9FcoymkZpKAFY4U4u2hTNaZc+Yjl62nIhQamnuXtguVz8eLAjt\nvC1VykQNSoaP6IGt8BuyDzKuw1jlUMlesAFfaQHuse5GBrkV9lBkXM/VeIMAg12c\njzNnz/Hql4CTAgMBAAGjQjBAMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZA\nMAsGA1UdDwQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsF\nAAOCAQEAnLqulP1n8fggiY5e6WukL8MmBitpalxLkzRRL7xBihQJzsiNjpA/Lt95\nSyabg73nDOTl98wCQJw/PWmZjDCgKf2c1UA+L9fSkLxQf5gLTMnVz6g6tB5m2LdA\nQAG+B2uucye4e/uFxJ1wkXDC508ipi0NzgG1/tO8ZX0SzeMxV2nH4jjn1X3jOHcM\nxvB32+sZdYCk8tmZ4s3cSKLKjoP8YaTiII7MjmYNTGuuUE4QL5Q1t8u1NcGvLIc/\nyfjHjvifA2aoyS20H06wOxUMtqP0n5yRYv7aLSqX7ZG9Srrnraaa1lSrv9mNR6GR\ni4v43IXfyTa69dx2otUQpF2/NkUBNQ==\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/without_ca/cert-key-pass.pem",
    "content": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,FDFF12531E3F46EA8FB2AF34CF90DAB0\n\nwZAwCeW2uOYVFeZPKR9x1og5wXZl2HJiLTyG5wK5P+6naKMhr2sKtyB3QWpVUnTL\nTPkqn3J8C0sWDNByI6nG4xtceFi9HMoBzwfhtmThnFE1BDqCaDktkg7SCc43/w6y\npw0GpJzgldGiMV1HSpFNAP/yUqV1Ybbro7wk7kRahT5ryKNqeWWgr9DvQ+O61dKK\njChFmDZgbokAFYl8SicAcUt8KIZp0UonLYe8neXki/gVip10+M9pphdntO7r68hJ\nlDwuHoos9YNkfsaTW3Ef6e+nbnw6FwY+Yms/g/2ItkuhO+6zohhLniynB3kOsBnb\n9BxtY9EdkNbFijPZpAMOLprfmuVT/MpoHxGeCdIvS2iaOKPMzDWGD6QvFd+0cyoP\nDl8imzFf36FzBexdj3/0Luq/O4blxfaiUpWUTMFUxhwMI/FP60wANpBFMCLOkKZw\nuJCmypJ3Bq4Ay72KGOp0/SuTn6gthkR7e1zqjOrncc4feEIWNN3GvGkJX/0hF9/h\nValvPboYB0C3Um3vP1lcva6HNTbnBROfHlpXs2PwuKNkHhifynY3cnU9qSR2LXEK\n4atBC3aJlcsZrAxYKLUhI/rCvZ2buLPqsELUNcauP1SR6OJtSG2QokAd8Tt+070D\nWDPNhIP+06AaKhQI9r+eiKY12SZboxns+Fg8/0VTBxiX9OzOTz+4z5WHX7epPOKj\nITvI3lq577RHQvvc9BPTfWYvLv5GU00UQbbJ4BalUlV7j9EDpp+jyExtFMnucd/K\nf8/WFchPbL/yqOf8T7BCTywk+nGho6pPNkMoja/ACtVau07MjizRT1MhrX2CMNuP\nO/8BsjTDYRCtuM4bqe5hvOmyfZ4i2yx9CiAFc8iWQM3dVEchz5azaxKjX2YlSmNr\nCh4zbyBq7H0rHjLqSezgQivkLVDHhIe5Nt/lj5sF/QUT12sD2WJIt7rAk+Fb8qZX\ncjSnirLC0qNhsLwLrS425r1vJgHB9VDQPlL5NJQW4kMmoNXv4+vHYVkxU7hxzw21\nrFgOWoBXVRK1vO5nE95gX3zgF1F2JxmWjBTrAxG9fE7zbrcz2Bokz0jbUn+6UEyA\n79aYnuPBZL2DbHE7fsVZL19sb4LUVA+HtVzCEQDrnCR1ew0TXG9LLtVYzP/XiAgd\nPaHrlYoyA/46+ho8bGckj37hFALulOP0a/RlJ1s7Axnl4R8GsK7IBA4igGaPyEET\nr4IK1skY3xjCtiEDI/inuIr51hnqUoVhTqmA5YCT3qG3CoJ/glaaUL9d0/Er2IWp\nnBcOIBEuE5gxwgNQPKXSV0VG1Pkp8nWpVkG1hk4uwQL0w/pLiouQk86Sz9VxBDlp\nLlw+Y9gAdrSswZNRItOfUp7QZle48cbTnxhApGaAFKDjXDyXu2xEQsD+GXqsVrqE\n1ouiKcE5AIV6aZ7GiBm9Js8C5/sIQHqa900/R2H+avv6QYaQNrvE+ihQRRSQ5+79\n2EZWJhGLe5jfylVNjKGIZSN12GC7hSTbze0Lmr3W3FZnOOCm95XPuuZVaKVjXCW6\nG37029JZjeoLjouMYFTxgyN/v5TlsuhPRzoFrODqanL/w8sXFPijSJaYW7R0sVJu\n-----END RSA PRIVATE KEY-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/without_ca/cert-key.pem",
    "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAv99VXFkMqdT//4p0lXTIlA68tP3kTTAmYemyi3nX2K5I03ML\nO9OXpU6rNP/JA39fh2bk89la8l1jlDRzUO+dKxYZnyZ0BnYCeTsSM5JnJ96S97cV\nEZRRqdtpArjKpaIL3dnnSk0j+67e+lxj/YbKizcErvvjRnvK0P7F5JtbusUQMQgL\nUG1PwMviiNkrP40DiKOYjEaSYVSH9VtBgDX0x1oLjLBZUS7HD7lvO0cSIlcHns0N\nNyxlribDPW0VKg4wbEa1qZpVFlxSHudfi/ZUKlAm8Ib7jN4dzpA9pMrhFBl/vStn\nWpczYYg3ncHhMo6EOxyByyV+iEv9tR4ynbOu5QIDAQABAoIBAHPAly2kJ0iZ0Gro\nq1ay4oK3tU53UPDkGHO/WdBdDEOTOSofYGqSRw0aPZ02pp0ujYIRKBSYTw6iGn4B\neoQcWjPxHDnwJ9TyAqICagOk0giydPVuhQ5a7T1MubhDpTVxYjlPyQIBMbEXmTtu\n23/Kfoyn/3JVUZJnAj89REObGia8yP7Iz1BtPIN34NvXVUcyt+DbqBWksFDcUAC7\nB+yKHg9gbpQNlM8s4it1vWmmQNif/DBoV8/bUrCqh7G3mTPGr6vuovFR+m0PxZa2\nnWn4V+I3/O0LxpKstOyg13YEaaw6U6IBZ2881hNPaDuZ+QPIBaM8fTWZCGjIMAeH\n25uN1GECgYEA9xMrAodz3EG4E9tNza+gEF6pGFALYlaK7xsrlfY0b5V1Yab7NAjG\nXH4DBZ7V8KN86xFOZ36KlVw/3WgXW3Akw0LHkYPLtxKvArTxn6/gEUS0UiLS1mEi\ntZO3ackRr9ZB/Z+SuwXpgS0ogtFIi7XVRNOGFfBarSENplsFTRGLEV0CgYEAxs2t\nzi9sh+GmQW3CKYUFlH8bj/erYRy5ulV+sUxP94UUhTGz+XzlUn/k4lIu0vzRFIcY\nEd74KPzpDTZd1Sh8N5Cgq1ag83oYlTrpY1Dj3iShVYMnMCSYSHzNZGmTztnAiTq7\nwGfKcV63hb7HMpoODcdqfgx3a601O3C1YDZ0EykCgYAzHwwvIcefMxnbQez/Oe9s\nc6mZFjbEyhNFFeIWzTJcRBhddCdVxNsGGCyX5R1vBOS73oDSeSRDcyrjrL3odXOl\n0xNn0FU+M+dyL+4ulD1QVV7Z/8qQANklqMKyaJ/4OeS9jV7Ww4pk5+sR5iuUfLL7\n4qhDsc+RMt2YJg0avR6BIQKBgGj6jXAWbNooFvHv8pqYKsUqhbR1VWNXlt/Fa7xW\nDtgmxxo6j84L3NC+xm+YdC9n30RWvigZtLN4MdyPO+dmlImG7sDrYtyPOdH7pOiT\n+2//qc4NDrGGcX+9n3wJoEqXg+szP5evdyB8oq6WenvHC+rMxfg/vwP2F+kGcFCZ\nEJxZAoGBANiBa2CNHvK2E6Lqx5fRe//G9dvpjszQWIIWeFJ3uyEuQQ/mZu9S7Vmj\nbhyeH2Z9r9D9MLZDVoclXOzvPvT8KhguLwnb1zJUoGmWIjkTQv7LAhWal12fDfRg\nn4FhPKjvcRLMIf+IKYQ3ZDysJGUsGaduTBR5QjyZYxV1QZLNYMqp\n-----END RSA PRIVATE KEY-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/without_ca/cert-pass.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDVDCCAjygAwIBAgIKWcJH+2qScaQGFTANBgkqhkiG9w0BAQsFADBXMQswCQYD\nVQQGEwJVUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDU1vdW50YWluIFZpZXcxIzAh\nBgNVBAMMGnNlcnZlci50ZXN0aW5nLmZsdWVudGQub3JnMCAXDTcwMDEwMTAwMDAw\nMFoYDzIxMTgwOTE5MDYwNDU2WjBXMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0Ex\nFjAUBgNVBAcMDU1vdW50YWluIFZpZXcxIzAhBgNVBAMMGnNlcnZlci50ZXN0aW5n\nLmZsdWVudGQub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAouXb\ncQ3Ij+n8H4MyOk1CBOh8B7cblXsmyHtDnYstzFO6ouoYssZ8WbJmWetqA1z3Lcr9\n0Ft8gFIYkBywb4V1hFcJ/G4feIcAvVrjL22/jqqLA8dlXLCqCywo7yHffEzBAn59\nvckYL69WZEJWP0bGz+0My82pU5r5rKgzhhuQKzdlypVSaKVvpYF9LMfLj/aiCPRM\nhg+jwhVX0z2xYpu47IId/gmM2K81m+s/OSDBijuph6RxwoONkM3s40+HNLGxAaDz\n3XJenkcy4Zl+74n3yGxLYFoTUTapT1gd0/3uaA7Q+8rMc5aMKJtcksB/fAsz6v8B\nWdu1yyPtAmbAwA44FwIDAQABoyAwHjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQE\nAwIGQDANBgkqhkiG9w0BAQsFAAOCAQEAMyGTHFu4DRNx6HQMjcYEaNAFRYVjVkRV\n43hoTO5lBUNbe2RxwcRzIFW40PW3BQnO7xrMcgcy8X1sukTRV4iHR/RO1OSrOmI/\nDp1goB/XAhHHi6Q250M4HtuiBsfSCxk8ASNnXESYitT01Y9+WNIT5Ir65+3L8f2f\nOygQ/QXv3PlK+HtbZJMsYfmFn45d8iXSuIZzjAIJ+GzB1chgslfqcPtWIzwyi3py\nNLN5o7bnzhWzViq2g4MCh4nBJIPFRsCh7wzBy97KiU7Ex9L9WSRwTUbeRLx6K7YI\n4MV5v1pz3RbPbD4AS7QgIsBRlW0udWvc1zzNzHGEMGjMGz5K/UEsQQ==\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "test/plugin_helper/data/cert/without_ca/cert.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDVTCCAj2gAwIBAgILALCfeVLdPoGOQEcwDQYJKoZIhvcNAQELBQAwVzELMAkG\nA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1Nb3VudGFpbiBWaWV3MSMw\nIQYDVQQDDBpzZXJ2ZXIudGVzdGluZy5mbHVlbnRkLm9yZzAgFw03MDAxMDEwMDAw\nMDBaGA8yMTE4MDkxOTA2MDQ1NlowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNB\nMRYwFAYDVQQHDA1Nb3VudGFpbiBWaWV3MSMwIQYDVQQDDBpzZXJ2ZXIudGVzdGlu\nZy5mbHVlbnRkLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL/f\nVVxZDKnU//+KdJV0yJQOvLT95E0wJmHpsot519iuSNNzCzvTl6VOqzT/yQN/X4dm\n5PPZWvJdY5Q0c1DvnSsWGZ8mdAZ2Ank7EjOSZyfekve3FRGUUanbaQK4yqWiC93Z\n50pNI/uu3vpcY/2Gyos3BK7740Z7ytD+xeSbW7rFEDEIC1BtT8DL4ojZKz+NA4ij\nmIxGkmFUh/VbQYA19MdaC4ywWVEuxw+5bztHEiJXB57NDTcsZa4mwz1tFSoOMGxG\ntamaVRZcUh7nX4v2VCpQJvCG+4zeHc6QPaTK4RQZf70rZ1qXM2GIN53B4TKOhDsc\ngcslfohL/bUeMp2zruUCAwEAAaMgMB4wCQYDVR0TBAIwADARBglghkgBhvhCAQEE\nBAMCBkAwDQYJKoZIhvcNAQELBQADggEBAJ5U5HGK2lmK6NKQGw63tbBqQ38tSCix\na3OwOQ9M4KeVGgAiG6fgESgv0AAN37pzoNRQ1soPOIazovdsbKUcec/cbMQoODuO\ng9KqueecB0BkSgIQXJWtoFWn1m9I/11VnbCbv1fYsui5fKANL9V6QUa052oNu1X+\n8I6FQpgzcY3sGw8jiu1sQYBg6wIQJfl4bf6wQBqVhUeaSeY++1F1SNuB0k6H2DgA\nOgGULEvcIb0dvSh/luOpIHoUsyv77aKWnC3LSwUD/M8/Lv43bcJKfDT2KjLeW8DX\nlCMESmm0ppShIa/fobNxq2Ale6oFxHma2t7pG8ZxCxn5ojiFQ3R6Lz0=\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "test/plugin_helper/http_server/test_app.rb",
    "content": "require_relative '../../helper'\nrequire 'flexmock/test_unit'\n\nbegin\n  require 'fluent/plugin_helper/http_server/app'\n  skip = false\nrescue LoadError => _\n  skip = true\nend\n\nunless skip\n  class HttpHelperAppTest < Test::Unit::TestCase\n    NULL_LOGGER = Logger.new(nil)\n\n    class DummyRouter\n      def initialize(table = {})\n        @table = table\n      end\n\n      def route!(method, path, _req)\n        r = @table.fetch(method).fetch(path)\n        [200, {}, r]\n      end\n    end\n\n    sub_test_case '#call' do\n      data(\n        'GET request' => 'GET',\n        'POST request' => 'POST',\n        'DELETE request' => 'DELETE',\n        'PUT request' => 'PUT',\n        'PATCH request' => 'PATCH',\n        'OPTION request' => 'OPTIONS',\n        'CONNECT request' => 'CONNECT',\n        'TRACE request' => 'TRACE',\n      )\n      test 'dispatch correct path' do |method|\n        r = DummyRouter.new(method.downcase.to_sym => { '/path/' => 'hi' })\n        app = Fluent::PluginHelper::HttpServer::App.new(r, NULL_LOGGER)\n        m = flexmock('request', method: method, path: '/path/', body: nil)\n        r = app.call(m)\n        assert_equal(r.body.read, 'hi')\n        assert_equal(r.status, 200)\n      end\n\n      test 'dispatch correct path for head' do |method|\n        r = DummyRouter.new(head: { '/path/' => 'hi' })\n        app = Fluent::PluginHelper::HttpServer::App.new(r, NULL_LOGGER)\n        m = flexmock('request', method: method, path: '/path', body: nil)\n        r = app.call(m)\n        assert_equal(r.body.read, '')\n        assert_equal(r.status, 200)\n      end\n\n      test 'if path does not end with `/`' do |method|\n        r = DummyRouter.new(head: { '/path/' => 'hi' })\n        app = Fluent::PluginHelper::HttpServer::App.new(r, NULL_LOGGER)\n        m = flexmock('request', method: method, path: '/path', body: nil)\n        r = app.call(m)\n        assert_equal(r.body.read, '')\n        assert_equal(r.status, 200)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/http_server/test_request.rb",
    "content": "require_relative '../../helper'\nrequire 'flexmock/test_unit'\nrequire 'fluent/plugin_helper/http_server/request'\n\nclass HttpHelperRequestTest < Test::Unit::TestCase\n  def test_request\n    headers = Protocol::HTTP::Headers.new({ 'Content-Type' => 'text/html', 'Content-Encoding' => 'gzip' })\n    req = flexmock('request', path: '/path?foo=42', headers: headers)\n\n    request = Fluent::PluginHelper::HttpServer::Request.new(req)\n\n    assert_equal('/path', request.path)\n    assert_equal('foo=42', request.query_string)\n    assert_equal({'foo'=>['42']}, request.query)\n    assert_equal('text/html', request.headers['content-type'])\n    assert_equal(['gzip'], request.headers['content-encoding'])\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/http_server/test_route.rb",
    "content": "require_relative '../../helper'\nrequire 'flexmock/test_unit'\n\nbegin\n  require 'fluent/plugin_helper/http_server/router'\n  skip = false\nrescue LoadError => _\n  skip = true\nend\n\nunless skip\n  class HttpHelperRouterTest < Test::Unit::TestCase\n    sub_test_case '#mount' do\n      test 'mount with method and path' do\n        router = Fluent::PluginHelper::HttpServer::Router.new\n        router.mount(:get, '/path/', ->(req) { req })\n        assert_equal(router.route!(:get, '/path/', 'request'), 'request')\n      end\n\n      test 'use default app if path is not found' do\n        router = Fluent::PluginHelper::HttpServer::Router.new\n        req = flexmock('request', path: 'path/')\n        assert_equal(router.route!(:get, '/path/', req), [404, { 'Content-Type' => 'text/plain' }, \"404 Not Found\\n\"])\n      end\n\n      test 'default app is configurable' do\n        router = Fluent::PluginHelper::HttpServer::Router.new(->(req) { req })\n        assert_equal(router.route!(:get, '/path/', 'hello'), 'hello')\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/service_discovery/test_manager.rb",
    "content": "require_relative '../../helper'\nrequire 'fluent/plugin_helper/service_discovery/manager'\n\nclass TestServiceDiscoveryManager < ::Test::Unit::TestCase\n  setup do\n    @sd_file_dir = File.expand_path('../../plugin/data/sd_file', __dir__)\n  end\n\n  class TestSdPlugin < Fluent::Plugin::ServiceDiscovery\n    Fluent::Plugin.register_sd('test_sd', self)\n\n    def initialize\n      super\n    end\n\n    def service_in(host, port)\n      s = Fluent::Plugin::ServiceDiscovery::Service.new(:sd_test, host, port)\n      @queue << Fluent::Plugin::ServiceDiscovery.service_in_msg(s)\n    end\n\n    def service_out(host, port)\n      s = Fluent::Plugin::ServiceDiscovery::Service.new(:sd_test, host, port)\n      @queue << Fluent::Plugin::ServiceDiscovery.service_out_msg(s)\n    end\n\n    def start(queue)\n      @queue = queue\n\n      super\n    end\n  end\n\n  sub_test_case '#configure' do\n    test 'build sd plugins and services' do\n      sdm = Fluent::PluginHelper::ServiceDiscovery::Manager.new(log: $log)\n      sdm.configure(\n        [\n          { type: :file, conf: config_element('service_discovery', '', { 'path' => File.join(@sd_file_dir, 'config.yml') }) },\n          { type: :static, conf: config_element('root', '', {}, [config_element('service', '', { 'host' => '127.0.0.2', 'port' => '5432' })]) },\n        ],\n      )\n\n      assert_equal 3, sdm.services.size\n      assert_equal 24224, sdm.services[0].port\n      assert_equal '127.0.0.1', sdm.services[0].host\n\n      assert_equal 24225, sdm.services[1].port\n      assert_equal '127.0.0.1', sdm.services[1].host\n\n      assert_equal 5432, sdm.services[2].port\n      assert_equal '127.0.0.2', sdm.services[2].host\n\n      assert_false sdm.static_config?\n    end\n\n    test 'no need to timer if only static' do\n      sdm = Fluent::PluginHelper::ServiceDiscovery::Manager.new(log: $log)\n      sdm.configure(\n        [{ type: :static, conf: config_element('root', '', {}, [config_element('service', '', { 'host' => '127.0.0.2', 'port' => '5432' })]) }]\n      )\n\n      assert_equal 1, sdm.services.size\n      assert_equal 5432, sdm.services[0].port\n      assert_equal '127.0.0.2', sdm.services[0].host\n\n      assert_true sdm.static_config?\n    end\n  end\n\n  sub_test_case '#run_once' do\n    test 'if new service added and deleted' do\n      sdm = Fluent::PluginHelper::ServiceDiscovery::Manager.new(log: $log)\n      t = TestSdPlugin.new\n      mock(Fluent::Plugin).new_sd(:sd_test, parent: anything) { t }\n      sdm.configure([{ type: :sd_test, conf: config_element('service_discovery', '', {})}])\n      sdm.start\n\n      assert_equal 0, sdm.services.size\n\n      t.service_in('127.0.0.1', '1234')\n\n      sdm.run_once\n      assert_equal 1, sdm.services.size\n      assert_equal '127.0.0.1', sdm.services[0].host\n      assert_equal '1234', sdm.services[0].port\n\n      t.service_out('127.0.0.1', '1234')\n\n      sdm.run_once\n      assert_equal 0, sdm.services.size\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/service_discovery/test_round_robin_balancer.rb",
    "content": "require_relative '../../helper'\nrequire 'fluent/plugin_helper/service_discovery/round_robin_balancer'\n\nclass TestRoundRobinBalancer < ::Test::Unit::TestCase\n  test 'select_service' do\n    rrb = Fluent::PluginHelper::ServiceDiscovery::RoundRobinBalancer.new\n    rrb.rebalance([1, 2, 3])\n\n    rrb.select_service { |n| assert_equal 1, n }\n    rrb.select_service { |n| assert_equal 2, n }\n    rrb.select_service { |n| assert_equal 3, n }\n    rrb.select_service { |n| assert_equal 1, n }\n    rrb.select_service { |n| assert_equal 2, n }\n    rrb.select_service { |n| assert_equal 3, n }\n    rrb.rebalance([1, 2, 3, 4])\n    rrb.select_service { |n| assert_equal 1, n }\n    rrb.select_service { |n| assert_equal 2, n }\n    rrb.select_service { |n| assert_equal 3, n }\n    rrb.select_service { |n| assert_equal 4, n }\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_cert_option.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/server'\nrequire 'fluent/plugin_helper/cert_option'\n\nclass CertOptionPluginHelperTest < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :cert_option\n  end\n\n  class DummyServer < Fluent::Plugin::TestBase\n    helpers :server\n  end\n\n  test 'can load PEM encoded certificate file' do\n    d = Dummy.new\n    certs = d.cert_option_certificates_from_file(\"test/plugin_helper/data/cert/cert.pem\")\n    assert_equal(1, certs.length)\n    certs = d.cert_option_certificates_from_file(\"test/plugin_helper/data/cert/cert-with-no-newline.pem\")\n    assert_equal(1, certs.length)\n    certs = d.cert_option_certificates_from_file(\"test/plugin_helper/data/cert/cert-with-CRLF.pem\")\n    assert_equal(1, certs.length)\n  end\n\n  test 'raise an error for broken certificates_from_file file' do\n    d = Dummy.new\n    assert_raise Fluent::ConfigError do\n      d.cert_option_certificates_from_file(\"test/plugin_helper/data/cert/empty.pem\")\n    end\n  end\n\n  sub_test_case \"ensure OpenSSL FIPS mode\" do\n    setup do\n      cert_dir = File.expand_path(File.join(File.dirname(__FILE__), \"../plugin_helper/data/cert/\"))\n      @tls_options = {\n        cert_path: File.join(cert_dir, \"cert.pem\"),\n        private_key_path: File.join(cert_dir, \"cert-key.pem\"),\n      }\n      @d = DummyServer.new\n    end\n\n    data(\n      enabled_fips_mode: [true, true, nil],\n      skip_checking_fips_mode: [true, false, nil],\n      block_incompatible_fips_mode: [false, true,\n                                     Fluent::ConfigError.new(\"Cannot enable FIPS compliant mode. OpenSSL FIPS configuration is disabled\")],\n      not_care_fips_mode: [false, false, nil]\n    )\n    test 'ensure FIPS error' do |(fips_mode, ensure_fips, expected)|\n      stub(OpenSSL).fips_mode { fips_mode }\n      conf = @d.server_create_transport_section_object(@tls_options.merge({ensure_fips: ensure_fips}))\n      if expected\n        assert_raise(expected) do\n          @d.cert_option_create_context(Fluent::TLS::DEFAULT_VERSION,\n                                        false,\n                                        Fluent::TLS::CIPHERS_DEFAULT,\n                                        conf)\n        end\n      else\n        assert_nothing_raised do\n          @d.cert_option_create_context(Fluent::TLS::DEFAULT_VERSION,\n                                        false,\n                                        Fluent::TLS::CIPHERS_DEFAULT,\n                                        conf)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_child_process.rb",
    "content": "# coding: utf-8\nrequire_relative '../helper'\nrequire 'fluent/plugin_helper/child_process'\nrequire 'fluent/plugin/base'\nrequire 'timeout'\nrequire 'tempfile'\n\nclass ChildProcessTest < Test::Unit::TestCase\n  TEST_DEADLOCK_TIMEOUT = 30\n  TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING = 0.1 # This may be shorter than ruby's threading timer, but work well\n  # @nalsh says that ruby's cpu assignments for threads are almost 200ms or so.\n  # Loop interval (expected that it work as specified) should be longer than it.\n  TEST_WAIT_INTERVAL_FOR_LOOP = 0.5\n\n  setup do\n    @d = Dummy.new\n    @d.configure(config_element())\n    @d.start\n  end\n\n  teardown do\n    if @d\n      @d.stop      unless @d.stopped?\n      @d.shutdown  unless @d.shutdown?\n      @d.close     unless @d.closed?\n      @d.terminate unless @d.terminated?\n      @d.log.reset\n    end\n  end\n\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :child_process\n    def configure(conf)\n      super\n      @_child_process_kill_timeout = 1\n    end\n  end\n\n  test 'can be instantiated' do\n    d1 = Dummy.new\n    assert d1.respond_to?(:_child_process_processes)\n  end\n\n  test 'can be configured and started' do\n    d1 = Dummy.new\n    assert_nothing_raised do\n      d1.configure(config_element())\n    end\n    assert d1.plugin_id\n    assert d1.log\n\n    d1.start\n  end\n\n  test 'can execute external command asynchronously' do\n    m = Mutex.new\n    m.lock\n    ary = []\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      @d.child_process_execute(:t0, 'echo', arguments: ['foo', 'bar'], mode: [:read]) do |io|\n        m.lock\n        ran = true\n        io.read # discard\n        ary << 2\n        m.unlock\n      end\n      ary << 1\n      m.unlock\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      m.unlock\n    end\n    assert_equal [1,2], ary\n  end\n\n  test 'can execute external command at just once, which finishes immediately' do\n    m = Mutex.new\n    t1 = Time.now\n    ary = []\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      @d.child_process_execute(:t1, 'echo', arguments: ['foo', 'bar'], mode: [:read]) do |io|\n        m.lock\n        ran = true\n        ary << io.read\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      m.unlock\n    end\n    assert{ Time.now - t1 < 4.0 }\n  end\n\n  test 'can execute external command at just once, which can handle both of read and write' do\n    m = Mutex.new\n    ary = []\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      cmd = \"ruby -e 'while !STDIN.eof? && line = STDIN.readline; puts line.chomp; STDOUT.flush rescue nil; end'\"\n      @d.child_process_execute(:t2, cmd, mode: [:write, :read]) do |writeio, readio|\n        m.lock\n        ran = true\n\n        [[1,2],[3,4],[5,6]].each do |i,j|\n          writeio.write \"my data#{i}\\n\"\n          writeio.write \"my data#{j}\\n\"\n          writeio.flush\n        end\n        writeio.close\n\n        while line = readio.readline\n          ary << line\n        end\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      m.unlock\n    end\n\n    assert_equal [], @d.log.out.logs\n    expected = (1..6).map{|i| \"my data#{i}\\n\" }\n    assert_equal expected, ary\n  end\n\n  test 'can execute external command at just once, which can handle both of read and write. Ignore stderr message/no block' do\n    m = Mutex.new\n    ary = []\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      # lots of stderr message should not be blocked and message should not be printed in test.\n      cmd = \"ruby -e 'while !STDIN.eof? && line = STDIN.readline; STDERR.puts line.chomp * 1000; STDOUT.puts line.chomp; STDOUT.flush rescue nil; end'\"\n      @d.child_process_execute(:t2_and_ignore_stderr, cmd, mode: [:write, :read]) do |writeio, readio|\n        m.lock\n        ran = true\n\n        [[1,2],[3,4],[5,6]].each do |i,j|\n          writeio.write \"my data#{i}\\n\"\n          writeio.write \"my data#{j}\\n\"\n          writeio.flush\n        end\n        writeio.close\n\n        while line = readio.readline\n          ary << line\n        end\n        m.unlock\n      end\n      begin\n        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n        m.lock\n      rescue\n      ensure\n        m.unlock\n      end\n    end\n\n    assert_equal [], @d.log.out.logs\n    expected = (1..6).map{|i| \"my data#{i}\\n\" }\n    assert_equal expected, ary\n  end\n\n  test 'can execute external command at just once, which can handle all of read, write and stderr' do\n    m = Mutex.new\n    ary1 = []\n    ary2 = []\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      cmd = \"ruby -e 'while !STDIN.eof? && line = STDIN.readline; puts line.chomp; STDOUT.flush rescue nil; STDERR.puts line.chomp; STDERR.flush rescue nil; end'\"\n      @d.child_process_execute(:t2a, cmd, mode: [:write, :read, :stderr]) do |writeio, readio, stderrio|\n        m.lock\n        ran = true\n\n        [[1,2],[3,4],[5,6]].each do |i,j|\n          writeio.write \"my data#{i}\\n\"\n          writeio.write \"my data#{j}\\n\"\n          writeio.flush\n        end\n        writeio.close\n\n        while (line1 = readio.readline) && (line2 = stderrio.readline)\n          ary1 << line1\n          ary2 << line2\n        end\n\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      m.unlock\n    end\n\n    assert_equal [], @d.log.out.logs\n    expected = (1..6).map{|i| \"my data#{i}\\n\" }\n    assert_equal expected, ary1\n    assert_equal expected, ary2\n  end\n\n  test 'can execute external command at just once, which can handle both of write and read (with stderr)' do\n    m = Mutex.new\n    ary = []\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      cmd = \"ruby\"\n      args = ['-e', 'while !STDIN.eof? && line = STDIN.readline; puts \"[s]\" + line.chomp; STDOUT.flush rescue nil; STDERR.puts \"[e]\" + line.chomp; STDERR.flush rescue nil; end']\n      @d.child_process_execute(:t2b, cmd, arguments: args, mode: [:write, :read_with_stderr]) do |writeio, readio|\n        m.lock\n        ran = true\n\n        [[1,2],[3,4],[5,6]].each do |i,j|\n          writeio.write \"my data#{i}\\n\"\n          writeio.write \"my data#{j}\\n\"\n          writeio.flush\n        end\n        writeio.close\n\n        while line = readio.readline\n          ary << line\n        end\n\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      m.unlock\n    end\n\n    assert_equal [], @d.log.out.logs\n    expected = (1..6).map{|i| [\"[s]my data#{i}\\n\", \"[e]my data#{i}\\n\"] }.flatten\n    assert_equal expected, ary\n  end\n\n  test 'can execute external command at just once, which runs forever' do\n    ary = []\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      args = [\"-e\", \"while sleep 0.1; puts 1; STDOUT.flush; end\"]\n      @d.child_process_execute(:t3, \"ruby\", arguments: args, mode: [:read]) do |io|\n        begin\n          while @d.child_process_running? && line = io.readline\n            ran ||= true\n            ary << line\n          end\n        rescue\n          # ignore\n        end\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until ran\n      sleep 1\n\n      @d.stop # nothing occurs\n      @d.shutdown\n      assert { ary.size >= 4 }\n\n      @d.close\n\n      @d.terminate\n      assert @d._child_process_processes.empty?\n    end\n  end\n\n  # In windows environment, child_process try KILL at first (because there's no SIGTERM)\n  test 'can execute external command just once, and can terminate it forcedly when shutdown/terminate even if it ignore SIGTERM' do\n    omit \"SIGTERM is unavailable on Windows\" if Fluent.windows?\n\n    m = Mutex.new\n    ary = []\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      @d.child_process_execute(:t4, \"ruby -e 'Signal.trap(:TERM, nil); while sleep 0.1; puts 1; STDOUT.flush rescue nil; end'\", mode: [:read]) do |io|\n        begin\n          while line = io.readline\n            unless ran\n              m.lock\n              ran = true\n            end\n            ary << line\n          end\n        rescue\n          # ignore\n        ensure\n          m.unlock\n        end\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n\n      assert_equal [], @d.log.out.logs\n\n      @d.stop # nothing occurs\n      lines1 = nil\n      waiting(TEST_WAIT_INTERVAL_FOR_LOOP * 3) do\n        lines1 = ary.size\n        lines1 > 1\n      end\n\n      pid = @d._child_process_processes.keys.first\n      # default value 10 is too long for test\n      @d.instance_eval { @_child_process_exit_timeout = 1 }\n      @d.shutdown\n      sleep TEST_WAIT_INTERVAL_FOR_LOOP\n      lines2 = ary.size\n      assert { lines2 > lines1 }\n      @d.close\n\n      assert_nil((Process.waitpid(pid, Process::WNOHANG) rescue nil))\n\n      @d.terminate\n      assert @d._child_process_processes.empty?\n      begin\n        Process.waitpid(pid)\n      rescue Errno::ECHILD\n      end\n      # Process successfully KILLed if test reaches here\n      assert true\n    end\n  end\n\n  test 'can execute external command many times, which finishes immediately' do\n    ary = []\n    arguments = [\"okay\"]\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      start_time = Fluent::Clock.now\n      @d.child_process_execute(:t5, \"echo\", arguments: arguments, interval: 1, mode: [:read]) do |io|\n        ary << io.read.split(\"\\n\").map(&:chomp).join\n      end\n      1.upto(2) do |i|\n        sleep 0.1 while ary.size < i\n        elapsed = Fluent::Clock.now - start_time\n        assert_equal(i, ary.size)\n        assert_true(elapsed > i && elapsed < i + 0.5,\n                    \"actual elapsed: #{elapsed}\")\n      end\n      assert_equal [], @d.log.out.logs\n      @d.stop\n      assert_equal [], @d.log.out.logs\n      @d.shutdown; @d.close; @d.terminate\n    end\n  end\n\n  test 'can execute external command many times, with leading once executed immediately' do\n    ary = []\n    arguments = [\"okay\"]\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      start_time = Fluent::Clock.now\n      @d.child_process_execute(:t6, \"echo\", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|\n        ary << io.read.split(\"\\n\").map(&:chomp).join\n      end\n      0.upto(1) do |i|\n        sleep 0.1 while ary.size < i + 1\n        elapsed = Fluent::Clock.now - start_time\n        assert_equal(i + 1, ary.size)\n        assert_true(elapsed > i && elapsed < i + 0.5,\n                    \"actual elapsed: #{elapsed}\")\n      end\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n      assert_equal [], @d.log.out.logs\n    end\n  end\n\n  test 'does not execute long running external command in parallel in default' do\n    ary = []\n    arguments = [\"-e\", \"10.times{ sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }\"] # 5 sec\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      assert_equal [], @d.log.out.logs\n      @d.log.out.singleton_class.module_eval do\n        define_method(:write){|message|\n          raise \"boo\" if message.include?('test: {\"test\":\"test\"}') || message.include?('test: {\"test\"=>\"test\"}')\n          @logs.push message\n        }\n      end\n\n      @d.child_process_execute(:t7, \"ruby\", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|\n        ary << io.read.split(\"\\n\").map(&:chomp).join\n      end\n      sleep 2\n      assert_equal 1, @d._child_process_processes.size\n      @d.stop\n      warn_msg = '[warn]: previous child process is still running. skipped. title=:t7 command=\"ruby\" arguments=[\"-e\", \"10.times{ sleep 0.5 }\"] interval=1 parallel=false' + \"\\n\"\n      logs = @d.log.out.logs\n      assert{ logs.first.end_with?(warn_msg) }\n      assert{ logs.all?{|line| line.end_with?(warn_msg) } }\n      @d.shutdown; @d.close; @d.terminate\n      assert_equal [], @d.log.out.logs\n    end\n  end\n\n  test 'can execute long running external command in parallel if specified' do\n    ary = []\n    arguments = [\"-e\", \"10.times{  puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }\"] # 5 sec\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      @d.child_process_execute(:t8, \"ruby\", arguments: arguments, interval: 1, immediate: true, parallel: true, mode: [:read]) do |io|\n        ary << io.read.split(\"\\n\").map(&:chomp).join\n      end\n      sleep 2\n      processes = @d._child_process_processes.size\n      assert { processes >= 2 && processes <= 4 }\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n      assert_equal [], @d.log.out.logs\n    end\n  end\n\n  test 'execute external processes only for writing' do\n    m = Mutex.new\n    unreadable = false\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      @d.child_process_execute(:t9, \"ruby\", arguments: ['-e', 'a=\"\"; while b=STDIN.readline; a+=b; end'], mode: [:write]) do |io|\n        m.lock\n        ran = true\n        begin\n          io.read\n        rescue IOError\n          unreadable = true\n        end\n        50.times do\n          io.write \"hahaha\\n\"\n        end\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      m.unlock\n      assert unreadable\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n      assert_equal [], @d.log.out.logs\n    end\n  end\n\n  test 'execute external processes only for reading' do\n    m = Mutex.new\n    unwritable = false\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      @d.child_process_execute(:t10, \"ruby\", arguments: [\"-e\", \"while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush rescue nil; end\"], mode: [:read]) do |io|\n        m.lock\n        ran = true\n        begin\n          io.write \"foobar\"\n        rescue IOError\n          unwritable = true\n        end\n        _data = io.readline\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      m.unlock\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n      assert unwritable\n      assert_equal [], @d.log.out.logs\n    end\n  end\n\n  test 'can control external encodings' do\n    m = Mutex.new\n    encodings = []\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      @d.child_process_execute(:t11, \"ruby -e 'sleep 10'\", external_encoding: 'ascii-8bit') do |r, w|\n        m.lock\n        ran = true\n        encodings << r.external_encoding\n        encodings << w.external_encoding\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      assert_equal Encoding::ASCII_8BIT, encodings[0]\n      assert_equal Encoding::ASCII_8BIT, encodings[1]\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n    end\n  end\n\n  test 'can control internal encodings' do\n    m = Mutex.new\n    encodings = []\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      @d.child_process_execute(:t12, \"ruby -e 'sleep 10'\", external_encoding: 'utf-8', internal_encoding: 'ascii-8bit') do |r, w|\n        m.lock\n        ran = true\n        encodings << r.internal_encoding\n        encodings << w.internal_encoding\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      assert_equal Encoding::ASCII_8BIT, encodings[0]\n      assert_equal Encoding::ASCII_8BIT, encodings[1]\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n    end\n  end\n\n  test 'can convert encodings from ascii-8bit to utf-8' do\n    m = Mutex.new\n    str = nil\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      args = ['-e', 'STDOUT.set_encoding(\"ascii-8bit\"); STDOUT.write \"\\xA4\\xB5\\xA4\\xC8\\xA4\\xB7\"']\n      @d.child_process_execute(:t13, \"ruby\", arguments: args, external_encoding: 'euc-jp', internal_encoding: 'windows-31j', mode: [:read]) do |io|\n        m.lock\n        ran = true\n        str = io.read\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      assert_equal Encoding.find('windows-31j'), str.encoding\n      expected = \"さとし\".encode('windows-31j')\n      assert_equal expected, str\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n    end\n  end\n\n  test 'can scrub characters without exceptions' do\n    if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')\n      pend \"Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)\"\n    end\n    m = Mutex.new\n    str = nil\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      args = ['-e', 'STDOUT.set_encoding(\"ascii-8bit\"); STDOUT.write \"\\xFF\\xFF\\x00\\xF0\\xF0\"']\n      @d.child_process_execute(:t13a, \"ruby\", arguments: args, mode: [:read]) do |io|\n        m.lock\n        ran = true\n        str = io.read\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      assert_equal Encoding.find('utf-8'), str.encoding\n      replacement = \"\\uFFFD\" # U+FFFD (REPLACEMENT CHARACTER)\n      nul = \"\\x00\" # U+0000 (NUL)\n      expected = replacement * 2 + nul + replacement * 2\n      assert_equal expected, str\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n    end\n  end\n\n  test 'can scrub characters without exceptions and replace specified chars' do\n    if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')\n      pend \"Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)\"\n    end\n    m = Mutex.new\n    str = nil\n    replacement = \"?\"\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      args = ['-e', 'STDOUT.set_encoding(\"ascii-8bit\"); STDOUT.write \"\\xFF\\xFF\\x00\\xF0\\xF0\"']\n      @d.child_process_execute(:t13b, \"ruby\", arguments: args, mode: [:read], scrub: true, replace_string: replacement) do |io|\n        m.lock\n        ran = true\n        str = io.read\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      assert_equal Encoding.find('utf-8'), str.encoding\n      nul = \"\\x00\" # U+0000 (NUL)\n      expected = replacement * 2 + nul + replacement * 2\n      assert_equal expected, str\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n    end\n  end\n\n  unless Fluent.windows?\n    test 'can specify subprocess name' do\n      io = IO.popen([[\"cat\", \"caaaaaaaaaaat\"], '-'])\n      process_naming_enabled = (IO.popen([\"ps\", \"opid,cmd\"]){|_io| _io.readlines }.count{|line| line.include?(\"caaaaaaaaaaat\") } > 0)\n      Process.kill(:TERM, io.pid) rescue nil\n      io.close rescue nil\n\n      # Does TravisCI prohibit process renaming?\n      # This test will be passed in such environment\n      pend unless process_naming_enabled\n\n      m = Mutex.new\n      pids = []\n      proc_lines = []\n      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n        ran = false\n        @d.child_process_execute(:t14, \"/bin/sh\", arguments:['-c', 'sleep 10; echo \"hello\"'], subprocess_name: \"sleeeeeeeeeper\", mode: [:read]) do |readio|\n          m.lock\n          ran = true\n          pids << @d.child_process_id\n          proc_lines += IO.popen([\"ps\", \"opid,cmd\"]){|_io| _io.readlines }\n          m.unlock\n          readio.read\n        end\n        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n        m.lock\n        pid = pids.first\n        # 16357 sleeeeeeeeeper -e sleep 10; puts \"hello\"\n        assert{ proc_lines.find{|line| line =~ /^\\s*#{pid}\\s/ }.strip.split(/\\s+/)[1] == \"sleeeeeeeeeper\" }\n        @d.stop; @d.shutdown; @d.close; @d.terminate\n      end\n    end\n  end\n\n  test 'can set ENV variables' do\n    m = Mutex.new\n    str = nil\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      ran = false\n      args = ['-e', 'puts ENV[\"testing_child_process\"]']\n      @d.child_process_execute(:t15a, \"ruby\", arguments: args, mode: [:read], env: {'testing_child_process' => 'Yes! True!'}) do |io|\n        m.lock\n        ran = true\n        str = io.read\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      expected = \"Yes! True!\\n\"\n      assert_equal expected, str\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n    end\n  end\n\n  test 'can unset ENV variables of Fluentd process' do\n    m = Mutex.new\n    str = nil\n    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n      current_env_path = ENV['PATH']\n      ran = false\n      args = ['-e', 'puts ENV[\"testing_child_process1\"].to_s + ENV[\"testing_child_process2\"].to_s']\n      ENV['testing_child_process1'] = \"No! False!\"\n      @d.child_process_execute(:t15b, \"ruby\", arguments: args, mode: [:read], unsetenv: true, env: {'testing_child_process2' => 'Yes! True!', 'PATH' => current_env_path}) do |io|\n        m.lock\n        ran = true\n        str = io.read\n        m.unlock\n      end\n      sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n      m.lock\n      expected = \"Yes! True!\\n\"\n      assert_equal expected, str\n      @d.stop; @d.shutdown; @d.close; @d.terminate\n    end\n  end\n\n  unless Fluent.windows?\n    test 'can change working directory' do\n      # check my real /tmp directory (for mac)\n      cmd = ['ruby', '-e', 'Dir.chdir(\"/tmp\"); puts Dir.pwd']\n      mytmpdir = IO.popen(cmd){|io| io.read.chomp }\n\n      m = Mutex.new\n      str = nil\n      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n        ran = false\n        args = ['-e', 'puts Dir.pwd']\n        @d.child_process_execute(:t16, \"ruby\", arguments: args, mode: [:read], chdir: \"/tmp\") do |io|\n          m.lock\n          ran = true\n          str = io.read.chomp\n          m.unlock\n        end\n        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran\n        m.lock\n        assert_equal mytmpdir, str\n        @d.stop; @d.shutdown; @d.close; @d.terminate\n      end\n    end\n  end\n\n  sub_test_case 'on_exit_callback is specified' do\n    setup do\n      @temp = Tempfile.create(\"child_process_wait_with_on_exit_callback\")\n      @temp_path = @temp.path\n      @temp.close\n    end\n\n    teardown do\n      File.unlink @temp_path if File.exist?(@temp_path)\n    end\n\n    test 'can return exit status for child process successfully exits using on_exit_callback' do\n      assert File.exist?(@temp_path)\n\n      block_exits = false\n      callback_called = false\n      exit_status = nil\n      args = ['-e', 'puts \"yay\"; File.unlink ARGV[0]', @temp_path]\n      cb = ->(status){ exit_status = status; callback_called = true }\n\n      str = nil\n      pid = nil\n      @d.child_process_execute(:st1, \"ruby\", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|\n        assert !callback_called # ensure yet to be called\n        pid = @d.instance_eval { child_process_id }\n        str = readio.read.chomp\n        block_exits = true\n      end\n      waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) } # to get exit status\n      waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits }\n      waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called }\n\n      assert callback_called\n      assert exit_status\n      assert_equal 0, exit_status.exitstatus\n      assert !File.exist?(@temp_path)\n\n      assert_equal \"yay\", str\n    end\n\n    test 'can return exit status with signal code for child process killed by signal using on_exit_callback' do\n      omit \"SIGQUIT is unsupported on Windows\" if Fluent.windows?\n\n      assert File.exist?(@temp_path)\n\n      block_exits = false\n      callback_called = false\n      exit_status = nil\n      args = ['-e', 'sleep ARGV[0].to_i; puts \"yay\"; File.unlink ARGV[1]', '100', @temp_path]\n      cb = ->(status){ exit_status = status; callback_called = true }\n\n      str = nil\n      pid = nil\n      @d.child_process_execute(:st1, \"ruby\", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|\n        pid = @d.instance_eval { child_process_id }\n        sleep 1\n        Process.kill(:QUIT, pid)\n        str = readio.read.chomp rescue nil # empty string before EOF\n        block_exits = true\n      end\n      waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) } # to get exit status\n      waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits }\n      waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called }\n\n      assert callback_called\n      assert exit_status\n\n      assert_equal nil, exit_status.exitstatus\n      assert_equal 3, exit_status.termsig\n      assert File.exist?(@temp_path)\n      assert_equal '', str\n    end\n\n    test 'calls on_exit_callback for each process exits for interval call using on_exit_callback' do\n      read_data_list = []\n      exit_status_list = []\n\n      args = ['yay']\n      cb = ->(status){ exit_status_list << status }\n\n      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n        @d.child_process_execute(:st1, \"echo\", arguments: args, immediate: true, interval: 1, mode: [:read], on_exit_callback: cb) do |readio|\n          read_data_list << readio.read.chomp\n        end\n        sleep 2.5\n      end\n\n      assert { read_data_list.size >= 2 }\n      assert { exit_status_list.size >= 2 }\n    end\n\n    test 'waits lasting child process until wait_timeout if block is not specified' do\n      assert File.exist?(@temp_path)\n\n      callback_called = false\n      exit_status = nil\n      args = ['-e', 'File.unlink ARGV[0]', @temp_path]\n      cb = ->(status){ exit_status = status; callback_called = true }\n\n      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n        @d.child_process_execute(:t17, \"ruby\", arguments: args, on_exit_callback: cb, wait_timeout: 2)\n        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called\n      end\n\n      assert callback_called\n      assert exit_status\n      assert_equal 0, exit_status.exitstatus\n      assert !File.exist?(@temp_path)\n    end\n\n    test 'waits lasting child process until wait_timeout after block rans if block is specified' do\n      assert File.exist?(@temp_path)\n\n      callback_called = false\n      exit_status = nil\n      args = ['-e', 'File.unlink ARGV[0]', @temp_path]\n      cb = ->(status){ exit_status = status; callback_called = true }\n\n      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n        @d.child_process_execute(:t17, \"ruby\", arguments: args, mode: nil, on_exit_callback: cb, wait_timeout: 10) do\n          sleep 1\n        end\n        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called\n      end\n\n      assert callback_called\n      assert exit_status\n      assert_equal 0, exit_status.exitstatus\n      assert !File.exist?(@temp_path)\n    end\n\n    test 'kills lasting child process after wait_timeout if block is not specified' do\n      assert File.exist?(@temp_path)\n\n      callback_called = false\n      exit_status = nil\n      args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '20', @temp_path]\n      cb = ->(status){ exit_status = status; callback_called = true }\n\n      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n        @d.child_process_execute(:t17, \"ruby\", arguments: args, on_exit_callback: cb, wait_timeout: 1)\n        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called\n      end\n\n      assert callback_called\n      assert exit_status\n      unless Fluent.windows? # On Windows, exitstatus is always 0 and termsig is nil\n        assert_nil exit_status.exitstatus\n        assert_equal 9, exit_status.termsig # SIGKILL\n      end\n      assert File.exist?(@temp_path)\n    end\n\n    test 'kills lasting child process after block ran and wait_timeout expires if block is specified' do\n      assert File.exist?(@temp_path)\n\n      callback_called = false\n      exit_status = nil\n      args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '20', @temp_path]\n      cb = ->(status){ exit_status = status; callback_called = true }\n\n      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n        @d.child_process_execute(:t17, \"ruby\", arguments: args, mode: nil, on_exit_callback: cb, wait_timeout: 1) do\n          sleep 1\n        end\n        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called\n      end\n\n      assert callback_called\n      assert exit_status\n      unless Fluent.windows? # On Windows, exitstatus is always 0 and termsig is nil\n        assert_nil exit_status.exitstatus\n        assert_equal 9, exit_status.termsig # SIGKILL\n      end\n      assert File.exist?(@temp_path)\n    end\n\n    test 'execute child process writing data to stdout which is unread' do\n      callback_called = false\n      exit_status = nil\n      prog = \"echo writing to stdout\"\n      callback = ->(status){ exit_status = status; callback_called = true }\n      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do\n        @d.child_process_execute(:out_exec_process, prog, stderr: :connect, immediate: true, parallel: true, mode: [], wait_timeout: 1, on_exit_callback: callback)\n        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called\n      end\n      assert callback_called\n      assert exit_status\n      assert exit_status.success?\n      assert_equal 0, exit_status.exitstatus\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_compat_parameters.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/compat_parameters'\nrequire 'fluent/plugin/input'\nrequire 'fluent/plugin/output'\nrequire 'fluent/time'\n\nrequire 'time'\n\nclass CompatParameterTest < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n    @i = nil\n    @default_newline = if Fluent.windows?\n                         \"\\r\\n\"\n                       else\n                         \"\\n\"\n                       end\n  end\n\n  teardown do\n    if @i\n      @i.stop unless @i.stopped?\n      @i.before_shutdown unless @i.before_shutdown?\n      @i.shutdown unless @i.shutdown?\n      @i.after_shutdown unless @i.after_shutdown?\n      @i.close unless @i.closed?\n      @i.terminate unless @i.terminated?\n    end\n  end\n\n  class DummyI0 < Fluent::Plugin::Input\n    helpers :compat_parameters, :parser, :extract\n    attr_reader :parser\n    def configure(conf)\n      compat_parameters_convert(conf, :extract, :parser)\n      super\n    end\n    def start\n      super\n      @parser = parser_create\n    end\n    def produce_events(input_data)\n      emit_events = [] # tag, time, record\n      @parser.parse(input_data) do |time, record|\n        tag = extract_tag_from_record(record) || 'dummy_tag'\n        emit_events << [tag, time, record]\n      end\n      emit_events\n    end\n  end\n  class DummyO0 < Fluent::Plugin::Output\n    helpers :compat_parameters\n    def configure(conf)\n      compat_parameters_buffer(conf, default_chunk_key: '')\n      super\n    end\n    def write(chunk); end # dummy\n  end\n  class DummyO1 < Fluent::Plugin::Output\n    helpers :compat_parameters\n    def configure(conf)\n      compat_parameters_buffer(conf, default_chunk_key: 'time')\n      super\n    end\n    def write(chunk); end # dummy\n  end\n  class DummyO2 < Fluent::Plugin::Output\n    helpers :compat_parameters\n    def configure(conf)\n      compat_parameters_buffer(conf, default_chunk_key: 'time')\n      super\n    end\n    def write(chunk); end # dummy\n  end\n  class DummyO3 < Fluent::Plugin::Output\n    helpers :compat_parameters\n    def configure(conf)\n      compat_parameters_buffer(conf, default_chunk_key: 'tag')\n      super\n    end\n    def write(chunk); end # dummy\n  end\n  class DummyO4 < Fluent::Plugin::Output\n    helpers :compat_parameters, :inject, :formatter\n    attr_reader :f\n    def configure(conf)\n      compat_parameters_convert(conf, :buffer, :inject, :formatter, default_chunk_key: 'tag')\n      super\n    end\n    def start\n      super\n      @f = formatter_create()\n    end\n    def write(chunk); end # dummy\n  end\n\n  sub_test_case 'output plugins which does not have default chunk key' do\n    test 'plugin helper converts parameters into plugin configuration parameters' do\n      hash = {\n        'num_threads' => 8,\n        'flush_interval' => '10s',\n        'buffer_chunk_limit' => '8m',\n        'buffer_queue_limit' => '1024',\n        'flush_at_shutdown' => 'yes',\n      }\n      conf = config_element('ROOT', '', hash)\n      @i = DummyO0.new\n      @i.configure(conf)\n\n      assert_equal 'memory', @i.buffer_config[:@type]\n      assert_equal [], @i.buffer_config.chunk_keys\n      assert_equal 8, @i.buffer_config.flush_thread_count\n      assert_equal 10, @i.buffer_config.flush_interval\n      assert_equal :default, @i.buffer_config.flush_mode\n      assert @i.buffer_config.flush_at_shutdown\n\n      assert_equal 8*1024*1024, @i.buffer.chunk_limit_size\n      assert_equal 1024, @i.buffer.queue_limit_length\n    end\n  end\n\n  sub_test_case 'output plugins which has default chunk key: time' do\n    test 'plugin helper converts parameters into plugin configuration parameters' do\n      hash = {\n        'buffer_type' => 'file',\n        'buffer_path' => '/tmp/mybuffer',\n        'disable_retry_limit' => 'yes',\n        'max_retry_wait' => '1h',\n        'buffer_queue_full_action' => 'block',\n      }\n      conf = config_element('ROOT', '', hash)\n      @i = DummyO1.new\n      @i.configure(conf)\n\n      assert_equal 'file', @i.buffer_config[:@type]\n      assert_equal 24*60*60, @i.buffer_config.timekey\n      assert @i.buffer_config.retry_forever\n      assert_equal 60*60, @i.buffer_config.retry_max_interval\n      assert_equal :block, @i.buffer_config.overflow_action\n      assert_equal :default, @i.buffer_config.flush_mode\n\n      assert !@i.chunk_key_tag\n      assert_equal [], @i.chunk_keys\n\n      assert_equal '/tmp/mybuffer/buffer.*.log', @i.buffer.path\n    end\n  end\n\n  sub_test_case 'output plugins which does not have default chunk key' do\n    test 'plugin helper converts parameters into plugin configuration parameters' do\n      hash = {\n        'buffer_type' => 'file',\n        'buffer_path' => '/tmp/mybuffer',\n        'time_slice_format' => '%Y%m%d%H',\n        'time_slice_wait' => '10',\n        'retry_limit' => '1024',\n        'buffer_queue_full_action' => 'drop_oldest_chunk',\n      }\n      conf = config_element('ROOT', '', hash)\n      @i = DummyO2.new\n      @i.configure(conf)\n\n      assert_equal 'file', @i.buffer_config[:@type]\n      assert_equal 60*60, @i.buffer_config.timekey\n      assert_equal 10, @i.buffer_config.timekey_wait\n      assert_equal 1024, @i.buffer_config.retry_max_times\n      assert_equal :drop_oldest_chunk, @i.buffer_config.overflow_action\n\n      assert @i.chunk_key_time\n      assert !@i.chunk_key_tag\n      assert_equal [], @i.chunk_keys\n\n      assert_equal '/tmp/mybuffer/buffer.*.log', @i.buffer.path\n    end\n  end\n\n  sub_test_case 'output plugins which has default chunk key: tag' do\n    test 'plugin helper converts parameters into plugin configuration parameters' do\n      hash = {\n        'buffer_type' => 'memory',\n        'num_threads' => '10',\n        'flush_interval' => '10s',\n        'try_flush_interval' => '0.1',\n        'queued_chunk_flush_interval' => '0.5',\n      }\n      conf = config_element('ROOT', '', hash)\n      @i = DummyO3.new\n      @i.configure(conf)\n\n      assert_equal 'memory', @i.buffer_config[:@type]\n      assert_equal 10, @i.buffer_config.flush_thread_count\n      assert_equal 10, @i.buffer_config.flush_interval\n      assert_equal 0.1, @i.buffer_config.flush_thread_interval\n      assert_equal 0.5, @i.buffer_config.flush_thread_burst_interval\n\n      assert !@i.chunk_key_time\n      assert @i.chunk_key_tag\n      assert_equal [], @i.chunk_keys\n    end\n  end\n\n  sub_test_case 'output plugins which has default chunk key: tag, and enables inject and formatter' do\n    test 'plugin helper converts parameters into plugin configuration parameters for all of buffer, inject and formatter' do\n      hash = {\n        'buffer_type' => 'file',\n        'buffer_path' => File.expand_path('../../tmp/compat_parameters/mybuffer.*.log', __FILE__),\n        'num_threads' => '10',\n        'format' => 'ltsv',\n        'delimiter' => ',',\n        'label_delimiter' => '%',\n        'include_time_key' => 'true', # default time_key 'time' and default time format (iso8601: 2016-06-24T15:57:38) at localtime\n        'include_tag_key' => 'yes', # default tag_key 'tag'\n      }\n      conf = config_element('ROOT', '', hash)\n      @i = DummyO4.new\n      @i.configure(conf)\n      @i.start\n      @i.after_start\n\n      assert_equal 'file', @i.buffer_config[:@type]\n      assert_equal 10, @i.buffer_config.flush_thread_count\n      formatter = @i.f\n      assert{ formatter.is_a? Fluent::Plugin::LabeledTSVFormatter }\n      assert_equal ',', @i.f.delimiter\n      assert_equal '%', @i.f.label_delimiter\n\n      assert !@i.chunk_key_time\n      assert @i.chunk_key_tag\n      assert_equal [], @i.chunk_keys\n\n      t = event_time('2016-06-24 16:05:01') # localtime\n      iso8601str = Time.at(t.to_i).iso8601\n      formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {\"value\" => 1}))\n      assert_equal \"value%1,tag%tag.test,time%#{iso8601str}#{@default_newline}\", formatted\n    end\n\n    test 'plugin helper setups time injecting as unix time (integer from epoch)' do\n      hash = {\n        'buffer_type' => 'file',\n        'buffer_path' => File.expand_path('../../tmp/compat_parameters/mybuffer.*.log', __FILE__),\n        'num_threads' => '10',\n        'format' => 'ltsv',\n        'delimiter' => ',',\n        'label_delimiter' => '%',\n        'include_time_key' => 'true', # default time_key 'time' and default time format (iso8601: 2016-06-24T15:57:38) at localtime\n        'include_tag_key' => 'yes', # default tag_key 'tag'\n      }\n      conf = config_element('ROOT', '', hash)\n      @i = DummyO4.new\n      @i.configure(conf)\n      @i.start\n      @i.after_start\n\n      assert_equal 'file', @i.buffer_config[:@type]\n      assert_equal 10, @i.buffer_config.flush_thread_count\n      formatter = @i.f\n      assert{ formatter.is_a? Fluent::Plugin::LabeledTSVFormatter }\n      assert_equal ',', @i.f.delimiter\n      assert_equal '%', @i.f.label_delimiter\n\n      assert !@i.chunk_key_time\n      assert @i.chunk_key_tag\n      assert_equal [], @i.chunk_keys\n\n      t = event_time('2016-06-24 16:05:01') # localtime\n      iso8601str = Time.at(t.to_i).iso8601\n      formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {\"value\" => 1}))\n      assert_equal \"value%1,tag%tag.test,time%#{iso8601str}#{@default_newline}\", formatted\n    end\n  end\n\n  sub_test_case 'input plugins' do\n    test 'plugin helper converts parameters into plugin configuration parameters for extract and parser' do\n      hash = {\n        'format' => 'ltsv',\n        'delimiter' => ',',\n        'label_delimiter' => '%',\n        'tag_key' => 't2',\n        'time_key' => 't',\n        'time_format' => '%Y-%m-%d.%H:%M:%S.%N',\n        'utc' => 'yes',\n        'types' => 'A integer|B string|C bool',\n        'types_delimiter' => '|',\n        'types_label_delimiter' => ' ',\n      }\n      conf = config_element('ROOT', '', hash)\n      @i = DummyI0.new\n      @i.configure(conf)\n      @i.start\n      @i.after_start\n\n      parser = @i.parser\n      assert{ parser.is_a? Fluent::Plugin::LabeledTSVParser }\n      assert_equal ',', parser.delimiter\n      assert_equal '%', parser.label_delimiter\n\n      events = @i.produce_events(\"A%1,B%x,C%true,t2%mytag,t%2016-10-20.03:50:11.987654321\")\n      assert_equal 1, events.size\n\n      tag, time, record = events.first\n      assert_equal 'mytag', tag\n      assert_equal_event_time event_time(\"2016-10-20 03:50:11.987654321 +0000\"), time\n      assert_equal 3, record.keys.size\n      assert_equal ['A','B','C'], record.keys.sort\n      assert_equal 1, record['A']\n      assert_equal 'x', record['B']\n      assert_equal true, record['C']\n    end\n\n    test 'plugin helper converts parameters into plugin configuration parameters for extract and parser, using numeric time' do\n      hash = {\n        'format' => 'ltsv',\n        'delimiter' => ',',\n        'label_delimiter' => '%',\n        'tag_key' => 't2',\n        'time_key' => 't',\n        'time_type' => 'float',\n        'localtime' => 'yes',\n      }\n      conf = config_element('ROOT', '', hash)\n      @i = DummyI0.new\n      @i.configure(conf)\n      @i.start\n      @i.after_start\n\n      parser = @i.parser\n      assert{ parser.is_a? Fluent::Plugin::LabeledTSVParser }\n      assert_equal ',', parser.delimiter\n      assert_equal '%', parser.label_delimiter\n    end\n\n    test 'plugin helper setups time extraction as unix time (integer from epoch)' do\n      # TODO:\n    end\n  end\n\n  sub_test_case 'parser plugins' do\n    test 'syslog parser parameters' do\n      hash = {\n        'format' => 'syslog',\n        'message_format' => 'rfc5424',\n        'with_priority' => 'true',\n        'rfc5424_time_format' => '%Y'\n      }\n      conf = config_element('ROOT', '', hash)\n      @i = DummyI0.new\n      @i.configure(conf)\n      @i.start\n      @i.after_start\n\n      parser = @i.parser\n      assert_kind_of(Fluent::Plugin::SyslogParser, parser)\n      assert_equal :rfc5424, parser.message_format\n      assert_equal true, parser.with_priority\n      assert_equal '%Y', parser.rfc5424_time_format\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_event_emitter.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/event_emitter'\nrequire 'fluent/plugin/base'\nrequire 'flexmock/test_unit'\n\nclass EventEmitterTest < Test::Unit::TestCase\n  setup do\n    Fluent::Test.setup\n  end\n\n  class Dummy0 < Fluent::Plugin::TestBase\n  end\n\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :event_emitter\n  end\n\n  test 'can be instantiated to be able to emit with router' do\n    d0 = Dummy0.new\n    assert d0.respond_to?(:has_router?)\n    assert !d0.has_router?\n    assert !d0.respond_to?(:router)\n\n    d1 = Dummy.new\n    assert d1.respond_to?(:has_router?)\n    assert d1.has_router?\n    assert d1.respond_to?(:router)\n    d1.stop; d1.shutdown; d1.close; d1.terminate\n  end\n\n  test 'can be configured with valid router' do\n    d1 = Dummy.new\n    assert d1.has_router?\n    assert_nil d1.router\n\n    assert_nothing_raised do\n      d1.configure(config_element())\n    end\n\n    assert d1.router\n\n    d1.shutdown\n\n    assert d1.router\n\n    d1.close\n\n    assert_nil d1.router\n\n    d1.terminate\n  end\n\n  test 'should not have event_emitter_router' do\n    d0 = Dummy0.new\n    assert !d0.respond_to?(:event_emitter_router)\n  end\n\n  test 'should have event_emitter_router' do\n    d = Dummy.new\n    assert d.respond_to?(:event_emitter_router)\n  end\n\n  test 'get router' do\n    router_mock = flexmock('mytest')\n    label_mock = flexmock('mylabel')\n    label_mock.should_receive(:event_router).twice.and_return(router_mock)\n    Fluent::Engine.root_agent.labels['@mytest'] = label_mock\n\n    d = Dummy.new\n    d.configure(config_element('ROOT', '', {'@label' => '@mytest'}))\n    router = d.event_emitter_router(\"@mytest\")\n    assert_equal router_mock, router\n  end\n\n  test 'get root router' do\n    d = Dummy.new\n    router = d.event_emitter_router(\"@ROOT\")\n    assert_equal Fluent::Engine.root_agent.event_router, router\n  end\n\n  test '#router should return the root router by default' do\n    stub(Fluent::Engine.root_agent).event_router { \"root_event_router\" }\n    stub(Fluent::Engine.root_agent).source_only_router { \"source_only_router\" }\n\n    d = Dummy.new\n    d.configure(Fluent::Config::Element.new('source', '', {}, []))\n    assert_equal \"root_event_router\", d.router\n  end\n\n  test '#router should return source_only_router during source-only' do\n    stub(Fluent::Engine.root_agent).event_router { \"root_event_router\" }\n    stub(Fluent::Engine.root_agent).source_only_router { \"source_only_router\" }\n\n    d = Dummy.new\n    d.configure(Fluent::Config::Element.new('source', '', {}, []))\n    d.event_emitter_apply_source_only\n\n    router_when_source_only = d.router\n\n    d.event_emitter_cancel_source_only\n\n    router_after_canceled = d.router\n\n    assert_equal(\n      [\"source_only_router\", \"root_event_router\"],\n      [router_when_source_only, router_after_canceled]\n    )\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_event_loop.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/event_loop'\nrequire 'fluent/plugin/base'\n\nclass EventLoopTest < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :event_loop\n    def configure(conf)\n      super\n      @_event_loop_run_timeout = 0.1\n    end\n  end\n\n  test 'can be instantiated to be able to create event loop' do\n    d1 = Dummy.new\n    assert d1.respond_to?(:event_loop_attach)\n    assert d1.respond_to?(:event_loop_running?)\n    assert d1.respond_to?(:_event_loop)\n    assert d1._event_loop\n    assert !d1.event_loop_running?\n  end\n\n  test 'can be configured' do\n    d1 = Dummy.new\n    assert_nothing_raised do\n      d1.configure(config_element())\n    end\n    assert d1.plugin_id\n    assert d1.log\n  end\n\n  test 'can run event loop by start, stop by shutdown/close and clear by terminate' do\n    d1 = Dummy.new\n    d1.configure(config_element())\n    assert !d1.event_loop_running?\n\n    d1.start\n    d1.event_loop_wait_until_start\n\n    assert d1.event_loop_running?\n    assert_equal 1, d1._event_loop.watchers.size\n\n    d1.shutdown\n    d1.close\n\n    assert !d1.event_loop_running?\n\n    d1.terminate\n\n    assert_nil d1._event_loop\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_extract.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/extract'\nrequire 'fluent/time'\n\nclass ExtractHelperTest < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :extract\n  end\n\n  class Dummy2 < Fluent::Plugin::TestBase\n    helpers :extract\n    config_section :extract do\n      config_set_default :tag_key, 'tag2'\n    end\n  end\n\n  def config_extract_section(hash = {})\n    config_element('ROOT', '', {}, [config_element('extract', '', hash)])\n  end\n\n  setup do\n    Fluent::Test.setup\n    @d = Dummy.new\n  end\n\n  teardown do\n    if @d\n      @d.stop unless @d.stopped?\n      @d.shutdown unless @d.shutdown?\n      @d.close unless @d.closed?\n      @d.terminate unless @d.terminated?\n    end\n  end\n\n  test 'can override default parameters, but not overwrite whole definition' do\n    d = Dummy.new\n    d.configure(config_element())\n    assert_nil d.extract_config\n\n    d = Dummy2.new\n    d.configure(config_element('ROOT', '', {}, [config_element('extract')]))\n    assert d.extract_config\n    assert_equal 'tag2', d.extract_config.tag_key\n  end\n\n  test 'returns nil in default' do\n    @d.configure(config_extract_section())\n    @d.start\n    assert_nil @d.instance_eval{ @_extract_tag_key }\n    assert_nil @d.instance_eval{ @_extract_time_key }\n    assert_nil @d.instance_eval{ @_extract_time_parser }\n\n    record = {\"key1\" => \"value1\", \"key2\" => 2, \"tag\" => \"yay\", \"time\" => Time.now.to_i}\n\n    assert_nil @d.extract_tag_from_record(record)\n    assert_nil @d.extract_time_from_record(record)\n  end\n\n  test 'can be configured as specified' do\n    @d.configure(config_extract_section(\n        \"tag_key\" => \"tag\",\n        \"time_key\" => \"time\",\n        \"time_type\" => \"unixtime\",\n    ))\n\n    assert_equal \"tag\", @d.instance_eval{ @_extract_tag_key }\n    assert_equal \"time\", @d.instance_eval{ @_extract_time_key }\n    assert_equal :unixtime, @d.instance_eval{ @extract_config.time_type }\n    assert_not_nil @d.instance_eval{ @_extract_time_parser }\n  end\n\n  sub_test_case 'extract_tag_from_record' do\n    test 'returns tag string from specified tag_key field' do\n      @d.configure(config_extract_section(\"tag_key\" => \"tag\"))\n      @d.start\n      @d.after_start\n\n      tag = @d.extract_tag_from_record({\"tag\" => \"tag.test.code\", \"message\" => \"yay!\"})\n      assert_equal \"tag.test.code\", tag\n    end\n\n    test 'returns tag as string by stringifying values from specified key' do\n      @d.configure(config_extract_section(\"tag_key\" => \"tag\"))\n      @d.start\n      @d.after_start\n\n      tag = @d.extract_tag_from_record({\"tag\" => 100, \"message\" => \"yay!\"})\n      assert_equal \"100\", tag\n    end\n  end\n\n  sub_test_case 'extract_time_from_record' do\n    test 'returns EventTime object from specified time_key field, parsed as float in default' do\n      @d.configure(config_extract_section(\"time_key\" => \"t\"))\n      @d.start\n      @d.after_start\n\n      # 1473135272 => 2016-09-06 04:14:32 UTC\n      t = @d.extract_time_from_record({\"t\" => 1473135272.5, \"message\" => \"yay!\"})\n      assert_equal_event_time(Fluent::EventTime.new(1473135272, 500_000_000), t)\n\n      t = @d.extract_time_from_record({\"t\" => \"1473135272.5\", \"message\" => \"yay!\"})\n      assert_equal_event_time(Fluent::EventTime.new(1473135272, 500_000_000), t)\n    end\n\n    test 'returns EventTime object, parsed as unixtime when configured so' do\n      @d.configure(config_extract_section(\"time_key\" => \"t\", \"time_type\" => \"unixtime\"))\n      @d.start\n      @d.after_start\n\n      t = @d.extract_time_from_record({\"t\" => 1473135272, \"message\" => \"yay!\"})\n      assert_equal_event_time(Fluent::EventTime.new(1473135272, 0), t)\n\n      t = @d.extract_time_from_record({\"t\" => \"1473135272\", \"message\" => \"yay!\"})\n      assert_equal_event_time(Fluent::EventTime.new(1473135272, 0), t)\n\n      t = @d.extract_time_from_record({\"t\" => 1473135272.5, \"message\" => \"yay!\"})\n      assert_equal_event_time(Fluent::EventTime.new(1473135272, 0), t)\n    end\n\n    test 'returns EventTime object, parsed by default time parser of ruby with timezone in data' do\n      t = with_timezone(\"UTC-02\") do\n        @d.configure(config_extract_section(\"time_key\" => \"t\", \"time_type\" => \"string\"))\n        @d.start\n        @d.after_start\n        @d.extract_time_from_record({\"t\" => \"2016-09-06 13:27:01 +0900\", \"message\" => \"yay!\"})\n      end\n      assert_equal_event_time(event_time(\"2016-09-06 13:27:01 +0900\"), t)\n    end\n\n    test 'returns EventTime object, parsed by default time parser of ruby as localtime' do\n      t = with_timezone(\"UTC-02\") do\n        @d.configure(config_extract_section(\"time_key\" => \"t\", \"time_type\" => \"string\"))\n        @d.start\n        @d.after_start\n        @d.extract_time_from_record({\"t\" => \"2016-09-06 13:27:01\", \"message\" => \"yay!\"})\n      end\n      assert_equal_event_time(event_time(\"2016-09-06 13:27:01 +0200\"), t)\n    end\n\n    test 'returns EventTime object, parsed as configured time_format with timezone' do\n      t = with_timezone(\"UTC-02\") do\n        @d.configure(config_extract_section(\"time_key\" => \"t\", \"time_type\" => \"string\", \"time_format\" => \"%H:%M:%S, %m/%d/%Y, %z\"))\n        @d.start\n        @d.after_start\n        @d.extract_time_from_record({\"t\" => \"13:27:01, 09/06/2016, -0700\", \"message\" => \"yay!\"})\n      end\n      assert_equal_event_time(event_time(\"2016-09-06 13:27:01 -0700\"), t)\n    end\n\n    test 'returns EventTime object, parsed as configured time_format in localtime without timezone' do\n      t = with_timezone(\"UTC-02\") do\n        @d.configure(config_extract_section(\"time_key\" => \"t\", \"time_type\" => \"string\", \"time_format\" => \"%H:%M:%S, %m/%d/%Y\"))\n        @d.start\n        @d.after_start\n        @d.extract_time_from_record({\"t\" => \"13:27:01, 09/06/2016\", \"message\" => \"yay!\"})\n      end\n      assert_equal_event_time(event_time(\"2016-09-06 13:27:01 +0200\"), t)\n    end\n\n    test 'returns EventTime object, parsed as configured time_format in utc without timezone, localtime: false' do\n      t = with_timezone(\"UTC-02\") do\n        c = config_extract_section(\"time_key\" => \"t\", \"time_type\" => \"string\", \"time_format\" => \"%H:%M:%S, %m/%d/%Y\", \"localtime\" => \"false\")\n        @d.configure(c)\n        @d.start\n        @d.after_start\n        @d.extract_time_from_record({\"t\" => \"13:27:01, 09/06/2016\", \"message\" => \"yay!\"})\n      end\n      assert_equal_event_time(event_time(\"2016-09-06 13:27:01 UTC\"), t)\n    end\n\n    test 'returns EventTime object, parsed as configured time_format in utc without timezone, utc: true' do\n      t = with_timezone(\"UTC-02\") do\n        c = config_extract_section(\"time_key\" => \"t\", \"time_type\" => \"string\", \"time_format\" => \"%H:%M:%S, %m/%d/%Y\", \"utc\" => \"true\")\n        @d.configure(c)\n        @d.start\n        @d.after_start\n        @d.extract_time_from_record({\"t\" => \"13:27:01, 09/06/2016\", \"message\" => \"yay!\"})\n      end\n      assert_equal_event_time(event_time(\"2016-09-06 13:27:01 UTC\"), t)\n    end\n\n    test 'returns EventTime object, parsed as configured time_format in configured timezone' do\n      t = with_timezone(\"UTC-02\") do\n        c = config_extract_section(\"time_key\" => \"t\", \"time_type\" => \"string\", \"time_format\" => \"%H:%M:%S, %m/%d/%Y\", \"timezone\" => \"+09:00\")\n        @d.configure(c)\n        @d.start\n        @d.after_start\n        @d.extract_time_from_record({\"t\" => \"13:27:01, 09/06/2016\", \"message\" => \"yay!\"})\n      end\n      assert_equal_event_time(event_time(\"2016-09-06 13:27:01 +0900\"), t)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_formatter.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/formatter'\nrequire 'fluent/plugin/base'\n\nclass FormatterHelperTest < Test::Unit::TestCase\n  class ExampleFormatter < Fluent::Plugin::Formatter\n    Fluent::Plugin.register_formatter('example', self)\n    def format(tag, time, record)\n      \"#{tag},#{time.to_i},#{record.keys.sort.join(',')}\" # hey, you miss values! :P\n    end\n  end\n  class Example2Formatter < Fluent::Plugin::Formatter\n    Fluent::Plugin.register_formatter('example2', self)\n    def format(tag, time, record)\n      \"#{tag},#{time.to_i},#{record.values.sort.join(',')}\" # key...\n    end\n  end\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :formatter\n    config_section :format do\n      config_set_default :@type, 'example'\n    end\n  end\n\n  class Dummy2 < Fluent::Plugin::TestBase\n    helpers :formatter\n    config_section :format do\n      config_set_default :@type, 'example2'\n    end\n  end\n\n  setup do\n    @d = nil\n  end\n\n  teardown do\n    if @d\n      @d.stop unless @d.stopped?\n      @d.shutdown unless @d.shutdown?\n      @d.close unless @d.closed?\n      @d.terminate unless @d.terminated?\n    end\n  end\n\n  test 'can be initialized without any formatters at first' do\n    d = Dummy.new\n    assert_equal 0, d._formatters.size\n  end\n\n  test 'can override default configuration parameters, but not overwrite whole definition' do\n    d = Dummy.new\n    assert_equal [], d.formatter_configs\n\n    d = Dummy2.new\n    d.configure(config_element('ROOT', '', {}, [config_element('format', '', {}, [])]))\n    assert_raise NoMethodError do\n      d.format\n    end\n    assert_equal 1, d.formatter_configs.size\n    assert_equal 'example2', d.formatter_configs.first[:@type]\n  end\n\n  test 'creates instance of type specified by conf, or default_type if @type is missing in conf' do\n    d = Dummy2.new\n    d.configure(config_element())\n    i = d.formatter_create(conf: config_element('format', '', {'@type' => 'example'}), default_type: 'example2')\n    assert{ i.is_a?(ExampleFormatter) }\n\n    d = Dummy2.new\n    d.configure(config_element())\n    i = d.formatter_create(conf: nil, default_type: 'example2')\n    assert{ i.is_a?(Example2Formatter) }\n  end\n\n  test 'raises config error if config section is specified, but @type is not specified' do\n    d = Dummy2.new\n    d.configure(config_element())\n    assert_raise Fluent::ConfigError.new(\"@type is required in <format>\") do\n      d.formatter_create(conf: config_element('format', '', {}), default_type: 'example2')\n    end\n  end\n\n  test 'can be configured with default type without format sections' do\n    d = Dummy.new\n    assert_nothing_raised do\n      d.configure(config_element())\n    end\n    assert_equal 1, d._formatters.size\n  end\n\n  test 'can be configured with a format section' do\n    d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('format', '', {'@type' => 'example'})\n      ])\n    assert_nothing_raised do\n      d.configure(conf)\n    end\n    assert_equal 1, d._formatters.size\n    assert{ d._formatters.values.all?{ |formatter| !formatter.started? } }\n  end\n\n  test 'can be configured with 2 or more format sections with different usages with each other' do\n    d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('format', 'default', {'@type' => 'example'}),\n        config_element('format', 'extra', {'@type' => 'example2'}),\n      ])\n    assert_nothing_raised do\n      d.configure(conf)\n    end\n    assert_equal 2, d._formatters.size\n    assert{ d._formatters.values.all?{ |formatter| !formatter.started? } }\n  end\n\n  test 'cannot be configured with 2 format sections with same usage' do\n    d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('format', 'default', {'@type' => 'example'}),\n        config_element('format', 'extra', {'@type' => 'example2'}),\n        config_element('format', 'extra', {'@type' => 'example2'}),\n      ])\n    assert_raises Fluent::ConfigError do\n      d.configure(conf)\n    end\n  end\n\n  test 'creates a format plugin instance which is already configured without usage' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('format', '', {'@type' => 'example'})\n      ])\n    d.configure(conf)\n    d.start\n\n    formatter = d.formatter_create\n    assert{ formatter.is_a? ExampleFormatter }\n    assert formatter.started?\n  end\n\n  test 'creates a formatter plugin instance which is already configured with usage' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('format', 'mydata', {'@type' => 'example'})\n      ])\n    d.configure(conf)\n    d.start\n\n    formatter = d.formatter_create(usage: 'mydata')\n    assert{ formatter.is_a? ExampleFormatter }\n    assert formatter.started?\n  end\n\n  test 'creates a formatter plugin without configurations' do\n    @d = d = Dummy.new\n    d.configure(config_element())\n    d.start\n\n    formatter = d.formatter_create(usage: 'mydata', type: 'example', conf: config_element('format', 'mydata'))\n    assert{ formatter.is_a? ExampleFormatter }\n    assert formatter.started?\n  end\n\n  test 'creates 2 or more formatter plugin instances' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('format', 'mydata', {'@type' => 'example'}),\n        config_element('format', 'secret', {'@type' => 'example2'})\n      ])\n    d.configure(conf)\n    d.start\n\n    p1 = d.formatter_create(usage: 'mydata')\n    p2 = d.formatter_create(usage: 'secret')\n    assert{ p1.is_a? ExampleFormatter }\n    assert p1.started?\n    assert{ p2.is_a? Example2Formatter }\n    assert p2.started?\n  end\n\n  test 'calls lifecycle methods for all plugin instances via owner plugin' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [ config_element('format', '', {'@type' => 'example'}), config_element('format', 'e2', {'@type' => 'example'}) ])\n    d.configure(conf)\n    d.start\n\n    i1 = d.formatter_create(usage: '')\n    i2 = d.formatter_create(usage: 'e2')\n    i3 = d.formatter_create(usage: 'e3', type: 'example2')\n\n    assert i1.started?\n    assert i2.started?\n    assert i3.started?\n\n    assert !i1.stopped?\n    assert !i2.stopped?\n    assert !i3.stopped?\n\n    d.stop\n\n    assert i1.stopped?\n    assert i2.stopped?\n    assert i3.stopped?\n\n    assert !i1.before_shutdown?\n    assert !i2.before_shutdown?\n    assert !i3.before_shutdown?\n\n    d.before_shutdown\n\n    assert i1.before_shutdown?\n    assert i2.before_shutdown?\n    assert i3.before_shutdown?\n\n    assert !i1.shutdown?\n    assert !i2.shutdown?\n    assert !i3.shutdown?\n\n    d.shutdown\n\n    assert i1.shutdown?\n    assert i2.shutdown?\n    assert i3.shutdown?\n\n    assert !i1.after_shutdown?\n    assert !i2.after_shutdown?\n    assert !i3.after_shutdown?\n\n    d.after_shutdown\n\n    assert i1.after_shutdown?\n    assert i2.after_shutdown?\n    assert i3.after_shutdown?\n\n    assert !i1.closed?\n    assert !i2.closed?\n    assert !i3.closed?\n\n    d.close\n\n    assert i1.closed?\n    assert i2.closed?\n    assert i3.closed?\n\n    assert !i1.terminated?\n    assert !i2.terminated?\n    assert !i3.terminated?\n\n    d.terminate\n\n    assert i1.terminated?\n    assert i2.terminated?\n    assert i3.terminated?\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_http_server_helper.rb",
    "content": "require_relative '../helper'\nrequire 'flexmock/test_unit'\nrequire 'fluent/plugin_helper/http_server'\nrequire 'fluent/plugin/output'\nrequire 'fluent/event'\nrequire 'net/http'\nrequire 'uri'\nrequire 'openssl'\nrequire 'async'\n\nclass HttpHelperTest < Test::Unit::TestCase\n  NULL_LOGGER = Logger.new(nil)\n  CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/without_ca')\n  CERT_CA_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/with_ca')\n\n  def setup\n    @port = unused_port(protocol: :tcp)\n  end\n\n  def teardown\n    @port = nil\n  end\n\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :http_server\n  end\n\n  def on_driver(config = nil)\n    config ||= Fluent::Config.parse(config || '', '(name)', '')\n    Fluent::Test.setup\n    driver = Dummy.new\n    driver.configure(config)\n    driver.start\n    driver.after_start\n\n    yield(driver)\n  ensure\n    unless driver.stopped?\n      driver.stop rescue nil\n    end\n\n    unless driver.before_shutdown?\n      driver.before_shutdown rescue nil\n    end\n\n    unless driver.shutdown?\n      driver.shutdown rescue nil\n    end\n\n    unless driver.after_shutdown?\n      driver.after_shutdown rescue nil\n    end\n\n    unless driver.closed?\n      driver.close rescue nil\n    end\n\n    unless driver.terminated?\n      driver.terminated rescue nil\n    end\n  end\n\n  def on_driver_transport(opts = {}, &block)\n    transport_conf = config_element('transport', 'tls', opts)\n    c = config_element('ROOT', '', {}, [transport_conf])\n    on_driver(c, &block)\n  end\n\n  %w[get head].each do |n|\n    define_method(n) do |uri, header = {}|\n      url = URI.parse(uri)\n      headers = { 'Content-Type' => 'application/x-www-form-urlencoded/' }.merge(header)\n      req = Net::HTTP.const_get(n.capitalize).new(url, headers)\n      Net::HTTP.start(url.host, url.port) do |http|\n        http.request(req)\n      end\n    end\n\n    define_method(\"secure_#{n}\") do |uri, header = {}, verify: true, cert_path: nil, selfsigned: true, hostname: false|\n      url = URI.parse(uri)\n      headers = { 'Content-Type' => 'application/x-www-form-urlencoded/' }.merge(header)\n      start_https_request(url.host, url.port, verify: verify, cert_path: cert_path, selfsigned: selfsigned) do |https|\n        https.send(n, url.path, headers.to_a)\n      end\n    end\n  end\n\n  %w[post put patch delete options trace].each do |n|\n    define_method(n) do |uri, body = '', header = {}|\n      url = URI.parse(uri)\n      headers = { 'Content-Type' => 'application/x-www-form-urlencoded/' }.merge(header)\n      req = Net::HTTP.const_get(n.capitalize).new(url, headers)\n      req.body = body\n      Net::HTTP.start(url.host, url.port) do |http|\n        http.request(req)\n      end\n    end\n  end\n\n  # wrapper for net/http\n  Response = Struct.new(:code, :body, :headers)\n\n  # Use async-http as http client since net/http can't be set verify_hostname= now\n  # will be replaced when net/http supports verify_hostname=\n  def start_https_request(addr, port, verify: true, cert_path: nil, selfsigned: true, hostname: nil)\n    context = OpenSSL::SSL::SSLContext.new\n    context.set_params({})\n    if verify\n      cert_store = OpenSSL::X509::Store.new\n      cert_store.set_default_paths\n      if selfsigned && OpenSSL::X509.const_defined?('V_FLAG_CHECK_SS_SIGNATURE')\n        cert_store.flags = OpenSSL::X509::V_FLAG_CHECK_SS_SIGNATURE\n      end\n\n      if cert_path\n        cert_store.add_file(cert_path)\n      end\n\n      context.cert_store = cert_store\n      if !hostname\n        context.verify_hostname = false # In test code, using hostname to be connected is very difficult\n      end\n\n      context.verify_mode = OpenSSL::SSL::VERIFY_PEER\n    else\n      context.verify_mode = OpenSSL::SSL::VERIFY_NONE\n    end\n\n    client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse(\"https://#{addr}:#{port}\", ssl_context: context))\n    Console.logger = Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER)\n\n    resp = nil\n    error = nil\n\n    Sync do\n      Console.logger = Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER)\n      begin\n        response = yield(client)\n      rescue => e               # Async::Reactor rescue all error. handle it by myself\n        error = e\n      end\n\n      if response\n        resp = Response.new(response.status.to_s, response.read, response.headers)\n      end\n    end\n\n    if error\n      raise error\n    else\n      resp\n    end\n  end\n\n  sub_test_case 'Create a HTTP server' do\n    test 'mount given path' do\n      on_driver do |driver|\n        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|\n          s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }\n          s.post('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello post'] }\n          s.head('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello head'] }\n          s.put('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello put'] }\n          s.patch('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello patch'] }\n          s.delete('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello delete'] }\n          s.trace('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello trace'] }\n          s.options('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello options'] }\n        end\n\n        resp = head(\"http://127.0.0.1:#{@port}/example/hello\")\n        assert_equal('200', resp.code)\n        assert_equal(nil, resp.body)\n        assert_equal('text/plain', resp['Content-Type'])\n\n        %w[get put post put delete options trace].each do |n|\n          resp = send(n, \"http://127.0.0.1:#{@port}/example/hello\")\n          assert_equal('200', resp.code)\n          assert_equal(\"hello #{n}\", resp.body)\n          assert_equal('text/plain', resp['Content-Type'])\n        end\n      end\n    end\n\n    test 'mount frozen path' do\n      on_driver do |driver|\n        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|\n          s.get('/example/hello'.freeze) { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }\n        end\n\n        resp = get(\"http://127.0.0.1:#{@port}/example/hello/\")\n        assert_equal('200', resp.code)\n      end\n    end\n\n    test 'when path does not start with `/` or ends with `/`' do\n      on_driver do |driver|\n        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|\n          s.get('example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }\n          s.get('/example/hello2/') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }\n        end\n\n        resp = get(\"http://127.0.0.1:#{@port}/example/hello\")\n        assert_equal('404', resp.code)\n\n        resp = get(\"http://127.0.0.1:#{@port}/example/hello2\")\n        assert_equal('200', resp.code)\n      end\n    end\n\n    test 'when error raised' do\n      on_driver do |driver|\n        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|\n          s.get('/example/hello') { raise 'error!' }\n        end\n\n        resp = get(\"http://127.0.0.1:#{@port}/example/hello\")\n        assert_equal('500', resp.code)\n      end\n    end\n\n    test 'when path is not found' do\n      on_driver do |driver|\n        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|\n          s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }\n        end\n\n        resp = get(\"http://127.0.0.1:#{@port}/example/hello/not_found\")\n        assert_equal('404', resp.code)\n      end\n    end\n\n    test 'params and body' do\n      on_driver do |driver|\n        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|\n          s.get('/example/hello') do |req|\n            assert_equal(req.query_string, nil)\n            assert_equal(req.body, nil)\n            [200, { 'Content-Type' => 'text/plain' }, 'hello get']\n          end\n\n          s.post('/example/hello') do |req|\n            assert_equal(req.query_string, nil)\n            assert_equal(req.body, 'this is body')\n            [200, { 'Content-Type' => 'text/plain' }, 'hello post']\n          end\n\n          s.get('/example/hello/params') do |req|\n            assert_equal(req.query_string, 'test=true')\n            assert_equal(req.body, nil)\n            [200, { 'Content-Type' => 'text/plain' }, 'hello get']\n          end\n\n          s.post('/example/hello/params') do |req|\n            assert_equal(req.query_string, 'test=true')\n            assert_equal(req.body, 'this is body')\n            [200, { 'Content-Type' => 'text/plain' }, 'hello post']\n          end\n        end\n\n        resp = get(\"http://127.0.0.1:#{@port}/example/hello\")\n        assert_equal('200', resp.code)\n\n        resp = post(\"http://127.0.0.1:#{@port}/example/hello\", 'this is body')\n        assert_equal('200', resp.code)\n\n        resp = get(\"http://127.0.0.1:#{@port}/example/hello/params?test=true\")\n        assert_equal('200', resp.code)\n\n        resp = post(\"http://127.0.0.1:#{@port}/example/hello/params?test=true\", 'this is body')\n        assert_equal('200', resp.code)\n      end\n    end\n\n    sub_test_case 'create a HTTPS server' do\n      test '#configure' do\n        driver = Dummy.new\n\n        transport_conf = config_element('transport', 'tls', { 'version' => 'TLSv1_1' })\n        driver.configure(config_element('ROOT', '', {}, [transport_conf]))\n        assert_equal :tls, driver.transport_config.protocol\n        assert_equal :TLSv1_1, driver.transport_config.version\n      end\n\n      sub_test_case '#http_server_create_https_server' do\n        test 'can overwrite settings by using tls_context' do\n          on_driver_transport({ 'insecure' => 'false' }) do |driver|\n            tls = { 'insecure' => 'true' } # overwrite\n            driver.http_server_create_https_server(:http_server_helper_test_tls, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER, tls_opts: tls) do |s|\n              s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }\n            end\n\n            resp = secure_get(\"https://127.0.0.1:#{@port}/example/hello\", verify: false)\n            assert_equal('200', resp.code)\n            assert_equal('hello get', resp.body)\n          end\n        end\n\n        test 'with insecure in transport section' do\n          on_driver_transport({ 'insecure' => 'true' }) do |driver|\n            driver.http_server_create_https_server(:http_server_helper_test_tls, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|\n              s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }\n            end\n            omit \"TLS connection should be aborted due to `Errno::ECONNABORTED`. Need to debug.\" if Fluent.windows?\n\n            resp = secure_get(\"https://127.0.0.1:#{@port}/example/hello\", verify: false)\n            assert_equal('200', resp.code)\n            assert_equal('hello get', resp.body)\n\n            assert_raise OpenSSL::SSL::SSLError do\n              secure_get(\"https://127.0.0.1:#{@port}/example/hello\")\n            end\n          end\n        end\n\n        data(\n          'with passphrase' => ['apple', 'cert-pass.pem', 'cert-key-pass.pem'],\n          'without passphrase' => [nil, 'cert.pem', 'cert-key.pem'])\n        test 'load self-signed cert/key pair, verified from clients using cert files' do |(passphrase, cert, private_key)|\n          omit \"Self signed certificate blocks TLS connection. Need to debug.\" if Fluent.windows?\n\n          cert_path = File.join(CERT_DIR, cert)\n          private_key_path = File.join(CERT_DIR, private_key)\n          opt = { 'insecure' => 'false', 'private_key_path' => private_key_path, 'cert_path' => cert_path }\n          if passphrase\n            opt['private_key_passphrase'] = passphrase\n          end\n\n          on_driver_transport(opt) do |driver|\n            driver.http_server_create_https_server(:http_server_helper_test_tls, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|\n              s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }\n            end\n\n            resp = secure_get(\"https://127.0.0.1:#{@port}/example/hello\", cert_path: cert_path)\n            assert_equal('200', resp.code)\n            assert_equal('hello get', resp.body)\n          end\n        end\n\n        data(\n          'with passphrase' => ['apple', 'cert-pass.pem', 'cert-key-pass.pem', 'ca-cert-pass.pem'],\n          'without passphrase' => [nil, 'cert.pem', 'cert-key.pem', 'ca-cert.pem'])\n        test 'load cert by private CA cert file, verified from clients using CA cert file' do |(passphrase, cert, cert_key, ca_cert)|\n          omit \"Self signed certificate blocks TLS connection. Need to debug.\" if Fluent.windows?\n\n          cert_path = File.join(CERT_CA_DIR, cert)\n          private_key_path = File.join(CERT_CA_DIR, cert_key)\n\n          ca_cert_path = File.join(CERT_CA_DIR, ca_cert)\n\n          opt = { 'insecure' => 'false', 'cert_path' => cert_path, 'private_key_path' => private_key_path }\n          if passphrase\n            opt['private_key_passphrase'] = passphrase\n          end\n\n          on_driver_transport(opt) do |driver|\n            driver.http_server_create_https_server(:http_server_helper_test_tls, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|\n              s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }\n            end\n\n            resp = secure_get(\"https://127.0.0.1:#{@port}/example/hello\", cert_path: ca_cert_path)\n            assert_equal('200', resp.code)\n            assert_equal('hello get', resp.body)\n          end\n        end\n      end\n    end\n    \n    data(\n      'IPv6 loopback ::1' => [:ipv6_loopback_test, '::1', '::1'],\n      'IPv6 any address ::' => [:ipv6_any_test, '::', '::1'],\n      'pre-bracketed IPv6 [::1]' => [:ipv6_bracketed_test, '[::1]', '::1']\n    )\n    test 'http_server can bind and serve on IPv6 address' do |data|\n      return unless ipv6_enabled?\n      \n      test_name, bind_addr, connect_addr = data\n      \n      on_driver do |driver|\n        driver.http_server_create_http_server(test_name, addr: bind_addr, port: @port, logger: NULL_LOGGER) do |s|\n          s.get('/test') { [200, { 'Content-Type' => 'text/plain' }, 'OK'] }\n        end\n        \n        http = Net::HTTP.new(connect_addr, @port)\n        response = http.get('/test')\n        assert_equal('200', response.code)\n        assert_equal('OK', response.body)\n      end\n    end\n\n    test 'must be called #start and #stop' do\n      on_driver do |driver|\n        server = flexmock('Server') do |watcher|\n          watcher.should_receive(:start).once.and_return do |que|\n            que.push(:start)\n          end\n          watcher.should_receive(:stop).once\n        end\n\n        stub(Fluent::PluginHelper::HttpServer::Server).new(addr: anything, port: anything, logger: anything, default_app: anything) { server }\n        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do\n          # nothing\n        end\n        driver.stop\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_inject.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/inject'\nrequire 'fluent/plugin/output'\nrequire 'fluent/event'\nrequire 'time'\n\nclass InjectHelperTest < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :inject\n  end\n\n  class Dummy2 < Fluent::Plugin::TestBase\n    helpers :inject\n    config_section :inject do\n      config_set_default :hostname_key, 'host'\n    end\n  end\n\n  class Dummy3 < Fluent::Plugin::Output\n    helpers :inject\n    def write(chunk)\n      # dummy\n    end\n  end\n\n  def config_inject_section(hash = {})\n    config_element('ROOT', '', {}, [config_element('inject', '', hash)])\n  end\n\n  setup do\n    Fluent::Test.setup\n    @d = Dummy.new\n  end\n\n  teardown do\n    if @d\n      @d.stop unless @d.stopped?\n      @d.before_shutdown unless @d.before_shutdown?\n      @d.shutdown unless @d.shutdown?\n      @d.after_shutdown unless @d.after_shutdown?\n      @d.close unless @d.closed?\n      @d.terminate unless @d.terminated?\n    end\n  end\n\n  test 'can override default parameters, but not overwrite whole definition' do\n    d = Dummy.new\n    d.configure(config_element())\n    assert_nil d.inject_config\n\n    d = Dummy2.new\n    d.configure(config_element('ROOT', '', {}, [config_element('inject')]))\n    assert d.inject_config\n    assert_equal 'host', d.inject_config.hostname_key\n  end\n\n  test 'do nothing in default' do\n    @d.configure(config_inject_section())\n    @d.start\n    assert_nil @d.instance_eval{ @_inject_hostname_key }\n    assert_nil @d.instance_eval{ @_inject_hostname }\n    assert_nil @d.instance_eval{ @_inject_worker_id_key }\n    assert_nil @d.instance_eval{ @_inject_worker_id }\n    assert_nil @d.instance_eval{ @_inject_tag_key }\n    assert_nil @d.instance_eval{ @_inject_time_key }\n    assert_nil @d.instance_eval{ @_inject_time_formatter }\n\n    time = event_time()\n    record = {\"key1\" => \"value1\", \"key2\" => 2}\n    assert_equal record, @d.inject_values_to_record('tag', time, record)\n    assert_equal record.object_id, @d.inject_values_to_record('tag', time, record).object_id\n\n    es0 = Fluent::OneEventStream.new(time, {\"key1\" => \"v\", \"key2\" => 0})\n\n    es1 = Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"a\", \"key2\" => 1}], [time, {\"key1\" => \"b\", \"key2\" => 2}] ])\n\n    es2 = Fluent::MultiEventStream.new\n    es2.add(event_time(), {\"key1\" => \"a\", \"key2\" => 1})\n    es2.add(event_time(), {\"key1\" => \"b\", \"key2\" => 2})\n\n    es3 = Fluent::MessagePackEventStream.new(es2.to_msgpack_stream)\n\n    [es0, es1, es2, es3].each do |es|\n      assert_equal es, @d.inject_values_to_event_stream('tag', es), \"failed for #{es.class}\"\n      assert_equal es.object_id, @d.inject_values_to_event_stream('tag', es).object_id, \"failed for #{es.class}\"\n    end\n  end\n\n  test 'can be configured as specified' do\n    with_worker_config(workers: 1, worker_id: 0) do\n      @d.configure(config_inject_section(\n          \"hostname_key\" => \"hostname\",\n          \"hostname\" => \"myhost.local\",\n          \"worker_id_key\" => \"worker_id\",\n          \"tag_key\" => \"tag\",\n          \"time_key\" => \"time\",\n          \"time_type\" => \"string\",\n          \"time_format\" => \"%Y-%m-%d %H:%M:%S.%N\",\n          \"timezone\" => \"-0700\",\n      ))\n    end\n\n    assert_equal \"hostname\", @d.instance_eval{ @_inject_hostname_key }\n    assert_equal \"myhost.local\", @d.instance_eval{ @_inject_hostname }\n    assert_equal \"worker_id\", @d.instance_eval{ @_inject_worker_id_key }\n    assert_equal 0, @d.instance_eval{ @_inject_worker_id }\n    assert_equal \"tag\", @d.instance_eval{ @_inject_tag_key }\n    assert_equal \"time\", @d.instance_eval{ @_inject_time_key }\n    assert_equal :string, @d.instance_eval{ @inject_config.time_type }\n    assert_not_nil @d.instance_eval{ @_inject_time_formatter }\n  end\n\n  test 'raise an error when injected hostname is used in buffer chunk key too' do\n    @d = Dummy3.new\n    conf = config_element('ROOT', '', {}, [\n      config_element('inject', '', {'hostname_key' => 'h'}),\n      config_element('buffer', 'tag,h'),\n    ])\n    assert_raise Fluent::ConfigError.new(\"the key specified by 'hostname_key' in <inject> cannot be used in buffering chunk key.\") do\n      @d.configure(conf)\n    end\n  end\n\n  sub_test_case 'using inject_values_to_record' do\n    test 'injects hostname automatically detected' do\n      detected_hostname = `hostname`.chomp\n      @d.configure(config_inject_section(\"hostname_key\" => \"host\"))\n      logs = @d.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"[info]: using hostname for specified field host_key=\\\"host\\\" host_name=\\\"#{detected_hostname}\\\"\") } }\n      @d.start\n\n      time = event_time()\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"host\" => detected_hostname}), @d.inject_values_to_record('tag', time, record)\n    end\n\n    test 'injects hostname as specified value' do\n      @d.configure(config_inject_section(\"hostname_key\" => \"host\", \"hostname\" => \"myhost.yay.local\"))\n      @d.start\n\n      time = event_time()\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"host\" => \"myhost.yay.local\"}), @d.inject_values_to_record('tag', time, record)\n    end\n\n    test 'injects worker id' do\n      with_worker_config(workers: 3, worker_id: 2) do\n        @d.configure(config_inject_section(\"worker_id_key\" => \"workerid\"))\n      end\n      @d.start\n\n      time = event_time()\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"workerid\" => 2}), @d.inject_values_to_record('tag', time, record)\n    end\n\n    test 'injects tag into specified key' do\n      @d.configure(config_inject_section(\"tag_key\" => \"mytag\"))\n      @d.start\n\n      time = event_time()\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"mytag\" => \"tag.test\"}), @d.inject_values_to_record('tag.test', time, record)\n    end\n\n    test 'injects time as floating point value into specified key as default' do\n      time_in_unix = Time.parse(\"2016-06-21 08:10:11 +0900\").to_i # 1466464211 in unix time\n      time_subsecond = 320_101_224\n      time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n      float_time = 1466464211.320101 # microsecond precision in float\n\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\"))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"timedata\" => float_time}), @d.inject_values_to_record('tag', time, record)\n    end\n\n    test 'injects time as unix time millis into specified key' do\n      time_in_unix = Time.parse(\"2016-06-21 08:10:11 +0900\").to_i\n      time_subsecond = 320_101_224\n      time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n      unixtime_millis = 1466464211320\n\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"unixtime_millis\"))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"timedata\" => unixtime_millis}), @d.inject_values_to_record('tag', time, record)\n      assert_equal record.merge({\"timedata\" => time_in_unix * 1_000}), @d.inject_values_to_record('tag', time_in_unix, record)\n    end\n\n    test 'injects time as unix time micros into specified key' do\n      time_in_unix = Time.parse(\"2016-06-21 08:10:11 +0900\").to_i\n      time_subsecond = 320_101_224\n      time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n      unixtime_micros = 1466464211320101\n\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"unixtime_micros\"))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"timedata\" => unixtime_micros}), @d.inject_values_to_record('tag', time, record)\n      assert_equal record.merge({\"timedata\" => time_in_unix * 1_000_000}), @d.inject_values_to_record('tag', time_in_unix, record)\n    end\n\n    test 'injects time as unix time nanos into specified key' do\n      time_in_unix = Time.parse(\"2016-06-21 08:10:11 +0900\").to_i\n      time_subsecond = 320_101_224\n      time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n      unixtime_nanos = 1466464211320101224\n\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"unixtime_nanos\"))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"timedata\" => unixtime_nanos}), @d.inject_values_to_record('tag', time, record)\n      assert_equal record.merge({\"timedata\" => time_in_unix * 1_000_000_000}), @d.inject_values_to_record('tag', time_in_unix, record)\n    end\n\n    test 'injects time as unix time into specified key' do\n      time_in_unix = Time.parse(\"2016-06-21 08:10:11 +0900\").to_i\n      time_subsecond = 320_101_224\n      time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n      int_time = 1466464211\n\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"unixtime\"))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"timedata\" => int_time}), @d.inject_values_to_record('tag', time, record)\n    end\n\n    test 'injects time as formatted string in localtime if timezone not specified' do\n      local_timezone = Time.now.strftime('%z')\n      time_in_unix = Time.parse(\"2016-06-21 08:10:11 #{local_timezone}\").to_i\n      time_subsecond = 320_101_224\n      time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"string\", \"time_format\" => \"%Y_%m_%d %H:%M:%S %z\"))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"timedata\" => \"2016_06_21 08:10:11 #{local_timezone}\"}), @d.inject_values_to_record('tag', time, record)\n    end\n\n    test 'injects time as formatted string with nanosecond in localtime if timezone not specified' do\n      local_timezone = Time.now.strftime('%z')\n      time_in_unix = Time.parse(\"2016-06-21 08:10:11 #{local_timezone}\").to_i\n      time_subsecond = 320_101_224\n      time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"string\", \"time_format\" => \"%Y_%m_%d %H:%M:%S.%N %z\"))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"timedata\" => \"2016_06_21 08:10:11.320101224 #{local_timezone}\"}), @d.inject_values_to_record('tag', time, record)\n    end\n\n    test 'injects time as formatted string with millisecond in localtime if timezone not specified' do\n      local_timezone = Time.now.strftime('%z')\n      time_in_unix = Time.parse(\"2016-06-21 08:10:11 #{local_timezone}\").to_i\n      time_subsecond = 320_101_224\n      time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"string\", \"time_format\" => \"%Y_%m_%d %H:%M:%S.%3N %z\"))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"timedata\" => \"2016_06_21 08:10:11.320 #{local_timezone}\"}), @d.inject_values_to_record('tag', time, record)\n    end\n\n    test 'injects time as formatted string in specified timezone' do\n      time_in_unix = Time.parse(\"2016-06-21 08:10:11 +0000\").to_i\n      time_subsecond = 320_101_224\n      time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"string\", \"time_format\" => \"%Y_%m_%d %H:%M:%S %z\", \"timezone\" => \"-0800\"))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      assert_equal record.merge({\"timedata\" => \"2016_06_21 00:10:11 -0800\"}), @d.inject_values_to_record('tag', time, record)\n    end\n\n    test 'injects hostname, tag and time' do\n      time_in_unix = Time.parse(\"2016-06-21 08:10:11 +0900\").to_i\n      time_subsecond = 320_101_224\n      time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n\n      @d.configure(config_inject_section(\n          \"hostname_key\" => \"hostnamedata\",\n          \"hostname\" => \"myname.local\",\n          \"tag_key\" => \"tagdata\",\n          \"time_key\" => \"timedata\",\n          \"time_type\" => \"string\",\n          \"time_format\" => \"%Y_%m_%d %H:%M:%S.%N %z\",\n          \"timezone\" => \"+0000\",\n      ))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      injected = {\"hostnamedata\" => \"myname.local\", \"tagdata\" => \"tag\", \"timedata\" => \"2016_06_20 23:10:11.320101224 +0000\"}\n      assert_equal record.merge(injected), @d.inject_values_to_record('tag', time, record)\n    end\n  end\n\n  sub_test_case 'using inject_values_to_event_stream' do\n    local_timezone = Time.now.strftime('%z')\n    time_in_unix = Time.parse(\"2016-06-21 08:10:11 #{local_timezone}\").to_i\n    time_subsecond = 320_101_224\n    time_in_rational = Rational(time_in_unix * 1_000_000_000 + time_subsecond, 1_000_000_000)\n    time_in_localtime = Time.at(time_in_rational).localtime\n    time_in_utc = Time.at(time_in_rational).utc\n    time = Fluent::EventTime.new(time_in_unix, time_subsecond)\n    time_float = time.to_r.truncate(+6).to_f\n\n    data(\n      \"OneEventStream\" => Fluent::OneEventStream.new(time, {\"key1\" => \"value1\", \"key2\" => 0}),\n      \"ArrayEventStream\" => Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"value1\", \"key2\" => 1}], [time, {\"key1\" => \"value2\", \"key2\" => 2}] ]),\n    )\n    test 'injects hostname automatically detected' do |data|\n      detected_hostname = `hostname`.chomp\n      @d.configure(config_inject_section(\"hostname_key\" => \"host\"))\n      logs = @d.log.out.logs\n      assert{ logs.any?{|l| l.include?(\"[info]: using hostname for specified field host_key=\\\"host\\\" host_name=\\\"#{detected_hostname}\\\"\") } }\n      @d.start\n\n      injected = {\"host\" => detected_hostname}\n      expected_es = Fluent::MultiEventStream.new\n      data.each do |t, r|\n        expected_es.add(t, r.merge(injected))\n      end\n      assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)\n    end\n\n    data(\n      \"OneEventStream\" => Fluent::OneEventStream.new(time, {\"key1\" => \"value1\", \"key2\" => 0}),\n      \"ArrayEventStream\" => Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"value1\", \"key2\" => 1}], [time, {\"key1\" => \"value2\", \"key2\" => 2}] ]),\n    )\n    test 'injects hostname as specified value' do |data|\n      @d.configure(config_inject_section(\"hostname_key\" => \"host\", \"hostname\" => \"myhost.yay.local\"))\n      @d.start\n\n      injected = {\"host\" => \"myhost.yay.local\"}\n      expected_es = Fluent::MultiEventStream.new\n      data.each do |t, r|\n        expected_es.add(t, r.merge(injected))\n      end\n      assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)\n    end\n\n    data(\n      \"OneEventStream\" => Fluent::OneEventStream.new(time, {\"key1\" => \"value1\", \"key2\" => 0}),\n      \"ArrayEventStream\" => Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"value1\", \"key2\" => 1}], [time, {\"key1\" => \"value2\", \"key2\" => 2}] ]),\n    )\n    test 'injects tag into specified key' do |data|\n      @d.configure(config_inject_section(\"tag_key\" => \"mytag\"))\n      @d.start\n\n      injected = {\"mytag\" => \"tag\"}\n      expected_es = Fluent::MultiEventStream.new\n      data.each do |t, r|\n        expected_es.add(t, r.merge(injected))\n      end\n      assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)\n    end\n\n    data(\n      \"OneEventStream\" => Fluent::OneEventStream.new(time, {\"key1\" => \"value1\", \"key2\" => 0}),\n      \"ArrayEventStream\" => Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"value1\", \"key2\" => 1}], [time, {\"key1\" => \"value2\", \"key2\" => 2}] ]),\n    )\n    test 'injects time as floating point value into specified key as default' do |data|\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\"))\n      @d.start\n\n      injected = {\"timedata\" => time_float }\n      expected_es = Fluent::MultiEventStream.new\n      data.each do |t, r|\n        expected_es.add(t, r.merge(injected))\n      end\n      assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)\n    end\n\n    data(\n      \"OneEventStream\" => Fluent::OneEventStream.new(time, {\"key1\" => \"value1\", \"key2\" => 0}),\n      \"ArrayEventStream\" => Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"value1\", \"key2\" => 1}], [time, {\"key1\" => \"value2\", \"key2\" => 2}] ]),\n    )\n    test 'injects time as unix time into specified key' do |data|\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"unixtime\"))\n      @d.start\n\n      injected = {\"timedata\" => time_in_localtime.to_i}\n      expected_es = Fluent::MultiEventStream.new\n      data.each do |t, r|\n        expected_es.add(t, r.merge(injected))\n      end\n      assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)\n    end\n\n    data(\n      \"OneEventStream\" => Fluent::OneEventStream.new(time, {\"key1\" => \"value1\", \"key2\" => 0}),\n      \"ArrayEventStream\" => Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"value1\", \"key2\" => 1}], [time, {\"key1\" => \"value2\", \"key2\" => 2}] ]),\n    )\n    test 'injects time as formatted string in localtime if timezone not specified' do |data|\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"string\", \"time_format\" => \"%Y_%m_%d %H:%M:%S %z\"))\n      @d.start\n\n      injected = {\"timedata\" => time_in_localtime.strftime(\"%Y_%m_%d %H:%M:%S %z\")}\n      expected_es = Fluent::MultiEventStream.new\n      data.each do |t, r|\n        expected_es.add(t, r.merge(injected))\n      end\n      assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)\n    end\n\n    data(\n      \"OneEventStream\" => Fluent::OneEventStream.new(time, {\"key1\" => \"value1\", \"key2\" => 0}),\n      \"ArrayEventStream\" => Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"value1\", \"key2\" => 1}], [time, {\"key1\" => \"value2\", \"key2\" => 2}] ]),\n    )\n    test 'injects time as formatted string with nanosecond in localtime if timezone not specified' do |data|\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"string\", \"time_format\" => \"%Y_%m_%d %H:%M:%S.%N %z\"))\n      @d.start\n\n      injected = {\"timedata\" => time_in_localtime.strftime(\"%Y_%m_%d %H:%M:%S.%N %z\")}\n      expected_es = Fluent::MultiEventStream.new\n      data.each do |t, r|\n        expected_es.add(t, r.merge(injected))\n      end\n      assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)\n    end\n\n    data(\n      \"OneEventStream\" => Fluent::OneEventStream.new(time, {\"key1\" => \"value1\", \"key2\" => 0}),\n      \"ArrayEventStream\" => Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"value1\", \"key2\" => 1}], [time, {\"key1\" => \"value2\", \"key2\" => 2}] ]),\n    )\n    test 'injects time as formatted string with millisecond in localtime if timezone not specified' do |data|\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"string\", \"time_format\" => \"%Y_%m_%d %H:%M:%S.%3N %z\"))\n      @d.start\n\n      injected = {\"timedata\" => time_in_localtime.strftime(\"%Y_%m_%d %H:%M:%S.%3N %z\")}\n      expected_es = Fluent::MultiEventStream.new\n      data.each do |t, r|\n        expected_es.add(t, r.merge(injected))\n      end\n      assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)\n    end\n\n    data(\n      \"OneEventStream\" => Fluent::OneEventStream.new(time, {\"key1\" => \"value1\", \"key2\" => 0}),\n      \"ArrayEventStream\" => Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"value1\", \"key2\" => 1}], [time, {\"key1\" => \"value2\", \"key2\" => 2}] ]),\n    )\n    test 'injects time as formatted string in specified timezone' do |data|\n      @d.configure(config_inject_section(\"time_key\" => \"timedata\", \"time_type\" => \"string\", \"time_format\" => \"%Y_%m_%d %H:%M:%S %z\", \"timezone\" => \"-0800\"))\n      @d.start\n\n      injected = {\"timedata\" => Time.at(time_in_unix).localtime(\"-08:00\").strftime(\"%Y_%m_%d %H:%M:%S -0800\")}\n      expected_es = Fluent::MultiEventStream.new\n      data.each do |t, r|\n        expected_es.add(t, r.merge(injected))\n      end\n      assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)\n    end\n\n    data(\n      \"OneEventStream\" => Fluent::OneEventStream.new(time, {\"key1\" => \"value1\", \"key2\" => 0}),\n      \"ArrayEventStream\" => Fluent::ArrayEventStream.new([ [time, {\"key1\" => \"value1\", \"key2\" => 1}], [time, {\"key1\" => \"value2\", \"key2\" => 2}] ]),\n    )\n    test 'injects hostname, tag and time' do |data|\n      @d.configure(config_inject_section(\n          \"hostname_key\" => \"hostnamedata\",\n          \"hostname\" => \"myname.local\",\n          \"tag_key\" => \"tagdata\",\n          \"time_key\" => \"timedata\",\n          \"time_type\" => \"string\",\n          \"time_format\" => \"%Y_%m_%d %H:%M:%S.%N %z\",\n          \"timezone\" => \"+0000\",\n      ))\n      @d.start\n\n      injected = {\"hostnamedata\" => \"myname.local\", \"tagdata\" => \"tag\", \"timedata\" => time_in_utc.strftime(\"%Y_%m_%d %H:%M:%S.%N %z\")}\n      expected_es = Fluent::MultiEventStream.new\n      data.each do |t, r|\n        expected_es.add(t, r.merge(injected))\n      end\n      assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)\n    end\n  end\n\n  sub_test_case 'time formatting with modified timezone' do\n    setup do\n      @time = event_time(\"2014-09-27 00:00:00 +00:00\").to_i\n    end\n\n    def format(conf)\n      @d.configure(config_inject_section(\n          \"hostname_key\" => \"hostnamedata\",\n          \"hostname\" => \"myname.local\",\n          \"tag_key\" => \"tagdata\",\n          \"time_key\" => \"timedata\",\n          \"time_type\" => \"string\",\n          \"time_format\" => \"%Y_%m_%d %H:%M:%S.%N %z\",\n          \"timezone\" => \"+0000\",\n      ))\n      @d.start\n\n      record = {\"key1\" => \"value1\", \"key2\" => 2}\n      injected = {\"hostnamedata\" => \"myname.local\", \"tagdata\" => \"tag\", \"timedata\" => \"2016_06_20 23:10:11.320101224 +0000\"}\n      assert_equal record.merge(injected), @d.inject_values_to_record('tag', time, record)\n\n\n      d = create_driver({'include_time_key' => true}.merge(conf))\n      formatted = d.instance.format(\"tag\", @time, {})\n      # Drop the leading \"time:\" and the trailing \"\\n\".\n      formatted[5..-2]\n    end\n\n    def test_nothing_specified_about_time_formatting\n      with_timezone(\"UTC-01\") do\n        # 'localtime' is true by default.\n        @d.configure(config_inject_section(\"time_key\" => \"t\", \"time_type\" => \"string\"))\n        @d.start\n        record = @d.inject_values_to_record('tag', @time, {\"message\" => \"yay\"})\n\n        assert_equal(\"2014-09-27T01:00:00+01:00\", record['t'])\n      end\n    end\n\n    def test_utc\n      with_timezone(\"UTC-01\") do\n        # 'utc' takes precedence over 'localtime'.\n        @d.configure(config_inject_section(\"time_key\" => \"t\", \"time_type\" => \"string\", \"utc\" => \"true\"))\n        @d.start\n        record = @d.inject_values_to_record('tag', @time, {\"message\" => \"yay\"})\n\n        assert_equal(\"2014-09-27T00:00:00Z\", record['t'])\n      end\n    end\n\n    def test_timezone\n      with_timezone(\"UTC-01\") do\n        # 'timezone' takes precedence over 'localtime'.\n        @d.configure(config_inject_section(\"time_key\" => \"t\", \"time_type\" => \"string\", \"timezone\" => \"+02\"))\n        @d.start\n        record = @d.inject_values_to_record('tag', @time, {\"message\" => \"yay\"})\n\n        assert_equal(\"2014-09-27T02:00:00+02:00\", record['t'])\n      end\n    end\n\n    def test_utc_timezone\n      with_timezone(\"UTC-01\") do\n        # 'timezone' takes precedence over 'utc'.\n        @d.configure(config_inject_section(\"time_key\" => \"t\", \"time_type\" => \"string\", \"timezone\" => \"Asia/Tokyo\", \"utc\" => \"true\"))\n        @d.start\n        record = @d.inject_values_to_record('tag', @time, {\"message\" => \"yay\"})\n\n        assert_equal(\"2014-09-27T09:00:00+09:00\", record['t'])\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_metrics.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/metrics'\nrequire 'fluent/plugin/base'\n\nclass MetricsTest < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :metrics\n    def configure(conf)\n      super\n    end\n  end\n\n  setup do\n    @d = nil\n  end\n\n  teardown do\n    if @d\n      @d.stop unless @d.stopped?\n      @d.shutdown unless @d.shutdown?\n      @d.close unless @d.closed?\n      @d.terminate unless @d.terminated?\n    end\n  end\n\n  test 'can be initialized without any metrics at first' do\n    d = Dummy.new\n    assert_equal 0, d._metrics.size\n  end\n\n  test 'can be configured' do\n    d1 = Dummy.new\n    assert_nothing_raised do\n      d1.configure(config_element())\n    end\n    assert d1.plugin_id\n    assert d1.log\n  end\n\n  test 'creates metrics instances' do\n    d = Dummy.new\n    i = d.metrics_create(namespace: \"fluentd_test\", subsystem: \"unit-test\", name: \"metrics1\", help_text: \"metrics testing\")\n    d.configure(config_element())\n    assert do\n      d.instance_variable_get(:@plugin_type_or_id).include?(\"dummy.object\")\n    end\n    assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }\n    assert_true i.has_methods_for_counter\n    assert_false i.has_methods_for_gauge\n\n    d = Dummy.new\n    i = d.metrics_create(namespace: \"fluentd_test\", subsystem: \"unit-test\", name: \"metrics2\", help_text: \"metrics testing\", prefer_gauge: true)\n    d.configure(config_element())\n    assert do\n      d.instance_variable_get(:@plugin_type_or_id).include?(\"dummy.object\")\n    end\n    assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }\n    assert_false i.has_methods_for_counter\n    assert_true i.has_methods_for_gauge\n  end\n\n  test 'calls lifecycle methods for all plugin instances via owner plugin' do\n    @d = d = Dummy.new\n    i1 = d.metrics_create(namespace: \"fluentd_test\", subsystem: \"unit-test\", name: \"metrics1\", help_text: \"metrics testing\")\n    i2 = d.metrics_create(namespace: \"fluentd_test\", subsystem: \"unit-test\", name: \"metrics2\", help_text: \"metrics testing\", prefer_gauge: true)\n    i3 = d.metrics_create(namespace: \"fluentd_test\", subsystem: \"unit-test\", name: \"metrics3\", help_text: \"metrics testing\")\n    d.configure(config_element())\n    assert do\n      d.instance_variable_get(:@plugin_type_or_id).include?(\"dummy.object\")\n    end\n    d.start\n\n    assert i1.started?\n    assert i2.started?\n    assert i3.started?\n\n    assert !i1.stopped?\n    assert !i2.stopped?\n    assert !i3.stopped?\n\n    d.stop\n\n    assert i1.stopped?\n    assert i2.stopped?\n    assert i3.stopped?\n\n    assert !i1.before_shutdown?\n    assert !i2.before_shutdown?\n    assert !i3.before_shutdown?\n\n    d.before_shutdown\n\n    assert i1.before_shutdown?\n    assert i2.before_shutdown?\n    assert i3.before_shutdown?\n\n    assert !i1.shutdown?\n    assert !i2.shutdown?\n    assert !i3.shutdown?\n\n    d.shutdown\n\n    assert i1.shutdown?\n    assert i2.shutdown?\n    assert i3.shutdown?\n\n    assert !i1.after_shutdown?\n    assert !i2.after_shutdown?\n    assert !i3.after_shutdown?\n\n    d.after_shutdown\n\n    assert i1.after_shutdown?\n    assert i2.after_shutdown?\n    assert i3.after_shutdown?\n\n    assert !i1.closed?\n    assert !i2.closed?\n    assert !i3.closed?\n\n    d.close\n\n    assert i1.closed?\n    assert i2.closed?\n    assert i3.closed?\n\n    assert !i1.terminated?\n    assert !i2.terminated?\n    assert !i3.terminated?\n\n    d.terminate\n\n    assert i1.terminated?\n    assert i2.terminated?\n    assert i3.terminated?\n  end\n\n  test 'can create getter method by metrics name' do\n    @d = d = Dummy.new\n\n    assert_raise(NoMethodError) do\n      d.foobarbaz\n    end\n\n    metrics = d.metrics_create(namespace: \"fluentd_test\", subsystem: \"unit-test\", name: \"foobarbaz\", help_text: \"metrics testing\")\n    metrics.inc\n\n    assert_equal(1, d.foobarbaz)\n    assert_equal(1, metrics.get)\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_parser.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/parser'\nrequire 'fluent/plugin/base'\nrequire 'fluent/time'\n\nclass ParserHelperTest < Test::Unit::TestCase\n  class ExampleParser < Fluent::Plugin::Parser\n    Fluent::Plugin.register_parser('example', self)\n    def parse(text)\n      ary = text.split(/\\s*,\\s*/)\n      r = {}\n      ary.each_with_index do |v, i|\n        r[i.to_s] = v\n      end\n      yield Fluent::EventTime.now, r\n    end\n  end\n  class Example2Parser < Fluent::Plugin::Parser\n    Fluent::Plugin.register_parser('example2', self)\n    def parse(text)\n      ary = text.split(/\\s*,\\s*/)\n      r = {}\n      ary.each_with_index do |v, i|\n        r[i.to_s] = v\n      end\n      yield Fluent::EventTime.now, r\n    end\n  end\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :parser\n    config_section :parse do\n      config_set_default :@type, 'example'\n    end\n  end\n\n  class Dummy2 < Fluent::Plugin::TestBase\n    helpers :parser\n    config_section :parse do\n      config_set_default :@type, 'example2'\n    end\n  end\n\n  setup do\n    @d = nil\n  end\n\n  teardown do\n    if @d\n      @d.stop unless @d.stopped?\n      @d.shutdown unless @d.shutdown?\n      @d.close unless @d.closed?\n      @d.terminate unless @d.terminated?\n    end\n  end\n\n  test 'can be initialized without any parsers at first' do\n    d = Dummy.new\n    assert_equal 0, d._parsers.size\n  end\n\n  test 'can override default configuration parameters, but not overwrite whole definition' do\n    d = Dummy.new\n    assert_equal [], d.parser_configs\n\n    d = Dummy2.new\n    d.configure(config_element('ROOT', '', {}, [config_element('parse', '', {}, [])]))\n    assert_raise NoMethodError do\n      d.parse\n    end\n    assert_equal 1, d.parser_configs.size\n    assert_equal 'example2', d.parser_configs.first[:@type]\n  end\n\n  test 'creates instance of type specified by conf, or default_type if @type is missing in conf' do\n    d = Dummy2.new\n    d.configure(config_element())\n    i = d.parser_create(conf: config_element('parse', '', {'@type' => 'example'}), default_type: 'example2')\n    assert{ i.is_a?(ExampleParser) }\n\n    d = Dummy2.new\n    d.configure(config_element())\n    i = d.parser_create(conf: nil, default_type: 'example2')\n    assert{ i.is_a?(Example2Parser) }\n  end\n\n  test 'raises config error if config section is specified, but @type is not specified' do\n    d = Dummy2.new\n    d.configure(config_element())\n    assert_raise Fluent::ConfigError.new(\"@type is required in <parse>\") do\n      d.parser_create(conf: config_element('parse', '', {}), default_type: 'example2')\n    end\n  end\n\n  test 'can be configured with default type without parse sections' do\n    d = Dummy.new\n    d.configure(config_element())\n    assert_equal 1, d._parsers.size\n  end\n\n  test 'can be configured with a parse section' do\n    d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('parse', '', {'@type' => 'example'})\n      ])\n    assert_nothing_raised do\n      d.configure(conf)\n    end\n    assert_equal 1, d._parsers.size\n    assert{ d._parsers.values.all?{ |parser| !parser.started? } }\n  end\n\n  test 'can be configured with 2 or more parse sections with different usages with each other' do\n    d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('parse', 'default', {'@type' => 'example'}),\n        config_element('parse', 'extra', {'@type' => 'example2'}),\n      ])\n    assert_nothing_raised do\n      d.configure(conf)\n    end\n    assert_equal 2, d._parsers.size\n    assert{ d._parsers.values.all?{ |parser| !parser.started? } }\n  end\n\n  test 'cannot be configured with 2 parse sections with same usage' do\n    d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('parse', 'default', {'@type' => 'example'}),\n        config_element('parse', 'extra', {'@type' => 'example2'}),\n        config_element('parse', 'extra', {'@type' => 'example2'}),\n      ])\n    assert_raises Fluent::ConfigError do\n      d.configure(conf)\n    end\n  end\n\n  test 'creates a parse plugin instance which is already configured without usage' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('parse', '', {'@type' => 'example'})\n      ])\n    d.configure(conf)\n    d.start\n\n    parser = d.parser_create\n    assert{ parser.is_a? ExampleParser }\n    assert parser.started?\n  end\n\n  test 'creates a parser plugin instance which is already configured with usage' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('parse', 'mydata', {'@type' => 'example'})\n      ])\n    d.configure(conf)\n    d.start\n\n    parser = d.parser_create(usage: 'mydata')\n    assert{ parser.is_a? ExampleParser }\n    assert parser.started?\n  end\n\n  test 'creates a parser plugin without configurations' do\n    @d = d = Dummy.new\n    d.configure(config_element())\n    d.start\n\n    parser = d.parser_create(usage: 'mydata', type: 'example', conf: config_element('parse', 'mydata'))\n    assert{ parser.is_a? ExampleParser }\n    assert parser.started?\n  end\n\n  test 'creates 2 or more parser plugin instances' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('parse', 'mydata', {'@type' => 'example'}),\n        config_element('parse', 'secret', {'@type' => 'example2'})\n      ])\n    d.configure(conf)\n    d.start\n\n    p1 = d.parser_create(usage: 'mydata')\n    p2 = d.parser_create(usage: 'secret')\n    assert{ p1.is_a? ExampleParser }\n    assert p1.started?\n    assert{ p2.is_a? Example2Parser }\n    assert p2.started?\n  end\n\n  test 'calls lifecycle methods for all plugin instances via owner plugin' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [ config_element('parse', '', {'@type' => 'example'}), config_element('parse', 'e2', {'@type' => 'example'}) ])\n    d.configure(conf)\n    d.start\n\n    i1 = d.parser_create(usage: '')\n    i2 = d.parser_create(usage: 'e2')\n    i3 = d.parser_create(usage: 'e3', type: 'example2')\n\n    assert i1.started?\n    assert i2.started?\n    assert i3.started?\n\n    assert !i1.stopped?\n    assert !i2.stopped?\n    assert !i3.stopped?\n\n    d.stop\n\n    assert i1.stopped?\n    assert i2.stopped?\n    assert i3.stopped?\n\n    assert !i1.before_shutdown?\n    assert !i2.before_shutdown?\n    assert !i3.before_shutdown?\n\n    d.before_shutdown\n\n    assert i1.before_shutdown?\n    assert i2.before_shutdown?\n    assert i3.before_shutdown?\n\n    assert !i1.shutdown?\n    assert !i2.shutdown?\n    assert !i3.shutdown?\n\n    d.shutdown\n\n    assert i1.shutdown?\n    assert i2.shutdown?\n    assert i3.shutdown?\n\n    assert !i1.after_shutdown?\n    assert !i2.after_shutdown?\n    assert !i3.after_shutdown?\n\n    d.after_shutdown\n\n    assert i1.after_shutdown?\n    assert i2.after_shutdown?\n    assert i3.after_shutdown?\n\n    assert !i1.closed?\n    assert !i2.closed?\n    assert !i3.closed?\n\n    d.close\n\n    assert i1.closed?\n    assert i2.closed?\n    assert i3.closed?\n\n    assert !i1.terminated?\n    assert !i2.terminated?\n    assert !i3.terminated?\n\n    d.terminate\n\n    assert i1.terminated?\n    assert i2.terminated?\n    assert i3.terminated?\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_record_accessor.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/record_accessor'\nrequire 'fluent/plugin/base'\n\nrequire 'time'\n\nclass RecordAccessorHelperTest < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :record_accessor\n  end\n\n  sub_test_case 'parse nested key expression' do\n    data('normal' => 'key1',\n         'space' => 'ke y2',\n         'dot key' => 'this.is.key3')\n    test 'parse single key' do |param|\n      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)\n      assert_equal param, result\n    end\n\n    test \"nested bracket keys with dot\" do\n      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(\"$['key1']['this.is.key3']\")\n      assert_equal ['key1', 'this.is.key3'], result\n    end\n\n    data('dot' => '$.key1.key2[0]',\n         'bracket' => \"$['key1']['key2'][0]\",\n         'bracket w/ double quotes' => '$[\"key1\"][\"key2\"][0]')\n    test \"nested keys ['key1', 'key2', 0]\" do |param|\n      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)\n      assert_equal ['key1', 'key2', 0], result\n    end\n\n    data('bracket' => \"$['key1'][0]['ke y2']\",\n         'bracket w/ double quotes' => '$[\"key1\"][0][\"ke y2\"]')\n    test \"nested keys ['key1', 0, 'ke y2']\" do |param|\n      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)\n      assert_equal ['key1', 0, 'ke y2'], result\n    end\n\n    data('dot' => '$.[0].key1.[1].key2',\n         'bracket' => \"$[0]['key1'][1]['key2']\",\n         'bracket w/ double quotes' => '$[0][\"key1\"][1][\"key2\"]')\n    test \"nested keys [0, 'key1', 1, 'key2']\" do |param|\n      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)\n      assert_equal [0, 'key1', 1, 'key2'], result\n    end\n\n    data(\"missing ']'\" => \"$['key1'\",\n         \"missing array index with dot\" => \"$.hello[]\",\n         \"missing array index with bracket\" => \"$[]\",\n         \"more chars\" => \"$.key1[0]foo\",\n         \"whitespace char included key in dot notation\" => \"$.key[0].ke y\",\n         \"empty keys with dot\" => \"$.\",\n         \"empty keys with bracket\" => \"$[\",\n         \"mismatched quotes1\" => \"$['key1']['key2\\\"]\",\n         \"mismatched quotes2\" => '$[\"key1\"][\"key2\\']')\n    test 'invalid syntax' do |param|\n      assert_raise Fluent::ConfigError do\n        Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)\n      end\n    end\n  end\n\n  sub_test_case 'attr_reader :keys' do\n    setup do\n      @d = Dummy.new\n    end\n\n    data('normal' => 'key1',\n         'space' => 'ke y2',\n         'dot key' => 'this.is.key3')\n    test 'access single key' do |param|\n      accessor = @d.record_accessor_create(param)\n      assert_equal param, accessor.keys\n    end\n\n    test \"nested bracket keys with dot\" do\n      accessor = @d.record_accessor_create(\"$['key1']['this.is.key3']\")\n      assert_equal ['key1','this.is.key3'], accessor.keys\n    end\n\n    data('dot' => '$.key1.key2[0]',\n         'bracket' => \"$['key1']['key2'][0]\",\n         'bracket w/ double quotes' => '$[\"key1\"][\"key2\"][0]')\n    test \"nested keys ['key1', 'key2', 0]\" do |param|\n      accessor = @d.record_accessor_create(param)\n      assert_equal ['key1', 'key2', 0], accessor.keys\n    end\n\n    data('bracket' => \"$['key1'][0]['ke y2']\",\n         'bracket w/ double quotes' => '$[\"key1\"][0][\"ke y2\"]')\n    test \"nested keys ['key1', 0, 'ke y2']\" do |param|\n      accessor = @d.record_accessor_create(param)\n      assert_equal ['key1', 0, 'ke y2'], accessor.keys\n    end\n  end\n\n  sub_test_case Fluent::PluginHelper::RecordAccessor::Accessor do\n    setup do\n      @d = Dummy.new\n    end\n\n    data('normal' => 'key1',\n         'space' => 'ke y2',\n         'dot key' => 'this.is.key3')\n    test 'access single key' do |param|\n      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}\n      accessor = @d.record_accessor_create(param)\n      assert_equal r[param], accessor.call(r)\n    end\n\n    test \"access single dot key using bracket style\" do\n      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}\n      accessor = @d.record_accessor_create('$[\"this.is.key3\"]')\n      assert_equal 'v3', accessor.call(r)\n    end\n\n    test \"nested bracket keys with dot\" do\n      r = {'key1' => {'this.is.key3' => 'value'}}\n      accessor = @d.record_accessor_create(\"$['key1']['this.is.key3']\")\n      assert_equal 'value', accessor.call(r)\n    end\n\n    data('dot' => '$.key1.key2[0]',\n         'bracket' => \"$['key1']['key2'][0]\",\n         'bracket w/ double quotes' => '$[\"key1\"][\"key2\"][0]')\n    test \"nested keys ['key1', 'key2', 0]\" do |param|\n      r = {'key1' => {'key2' => [1, 2, 3]}}\n      accessor = @d.record_accessor_create(param)\n      assert_equal 1, accessor.call(r)\n    end\n\n    data('bracket' => \"$['key1'][0]['ke y2']\",\n         'bracket w/ double quotes' => '$[\"key1\"][0][\"ke y2\"]')\n    test \"nested keys ['key1', 0, 'ke y2']\" do |param|\n      r = {'key1' => [{'ke y2' => \"value\"}]}\n      accessor = @d.record_accessor_create(param)\n      assert_equal 'value', accessor.call(r)\n    end\n\n    data(\"missing ']'\" => \"$['key1'\",\n         \"missing array index with dot\" => \"$.hello[]\",\n         \"missing array index with bracket\" => \"$['hello'][]\",\n         \"whitespace char included key in dot notation\" => \"$.key[0].ke y\",\n         \"more chars\" => \"$.key1[0]foo\",\n         \"empty keys with dot\" => \"$.\",\n         \"empty keys with bracket\" => \"$[\",\n         \"mismatched quotes1\" => \"$['key1']['key2\\\"]\",\n         \"mismatched quotes2\" => '$[\"key1\"][\"key2\\']')\n    test 'invalid syntax' do |param|\n      assert_raise Fluent::ConfigError do\n        @d.record_accessor_create(param)\n      end\n    end\n  end\n\n  sub_test_case 'Fluent::PluginHelper::RecordAccessor::Accessor#delete' do\n    setup do\n      @d = Dummy.new\n    end\n\n    data('normal' => 'key1',\n         'space' => 'ke y2',\n         'dot key' => 'this.is.key3')\n    test 'delete top key' do |param|\n      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}\n      accessor = @d.record_accessor_create(param)\n      accessor.delete(r)\n      assert_not_include(r, param)\n    end\n\n    test \"delete top key using bracket style\" do\n      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}\n      accessor = @d.record_accessor_create('$[\"this.is.key3\"]')\n      accessor.delete(r)\n      assert_not_include(r, 'this.is.key3')\n    end\n\n    data('bracket' => \"$['key1'][0]['ke y2']\",\n         'bracket w/ double quotes' => '$[\"key1\"][0][\"ke y2\"]')\n    test \"delete nested keys ['key1', 0, 'ke y2']\" do |param|\n      r = {'key1' => [{'ke y2' => \"value\"}]}\n      accessor = @d.record_accessor_create(param)\n      accessor.delete(r)\n      assert_not_include(r['key1'][0], 'ke y2')\n    end\n\n    test \"don't raise an error when unexpected record is coming\" do\n      r = {'key1' => [{'key3' => \"value\"}]}\n      accessor = @d.record_accessor_create(\"$['key1']['key2']['key3']\")\n      assert_nothing_raised do\n        assert_nil accessor.delete(r)\n      end\n    end\n  end\n\n  sub_test_case 'Fluent::PluginHelper::RecordAccessor::Accessor#set' do\n    setup do\n      @d = Dummy.new\n    end\n\n    data('normal' => 'key1',\n         'space' => 'ke y2',\n         'dot key' => 'this.is.key3')\n    test 'set top key' do |param|\n      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}\n      accessor = @d.record_accessor_create(param)\n      accessor.set(r, \"test\")\n      assert_equal \"test\", r[param]\n    end\n\n    test \"set top key using bracket style\" do\n      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}\n      accessor = @d.record_accessor_create('$[\"this.is.key3\"]')\n      accessor.set(r, \"test\")\n      assert_equal \"test\", r[\"this.is.key3\"]\n    end\n\n    data('bracket' => \"$['key1'][0]['ke y2']\",\n         'bracket w/ double quotes' => '$[\"key1\"][0][\"ke y2\"]')\n    test \"set nested keys ['key1', 0, 'ke y2']\" do |param|\n      r = {'key1' => [{'ke y2' => \"value\"}]}\n      accessor = @d.record_accessor_create(param)\n      accessor.set(r, \"nested_message\")\n      assert_equal \"nested_message\", r['key1'][0]['ke y2']\n    end\n\n    test \"don't raise an error when unexpected record is coming\" do\n      r = {'key1' => [{'key3' => \"value\"}]}\n      accessor = @d.record_accessor_create(\"$['key1']['key2']['key3']\")\n      assert_nothing_raised do\n        accessor.set(r, \"unknown field\")\n      end\n      assert_equal({'key1' => [{'key3' => \"value\"}]}, r)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_retry_state.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/retry_state'\nrequire 'fluent/plugin/base'\n\nrequire 'time'\n\nclass RetryStateHelperTest < Test::Unit::TestCase\n  def override_current_time(state, time)\n    mojule = Module.new do\n      define_method(:current_time){ time }\n    end\n    state.singleton_class.module_eval do\n      prepend mojule\n    end\n  end\n\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :retry_state\n  end\n\n  class RetryRecord\n    attr_reader :retry_count, :elapsed_sec, :is_secondary\n    def initialize(retry_count, elapsed_sec, is_secondary)\n      @retry_count = retry_count # This is Nth retryment\n      @elapsed_sec = elapsed_sec\n      @is_secondary = is_secondary\n    end\n\n    def ==(obj)\n      @retry_count == obj.retry_count &&\n      @elapsed_sec == obj.elapsed_sec &&\n      @is_secondary == obj.is_secondary\n    end\n  end\n\n  setup do\n    @d = Dummy.new\n  end\n\n  test 'randomize can generate value within specified +/- range' do\n    s = @d.retry_state_create(:t1, :exponential_backoff, 0.1, 30) # default enabled w/ 0.125\n    500.times do\n      r = s.randomize(1000)\n      assert{ r >= 875 && r < 1125 }\n    end\n\n    s = @d.retry_state_create(:t1, :exponential_backoff, 0.1, 30, randomize_width: 0.25)\n    500.times do\n      r = s.randomize(1000)\n      assert{ r >= 750 && r < 1250 }\n    end\n  end\n\n  test 'plugin can create retry_state machine' do\n    s = @d.retry_state_create(:t1, :exponential_backoff, 0.1, 30)\n    # attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_times\n\n    assert_equal :t1, s.title\n    start_time = s.start\n\n    assert_equal 0, s.steps\n    assert_equal (start_time + 0.1).to_i, s.next_time.to_i\n    assert_equal (start_time + 0.1).nsec, s.next_time.nsec\n    assert_equal (start_time + 30), s.timeout_at\n\n    assert_equal :primary, s.current\n    assert{ s.is_a? Fluent::PluginHelper::RetryState::ExponentialBackOffRetry }\n  end\n\n  test 'periodic retries' do\n    s = @d.retry_state_create(:t2, :periodic, 3, 29, randomize: false)\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n    assert_equal (dummy_current_time + 29), s.timeout_at\n    assert_equal (dummy_current_time + 3), s.next_time\n\n    i = 1\n    while i < 9\n      override_current_time(s, s.next_time)\n      s.step\n      assert_equal i, s.steps\n      assert_equal (s.current_time + 3), s.next_time\n      assert !s.limit?\n      i += 1\n    end\n\n    assert_equal 9, i\n    override_current_time(s, s.next_time)\n    s.step\n    assert_equal s.timeout_at, s.next_time\n    s.step\n    assert s.limit?\n  end\n\n  test 'periodic retries with max_steps' do\n    s = @d.retry_state_create(:t2, :periodic, 3, 29, randomize: false, max_steps: 5)\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n    assert_equal (dummy_current_time + 29), s.timeout_at\n    assert_equal (dummy_current_time + 3), s.next_time\n\n    i = 1\n    while i < 5\n      override_current_time(s, s.next_time)\n      s.step\n      assert_equal i, s.steps\n      assert_equal (s.current_time + 3), s.next_time\n      assert !s.limit?\n      i += 1\n    end\n\n    assert_equal 5, i\n    override_current_time(s, s.next_time)\n    s.step\n    assert s.limit?\n  end\n\n  test 'periodic retries with secondary' do\n    s = @d.retry_state_create(:t3, :periodic, 3, 100, randomize: false, secondary: true) # threshold 0.8\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n    assert_equal (dummy_current_time + 100), s.timeout_at\n    assert_equal (dummy_current_time + 100 * 0.8), s.secondary_transition_at\n\n    assert_equal (dummy_current_time + 3), s.next_time\n    assert !s.secondary?\n\n    i = 1\n    while i < 26\n      override_current_time(s, s.next_time)\n      assert !s.secondary?\n\n      s.step\n      assert_equal i, s.steps\n      assert_equal (s.current_time + 3), s.next_time\n      assert !s.limit?\n      i += 1\n    end\n\n    assert_equal 26, i\n    override_current_time(s, s.next_time) # 78\n    assert !s.secondary?\n\n    s.step\n    assert_equal 26, s.steps\n    assert_equal s.secondary_transition_at, s.next_time\n    assert !s.limit?\n\n    i += 1\n    assert_equal 27, i\n    override_current_time(s, s.next_time) # 80\n    assert s.secondary?\n\n    s.step\n    assert_equal (s.current_time + 3), s.next_time\n    assert_equal s.steps, s.secondary_transition_steps\n    assert !s.limit?\n\n    i += 1\n\n    while i < 33\n      override_current_time(s, s.next_time)\n      assert s.secondary?\n\n      s.step\n      assert_equal (s.current_time + 3), s.next_time\n      assert !s.limit?\n      i += 1\n    end\n\n    assert_equal 33, i\n    override_current_time(s, s.next_time) # 98\n    assert s.secondary?\n\n    s.step\n    assert_equal s.timeout_at, s.next_time # 100\n\n    s.step\n    assert s.limit?\n  end\n\n  test 'periodic retries with secondary and specified threshold' do\n    s = @d.retry_state_create(:t3, :periodic, 3, 100, randomize: false, secondary: true, secondary_threshold: 0.75)\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n    assert_equal (dummy_current_time + 100), s.timeout_at\n    assert_equal (dummy_current_time + 100 * 0.75), s.secondary_transition_at\n  end\n\n  test 'periodic retries with secondary and max_steps' do\n    s = @d.retry_state_create(:t3, :periodic, 3, 100, max_steps: 5, randomize: false, secondary: true)\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n    assert_equal (dummy_current_time + 100), s.timeout_at\n    assert_equal (dummy_current_time + 3 * 5 * 0.8), s.secondary_transition_at\n  end\n\n  test 'exponential backoff forever without randomization' do\n    s = @d.retry_state_create(:t11, :exponential_backoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2)\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n\n    assert_equal 0, s.steps\n    assert_equal (dummy_current_time + 0.1), s.next_time\n\n    i = 1\n    while i < 300\n      s.step\n      assert_equal i, s.steps\n      assert_equal (dummy_current_time + 0.1 * (2 ** i)), s.next_time\n      assert !s.limit?\n      i += 1\n    end\n  end\n\n  test 'exponential backoff with max_interval' do\n    s = @d.retry_state_create(:t12, :exponential_backoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2, max_interval: 100)\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n\n    assert_equal 0, s.steps\n    assert_equal (dummy_current_time + 0.1), s.next_time\n\n    # 0.1 * 2 ** 9 == 51.2\n    # 0.1 * 2 ** 10 == 102.4\n    i = 1\n    while i < 10\n      s.step\n      assert_equal i, s.steps\n      assert_equal (dummy_current_time + 0.1 * (2 ** i)), s.next_time, \"start:#{dummy_current_time}, i:#{i}\"\n      i += 1\n    end\n\n    s.step\n    assert_equal 10, s.steps\n    assert_equal (dummy_current_time + 100), s.next_time\n\n    s.step\n    assert_equal 11, s.steps\n    assert_equal (dummy_current_time + 100), s.next_time\n  end\n\n  test 'exponential backoff with shorter timeout' do\n    s = @d.retry_state_create(:t13, :exponential_backoff, 1, 12, randomize: false, backoff_base: 2, max_interval: 10)\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n\n    assert_equal (dummy_current_time + 12), s.timeout_at\n\n    assert_equal 0, s.steps\n    assert_equal (dummy_current_time + 1), s.next_time\n\n    # 1 + 2 + 4 (=7)\n\n    override_current_time(s, s.next_time)\n    s.step\n    assert_equal 1, s.steps\n    assert_equal (s.current_time + 2), s.next_time\n\n    override_current_time(s, s.next_time)\n    s.step\n    assert_equal 2, s.steps\n    assert_equal (s.current_time + 4), s.next_time\n\n    assert !s.limit?\n\n    # + 8 (=15) > 12\n\n    override_current_time(s, s.next_time)\n    s.step\n    assert_equal 3, s.steps\n    assert_equal s.timeout_at, s.next_time\n\n    s.step\n    assert s.limit?\n  end\n\n  test 'exponential backoff with max_steps' do\n    s = @d.retry_state_create(:t14, :exponential_backoff, 1, 120, randomize: false, backoff_base: 2, max_interval: 10, max_steps: 6)\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n\n    assert_equal (dummy_current_time + 120), s.timeout_at\n\n    assert_equal 0, s.steps\n    assert_equal (dummy_current_time + 1), s.next_time\n\n    override_current_time(s, s.next_time)\n    s.step\n    assert_equal 1, s.steps\n    assert_equal (s.current_time + 2), s.next_time\n\n    override_current_time(s, s.next_time)\n    s.step\n    assert_equal 2, s.steps\n    assert_equal (s.current_time + 4), s.next_time\n\n    override_current_time(s, s.next_time)\n    s.step\n    assert_equal 3, s.steps\n    assert_equal (s.current_time + 8), s.next_time\n\n    assert !s.limit?\n\n    override_current_time(s, s.next_time)\n    s.step\n    assert_equal 4, s.steps\n    assert_equal (s.current_time + 10), s.next_time\n\n    assert !s.limit?\n\n    override_current_time(s, s.next_time)\n    s.step\n    assert_equal 5, s.steps\n    assert_equal (s.current_time + 10), s.next_time\n\n    assert !s.limit?\n\n    override_current_time(s, s.next_time)\n    s.step\n    assert_equal 6, s.steps\n    assert s.limit?\n  end\n\n  test 'exponential backoff retries with secondary' do\n    s = @d.retry_state_create(:t15, :exponential_backoff, 1, 100, randomize: false, backoff_base: 2, secondary: true) # threshold 0.8\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n    assert_equal (dummy_current_time + 100), s.timeout_at\n    assert_equal (dummy_current_time + 100 * 0.8), s.secondary_transition_at\n\n    assert_equal (dummy_current_time + 1), s.next_time\n    assert !s.secondary?\n\n    # primary: 3, 7, 15, 31, 63, 80 (timeout * threshold)\n    # secondary: 81, 83, 87, 95, 100\n    i = 1\n    while i < 6\n      override_current_time(s, s.next_time)\n      assert !s.secondary?\n\n      s.step\n      assert_equal i, s.steps\n      assert_equal (s.current_time + 1 * (2 ** i)), s.next_time\n      assert !s.limit?\n      i += 1\n    end\n\n    assert_equal 6, i\n    override_current_time(s, s.next_time) # 63\n    assert !s.secondary?\n\n    s.step\n    assert_equal 6, s.steps\n    assert_equal s.secondary_transition_at, s.next_time\n    assert !s.limit?\n\n    i += 1\n    assert_equal 7, i\n    override_current_time(s, s.next_time) # 80\n    assert s.secondary?\n\n    s.step\n    assert_equal 7, s.steps\n    assert_equal s.steps, s.secondary_transition_steps\n    assert_equal (s.secondary_transition_at + 1.0), s.next_time # 81\n    assert !s.limit?\n    assert_equal :secondary, s.current\n\n    # 83, 87, 95, 100\n    j = 1\n    while j < 4\n      override_current_time(s, s.next_time)\n      assert s.secondary?\n      assert_equal :secondary, s.current\n\n      s.step\n      assert_equal (7 + j), s.steps\n      assert_equal (s.current_time + (1 * (2 ** j))), s.next_time\n      assert !s.limit?, \"j:#{j}\"\n      j += 1\n    end\n\n    assert_equal 4, j\n    override_current_time(s, s.next_time) # 95\n    assert s.secondary?\n\n    s.step\n    assert_equal s.timeout_at, s.next_time # 100\n\n    s.step\n    assert s.limit?\n  end\n\n  test 'exponential backoff retries with secondary and specified threshold' do\n    s = @d.retry_state_create(:t16, :exponential_backoff, 1, 100, randomize: false, secondary: true, backoff_base: 2, secondary_threshold: 0.75)\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    assert_equal dummy_current_time, s.current_time\n    assert_equal (dummy_current_time + 100), s.timeout_at\n    assert_equal (dummy_current_time + 100 * 0.75), s.secondary_transition_at\n  end\n\n  test 'exponential backoff retries with secondary and max_steps' do\n    s = @d.retry_state_create(:t15, :exponential_backoff, 1, 100, randomize: false, max_steps: 5, backoff_base: 2, secondary: true) # threshold 0.8\n    dummy_current_time = s.start\n    override_current_time(s, dummy_current_time)\n\n    timeout = 0\n    5.times { |i| timeout += 1.0 * (2 ** i) }\n\n    assert_equal dummy_current_time, s.current_time\n    assert_equal (dummy_current_time + 100), s.timeout_at\n    assert_equal (dummy_current_time + timeout * 0.8), s.secondary_transition_at\n  end\n\n  sub_test_case 'exponential backoff' do\n    test 'too big steps(check inf handling)' do\n      s = @d.retry_state_create(:t11, :exponential_backoff, 1, 300, randomize: false, forever: true, backoff_base: 2)\n      dummy_current_time = s.start\n      override_current_time(s, dummy_current_time)\n\n      i = 1\n      while i < 1027\n        if i >= 1025\n          # With this setting, 1025+ number causes inf in `calc_interval`, so 1024 value is used for next_time\n          assert_nothing_raised(FloatDomainError) { s.step }\n          assert_equal (dummy_current_time + (2 ** (1024 - 1))), s.next_time\n        else\n          s.step\n        end\n        i += 1\n      end\n    end\n  end\n\n  sub_test_case \"ExponentialBackOff_ScenarioTests\" do\n    data(\"Simple timeout\", {\n      timeout: 100, max_steps: nil, max_interval: nil, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 100, false),\n      ],\n    })\n    data(\"Simple timeout with secondary\", {\n      timeout: 100, max_steps: nil, max_interval: nil, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 80, true),\n        RetryRecord.new(8, 81, true),\n        RetryRecord.new(9, 83, true),\n        RetryRecord.new(10, 87, true),\n        RetryRecord.new(11, 95, true),\n        RetryRecord.new(12, 100, true),\n      ],\n    })\n    data(\"Simple timeout with custom wait and backoff_base\", {\n      timeout: 1000, max_steps: nil, max_interval: nil, use_sec: false, sec_thres: 0.8, wait: 2, backoff_base: 3,\n      expected: [\n        RetryRecord.new(1, 2, false),\n        RetryRecord.new(2, 8, false),\n        RetryRecord.new(3, 26, false),\n        RetryRecord.new(4, 80, false),\n        RetryRecord.new(5, 242, false),\n        RetryRecord.new(6, 728, false),\n        RetryRecord.new(7, 1000, false),\n      ],\n    })\n    data(\"Simple timeout with custom wait and backoff_base and secondary\", {\n      timeout: 1000, max_steps: nil, max_interval: nil, use_sec: true, sec_thres: 0.8, wait: 2, backoff_base: 3,\n      expected: [\n        RetryRecord.new(1, 2, false),\n        RetryRecord.new(2, 8, false),\n        RetryRecord.new(3, 26, false),\n        RetryRecord.new(4, 80, false),\n        RetryRecord.new(5, 242, false),\n        RetryRecord.new(6, 728, false),\n        RetryRecord.new(7, 800, true),\n        RetryRecord.new(8, 802, true),\n        RetryRecord.new(9, 808, true),\n        RetryRecord.new(10, 826, true),\n        RetryRecord.new(11, 880, true),\n        RetryRecord.new(12, 1000, true),\n      ],\n    })\n    data(\"Default timeout\", {\n      timeout: 72*3600, max_steps: nil, max_interval: nil, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 1023, false),\n        RetryRecord.new(11, 2047, false),\n        RetryRecord.new(12, 4095, false),\n        RetryRecord.new(13, 8191, false),\n        RetryRecord.new(14, 16383, false),\n        RetryRecord.new(15, 32767, false),\n        RetryRecord.new(16, 65535, false),\n        RetryRecord.new(17, 131071, false),\n        RetryRecord.new(18, 259200, false),\n      ],\n    })\n    data(\"Default timeout with secondary\", {\n      timeout: 72*3600, max_steps: nil, max_interval: nil, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 1023, false),\n        RetryRecord.new(11, 2047, false),\n        RetryRecord.new(12, 4095, false),\n        RetryRecord.new(13, 8191, false),\n        RetryRecord.new(14, 16383, false),\n        RetryRecord.new(15, 32767, false),\n        RetryRecord.new(16, 65535, false),\n        RetryRecord.new(17, 131071, false),\n        RetryRecord.new(18, 207360, true),\n        RetryRecord.new(19, 207361, true),\n        RetryRecord.new(20, 207363, true),\n        RetryRecord.new(21, 207367, true),\n        RetryRecord.new(22, 207375, true),\n        RetryRecord.new(23, 207391, true),\n        RetryRecord.new(24, 207423, true),\n        RetryRecord.new(25, 207487, true),\n        RetryRecord.new(26, 207615, true),\n        RetryRecord.new(27, 207871, true),\n        RetryRecord.new(28, 208383, true),\n        RetryRecord.new(29, 209407, true),\n        RetryRecord.new(30, 211455, true),\n        RetryRecord.new(31, 215551, true),\n        RetryRecord.new(32, 223743, true),\n        RetryRecord.new(33, 240127, true),\n        RetryRecord.new(34, 259200, true),\n      ],\n    })\n    data(\"Default timeout with secondary and custom threshold\", {\n      timeout: 72*3600, max_steps: nil, max_interval: nil, use_sec: true, sec_thres: 0.5, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 1023, false),\n        RetryRecord.new(11, 2047, false),\n        RetryRecord.new(12, 4095, false),\n        RetryRecord.new(13, 8191, false),\n        RetryRecord.new(14, 16383, false),\n        RetryRecord.new(15, 32767, false),\n        RetryRecord.new(16, 65535, false),\n        RetryRecord.new(17, 129600, true),\n        RetryRecord.new(18, 129601, true),\n        RetryRecord.new(19, 129603, true),\n        RetryRecord.new(20, 129607, true),\n        RetryRecord.new(21, 129615, true),\n        RetryRecord.new(22, 129631, true),\n        RetryRecord.new(23, 129663, true),\n        RetryRecord.new(24, 129727, true),\n        RetryRecord.new(25, 129855, true),\n        RetryRecord.new(26, 130111, true),\n        RetryRecord.new(27, 130623, true),\n        RetryRecord.new(28, 131647, true),\n        RetryRecord.new(29, 133695, true),\n        RetryRecord.new(30, 137791, true),\n        RetryRecord.new(31, 145983, true),\n        RetryRecord.new(32, 162367, true),\n        RetryRecord.new(33, 195135, true),\n        RetryRecord.new(34, 259200, true),\n      ],\n    })\n    data(\"Simple max_steps\", {\n      timeout: 72*3600, max_steps: 10, max_interval: nil, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 1023, false),\n      ],\n    })\n    data(\"Simple max_steps with secondary\", {\n      timeout: 72*3600, max_steps: 10, max_interval: nil, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 818, true),\n      ],\n    })\n    data(\"Simple interval\", {\n      timeout: 72*3600, max_steps: nil, max_interval: 3600, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 1023, false),\n        RetryRecord.new(11, 2047, false),\n        RetryRecord.new(12, 4095, false),\n        RetryRecord.new(13, 7695, false),\n        RetryRecord.new(14, 11295, false),\n        RetryRecord.new(15, 14895, false),\n        RetryRecord.new(16, 18495, false),\n        RetryRecord.new(17, 22095, false),\n        RetryRecord.new(18, 25695, false),\n        RetryRecord.new(19, 29295, false),\n        RetryRecord.new(20, 32895, false),\n        RetryRecord.new(21, 36495, false),\n        RetryRecord.new(22, 40095, false),\n        RetryRecord.new(23, 43695, false),\n        RetryRecord.new(24, 47295, false),\n        RetryRecord.new(25, 50895, false),\n        RetryRecord.new(26, 54495, false),\n        RetryRecord.new(27, 58095, false),\n        RetryRecord.new(28, 61695, false),\n        RetryRecord.new(29, 65295, false),\n        RetryRecord.new(30, 68895, false),\n        RetryRecord.new(31, 72495, false),\n        RetryRecord.new(32, 76095, false),\n        RetryRecord.new(33, 79695, false),\n        RetryRecord.new(34, 83295, false),\n        RetryRecord.new(35, 86895, false),\n        RetryRecord.new(36, 90495, false),\n        RetryRecord.new(37, 94095, false),\n        RetryRecord.new(38, 97695, false),\n        RetryRecord.new(39, 101295, false),\n        RetryRecord.new(40, 104895, false),\n        RetryRecord.new(41, 108495, false),\n        RetryRecord.new(42, 112095, false),\n        RetryRecord.new(43, 115695, false),\n        RetryRecord.new(44, 119295, false),\n        RetryRecord.new(45, 122895, false),\n        RetryRecord.new(46, 126495, false),\n        RetryRecord.new(47, 130095, false),\n        RetryRecord.new(48, 133695, false),\n        RetryRecord.new(49, 137295, false),\n        RetryRecord.new(50, 140895, false),\n        RetryRecord.new(51, 144495, false),\n        RetryRecord.new(52, 148095, false),\n        RetryRecord.new(53, 151695, false),\n        RetryRecord.new(54, 155295, false),\n        RetryRecord.new(55, 158895, false),\n        RetryRecord.new(56, 162495, false),\n        RetryRecord.new(57, 166095, false),\n        RetryRecord.new(58, 169695, false),\n        RetryRecord.new(59, 173295, false),\n        RetryRecord.new(60, 176895, false),\n        RetryRecord.new(61, 180495, false),\n        RetryRecord.new(62, 184095, false),\n        RetryRecord.new(63, 187695, false),\n        RetryRecord.new(64, 191295, false),\n        RetryRecord.new(65, 194895, false),\n        RetryRecord.new(66, 198495, false),\n        RetryRecord.new(67, 202095, false),\n        RetryRecord.new(68, 205695, false),\n        RetryRecord.new(69, 209295, false),\n        RetryRecord.new(70, 212895, false),\n        RetryRecord.new(71, 216495, false),\n        RetryRecord.new(72, 220095, false),\n        RetryRecord.new(73, 223695, false),\n        RetryRecord.new(74, 227295, false),\n        RetryRecord.new(75, 230895, false),\n        RetryRecord.new(76, 234495, false),\n        RetryRecord.new(77, 238095, false),\n        RetryRecord.new(78, 241695, false),\n        RetryRecord.new(79, 245295, false),\n        RetryRecord.new(80, 248895, false),\n        RetryRecord.new(81, 252495, false),\n        RetryRecord.new(82, 256095, false),\n        RetryRecord.new(83, 259200, false),\n      ],\n    })\n    data(\"Simple interval with secondary\", {\n      timeout: 72*3600, max_steps: nil, max_interval: 3600, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 1023, false),\n        RetryRecord.new(11, 2047, false),\n        RetryRecord.new(12, 4095, false),\n        RetryRecord.new(13, 7695, false),\n        RetryRecord.new(14, 11295, false),\n        RetryRecord.new(15, 14895, false),\n        RetryRecord.new(16, 18495, false),\n        RetryRecord.new(17, 22095, false),\n        RetryRecord.new(18, 25695, false),\n        RetryRecord.new(19, 29295, false),\n        RetryRecord.new(20, 32895, false),\n        RetryRecord.new(21, 36495, false),\n        RetryRecord.new(22, 40095, false),\n        RetryRecord.new(23, 43695, false),\n        RetryRecord.new(24, 47295, false),\n        RetryRecord.new(25, 50895, false),\n        RetryRecord.new(26, 54495, false),\n        RetryRecord.new(27, 58095, false),\n        RetryRecord.new(28, 61695, false),\n        RetryRecord.new(29, 65295, false),\n        RetryRecord.new(30, 68895, false),\n        RetryRecord.new(31, 72495, false),\n        RetryRecord.new(32, 76095, false),\n        RetryRecord.new(33, 79695, false),\n        RetryRecord.new(34, 83295, false),\n        RetryRecord.new(35, 86895, false),\n        RetryRecord.new(36, 90495, false),\n        RetryRecord.new(37, 94095, false),\n        RetryRecord.new(38, 97695, false),\n        RetryRecord.new(39, 101295, false),\n        RetryRecord.new(40, 104895, false),\n        RetryRecord.new(41, 108495, false),\n        RetryRecord.new(42, 112095, false),\n        RetryRecord.new(43, 115695, false),\n        RetryRecord.new(44, 119295, false),\n        RetryRecord.new(45, 122895, false),\n        RetryRecord.new(46, 126495, false),\n        RetryRecord.new(47, 130095, false),\n        RetryRecord.new(48, 133695, false),\n        RetryRecord.new(49, 137295, false),\n        RetryRecord.new(50, 140895, false),\n        RetryRecord.new(51, 144495, false),\n        RetryRecord.new(52, 148095, false),\n        RetryRecord.new(53, 151695, false),\n        RetryRecord.new(54, 155295, false),\n        RetryRecord.new(55, 158895, false),\n        RetryRecord.new(56, 162495, false),\n        RetryRecord.new(57, 166095, false),\n        RetryRecord.new(58, 169695, false),\n        RetryRecord.new(59, 173295, false),\n        RetryRecord.new(60, 176895, false),\n        RetryRecord.new(61, 180495, false),\n        RetryRecord.new(62, 184095, false),\n        RetryRecord.new(63, 187695, false),\n        RetryRecord.new(64, 191295, false),\n        RetryRecord.new(65, 194895, false),\n        RetryRecord.new(66, 198495, false),\n        RetryRecord.new(67, 202095, false),\n        RetryRecord.new(68, 205695, false),\n        RetryRecord.new(69, 207360, true),\n        RetryRecord.new(70, 207361, true),\n        RetryRecord.new(71, 207363, true),\n        RetryRecord.new(72, 207367, true),\n        RetryRecord.new(73, 207375, true),\n        RetryRecord.new(74, 207391, true),\n        RetryRecord.new(75, 207423, true),\n        RetryRecord.new(76, 207487, true),\n        RetryRecord.new(77, 207615, true),\n        RetryRecord.new(78, 207871, true),\n        RetryRecord.new(79, 208383, true),\n        RetryRecord.new(80, 209407, true),\n        RetryRecord.new(81, 211455, true),\n        RetryRecord.new(82, 215055, true),\n        RetryRecord.new(83, 218655, true),\n        RetryRecord.new(84, 222255, true),\n        RetryRecord.new(85, 225855, true),\n        RetryRecord.new(86, 229455, true),\n        RetryRecord.new(87, 233055, true),\n        RetryRecord.new(88, 236655, true),\n        RetryRecord.new(89, 240255, true),\n        RetryRecord.new(90, 243855, true),\n        RetryRecord.new(91, 247455, true),\n        RetryRecord.new(92, 251055, true),\n        RetryRecord.new(93, 254655, true),\n        RetryRecord.new(94, 258255, true),\n        RetryRecord.new(95, 259200, true),\n      ],\n    })\n    data(\"Max_steps and max_interval\", {\n      timeout: 72*3600, max_steps: 30, max_interval: 3600, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 1023, false),\n        RetryRecord.new(11, 2047, false),\n        RetryRecord.new(12, 4095, false),\n        RetryRecord.new(13, 7695, false),\n        RetryRecord.new(14, 11295, false),\n        RetryRecord.new(15, 14895, false),\n        RetryRecord.new(16, 18495, false),\n        RetryRecord.new(17, 22095, false),\n        RetryRecord.new(18, 25695, false),\n        RetryRecord.new(19, 29295, false),\n        RetryRecord.new(20, 32895, false),\n        RetryRecord.new(21, 36495, false),\n        RetryRecord.new(22, 40095, false),\n        RetryRecord.new(23, 43695, false),\n        RetryRecord.new(24, 47295, false),\n        RetryRecord.new(25, 50895, false),\n        RetryRecord.new(26, 54495, false),\n        RetryRecord.new(27, 58095, false),\n        RetryRecord.new(28, 61695, false),\n        RetryRecord.new(29, 65295, false),\n        RetryRecord.new(30, 68895, false),\n      ],\n    })\n    data(\"Max_steps and max_interval with secondary\", {\n      timeout: 72*3600, max_steps: 30, max_interval: 3600, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 1023, false),\n        RetryRecord.new(11, 2047, false),\n        RetryRecord.new(12, 4095, false),\n        RetryRecord.new(13, 7695, false),\n        RetryRecord.new(14, 11295, false),\n        RetryRecord.new(15, 14895, false),\n        RetryRecord.new(16, 18495, false),\n        RetryRecord.new(17, 22095, false),\n        RetryRecord.new(18, 25695, false),\n        RetryRecord.new(19, 29295, false),\n        RetryRecord.new(20, 32895, false),\n        RetryRecord.new(21, 36495, false),\n        RetryRecord.new(22, 40095, false),\n        RetryRecord.new(23, 43695, false),\n        RetryRecord.new(24, 47295, false),\n        RetryRecord.new(25, 50895, false),\n        RetryRecord.new(26, 54495, false),\n        RetryRecord.new(27, 55116, true),\n        RetryRecord.new(28, 55117, true),\n        RetryRecord.new(29, 55119, true),\n        RetryRecord.new(30, 55123, true),\n      ],\n    })\n    data(\"Max_steps and max_interval with timeout\", {\n      timeout: 10000, max_steps: 30, max_interval: 1000, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 1023, false),\n        RetryRecord.new(11, 2023, false),\n        RetryRecord.new(12, 3023, false),\n        RetryRecord.new(13, 4023, false),\n        RetryRecord.new(14, 5023, false),\n        RetryRecord.new(15, 6023, false),\n        RetryRecord.new(16, 7023, false),\n        RetryRecord.new(17, 8023, false),\n        RetryRecord.new(18, 9023, false),\n        RetryRecord.new(19, 10000, false),\n      ],\n    })\n    data(\"Max_steps and max_interval with timeout and secondary\", {\n      timeout: 10000, max_steps: 30, max_interval: 1000, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2,\n      expected: [\n        RetryRecord.new(1, 1, false),\n        RetryRecord.new(2, 3, false),\n        RetryRecord.new(3, 7, false),\n        RetryRecord.new(4, 15, false),\n        RetryRecord.new(5, 31, false),\n        RetryRecord.new(6, 63, false),\n        RetryRecord.new(7, 127, false),\n        RetryRecord.new(8, 255, false),\n        RetryRecord.new(9, 511, false),\n        RetryRecord.new(10, 1023, false),\n        RetryRecord.new(11, 2023, false),\n        RetryRecord.new(12, 3023, false),\n        RetryRecord.new(13, 4023, false),\n        RetryRecord.new(14, 5023, false),\n        RetryRecord.new(15, 6023, false),\n        RetryRecord.new(16, 7023, false),\n        RetryRecord.new(17, 8000, true),\n        RetryRecord.new(18, 8001, true),\n        RetryRecord.new(19, 8003, true),\n        RetryRecord.new(20, 8007, true),\n        RetryRecord.new(21, 8015, true),\n        RetryRecord.new(22, 8031, true),\n        RetryRecord.new(23, 8063, true),\n        RetryRecord.new(24, 8127, true),\n        RetryRecord.new(25, 8255, true),\n        RetryRecord.new(26, 8511, true),\n        RetryRecord.new(27, 9023, true),\n        RetryRecord.new(28, 10000, true),\n      ],\n    })\n    test \"exponential backoff with scenario\" do |data|\n      print_for_debug = false # change this value true if need to see msg always.\n      trying_count = 1000 # just for avoiding infinite loop\n\n      retry_records = []\n      msg = \"\"\n\n      s = @d.retry_state_create(\n        :t15, :exponential_backoff, data[:wait], data[:timeout],\n        max_steps: data[:max_steps], max_interval: data[:max_interval],\n        secondary: data[:use_sec], secondary_threshold: data[:sec_thres],\n        backoff_base: data[:backoff_base], randomize: false\n      )\n      override_current_time(s, s.start)\n\n      retry_count = 0\n      trying_count.times do\n        next_elapsed = (s.next_time - s.start).to_i\n\n        msg << \"step: #{s.steps}, next: #{next_elapsed}s (#{next_elapsed / 3600}h)\\n\"\n\n        # Wait until next time to trigger the next retry\n        override_current_time(s, s.next_time)\n\n        # Retry will be triggered at this point.\n        retry_count += 1\n        rec = RetryRecord.new(retry_count, next_elapsed, s.secondary?)\n        retry_records.append(rec)\n        msg << \"[#{next_elapsed}s elapsed point] #{retry_count}th-Retry(#{s.secondary? ? \"SEC\" : \"PRI\"}) is triggered.\\n\"\n\n        # Update retry statement\n        s.step\n        if s.limit?\n          msg << \"--- Reach limit. ---\\n\"\n          break\n        end\n      end\n\n      assert_equal(data[:expected], retry_records, msg)\n\n      print(msg) if print_for_debug\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_server.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/server'\nrequire 'fluent/plugin_helper/cert_option' # to create certs for tests\nrequire 'fluent/plugin/base'\nrequire 'timeout'\n\nrequire 'serverengine'\nrequire 'fileutils'\n\nclass ServerPluginHelperTest < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :server\n  end\n\n  TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/../tmp/plugin_helper_server\")\n\n  setup do\n    @port = unused_port(protocol: :tcp)\n    if Fluent.windows?\n      @socket_manager_server = ServerEngine::SocketManager::Server.open\n      @socket_manager_path = @socket_manager_server.path\n    else\n      @socket_manager_path = ServerEngine::SocketManager::Server.generate_path\n      if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)\n        FileUtils.rm_f @socket_manager_path\n      end\n      @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)\n    end\n    ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s\n\n    @d = Dummy.new\n    @d.under_plugin_development = true\n    @d.start\n    @d.after_start\n  end\n\n  teardown do\n    (@d.stopped? || @d.stop) rescue nil\n    (@d.before_shutdown? || @d.before_shutdown) rescue nil\n    (@d.shutdown? || @d.shutdown) rescue nil\n    (@d.after_shutdown? || @d.after_shutdown) rescue nil\n    (@d.closed? || @d.close) rescue nil\n    (@d.terminated? || @d.terminate) rescue nil\n\n    @socket_manager_server.close\n    if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)\n      FileUtils.rm_f @socket_manager_path\n    end\n  end\n\n  sub_test_case 'plugin instance' do\n    test 'can be instantiated to be able to create threads' do\n      d = Dummy.new\n      assert d.respond_to?(:_servers)\n      assert d._servers.empty?\n\n      assert d.respond_to?(:server_wait_until_start)\n      assert d.respond_to?(:server_wait_until_stop)\n      assert d.respond_to?(:server_create_connection)\n      assert d.respond_to?(:server_create)\n      assert d.respond_to?(:server_create_tcp)\n      assert d.respond_to?(:server_create_udp)\n      assert d.respond_to?(:server_create_tls)\n    end\n\n    test 'can be configured' do\n      d = Dummy.new\n      assert_nothing_raised do\n        d.configure(config_element())\n      end\n      assert d.plugin_id\n      assert d.log\n      assert_equal 0, d.transport_config.linger_timeout\n    end\n\n    test 'can change linger_timeout option' do\n      d = Dummy.new\n\n      transport_opts = {\n        'linger_timeout' => 1,\n      }\n      transport_conf = config_element('transport', 'tcp', transport_opts)\n      conf = config_element('source', 'tag.*', {}, [transport_conf])\n\n      assert_nothing_raised do\n        d.configure(conf)\n      end\n      assert d.plugin_id\n      assert d.log\n      assert_equal 1, d.transport_config.linger_timeout\n    end\n\n    test 'can change receive_buffer_size option' do\n      d = Dummy.new\n\n      transport_opts = {\n        'receive_buffer_size' => 1024,\n      }\n      transport_conf = config_element('transport', 'tcp', transport_opts)\n      conf = config_element('source', 'tag.*', {}, [transport_conf])\n\n      assert_nothing_raised do\n        d.configure(conf)\n      end\n      assert d.plugin_id\n      assert d.log\n      assert_equal 1024, d.transport_config.receive_buffer_size\n    end\n  end\n\n  # run tests for tcp, udp, tls and unix\n  sub_test_case '#server_create and #server_create_connection' do\n    methods = {server_create: :server_create, server_create_connection: :server_create_connection}\n\n    data(methods)\n    test 'raise error if title is not specified or not a symbol' do |m|\n      assert_raise(ArgumentError.new(\"BUG: title must be a symbol\")) do\n        @d.__send__(m, nil, @port){|x| x }\n      end\n      assert_raise(ArgumentError.new(\"BUG: title must be a symbol\")) do\n        @d.__send__(m, \"\", @port){|x| x }\n      end\n      assert_raise(ArgumentError.new(\"BUG: title must be a symbol\")) do\n        @d.__send__(m, \"title\", @port){|x| x }\n      end\n      assert_nothing_raised do\n        @d.__send__(m, :myserver, @port){|x| x }\n      end\n    end\n\n    data(methods)\n    test 'raise error if port is not specified or not an integer' do |m|\n      assert_raise(ArgumentError.new(\"BUG: port must be an integer\")) do\n        @d.__send__(m, :myserver, nil){|x| x }\n      end\n      assert_raise(ArgumentError.new(\"BUG: port must be an integer\")) do\n        @d.__send__(m, :myserver, \"1\"){|x| x }\n      end\n      assert_raise(ArgumentError.new(\"BUG: port must be an integer\")) do\n        @d.__send__(m, :myserver, 1.5){|x| x }\n      end\n      assert_nothing_raised do\n        @d.__send__(m, :myserver, @port){|x| x }\n      end\n    end\n\n    data(methods)\n    test 'raise error if block is not specified' do |m|\n      assert_raise(ArgumentError) do\n        @d.__send__(m, :myserver, @port)\n      end\n      assert_nothing_raised do\n        @d.__send__(m, :myserver, @port){|x| x }\n      end\n    end\n\n    data(methods)\n    test 'creates tcp server, binds 0.0.0.0 in default' do |m|\n      @d.__send__(m, :myserver, @port){|x| x }\n\n      assert_equal 1, @d._servers.size\n\n      created_server_info = @d._servers.first\n\n      assert_equal :myserver, created_server_info.title\n      assert_equal @port, created_server_info.port\n\n      assert_equal :tcp, created_server_info.proto\n      assert_equal \"0.0.0.0\", created_server_info.bind\n\n      created_server = created_server_info.server\n\n      assert created_server.is_a?(Coolio::TCPServer)\n      assert_equal \"0.0.0.0\", created_server.instance_eval{ @listen_socket }.addr[3]\n    end\n\n    data(methods)\n    test 'creates tcp server if specified in proto' do |m|\n      @d.__send__(m, :myserver, @port, proto: :tcp){|x| x }\n\n      created_server_info = @d._servers.first\n      assert_equal :tcp, created_server_info.proto\n      created_server = created_server_info.server\n      assert created_server.is_a?(Coolio::TCPServer)\n    end\n\n    data(methods)\n    test 'creates tls server in default if transport section and tcp protocol specified' do |m|\n      @d = d = Dummy.new\n      transport_conf = config_element('transport', 'tcp', {}, [])\n      d.configure(config_element('ROOT', '', {}, [transport_conf]))\n      d.start\n      d.after_start\n\n      d.__send__(m, :myserver, @port){|x| x }\n\n      created_server_info = @d._servers.first\n      assert_equal :tcp, created_server_info.proto\n      created_server = created_server_info.server\n      assert created_server.is_a?(Coolio::TCPServer)\n    end\n\n    data(methods)\n    test 'creates tls server if specified in proto' do |m|\n      assert_raise(ArgumentError.new(\"BUG: TLS transport specified, but certification options are not specified\")) do\n        @d.__send__(m, :myserver, @port, proto: :tls){|x| x }\n      end\n      @d.__send__(m, :myserver, @port, proto: :tls, tls_options: {insecure: true}){|x| x }\n\n      created_server_info = @d._servers.first\n      assert_equal :tls, created_server_info.proto\n      created_server = created_server_info.server\n      assert created_server.is_a?(Coolio::TCPServer) # yes, TCP here\n    end\n\n    data(methods)\n    test 'creates tls server in default if transport section and tls protocol specified' do |m|\n      @d = d = Dummy.new\n      transport_conf = config_element('transport', 'tls', {'insecure' => 'true'}, [])\n      d.configure(config_element('ROOT', '', {}, [transport_conf]))\n      d.start\n      d.after_start\n\n      d.__send__(m, :myserver, @port){|x| x }\n\n      created_server_info = @d._servers.first\n      assert_equal :tls, created_server_info.proto\n      created_server = created_server_info.server\n      assert created_server.is_a?(Coolio::TCPServer) # OK, it's Coolio::TCPServer\n    end\n\n    data(methods)\n    test 'creates unix server if specified in proto' do |m|\n      # pend \"not implemented yet\"\n    end\n\n    data(methods)\n    test 'raise error if unknown protocol specified' do |m|\n      assert_raise(ArgumentError.new(\"BUG: invalid protocol name\")) do\n        @d.__send__(m, :myserver, @port, proto: :quic){|x| x }\n      end\n    end\n\n    data(\n      'server_create tcp' => [:server_create, :tcp],\n      'server_create tls' => [:server_create, :tls],\n      # 'server_create unix' => [:server_create, :unix],\n      'server_create_connection tcp' => [:server_create_connection, :tcp],\n      'server_create_connection tls' => [:server_create_connection, :tls],\n      # 'server_create_connection tcp' => [:server_create_connection, :unix],\n    )\n    test 'raise error if udp options specified for tcp/tls/unix' do |(m, proto)|\n      port = unused_port(protocol: proto)\n      assert_raise ArgumentError do\n        @d.__send__(m, :myserver, port, proto: proto, max_bytes: 128){|x| x }\n      end\n      assert_raise ArgumentError do\n        @d.__send__(m, :myserver, port, proto: proto, flags: 1){|x| x }\n      end\n    end\n\n    data(\n      'server_create udp' => [:server_create, :udp],\n    )\n    test 'raise error if tcp/tls options specified for udp' do |(m, proto)|\n      port = unused_port(protocol: proto)\n      assert_raise(ArgumentError.new(\"BUG: linger_timeout is available for tcp/tls\")) do\n        @d.__send__(m, :myserver, port, proto: proto, linger_timeout: 1, max_bytes: 128){|x| x }\n      end\n    end\n\n    data(\n      'server_create udp' => [:server_create, :udp],\n    )\n    test 'raise error if tcp/tls/unix backlog options specified for udp' do |(m, proto)|\n      port = unused_port(protocol: proto)\n      assert_raise(ArgumentError.new(\"BUG: backlog is available for tcp/tls\")) do\n        @d.__send__(m, :myserver, port, proto: proto, backlog: 500){|x| x }\n      end\n    end\n\n    data(\n      'server_create udp' => [:server_create, :udp],\n    )\n    test 'raise error if tcp/tls send_keepalive_packet option is specified for udp' do |(m, proto)|\n      port = unused_port(protocol: proto)\n      assert_raise(ArgumentError.new(\"BUG: send_keepalive_packet is available for tcp/tls\")) do\n        @d.__send__(m, :myserver, port, proto: proto, send_keepalive_packet: true){|x| x }\n      end\n    end\n\n    data(\n      'server_create tcp' => [:server_create, :tcp, {}],\n      'server_create udp' => [:server_create, :udp, {max_bytes: 128}],\n      # 'server_create unix' => [:server_create, :unix, {}],\n      'server_create_connection tcp' => [:server_create_connection, :tcp, {}],\n      # 'server_create_connection unix' => [:server_create_connection, :unix, {}],\n    )\n    test 'raise error if tls options specified for tcp/udp/unix' do |(m, proto, kwargs)|\n      port = unused_port(protocol: proto)\n      assert_raise(ArgumentError.new(\"BUG: tls_options is available only for tls\")) do\n        @d.__send__(m, :myserver, port, proto: proto, tls_options: {}, **kwargs){|x| x }\n      end\n    end\n\n    data(\n      'server_create tcp' => [:server_create, :tcp, {}],\n      'server_create udp' => [:server_create, :udp, {max_bytes: 128}],\n      'server_create tls' => [:server_create, :tls, {tls_options: {insecure: true}}],\n      'server_create_connection tcp' => [:server_create_connection, :tcp, {}],\n      'server_create_connection tls' => [:server_create_connection, :tls, {tls_options: {insecure: true}}],\n    )\n    test 'can bind specified IPv4 address' do |(m, proto, kwargs)|\n      port = unused_port(protocol: proto)\n      @d.__send__(m, :myserver, port, proto: proto, bind: \"127.0.0.1\", **kwargs){|x| x }\n      assert_equal \"127.0.0.1\", @d._servers.first.bind\n      assert_equal \"127.0.0.1\", @d._servers.first.server.instance_eval{ instance_variable_defined?(:@listen_socket) ? @listen_socket : @_io }.addr[3]\n    end\n\n    data(\n      'server_create tcp' => [:server_create, :tcp, {}],\n      'server_create udp' => [:server_create, :udp, {max_bytes: 128}],\n      'server_create tls' => [:server_create, :tls, {tls_options: {insecure: true}}],\n      'server_create_connection tcp' => [:server_create_connection, :tcp, {}],\n      'server_create_connection tls' => [:server_create_connection, :tls, {tls_options: {insecure: true}}],\n    )\n    test 'can bind specified IPv6 address' do |(m, proto, kwargs)| # if available\n      omit \"IPv6 unavailable here\" unless ipv6_enabled?\n      port = unused_port(protocol: proto)\n      @d.__send__(m, :myserver, port, proto: proto, bind: \"::1\", **kwargs){|x| x }\n      assert_equal \"::1\", @d._servers.first.bind\n      assert_equal \"::1\", @d._servers.first.server.instance_eval{ instance_variable_defined?(:@listen_socket) ? @listen_socket : @_io }.addr[3]\n    end\n\n    data(\n      'server_create tcp' => [:server_create, :tcp, {}],\n      'server_create udp' => [:server_create, :udp, {max_bytes: 128}],\n      'server_create tls' => [:server_create, :tls, {tls_options: {insecure: true}}],\n      # 'server_create unix' => [:server_create, :unix, {}],\n      'server_create_connection tcp' => [:server_create, :tcp, {}],\n      'server_create_connection tls' => [:server_create, :tls, {tls_options: {insecure: true}}],\n      # 'server_create_connection unix' => [:server_create, :unix, {}],\n    )\n    test 'can create 2 or more servers which share same bind address and port if shared option is true' do |(m, proto, kwargs)|\n      begin\n        d2 = Dummy.new; d2.start; d2.after_start\n        port = unused_port(protocol: proto)\n\n        assert_nothing_raised do\n          @d.__send__(m, :myserver, port, proto: proto, **kwargs){|x| x }\n          d2.__send__(m, :myserver, port, proto: proto, **kwargs){|x| x }\n        end\n      ensure\n        d2.stop; d2.before_shutdown; d2.shutdown; d2.after_shutdown; d2.close; d2.terminate\n      end\n    end\n\n    data(\n      'server_create tcp' => [:server_create, :tcp, {}],\n      # Disable udp test because the behaviour of SO_REUSEXXX option is different between BSD, Linux and others...\n      # Need to find good way for testing on local, CI service and others.\n      #'server_create udp' => [:server_create, :udp, {max_bytes: 128}],\n      'server_create tls' => [:server_create, :tls, {tls_options: {insecure: true}}],\n      # 'server_create unix' => [:server_create, :unix, {}],\n      'server_create_connection tcp' => [:server_create, :tcp, {}],\n      'server_create_connection tls' => [:server_create, :tls, {tls_options: {insecure: true}}],\n      # 'server_create_connection unix' => [:server_create, :unix, {}],\n    )\n    test 'cannot create 2 or more servers using same bind address and port if shared option is false' do |(m, proto, kwargs)|\n      begin\n        d2 = Dummy.new; d2.start; d2.after_start\n        port = unused_port(protocol: proto)\n\n        assert_nothing_raised do\n          @d.__send__(m, :myserver, port, proto: proto, shared: false, **kwargs){|x| x }\n        end\n        assert_raise(Errno::EADDRINUSE, Errno::EACCES) do\n          d2.__send__(m, :myserver, port, proto: proto, **kwargs){|x| x }\n        end\n      ensure\n        d2.stop; d2.before_shutdown; d2.shutdown; d2.after_shutdown; d2.close; d2.terminate\n      end\n    end\n\n    test 'close all connections by shutdown' do\n      @d.server_create_tcp(:s, @port) do |data, conn|\n      end\n\n      client_sockets = []\n      5.times do\n        client_sockets << TCPSocket.open(\"127.0.0.1\", @port)\n      end\n      waiting(4){ sleep 0.1 until @d.instance_variable_get(:@_server_connections).size == 5 }\n\n      @d.stop\n      @d.before_shutdown\n      @d.shutdown\n\n      assert_true @d.instance_variable_get(:@_server_connections).empty?\n    ensure\n      client_sockets.each(&:close)\n    end\n  end\n\n  sub_test_case '#server_create' do\n    data(\n      'tcp' => [:tcp, {}],\n      'udp' => [:udp, {max_bytes: 128}],\n      'tls' => [:tls, {tls_options: {insecure: true}}],\n      # 'unix' => [:unix, {}],\n    )\n    test 'raise error if block argument is not specified or too many' do |(proto, kwargs)|\n      port = unused_port(protocol: proto)\n      assert_raise(ArgumentError.new(\"BUG: block must have 1 or 2 arguments\")) do\n        @d.server_create(:myserver, port, proto: proto, **kwargs){ 1 }\n      end\n      assert_raise(ArgumentError.new(\"BUG: block must have 1 or 2 arguments\")) do\n        @d.server_create(:myserver, port, proto: proto, **kwargs){|sock, conn, what_is_this| 1 }\n      end\n    end\n\n    test 'creates udp server if specified in proto' do\n      port = unused_port(protocol: :udp)\n      @d.server_create(:myserver, port, proto: :udp, max_bytes: 512){|x| x }\n\n      created_server_info = @d._servers.first\n      assert_equal :udp, created_server_info.proto\n      created_server = created_server_info.server\n      assert created_server.is_a?(Fluent::PluginHelper::Server::EventHandler::UDPServer)\n    end\n  end\n\n  sub_test_case '#server_create_tcp' do\n    test 'can accept all keyword arguments valid for tcp server' do\n      assert_nothing_raised do\n        @d.server_create_tcp(:s, @port, bind: '127.0.0.1', shared: false, resolve_name: true, linger_timeout: 10, backlog: 500, send_keepalive_packet: true) do |data, conn|\n          # ...\n        end\n      end\n    end\n\n    test 'creates a tcp server just to read data' do\n      received = \"\"\n      @d.server_create_tcp(:s, @port) do |data|\n        received << data\n      end\n      3.times do\n        sock = TCPSocket.new(\"127.0.0.1\", @port)\n        sock.puts \"yay\"\n        sock.puts \"foo\"\n        sock.close\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 24 }\n      assert_equal \"yay\\nfoo\\nyay\\nfoo\\nyay\\nfoo\\n\", received\n    end\n\n    test 'creates a tcp server to read and write data' do\n      received = \"\"\n      responses = []\n      @d.server_create_tcp(:s, @port) do |data, conn|\n        received << data\n        conn.write \"ack\\n\"\n      end\n      3.times do\n        TCPSocket.open(\"127.0.0.1\", @port) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n          responses << sock.readline\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 24 }\n      assert_equal \"yay\\nfoo\\nyay\\nfoo\\nyay\\nfoo\\n\", received\n      assert_equal [\"ack\\n\",\"ack\\n\",\"ack\\n\"], responses\n    end\n\n    test 'creates a tcp server to read and write data using IPv6' do\n      omit \"IPv6 unavailable here\" unless ipv6_enabled?\n\n      received = \"\"\n      responses = []\n      @d.server_create_tcp(:s, @port, bind: \"::1\") do |data, conn|\n        received << data\n        conn.write \"ack\\n\"\n      end\n      3.times do\n        TCPSocket.open(\"::1\", @port) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n          responses << sock.readline\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 24 }\n      assert_equal \"yay\\nfoo\\nyay\\nfoo\\nyay\\nfoo\\n\", received\n      assert_equal [\"ack\\n\",\"ack\\n\",\"ack\\n\"], responses\n    end\n\n    test 'does not resolve name of client address in default' do\n      received = \"\"\n      sources = []\n      @d.server_create_tcp(:s, @port) do |data, conn|\n        received << data\n        sources << conn.remote_host\n      end\n      3.times do\n        TCPSocket.open(\"127.0.0.1\", @port) do |sock|\n          sock.puts \"yay\"\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 12 }\n      assert_equal \"yay\\nyay\\nyay\\n\", received\n      assert{ sources.all?(\"127.0.0.1\") }\n    end\n\n    test 'does resolve name of client address if resolve_name is true' do\n      hostname = Socket.getnameinfo([nil, nil, nil, \"127.0.0.1\"])[0]\n\n      received = \"\"\n      sources = []\n      @d.server_create_tcp(:s, @port, resolve_name: true) do |data, conn|\n        received << data\n        sources << conn.remote_host\n      end\n      3.times do\n        TCPSocket.open(\"127.0.0.1\", @port) do |sock|\n          sock.puts \"yay\"\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 12 }\n      assert_equal \"yay\\nyay\\nyay\\n\", received\n      assert{ sources.all?(hostname) }\n    end\n\n    test 'can keep connections alive for tcp if keepalive specified' do\n      # pend \"not implemented yet\"\n    end\n\n    test 'raises error if plugin registers data callback for connection object from #server_create' do\n      received = \"\"\n      errors = []\n      @d.server_create_tcp(:s, @port) do |data, conn|\n        received << data\n        begin\n          conn.data{|d| received << d.upcase }\n        rescue => e\n          errors << e\n        end\n      end\n      TCPSocket.open(\"127.0.0.1\", @port) do |sock|\n        sock.puts \"foo\"\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 4 || errors.size == 1 }\n      assert_equal \"foo\\n\", received\n      assert{ errors.size > 0 } # it might be called twice (or more) when connection was accepted, and then data arrived (or more)\n      assert_equal \"data callback can be registered just once, but registered twice\", errors.first.message\n    end\n\n    test 'can call write_complete callback if registered' do\n      buffer = \"\"\n      lines = []\n      responses = []\n      response_completes = []\n      @d.server_create_tcp(:s, @port) do |data, conn|\n        conn.on(:write_complete){|c| response_completes << true }\n        buffer << data\n        if idx = buffer.index(\"\\n\")\n          lines << buffer.slice!(0,idx+1)\n          conn.write \"ack\\n\"\n        end\n      end\n      3.times do\n        TCPSocket.open(\"127.0.0.1\", @port) do |sock|\n          sock.write \"yay\"\n          sock.write \"foo\\n\"\n          begin\n            responses << sock.readline\n          rescue EOFError, IOError, Errno::ECONNRESET\n            # ignore\n          end\n          sock.close\n        end\n      end\n      waiting(10){ sleep 0.1 until lines.size == 3 && response_completes.size == 3 }\n      assert_equal [\"yayfoo\\n\", \"yayfoo\\n\", \"yayfoo\\n\"], lines\n      assert_equal [\"ack\\n\",\"ack\\n\",\"ack\\n\"], responses\n      assert_equal [true, true, true], response_completes\n    end\n\n    test 'can call close callback if registered' do\n      buffer = \"\"\n      lines = []\n      callback_results = []\n      @d.server_create_tcp(:s, @port) do |data, conn|\n        conn.on(:close){|c| callback_results << \"closed\" }\n        buffer << data\n        if idx = buffer.index(\"\\n\")\n          lines << buffer.slice!(0,idx+1)\n          conn.write \"ack\\n\"\n        end\n      end\n      3.times do\n        TCPSocket.open(\"127.0.0.1\", @port) do |sock|\n          sock.write \"yay\"\n          sock.write \"foo\\n\"\n          begin\n            while line = sock.readline\n              if line == \"ack\\n\"\n                sock.close\n              end\n            end\n          rescue EOFError, IOError, Errno::ECONNRESET\n            # ignore\n          end\n        end\n      end\n      waiting(10){ sleep 0.1 until lines.size == 3 && callback_results.size == 3 }\n      assert_equal [\"yayfoo\\n\", \"yayfoo\\n\", \"yayfoo\\n\"], lines\n      assert_equal [\"closed\", \"closed\", \"closed\"], callback_results\n    end\n\n    test 'can listen IPv4 / IPv6 together' do\n      omit \"IPv6 unavailable here\" unless ipv6_enabled?\n\n      assert_nothing_raised do\n        @d.server_create_tcp(:s_ipv4, @port, bind: '0.0.0.0', shared: false) do |data, conn|\n          # ...\n        end\n        @d.server_create_tcp(:s_ipv6, @port, bind: '::', shared: false) do |data, conn|\n          # ...\n        end\n      end\n    end\n  end\n\n  sub_test_case '#server_create_udp' do\n    test 'can accept all keyword arguments valid for udp server' do\n      assert_nothing_raised do\n        port = unused_port(protocol: :udp)\n        @d.server_create_udp(:s, port, bind: '127.0.0.1', shared: false, resolve_name: true, max_bytes: 100, flags: 1) do |data, conn|\n          # ...\n        end\n      end\n    end\n\n    test 'creates a udp server just to read data' do\n      received = \"\"\n      port = unused_port(protocol: :udp)\n      @d.server_create_udp(:s, port, max_bytes: 128) do |data|\n        received << data\n      end\n      bind_port = unused_port(protocol: :udp, bind: \"127.0.0.1\")\n      3.times do\n        sock = UDPSocket.new(Socket::AF_INET)\n        sock.bind(\"127.0.0.1\", bind_port)\n        sock.connect(\"127.0.0.1\", port)\n        sock.puts \"yay\"\n        sock.puts \"foo\"\n        sock.close\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 24 }\n      assert_equal \"yay\\nfoo\\nyay\\nfoo\\nyay\\nfoo\\n\", received\n    end\n\n    test 'creates a udp server to read and write data' do\n      received = \"\"\n      responses = []\n      port = unused_port(protocol: :udp)\n      @d.server_create_udp(:s, port, max_bytes: 128) do |data, sock|\n        received << data\n        sock.write \"ack\\n\"\n      end\n      bind_port = unused_port(protocol: :udp)\n      3.times do\n        begin\n          sock = UDPSocket.new(Socket::AF_INET)\n          sock.bind(\"127.0.0.1\", bind_port)\n          sock.connect(\"127.0.0.1\", port)\n          th = Thread.new do\n            while true\n              begin\n                in_data, _addr = sock.recvfrom_nonblock(16)\n                if in_data\n                  responses << in_data\n                  break\n                end\n              rescue IO::WaitReadable\n                IO.select([sock])\n              end\n            end\n            true\n          end\n          sock.write \"yay\\nfoo\\n\"\n          th.join(5)\n        ensure\n          sock.close\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 24 }\n      assert_equal \"yay\\nfoo\\nyay\\nfoo\\nyay\\nfoo\\n\", received\n      assert_equal [\"ack\\n\",\"ack\\n\",\"ack\\n\"], responses\n    end\n\n    test 'creates a udp server to read and write data using IPv6' do\n      omit \"IPv6 unavailable here\" unless ipv6_enabled?\n\n      received = \"\"\n      responses = []\n      port = unused_port(protocol: :udp, bind: \"::1\")\n      @d.server_create_udp(:s, port, bind: \"::1\", max_bytes: 128) do |data, sock|\n        received << data\n        sock.write \"ack\\n\"\n      end\n      bind_port = unused_port(protocol: :udp, bind: \"::1\")\n      3.times do\n        begin\n          sock = UDPSocket.new(Socket::AF_INET6)\n          sock.bind(\"::1\", bind_port)\n          th = Thread.new do\n            responses << sock.recv(16)\n            true\n          end\n          sock.connect(\"::1\", port)\n          sock.write \"yay\\nfoo\\n\"\n          th.join(5)\n        ensure\n          sock.close\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 24 }\n      assert_equal \"yay\\nfoo\\nyay\\nfoo\\nyay\\nfoo\\n\", received\n      assert_equal [\"ack\\n\",\"ack\\n\",\"ack\\n\"], responses\n    end\n\n    test 'does not resolve name of client address in default' do\n      received = \"\"\n      sources = []\n      port = unused_port(protocol: :udp)\n      @d.server_create_udp(:s, port, max_bytes: 128) do |data, sock|\n        received << data\n        sources << sock.remote_host\n      end\n      3.times do\n        sock = UDPSocket.new(Socket::AF_INET)\n        sock.connect(\"127.0.0.1\", port)\n        sock.puts \"yay\"\n        sock.close\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 12 }\n      assert_equal \"yay\\nyay\\nyay\\n\", received\n      assert{ sources.all?(\"127.0.0.1\") }\n    end\n\n    test 'does resolve name of client address if resolve_name is true' do\n      hostname = Socket.getnameinfo([nil, nil, nil, \"127.0.0.1\"])[0]\n\n      received = \"\"\n      sources = []\n      port = unused_port(protocol: :udp)\n      @d.server_create_udp(:s, port, resolve_name: true, max_bytes: 128) do |data, sock|\n        received << data\n        sources << sock.remote_host\n      end\n      3.times do\n        sock = UDPSocket.new(Socket::AF_INET)\n        sock.connect(\"127.0.0.1\", port)\n        sock.puts \"yay\"\n        sock.close\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 12 }\n      assert_equal \"yay\\nyay\\nyay\\n\", received\n      assert{ sources.all?(hostname) }\n    end\n\n    test 'raises error if plugin registers data callback for connection object from #server_create' do\n      received = \"\"\n      errors = []\n      port = unused_port(protocol: :udp)\n      @d.server_create_udp(:s, port, max_bytes: 128) do |data, sock|\n        received << data\n        begin\n          sock.data{|d| received << d.upcase }\n        rescue => e\n          errors << e\n        end\n      end\n      sock = UDPSocket.new(Socket::AF_INET)\n      sock.connect(\"127.0.0.1\", port)\n      sock.write \"foo\\n\"\n      sock.close\n\n      waiting(10){ sleep 0.1 until received.bytesize == 4 && errors.size == 1 }\n      assert_equal \"foo\\n\", received\n      assert_equal 1, errors.size\n      assert_equal \"BUG: this event is disabled for udp: data\", errors.first.message\n    end\n\n    test 'raise error if plugin registers write_complete callback for udp' do\n      received = \"\"\n      errors = []\n      port = unused_port(protocol: :udp)\n      @d.server_create_udp(:s, port, max_bytes: 128) do |data, sock|\n        received << data\n        begin\n          sock.on(:write_complete){|conn| \"\" }\n        rescue => e\n          errors << e\n        end\n      end\n      sock = UDPSocket.new(Socket::AF_INET)\n      sock.connect(\"127.0.0.1\", port)\n      sock.write \"foo\\n\"\n      sock.close\n\n      waiting(10){ sleep 0.1 until received.bytesize == 4 && errors.size == 1 }\n      assert_equal \"foo\\n\", received\n      assert_equal 1, errors.size\n      assert_equal \"BUG: this event is disabled for udp: write_complete\", errors.first.message\n    end\n\n    test 'raises error if plugin registers close callback for udp' do\n      received = \"\"\n      errors = []\n      port = unused_port(protocol: :udp)\n      @d.server_create_udp(:s, port, max_bytes: 128) do |data, sock|\n        received << data\n        begin\n          sock.on(:close){|d| \"\" }\n        rescue => e\n          errors << e\n        end\n      end\n      sock = UDPSocket.new(Socket::AF_INET)\n      sock.connect(\"127.0.0.1\", port)\n      sock.write \"foo\\n\"\n      sock.close\n\n      waiting(10){ sleep 0.1 until received.bytesize == 4 && errors.size == 1 }\n      assert_equal \"foo\\n\", received\n      assert_equal 1, errors.size\n      assert_equal \"BUG: this event is disabled for udp: close\", errors.first.message\n    end\n\n    test 'can bind IPv4 / IPv6 together' do\n      omit \"IPv6 unavailable here\" unless ipv6_enabled?\n\n      port = unused_port(protocol: :udp)\n      assert_nothing_raised do\n        @d.server_create_udp(:s_ipv4_udp, port, bind: '0.0.0.0', shared: false, max_bytes: 128) do |data, sock|\n          # ...\n        end\n        @d.server_create_udp(:s_ipv6_udp, port, bind: '::', shared: false, max_bytes: 128) do |data, sock|\n          # ...\n        end\n      end\n    end\n\n    sub_test_case 'over max_bytes' do\n      data(\"cut off on Non-Windows\", { max_bytes: 32, records: [\"a\" * 40], expected: [\"a\" * 32] }, keep: true) unless Fluent.windows?\n      data(\"drop on Windows\", { max_bytes: 32, records: [\"a\" * 40], expected: [] }, keep: true) if Fluent.windows?\n      test 'with sock' do |data|\n        max_bytes, records, expected = data.values\n\n        actual_records = []\n        port = unused_port(protocol: :udp)\n        @d.server_create_udp(:myserver, port, max_bytes: max_bytes) do |data, sock|\n          actual_records << data\n        end\n\n        open_client(:udp, \"127.0.0.1\", port) do |sock|\n          records.each do |record|\n            sock.send(record, 0)\n          end\n        end\n\n        waiting(10) { sleep 0.1 until actual_records.size >= expected.size }\n        sleep 1 if expected.size == 0 # To confirm no record recieved.\n\n        assert_equal expected, actual_records\n      end\n\n      test 'without sock' do |data|\n        max_bytes, records, expected = data.values\n\n        actual_records = []\n        port = unused_port(protocol: :udp)\n        @d.server_create_udp(:myserver, port, max_bytes: max_bytes) do |data|\n          actual_records << data\n        end\n\n        open_client(:udp, \"127.0.0.1\", port) do |sock|\n          records.each do |record|\n            sock.send(record, 0)\n          end\n        end\n\n        waiting(10) { sleep 0.1 until actual_records.size >= expected.size }\n        sleep 1 if expected.size == 0 # To confirm no record received.\n\n        assert_equal expected, actual_records\n      end\n    end\n  end\n\n  module CertUtil\n    extend Fluent::PluginHelper::CertOption\n  end\n\n  def create_ca_options\n    {\n      private_key_length: 2048,\n      country: 'US',\n      state: 'CA',\n      locality: 'Mountain View',\n      common_name: 'ca.testing.fluentd.org',\n      expiration: 30 * 86400,\n      digest: :sha256,\n    }\n  end\n\n  def create_server_options\n    {\n      private_key_length: 2048,\n      country: 'US',\n      state: 'CA',\n      locality: 'Mountain View',\n      common_name: 'server.testing.fluentd.org',\n      expiration: 30 * 86400,\n      digest: :sha256,\n    }\n  end\n\n  def write_cert_and_key(cert_path, cert, key_path, key, passphrase)\n    File.open(cert_path, \"w\"){|f| f.write(cert.to_pem) }\n    # Write the secret key (raw or encrypted by AES256) in PEM format\n    key_str = passphrase ? key.export(OpenSSL::Cipher.new(\"AES-256-CBC\"), passphrase) : key.export\n    File.open(key_path, \"w\"){|f| f.write(key_str) }\n    File.chmod(0600, cert_path, key_path)\n  end\n\n  def create_server_pair_signed_by_self(cert_path, private_key_path, passphrase)\n    cert, key, _ = CertUtil.cert_option_generate_server_pair_self_signed(create_server_options)\n    write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)\n    return cert\n  end\n\n  def create_ca_pair_signed_by_self(cert_path, private_key_path, passphrase)\n    cert, key, _ = CertUtil.cert_option_generate_ca_pair_self_signed(create_ca_options)\n    write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)\n  end\n\n  def create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)\n    cert, key, _ = CertUtil.cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, create_server_options)\n    write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)\n    return cert\n  end\n\n  def create_server_pair_chained_with_root_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)\n    root_cert, root_key, _ = CertUtil.cert_option_generate_ca_pair_self_signed(create_ca_options)\n    write_cert_and_key(ca_cert_path, root_cert, ca_key_path, root_key, ca_key_passphrase)\n\n    intermediate_ca_options = create_ca_options\n    intermediate_ca_options[:common_name] = 'ca2.testing.fluentd.org'\n    chain_cert, chain_key = CertUtil.cert_option_generate_pair(intermediate_ca_options, root_cert.subject)\n    chain_cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))\n    chain_cert.sign(root_key, \"sha256\")\n\n    server_cert, server_key, _ = CertUtil.cert_option_generate_pair(create_server_options, chain_cert.subject)\n    factory = OpenSSL::X509::ExtensionFactory.new\n    server_cert.add_extension(factory.create_extension('basicConstraints', 'CA:FALSE'))\n    server_cert.add_extension(factory.create_extension('nsCertType', 'server'))\n    server_cert.sign(chain_key, \"sha256\")\n\n    # write chained cert\n    File.open(cert_path, \"w\") do |f|\n      f.write server_cert.to_pem\n      f.write chain_cert.to_pem\n    end\n    key_str = passphrase ? server_key.export(OpenSSL::Cipher.new(\"AES-256-CBC\"), passphrase) : server_key.export\n    File.open(private_key_path, \"w\"){|f| f.write(key_str) }\n    File.chmod(0600, cert_path, private_key_path)\n  end\n\n  def open_tls_session(addr, port, version: Fluent::TLS::DEFAULT_VERSION, verify: true, cert_path: nil, selfsigned: true, hostname: nil)\n    context = OpenSSL::SSL::SSLContext.new\n    context.set_params({})\n    if verify\n      cert_store = OpenSSL::X509::Store.new\n      cert_store.set_default_paths\n      if selfsigned && OpenSSL::X509.const_defined?('V_FLAG_CHECK_SS_SIGNATURE')\n        cert_store.flags = OpenSSL::X509::V_FLAG_CHECK_SS_SIGNATURE\n      end\n      if cert_path\n        cert_store.add_file(cert_path)\n      end\n      context.verify_mode = OpenSSL::SSL::VERIFY_PEER\n      context.cert_store = cert_store\n      if !hostname\n        context.verify_hostname = false # In test code, using hostname to be connected is very difficult\n      end\n    else\n      context.verify_mode = OpenSSL::SSL::VERIFY_NONE\n    end\n    Fluent::TLS.set_version_to_context(context, version, nil, nil)\n\n    sock = OpenSSL::SSL::SSLSocket.new(TCPSocket.new(addr, port), context)\n    sock.hostname = hostname if hostname && sock.respond_to?(:hostname)\n    sock.connect\n    yield sock\n  ensure\n    sock.close rescue nil\n  end\n\n  def assert_certificate(cert, expected_extensions)\n    get_extension = lambda do |oid|\n      cert.extensions.detect { |e| e.oid == oid }\n    end\n\n    assert_true cert.serial > 1\n    assert_equal 2, cert.version\n\n    expected_extensions.each do |ext|\n      expected_oid, expected_value = ext\n      assert_equal expected_value, get_extension.call(expected_oid).value\n    end\n  end\n\n  sub_test_case '#server_create_tls with various certificate options' do\n    setup do\n      @d = Dummy.new # to get plugin not configured/started yet\n\n      @certs_dir = File.join(TMP_DIR, \"tls_certs\")\n      @server_cert_dir = File.join(@certs_dir, \"server\")\n      FileUtils.rm_rf @certs_dir\n      FileUtils.mkdir_p @server_cert_dir\n    end\n\n    sub_test_case 'using tls_options arguments to specify cert options' do\n      setup do\n        @d.configure(config_element()); @d.start; @d.after_start\n      end\n\n      test 'create dynamic self-signed cert/key pair (without any verification from clients)' do\n        # insecure\n        tls_options = {\n          protocol: :tls,\n          version: :'TLSv1_2',\n          ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',\n          insecure: true,\n          generate_private_key_length: 2048,\n          generate_cert_country: 'US',\n          generate_cert_state: 'CA',\n          generate_cert_locality: 'Mountain View',\n          generate_cert_common_name: 'myserver.testing.fluentd.org',\n          generate_cert_expiration: 10 * 365 * 86400,\n          generate_cert_digest: :sha256,\n        }\n\n        received = \"\"\n        @d.server_create_tls(:s, @port, tls_options: tls_options) do |data, conn|\n          received << data\n        end\n        assert_raise \"\" do\n          open_tls_session('127.0.0.1', @port) do |sock|\n            sock.post_connection_check('myserver.testing.fluentd.org')\n            # cannot connect ....\n          end\n        end\n        open_tls_session('127.0.0.1', @port, verify: false) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n        waiting(10){ sleep 0.1 until received.bytesize == 8 }\n        assert_equal \"yay\\nfoo\\n\", received\n      end\n\n      data('with passphrase' => 'yaaaaaaaaaaaaaaaaaaay',\n           'without passphrase'  => nil)\n      test 'load self-signed cert/key pair (files), verified from clients using cert files' do |private_key_passphrase|\n        cert_path = File.join(@server_cert_dir, \"cert.pem\")\n        private_key_path = File.join(@certs_dir, \"server.key.pem\")\n        cert = create_server_pair_signed_by_self(cert_path, private_key_path, private_key_passphrase)\n\n        assert_certificate(cert,[\n          ['basicConstraints', 'CA:FALSE'],\n          ['nsCertType', 'SSL Server']\n        ])\n\n        tls_options = {\n          protocol: :tls,\n          version: :'TLSv1_2',\n          ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',\n          insecure: false,\n          cert_path: cert_path,\n          private_key_path: private_key_path,\n        }\n        tls_options[:private_key_passphrase] = private_key_passphrase if private_key_passphrase\n        received = \"\"\n        @d.server_create_tls(:s, @port, tls_options: tls_options) do |data, conn|\n          received << data\n        end\n        assert_raise \"\" do\n          open_tls_session('127.0.0.1', @port) do |sock|\n            sock.post_connection_check('server.testing.fluentd.org')\n            # cannot connect by failing verification without server cert\n          end\n        end\n        open_tls_session('127.0.0.1', @port, cert_path: cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n        waiting(10){ sleep 0.1 until received.bytesize == 8 }\n        assert_equal \"yay\\nfoo\\n\", received\n      end\n\n      data('with passphrase' => \"fooooooooooooooooooooooooo\",\n           'without passphrase'  => nil)\n      test 'create dynamic server cert by private CA cert file, verified from clients using CA cert file' do |ca_key_passphrase|\n        ca_cert_path = File.join(@certs_dir, \"ca_cert.pem\")\n        ca_key_path = File.join(@certs_dir, \"ca.key.pem\")\n        create_ca_pair_signed_by_self(ca_cert_path, ca_key_path, ca_key_passphrase)\n\n        tls_options = {\n          protocol: :tls,\n          version: :'TLSv1_2',\n          ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',\n          insecure: false,\n          ca_cert_path: ca_cert_path,\n          ca_private_key_path: ca_key_path,\n          generate_private_key_length: 2048,\n        }\n        tls_options[:ca_private_key_passphrase] = ca_key_passphrase if ca_key_passphrase\n        received = \"\"\n        @d.server_create_tls(:s, @port, tls_options: tls_options) do |data, conn|\n          received << data\n        end\n        open_tls_session('127.0.0.1', @port, cert_path: ca_cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n        waiting(10){ sleep 0.1 until received.bytesize == 8 }\n        assert_equal \"yay\\nfoo\\n\", received\n      end\n\n      data('with passphrase' => [\"foooooooo\", \"yaaaaaaaaaaaaaaaaaaay\"],\n           'without passphrase'  => [nil, nil])\n      test 'load static server cert by private CA cert file, verified from clients using CA cert file' do |(ca_key_passphrase, private_key_passphrase)|\n        ca_cert_path = File.join(@certs_dir, \"ca_cert.pem\")\n        ca_key_path = File.join(@certs_dir, \"ca.key.pem\")\n        create_ca_pair_signed_by_self(ca_cert_path, ca_key_path, ca_key_passphrase)\n\n        cert_path = File.join(@server_cert_dir, \"cert.pem\")\n        private_key_path = File.join(@certs_dir, \"server.key.pem\")\n        cert = create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)\n\n        assert_certificate(cert,[\n            ['basicConstraints', 'CA:FALSE'],\n            ['nsCertType', 'SSL Server'],\n            ['keyUsage', 'Digital Signature, Key Encipherment'],\n            ['extendedKeyUsage', 'TLS Web Server Authentication']\n        ])\n\n        tls_options = {\n          protocol: :tls,\n          version: :'TLSv1_2',\n          ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',\n          insecure: false,\n          cert_path: cert_path,\n          private_key_path: private_key_path,\n        }\n        tls_options[:private_key_passphrase] = private_key_passphrase if private_key_passphrase\n        received = \"\"\n        @d.server_create_tls(:s, @port, tls_options: tls_options) do |data, conn|\n          received << data\n        end\n        open_tls_session('127.0.0.1', @port, cert_path: ca_cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n        waiting(10){ sleep 0.1 until received.bytesize == 8 }\n        assert_equal \"yay\\nfoo\\n\", received\n      end\n\n      data('with passphrase' => [\"foooooooo\", \"yaaaaaaaaaaaaaaaaaaay\"],\n           'without passphrase'  => [nil, nil])\n      test 'load chained server cert by private CA cert file, verified from clients using CA cert file as root' do |(ca_key_passphrase, private_key_passphrase)|\n        ca_cert_path = File.join(@certs_dir, \"ca_cert.pem\")\n        ca_key_path = File.join(@certs_dir, \"ca.key.pem\")\n        cert_path = File.join(@server_cert_dir, \"cert.pem\")\n        private_key_path = File.join(@certs_dir, \"server.key.pem\")\n        create_server_pair_chained_with_root_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)\n\n        tls_options = {\n          protocol: :tls,\n          version: :'TLSv1_2',\n          ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',\n          insecure: false,\n          cert_path: cert_path,\n          private_key_path: private_key_path,\n        }\n        tls_options[:private_key_passphrase] = private_key_passphrase if private_key_passphrase\n        received = \"\"\n        @d.server_create_tls(:s, @port, tls_options: tls_options) do |data, conn|\n          received << data\n        end\n        open_tls_session('127.0.0.1', @port, cert_path: ca_cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n        waiting(10){ sleep 0.1 until received.bytesize == 8 }\n        assert_equal \"yay\\nfoo\\n\", received\n      end\n    end\n\n    sub_test_case 'using configurations to specify cert options' do\n      test 'create dynamic self-signed cert/key pair (without any verification from clients)' do\n        # insecure\n        transport_opts = {\n          'insecure' => 'true',\n        }\n        transport_conf = config_element('transport', 'tls', transport_opts)\n        conf = config_element('match', 'tag.*', {}, [transport_conf])\n\n        @d.configure(conf); @d.start; @d.after_start\n\n        received = \"\"\n        @d.server_create_tls(:s, @port) do |data, conn|\n          received << data\n        end\n        assert_raise \"\" do\n          open_tls_session('127.0.0.1', @port) do |sock|\n            sock.post_connection_check('myserver.testing.fluentd.org')\n            # cannot connect ....\n          end\n        end\n        open_tls_session('127.0.0.1', @port, verify: false) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n        waiting(10){ sleep 0.1 until received.bytesize == 8 }\n        assert_equal \"yay\\nfoo\\n\", received\n      end\n\n      data('with passphrase' => \"yaaaaaaaaaaaaaaaaaaay\",\n           'without passphrase'  => nil)\n      test 'load self-signed cert/key pair (files), verified from clients using cert files' do |private_key_passphrase|\n        cert_path = File.join(@server_cert_dir, \"cert.pem\")\n        private_key_path = File.join(@certs_dir, \"server.key.pem\")\n        create_server_pair_signed_by_self(cert_path, private_key_path, private_key_passphrase)\n\n        transport_opts = {\n          'cert_path' => cert_path,\n          'private_key_path' => private_key_path,\n        }\n        transport_opts['private_key_passphrase'] = private_key_passphrase if private_key_passphrase\n        transport_conf = config_element('transport', 'tls', transport_opts)\n        conf = config_element('match', 'tag.*', {}, [transport_conf])\n\n        @d.configure(conf); @d.start; @d.after_start\n\n        received = \"\"\n        @d.server_create_tls(:s, @port) do |data, conn|\n          received << data\n        end\n        assert_raise \"\" do\n          open_tls_session('127.0.0.1', @port) do |sock|\n            sock.post_connection_check('server.testing.fluentd.org')\n            # cannot connect by failing verification without server cert\n          end\n        end\n        open_tls_session('127.0.0.1', @port, cert_path: cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n        waiting(10){ sleep 0.1 until received.bytesize == 8 }\n        assert_equal \"yay\\nfoo\\n\", received\n      end\n\n      data('with passphrase' => \"fooooooooooooooooooooooooo\",\n           'without passphrase'  => nil)\n      test 'create dynamic server cert by private CA cert file, verified from clients using CA cert file' do |ca_key_passphrase|\n        ca_cert_path = File.join(@certs_dir, \"ca_cert.pem\")\n        ca_key_path = File.join(@certs_dir, \"ca.key.pem\")\n        create_ca_pair_signed_by_self(ca_cert_path, ca_key_path, ca_key_passphrase)\n\n        transport_opts = {\n          'ca_cert_path' => ca_cert_path,\n          'ca_private_key_path' => ca_key_path,\n        }\n        transport_opts['ca_private_key_passphrase'] = ca_key_passphrase if ca_key_passphrase\n        transport_conf = config_element('transport', 'tls', transport_opts)\n        conf = config_element('match', 'tag.*', {}, [transport_conf])\n\n        @d.configure(conf); @d.start; @d.after_start\n\n        received = \"\"\n        @d.server_create_tls(:s, @port) do |data, conn|\n          received << data\n        end\n        open_tls_session('127.0.0.1', @port, cert_path: ca_cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n        waiting(10){ sleep 0.1 until received.bytesize == 8 }\n        assert_equal \"yay\\nfoo\\n\", received\n      end\n\n      data('with passphrase' => [\"foooooooo\", \"yaaaaaaaaaaaaaaaaaaay\"],\n           'without passphrase'  => [nil, nil])\n      test 'load static server cert by private CA cert file, verified from clients using CA cert file' do |(ca_key_passphrase, private_key_passphrase)|\n        ca_cert_path = File.join(@certs_dir, \"ca_cert.pem\")\n        ca_key_path = File.join(@certs_dir, \"ca.key.pem\")\n        create_ca_pair_signed_by_self(ca_cert_path, ca_key_path, ca_key_passphrase)\n\n        cert_path = File.join(@server_cert_dir, \"cert.pem\")\n        private_key_path = File.join(@certs_dir, \"server.key.pem\")\n        create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)\n\n        transport_opts = {\n          'cert_path' => cert_path,\n          'private_key_path' => private_key_path,\n        }\n        transport_opts['private_key_passphrase'] = private_key_passphrase if private_key_passphrase\n        transport_conf = config_element('transport', 'tls', transport_opts)\n        conf = config_element('match', 'tag.*', {}, [transport_conf])\n\n        @d.configure(conf); @d.start; @d.after_start\n\n        received = \"\"\n        @d.server_create_tls(:s, @port) do |data, conn|\n          received << data\n        end\n        open_tls_session('127.0.0.1', @port, cert_path: ca_cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n        waiting(10){ sleep 0.1 until received.bytesize == 8 }\n        assert_equal \"yay\\nfoo\\n\", received\n      end\n\n      data('with passphrase' => [\"foooooooo\", \"yaaaaaaaaaaaaaaaaaaay\"],\n           'without passphrase'  => [nil, nil])\n      test 'load chained server cert by private CA cert file, verified from clients using CA cert file as root' do |(ca_key_passphrase, private_key_passphrase)|\n        ca_cert_path = File.join(@certs_dir, \"ca_cert.pem\")\n        ca_key_path = File.join(@certs_dir, \"ca.key.pem\")\n        cert_path = File.join(@server_cert_dir, \"cert.pem\")\n        private_key_path = File.join(@certs_dir, \"server.key.pem\")\n        create_server_pair_chained_with_root_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)\n\n        transport_opts = {\n          'cert_path' => cert_path,\n          'private_key_path' => private_key_path,\n        }\n        transport_opts['private_key_passphrase'] = private_key_passphrase if private_key_passphrase\n        transport_conf = config_element('transport', 'tls', transport_opts)\n        conf = config_element('match', 'tag.*', {}, [transport_conf])\n\n        @d.configure(conf); @d.start; @d.after_start\n\n        received = \"\"\n        @d.server_create_tls(:s, @port) do |data, conn|\n          received << data\n        end\n        open_tls_session('127.0.0.1', @port, cert_path: ca_cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n        waiting(10){ sleep 0.1 until received.bytesize == 8 }\n        assert_equal \"yay\\nfoo\\n\", received\n      end\n\n      test 'set ciphers' do\n        cert_path = File.join(@server_cert_dir, \"cert.pem\")\n        private_key_path = File.join(@certs_dir, \"server.key.pem\")\n        create_server_pair_signed_by_self(cert_path, private_key_path, nil)\n        tls_options = {\n          protocol: :tls,\n          version: :TLSv1_2,\n          ciphers: 'SHA256',\n          insecure: false,\n          cert_path: cert_path,\n          private_key_path: private_key_path,\n        }\n        conf = @d.server_create_transport_section_object(tls_options)\n        ctx = @d.cert_option_create_context(conf.version, conf.insecure, conf.ciphers, conf)\n        matched = false\n        ctx.ciphers.each do |cipher|\n          cipher_name, tls_version = cipher\n          # OpenSSL 1.0.2: \"TLSv1/SSLv3\"\n          # OpenSSL 1.1.1: \"TLSv1.2\"\n          if tls_version == \"TLSv1/SSLv3\" || tls_version == \"TLSv1.2\"\n            matched = true\n            unless cipher_name.match?(/#{conf.ciphers}/)\n              matched = false\n              break\n            end\n          end\n        end\n\n        error_msg = build_message(\"Unexpected ciphers for #{conf.version}\",\n                                  \"<?>\\nwas expected to include only <?> ciphers for #{conf.version}\",\n                                  ctx.ciphers, conf.ciphers)\n        assert(matched, error_msg)\n      end\n    end\n  end\n\n  sub_test_case '#server_create_tls' do\n    setup do\n      @certs_dir = File.join(TMP_DIR, \"tls_certs\")\n      FileUtils.rm_rf @certs_dir\n      FileUtils.mkdir_p @certs_dir\n\n      @server_cert_dir = File.join(@certs_dir, \"server\")\n      FileUtils.mkdir_p @server_cert_dir\n\n      @cert_path = File.join(@server_cert_dir, \"cert.pem\")\n      private_key_path = File.join(@certs_dir, \"server.key.pem\")\n      private_key_passphrase = \"yaaaaaaaaaaaaaaaaaaay\"\n      create_server_pair_signed_by_self(@cert_path, private_key_path, private_key_passphrase)\n\n      @default_hostname = ::Socket.gethostname\n\n      @tls_options = {\n        protocol: :tls,\n        version: :'TLSv1_2',\n        ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',\n        insecure: false,\n        cert_path: @cert_path,\n        private_key_path: private_key_path,\n        private_key_passphrase: private_key_passphrase,\n      }\n    end\n\n    test 'can accept all keyword arguments valid for tcp/tls server' do\n      assert_nothing_raised do\n        @d.server_create_tls(:s, @port, bind: '127.0.0.1', shared: false, resolve_name: true, linger_timeout: 10, backlog: 500, tls_options: @tls_options, send_keepalive_packet: true) do |data, conn|\n          # ...\n        end\n      end\n    end\n\n    test 'creates a tls server just to read data' do\n      received = \"\"\n      @d.server_create_tls(:s, @port, tls_options: @tls_options) do |data, conn|\n        received << data\n      end\n      3.times do\n        open_tls_session('127.0.0.1', @port, cert_path: @cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 24 }\n      assert_equal 3, received.scan(\"yay\\n\").size\n      assert_equal 3, received.scan(\"foo\\n\").size\n    end\n\n    test 'creates a tls server to read and write data' do\n      received = \"\"\n      responses = []\n      @d.server_create_tls(:s, @port, tls_options: @tls_options) do |data, conn|\n        received << data\n        conn.write \"ack\\n\"\n      end\n      3.times do\n        # open_tls_session('127.0.0.1', @port, cert_path: @cert_path, hostname: @default_hostname) do |sock|\n        open_tls_session('127.0.0.1', @port, cert_path: @cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n          responses << sock.readline\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 24 }\n      assert_equal 3, received.scan(\"yay\\n\").size\n      assert_equal 3, received.scan(\"foo\\n\").size\n      assert_equal [\"ack\\n\",\"ack\\n\",\"ack\\n\"], responses\n    end\n\n    test 'creates a tls server to read and write data using IPv6' do\n      omit \"IPv6 unavailable here\" unless ipv6_enabled?\n\n      received = \"\"\n      responses = []\n      @d.server_create_tls(:s, @port, bind: \"::1\",  tls_options: @tls_options) do |data, conn|\n        received << data\n        conn.write \"ack\\n\"\n      end\n      3.times do\n        # open_tls_session('::1', @port, cert_path: @cert_path, hostname: @default_hostname) do |sock|\n        open_tls_session('::1', @port, cert_path: @cert_path) do |sock|\n          sock.puts \"yay\"\n          sock.puts \"foo\"\n          responses << sock.readline\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 24 }\n      assert_equal 3, received.scan(\"yay\\n\").size\n      assert_equal 3, received.scan(\"foo\\n\").size\n      assert_equal [\"ack\\n\",\"ack\\n\",\"ack\\n\"], responses\n    end\n\n    test 'does not resolve name of client address in default' do\n      received = \"\"\n      sources = []\n      @d.server_create_tls(:s, @port, tls_options: @tls_options) do |data, conn|\n        received << data\n        sources << conn.remote_host\n      end\n      3.times do\n        # open_tls_session('127.0.0.1', @port, cert_path: @cert_path, hostname: @default_hostname) do |sock|\n        open_tls_session('127.0.0.1', @port, cert_path: @cert_path) do |sock|\n          sock.puts \"yay\"\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 12 }\n      assert_equal 3, received.scan(\"yay\\n\").size\n      assert{ sources.all?(\"127.0.0.1\") }\n    end\n\n    test 'does resolve name of client address if resolve_name is true' do\n      hostname = Socket.getnameinfo([nil, nil, nil, \"127.0.0.1\"])[0]\n\n      received = \"\"\n      sources = []\n      @d.server_create_tls(:s, @port, resolve_name: true, tls_options: @tls_options) do |data, conn|\n        received << data\n        sources << conn.remote_host\n      end\n      3.times do\n        # open_tls_session('127.0.0.1', @port, cert_path: @cert_path, hostname: @default_hostname) do |sock|\n        open_tls_session('127.0.0.1', @port, cert_path: @cert_path) do |sock|\n          sock.puts \"yay\"\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 12 }\n      assert_equal 3, received.scan(\"yay\\n\").size\n      assert{ sources.all?(hostname) }\n    end\n\n    test 'can keep connections alive for tls if keepalive specified' do\n      # pend \"not implemented yet\"\n    end\n\n    test 'raises error if plugin registers data callback for connection object from #server_create' do\n      received = \"\"\n      errors = []\n      @d.server_create_tls(:s, @port, tls_options: @tls_options) do |data, conn|\n        received << data\n        begin\n          conn.data{|d| received << d.upcase }\n        rescue => e\n          errors << e\n        end\n      end\n      open_tls_session('127.0.0.1', @port, cert_path: @cert_path) do |sock|\n        sock.puts \"foo\"\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 4 || errors.size == 1 }\n      assert_equal \"foo\\n\", received\n      assert_equal 1, errors.size\n      assert_equal \"data callback can be registered just once, but registered twice\", errors.first.message\n    end\n\n    test 'can call write_complete callback if registered' do\n      buffer = \"\"\n      lines = []\n      responses = []\n      response_completes = []\n      @d.server_create_tls(:s, @port, tls_options: @tls_options) do |data, conn|\n        conn.on(:write_complete){|c| response_completes << true }\n        buffer << data\n        if idx = buffer.index(\"\\n\")\n          lines << buffer.slice!(0,idx+1)\n          conn.write \"ack\\n\"\n        end\n      end\n      3.times do\n        open_tls_session('127.0.0.1', @port, cert_path: @cert_path) do |sock|\n          sock.write \"yay\"\n          sock.write \"foo\\n\"\n          begin\n            responses << sock.readline\n          rescue EOFError, IOError, Errno::ECONNRESET\n            # ignore\n          end\n          sock.close\n        end\n      end\n      waiting(10){ sleep 0.1 until lines.size == 3 && response_completes.size == 3 }\n      assert_equal [\"yayfoo\\n\", \"yayfoo\\n\", \"yayfoo\\n\"], lines\n      assert_equal [\"ack\\n\",\"ack\\n\",\"ack\\n\"], responses\n      assert_equal [true, true, true], response_completes\n    end\n\n    test 'can call close callback if registered' do\n      buffer = \"\"\n      lines = []\n      callback_results = []\n      @d.server_create_tls(:s, @port, tls_options: @tls_options) do |data, conn|\n        conn.on(:close){|c| callback_results << \"closed\" }\n        buffer << data\n        if idx = buffer.index(\"\\n\")\n          lines << buffer.slice!(0,idx+1)\n          conn.write \"ack\\n\"\n        end\n      end\n      3.times do\n        open_tls_session('127.0.0.1', @port, cert_path: @cert_path) do |sock|\n          sock.write \"yay\"\n          sock.write \"foo\\n\"\n          begin\n            while line = sock.readline\n              if line == \"ack\\n\"\n                sock.close\n              end\n            end\n          rescue EOFError, IOError, Errno::ECONNRESET\n            # ignore\n          end\n        end\n      end\n      waiting(10){ sleep 0.1 until lines.size == 3 && callback_results.size == 3 }\n      assert_equal [\"yayfoo\\n\", \"yayfoo\\n\", \"yayfoo\\n\"], lines\n      assert_equal [\"closed\", \"closed\", \"closed\"], callback_results\n    end\n\n    sub_test_case 'TLS version connection check' do\n      test \"can't connect with different TLS version\" do\n        @d.server_create_tls(:s, @port, tls_options: @tls_options) do |data, conn|\n        end\n        if defined?(OpenSSL::SSL::TLS1_3_VERSION)\n          version = :'TLS1_3'\n        else\n          version = :'TLS1_1'\n        end\n        assert_raise(OpenSSL::SSL::SSLError, Errno::ECONNRESET) {\n          open_tls_session('127.0.0.1', @port, cert_path: @cert_path, version: version) do |sock|\n          end\n        }\n      end\n\n      test \"can specify multiple TLS versions by min_version/max_version\" do\n        omit \"min_version=/max_version= is not supported\" unless Fluent::TLS::MIN_MAX_AVAILABLE\n\n        min_version = :'TLS1_2'\n        if defined?(OpenSSL::SSL::TLS1_3_VERSION)\n          max_version = :'TLS1_3'\n        else\n          max_version = :'TLS1_2'\n        end\n\n        opts = @tls_options.merge(min_version: min_version, max_version: max_version)\n        @d.server_create_tls(:s, @port, tls_options: opts) do |data, conn|\n        end\n        assert_raise(OpenSSL::SSL::SSLError, Errno::ECONNRESET) {\n          open_tls_session('127.0.0.1', @port, cert_path: @cert_path, version: :'TLS1') do |sock|\n          end\n        }\n        [min_version, max_version].each { |ver|\n          assert_nothing_raised {\n            open_tls_session('127.0.0.1', @port, cert_path: @cert_path, version: ver) do |sock|\n            end\n          }\n        }\n      end\n    end\n  end\n\n  sub_test_case '#server_create_unix' do\n    # not implemented yet\n\n    # test 'can accept all keyword arguments valid for unix server'\n    # test 'creates a unix server just to read data'\n    # test 'creates a unix server to read and write data'\n\n    # test 'raises error if plugin registers data callback for connection object from #server_create'\n    # test 'can call write_complete callback if registered'\n    # test 'can call close callback if registered'\n  end\n\n  def open_client(proto, addr, port)\n    client = case proto\n             when :udp\n               c = UDPSocket.open\n               c.connect(addr, port)\n               c\n             when :tcp\n               TCPSocket.open(addr, port)\n             when :tls\n               c = OpenSSL::SSL::SSLSocket.new(TCPSocket.open(addr, port))\n               c.sync_close = true\n               c.connect\n             else\n               raise ArgumentError, \"unknown proto:#{proto}\"\n             end\n    yield client\n  ensure\n    client.close rescue nil\n  end\n\n  # run tests for tcp, tls and unix\n  sub_test_case '#server_create_connection' do\n    test 'raise error if udp is specified in proto' do\n      assert_raise(ArgumentError.new(\"BUG: cannot create connection for UDP\")) do\n        @d.server_create_connection(:myserver, @port, proto: :udp){|c| c }\n      end\n    end\n\n    # def server_create_connection(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, tls_options: nil, resolve_name: false, linger_timeout: 0, backlog: nil, &block)\n    protocols = {\n      'tcp' => [:tcp, {}],\n      'tls' => [:tls, {tls_options: {insecure: true}}],\n      # 'unix' => [:unix, {path: \"\"}],\n    }\n\n    data(protocols)\n    test 'raise error if block argument is not specified or too many' do |(proto, kwargs)|\n      empty_block = ->(){}\n      assert_raise(ArgumentError.new(\"BUG: block must have just one argument\")) do\n        @d.server_create_connection(:myserver, @port, proto: proto, **kwargs, &empty_block)\n      end\n      assert_raise(ArgumentError.new(\"BUG: block must have just one argument\")) do\n        @d.server_create_connection(:myserver, @port, proto: proto, **kwargs){|conn, what_is_this| [conn, what_is_this] }\n      end\n    end\n\n    data(protocols)\n    test 'does not resolve name of client address in default' do |(proto, kwargs)|\n      received = \"\"\n      sources = []\n      @d.server_create_connection(:s, @port, proto: proto, **kwargs) do |conn|\n        sources << conn.remote_host\n        conn.data do |d|\n          received << d\n        end\n      end\n      3.times do\n        open_client(proto, \"127.0.0.1\", @port) do |sock|\n          sock.puts \"yay\"\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 12 }\n      assert_equal \"yay\\nyay\\nyay\\n\", received\n      assert{ sources.all?(\"127.0.0.1\") }\n    end\n\n    data(protocols)\n    test 'does resolve name of client address if resolve_name is true' do |(proto, kwargs)|\n      hostname = Socket.getnameinfo([nil, nil, nil, \"127.0.0.1\"])[0]\n\n      received = \"\"\n      sources = []\n      @d.server_create_connection(:s, @port, proto: proto, resolve_name: true, **kwargs) do |conn|\n        sources << conn.remote_host\n        conn.data do |d|\n          received << d\n        end\n      end\n      3.times do\n        open_client(proto, \"127.0.0.1\", @port) do |sock|\n          sock.puts \"yay\"\n        end\n      end\n      waiting(10){ sleep 0.1 until received.bytesize == 12 }\n      assert_equal \"yay\\nyay\\nyay\\n\", received\n      assert{ sources.all?(hostname) }\n    end\n\n    data(protocols)\n    test 'creates a server to provide connection, which can read, write and close' do |(proto, kwargs)|\n      lines = []\n      buffer = \"\"\n      @d.server_create_connection(:s, @port, proto: proto, **kwargs) do |conn|\n        conn.data do |d|\n          buffer << d\n          if buffer == \"x\"\n            buffer.slice!(0, 1)\n            conn.close\n          end\n          if idx = buffer.index(\"\\n\")\n            lines << buffer.slice!(0, idx + 1)\n            conn.write \"foo!\\n\"\n          end\n        end\n      end\n      replied = []\n      disconnecteds = []\n      3.times do |i|\n        open_client(proto, \"127.0.0.1\", @port) do |sock|\n          sock.puts \"yay\"\n          while line = sock.readline\n            replied << line\n            break\n          end\n          sock.write \"x\"\n          connection_closed = false\n          begin\n            data = sock.read\n            if data.empty?\n              connection_closed = true\n            end\n          rescue => e\n            if e.is_a?(Errno::ECONNRESET)\n              connection_closed = true\n            end\n          ensure\n            disconnecteds << connection_closed\n          end\n        end\n      end\n      waiting(10){ sleep 0.1 until lines.size == 3 }\n      waiting(10){ sleep 0.1 until replied.size == 3 }\n      waiting(10){ sleep 0.1 until disconnecteds.size == 3 }\n      assert_equal [\"yay\\n\", \"yay\\n\", \"yay\\n\"], lines\n      assert_equal [\"foo!\\n\", \"foo!\\n\", \"foo!\\n\"], replied\n      assert_equal [true, true, true], disconnecteds\n    end\n\n    data(protocols)\n    test 'creates a server to provide connection, which accepts callbacks for data, write_complete, and close' do |(proto, kwargs)|\n      lines = []\n      buffer = \"\"\n      written = 0\n      closed = 0\n      @d.server_create_connection(:s, @port, proto: proto, **kwargs) do |conn|\n        conn.on(:write_complete){|_conn| written += 1 }\n        conn.on(:close){|_conn| closed += 1 }\n        conn.on(:data) do |d|\n          buffer << d\n          if idx = buffer.index(\"\\n\")\n            lines << buffer.slice!(0, idx + 1)\n            conn.write \"foo!\\n\"\n          end\n        end\n      end\n      replied = []\n      3.times do\n        open_client(proto, \"127.0.0.1\", @port) do |sock|\n          sock.puts \"yay\"\n          while line = sock.readline\n            replied << line\n            break\n          end\n        end # TCP socket is closed here\n      end\n      waiting(10){ sleep 0.1 until lines.size == 3 }\n      waiting(10){ sleep 0.1 until replied.size == 3 }\n      waiting(10){ sleep 0.1 until closed == 3 }\n      assert_equal [\"yay\\n\", \"yay\\n\", \"yay\\n\"], lines\n      assert_equal 3, written\n      assert_equal 3, closed\n      assert_equal [\"foo!\\n\", \"foo!\\n\", \"foo!\\n\"], replied\n    end\n\n    data(protocols)\n    test 'creates a server, and does not leak connections' do |(proto, kwargs)|\n      buffer = \"\"\n      closed = 0\n      @d.server_create_connection(:s, @port, proto: proto, **kwargs) do |conn|\n        conn.on(:close){|_c| closed += 1 }\n        conn.on(:data) do |d|\n          buffer << d\n        end\n      end\n      3.times do\n        open_client(proto, \"127.0.0.1\", @port) do |sock|\n          sock.puts \"yay\"\n        end\n      end\n      waiting(10){ sleep 0.1 until buffer.bytesize == 12 }\n      waiting(10){ sleep 0.1 until closed == 3 }\n      assert_equal 0, @d.instance_eval{ @_server_connections.size }\n    end\n\n    data(protocols)\n    test 'will refuse more connect requests after stop, but read data from sockets already connected, in non-shared server' do |(proto, kwargs)|\n      connected = false\n      begin\n        open_client(proto, \"127.0.0.1\", @port) do |sock|\n          # expected behavior is connection refused...\n          connected = true\n        end\n      rescue\n      end\n\n      assert_false connected\n\n      received = \"\"\n      @d.server_create_connection(:s, @port, proto: proto, shared: false, **kwargs) do |conn|\n        conn.on(:data) do |data|\n          received << data\n          conn.write \"ack\\n\"\n        end\n      end\n\n      th0 = Thread.new do\n        open_client(proto, \"127.0.0.1\", @port) do |sock|\n          sock.puts \"yay\"\n          sock.readline\n        end\n      end\n\n      value0 = waiting(5){ th0.value }\n      assert_equal \"ack\\n\", value0\n\n      stopped = false\n      sleeping = false\n      ending = false\n\n      th1 = Thread.new do\n        open_client(proto, \"127.0.0.1\", @port) do |sock|\n          sleeping = true\n          sleep 0.1 until stopped\n          sock.puts \"yay\"\n          res = sock.readline\n          ending = true\n          res\n        end\n      end\n\n      sleep 0.1 until sleeping\n\n      @d.stop\n      assert @d.stopped?\n      stopped = true\n\n      sleep 0.1 until ending\n\n      @d.before_shutdown\n      @d.shutdown\n\n      th2 = Thread.new do\n        begin\n          open_client(proto, \"127.0.0.1\", @port) do |sock|\n            sock.puts \"foo\"\n          end\n          false # failed\n        rescue\n          true # success\n        end\n      end\n\n      value1 = waiting(5){ th1.value }\n      value2 = waiting(5){ th2.value }\n\n      assert_equal \"yay\\nyay\\n\", received\n      assert_equal \"ack\\n\", value1\n      assert value2, \"should be truthy value to show connection was correctly refused\"\n    end\n\n    test 'can keep connections alive for tcp/tls if keepalive specified' do\n      # pend \"not implemented yet\"\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_service_discovery.rb",
    "content": "require_relative '../helper'\nrequire 'flexmock/test_unit'\nrequire 'fluent/plugin_helper/service_discovery'\nrequire 'fluent/plugin/output'\n\nclass ServiceDiscoveryHelper < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :service_discovery\n\n    # Make these method public\n    def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)\n      super\n    end\n\n    def discovery_manager\n      super\n    end\n  end\n\n  class DummyPlugin < Fluent::Plugin::TestBase\n    helpers :service_discovery\n\n    def configure(conf)\n      super\n      service_discovery_configure(:service_discovery_helper_test, static_default_service_directive: 'node')\n    end\n\n    def select_service(&block)\n      service_discovery_select_service(&block)\n    end\n\n    # Make these method public\n    def discovery_manager\n      super\n    end\n  end\n\n  setup do\n    @sd_file_dir = File.expand_path('../plugin/data/sd_file', __dir__)\n\n    @d = nil\n  end\n\n  teardown do\n    if @d\n      @d.stop unless @d.stopped?\n      @d.shutdown unless @d.shutdown?\n      @d.after_shutdown unless @d.after_shutdown?\n      @d.close unless @d.closed?\n      @d.terminate unless @d.terminated?\n    end\n  end\n\n  test 'support calling #service_discovery_create_manager and #discovery_manager from plugin' do\n    d = @d = Dummy.new\n\n    d.service_discovery_create_manager(\n      :service_discovery_helper_test,\n      configurations: [{ type: :static, conf: config_element('root', '', {}, [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]) }],\n    )\n\n    assert_true !!d.discovery_manager\n\n    mock.proxy(d.discovery_manager).start.once\n    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never\n\n    d.start\n    d.event_loop_wait_until_start\n\n    services = d.discovery_manager.services\n    assert_equal 1, services.size\n    assert_equal '127.0.0.1', services[0].host\n    assert_equal 1234, services[0].port\n  end\n\n  test 'start discovery manager' do\n    d = @d = DummyPlugin.new\n\n    services = [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]\n    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', {'@type' => 'static'}, services)]))\n\n    assert_true !!d.discovery_manager\n\n    mock.proxy(d.discovery_manager).start.once\n    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never\n\n    d.start\n    d.event_loop_wait_until_start\n\n    assert_equal 1, d.discovery_manager.services.size\n    d.select_service do |serv|\n      assert_equal \"127.0.0.1\", serv.host\n      assert_equal 1234, serv.port\n    end\n  end\n\n  test 'call timer_execute if dynamic configuration' do\n    d = @d = DummyPlugin.new\n    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', { '@type' => 'file', 'path' => File.join(@sd_file_dir, 'config.yml' )})]))\n\n    assert_true !!d.discovery_manager\n    mock.proxy(d.discovery_manager).start.once\n    mock(d).timer_execute(:service_discovery_helper_test, anything).once\n    d.start\n    d.event_loop_wait_until_start\n  end\n\n  test 'exits service discovery instances without any errors' do\n    d = @d = DummyPlugin.new\n    mockv = flexmock('dns_resolver', getaddress: '127.0.0.1')\n              .should_receive(:getresources)\n              .and_return([Resolv::DNS::Resource::IN::SRV.new(1, 10, 8081, 'service1.example.com')])\n              .mock\n    mock(Resolv::DNS).new { mockv }\n\n    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', { '@type' => 'srv', 'service' => 'service1', 'hostname' => 'example.com' })]))\n\n    assert_true !!d.discovery_manager\n    mock.proxy(d.discovery_manager).start.once\n    mock(d).timer_execute(:service_discovery_helper_test, anything).once\n\n    # To avoid clearing `@logs` during `terminate` step\n    # https://github.com/fluent/fluentd/blob/bc78d889f93dad8c2a4e0ad1ca802546185dacba/lib/fluent/test/log.rb#L33\n    mock(d.log).reset.times(3)\n\n    d.start\n    d.event_loop_wait_until_start\n\n    d.stop unless d.stopped?\n    d.shutdown unless d.shutdown?\n    d.after_shutdown unless d.after_shutdown?\n    d.close unless d.closed?\n    d.terminate unless d.terminated?\n\n    assert_false(d.log.out.logs.any? { |e| e.match?(/thread doesn't exit correctly/) })\n  end\n\n  test 'static service discovery will be configured automatically when default service directive is specified' do\n    d = @d = DummyPlugin.new\n\n    nodes = [\n      config_element('node', '', { 'host' => '192.168.0.1', 'port' => '24224' }),\n      config_element('node', '', { 'host' => '192.168.0.2', 'port' => '24224' })\n    ]\n    d.configure(config_element('root', '', {}, nodes))\n\n    assert_true !!d.discovery_manager\n\n    mock.proxy(d.discovery_manager).start.once\n    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never\n\n    d.start\n    d.event_loop_wait_until_start\n\n    assert_equal 2, d.discovery_manager.services.size\n    d.select_service do |serv|\n      assert_equal \"192.168.0.1\", serv.host\n      assert_equal 24224, serv.port\n    end\n    d.select_service do |serv|\n      assert_equal \"192.168.0.2\", serv.host\n      assert_equal 24224, serv.port\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_socket.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/socket'\nrequire 'fluent/plugin/base'\n\nrequire 'socket'\nrequire 'openssl'\n\nclass SocketHelperTest < Test::Unit::TestCase\n  CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/without_ca')\n  CA_CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/with_ca')\n  CERT_CHAINS_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/cert_chains')\n\n  def setup\n    @port = unused_port(protocol: :tcp)\n  end\n\n  def teardown\n    @port = nil\n  end\n\n  class SocketHelperTestPlugin < Fluent::Plugin::TestBase\n    helpers :socket\n  end\n\n  class EchoTLSServer\n    def initialize(port, host: '127.0.0.1', cert_path: nil, private_key_path: nil, ca_path: nil)\n      server = TCPServer.open(host, port)\n      ctx = OpenSSL::SSL::SSLContext.new\n      ctx.cert = OpenSSL::X509::Certificate.new(File.open(cert_path)) if cert_path\n\n      cert_store = OpenSSL::X509::Store.new\n      cert_store.set_default_paths\n      cert_store.add_file(ca_path) if ca_path\n      ctx.cert_store = cert_store\n\n      ctx.key = OpenSSL::PKey::RSA.new(File.open(private_key_path)) if private_key_path\n      ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER\n      ctx.verify_hostname = false\n\n      @server = OpenSSL::SSL::SSLServer.new(server, ctx)\n      @thread = nil\n      @r, @w = IO.pipe\n    end\n\n    def start\n      do_start\n\n      if block_given?\n        begin\n          yield\n          @thread.join(5)\n        ensure\n          stop\n        end\n      end\n    end\n\n    def stop\n      unless @w.closed?\n        @w.write('stop')\n      end\n\n      [@server, @w, @r].each do |s|\n        next if s.closed?\n        s.close\n      end\n\n      @thread.join(5)\n    end\n\n    private\n\n    def do_start\n      @thread = Thread.new(@server) do |s|\n        socks, _, _ = IO.select([s.accept, @r], nil, nil)\n\n        if socks.include?(@r)\n          break\n        end\n\n        sock = socks.first\n        buf = +''\n        loop do\n          b = sock.read_nonblock(1024, nil, exception: false)\n          if b == :wait_readable || b.nil?\n            break\n          end\n          buf << b\n        end\n\n        sock.write(buf)\n        sock.close\n      end\n    end\n  end\n\n  test 'with self-signed cert/key pair' do\n    cert_path = File.join(CERT_DIR, 'cert.pem')\n    private_key_path = File.join(CERT_DIR, 'cert-key.pem')\n\n    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path).start do\n      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_paths: [cert_path])\n      client.write('hello')\n      assert_equal 'hello', client.readpartial(100)\n      client.close\n    end\n  end\n\n  test 'with cert/key signed by self-signed CA' do\n    cert_path = File.join(CA_CERT_DIR, 'cert.pem')\n    private_key_path = File.join(CA_CERT_DIR, 'cert-key.pem')\n\n    ca_cert_path = File.join(CA_CERT_DIR, 'ca-cert.pem')\n\n    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path).start do\n      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_paths: [ca_cert_path])\n      client.write('hello')\n      assert_equal 'hello', client.readpartial(100)\n      client.close\n    end\n  end\n\n  test 'with cert/key signed by self-signed CA in server and client cert chain' do\n    cert_path = File.join(CERT_DIR, 'cert.pem')\n    private_key_path = File.join(CERT_DIR, 'cert-key.pem')\n\n    client_ca_cert_path = File.join(CERT_CHAINS_DIR, 'ca-cert.pem')\n    client_cert_path = File.join(CERT_CHAINS_DIR, 'cert.pem')\n    client_private_key_path = File.join(CERT_CHAINS_DIR, 'cert-key.pem')\n\n    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path, ca_path: client_ca_cert_path).start do\n      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_path: client_cert_path, private_key_path: client_private_key_path, cert_paths: [cert_path])\n      client.write('hello')\n      assert_equal 'hello', client.readpartial(100)\n      client.close\n    end\n  end\n\n  test 'with empty cert file' do\n    cert_path = File.expand_path(File.dirname(__FILE__) + '/data/cert/empty.pem')\n\n    assert_raise Fluent::ConfigError do\n      SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, cert_path: cert_path)\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_storage.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/storage'\nrequire 'fluent/plugin/base'\n\nclass ExampleStorage < Fluent::Plugin::Storage\n  Fluent::Plugin.register_storage('example', self)\n\n  attr_reader :data, :saved, :load_times, :save_times\n\n  def initialize\n    super\n    @data = {}\n    @saved = {}\n    @load_times = 0\n    @save_times = 0\n  end\n  def load\n    @data ||= {}\n    @load_times += 1\n  end\n  def save\n    @saved = @data.dup\n    @save_times += 1\n  end\n  def get(key)\n    @data[key]\n  end\n  def fetch(key, defval)\n    @data.fetch(key, defval)\n  end\n  def put(key, value)\n    @data[key] = value\n  end\n  def delete(key)\n    @data.delete(key)\n  end\n  def update(key, &block)\n    @data[key] = block.call(@data[key])\n  end\n  def close\n    @data = {}\n    super\n  end\n  def terminate\n    @saved = {}\n    @load_times = @save_times = 0\n    super\n  end\nend\n\nclass Example2Storage < ExampleStorage\n  Fluent::Plugin.register_storage('ex2', self)\n  config_param :dummy_path, :string, default: 'dummy'\nend\nclass Example3Storage < ExampleStorage\n  Fluent::Plugin.register_storage('ex3', self)\n  def synchronized?\n    true\n  end\nend\nclass Example4Storage < ExampleStorage\n  Fluent::Plugin.register_storage('ex4', self)\n  def persistent_always?\n    true\n  end\n  def synchronized?\n    true\n  end\nend\n\nclass StorageHelperTest < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :storage\n  end\n\n  class Dummy2 < Fluent::Plugin::TestBase\n    helpers :storage\n    config_section :storage do\n      config_set_default :@type, 'ex2'\n      config_set_default :dummy_path, '/tmp/yay'\n    end\n  end\n\n  setup do\n    @d = nil\n  end\n\n  teardown do\n    if @d\n      @d.stop unless @d.stopped?\n      @d.shutdown unless @d.shutdown?\n      @d.close unless @d.closed?\n      @d.terminate unless @d.terminated?\n    end\n  end\n\n  test 'can be initialized without any storages at first' do\n    d = Dummy.new\n    assert_equal 0, d._storages.size\n  end\n\n  test 'can be configured with hash' do\n    d = Dummy.new\n    d.configure(config_element())\n    conf = { '@type' => 'example' }\n    assert_nothing_raised do\n      d.storage_create(conf: conf)\n    end\n  end\n\n  test 'can override default configuration parameters, but not overwrite whole definition' do\n    d = Dummy.new\n    d.configure(config_element())\n    assert_equal 1, d.storage_configs.size\n    assert_equal 'local', d.storage_configs.first[:@type]\n\n    d = Dummy2.new\n    d.configure(config_element('ROOT', '', {}, [config_element('storage', '', {}, [])]))\n    assert_raise NoMethodError do\n      d.storage\n    end\n    assert_equal 1, d.storage_configs.size\n    assert_equal 'ex2', d.storage_configs.first[:@type]\n    assert_equal '/tmp/yay', d.storage_configs.first.dummy_path\n  end\n\n  test 'creates instance of type specified by conf, or default_type if @type is missing in conf' do\n    d = Dummy2.new\n    d.configure(config_element())\n    i = d.storage_create(conf: config_element('format', '', {'@type' => 'example'}), default_type: 'ex2')\n    assert{ i.is_a?(Fluent::PluginHelper::Storage::SynchronizeWrapper) && i.instance_eval{ @storage }.is_a?(ExampleStorage) }\n\n    d = Dummy2.new\n    d.configure(config_element())\n    i = d.storage_create(conf: nil, default_type: 'ex2')\n    assert{ i.is_a?(Fluent::PluginHelper::Storage::SynchronizeWrapper) && i.instance_eval{ @storage }.is_a?(Example2Storage) }\n  end\n\n  test 'raises config error if config section is specified, but @type is not specified' do\n    d = Dummy2.new\n    d.configure(config_element())\n    assert_raise Fluent::ConfigError.new(\"@type is required in <storage>\") do\n      d.storage_create(conf: config_element('storage', 'foo', {}), default_type: 'ex2')\n    end\n  end\n\n  test 'raises config error if config argument has invalid characters' do\n    d = Dummy.new\n    assert_raise Fluent::ConfigError.new(\"Argument in <storage ARG> uses invalid characters: 'yaa y'\") do\n      d.configure(config_element('root', '', {}, [config_element('storage', 'yaa y', {'@type' => 'local'})]))\n    end\n    d.configure(config_element())\n    assert_raise Fluent::ConfigError.new(\"Argument in <storage ARG> uses invalid characters: 'a,b'\") do\n      d.storage_create(usage: 'a,b', type: 'local')\n    end\n  end\n\n  test 'can be configured without storage sections' do\n    d = Dummy.new\n    assert_nothing_raised do\n      d.configure(config_element())\n    end\n    assert_equal 1, d._storages.size\n  end\n\n  test 'can be configured with a storage section' do\n    d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', '', {'@type' => 'example'})\n      ])\n    assert_nothing_raised do\n      d.configure(conf)\n    end\n    assert_equal 1, d._storages.size\n    assert{ d._storages.values.all?{ |s| !s.running } }\n  end\n\n  test 'can be configured with 2 or more storage sections with different usages with each other' do\n    d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'default', {'@type' => 'example'}),\n        config_element('storage', 'extra', {'@type' => 'ex2', 'dummy_path' => 'v'}),\n      ])\n    assert_nothing_raised do\n      d.configure(conf)\n    end\n    assert_equal 2, d._storages.size\n    assert{ d._storages.values.all?{ |s| !s.running } }\n  end\n\n  test 'cannot be configured with 2 storage sections with same usage' do\n    d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'default', {'@type' => 'example'}),\n        config_element('storage', 'extra', {'@type' => 'ex2', 'dummy_path' => 'v'}),\n        config_element('storage', 'extra', {'@type' => 'ex2', 'dummy_path' => 'v2'}),\n      ])\n    assert_raises Fluent::ConfigError do\n      d.configure(conf)\n    end\n  end\n\n  test 'creates a storage plugin instance which is already configured without usage' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', '', {'@type' => 'example'})\n      ])\n    d.configure(conf)\n    d.start\n\n    s = d.storage_create\n    assert{ s.implementation.is_a? ExampleStorage }\n  end\n\n  test 'creates a storage plugin instance which is already configured with usage' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'example'})\n      ])\n    d.configure(conf)\n    d.start\n\n    s = d.storage_create(usage: 'mydata')\n    assert{ s.implementation.is_a? ExampleStorage }\n  end\n\n  test 'creates a storage plugin without configurations' do\n    @d = d = Dummy.new\n    d.configure(config_element())\n    d.start\n\n    s = d.storage_create(usage: 'mydata', type: 'example', conf: config_element('storage', 'mydata'))\n    assert{ s.implementation.is_a? ExampleStorage }\n  end\n\n  test 'creates 2 or more storage plugin instances' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'example'}),\n        config_element('storage', 'secret', {'@type' => 'ex2', 'dummy_path' => 'yay!'}),\n      ])\n    d.configure(conf)\n    d.start\n\n    s1 = d.storage_create(usage: 'mydata')\n    s2 = d.storage_create(usage: 'secret')\n    assert{ s1.implementation.is_a? ExampleStorage }\n    assert{ s2.implementation.is_a? Example2Storage }\n    assert_equal 'yay!', s2.implementation.dummy_path\n  end\n\n  test 'creates wrapped instances for non-synchronized plugin in default' do # and check operations\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'example'})\n      ])\n    d.configure(conf)\n    d.start\n\n    s = d.storage_create(usage: 'mydata')\n    assert !s.implementation.synchronized?\n    assert{ s.is_a? Fluent::PluginHelper::Storage::SynchronizeWrapper }\n    assert s.synchronized?\n    assert s.autosave\n    assert s.save_at_shutdown\n\n    assert_nil s.get('key')\n    assert_equal 'value', s.put('key', 'value')\n    assert_equal 'value', s.fetch('key', 'v1')\n    assert_equal 'v2', s.update('key'){|v| v[0] + '2' }\n    assert_equal 'v2', s.get('key')\n    assert_equal 'v2', s.delete('key')\n  end\n\n  test 'creates wrapped instances for non-persistent plugins when configured as persistent' do # and check operations\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'example', 'persistent' => 'true'})\n      ])\n    d.configure(conf)\n    d.start\n\n    s = d.storage_create(usage: 'mydata')\n    assert !s.implementation.persistent_always?\n    assert{ s.is_a? Fluent::PluginHelper::Storage::PersistentWrapper }\n    assert s.persistent\n    assert s.persistent_always?\n    assert s.synchronized?\n    assert !s.autosave\n    assert s.save_at_shutdown\n\n    assert_nil s.get('key')\n    assert_equal 'value', s.put('key', 'value')\n    assert_equal 'value', s.fetch('key', 'v1')\n    assert_equal 'v2', s.update('key'){|v| v[0] + '2' }\n    assert_equal 'v2', s.get('key')\n    assert_equal 'v2', s.delete('key')\n  end\n\n  test 'creates bare instances for synchronized plugin in default' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'ex3'})\n      ])\n    d.configure(conf)\n    d.start\n\n    s = d.storage_create(usage: 'mydata')\n    assert s.implementation.synchronized?\n    assert{ s.is_a? Example3Storage }\n    assert s.synchronized?\n  end\n\n  test 'creates bare instances for persistent-always plugin when configured as persistent' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'ex4', 'persistent' => 'true'})\n      ])\n    d.configure(conf)\n    d.start\n\n    s = d.storage_create(usage: 'mydata')\n    assert s.implementation.persistent_always?\n    assert{ s.is_a? Example4Storage }\n    assert s.persistent\n    assert s.persistent_always?\n    assert s.synchronized?\n  end\n\n  test 'does not execute timer if autosave is not specified' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'example', 'autosave' => 'false'})\n      ])\n    d.configure(conf)\n    d.start\n\n    d.storage_create(usage: 'mydata')\n    assert_equal 0, d._timers.size\n  end\n\n  test 'executes timer if autosave is specified and plugin is not persistent' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'example', 'autosave_interval' => '1s', 'persistent' => 'false'})\n      ])\n    d.configure(conf)\n    d.start\n\n    d.storage_create(usage: 'mydata')\n    assert_equal 1, d._timers.size\n  end\n\n  test 'executes timer for autosave, which calls #save periodically' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'example', 'autosave_interval' => '1s', 'persistent' => 'false'})\n      ])\n    d.configure(conf)\n    d.start\n\n    s = d.storage_create(usage: 'mydata')\n    assert_equal 1, d._timers.size\n    timeout = Time.now + 3\n    while Time.now < timeout\n      s.put('k', 'v')\n      sleep 0.2\n    end\n\n    d.stop\n    assert{ s.implementation.save_times > 0 }\n  end\n\n  test 'saves data for each operations if plugin storage is configured as persistent, and wrapped' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'example', 'persistent' => 'true'})\n      ])\n    d.configure(conf)\n    d.start\n    s = d.storage_create(usage: 'mydata')\n    assert_equal 1, s.implementation.load_times\n    assert_equal 0, s.implementation.save_times\n\n    s.get('k1')\n    assert_equal 2, s.implementation.load_times\n    assert_equal 0, s.implementation.save_times\n\n    s.put('k1', 'v1')\n    assert_equal 3, s.implementation.load_times\n    assert_equal 1, s.implementation.save_times\n\n    s.fetch('k2', 'v2')\n    assert_equal 4, s.implementation.load_times\n    assert_equal 1, s.implementation.save_times\n\n    s.delete('k1')\n    assert_equal 5, s.implementation.load_times\n    assert_equal 2, s.implementation.save_times\n  end\n\n  test 'stops timer for autosave by #stop, calls #save by #shutdown if save_at_shutdown is specified' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'example', 'save_at_shutdown' => 'true'})\n      ])\n    d.configure(conf)\n\n    assert !d.timer_running?\n\n    d.start\n\n    assert d.timer_running?\n\n    s = d.storage_create(usage: 'mydata')\n    assert s.autosave\n    assert_equal 1, d._timers.size\n\n    d.stop\n\n    assert !d.timer_running?\n\n    assert_equal 1, s.implementation.load_times\n    assert_equal 0, s.implementation.save_times\n\n    d.before_shutdown\n\n    d.shutdown\n\n    assert_equal 1, s.implementation.load_times\n    assert_equal 1, s.implementation.save_times\n  end\n\n  test 'calls #close and #terminate for all plugin instances by #close/#shutdown' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [\n        config_element('storage', 'mydata', {'@type' => 'example', 'autosave' => 'false', 'save_at_shutdown' => 'true'})\n      ])\n    d.configure(conf)\n    d.start\n    s = d.storage_create(usage: 'mydata')\n\n    s.put('k1', 'v1')\n    s.put('k2', 2)\n    s.put('k3', true)\n\n    d.stop\n\n    assert_equal 3, s.implementation.data.size\n    assert_equal 0, s.implementation.saved.size\n\n    d.shutdown\n\n    assert_equal 3, s.implementation.data.size\n    assert_equal 3, s.implementation.saved.size\n\n    d.close\n\n    assert_equal 0, s.implementation.data.size\n    assert_equal 3, s.implementation.saved.size\n\n    d.shutdown\n\n    assert_equal 0, s.implementation.data.size\n    assert_equal 0, s.implementation.saved.size\n  end\n\n  test 'calls lifecycle methods for all plugin instances via owner plugin' do\n    @d = d = Dummy.new\n    conf = config_element('ROOT', '', {}, [ config_element('storage', '', {'@type' => 'example'}), config_element('storage', 'e2', {'@type' => 'example'}) ])\n    d.configure(conf)\n    d.start\n\n    i1 = d.storage_create(usage: '')\n    i2 = d.storage_create(usage: 'e2')\n    i3 = d.storage_create(usage: 'e3', type: 'ex2')\n\n    assert i1.started?\n    assert i2.started?\n    assert i3.started?\n\n    assert !i1.stopped?\n    assert !i2.stopped?\n    assert !i3.stopped?\n\n    d.stop\n\n    assert i1.stopped?\n    assert i2.stopped?\n    assert i3.stopped?\n\n    assert !i1.before_shutdown?\n    assert !i2.before_shutdown?\n    assert !i3.before_shutdown?\n\n    d.before_shutdown\n\n    assert i1.before_shutdown?\n    assert i2.before_shutdown?\n    assert i3.before_shutdown?\n\n    assert !i1.shutdown?\n    assert !i2.shutdown?\n    assert !i3.shutdown?\n\n    d.shutdown\n\n    assert i1.shutdown?\n    assert i2.shutdown?\n    assert i3.shutdown?\n\n    assert !i1.after_shutdown?\n    assert !i2.after_shutdown?\n    assert !i3.after_shutdown?\n\n    d.after_shutdown\n\n    assert i1.after_shutdown?\n    assert i2.after_shutdown?\n    assert i3.after_shutdown?\n\n    assert !i1.closed?\n    assert !i2.closed?\n    assert !i3.closed?\n\n    d.close\n\n    assert i1.closed?\n    assert i2.closed?\n    assert i3.closed?\n\n    assert !i1.terminated?\n    assert !i2.terminated?\n    assert !i3.terminated?\n\n    d.terminate\n\n    assert i1.terminated?\n    assert i2.terminated?\n    assert i3.terminated?\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_thread.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/thread'\nrequire 'fluent/plugin/base'\nrequire 'timeout'\n\nclass ThreadTest < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :thread\n    def configure(conf)\n      super\n      @_thread_wait_seconds = 0.1\n      self\n    end\n  end\n\n  test 'can be instantiated to be able to create threads' do\n    d1 = Dummy.new\n    assert d1.respond_to?(:thread_current_running?)\n    assert d1.respond_to?(:thread_create)\n    assert d1.respond_to?(:_threads)\n    assert !d1.thread_current_running?\n    assert d1._threads.empty?\n  end\n\n  test 'can be configured' do\n    d1 = Dummy.new\n    assert_nothing_raised do\n      d1.configure(config_element())\n    end\n    assert d1.plugin_id\n    assert d1.log\n  end\n\n  test 'can create thread after prepared' do\n    d1 = Dummy.new\n    d1.configure(config_element())\n    d1.start\n\n    m1 = Mutex.new\n    m2 = Mutex.new\n\n    m1.lock\n    thread_run = false\n\n    Timeout.timeout(10) do\n      t = d1.thread_create(:test1) do\n        m2.lock\n\n        assert !d1._threads.empty? # this must be true always\n        assert d1.thread_current_running?\n\n        thread_run = true\n        m2.unlock\n        m1.lock\n      end\n      Thread.pass until m2.locked? || thread_run\n\n      m2.lock; m2.unlock\n      assert_equal 1, d1._threads.size\n\n      assert_equal :test1, t[:_fluentd_plugin_helper_thread_title]\n      assert t[:_fluentd_plugin_helper_thread_running]\n      assert !d1._threads.empty?\n\n      m1.unlock\n\n      while t[:_fluentd_plugin_helper_thread_running]\n        Thread.pass\n      end\n    end\n\n    assert d1._threads.empty?\n\n    d1.stop; d1.shutdown; d1.close; d1.terminate\n  end\n\n  test 'can wait until all threads start' do\n    d1 = Dummy.new.configure(config_element()).start\n    ary = []\n    d1.thread_create(:t1) do\n      ary << 1\n    end\n    d1.thread_create(:t2) do\n      ary << 2\n    end\n    d1.thread_create(:t3) do\n      ary << 3\n    end\n    Timeout.timeout(10) do\n      d1.thread_wait_until_start\n    end\n    assert_equal [1,2,3], ary.sort\n\n    d1.stop; d1.shutdown; d1.close; d1.terminate\n  end\n\n  test 'can stop threads which is watching thread_current_running?, and then close it' do\n    d1 = Dummy.new.configure(config_element()).start\n\n    m1 = Mutex.new\n    thread_in_run = false\n    Timeout.timeout(10) do\n      t = d1.thread_create(:test2) do\n        thread_in_run = true\n        m1.lock\n        while d1.thread_current_running?\n          Thread.pass\n        end\n        thread_in_run = false\n        m1.unlock\n      end\n      Thread.pass until m1.locked?\n\n      assert thread_in_run\n      assert !d1._threads.empty?\n\n      d1.stop\n      Thread.pass while m1.locked?\n      assert !t[:_fluentd_plugin_helper_thread_running]\n      assert t.stop?\n    end\n\n    assert d1._threads.empty?\n\n    d1.stop; d1.shutdown; d1.close; d1.terminate\n  end\n\n  test 'can terminate threads forcedly which is running forever' do\n    d1 = Dummy.new.configure(config_element()).start\n\n    m1 = Mutex.new\n    thread_in_run = false\n    Timeout.timeout(10) do\n      t = d1.thread_create(:test2) do\n        thread_in_run = true\n        m1.lock\n        while true\n          Thread.pass\n        end\n        thread_in_run = false\n      end\n      Thread.pass until m1.locked?\n\n      assert thread_in_run\n      assert !d1._threads.empty?\n\n      d1.stop\n      assert !t[:_fluentd_plugin_helper_thread_running]\n      assert t.alive?\n\n      d1.shutdown\n      assert t.alive?\n      assert !d1._threads.empty?\n\n      d1.close\n      assert t.alive?\n      assert !d1._threads.empty?\n\n      d1.terminate\n      assert t.stop?\n      assert d1._threads.empty?\n    end\n  end\nend\n"
  },
  {
    "path": "test/plugin_helper/test_timer.rb",
    "content": "require_relative '../helper'\nrequire 'fluent/plugin_helper/timer'\nrequire 'fluent/plugin/base'\n\nclass TimerTest < Test::Unit::TestCase\n  class Dummy < Fluent::Plugin::TestBase\n    helpers :timer\n  end\n\n  test 'can be instantiated under state that timer is not running' do\n    d1 = Dummy.new\n    assert d1.respond_to?(:timer_running?)\n    assert !d1.timer_running?\n  end\n\n  test 'can be configured' do\n    d1 = Dummy.new\n    assert_nothing_raised do\n      d1.configure(config_element())\n    end\n    assert d1.plugin_id\n    assert d1.log\n  end\n\n  test 'can start timers by start' do\n    d1 = Dummy.new\n    d1.configure(config_element())\n    assert !d1.timer_running?\n    d1.start\n    assert d1.timer_running?\n\n    counter = 0\n    d1.timer_execute(:test, 1) do\n      counter += 1\n    end\n\n    sleep 2\n\n    d1.stop\n    assert !d1.timer_running?\n\n    assert{ counter >= 1 && counter <= 2 }\n\n    d1.shutdown; d1.close; d1.terminate\n  end\n\n  test 'can run many timers' do\n    d1 = Dummy.new\n    d1.configure(config_element())\n    d1.start\n\n    counter1 = 0\n    counter2 = 0\n\n    d1.timer_execute(:t1, 0.2) do\n      counter1 += 1\n    end\n    d1.timer_execute(:t2, 0.2) do\n      counter2 += 1\n    end\n\n    sleep 1\n    d1.stop\n\n    assert{ counter1 >= 4 && counter1 <= 5 }\n    assert{ counter2 >= 4 && counter2 <= 5 }\n\n    d1.shutdown; d1.close; d1.terminate\n  end\n\n  test 'aborts timer which raises exceptions' do\n    d1 = Dummy.new\n    d1.configure(config_element())\n    d1.start\n\n    counter1 = 0\n    counter2 = 0\n\n    d1.timer_execute(:t1, 0.2) do\n      counter1 += 1\n    end\n    d1.timer_execute(:t2, 0.2) do\n      raise \"abort!!!!!!\" if counter2 > 1\n      counter2 += 1\n    end\n\n    sleep 1\n    d1.stop\n\n    assert{ counter1 >= 4 && counter1 <= 5 }\n    assert{ counter2 == 2 }\n    msg = \"Unexpected error raised. Stopping the timer. title=:t2\"\n    assert(d1.log.out.logs.any?{|line| line.include?(\"[error]:\") && line.include?(msg) && line.include?(\"abort!!!!!!\") })\n    assert(d1.log.out.logs.any?{|line| line.include?(\"[error]:\") && line.include?(\"Timer detached. title=:t2\") })\n\n    d1.shutdown; d1.close; d1.terminate\n  end\n\n  test 'can run at once' do\n    d1 = Dummy.new\n    d1.configure(config_element())\n    assert !d1.timer_running?\n    d1.start\n    assert d1.timer_running?\n\n    waiting_assertion = true\n    waiting_timer = true\n    counter = 0\n    d1.timer_execute(:test, 1, repeat: false) do\n      sleep(0.1) while waiting_assertion\n      counter += 1\n      waiting_timer = false\n    end\n\n    watchers = d1._event_loop.watchers.reject {|w| w.is_a?(Fluent::PluginHelper::EventLoop::DefaultWatcher) }\n    assert_equal(1, watchers.size)\n    assert(watchers.first.attached?)\n\n    waiting_assertion = false\n    sleep(0.1) while waiting_timer\n\n    assert_equal(1, counter)\n    waiting(4){ sleep 0.1 while watchers.first.attached? }\n    assert_false(watchers.first.attached?)\n    watchers = d1._event_loop.watchers.reject {|w| w.is_a?(Fluent::PluginHelper::EventLoop::DefaultWatcher) }\n    assert_equal(0, watchers.size)\n\n    d1.shutdown; d1.close; d1.terminate\n  end\nend\n"
  },
  {
    "path": "test/scripts/exec_script.rb",
    "content": "require 'json'\nrequire 'msgpack'\n\ndef gen_tsv(time)\n  \"#{time}\\ttag1\\tok\"\nend\n\ndef gen_json(time)\n  {'tag' => 'tag1', 'time' => time, 'k1' => 'ok'}.to_json\nend\n\ndef gen_msgpack(time)\n  {'tagger' => 'tag1', 'datetime' => time, 'k1' => 'ok'}.to_msgpack\nend\n\ndef gen_raw_string(time)\n  \"#{time} hello\"\nend\n\ntime = ARGV.first\ntime = Integer(time) rescue time\n\ncase ARGV.last.to_i\nwhen 0\n  puts gen_tsv(time)\nwhen 1\n  puts gen_json(time)\nwhen 2\n  print gen_msgpack(time)\nwhen 3\n  print gen_raw_string(time)\nend\n"
  },
  {
    "path": "test/scripts/fluent/plugin/formatter1/formatter_test1.rb",
    "content": "module Fluent\n  Plugin.register_formatter(\n    'test1',\n    Proc.new { |tag, time, record|\n      \"#{tag}:#{time}:#{record.size}\"\n    })\nend\n"
  },
  {
    "path": "test/scripts/fluent/plugin/formatter2/formatter_test2.rb",
    "content": "module Fluent\n  Plugin.register_formatter(\n    'test2',\n    Proc.new { |tag, time, record|\n      \"#{tag}:#{time}:#{record.size}\"\n    })\nend\n"
  },
  {
    "path": "test/scripts/fluent/plugin/formatter_known.rb",
    "content": "module Fluent\n  TextFormatter.register_template('known_old', Proc.new { |tag, time, record|\n      \"#{tag}:#{time}:#{record.size}\"\n    })\n  Plugin.register_formatter('known', Proc.new { |tag, time, record|\n      \"#{tag}:#{time}:#{record.size}\"\n    })\nend\n"
  },
  {
    "path": "test/scripts/fluent/plugin/out_test.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nrequire 'fluent/plugin/output'\nrequire 'fluent/event'\n\nmodule Fluent::Plugin\n  class TestOutput < Output\n    Fluent::Plugin.register_output('test', self)\n\n    config_param :name, :string\n\n    config_section :buffer do\n      config_set_default :chunk_keys, ['tag']\n    end\n\n    def initialize\n      super\n      @emit_streams = []\n    end\n\n    attr_reader :emit_streams\n\n    def emits\n      all = []\n      @emit_streams.each {|tag,events|\n        events.each {|time,record|\n          all << [tag, time, record]\n        }\n      }\n      all\n    end\n\n    def events\n      all = []\n      @emit_streams.each {|tag,events|\n        all.concat events\n      }\n      all\n    end\n\n    def records\n      all = []\n      @emit_streams.each {|tag,events|\n        events.each {|time,record|\n          all << record\n        }\n      }\n      all\n    end\n\n    def prefer_buffered_processing\n      false\n    end\n\n    def process(tag, es)\n      @emit_streams << [tag, es.to_a]\n    end\n\n    def write(chunk)\n      es = Fluent::ArrayEventStream.new\n      chunk.each do |time, record|\n        es.add(time, record)\n      end\n      @emit_streams << [tag, es]\n    end\n  end\nend\n"
  },
  {
    "path": "test/scripts/fluent/plugin/out_test2.rb",
    "content": "#\n# Fluentd\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n#\n\nmodule Fluent::Plugin\n  class Test2Output < Output\n    Fluent::Plugin.register_output('test2', self)\n\n    helpers :event_emitter\n\n    config_param :name, :string\n\n    config_section :buffer do\n      config_set_default :chunk_keys, ['tag']\n    end\n\n    def initialize\n      super\n      @emit_streams = []\n    end\n\n    attr_reader :emit_streams\n\n    def emits\n      all = []\n      @emit_streams.each {|tag,events|\n        events.each {|time,record|\n          all << [tag, time, record]\n        }\n      }\n      all\n    end\n\n    def events\n      all = []\n      @emit_streams.each {|tag,events|\n        all.concat events\n      }\n      all\n    end\n\n    def records\n      all = []\n      @emit_streams.each {|tag,events|\n        events.each {|time,record|\n          all << record\n        }\n      }\n      all\n    end\n\n    def prefer_buffered_processing\n      false\n    end\n\n    def process(tag, es)\n      @emit_streams << [tag, es.to_a]\n    end\n\n    def write(chunk)\n      es = Fluent::ArrayEventStream.new\n      chunk.each do |time, record|\n        es.add(time, record)\n      end\n      @emit_streams << [tag, es]\n    end\n  end\nend\n"
  },
  {
    "path": "test/scripts/fluent/plugin/parser_known.rb",
    "content": "module Fluent\n  TextParser.register_template('known_old', /^(?<message>.*)$/)\n  Plugin.register_parser('known', /^(?<message>.*)$/)\nend\n"
  },
  {
    "path": "test/scripts/windows_service_test.ps1",
    "content": "$ErrorActionPreference = \"Stop\"\r\nSet-PSDebug -Trace 1\r\n\r\n$default_conf_path = (Resolve-Path fluent.conf).Path\r\n$current_path = (Get-Location).Path\r\n$log_path = \"$current_path/fluentd.log\"\r\n\r\nruby bin/fluentd --reg-winsvc i --reg-winsvc-fluentdopt \"-c '$default_conf_path' -o '$log_path'\"\r\n\r\n# Test: must not start automatically\r\nif ((Get-Service fluentdwinsvc).Status -ne \"Stopped\") {\r\n    Write-Error \"The service should not start automatically.\"\r\n}\r\n\r\nStart-Service fluentdwinsvc\r\nStart-Sleep 30\r\n\r\n# Test: the service should be running after started\r\nif ((Get-Service fluentdwinsvc).Status -ne \"Running\") {\r\n    Write-Error \"The service should be running after started.\"\r\n}\r\n\r\n# Test: no warn/error/fatal logs\r\nGet-ChildItem \"*.log\" | %{\r\n    Get-Content $_\r\n    if (Select-String -Path $_ -Pattern \"[warn]\", \"[error]\", \"[fatal]\" -SimpleMatch -Quiet) {\r\n        Select-String -Path $_ -Pattern \"[warn]\", \"[error]\", \"[fatal]\" -SimpleMatch\r\n        Write-Error \"There are abnormal level logs in ${_}:\"\r\n    }\r\n}\r\n\r\nStop-Service fluentdwinsvc\r\nStart-Sleep 10 # Somehow it is possible that some processes stay alive for a while. (This could be not good behavior...)\r\n\r\n# Test: status after stopped\r\nif ((Get-Service fluentdwinsvc).Status -ne \"Stopped\") {\r\n    Write-Error \"The service should be in 'Stopped' status after stopped.\"\r\n}\r\n# Test: all Ruby processes should stop\r\n$ruby_processes = Get-Process -name ruby -ErrorAction SilentlyContinue\r\nif ($ruby_processes.Count -ne 0) {\r\n    Write-Output $ruby_processes\r\n    Write-Error \"All Ruby processes should stop.\"\r\n}\r\n\r\n# Test: service should stop when the supervisor fails to launch\r\n# https://github.com/fluent/fluentd/pull/4909\r\n$test_setting = @'\r\n<source>\r\n  @type sample\r\n  @id DUPLICATED_ID\r\n  tag test\r\n</source>\r\n<match test>\r\n  @type stdout\r\n  @id DUPLICATED_ID\r\n</match>\r\n'@\r\nAdd-Content -Path \"duplicated_id.conf\" -Encoding UTF8 -Value $test_setting\r\nruby bin/fluentd --reg-winsvc-fluentdopt \"-c '$current_path/duplicated_id.conf' -o '$log_path'\"\r\nStart-Service fluentdwinsvc\r\nStart-Sleep 30\r\nif ((Get-Service fluentdwinsvc).Status -ne \"Stopped\") {\r\n    Write-Error \"The service should be in 'Stopped' status when the supervisor fails to launch.\"\r\n}\r\n$ruby_processes = Get-Process -name ruby -ErrorAction SilentlyContinue\r\nif ($ruby_processes.Count -ne 0) {\r\n    Write-Output $ruby_processes\r\n    Write-Error \"All Ruby processes should stop.\"\r\n}\r\n\r\nruby bin/fluentd --reg-winsvc u\r\nRemove-Item $log_path\r\n"
  },
  {
    "path": "test/test_capability.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/test'\nrequire 'fluent/capability'\n\nclass FluentCapabilityTest < ::Test::Unit::TestCase\n  setup do\n    @capability = Fluent::Capability.new(:current_process)\n    omit \"Fluent::Capability class is not usable on this environment\" unless @capability.usable?\n  end\n\n  sub_test_case \"check capability\" do\n    test \"effective\" do\n      @capability.clear(:both)\n      assert_true @capability.update(:add, :effective, :dac_read_search)\n      assert_equal CapNG::Result::PARTIAL, @capability.have_capabilities?(:caps)\n      assert_nothing_raised do\n        @capability.apply(:caps)\n      end\n      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:bounds)\n      assert_true @capability.have_capability?(:effective, :dac_read_search)\n      assert_false @capability.have_capability?(:inheritable, :dac_read_search)\n      assert_false @capability.have_capability?(:permitted, :dac_read_search)\n    end\n\n    test \"inheritable\" do\n      @capability.clear(:both)\n      capabilities = [:chown, :dac_override]\n      assert_equal [true, true], @capability.update(:add, :inheritable, capabilities)\n      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:caps)\n      assert_nothing_raised do\n        @capability.apply(:caps)\n      end\n      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:bounds)\n      capabilities.each do |capability|\n        assert_false @capability.have_capability?(:effective, capability)\n        assert_true @capability.have_capability?(:inheritable, capability)\n        assert_false @capability.have_capability?(:permitted, capability)\n      end\n    end\n\n    test \"permitted\" do\n      @capability.clear(:both)\n      capabilities = [:fowner, :fsetid, :kill]\n      assert_equal [true, true, true], @capability.update(:add, :permitted, capabilities)\n      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:caps)\n      assert_nothing_raised do\n        @capability.apply(:caps)\n      end\n      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:bounds)\n      capabilities.each do |capability|\n        assert_false @capability.have_capability?(:effective, capability)\n        assert_false @capability.have_capability?(:inheritable, capability)\n        assert_true @capability.have_capability?(:permitted, capability)\n      end\n    end\n\n    test \"effective/inheritable/permitted\" do\n      @capability.clear(:both)\n      capabilities = [:setpcap, :net_admin, :net_raw, :sys_boot, :sys_time]\n      update_type = CapNG::Type::EFFECTIVE | CapNG::Type::INHERITABLE | CapNG::Type::PERMITTED\n      assert_equal [true, true, true, true, true], @capability.update(:add, update_type, capabilities)\n      assert_equal CapNG::Result::PARTIAL, @capability.have_capabilities?(:caps)\n      assert_nothing_raised do\n        @capability.apply(:caps)\n      end\n      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:bounds)\n      capabilities.each do |capability|\n        assert_true @capability.have_capability?(:effective, capability)\n        assert_true @capability.have_capability?(:inheritable, capability)\n        assert_true @capability.have_capability?(:permitted, capability)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_clock.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/clock'\n\nrequire 'timecop'\n\nclass ClockTest < ::Test::Unit::TestCase\n  teardown do\n    Fluent::Clock.return # call it always not to affect other tests\n  end\n\n  sub_test_case 'without any pre-operation' do\n    test 'clock can provides incremental floating point number based on second' do\n      c1 = Fluent::Clock.now\n      assert_kind_of Float, c1\n      sleep 1.1\n      c2 = Fluent::Clock.now\n      assert{ c2 >= c1 + 1.0 && c2 < c1 + 9.0 } # if clock returns deci-second (fantastic!), c2 should be larger than c1 + 10\n    end\n\n    test 'clock value will proceed even if timecop freezes Time' do\n      Timecop.freeze(Time.now) do\n        c1 = Fluent::Clock.now\n        assert_kind_of Float, c1\n        sleep 1.1\n        c2 = Fluent::Clock.now\n        assert{ c2 >= c1 + 1.0 && c2 < c1 + 9.0 }\n      end\n    end\n  end\n\n  sub_test_case 'using #freeze without any arguments' do\n    test 'Clock.freeze without arguments freezes clock with current clock value' do\n      c0 = Fluent::Clock.now\n      Fluent::Clock.freeze\n      c1 = Fluent::Clock.now\n      Fluent::Clock.return\n      c2 = Fluent::Clock.now\n      assert{ c0 <= c1 && c1 <= c2 }\n    end\n\n    test 'Clock.return raises an error if it is called in block' do\n      assert_raise RuntimeError.new(\"invalid return while running code in blocks\") do\n        Fluent::Clock.freeze do\n          Fluent::Clock.return\n        end\n      end\n    end\n  end\n\n  sub_test_case 'using #freeze with clock value' do\n    test 'Clock.now always returns frozen time until #return called' do\n      c0 = Fluent::Clock.now\n      Fluent::Clock.freeze(c0)\n      assert_equal c0, Fluent::Clock.now\n      sleep 0.5\n      assert_equal c0, Fluent::Clock.now\n      sleep 0.6\n      assert_equal c0, Fluent::Clock.now\n\n      Fluent::Clock.return\n      c1 = Fluent::Clock.now\n      assert{ c1 >= c0 + 1.0 }\n    end\n\n    test 'Clock.now returns frozen time in the block argument of #freeze' do\n      c0 = Fluent::Clock.now\n      Fluent::Clock.freeze(c0) do\n        assert_equal c0, Fluent::Clock.now\n        sleep 0.5\n        assert_equal c0, Fluent::Clock.now\n        sleep 0.6\n        assert_equal c0, Fluent::Clock.now\n      end\n      c1 = Fluent::Clock.now\n      assert{ c1 >= c0 + 1.0 }\n    end\n\n    test 'Clock.now returns unfrozen value after jumping out from block by raising errors' do\n      c0 = Fluent::Clock.now\n      rescued_error = nil\n      begin\n        Fluent::Clock.freeze(c0) do\n          assert_equal c0, Fluent::Clock.now\n          sleep 0.5\n          assert_equal c0, Fluent::Clock.now\n          sleep 0.6\n          assert_equal c0, Fluent::Clock.now\n          raise \"bye!\"\n        end\n      rescue => e\n        rescued_error = e\n      end\n      assert rescued_error # ensure to rescue an error\n      c1 = Fluent::Clock.now\n      assert{ c1 >= c0 + 1.0 }\n    end\n\n    test 'Clock.return cancels all Clock.freeze effects by just once' do\n      c0 = Fluent::Clock.now\n      sleep 0.1\n      c1 = Fluent::Clock.now\n      sleep 0.1\n      c2 = Fluent::Clock.now\n      Fluent::Clock.freeze(c0)\n      sleep 0.1\n      assert_equal c0, Fluent::Clock.now\n      Fluent::Clock.freeze(c1)\n      sleep 0.1\n      assert_equal c1, Fluent::Clock.now\n      Fluent::Clock.freeze(c2)\n      sleep 0.1\n      assert_equal c2, Fluent::Clock.now\n\n      Fluent::Clock.return\n      assert{ Fluent::Clock.now > c2 }\n    end\n\n    test 'Clock.freeze allows nested blocks by itself' do\n      c0 = Fluent::Clock.now\n      sleep 0.1\n      c1 = Fluent::Clock.now\n      sleep 0.1\n      c2 = Fluent::Clock.now\n      Fluent::Clock.freeze(c0) do\n        sleep 0.1\n        assert_equal c0, Fluent::Clock.now\n        Fluent::Clock.freeze(c1) do\n          sleep 0.1\n          assert_equal c1, Fluent::Clock.now\n          Fluent::Clock.freeze(c2) do\n            sleep 0.1\n            assert_equal c2, Fluent::Clock.now\n          end\n          assert_equal c1, Fluent::Clock.now\n        end\n        assert_equal c0, Fluent::Clock.now\n      end\n      assert{ Fluent::Clock.now > c0 }\n    end\n  end\n\n  sub_test_case 'using #freeze with Time argument' do\n    test 'Clock.freeze returns the clock value which should be produced when the time is at the specified time' do\n      c0 = Fluent::Clock.now\n      t0 = Time.now\n      t1 = t0 - 30\n      assert_kind_of Time, t1\n      t2 = t0 + 30\n      assert_kind_of Time, t2\n\n      # 31 is for error of floating point value\n      Fluent::Clock.freeze(t1) do\n        c1 = Fluent::Clock.now\n        assert{ c1 >= c0 - 31 && c1 <= c0 - 31 + 10 } # +10 is for threading schedule error\n      end\n\n      # 29 is for error of floating point value\n      Fluent::Clock.freeze(t2) do\n        c2 = Fluent::Clock.now\n        assert{ c2 >= c0 + 29 && c2 <= c0 + 29 + 10 } # +10 is for threading schedule error\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_config.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/config'\nrequire 'fluent/config/parser'\nrequire 'fluent/supervisor'\nrequire 'fluent/load'\nrequire 'fileutils'\n\nclass ConfigTest < Test::Unit::TestCase\n  include Fluent\n\n  TMP_DIR = File.dirname(__FILE__) + \"/tmp/config#{ENV['TEST_ENV_NUMBER']}\"\n\n  def read_config(path, use_yaml: false)\n    path = File.expand_path(path)\n    if use_yaml\n      context = Kernel.binding\n\n      s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path))\n      Fluent::Config::YamlParser::Parser.new(s).build.to_element\n    else\n      File.open(path) { |io|\n        Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))\n      }\n    end\n  end\n\n  def prepare_config\n    write_config \"#{TMP_DIR}/config_test_1.conf\", %[\n      k1 root_config\n      include dir/config_test_2.conf  #\n      include #{TMP_DIR}/config_test_4.conf\n      include file://#{TMP_DIR}/config_test_5.conf\n      <include config.d/*.conf />\n    ]\n    write_config \"#{TMP_DIR}/dir/config_test_2.conf\", %[\n      k2 relative_path_include\n      include ../config_test_3.conf\n    ]\n    write_config \"#{TMP_DIR}/config_test_3.conf\", %[\n      k3 relative_include_in_included_file\n    ]\n    write_config \"#{TMP_DIR}/config_test_4.conf\", %[\n      k4 absolute_path_include\n    ]\n    write_config \"#{TMP_DIR}/config_test_5.conf\", %[\n      k5 uri_include\n    ]\n    write_config \"#{TMP_DIR}/config.d/config_test_6.conf\", %[\n      k6 wildcard_include_1\n      <elem1 name>\n        include normal_parameter\n      </elem1>\n    ]\n    write_config \"#{TMP_DIR}/config.d/config_test_7.conf\", %[\n      k7 wildcard_include_2\n    ]\n    write_config \"#{TMP_DIR}/config.d/config_test_8.conf\", %[\n      <elem2 name>\n        <include ../dir/config_test_9.conf />\n      </elem2>\n    ]\n    write_config \"#{TMP_DIR}/dir/config_test_9.conf\", %[\n      k9 embedded\n      <elem3 name>\n        nested nested_value\n        include hoge\n      </elem3>\n    ]\n    write_config \"#{TMP_DIR}/config.d/00_config_test_8.conf\", %[\n      k8 wildcard_include_3\n      <elem4 name>\n        include normal_parameter\n      </elem4>\n    ]\n\n  end\n\n  def test_include\n    prepare_config\n    c = read_config(\"#{TMP_DIR}/config_test_1.conf\")\n    assert_equal 'root_config', c['k1']\n    assert_equal 'relative_path_include', c['k2']\n    assert_equal 'relative_include_in_included_file', c['k3']\n    assert_equal 'absolute_path_include', c['k4']\n    assert_equal 'uri_include', c['k5']\n    assert_equal 'wildcard_include_1', c['k6']\n    assert_equal 'wildcard_include_2', c['k7']\n    assert_equal 'wildcard_include_3', c['k8']\n    assert_equal [\n      'k1',\n      'k2',\n      'k3',\n      'k4',\n      'k5',\n      'k8', # Because of the file name this comes first.\n      'k6',\n      'k7',\n    ], c.keys\n\n    elem1 = c.elements.find { |e| e.name == 'elem1' }\n    assert_not_nil elem1\n    assert_equal 'name', elem1.arg\n    assert_equal 'normal_parameter', elem1['include']\n\n    elem2 = c.elements.find { |e| e.name == 'elem2' }\n    assert_not_nil elem2\n    assert_equal 'name', elem2.arg\n    assert_equal 'embedded', elem2['k9']\n    assert !elem2.has_key?('include')\n\n    elem3 = elem2.elements.find { |e| e.name == 'elem3' }\n    assert_not_nil elem3\n    assert_equal 'nested_value', elem3['nested']\n    assert_equal 'hoge', elem3['include']\n  end\n\n  def test_check_not_fetchd\n    write_config \"#{TMP_DIR}/config_test_not_fetched.conf\", %[\n      <match dummy>\n       type          rewrite\n       add_prefix    filtered\n       <rule>\n         key     path\n         pattern ^[A-Z]+\n         replace\n       </rule>\n     </match>\n    ]\n    root_conf  = read_config(\"#{TMP_DIR}/config_test_not_fetched.conf\")\n    match_conf = root_conf.elements.first\n    rule_conf  = match_conf.elements.first\n\n    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }\n    assert_equal %w[type add_prefix key pattern replace], not_fetched\n\n    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }\n    assert_equal %w[type add_prefix key pattern replace], not_fetched\n\n    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }\n    assert_equal %w[key pattern replace], not_fetched\n\n    # accessing should delete\n    match_conf['type']\n    rule_conf['key']\n\n    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }\n    assert_equal %w[add_prefix pattern replace], not_fetched\n\n    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }\n    assert_equal %w[add_prefix pattern replace], not_fetched\n\n    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }\n    assert_equal %w[pattern replace], not_fetched\n\n    # repeatedly accessing should not grow memory usage\n    before_size = match_conf.unused.size\n    10.times { match_conf['type'] }\n    assert_equal before_size, match_conf.unused.size\n  end\n\n  sub_test_case \"yaml config\" do\n    def test_included\n      write_config \"#{TMP_DIR}/config_test_not_fetched.yaml\", <<-EOS\n      config:\n        - source:\n            $type: dummy\n            tag: tag.dummy\n        - source:\n            $type: tcp\n            $log_level: info\n            tag: tag.tcp\n            parse:\n              $arg:\n                - why.parse.section.doesnot.have.arg\n                - huh\n              $type: none\n        - match:\n            $tag: tag.*\n            $type: stdout\n            $log_level: debug\n            buffer:\n              $type: memory\n              flush_interval: 1s\n        - !include fluent-included.yaml\n      EOS\n      write_config \"#{TMP_DIR}/fluent-included.yaml\", <<-EOS\n      - label:\n          $name: '@FLUENT_LOG'\n          config:\n            - match:\n                $type: \"null\"\n                $tag: \"**\"\n                buffer:\n                  $type: memory\n                  flush_mode: interval\n                  flush_interval: 1s\n      EOS\n      root_conf  = read_config(\"#{TMP_DIR}/config_test_not_fetched.yaml\", use_yaml: true)\n      dummy_source_conf = root_conf.elements.first\n      tcp_source_conf = root_conf.elements[1]\n      parse_tcp_conf = tcp_source_conf.elements.first\n      match_conf = root_conf.elements[2]\n      label_conf = root_conf.elements[3]\n      fluent_log_conf = label_conf.elements.first\n      fluent_log_buffer_conf = fluent_log_conf.elements.first\n\n      assert_equal(\n        [\n          'dummy',\n          'tag.dummy',\n          'tcp',\n          'tag.tcp',\n          'info',\n          'none',\n          'why.parse.section.doesnot.have.arg,huh',\n          'stdout',\n          'tag.*',\n          'debug',\n          'null',\n          '**',\n          '@FLUENT_LOG',\n          'memory',\n          'interval',\n          '1s',\n        ],\n        [\n          dummy_source_conf['@type'],\n          dummy_source_conf['tag'],\n          tcp_source_conf['@type'],\n          tcp_source_conf['tag'],\n          tcp_source_conf['@log_level'],\n          parse_tcp_conf['@type'],\n          parse_tcp_conf.arg,\n          match_conf['@type'],\n          match_conf.arg,\n          match_conf['@log_level'],\n          fluent_log_conf['@type'],\n          fluent_log_conf.arg,\n          label_conf.arg,\n          fluent_log_buffer_conf['@type'],\n          fluent_log_buffer_conf['flush_mode'],\n          fluent_log_buffer_conf['flush_interval'],\n        ])\n    end\n\n    def test_included_glob\n      write_config \"#{TMP_DIR}/config.yaml\", <<-EOS\n      config:\n        - !include \"include/*.yaml\"\n      EOS\n      write_config \"#{TMP_DIR}/include/02_source2.yaml\", <<-EOS\n      - source:\n          $type: dummy\n          tag: tag.dummy\n      EOS\n      write_config \"#{TMP_DIR}/include/01_source1.yaml\", <<-EOS\n      - source:\n          $type: tcp\n          tag: tag.tcp\n          parse:\n            $arg:\n              - why.parse.section.doesnot.have.arg\n              - huh\n            $type: none\n      EOS\n      write_config \"#{TMP_DIR}/include/03_match1.yaml\", <<-EOS\n      - match:\n          $tag: tag.*\n          $type: stdout\n          buffer:\n            $type: memory\n            flush_interval: 1s\n      EOS\n      root_conf = read_config(\"#{TMP_DIR}/config.yaml\", use_yaml: true)\n      tcp_source_conf = root_conf.elements.first\n      dummy_source_conf = root_conf.elements[1]\n      parse_tcp_conf = tcp_source_conf.elements.first\n      match_conf = root_conf.elements[2]\n\n      assert_equal(\n        [\n          'tcp',\n          'tag.tcp',\n          'none',\n          'why.parse.section.doesnot.have.arg,huh',\n          'dummy',\n          'tag.dummy',\n          'stdout',\n          'tag.*',\n        ],\n        [\n          tcp_source_conf['@type'],\n          tcp_source_conf['tag'],\n          parse_tcp_conf['@type'],\n          parse_tcp_conf.arg,\n          dummy_source_conf['@type'],\n          dummy_source_conf['tag'],\n          match_conf['@type'],\n          match_conf.arg,\n        ])\n    end\n\n    def test_check_not_fetchd\n      write_config \"#{TMP_DIR}/config_test_not_fetched.yaml\", <<-EOS\n      config:\n        - match:\n            $arg: dummy\n            $type: rewrite\n            add_prefix:    filtered\n            rule:\n              key:     path\n              pattern: \"^[A-Z]+\"\n              replace: true\n      EOS\n      root_conf  = read_config(\"#{TMP_DIR}/config_test_not_fetched.yaml\", use_yaml: true)\n      match_conf = root_conf.elements.first\n      rule_conf  = match_conf.elements.first\n\n      not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }\n      assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched\n\n      not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }\n      assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched\n\n      not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }\n      assert_equal %w[key pattern replace], not_fetched\n\n      # accessing should delete\n      match_conf['type']\n      rule_conf['key']\n\n      not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }\n      assert_equal %w[@type $arg add_prefix pattern replace], not_fetched\n\n      not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }\n      assert_equal %w[@type $arg add_prefix pattern replace], not_fetched\n\n      not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }\n      assert_equal %w[pattern replace], not_fetched\n\n      # repeatedly accessing should not grow memory usage\n      before_size = match_conf.unused.size\n      10.times { match_conf['type'] }\n      assert_equal before_size, match_conf.unused.size\n    end\n\n    data(\n      \"One String for $arg\" => <<~CONF,\n        config:\n          - source:\n              $type: sample\n              tag: test\n          - match:\n              $type: stdout\n              $tag: test.**\n              buffer:\n                $arg: tag\n                $type: memory\n                flush_mode: immediate\n      CONF\n      \"Comma-separated String for $arg\" => <<~CONF,\n        config:\n          - source:\n              $type: sample\n              tag: test\n          - match:\n              $type: stdout\n              $tag: test.**\n              buffer:\n                $arg: tag, time\n                $type: memory\n                timekey: 1h\n                flush_mode: immediate\n      CONF\n      \"One-liner Array for $arg\" => <<~CONF,\n        config:\n          - source:\n              $type: sample\n              tag: test\n          - match:\n              $type: stdout\n              $tag: test.**\n              buffer:\n                $arg: [tag, time]\n                $type: memory\n                timekey: 1h\n                flush_mode: immediate\n      CONF\n      \"Multi-liner Array for $arg\" => <<~CONF,\n        config:\n          - source:\n              $type: sample\n              tag: test\n          - match:\n              $type: stdout\n              $tag: test.**\n              buffer:\n                $arg:\n                  - tag\n                  - time\n                $type: memory\n                timekey: 1h\n                flush_mode: immediate\n      CONF\n      \"One String for normal Array option\" => <<~CONF,\n        config:\n          - source:\n              $type: sample\n              tag: test\n          - match:\n              $type: stdout\n              $tag: test.**\n              format:\n                $type: csv\n                fields: message\n      CONF\n      \"Comma-separated String for normal Array option\" => <<~CONF,\n        config:\n          - source:\n              $type: sample\n              tag: test\n          - match:\n              $type: stdout\n              $tag: test.**\n              inject:\n                time_key: timestamp\n                time_type: string\n              format:\n                $type: csv\n                fields: timestamp, message\n      CONF\n      \"One-liner Array for normal Array option\" => <<~CONF,\n        config:\n          - source:\n              $type: sample\n              tag: test\n          - match:\n              $type: stdout\n              $tag: test.**\n              inject:\n                time_key: timestamp\n                time_type: string\n              format:\n                $type: csv\n                fields: [timestamp, message]\n      CONF\n      \"Multi-liner Array for normal Array option\" => <<~CONF,\n        config:\n          - source:\n              $type: sample\n              tag: test\n          - match:\n              $type: stdout\n              $tag: test.**\n              inject:\n                time_key: timestamp\n                time_type: string\n              format:\n                $type: csv\n                fields:\n                  - timestamp\n                  - message\n      CONF\n      \"Multiple sections\" => <<~CONF,\n        config:\n          - source:\n              $type: sample\n              tag: test\n          - match:\n              $type: copy\n              $tag: test.**\n              store:\n                - $type: relabel\n                  $label: \"@foo\"\n                - $type: relabel\n                  $label: \"@bar\"\n          - label:\n              $name: \"@foo\"\n              config:\n                - match:\n                    $type: stdout\n                    $tag: test.**\n          - label:\n              $name: \"@bar\"\n              config:\n                - match:\n                    $type: stdout\n                    $tag: test.**\n      CONF\n    )\n    test \"Can parse config without error\" do |conf|\n      write_config \"#{TMP_DIR}/config.yaml\", conf\n      read_config(\"#{TMP_DIR}/config.yaml\", use_yaml: true)\n    end\n  end\n\n  def write_config(path, data, encoding: 'utf-8')\n    FileUtils.mkdir_p(File.dirname(path))\n    File.open(path, \"w:#{encoding}:utf-8\") {|f| f.write data }\n  end\n\n  sub_test_case '.build' do\n    test 'read config' do\n      write_config(\"#{TMP_DIR}/build/config_build.conf\", 'key value')\n      c = Fluent::Config.build(config_path: \"#{TMP_DIR}/build/config_build.conf\")\n      assert_equal('value', c['key'])\n    end\n\n    test 'read config with encoding' do\n      write_config(\"#{TMP_DIR}/build/config_build2.conf\", \"#てすと\\nkey value\", encoding: 'shift_jis')\n\n      c = Fluent::Config.build(config_path: \"#{TMP_DIR}/build/config_build2.conf\", encoding: 'shift_jis')\n      assert_equal('value', c['key'])\n    end\n\n    test 'read config with additional_config' do\n      write_config(\"#{TMP_DIR}/build/config_build2.conf\", \"key value\")\n\n      c = Fluent::Config.build(config_path: \"#{TMP_DIR}/build/config_build2.conf\", additional_config: 'key2 value2')\n      assert_equal('value', c['key'])\n      assert_equal('value2', c['key2'])\n    end\n\n    sub_test_case 'on_file_parsed' do\n      test \"calling order on normal configuration files\" do\n        write_config(\"#{TMP_DIR}/build/common_param.conf\", <<~EOS)\n          flush_interval 5s\n          total_limit_size 100m\n          chunk_limit_size 1m\n        EOS\n        write_config(\"#{TMP_DIR}/build/server.conf\", <<~EOS)\n          <server>\n            host 127.0.0.1\n            port 24224\n          </server>\n        EOS\n        write_config(\"#{TMP_DIR}/build/forward.conf\", <<~EOS)\n          <match test.*>\n            @type forward\n\n            @include server.conf\n          </match>\n        EOS\n        write_config(\"#{TMP_DIR}/build/inline.conf\", <<~EOS)\n          <source>\n            @type stdout\n            tag test\n          </source>\n        EOS\n        write_config(\"#{TMP_DIR}/build/fluent.conf\", <<~EOS)\n          <match sample.*>\n            @type file\n            <buffer>\n              @include common_param.conf\n            </buffer>\n          </match>\n          <match debug.*>\n            @type stdout\n            <buffer>\n              @include common_param.conf\n            </buffer>\n          </match>\n\n          @include forward.conf\n        EOS\n\n        # parsed_files contains file paths in the order of file parsed\n        parsed_files = []\n        Fluent::Config.build(\n          config_path: \"#{TMP_DIR}/build/fluent.conf\",\n          additional_config: \"@include inline.conf\",\n          on_file_parsed: ->(path) { parsed_files << path },\n        )\n\n        assert_equal(\n          [\n            \"#{TMP_DIR}/build/common_param.conf\",\n            \"#{TMP_DIR}/build/common_param.conf\",\n            \"#{TMP_DIR}/build/server.conf\",\n            \"#{TMP_DIR}/build/forward.conf\",\n            \"#{TMP_DIR}/build/inline.conf\",\n            \"#{TMP_DIR}/build/fluent.conf\"\n          ],\n          parsed_files\n        )\n      end\n\n      test \"calling order on YAML configuration files\" do\n        write_config(\"#{TMP_DIR}/build/common_buffer.yaml\", <<~EOS)\n          - buffer:\n              flush_interval: 5s\n              total_limit_size: 100m\n              chunk_limit_size: 1m\n        EOS\n        write_config(\"#{TMP_DIR}/build/forward.yaml\", <<~EOS)\n          - match:\n              $tag: test.*\n              server:\n                host: 127.0.0.1\n                port: 24224\n        EOS\n        write_config(\"#{TMP_DIR}/build/fluent.yaml\", <<~EOS)\n          config:\n            - match:\n                $tag: sample.*\n                $type: file\n                <<: !include common_buffer.yaml\n            - match:\n                $tag: debug.*\n                $type: stdout\n                <<: !include common_buffer.yaml\n            - !include forward.yaml\n        EOS\n\n        # parsed_files contains file paths in the order of file parsed\n        # `additional_config` does not support YAML config\n        parsed_files = []\n        Fluent::Config.build(\n          config_path: \"#{TMP_DIR}/build/fluent.yaml\",\n          type: :yaml,\n          on_file_parsed: ->(path) { parsed_files << path },\n          )\n\n        assert_equal(\n          [\n            \"#{TMP_DIR}/build/common_buffer.yaml\",\n            \"#{TMP_DIR}/build/common_buffer.yaml\",\n            \"#{TMP_DIR}/build/forward.yaml\",\n            \"#{TMP_DIR}/build/fluent.yaml\"\n          ],\n          parsed_files\n        )\n      end\n\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_configdsl.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/config/dsl'\nrequire 'fluent/test'\nrequire 'tempfile'\n\nclass ConfigDSLTest < Test::Unit::TestCase\n  # TEST_CONFIG1 = %[\n  #   <source>\n  #     type forward\n  #     port 24224\n  #   </source>\n  #   <match test.**>\n  #     type forward\n  #     flush_interval 1s\n  #     <server>\n  #       host host0.example.com\n  #       port 24224\n  #     </server>\n  #     <server>\n  #       host host1.example.com\n  #       port 24224\n  #     </server>\n  #   </match>\n  # ]\n  TEST_DSL_CONFIG1 = %q[\nsource {\n  type \"forward\"\n  port 24224\n}\nmatch('test.**') {\n  type \"forward\"\n  flush_interval \"1s\"\n  (0..1).each do |i|\n    server {\n      host \"host#{i}.example.com\"\n      port 24224\n    }\n  end\n}\n]\n\n  TEST_DSL_CONFIG2 = %q[\nv = [0, 1, 2]\n]\n\n  TEST_DSL_CONFIG3 = %q[\nmatch\n]\n\n  TEST_DSL_CONFIG4 = %q[\nmatch('aa', 'bb'){\n  type :null\n}\n]\n\n  TEST_DSL_CONFIG5 = %q[\nmatch('aa')\n]\n\n  def test_parse\n    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG1)\n\n    assert_equal 0, root.keys.size\n    assert_equal 2, root.elements.size\n\n    e0 = root.elements[0]\n    assert_equal 'source', e0.name\n    assert_equal '', e0.arg\n    assert_equal 'forward', e0['@type']\n    assert_equal '24224', e0['port']\n\n    e1 = root.elements[1]\n    assert_equal 'match', e1.name\n    assert_equal 'test.**', e1.arg\n    assert_equal 'forward', e1['@type']\n    assert_equal '1s', e1['flush_interval']\n    assert_equal 2, e1.elements.size\n    e1s0 = e1.elements[0]\n    assert_equal 'server', e1s0.name\n    assert_equal 'host0.example.com', e1s0['host']\n    assert_equal '24224', e1s0['port']\n    e1s1 = e1.elements[1]\n    assert_equal 'server', e1s1.name\n    assert_equal 'host1.example.com', e1s1['host']\n    assert_equal '24224', e1s1['port']\n  end\n\n  def test_parse2\n    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG2)\n\n    assert_equal 0, root.keys.size\n    assert_equal 0, root.elements.size\n  end\n\n  def test_config_error\n    assert_raise(ArgumentError) {\n      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG3)\n    }\n\n    assert_raise(ArgumentError) {\n      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG4)\n    }\n\n    assert_raise(ArgumentError) {\n      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG5)\n    }\n  end\n\n  def test_with_ruby_keyword\n    uname_string = `uname -a`\n    tmpfile = Tempfile.create('fluentd-test')\n    tmpfile.write(uname_string)\n    tmpfile.close\n\n    root1 = Fluent::Config::DSL::Parser.parse(<<DSL)\nuname_str = ruby.open(\"#{tmpfile.path}\"){|out| out.read}\nsource {\n  uname uname_str\n}\nDSL\n    source1 = root1.elements.first\n    assert_equal 'source', source1.name\n    assert_equal 1, source1.keys.size\n    assert_equal uname_string, source1['uname']\n\n    root2 = Fluent::Config::DSL::Parser.parse(<<DSL)\nruby_version = ruby {\n  require 'erb'\n  ERB.new('<%= RUBY_VERSION %> from erb').result\n}\nsource {\n  version ruby_version\n}\nDSL\n    source2 = root2.elements.first\n    assert_equal 'source', source2.name\n    assert_equal 1, source2.keys.size\n    assert_equal \"#{RUBY_VERSION} from erb\", source2['version']\n\n    # Parser#parse raises NoMethodError when configuration dsl elements are written in ruby block\n    conf3 = <<DSL\nruby {\n  source {\n    type \"tail\"\n  }\n}\nsource {\n  uname uname_str\n}\nDSL\n    assert_raise (NoMethodError) { Fluent::Config::DSL::Parser.parse(conf3) }\n  ensure\n    File.delete(tmpfile.path)\n  end\nend\n"
  },
  {
    "path": "test/test_daemonizer.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/daemonizer'\n\nclass DaemonizerTest < ::Test::Unit::TestCase\n  TMP_DIR = File.join(File.dirname(__FILE__), 'tmp', 'daemonizer')\n\n  setup do\n    FileUtils.mkdir_p(TMP_DIR)\n  end\n\n  teardown do\n    FileUtils.rm_rf(TMP_DIR) rescue nil\n  end\n\n  test 'makes pid file' do\n    pid_path = File.join(TMP_DIR, 'file.pid')\n\n    mock(Process).daemon(anything, anything).once\n    r = Fluent::Daemonizer.daemonize(pid_path) { 'ret' }\n    assert_equal 'ret', r\n    assert File.exist?(pid_path)\n    assert Process.pid.to_s, File.read(pid_path).to_s\n  end\n\n  test 'in platforms which do not support fork' do\n    pid_path = File.join(TMP_DIR, 'file.pid')\n\n    mock(Process).daemon(anything, anything) { raise NotImplementedError }\n    args = ['-c', 'test.conf']\n    mock(Process).spawn(anything, *args) { Process.pid }\n\n    Fluent::Daemonizer.daemonize(pid_path, args) { 'ret' }\n    assert File.exist?(pid_path)\n    assert Process.pid.to_s, File.read(pid_path).to_s\n  end\n\n  sub_test_case 'when pid file already exists' do\n    test 'raise an error when process is running' do\n      omit 'chmod of file does not affect root user' if Process.uid.zero?\n      pid_path = File.join(TMP_DIR, 'file.pid')\n      File.write(pid_path, '1')\n\n      mock(Process).daemon(anything, anything).never\n      mock(Process).kill(0, 1).once\n\n      assert_raise(Fluent::ConfigError.new('pid(1) is running')) do\n        Fluent::Daemonizer.daemonize(pid_path) { 'ret' }\n      end\n    end\n\n    test 'raise an error when file is not readable' do\n      omit 'chmod of file does not affect root user' if Process.uid.zero?\n      not_readable_path = File.join(TMP_DIR, 'not_readable.pid')\n\n      File.write(not_readable_path, '1')\n      FileUtils.chmod(0333, not_readable_path)\n\n      mock(Process).daemon(anything, anything).never\n      assert_raise(Fluent::ConfigError.new(\"Cannot access pid file: #{File.absolute_path(not_readable_path)}\")) do\n        Fluent::Daemonizer.daemonize(not_readable_path) { 'ret' }\n      end\n    end\n\n    test 'raise an error when file is not writable' do\n      omit 'chmod of file does not affect root user' if Process.uid.zero?\n      not_writable_path = File.join(TMP_DIR, 'not_writable.pid')\n\n      File.write(not_writable_path, '1')\n      FileUtils.chmod(0555, not_writable_path)\n\n      mock(Process).daemon(anything, anything).never\n      assert_raise(Fluent::ConfigError.new(\"Cannot access pid file: #{File.absolute_path(not_writable_path)}\")) do\n        Fluent::Daemonizer.daemonize(not_writable_path) { 'ret' }\n      end\n    end\n\n    test 'raise an error when directory is not writable' do\n      omit 'chmod of file does not affect root user' if Process.uid.zero?\n      not_writable_dir = File.join(TMP_DIR, 'not_writable')\n      pid_path = File.join(not_writable_dir, 'file.pid')\n\n      FileUtils.mkdir_p(not_writable_dir)\n      FileUtils.chmod(0555, not_writable_dir)\n\n      mock(Process).daemon(anything, anything).never\n      assert_raise(Fluent::ConfigError.new(\"Cannot access directory for pid file: #{File.absolute_path(not_writable_dir)}\")) do\n        Fluent::Daemonizer.daemonize(pid_path) { 'ret' }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_engine.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/engine'\nrequire 'fluent/config'\nrequire 'fluent/input'\nrequire 'fluent/system_config'\n\nclass EngineTest < ::Test::Unit::TestCase\n  class DummyEngineTestOutput < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('dummy_engine_test', self)\n    def write(chunk); end\n  end\n\n  class DummyEngineTest2Output < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('dummy_engine_test2', self)\n    def write(chunk); end\n  end\n\n  class DummyEngineTestInput < Fluent::Plugin::Input\n    Fluent::Plugin.register_input('dummy_engine_test', self)\n    def multi_workers_ready?; true; end\n  end\n\n  class DummyEngineTest2Input < Fluent::Plugin::Input\n    Fluent::Plugin.register_input('dummy_engine_test2', self)\n    def multi_workers_ready?; true; end\n  end\n\n  class DummyEngineClassVarTestInput < Fluent::Plugin::Input\n    Fluent::Plugin.register_input('dummy_engine_class_var_test', self)\n    @@test = nil\n    def multi_workers_ready?; true; end\n  end\n\n  sub_test_case '#reload_config' do\n    test 'reload new configuration' do\n      conf_data = <<-CONF\n        <source>\n          @type dummy_engine_test\n        </source>\n        <match>\n          @type dummy_engine_test\n        </match>\n      CONF\n\n      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n      system_config = Fluent::SystemConfig.create(conf)\n\n      engine = Fluent::EngineClass.new\n      engine.init(system_config)\n      engine.configure(conf)\n\n      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]\n      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]\n\n      new_conf_data = <<-CONF\n        <source>\n          @type dummy_engine_test2\n        </source>\n        <match>\n          @type dummy_engine_test2\n        </match>\n      CONF\n\n      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)\n\n      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)\n      stub(Fluent::RootAgent).new do\n        stub(agent).start.once\n        agent\n      end\n\n      engine.reload_config(new_conf)\n\n      assert_kind_of DummyEngineTest2Input, engine.root_agent.inputs[0]\n      assert_kind_of DummyEngineTest2Output, engine.root_agent.outputs[0]\n    end\n\n    test \"doesn't start RootAgent when supervisor is true\" do\n      conf_data = <<-CONF\n        <source>\n          @type dummy_engine_test\n        </source>\n        <match>\n          @type dummy_engine_test\n        </match>\n      CONF\n\n      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n      system_config = Fluent::SystemConfig.create(conf)\n\n      engine = Fluent::EngineClass.new\n      engine.init(system_config)\n      engine.configure(conf)\n\n      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]\n      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]\n\n      new_conf_data = <<-CONF\n        <source>\n          @type dummy_engine_test2\n        </source>\n        <match>\n          @type dummy_engine_test2\n        </match>\n      CONF\n\n      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)\n\n      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)\n      stub(Fluent::RootAgent).new do\n        stub(agent).start.never\n        agent\n      end\n\n      engine.reload_config(new_conf, supervisor: true)\n\n      assert_kind_of DummyEngineTest2Input, engine.root_agent.inputs[0]\n      assert_kind_of DummyEngineTest2Output, engine.root_agent.outputs[0]\n    end\n\n    test 'raise an error when conf is invalid' do\n      conf_data = <<-CONF\n        <source>\n          @type dummy_engine_test\n        </source>\n        <match>\n          @type dummy_engine_test\n        </match>\n      CONF\n\n      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n      system_config = Fluent::SystemConfig.create(conf)\n\n      engine = Fluent::EngineClass.new\n      engine.init(system_config)\n      engine.configure(conf)\n\n      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]\n      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]\n\n      new_conf_data = <<-CONF\n        <source>\n          @type\n        </source>\n      CONF\n\n      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)\n\n      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)\n      stub(Fluent::RootAgent).new do\n        stub(agent).start.never\n        agent\n      end\n\n      assert_raise(Fluent::ConfigError.new(\"Missing '@type' parameter on <source> directive\")) do\n        engine.reload_config(new_conf)\n      end\n\n      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]\n      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]\n    end\n\n    test 'raise an error when unreloadable exists' do\n      conf_data = <<-CONF\n        <source>\n          @type dummy_engine_test\n        </source>\n        <match>\n          @type dummy_engine_test\n        </match>\n      CONF\n\n      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n      system_config = Fluent::SystemConfig.create(conf)\n\n      engine = Fluent::EngineClass.new\n      engine.init(system_config)\n      engine.configure(conf)\n\n      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]\n      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]\n\n      conf_data = <<-CONF\n        <source>\n          @type dummy_engine_class_var_test\n        </source>\n        <match>\n          @type dummy_engine_test\n        </match>\n      CONF\n\n      new_conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n\n      e = assert_raise(Fluent::ConfigError) do\n        engine.reload_config(new_conf)\n      end\n      assert e.message.match?('Unreloadable plugin plugin: dummy_engine_class_var_test')\n\n      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]\n      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_event.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/test'\nrequire 'fluent/event'\nrequire 'fluent/plugin/compressable'\n\nmodule EventTest\n  module DeepCopyAssertion\n    def assert_duplicated_records(es1, es2)\n      ary1 = []\n      es1.each do |_, record|\n        ary1 << record\n      end\n      ary2 = []\n      es2.each do |_, record|\n        ary2 << record\n      end\n      assert_equal ary1.size, ary2.size\n      ary1.each_with_index do |r, i|\n        assert_not_equal r.object_id, ary2[i].object_id\n      end\n    end\n  end\n\n  class OneEventStreamTest < ::Test::Unit::TestCase\n    include Fluent\n    include DeepCopyAssertion\n    include Fluent::Plugin::Compressable\n\n    def setup\n      @time = event_time()\n      @record = {'k' => 'v', 'n' => 1}\n      @es = OneEventStream.new(@time, @record)\n    end\n\n    test 'empty?' do\n      assert_false @es.empty?\n    end\n\n    test 'size' do\n      assert_equal 1, @es.size\n    end\n\n    test 'repeatable?' do\n      assert_true @es.repeatable?\n    end\n\n    test 'dup' do\n      dupped = @es.dup\n      assert_kind_of OneEventStream, dupped\n      assert_not_equal @es.object_id, dupped.object_id\n      assert_duplicated_records @es, dupped\n    end\n\n    test 'slice' do\n      assert_equal 0, @es.slice(1, 1).size\n      assert_equal 0, @es.slice(0, 0).size\n\n      sliced = @es.slice(0, 1)\n      assert_kind_of EventStream, sliced\n      assert_equal 1, sliced.size\n\n      sliced.each do |time, record|\n        assert_equal @time, time\n        assert_equal @record, record\n      end\n    end\n\n    test 'each' do\n      @es.each { |time, record|\n        assert_equal @time, time\n        assert_equal @record, record\n      }\n    end\n\n    test 'to_msgpack_stream' do\n      stream = @es.to_msgpack_stream\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @time, time\n        assert_equal @record, record\n      }\n    end\n\n    test 'to_msgpack_stream with time_int argument' do\n      stream = @es.to_msgpack_stream(time_int: true)\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @time.to_i, time\n        assert_equal @record, record\n      }\n    end\n\n    test 'to_compressed_msgpack_stream' do\n      stream = @es.to_compressed_msgpack_stream\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(decompress(stream)) { |time, record|\n        assert_equal @time, time\n        assert_equal @record, record\n      }\n    end\n\n    test 'to_compressed_msgpack_stream with time_int argument' do\n      stream = @es.to_compressed_msgpack_stream(time_int: true)\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(decompress(stream)) { |time, record|\n        assert_equal @time.to_i, time\n        assert_equal @record, record\n      }\n    end\n  end\n\n  class ArrayEventStreamTest < ::Test::Unit::TestCase\n    include Fluent\n    include DeepCopyAssertion\n    include Fluent::Plugin::Compressable\n\n    def setup\n      time = Engine.now\n      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]\n      @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]\n      @es = ArrayEventStream.new(@times.zip(@records))\n    end\n\n    test 'repeatable?' do\n      assert_true @es.repeatable?\n    end\n\n    test 'dup' do\n      dupped = @es.dup\n      assert_kind_of ArrayEventStream, dupped\n      assert_not_equal @es.object_id, dupped.object_id\n      assert_duplicated_records @es, dupped\n    end\n\n    test 'empty?' do\n      assert_not_empty @es\n      assert_true ArrayEventStream.new([]).empty?\n    end\n\n    test 'size' do\n      assert_equal 2, @es.size\n      assert_equal 0, ArrayEventStream.new([]).size\n    end\n\n    test 'slice' do\n      sliced = @es.slice(1,1)\n      assert_kind_of EventStream, sliced\n      assert_equal 1, sliced.size\n\n      sliced.each do |time, record|\n        assert_equal @times[1], time\n        assert_equal 'v2', record['k']\n        assert_equal 2, record['n']\n      end\n\n      sliced = @es.slice(0,2)\n      assert_kind_of EventStream, sliced\n      assert_equal 2, sliced.size\n\n      counter = 0\n      sliced.each do |time, record|\n        assert_equal @times[counter], time\n        assert_equal @records[counter]['k'], record['k']\n        assert_equal @records[counter]['n'], record['n']\n        counter += 1\n      end\n    end\n\n    test 'each' do\n      i = 0\n      @es.each { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n\n    test 'to_msgpack_stream' do\n      i = 0\n      stream = @es.to_msgpack_stream\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n\n    test 'to_compressed_msgpack_stream' do\n      i = 0\n      compressed_stream = @es.to_compressed_msgpack_stream\n      stream = decompress(compressed_stream)\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n\n    test 'to_compressed_msgpack_stream with time_int argument' do\n      i = 0\n      compressed_stream = @es.to_compressed_msgpack_stream(time_int: true)\n      stream = decompress(compressed_stream)\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @times[i].to_i, time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n  end\n\n  class MultiEventStreamTest < ::Test::Unit::TestCase\n    include Fluent\n    include DeepCopyAssertion\n    include Fluent::Plugin::Compressable\n\n    def setup\n      time = Engine.now\n      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]\n      @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]\n      @es = MultiEventStream.new\n      @times.zip(@records).each { |_time, record|\n        @es.add(_time, record)\n      }\n    end\n\n    test 'repeatable?' do\n      assert_true @es.repeatable?\n    end\n\n    test 'dup' do\n      dupped = @es.dup\n      assert_kind_of MultiEventStream, dupped\n      assert_not_equal @es.object_id, dupped.object_id\n      assert_duplicated_records @es, dupped\n    end\n\n    test 'empty?' do\n      assert_not_empty @es\n      assert_true MultiEventStream.new.empty?\n    end\n\n    test 'size' do\n      assert_equal 2, @es.size\n      assert_equal 0, MultiEventStream.new.size\n    end\n\n    test 'slice' do\n      sliced = @es.slice(1,1)\n      assert_kind_of EventStream, sliced\n      assert_equal 1, sliced.size\n\n      sliced.each do |time, record|\n        assert_equal @times[1], time\n        assert_equal 'v2', record['k']\n        assert_equal 2, record['n']\n      end\n\n      sliced = @es.slice(0,2)\n      assert_kind_of EventStream, sliced\n      assert_equal 2, sliced.size\n\n      counter = 0\n      sliced.each do |time, record|\n        assert_equal @times[counter], time\n        assert_equal @records[counter]['k'], record['k']\n        assert_equal @records[counter]['n'], record['n']\n        counter += 1\n      end\n    end\n\n    test 'each' do\n      i = 0\n      @es.each { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n\n    test 'to_msgpack_stream' do\n      i = 0\n      stream = @es.to_msgpack_stream\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n\n    test 'to_compressed_msgpack_stream' do\n      i = 0\n      compressed_stream = @es.to_compressed_msgpack_stream\n      stream = decompress(compressed_stream)\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n\n    test 'to_compressed_msgpack_stream with time_int argument' do\n      i = 0\n      compressed_stream = @es.to_compressed_msgpack_stream(time_int: true)\n      stream = decompress(compressed_stream)\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @times[i].to_i, time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n  end\n\n  class MessagePackEventStreamTest < ::Test::Unit::TestCase\n    include Fluent\n    include DeepCopyAssertion\n    include Fluent::Plugin::Compressable\n\n    def setup\n      pk = Fluent::MessagePackFactory.msgpack_packer\n      time = Engine.now\n      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]\n      @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]\n      @times.zip(@records).each { |_time, record|\n        pk.write([_time, record])\n      }\n      @es = MessagePackEventStream.new(pk.to_s)\n    end\n\n    test 'dup' do\n      dupped = @es.dup\n      assert_kind_of MessagePackEventStream, dupped\n      assert_not_equal @es.object_id, dupped.object_id\n      assert_duplicated_records @es, dupped\n\n      # After iteration of events (done in assert_duplicated_records),\n      # duplicated event stream still has unpacked objects and correct size\n      dupped = @es.dup\n      assert_equal 2, dupped.instance_eval{ @size }\n    end\n\n    test 'empty?' do\n      assert_false @es.empty?\n      assert_true MessagePackEventStream.new('', 0).empty?\n    end\n\n    test 'size' do\n      assert_equal 2, @es.size\n      assert_equal 0, MessagePackEventStream.new('').size\n    end\n\n    test 'repeatable?' do\n      assert_true @es.repeatable?\n    end\n\n    test 'slice' do\n      sliced = @es.slice(1,1)\n      assert_kind_of EventStream, sliced\n      assert_equal 1, sliced.size\n\n      sliced.each do |time, record|\n        assert_equal @times[1], time\n        assert_equal 'v2', record['k']\n        assert_equal 2, record['n']\n      end\n\n      sliced = @es.slice(0,2)\n      assert_kind_of EventStream, sliced\n      assert_equal 2, sliced.size\n\n      counter = 0\n      sliced.each do |time, record|\n        assert_equal @times[counter], time\n        assert_equal @records[counter]['k'], record['k']\n        assert_equal @records[counter]['n'], record['n']\n        counter += 1\n      end\n    end\n\n    test 'each' do\n      i = 0\n      @es.each { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n\n    test 'to_msgpack_stream' do\n      i = 0\n      stream = @es.to_msgpack_stream\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n\n    test 'to_compressed_msgpack_stream' do\n      i = 0\n      compressed_stream = @es.to_compressed_msgpack_stream\n      stream = decompress(compressed_stream)\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n\n    # `any?` represents an Enumerable method which calls `each` internally\n    test 'size_after_any' do\n      @es.any?\n\n      assert_equal 2, @es.size\n    end\n\n    # `any?` represents an Enumerable method which calls `each` internally\n    test 'each_after_any' do\n      @es.any?\n\n      count = 0\n      @es.each { |time, record| count += 1 }\n      assert_equal 2, count\n    end\n  end\n\n  class CompressedMessagePackEventStreamTest < ::Test::Unit::TestCase\n    include Fluent\n    include DeepCopyAssertion\n    include Fluent::Plugin::Compressable\n\n    def setup\n      time = Engine.now\n      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]\n      @records = [{ 'k' => 'v1', 'n' => 1 }, { 'k' => 'v2', 'n' => 2 }]\n      @packed_record = ''\n      @entries = ''\n      @times.zip(@records).each do |_time, record|\n        v = [_time, record].to_msgpack\n        @packed_record += v\n        @entries += compress(v)\n      end\n      @es = CompressedMessagePackEventStream.new(@entries)\n    end\n\n    def ensure_data_is_decompressed\n      assert_equal @entries, @es.instance_variable_get(:@data)\n      yield\n      assert_equal @packed_record, @es.instance_variable_get(:@data)\n    end\n\n    test 'dup' do\n      dupped = @es.dup\n      assert_kind_of CompressedMessagePackEventStream, dupped\n      assert_not_equal @es.object_id, dupped.object_id\n      assert_duplicated_records @es, dupped\n\n      # After iteration of events (done in assert_duplicated_records),\n      # duplicated event stream still has unpacked objects and correct size\n      dupped = @es.dup\n      assert_equal 2, dupped.instance_eval{ @size }\n    end\n\n    test 'repeatable?' do\n      assert_true @es.repeatable?\n    end\n\n    test 'size' do\n      assert_equal 0, CompressedMessagePackEventStream.new('').size\n      ensure_data_is_decompressed { assert_equal 2, @es.size  }\n    end\n\n    test 'each' do\n      i = 0\n      ensure_data_is_decompressed do\n        @es.each do |time, record|\n          assert_equal @times[i], time\n          assert_equal @records[i], record\n          i += 1\n        end\n      end\n    end\n\n    test 'slice' do\n      sliced = nil\n      ensure_data_is_decompressed { sliced = @es.slice(1,1) }\n      assert_kind_of EventStream, sliced\n      assert_equal 1, sliced.size\n\n      sliced.each do |time, record|\n        assert_equal @times[1], time\n        assert_equal 'v2', record['k']\n        assert_equal 2, record['n']\n      end\n\n      sliced = @es.slice(0,2)\n      assert_kind_of EventStream, sliced\n      assert_equal 2, sliced.size\n\n      counter = 0\n      sliced.each do |time, record|\n        assert_equal @times[counter], time\n        assert_equal @records[counter]['k'], record['k']\n        assert_equal @records[counter]['n'], record['n']\n        counter += 1\n      end\n    end\n\n    test 'to_msgpack_stream' do\n      i = 0\n      stream = nil\n      ensure_data_is_decompressed { stream = @es.to_msgpack_stream }\n\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n\n    test 'to_compressed_msgpack_stream' do\n      i = 0\n      # Do not call ensure_decompressed!\n      assert_equal @entries, @es.instance_variable_get(:@data)\n      compressed_stream = @es.to_compressed_msgpack_stream\n      assert_equal @entries, @es.instance_variable_get(:@data)\n\n      stream = decompress(compressed_stream)\n      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|\n        assert_equal @times[i], time\n        assert_equal @records[i], record\n        i += 1\n      }\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_event_router.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/event_router'\nrequire_relative 'test_plugin_classes'\n\nclass EventRouterTest < ::Test::Unit::TestCase\n  include Fluent\n  include FluentTest\n\n  teardown do\n    @output = nil\n    @filter = nil\n    @compat_filter = nil\n    @error_output = nil\n    @emit_handler = nil\n    @default_collector = nil\n  end\n\n  def output\n    @output ||= FluentTestOutput.new\n  end\n\n  def filter\n    @filter ||= FluentTestFilter.new\n  end\n\n  def compat_filter\n    @compat_filter ||= FluentCompatTestFilter.new\n  end\n\n  def error_output\n    @error_output ||= FluentTestErrorOutput.new\n  end\n\n  def emit_handler\n    @emit_handler ||= TestEmitErrorHandler.new\n  end\n\n  def default_collector\n    @default_collector ||= FluentTestOutput.new\n  end\n\n  def event(record, time = Engine.now)\n    OneEventStream.new(time, record)\n  end\n\n  DEFAULT_EVENT_NUM = 5\n\n  def events(num = DEFAULT_EVENT_NUM)\n    es = MultiEventStream.new\n    num.times { |i|\n      es.add(Engine.now, 'key' => \"value#{i}\")\n    }\n    es\n  end\n\n  sub_test_case EventRouter::MatchCache do\n    setup do\n      @match_cache = EventRouter::MatchCache.new\n    end\n\n    test \"call block when non-cached key\" do\n      assert_raise(RuntimeError.new('Test!')) {\n        @match_cache.get('test') { raise 'Test!' }\n      }\n    end\n\n    test \"don't call block when cached key\" do\n      @match_cache.get('test') { \"I'm cached\" }\n      assert_nothing_raised {\n        @match_cache.get('test') { raise 'Test!' }\n      }\n      assert_equal \"I'm cached\", @match_cache.get('test') { raise 'Test!' }\n    end\n\n    test \"call block when keys are expired\" do\n      cache_size = EventRouter::MatchCache::MATCH_CACHE_SIZE\n      cache_size.times { |i|\n        @match_cache.get(\"test#{i}\") { \"I'm cached #{i}\" }\n      }\n      assert_nothing_raised {\n        cache_size.times { |i|\n          @match_cache.get(\"test#{i}\") { raise \"Why called?\" }\n        }\n      }\n      # expire old keys\n      cache_size.times { |i|\n        @match_cache.get(\"new_test#{i}\") { \"I'm young #{i}\" }\n      }\n      num_called = 0\n      cache_size.times { |i|\n        @match_cache.get(\"test#{i}\") { num_called += 1 }\n      }\n      assert_equal cache_size, num_called\n    end\n  end\n\n  sub_test_case EventRouter::Pipeline do\n    setup do\n      @pipeline = EventRouter::Pipeline.new\n      @es = event('key' => 'value')\n    end\n\n    test 'set one output' do\n      @pipeline.set_output(output)\n      @pipeline.emit_events('test', @es)\n      assert_equal 1, output.events.size\n      assert_equal 'value', output.events['test'].first['key']\n    end\n\n    sub_test_case 'with filter' do\n      setup do\n        @pipeline.set_output(output)\n      end\n\n      data('Filter plugin' => 'filter',\n           'Compat::Filter plugin' => 'compat_filter')\n      test 'set one filer' do |filter_type|\n        @pipeline.add_filter(filter_type == 'filter' ? filter : compat_filter)\n        @pipeline.emit_events('test', @es)\n        assert_equal 1, output.events.size\n        assert_equal 'value', output.events['test'].first['key']\n        assert_equal 0, output.events['test'].first['__test__']\n      end\n\n      data('Filter plugin' => 'filter',\n           'Compat::Filter plugin' => 'compat_filter')\n      test 'set one filer with multi events' do |filter_type|\n        @pipeline.add_filter(filter_type == 'filter' ? filter : compat_filter)\n        @pipeline.emit_events('test', events)\n        assert_equal 1, output.events.size\n        assert_equal 5, output.events['test'].size\n        DEFAULT_EVENT_NUM.times { |i|\n          assert_equal \"value#{i}\", output.events['test'][i]['key']\n          assert_equal i, output.events['test'][i]['__test__']\n        }\n      end\n    end\n  end\n\n  sub_test_case EventRouter do\n    teardown do\n      @event_router = nil\n    end\n\n    def event_router\n      @event_router ||= EventRouter.new(default_collector, emit_handler)\n    end\n\n    sub_test_case 'default collector' do\n      test 'call default collector when no output' do\n        assert_rr do\n          mock(default_collector).emit_events('test', is_a(OneEventStream))\n          event_router.emit('test', Engine.now, 'k' => 'v')\n        end\n      end\n\n      test \"call default collector when only filter\" do\n        event_router.add_rule('test', filter)\n        assert_rr do\n          # After apply Filter, EventStream becomes MultiEventStream by default\n          mock(default_collector).emit_events('test', is_a(MultiEventStream))\n          event_router.emit('test', Engine.now, 'k' => 'v')\n        end\n        assert_equal 1, filter.num\n      end\n\n      test \"call default collector when no matched with output\" do\n        event_router.add_rule('test', output)\n        assert_rr do\n          mock(default_collector).emit_events('dummy', is_a(OneEventStream))\n          event_router.emit('dummy', Engine.now, 'k' => 'v')\n        end\n      end\n\n      test \"don't call default collector when tag matched\" do\n        event_router.add_rule('test', output)\n        assert_rr do\n          mock(default_collector).emit_events('test', is_a(OneEventStream)).never\n          event_router.emit('test', Engine.now, 'k' => 'v')\n        end\n        # check emit handler doesn't catch rr error\n        assert_empty emit_handler.events\n      end\n    end\n\n    sub_test_case 'filter' do\n      test 'filter should be called when tag matched' do\n        filter = Class.new(FluentTestFilter) { |x|\n          def filter_stream(_tag, es); end\n        }.new\n\n        event_router.add_rule('test', filter)\n\n        assert_rr do\n          mock(filter).filter_stream('test', is_a(OneEventStream)) { events }\n          event_router.emit('test', Engine.now, 'k' => 'v')\n        end\n      end\n\n      test 'filter should not be called when tag mismatched' do\n        event_router.add_rule('test', filter)\n\n        assert_rr do\n          mock(filter).filter_stream('test', is_a(OneEventStream)).never\n          event_router.emit('foo', Engine.now, 'k' => 'v')\n        end\n      end\n\n      test 'filter changes records' do\n        event_router.add_rule('test', filter)\n        event_router.add_rule('test', output)\n        event_router.emit('test', Engine.now, 'k' => 'v')\n\n        assert_equal 1, filter.num\n        assert_equal 1, output.events['test'].size\n        assert_equal 0, output.events['test'].first['__test__']\n        assert_equal 'v', output.events['test'].first['k']\n      end\n\n      test 'filter can be chained' do\n        other_filter = FluentTestFilter.new('__hoge__')\n        event_router.add_rule('test', filter)\n        event_router.add_rule('test', other_filter)\n        event_router.add_rule('test', output)\n        event_router.emit('test', Engine.now, 'k' => 'v')\n\n        assert_equal 1, filter.num\n        assert_equal 1, other_filter.num\n        assert_equal 1, output.events['test'].size\n        assert_equal 0, output.events['test'].first['__test__']\n        assert_equal 0, output.events['test'].first['__hoge__']\n        assert_equal 'v', output.events['test'].first['k']\n      end\n    end\n\n    sub_test_case 'optimized filter' do\n      setup do\n        @record = { 'k' => 'v' }\n        @now = Engine.now\n      end\n\n      test 'call optimized filter when the filter plugin implements #filter without #filter_stream' do\n        event_router.add_rule('test', filter)\n\n        assert_rr do\n          mock(filter).filter('test', @now, @record) { @record }\n          event_router.emit('test', @now, @record)\n        end\n      end\n\n      test 'call optimized filter when the filter plugin implements #filter_with_time without #filter_stream' do\n        filter = Class.new(FluentTestFilter) {\n          undef_method :filter\n          def filter_with_time(tag, time, record); end\n        }.new\n\n        event_router.add_rule('test', filter)\n\n        assert_rr do\n          mock(filter).filter_with_time('test', @now, @record) { [time, @record] }\n          event_router.emit('test', @now, @record)\n        end\n      end\n\n      test \"don't call optimized filter when filter plugins implement #filter_stream\" do\n        filter = Class.new(FluentTestFilter) {\n          undef_method :filter\n          def filter_stream(tag, time, record); end\n        }.new\n\n        event_router.add_rule('test', filter)\n\n        assert_rr do\n          mock(filter).filter_stream('test', is_a(OneEventStream)) { OneEventStream.new(@now, @record) }\n          event_router.emit('test', @now, @record)\n        end\n      end\n\n      test 'call optimized filter when filter plugins have #filter_with_time instead of #filter' do\n        filter_with_time = Class.new(FluentTestFilter) {\n          undef_method :filter\n          def filter_with_time(tag, time, record); end\n        }.new\n\n        event_router.add_rule('test', filter_with_time)\n        event_router.add_rule('test', filter)\n\n        assert_rr do\n          mock(filter_with_time).filter_with_time('test', @now, @record) { [@now + 1, @record] }\n          mock(filter).filter('test', @now + 1, @record) { @record }\n          event_router.emit('test', @now, @record)\n        end\n      end\n\n      test \"don't call optimized filter even if just a filter of some filters implements #filter_stream method\" do\n        filter_stream = Class.new(FluentTestFilter) {\n          def filter_stream(tag, record); end\n        }.new\n\n        filter_with_time = Class.new(FluentTestFilter) {\n          undef_method :filter\n          def filter_with_time(tag, time, record); end\n        }.new\n\n        filters = [filter_stream, filter_with_time, filter]\n        filters.each { |f| event_router.add_rule('test', f) }\n\n        e = OneEventStream.new(@now, @record)\n        assert_rr do\n          mock($log).info(\"disable filter chain optimization because #{[filter_stream].map(&:class)} uses `#filter_stream` method.\")\n          mock(filter_stream).filter_stream('test', is_a(OneEventStream)) { e }\n          mock(filter).filter_stream('test', is_a(OneEventStream)) { e }\n          mock(filter_with_time).filter_stream('test', is_a(OneEventStream)) { e }\n          event_router.emit('test', @now, @record)\n        end\n      end\n    end\n\n    sub_test_case 'emit_error_handler' do\n      test 'call handle_emits_error when emit failed' do\n        event_router.add_rule('test', error_output)\n\n        event_router.emit('test', Engine.now, 'k' => 'v')\n        assert_rr do\n          mock(emit_handler).handle_emits_error('test', is_a(OneEventStream), is_a(RuntimeError))\n          event_router.emit('test', Engine.now, 'k' => 'v')\n        end\n      end\n\n      test 'can pass records modified by filters to handle_emits_error' do\n        filter = Class.new(FluentTestFilter) {\n          def filter_stream(_tag, es); end\n        }.new\n        event_router.add_rule('test', filter)\n        event_router.add_rule('test', error_output)\n\n        time = Engine.now\n        modified_es = OneEventStream.new(time, 'modified_label' => 'modified_value')\n\n        assert_rr do\n          stub(filter).filter_stream { modified_es }\n          mock(emit_handler).handle_emits_error('test', modified_es, is_a(RuntimeError))\n          event_router.emit('test', time, 'pre_label' => 'pre_value')\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_event_time.rb",
    "content": "require_relative 'helper'\nrequire 'timecop'\nrequire 'oj'\nrequire 'yajl'\n\nclass EventTimeTest < Test::Unit::TestCase\n  setup do\n    @now = Time.now\n    Timecop.freeze(@now)\n  end\n\n  teardown do\n    Timecop.return\n  end\n\n  test '#sec' do\n    assert_equal(1, Fluent::EventTime.new(1, 2).sec)\n  end\n\n  test '#nsec' do\n    assert_equal(2, Fluent::EventTime.new(1, 2).nsec)\n    assert_equal(0, Fluent::EventTime.new(1).nsec)\n  end\n\n  test '#to_int' do\n    assert_equal(1, Fluent::EventTime.new(1, 2).to_int)\n  end\n\n  test '#to_r' do\n    assert_equal(Rational(1_000_000_002, 1_000_000_000), Fluent::EventTime.new(1, 2).to_r)\n  end\n\n  test '#to_s' do\n    time = Fluent::EventTime.new(100)\n    assert_equal('100', time.to_s)\n    assert_equal('100', \"#{time}\")\n  end\n\n  test '#to_time' do\n    time = Fluent::EventTime.new(@now.to_i, @now.nsec).to_time\n    assert_instance_of(Time, time)\n    assert_equal(@now.to_i, time.to_i)\n    begin\n      ::Time.at(0, 0, :nanosecond)\n      assert_equal(@now.nsec, time.nsec)\n    rescue\n      # Time.at(@sec, @nsec / 1000.0) sometimes cause 1 diff error in nsec by 1000.0\n      assert_in_delta(@now.nsec, time.nsec, 1)\n    end\n  end\n\n  test '#to_json' do\n    time = Fluent::EventTime.new(100)\n    assert_equal('100', time.to_json)\n    assert_equal('{\"time\":100}', {'time' => time}.to_json)\n    assert_equal('[\"tag\",100,{\"key\":\"value\"}]', [\"tag\", time, {\"key\" => \"value\"}].to_json)\n  end\n\n  test 'JSON.dump' do\n    time = Fluent::EventTime.new(100)\n    assert_equal('{\"time\":100}', JSON.dump({'time' => time}))\n    assert_equal('[\"tag\",100,{\"key\":\"value\"}]', JSON.dump([\"tag\", time, {\"key\" => \"value\"}]))\n  end\n\n  test 'Oj.dump' do\n    time = Fluent::EventTime.new(100)\n    require 'fluent/oj_options'\n    Fluent::OjOptions.load_env\n    assert_equal('{\"time\":100}', Oj.dump({'time' => time}))\n    assert_equal('[\"tag\",100,{\"key\":\"value\"}]', Oj.dump([\"tag\", time, {\"key\" => \"value\"}], mode: :compat))\n  end\n\n  test 'Yajl.dump' do\n    time = Fluent::EventTime.new(100)\n    assert_equal('{\"time\":100}', Yajl.dump({'time' => time}))\n    assert_equal('[\"tag\",100,{\"key\":\"value\"}]', Yajl.dump([\"tag\", time, {\"key\" => \"value\"}]))\n  end\n\n  test '.from_time' do\n    sec = 1000\n    usec = 2\n    time = Fluent::EventTime.from_time(Time.at(sec, usec))\n    assert_equal(time.sec, sec)\n    assert_equal(time.nsec, usec * 1000)\n  end\n\n  test 'now' do\n    assert_equal(@now.to_i, Fluent::EventTime.now.sec)\n    assert_equal(@now.nsec, Fluent::EventTime.now.nsec)\n  end\n\n  test 'parse' do\n    assert_equal(Time.parse(\"2011-01-02 13:14:15\").to_i, Fluent::EventTime.parse(\"2011-01-02 13:14:15\").sec)\n    assert_equal(Time.parse(\"2011-01-02 13:14:15\").nsec, Fluent::EventTime.parse(\"2011-01-02 13:14:15\").nsec)\n  end\n\n  test 'eq?' do\n    assert(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), Fluent::EventTime.new(1, 2)))\n    refute(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), Fluent::EventTime.new(1, 3)))\n    refute(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), Fluent::EventTime.new(3, 2)))\n    refute(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), Fluent::EventTime.new(3, 4)))\n\n    assert(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), 1))\n    refute(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), 2))\n\n    assert(Fluent::EventTime.eq?(1, Fluent::EventTime.new(1, 2)))\n    refute(Fluent::EventTime.eq?(2, Fluent::EventTime.new(1, 2)))\n  end\n\n  test '==' do\n    assert(Fluent::EventTime.new(1, 2) == Fluent::EventTime.new(1, 2))\n    assert(Fluent::EventTime.new(1, 2) == Fluent::EventTime.new(1, 3))\n    refute(Fluent::EventTime.new(1, 2) == Fluent::EventTime.new(3, 2))\n    refute(Fluent::EventTime.new(1, 2) == Fluent::EventTime.new(3, 4))\n\n    assert(Fluent::EventTime.new(1, 2) == 1)\n    refute(Fluent::EventTime.new(1, 2) == 2)\n\n    assert(1 == Fluent::EventTime.new(1, 2))\n    refute(2 == Fluent::EventTime.new(1, 2))\n  end\n\n  test '+' do\n    assert_equal(4, Fluent::EventTime.new(1, 2) + Fluent::EventTime.new(3, 4))\n    assert_equal(6, Fluent::EventTime.new(1, 2) + 5)\n    assert_equal(6, 5 + Fluent::EventTime.new(1, 2))\n  end\n\n  test '-' do\n    assert_equal(-2, Fluent::EventTime.new(1, 2) - Fluent::EventTime.new(3, 4))\n    assert_equal(-4, Fluent::EventTime.new(1, 2) - 5)\n    assert_equal(4, 5 - Fluent::EventTime.new(1, 2))\n  end\n\n  test '>' do\n    assert(Fluent::EventTime.new(2) > Fluent::EventTime.new(1))\n    refute(Fluent::EventTime.new(1) > Fluent::EventTime.new(1))\n    refute(Fluent::EventTime.new(1) > Fluent::EventTime.new(2))\n\n    assert(Fluent::EventTime.new(2) > 1)\n    refute(Fluent::EventTime.new(1) > 1)\n    refute(Fluent::EventTime.new(1) > 2)\n\n    assert(2 > Fluent::EventTime.new(1))\n    refute(1 > Fluent::EventTime.new(1))\n    refute(1 > Fluent::EventTime.new(2))\n  end\n\n  test '>=' do\n    assert(Fluent::EventTime.new(2) >= Fluent::EventTime.new(1))\n    assert(Fluent::EventTime.new(1) >= Fluent::EventTime.new(1))\n    refute(Fluent::EventTime.new(1) >= Fluent::EventTime.new(2))\n\n    assert(Fluent::EventTime.new(2) >= 1)\n    assert(Fluent::EventTime.new(1) >= 1)\n    refute(Fluent::EventTime.new(1) >= 2)\n\n    assert(2 >= Fluent::EventTime.new(1))\n    assert(1 >= Fluent::EventTime.new(1))\n    refute(1 >= Fluent::EventTime.new(2))\n  end\n\n  test '<' do\n    assert(Fluent::EventTime.new(1) < Fluent::EventTime.new(2))\n    refute(Fluent::EventTime.new(1) < Fluent::EventTime.new(1))\n    refute(Fluent::EventTime.new(2) < Fluent::EventTime.new(1))\n\n    assert(Fluent::EventTime.new(1) < 2)\n    refute(Fluent::EventTime.new(1) < 1)\n    refute(Fluent::EventTime.new(2) < 1)\n\n    assert(1 < Fluent::EventTime.new(2))\n    refute(1 < Fluent::EventTime.new(1))\n    refute(2 < Fluent::EventTime.new(1))\n  end\n\n  test '=<' do\n    assert(Fluent::EventTime.new(1) <= Fluent::EventTime.new(2))\n    assert(Fluent::EventTime.new(1) <= Fluent::EventTime.new(1))\n    refute(Fluent::EventTime.new(2) <= Fluent::EventTime.new(1))\n\n    assert(Fluent::EventTime.new(1) <= 2)\n    assert(Fluent::EventTime.new(1) <= 1)\n    refute(Fluent::EventTime.new(2) <= 1)\n\n    assert(1 <= Fluent::EventTime.new(2))\n    assert(1 <= Fluent::EventTime.new(1))\n    refute(2 <= Fluent::EventTime.new(1))\n  end\n\n  test 'Time.at' do\n    sec = 1000\n    nsec = 2000\n    ntime = Fluent::EventTime.new(sec, nsec)\n    time = Time.at(ntime)\n    assert_equal(sec, time.to_i)\n    assert_equal(nsec, time.nsec)\n  end\nend\n"
  },
  {
    "path": "test/test_file_wrapper.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/file_wrapper'\n\nclass FileWrapperTest < Test::Unit::TestCase\n  TMP_DIR = File.dirname(__FILE__) + \"/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}\"\n\n  def setup\n    FileUtils.mkdir_p(TMP_DIR)\n  end\n\n  def teardown\n    FileUtils.rm_rf(TMP_DIR)\n  end\n\n  sub_test_case 'WindowsFile exceptions' do\n    test 'nothing raised' do\n      begin\n        path = \"#{TMP_DIR}/test_windows_file.txt\"\n        file1 = file2 = nil\n        file1 = File.open(path, \"wb\") do |f|\n        end\n        assert_nothing_raised do\n          file2 = Fluent::WindowsFile.new(path)\n        ensure\n          file2.close\n        end\n      ensure\n        file1.close if file1\n      end\n    end\n\n    test 'Errno::ENOENT raised' do\n      path = \"#{TMP_DIR}/nofile.txt\"\n      file = nil\n      assert_raise(Errno::ENOENT) do\n        file = Fluent::WindowsFile.new(path)\n      ensure\n        file.close if file\n      end\n    end\n\n    test 'Errno::ENOENT raised on DeletePending' do\n      path = \"#{TMP_DIR}/deletepending.txt\"\n      file = Fluent::WindowsFile.new(path, mode='w')\n      File.delete(path)\n      assert_raise(Errno::ENOENT) do\n        file.stat\n      ensure\n        file.close if file\n      end\n    end\n  end\nend if Fluent.windows?\n"
  },
  {
    "path": "test/test_filter.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/filter'\n\nclass FilterTest < Test::Unit::TestCase\n  include Fluent\n\n  setup do\n    Fluent::Test.setup\n    @time = Fluent::Engine.now\n  end\n\n  def create_driver(klass = Fluent::Filter, conf = '')\n    Test::FilterTestDriver.new(klass).configure(conf, true)\n  end\n\n  def emit(klass, msgs, conf = '')\n    d = create_driver(klass, conf)\n    d.run {\n      msgs.each {|msg|\n        d.emit({'message' => msg}, @time)\n      }\n    }.filtered\n  end\n\n  sub_test_case 'configure' do\n    test 'check to implement `filter` method' do\n      klass = Class.new(Fluent::Filter) do |c|\n        def filter(tag, time, record); end\n      end\n\n      assert_nothing_raised do\n        klass.new\n      end\n    end\n\n    test 'check to implement `filter_with_time` method' do\n      klass = Class.new(Fluent::Filter) do |c|\n        def filter_with_time(tag, time, record); end\n      end\n\n      assert_nothing_raised do\n        klass.new\n      end\n    end\n\n    test 'DO NOT check when implement `filter_stream`' do\n      klass = Class.new(Fluent::Filter) do |c|\n        def filter_stream(tag, es); end\n      end\n\n      assert_nothing_raised do\n        klass.new\n      end\n    end\n\n    test 'NotImplementedError' do\n      klass = Class.new(Fluent::Filter)\n\n      assert_raise NotImplementedError do\n        klass.new\n      end\n    end\n\n    test 'duplicated method implementation' do\n      klass = Class.new(Fluent::Filter) do |c|\n        def filter(tag, time, record); end\n        def filter_with_time(tag, time, record); end\n      end\n\n      assert_raise do\n        klass.new\n      end\n    end\n  end\n\n  sub_test_case 'filter' do\n    test 'null filter' do\n      null_filter = Class.new(Fluent::Filter) do |c|\n        def filter(tag, time, record)\n          nil\n        end\n      end\n      es = emit(null_filter, ['foo'])\n      assert_equal(0, es.instance_variable_get(:@record_array).size)\n    end\n\n    test 'pass filter' do\n      pass_filter = Class.new(Fluent::Filter) do |c|\n        def filter(tag, time, record)\n          record\n        end\n      end\n      es = emit(pass_filter, ['foo'])\n      assert_equal(1, es.instance_variable_get(:@record_array).size)\n    end\n  end\n\n  sub_test_case 'filter_stream' do\n    test 'null filter' do\n      null_filter = Class.new(Fluent::Filter) do |c|\n        def filter_stream(tag, es)\n          MultiEventStream.new\n        end\n        def filter(tag, time, record); record; end\n      end\n      es = emit(null_filter, ['foo'])\n      assert_equal(0, es.instance_variable_get(:@record_array).size)\n    end\n\n    test 'pass filter' do\n      pass_filter = Class.new(Fluent::Filter) do |c|\n        def filter_stream(tag, es)\n          es\n        end\n        def filter(tag, time, record); record; end\n      end\n      es = emit(pass_filter, ['foo'])\n      assert_equal(1, es.instance_variable_get(:@record_array).size)\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_fluent_log_event_router.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/fluent_log_event_router'\nrequire 'fluent/root_agent'\nrequire 'fluent/system_config'\n\nclass FluentLogEventRouterTest < ::Test::Unit::TestCase\n  # @param config [String]\n  def build_config(config)\n    Fluent::Config.parse(config, 'fluent_log_event', '', syntax: :v1)\n  end\n\n  sub_test_case 'NullFluentLogEventRouter does nothing' do\n    test 'emittable? returns false but others does nothing' do\n      null_event_router = Fluent::NullFluentLogEventRouter.new\n      null_event_router.start\n      null_event_router.stop\n      null_event_router.graceful_stop\n      null_event_router.emit_event(nil)\n      assert_false null_event_router.emittable?\n    end\n  end\n\n  sub_test_case '#build' do\n    test 'NullFluentLogEventRouter if root_agent have not internal logger' do\n      root_agent = Fluent::RootAgent.new(log: $log, system_config: Fluent::SystemConfig.new)\n      root_agent.configure(build_config(''))\n\n      d = Fluent::FluentLogEventRouter.build(root_agent)\n      assert_equal Fluent::NullFluentLogEventRouter, d.class\n    end\n\n    test 'FluentLogEventRouter if <match fluent.*> exists in config' do\n      root_agent = Fluent::RootAgent.new(log: $log, system_config: Fluent::SystemConfig.new)\n      root_agent.configure(build_config(<<-CONFIG))\n        <match fluent.*>\n          @type null\n        </match>\n      CONFIG\n\n      d = Fluent::FluentLogEventRouter.build(root_agent)\n      assert_equal Fluent::FluentLogEventRouter, d.class\n    end\n\n    test 'FluentLogEventRouter if <label @FLUENT_LOG> exists in config' do\n      root_agent = Fluent::RootAgent.new(log: $log, system_config: Fluent::SystemConfig.new)\n      root_agent.configure(build_config(<<-CONFIG))\n       <label @FLUENT_LOG>\n         <match *>\n           @type null\n         </match>\n        </label>\n      CONFIG\n\n      d = Fluent::FluentLogEventRouter.build(root_agent)\n      assert_equal Fluent::FluentLogEventRouter, d.class\n    end\n  end\n\n  test 'when calling graceful_stop, it flushes all events' do\n    event_router = []\n    stub(event_router).emit do |tag, time, record|\n      event_router.push([tag, time, record])\n    end\n\n    d = Fluent::FluentLogEventRouter.new(event_router)\n\n    t = Time.now\n    msg = ['tag', t, { 'key' => 'value' }]\n    d.emit_event(msg)\n    d.graceful_stop\n    d.emit_event(msg)\n    d.start\n\n    d.graceful_stop # to call join\n    assert_equal 2, event_router.size\n    assert_equal msg, event_router[0]\n    assert_equal msg, event_router[1]\n  end\n\n  test 'when calling stop, it ignores existing events' do\n    event_router = []\n    stub(event_router).emit do |tag, time, record|\n      event_router.push([tag, time, record])\n    end\n\n    d = Fluent::FluentLogEventRouter.new(event_router)\n\n    t = Time.now\n    msg = ['tag', t, { 'key' => 'value' }]\n    d.emit_event(msg)\n    d.stop\n    d.emit_event(msg)\n    d.start\n\n    d.stop # to call join\n    assert_equal 1, event_router.size\n    assert_equal msg, event_router[0]\n  end\nend\n"
  },
  {
    "path": "test/test_formatter.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/test'\nrequire 'fluent/formatter'\n\nmodule FormatterTest\n  include Fluent\n\n  def tag\n    'tag'\n  end\n\n  def record\n    {'message' => 'awesome', 'greeting' => 'hello'}\n  end\n\n  class BaseFormatterTest < ::Test::Unit::TestCase\n    include FormatterTest\n\n    def test_call\n      formatter = Formatter.new\n      formatter.configure(config_element())\n      assert_raise NotImplementedError do\n        formatter.format('tag', Engine.now, {})\n      end\n    end\n  end\n\n  class BaseFormatterTestWithTestDriver < ::Test::Unit::TestCase\n    include FormatterTest\n\n    def create_driver(conf={})\n      Fluent::Test::FormatterTestDriver.new(Formatter).configure(conf)\n    end\n\n    def test_call\n      d = create_driver\n      assert_raise NotImplementedError do\n        d.format('tag', Engine.now, {})\n      end\n    end\n\n    def test_call_with_string_literal_configure\n      d = create_driver('')\n      assert_raise NotImplementedError do\n        d.format('tag', Engine.now, {})\n      end\n    end\n  end\n\n  class OutFileFormatterTest < ::Test::Unit::TestCase\n    include FormatterTest\n\n    def setup\n      @formatter = Fluent::Test::FormatterTestDriver.new('out_file')\n      @time = Engine.now\n      @newline = if Fluent.windows?\n                   \"\\r\\n\"\n                 else\n                   \"\\n\"\n                 end\n    end\n\n    def configure(conf)\n      @formatter.configure({'utc' => true}.merge(conf))\n    end\n\n    def test_format\n      configure({})\n      formatted = @formatter.format(tag, @time, record)\n\n      assert_equal(\"#{time2str(@time)}\\t#{tag}\\t#{Yajl.dump(record)}#{@newline}\", formatted)\n    end\n\n    def test_format_without_time\n      configure('output_time' => 'false')\n      formatted = @formatter.format(tag, @time, record)\n\n      assert_equal(\"#{tag}\\t#{Yajl.dump(record)}#{@newline}\", formatted)\n    end\n\n    def test_format_without_tag\n      configure('output_tag' => 'false')\n      formatted = @formatter.format(tag, @time, record)\n\n      assert_equal(\"#{time2str(@time)}\\t#{Yajl.dump(record)}#{@newline}\", formatted)\n    end\n\n    def test_format_without_time_and_tag\n      configure('output_tag' => 'false', 'output_time' => 'false')\n      formatted = @formatter.format('tag', @time, record)\n\n      assert_equal(\"#{Yajl.dump(record)}#{@newline}\", formatted)\n    end\n\n    def test_format_without_time_and_tag_against_string_literal_configure\n      @formatter.configure(%[\n        utc         true\n        output_tag  false\n        output_time false\n      ])\n      formatted = @formatter.format('tag', @time, record)\n\n      assert_equal(\"#{Yajl.dump(record)}#{@newline}\", formatted)\n    end\n  end\n\n  class JsonFormatterTest < ::Test::Unit::TestCase\n    include FormatterTest\n\n    def setup\n      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::JSONFormatter)\n      @time = Engine.now\n      @newline = if Fluent.windows?\n                   \"\\r\\n\"\n                 else\n                   \"\\n\"\n                 end\n    end\n\n    data('oj' => 'oj', 'yajl' => 'yajl')\n    def test_format(data)\n      @formatter.configure('json_parser' => data)\n      formatted = @formatter.format(tag, @time, record)\n\n      assert_equal(\"#{Yajl.dump(record)}#{@newline}\", formatted)\n    end\n  end\n\n  class MessagePackFormatterTest < ::Test::Unit::TestCase\n    include FormatterTest\n\n    def setup\n      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::MessagePackFormatter)\n      @time = Engine.now\n    end\n\n    def test_format\n      @formatter.configure({})\n      formatted = @formatter.format(tag, @time, record)\n\n      assert_equal(record.to_msgpack, formatted)\n    end\n  end\n\n  class LabeledTSVFormatterTest < ::Test::Unit::TestCase\n    include FormatterTest\n\n    def setup\n      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::LabeledTSVFormatter)\n      @time = Engine.now\n      @newline = if Fluent.windows?\n                   \"\\r\\n\"\n                 else\n                   \"\\n\"\n                 end\n    end\n\n    def test_config_params\n      assert_equal \"\\t\", @formatter.instance.delimiter\n      assert_equal  \":\", @formatter.instance.label_delimiter\n\n      @formatter.configure(\n        'delimiter'       => ',',\n        'label_delimiter' => '=',\n      )\n\n      assert_equal \",\", @formatter.instance.delimiter\n      assert_equal \"=\", @formatter.instance.label_delimiter\n    end\n\n    def test_format\n      @formatter.configure({})\n      formatted = @formatter.format(tag, @time, record)\n\n      assert_equal(\"message:awesome\\tgreeting:hello#{@newline}\", formatted)\n    end\n\n    def test_format_with_customized_delimiters\n      @formatter.configure(\n        'delimiter'       => ',',\n        'label_delimiter' => '=',\n      )\n      formatted = @formatter.format(tag, @time, record)\n\n      assert_equal(\"message=awesome,greeting=hello#{@newline}\", formatted)\n    end\n\n    def record_with_tab\n      {'message' => \"awe\\tsome\", 'greeting' => \"hello\\t\"}\n    end\n\n    def test_format_suppresses_tab\n      @formatter.configure({})\n      formatted = @formatter.format(tag, @time, record_with_tab)\n\n      assert_equal(\"message:awe some\\tgreeting:hello #{@newline}\", formatted)\n    end\n\n    def test_format_suppresses_tab_custom_replacement\n      @formatter.configure(\n        'replacement'      => 'X',\n      )\n      formatted = @formatter.format(tag, @time, record_with_tab)\n\n      assert_equal(\"message:aweXsome\\tgreeting:helloX#{@newline}\", formatted)\n    end\n\n    def test_format_suppresses_custom_delimiter\n      @formatter.configure(\n        'delimiter'       => 'w',\n        'label_delimiter' => '=',\n      )\n      formatted = @formatter.format(tag, @time, record)\n\n      assert_equal(\"message=a esomewgreeting=hello#{@newline}\", formatted)\n    end\n  end\n\n  class CsvFormatterTest < ::Test::Unit::TestCase\n    include FormatterTest\n\n    def setup\n      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::CsvFormatter)\n      @time = Engine.now\n    end\n\n    def test_config_params\n      assert_equal ',', @formatter.instance.delimiter\n      assert_equal true, @formatter.instance.force_quotes\n      assert_nil @formatter.instance.fields\n    end\n\n    data(\n      'tab_char' => [\"\\t\", '\\t'],\n      'tab_string' => [\"\\t\", 'TAB'],\n      'pipe' => ['|', '|'])\n    def test_config_params_with_customized_delimiters(data)\n      expected, target = data\n      @formatter.configure('delimiter' => target, 'fields' => 'a,b,c')\n      assert_equal expected, @formatter.instance.delimiter\n      assert_equal ['a', 'b', 'c'], @formatter.instance.fields\n    end\n\n    def test_format\n      @formatter.configure('fields' => 'message,message2')\n      formatted = @formatter.format(tag, @time, {\n        'message' => 'awesome',\n        'message2' => 'awesome2'\n      })\n      assert_equal(\"\\\"awesome\\\",\\\"awesome2\\\"\\n\", formatted)\n    end\n\n    def test_format_with_customized_delimiters\n      @formatter.configure(\n        'fields' => 'message,message2',\n        'delimiter' => '\\t'\n      )\n      formatted = @formatter.format(tag, @time, {\n        'message' => 'awesome',\n        'message2' => 'awesome2'\n      })\n      assert_equal(\"\\\"awesome\\\"\\t\\\"awesome2\\\"\\n\", formatted)\n    end\n\n    def test_format_with_non_quote\n      @formatter.configure(\n        'fields' => 'message,message2',\n        'force_quotes' => 'false'\n      )\n      formatted = @formatter.format(tag, @time, {\n        'message' => 'awesome',\n        'message2' => 'awesome2'\n      })\n      assert_equal(\"awesome,awesome2\\n\", formatted)\n    end\n\n    data(\n      'nil' => {\n        'message' => 'awesome',\n        'message2' => nil,\n        'message3' => 'awesome3'\n      },\n      'blank' => {\n        'message' => 'awesome',\n        'message2' => '',\n        'message3' => 'awesome3'\n      })\n    def test_format_with_empty_fields(data)\n      @formatter.configure(\n        'fields' => 'message,message2,message3'\n      )\n      formatted = @formatter.format(tag, @time, data)\n      assert_equal(\"\\\"awesome\\\",\\\"\\\",\\\"awesome3\\\"\\n\", formatted)\n    end\n\n    data(\n      'normally' => 'one,two,three',\n      'white_space' => 'one , two , three',\n      'blank' => 'one,,two,three')\n    def test_config_params_with_fields(data)\n      @formatter.configure('fields' => data)\n      assert_equal %w(one two three), @formatter.instance.fields\n    end\n  end\n\n  class SingleValueFormatterTest < ::Test::Unit::TestCase\n    include FormatterTest\n    def setup\n      @newline = if Fluent.windows?\n                   \"\\r\\n\"\n                 else\n                   \"\\n\"\n                 end\n    end\n\n    def create_driver(klass_or_str)\n      Fluent::Test::FormatterTestDriver.new(klass_or_str)\n    end\n\n    def test_config_params\n      formatter = create_driver(TextFormatter::SingleValueFormatter)\n      assert_equal \"message\", formatter.instance.message_key\n\n      formatter.configure('message_key' => 'foobar')\n      assert_equal \"foobar\", formatter.instance.message_key\n    end\n\n    def test_format\n      formatter = create_driver('single_value')\n      formatter.configure({})\n      formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})\n      assert_equal(\"awesome#{@newline}\", formatted)\n    end\n\n    def test_format_without_newline\n      formatter = create_driver('single_value')\n      formatter.configure('add_newline' => 'false')\n      formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})\n      assert_equal(\"awesome\", formatted)\n    end\n\n    def test_format_with_message_key\n      formatter = create_driver(TextFormatter::SingleValueFormatter)\n      formatter.configure('message_key' => 'foobar')\n      formatted = formatter.format('tag', Engine.now, {'foobar' => 'foo'})\n\n      assert_equal(\"foo#{@newline}\", formatted)\n    end\n  end\n\n  class FormatterLookupTest < ::Test::Unit::TestCase\n    include FormatterTest\n\n    def test_unknown_format\n      assert_raise NotFoundPluginError do\n        Fluent::Plugin.new_formatter('unknown')\n      end\n    end\n\n    data('register_formatter' => 'known', 'register_template' => 'known_old')\n    def test_find_formatter(data)\n      $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), 'scripts'))\n      assert_nothing_raised ConfigError do\n        Fluent::Plugin.new_formatter(data)\n      end\n      $LOAD_PATH.shift\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_input.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/input'\n\nclass FluentInputTest < ::Test::Unit::TestCase\n  include Fluent\n\n  def setup\n    Fluent::Test.setup\n  end\n\n  def create_driver(conf = '')\n    Fluent::Test::InputTestDriver.new(Fluent::Input).configure(conf, true)\n  end\n\n  def test_router\n    d = create_driver\n    assert_equal Engine.root_agent.event_router, d.instance.router\n\n    d = nil\n    assert_nothing_raised {\n      d = create_driver('@label @known')\n    }\n    expected = Engine.root_agent.find_label('@known').event_router\n    assert_equal expected, d.instance.router\n\n    # TestDriver helps to create a label instance automatically, so directly test here\n    assert_raise(ArgumentError) {\n      Fluent::Input.new.configure(Config.parse('@label @unknown', '(test)', '(test_dir)', true))\n    }\n  end\nend\n"
  },
  {
    "path": "test/test_log.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/engine'\nrequire 'fluent/log'\nrequire 'timecop'\nrequire 'logger'\nrequire 'securerandom'\nrequire 'pathname'\n\nclass LogTest < Test::Unit::TestCase\n  def tmp_dir\n    File.join(File.dirname(__FILE__), \"tmp\", \"log\", \"#{ENV['TEST_ENV_NUMBER']}\", SecureRandom.hex(10))\n  end\n\n  def setup\n    @tmp_dir = tmp_dir\n    FileUtils.mkdir_p(@tmp_dir)\n    @log_device = Fluent::Test::DummyLogDevice.new\n    @timestamp = Time.parse(\"2016-04-21 02:58:41 +0000\")\n    @timestamp_str = @timestamp.strftime(\"%Y-%m-%d %H:%M:%S %z\")\n    Timecop.freeze(@timestamp)\n  end\n\n  def teardown\n    @log_device.reset\n    Timecop.return\n    Thread.current[:last_repeated_stacktrace] = nil\n    begin\n      FileUtils.rm_rf(@tmp_dir)\n    rescue Errno::EACCES\n      # It may occur on Windows because of delete pending state due to delayed GC.\n      # Ruby 3.2 or later doesn't ignore Errno::EACCES:\n      # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81\n    end\n  end\n\n  def test_per_process_path\n    path = Fluent::Log.per_process_path(\"C:/tmp/test.log\", :supervisor, 0)\n    assert_equal(path, \"C:/tmp/test-supervisor-0.log\")\n\n    path = Fluent::Log.per_process_path(\"C:/tmp/test.log\", :worker, 1)\n    assert_equal(path, \"C:/tmp/test-1.log\")\n  end\n\n  sub_test_case \"log level\" do\n    data(\n      trace: [Fluent::Log::LEVEL_TRACE, 0],\n      debug: [Fluent::Log::LEVEL_DEBUG, 1],\n      info: [Fluent::Log::LEVEL_INFO, 2],\n      warn: [Fluent::Log::LEVEL_WARN, 3],\n      error: [Fluent::Log::LEVEL_ERROR, 4],\n      fatal: [Fluent::Log::LEVEL_FATAL, 5],\n    )\n    def test_output(data)\n      log_level, start = data\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev)\n      log = Fluent::Log.new(logger)\n      log.level = log_level\n      log.trace \"trace log\"\n      log.debug \"debug log\"\n      log.info \"info log\"\n      log.warn \"warn log\"\n      log.error \"error log\"\n      log.fatal \"fatal log\"\n      expected = [\n        \"#{@timestamp_str} [trace]: trace log\\n\",\n        \"#{@timestamp_str} [debug]: debug log\\n\",\n        \"#{@timestamp_str} [info]: info log\\n\",\n        \"#{@timestamp_str} [warn]: warn log\\n\",\n        \"#{@timestamp_str} [error]: error log\\n\",\n        \"#{@timestamp_str} [fatal]: fatal log\\n\"\n      ][start..-1]\n      assert_equal(expected, log.out.logs)\n    end\n\n    data(\n        trace: [ServerEngine::DaemonLogger::TRACE, 0],\n        debug: [ServerEngine::DaemonLogger::DEBUG, 1],\n        info: [ServerEngine::DaemonLogger::INFO, 2],\n        warn: [ServerEngine::DaemonLogger::WARN, 3],\n        error: [ServerEngine::DaemonLogger::ERROR, 4],\n        fatal: [ServerEngine::DaemonLogger::FATAL, 5],\n    )\n    def test_output_with_serverengine_loglevel(data)\n      log_level, start = data\n\n      dl_opts = {}\n      dl_opts[:log_level] = log_level\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      log = Fluent::Log.new(logger)\n      log.trace \"trace log\"\n      log.debug \"debug log\"\n      log.info \"info log\"\n      log.warn \"warn log\"\n      log.error \"error log\"\n      log.fatal \"fatal log\"\n      expected = [\n        \"#{@timestamp_str} [trace]: trace log\\n\",\n        \"#{@timestamp_str} [debug]: debug log\\n\",\n        \"#{@timestamp_str} [info]: info log\\n\",\n        \"#{@timestamp_str} [warn]: warn log\\n\",\n        \"#{@timestamp_str} [error]: error log\\n\",\n        \"#{@timestamp_str} [fatal]: fatal log\\n\"\n      ][start..-1]\n      assert_equal(expected, log.out.logs)\n    end\n\n    data(\n      trace: [Fluent::Log::LEVEL_TRACE, 0],\n      debug: [Fluent::Log::LEVEL_DEBUG, 1],\n      info: [Fluent::Log::LEVEL_INFO, 2],\n      warn: [Fluent::Log::LEVEL_WARN, 3],\n      error: [Fluent::Log::LEVEL_ERROR, 4],\n      fatal: [Fluent::Log::LEVEL_FATAL, 5],\n    )\n    def test_output_with_block(data)\n      log_level, start = data\n\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev)\n      log = Fluent::Log.new(logger)\n      log.level = log_level\n      log.trace { \"trace log\" }\n      log.debug { \"debug log\" }\n      log.info { \"info log\" }\n      log.warn { \"warn log\" }\n      log.error { \"error log\" }\n      log.fatal { \"fatal log\" }\n      expected = [\n        \"#{@timestamp_str} [trace]: trace log\\n\",\n        \"#{@timestamp_str} [debug]: debug log\\n\",\n        \"#{@timestamp_str} [info]: info log\\n\",\n        \"#{@timestamp_str} [warn]: warn log\\n\",\n        \"#{@timestamp_str} [error]: error log\\n\",\n        \"#{@timestamp_str} [fatal]: fatal log\\n\"\n      ][start..-1]\n      assert_equal(expected, log.out.logs)\n    end\n\n    data(\n        trace: [ServerEngine::DaemonLogger::TRACE, 0],\n        debug: [ServerEngine::DaemonLogger::DEBUG, 1],\n        info: [ServerEngine::DaemonLogger::INFO, 2],\n        warn: [ServerEngine::DaemonLogger::WARN, 3],\n        error: [ServerEngine::DaemonLogger::ERROR, 4],\n        fatal: [ServerEngine::DaemonLogger::FATAL, 5],\n    )\n    def test_output_with_block_with_serverengine_loglevel(data)\n      log_level, start = data\n\n      dl_opts = {}\n      dl_opts[:log_level] = log_level\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      log = Fluent::Log.new(logger)\n      log.trace { \"trace log\" }\n      log.debug { \"debug log\" }\n      log.info { \"info log\" }\n      log.warn { \"warn log\" }\n      log.error { \"error log\" }\n      log.fatal { \"fatal log\" }\n      expected = [\n        \"#{@timestamp_str} [trace]: trace log\\n\",\n        \"#{@timestamp_str} [debug]: debug log\\n\",\n        \"#{@timestamp_str} [info]: info log\\n\",\n        \"#{@timestamp_str} [warn]: warn log\\n\",\n        \"#{@timestamp_str} [error]: error log\\n\",\n        \"#{@timestamp_str} [fatal]: fatal log\\n\"\n      ][start..-1]\n      assert_equal(expected, log.out.logs)\n    end\n\n    data(\n      trace: [Fluent::Log::LEVEL_TRACE, { trace: true, debug: true, info: true, warn: true, error: true, fatal: true }],\n      debug: [Fluent::Log::LEVEL_DEBUG, { trace: false, debug: true, info: true, warn: true, error: true, fatal: true }],\n      info: [Fluent::Log::LEVEL_INFO, { trace: false, debug: false, info: true, warn: true, error: true, fatal: true }],\n      warn: [Fluent::Log::LEVEL_WARN, { trace: false, debug: false, info: false, warn: true, error: true, fatal: true }],\n      error: [Fluent::Log::LEVEL_ERROR, { trace: false, debug: false, info: false, warn: false, error: true, fatal: true }],\n      fatal: [Fluent::Log::LEVEL_FATAL, { trace: false, debug: false, info: false, warn: false, error: false, fatal: true }],\n    )\n    def test_execute_block(data)\n      log_level, expected = data\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev)\n      log = Fluent::Log.new(logger)\n      log.level = log_level\n      block_called = {\n        trace: false,\n        debug: false,\n        info: false,\n        warn: false,\n        error: false,\n        fatal: false,\n      }\n      log.trace { block_called[:trace] = true }\n      log.debug { block_called[:debug] = true }\n      log.info { block_called[:info] = true }\n      log.warn { block_called[:warn] = true }\n      log.error { block_called[:error] = true }\n      log.fatal { block_called[:fatal] = true }\n      assert_equal(expected, block_called)\n    end\n\n    data(\n      trace: [ServerEngine::DaemonLogger::TRACE, { trace: true, debug: true, info: true, warn: true, error: true, fatal: true }],\n      debug: [ServerEngine::DaemonLogger::DEBUG, { trace: false, debug: true, info: true, warn: true, error: true, fatal: true }],\n      info: [ServerEngine::DaemonLogger::INFO, { trace: false, debug: false, info: true, warn: true, error: true, fatal: true }],\n      warn: [ServerEngine::DaemonLogger::WARN, { trace: false, debug: false, info: false, warn: true, error: true, fatal: true }],\n      error: [ServerEngine::DaemonLogger::ERROR, { trace: false, debug: false, info: false, warn: false, error: true, fatal: true }],\n      fatal: [ServerEngine::DaemonLogger::FATAL, { trace: false, debug: false, info: false, warn: false, error: false, fatal: true }],\n    )\n    def test_execute_block_with_serverengine_loglevel(data)\n      log_level, expected = data\n      dl_opts = {}\n      dl_opts[:log_level] = log_level\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      log = Fluent::Log.new(logger)\n      block_called = {\n        trace: false,\n        debug: false,\n        info: false,\n        warn: false,\n        error: false,\n        fatal: false,\n      }\n      log.trace { block_called[:trace] = true }\n      log.debug { block_called[:debug] = true }\n      log.info { block_called[:info] = true }\n      log.warn { block_called[:warn] = true }\n      log.error { block_called[:error] = true }\n      log.fatal { block_called[:fatal] = true }\n      assert_equal(expected, block_called)\n    end\n\n    data(\n      trace: [Fluent::Log::LEVEL_TRACE, 0],\n      debug: [Fluent::Log::LEVEL_DEBUG, 3],\n      info: [Fluent::Log::LEVEL_INFO, 6],\n      warn: [Fluent::Log::LEVEL_WARN, 9],\n      error: [Fluent::Log::LEVEL_ERROR, 12],\n      fatal: [Fluent::Log::LEVEL_FATAL, 15],\n    )\n    def test_backtrace(data)\n      log_level, start = data\n      backtrace = [\"line 1\", \"line 2\", \"line 3\"]\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev)\n      log = Fluent::Log.new(logger)\n      log.level = log_level\n      log.trace_backtrace(backtrace)\n      log.debug_backtrace(backtrace)\n      log.info_backtrace(backtrace)\n      log.warn_backtrace(backtrace)\n      log.error_backtrace(backtrace)\n      log.fatal_backtrace(backtrace)\n      expected = [\n        \"  #{@timestamp_str} [trace]: line 1\\n\",\n        \"  #{@timestamp_str} [trace]: line 2\\n\",\n        \"  #{@timestamp_str} [trace]: line 3\\n\",\n        \"  #{@timestamp_str} [debug]: line 1\\n\",\n        \"  #{@timestamp_str} [debug]: line 2\\n\",\n        \"  #{@timestamp_str} [debug]: line 3\\n\",\n        \"  #{@timestamp_str} [info]: line 1\\n\",\n        \"  #{@timestamp_str} [info]: line 2\\n\",\n        \"  #{@timestamp_str} [info]: line 3\\n\",\n        \"  #{@timestamp_str} [warn]: line 1\\n\",\n        \"  #{@timestamp_str} [warn]: line 2\\n\",\n        \"  #{@timestamp_str} [warn]: line 3\\n\",\n        \"  #{@timestamp_str} [error]: line 1\\n\",\n        \"  #{@timestamp_str} [error]: line 2\\n\",\n        \"  #{@timestamp_str} [error]: line 3\\n\",\n        \"  #{@timestamp_str} [fatal]: line 1\\n\",\n        \"  #{@timestamp_str} [fatal]: line 2\\n\",\n        \"  #{@timestamp_str} [fatal]: line 3\\n\"\n      ][start..-1]\n      assert_equal(expected, log.out.logs)\n    end\n\n    data(\n      trace: [ServerEngine::DaemonLogger::TRACE, 0],\n      debug: [ServerEngine::DaemonLogger::DEBUG, 3],\n      info: [ServerEngine::DaemonLogger::INFO, 6],\n      warn: [ServerEngine::DaemonLogger::WARN, 9],\n      error: [ServerEngine::DaemonLogger::ERROR, 12],\n      fatal: [ServerEngine::DaemonLogger::FATAL, 15],\n    )\n    def test_backtrace_with_serverengine_loglevel(data)\n      log_level, start = data\n      backtrace = [\"line 1\", \"line 2\", \"line 3\"]\n      dl_opts = {}\n      dl_opts[:log_level] = log_level\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      log = Fluent::Log.new(logger)\n      log.trace_backtrace(backtrace)\n      log.debug_backtrace(backtrace)\n      log.info_backtrace(backtrace)\n      log.warn_backtrace(backtrace)\n      log.error_backtrace(backtrace)\n      log.fatal_backtrace(backtrace)\n      expected = [\n        \"  #{@timestamp_str} [trace]: line 1\\n\",\n        \"  #{@timestamp_str} [trace]: line 2\\n\",\n        \"  #{@timestamp_str} [trace]: line 3\\n\",\n        \"  #{@timestamp_str} [debug]: line 1\\n\",\n        \"  #{@timestamp_str} [debug]: line 2\\n\",\n        \"  #{@timestamp_str} [debug]: line 3\\n\",\n        \"  #{@timestamp_str} [info]: line 1\\n\",\n        \"  #{@timestamp_str} [info]: line 2\\n\",\n        \"  #{@timestamp_str} [info]: line 3\\n\",\n        \"  #{@timestamp_str} [warn]: line 1\\n\",\n        \"  #{@timestamp_str} [warn]: line 2\\n\",\n        \"  #{@timestamp_str} [warn]: line 3\\n\",\n        \"  #{@timestamp_str} [error]: line 1\\n\",\n        \"  #{@timestamp_str} [error]: line 2\\n\",\n        \"  #{@timestamp_str} [error]: line 3\\n\",\n        \"  #{@timestamp_str} [fatal]: line 1\\n\",\n        \"  #{@timestamp_str} [fatal]: line 2\\n\",\n        \"  #{@timestamp_str} [fatal]: line 3\\n\"\n      ][start..-1]\n      assert_equal(expected, log.out.logs)\n    end\n  end\n\n  sub_test_case \"suppress repeated backtrace\" do\n    def test_same_log_level\n      backtrace = [\"line 1\", \"line 2\", \"line 3\"]\n      dl_opts = {}\n      dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      opts = {}\n      opts[:suppress_repeated_stacktrace] = true\n      log = Fluent::Log.new(logger, opts)\n      log.trace_backtrace(backtrace)\n      log.trace_backtrace(backtrace)\n      log.trace_backtrace(backtrace + [\"line 4\"])\n      log.trace_backtrace(backtrace)\n      log.trace_backtrace(backtrace)\n      expected = [\n        \"  #{@timestamp_str} [trace]: line 1\\n\",\n        \"  #{@timestamp_str} [trace]: line 2\\n\",\n        \"  #{@timestamp_str} [trace]: line 3\\n\",\n        \"  #{@timestamp_str} [trace]: suppressed same stacktrace\\n\",\n        \"  #{@timestamp_str} [trace]: line 1\\n\",\n        \"  #{@timestamp_str} [trace]: line 2\\n\",\n        \"  #{@timestamp_str} [trace]: line 3\\n\",\n        \"  #{@timestamp_str} [trace]: line 4\\n\",\n        \"  #{@timestamp_str} [trace]: line 1\\n\",\n        \"  #{@timestamp_str} [trace]: line 2\\n\",\n        \"  #{@timestamp_str} [trace]: line 3\\n\",\n        \"  #{@timestamp_str} [trace]: suppressed same stacktrace\\n\",\n      ]\n      assert_equal(expected, log.out.logs)\n    end\n\n    def test_different_log_level\n      backtrace = [\"line 1\", \"line 2\", \"line 3\"]\n      dl_opts = {}\n      dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      opts = {}\n      opts[:suppress_repeated_stacktrace] = true\n      log = Fluent::Log.new(logger, opts)\n      log.trace_backtrace(backtrace)\n      log.debug_backtrace(backtrace)\n      log.info_backtrace(backtrace)\n      log.warn_backtrace(backtrace)\n      log.error_backtrace(backtrace)\n      log.fatal_backtrace(backtrace)\n      expected = [\n        \"  #{@timestamp_str} [trace]: line 1\\n\",\n        \"  #{@timestamp_str} [trace]: line 2\\n\",\n        \"  #{@timestamp_str} [trace]: line 3\\n\",\n        \"  #{@timestamp_str} [debug]: suppressed same stacktrace\\n\",\n        \"  #{@timestamp_str} [info]: suppressed same stacktrace\\n\",\n        \"  #{@timestamp_str} [warn]: suppressed same stacktrace\\n\",\n        \"  #{@timestamp_str} [error]: suppressed same stacktrace\\n\",\n        \"  #{@timestamp_str} [fatal]: suppressed same stacktrace\\n\",\n      ]\n      assert_equal(expected, log.out.logs)\n    end\n  end\n\n  sub_test_case \"force_stacktrace_level\" do\n    data(\n      none: [ nil, [\"trace\", \"debug\", \"info\", \"warn\", \"error\", \"fatal\"] ],\n      trace: [ Fluent::Log::LEVEL_TRACE, [\"trace\", \"trace\", \"trace\", \"trace\", \"trace\", \"trace\"] ],\n      debug: [ Fluent::Log::LEVEL_DEBUG, [\"debug\", \"debug\", \"debug\", \"debug\", \"debug\", \"debug\"] ],\n      info: [ Fluent::Log::LEVEL_INFO, [\"info\", \"info\", \"info\", \"info\", \"info\", \"info\"] ],\n      warn: [ Fluent::Log::LEVEL_WARN, [\"warn\", \"warn\", \"warn\", \"warn\", \"warn\", \"warn\"] ],\n      error: [ Fluent::Log::LEVEL_ERROR, [\"error\", \"error\", \"error\", \"error\", \"error\", \"error\"] ],\n      fatal: [ Fluent::Log::LEVEL_FATAL, [\"fatal\", \"fatal\", \"fatal\", \"fatal\", \"fatal\", \"fatal\"] ],\n    )\n    test \"level should be forced\" do |(level, expected)|\n      backtrace = [\"backtrace\"]\n      logger = Fluent::Log.new(\n        ServerEngine::DaemonLogger.new(\n          @log_device,\n          log_level: ServerEngine::DaemonLogger::TRACE,\n        )\n      )\n      logger.force_stacktrace_level(level) unless level.nil?\n\n      logger.trace_backtrace(backtrace)\n      logger.debug_backtrace(backtrace)\n      logger.info_backtrace(backtrace)\n      logger.warn_backtrace(backtrace)\n      logger.error_backtrace(backtrace)\n      logger.fatal_backtrace(backtrace)\n\n      assert do\n        expected == logger.out.logs.map { |log| log.match(/ \\[([a-z]+)\\]: backtrace$/)[1] }\n      end\n    end\n\n    test \"stacktraces that do not meet log_level initially should be discarded\" do\n      logger = Fluent::Log.new(\n        ServerEngine::DaemonLogger.new(\n          @log_device,\n          log_level: ServerEngine::DaemonLogger::INFO,\n        )\n      )\n      logger.force_stacktrace_level(Fluent::Log::LEVEL_INFO)\n\n      logger.trace_backtrace([\"trace\"])\n      logger.debug_backtrace([\"debug\"])\n      logger.info_backtrace([\"info\"])\n      logger.warn_backtrace([\"warn\"])\n      logger.error_backtrace([\"error\"])\n      logger.fatal_backtrace([\"fatal\"])\n\n      assert_equal(\n        [\n          \"  #{@timestamp_str} [info]: info\\n\",\n          \"  #{@timestamp_str} [info]: warn\\n\",\n          \"  #{@timestamp_str} [info]: error\\n\",\n          \"  #{@timestamp_str} [info]: fatal\\n\",\n        ],\n        logger.out.logs,\n      )\n    end\n\n    test \"stacktraces that do not meet log_level finally should be discarded\" do\n      logger = Fluent::Log.new(\n        ServerEngine::DaemonLogger.new(\n          @log_device,\n          log_level: ServerEngine::DaemonLogger::INFO,\n        )\n      )\n      logger.force_stacktrace_level(Fluent::Log::LEVEL_DEBUG)\n\n      logger.trace_backtrace([\"trace\"])\n      logger.debug_backtrace([\"debug\"])\n      logger.info_backtrace([\"info\"])\n      logger.warn_backtrace([\"warn\"])\n      logger.error_backtrace([\"error\"])\n      logger.fatal_backtrace([\"fatal\"])\n\n      assert_equal([], logger.out.logs)\n    end\n  end\n\n  sub_test_case \"ignore_repeated_log_interval\" do\n    def test_same_message\n      message = \"This is test\"\n      logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})\n      log = Fluent::Log.new(logger, {ignore_repeated_log_interval: 5})\n\n      log.error message\n      10.times { |i|\n        Timecop.freeze(@timestamp + i)\n        log.error message\n      }\n\n      expected = [\n        \"2016-04-21 02:58:41 +0000 [error]: This is test\\n\",\n        \"2016-04-21 02:58:47 +0000 [error]: This is test\\n\"\n      ]\n      assert_equal(expected, log.out.logs)\n    end\n\n    def test_different_message\n      message = \"This is test\"\n      logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})\n      log = Fluent::Log.new(logger, {ignore_repeated_log_interval: 10})\n\n      log.error message\n      3.times { |i|\n        Timecop.freeze(@timestamp + i)\n        log.error message\n        log.error message\n        log.info \"Hello! \" + message\n      }\n\n      expected = [\n        \"2016-04-21 02:58:41 +0000 [error]: This is test\\n\",\n        \"2016-04-21 02:58:41 +0000 [info]: Hello! This is test\\n\",\n        \"2016-04-21 02:58:42 +0000 [error]: This is test\\n\",\n        \"2016-04-21 02:58:42 +0000 [info]: Hello! This is test\\n\",\n        \"2016-04-21 02:58:43 +0000 [error]: This is test\\n\",\n        \"2016-04-21 02:58:43 +0000 [info]: Hello! This is test\\n\",\n      ]\n      assert_equal(expected, log.out.logs)\n    end\n  end\n\n  sub_test_case \"ignore_same_log_interval\" do\n    teardown do\n      Thread.current[:last_same_log] = nil\n    end\n\n    def test_same_message\n      message = \"This is test\"\n      logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})\n      log = Fluent::Log.new(logger, {ignore_same_log_interval: 5})\n\n      log.error message\n      10.times { |i|\n        Timecop.freeze(@timestamp + i + 1)\n        log.error message\n      }\n\n      expected = [\n        \"2016-04-21 02:58:41 +0000 [error]: This is test\\n\",\n        \"2016-04-21 02:58:47 +0000 [error]: This is test\\n\"\n      ]\n      assert_equal(expected, log.out.logs)\n    end\n\n    def test_different_message\n      message = \"This is test\"\n      logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})\n      log = Fluent::Log.new(logger, {ignore_same_log_interval: 10})\n\n      log.error message\n      3.times { |i|\n        Timecop.freeze(@timestamp + i)\n        log.error message\n        log.error message\n        log.info \"Hello! \" + message\n      }\n\n      expected = [\n        \"2016-04-21 02:58:41 +0000 [error]: This is test\\n\",\n        \"2016-04-21 02:58:41 +0000 [info]: Hello! This is test\\n\",\n      ]\n      assert_equal(expected, log.out.logs)\n    end\n\n    def test_reject_on_max_size\n      ignore_same_log_interval = 10\n\n      logger = Fluent::Log.new(\n        ServerEngine::DaemonLogger.new(@log_device, log_level: ServerEngine::DaemonLogger::INFO),\n        ignore_same_log_interval: ignore_same_log_interval,\n      )\n\n      # Output unique log every second.\n      Fluent::Log::IGNORE_SAME_LOG_MAX_CACHE_SIZE.times do |i|\n        logger.info \"Test #{i}\"\n        Timecop.freeze(@timestamp + i)\n      end\n      logger.info \"Over max size!\"\n\n      # The newest cache and the latest caches in `ignore_same_log_interval` should exist.\n      assert { Thread.current[:last_same_log].size == ignore_same_log_interval + 1 }\n    end\n\n    def test_clear_on_max_size\n      ignore_same_log_interval = 10\n\n      logger = Fluent::Log.new(\n        ServerEngine::DaemonLogger.new(@log_device, log_level: ServerEngine::DaemonLogger::INFO),\n        ignore_same_log_interval: ignore_same_log_interval,\n      )\n\n      # Output unique log at the same time.\n      Fluent::Log::IGNORE_SAME_LOG_MAX_CACHE_SIZE.times do |i|\n        logger.info \"Test #{i}\"\n      end\n      logger.info \"Over max size!\"\n\n      # Can't reject old logs, so all cache should be cleared and only the newest should exist.\n      assert { Thread.current[:last_same_log].size == 1 }\n    end\n  end\n\n  def test_dup\n    dl_opts = {}\n    dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\n    logdev = @log_device\n    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n    log1 = Fluent::Log.new(logger)\n    log2 = log1.dup\n    log1.level = Fluent::Log::LEVEL_DEBUG\n    assert_equal(Fluent::Log::LEVEL_DEBUG, log1.level)\n    assert_equal(Fluent::Log::LEVEL_TRACE, log2.level)\n  end\n\n  def test_format_json\n    logdev = @log_device\n    logger = ServerEngine::DaemonLogger.new(logdev)\n    log = Fluent::Log.new(logger)\n    log.format = :json\n    log.level = Fluent::Log::LEVEL_TRACE\n    log.trace \"trace log\"\n    log.debug \"debug log\"\n    log.info \"info log\"\n    log.warn \"warn log\"\n    log.error \"error log\"\n    log.fatal \"fatal log\"\n    expected = [\n      \"#{@timestamp_str} [trace]: trace log\\n\",\n      \"#{@timestamp_str} [debug]: debug log\\n\",\n      \"#{@timestamp_str} [info]: info log\\n\",\n      \"#{@timestamp_str} [warn]: warn log\\n\",\n      \"#{@timestamp_str} [error]: error log\\n\",\n      \"#{@timestamp_str} [fatal]: fatal log\\n\"\n    ]\n    assert_equal(expected, log.out.logs.map { |l|\n                   r = JSON.parse(l)\n                   \"#{r['time']} [#{r['level']}]: #{r['message']}\\n\"\n                 })\n  end\n\n  def test_time_format\n    logdev = @log_device\n    logger = ServerEngine::DaemonLogger.new(logdev)\n    log = Fluent::Log.new(logger)\n    log.time_format = \"%Y\"\n    log.level = Fluent::Log::LEVEL_TRACE\n    log.trace \"trace log\"\n    log.debug \"debug log\"\n    log.info \"info log\"\n    log.warn \"warn log\"\n    log.error \"error log\"\n    log.fatal \"fatal log\"\n    timestamp_str = @timestamp.strftime(\"%Y\")\n    expected = [\n      \"#{timestamp_str} [trace]: trace log\\n\",\n      \"#{timestamp_str} [debug]: debug log\\n\",\n      \"#{timestamp_str} [info]: info log\\n\",\n      \"#{timestamp_str} [warn]: warn log\\n\",\n      \"#{timestamp_str} [error]: error log\\n\",\n      \"#{timestamp_str} [fatal]: fatal log\\n\"\n    ]\n    assert_equal(expected, log.out.logs)\n  end\n\n  def test_disable_events\n    dl_opts = {}\n    dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\n    logdev = @log_device\n    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n    log = Fluent::Log.new(logger)\n    log.enable_event(true)\n    engine = log.instance_variable_get(:@engine)\n    mock(engine).push_log_event(anything, anything, anything).once\n    log.trace \"trace log\"\n    log.disable_events(Thread.current)\n    log.trace \"trace log\"\n  end\n\n  def test_level_reload\n    dl_opts = {}\n    dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\n    logdev = @log_device\n    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n    log = Fluent::Log.new(logger)\n    assert_equal(ServerEngine::DaemonLogger::TRACE, logger.level)\n    assert_equal(Fluent::Log::LEVEL_TRACE, log.level)\n    # change daemon logger side level\n    logger.level = ServerEngine::DaemonLogger::DEBUG\n    assert_equal(ServerEngine::DaemonLogger::DEBUG, logger.level)\n    # check fluentd log side level is also changed\n    assert_equal(Fluent::Log::LEVEL_DEBUG, log.level)\n  end\n\n  DAY_SEC = 60 * 60 * 24\n  data(\n    rotate_daily_age: ['daily', 100000, DAY_SEC + 1],\n    rotate_weekly_age: ['weekly', 100000, DAY_SEC * 7 + 1],\n    rotate_monthly_age: ['monthly', 100000, DAY_SEC * 31 + 1],\n    rotate_size: [1, 100, 0, '0'],\n  )\n  def test_log_with_logdevio(expected)\n    with_timezone('utc') do\n      @timestamp = Time.parse(\"2016-04-21 00:00:00 +0000\")\n      @timestamp_str = @timestamp.strftime(\"%Y-%m-%d %H:%M:%S %z\")\n      Timecop.freeze(@timestamp)\n\n      rotate_age, rotate_size, travel_term = expected\n      path = \"#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}\"\n\n      logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)\n      logger = ServerEngine::DaemonLogger.new(logdev)\n      log = Fluent::Log.new(logger)\n\n      msg = 'a' * 101\n      log.info msg\n      assert_match msg, File.read(path)\n\n      Timecop.freeze(@timestamp + travel_term)\n\n      msg2 = 'b' * 101\n      log.info msg2\n      c = File.read(path)\n\n      assert_match msg2, c\n      assert_not_equal msg, c\n    end\n  end\n\n  def test_log_rotates_specified_size_with_logdevio\n    with_timezone('utc') do\n      begin\n        rotate_age = 2\n        rotate_size = 100\n        path = \"#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}\"\n        path0 = path + '.0'\n        path1 = path + '.1'\n\n        logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)\n        logger = ServerEngine::DaemonLogger.new(logdev)\n        log = Fluent::Log.new(logger)\n\n        msg = 'a' * 101\n        log.info msg\n        assert_match msg, File.read(path)\n        assert_true File.exist?(path)\n        assert_true !File.exist?(path0)\n        assert_true !File.exist?(path1)\n\n        # create log.0\n        msg2 = 'b' * 101\n        log.info msg2\n        c = File.read(path)\n        c0 = File.read(path0)\n        assert_match msg2, c\n        assert_match msg, c0\n        assert_true File.exist?(path)\n        assert_true File.exist?(path0)\n        assert_true !File.exist?(path1)\n\n        # rotate\n        msg3 = 'c' * 101\n        log.info msg3\n        c = File.read(path)\n        c0 = File.read(path0)\n        assert_match msg3, c\n        assert_match msg2, c0\n        assert_true File.exist?(path)\n        assert_true File.exist?(path0)\n        assert_true !File.exist?(path1)\n      ensure\n        logdev&.close\n      end\n    end\n  end\n\n  def test_reopen\n    path = Pathname(@tmp_dir) + \"fluent.log\"\n\n    logdev = Fluent::LogDeviceIO.new(path.to_s)\n    logger = ServerEngine::DaemonLogger.new(logdev)\n    log = Fluent::Log.new(logger, path: path)\n\n    message = \"This is test message.\"\n\n    log.info message\n    log.reopen!\n    log.info message\n\n    assert { path.read.lines.count{ |line| line.include?(message) } == 2 }\n    # Assert reopening the same file.\n    # Especially, on Windows, the filepath is fixed for each process with rotate,\n    # so we need to care about this.\n    assert { path.parent.entries.size == 3 } # [\".\", \"..\", \"fluent.log\"]\n  ensure\n    logdev&.close\n  end\nend\n\nclass PluginLoggerTest < Test::Unit::TestCase\n  def setup\n    @log_device = Fluent::Test::DummyLogDevice.new\n    @timestamp = Time.parse(\"2016-04-21 02:58:41 +0000\")\n    @timestamp_str = @timestamp.strftime(\"%Y-%m-%d %H:%M:%S %z\")\n    Timecop.freeze(@timestamp)\n    dl_opts = {}\n    dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\n    logdev = @log_device\n    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n    @logger = Fluent::Log.new(logger)\n  end\n\n  def teardown\n    @log_device.reset\n    Timecop.return\n    Thread.current[:last_repeated_stacktrace] = nil\n  end\n\n  def test_initialize\n    log = Fluent::PluginLogger.new(@logger)\n    logger = log.instance_variable_get(:@logger)\n    assert_equal(logger, @logger)\n  end\n\n  def test_enable_color\n    log = Fluent::PluginLogger.new(@logger)\n    log.enable_color(true)\n    assert_equal(true, log.enable_color?)\n    assert_equal(true, @logger.enable_color?)\n    log.enable_color(false)\n    assert_equal(false, log.enable_color?)\n    assert_equal(false, @logger.enable_color?)\n    log.enable_color\n    assert_equal(true, log.enable_color?)\n    assert_equal(true, @logger.enable_color?)\n  end\n\n  def test_log_type_in_default\n    mock(@logger).caller_line(:default, Time.now, 1, Fluent::Log::LEVEL_TRACE).once\n    mock(@logger).caller_line(:default, Time.now, 1, Fluent::Log::LEVEL_DEBUG).once\n    mock(@logger).caller_line(:default, Time.now, 1, Fluent::Log::LEVEL_INFO).once\n    mock(@logger).caller_line(:default, Time.now, 1, Fluent::Log::LEVEL_WARN).once\n    mock(@logger).caller_line(:default, Time.now, 1, Fluent::Log::LEVEL_ERROR).once\n    mock(@logger).caller_line(:default, Time.now, 1, Fluent::Log::LEVEL_FATAL).once\n\n    @logger.trace \"trace log 1\"\n    @logger.debug \"debug log 2\"\n    @logger.info  \"info log 3\"\n    @logger.warn  \"warn log 4\"\n    @logger.error \"error log 5\"\n    @logger.fatal \"fatal log 6\"\n  end\n\n  def test_log_types\n    mock(@logger).caller_line(:default, Time.now, 1, Fluent::Log::LEVEL_TRACE).once\n    mock(@logger).caller_line(:supervisor, Time.now, 1, Fluent::Log::LEVEL_DEBUG).once\n    mock(@logger).caller_line(:worker0, Time.now, 1, Fluent::Log::LEVEL_INFO).once\n    mock(@logger).caller_line(:default, Time.now, 1, Fluent::Log::LEVEL_WARN).once\n    mock(@logger).caller_line(:supervisor, Time.now, 1, Fluent::Log::LEVEL_ERROR).once\n    mock(@logger).caller_line(:worker0, Time.now, 1, Fluent::Log::LEVEL_FATAL).once\n\n    @logger.trace :default, \"trace log 1\"\n    @logger.debug :supervisor, \"debug log 2\"\n    @logger.info  :worker0, \"info log 3\"\n    @logger.warn  :default, \"warn log 4\"\n    @logger.error :supervisor, \"error log 5\"\n    @logger.fatal :worker0, \"fatal log 6\"\n  end\n\n  sub_test_case \"take over the parent logger\" do\n    def test_level\n      log = Fluent::PluginLogger.new(@logger)\n      assert_equal(log.level, @logger.level)\n      log.level = \"fatal\"\n      assert_equal(Fluent::Log::LEVEL_FATAL, log.level)\n      assert_equal(Fluent::Log::LEVEL_TRACE, @logger.level)\n    end\n\n    def test_options\n      parent_log = Fluent::Log.new(\n        ServerEngine::DaemonLogger.new(\n          @log_device,\n          log_level: ServerEngine::DaemonLogger::INFO,\n        ),\n        suppress_repeated_stacktrace: true,\n        ignore_repeated_log_interval: 10,\n        ignore_same_log_interval: 10,\n      )\n      parent_log.force_stacktrace_level(Fluent::Log::LEVEL_INFO)\n\n      log = Fluent::PluginLogger.new(parent_log)\n      assert_equal(\n        [\n          true,\n          Fluent::Log::LEVEL_INFO,\n          10,\n          10,\n        ],\n        [\n          log.instance_variable_get(:@suppress_repeated_stacktrace),\n          log.instance_variable_get(:@forced_stacktrace_level),\n          log.instance_variable_get(:@ignore_repeated_log_interval),\n          log.instance_variable_get(:@ignore_same_log_interval),\n        ]\n      )\n    end\n  end\n\n  sub_test_case \"supervisor process type\" do\n    setup do\n      dl_opts = {}\n      dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      @logger = Fluent::Log.new(logger, process_type: :supervisor)\n    end\n\n    test 'default type logs are shown    w/o worker id' do\n      @logger.info \"yaaay\"\n      @logger.info :default, \"booo\"\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: yaaay\\n\") }\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: booo\\n\") }\n    end\n\n    test 'supervisor type logs are shown w/o worker id' do\n      @logger.info :supervisor, \"yaaay\"\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: yaaay\\n\") }\n    end\n\n    test 'worker0 type logs are not shown' do\n      @logger.info :worker0, \"yaaay\"\n      assert{ !@log_device.logs.include?(\"#{@timestamp_str} [info]: yaaay\\n\") }\n    end\n  end\n\n  sub_test_case \"worker0 process type\" do\n    setup do\n      dl_opts = {}\n      dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      @logger = Fluent::Log.new(logger, process_type: :worker0, worker_id: 10)\n    end\n\n    test 'default type logs are shown w/ worker id' do\n      @logger.info \"yaaay\"\n      @logger.info :default, \"booo\"\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: #10 yaaay\\n\") }\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: #10 booo\\n\") }\n    end\n\n    test 'supervisor type logs are not shown' do\n      @logger.info :supervisor, \"yaaay\"\n      assert{ !@log_device.logs.include?(\"#{@timestamp_str} [info]: yaaay\\n\") }\n    end\n\n    test 'worker0 type logs are shown w/o worker id' do\n      @logger.info :worker0, \"yaaay\"\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: yaaay\\n\") }\n    end\n  end\n\n  sub_test_case \"workers process type\" do\n    setup do\n      dl_opts = {}\n      dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      @logger = Fluent::Log.new(logger, process_type: :workers, worker_id: 7)\n    end\n\n    test 'default type logs are shown w/ worker id' do\n      @logger.info \"yaaay\"\n      @logger.info :default, \"booo\"\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: #7 yaaay\\n\") }\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: #7 booo\\n\") }\n    end\n\n    test 'supervisor type logs are not shown' do\n      @logger.info :supervisor, \"yaaay\"\n      assert{ !@log_device.logs.include?(\"#{@timestamp_str} [info]: yaaay\\n\") }\n    end\n\n    test 'worker0 type logs are not shown' do\n      @logger.info :worker0, \"yaaay\"\n      assert{ !@log_device.logs.include?(\"#{@timestamp_str} [info]: yaaay\\n\") }\n    end\n  end\n\n  sub_test_case \"standalone process type\" do\n    setup do\n      dl_opts = {}\n      dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE\n      logdev = @log_device\n      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n      @logger = Fluent::Log.new(logger, process_type: :standalone, worker_id: 0)\n    end\n\n    test 'default type logs are shown w/o worker id' do\n      @logger.info \"yaaay\"\n      @logger.info :default, \"booo\"\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: yaaay\\n\") }\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: booo\\n\") }\n    end\n\n    test 'supervisor type logs are shown w/o worker id' do\n      @logger.info :supervisor, \"yaaay\"\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: yaaay\\n\") }\n    end\n\n    test 'worker0 type logs are shown w/o worker id' do\n      @logger.info :worker0, \"yaaay\"\n      assert{ @log_device.logs.include?(\"#{@timestamp_str} [info]: yaaay\\n\") }\n    end\n  end\n\n  sub_test_case \"delegators\" do\n    def setup\n      super\n      @log = Fluent::PluginLogger.new(@logger)\n    end\n\n    def test_enable_debug\n      mock(@logger).enable_debug\n      @log.enable_debug\n    end\n\n    def test_enable_event\n      mock(@logger).enable_event\n      @log.enable_event\n    end\n\n    def test_disable_events\n      mock(@logger).disable_events(Thread.current)\n      @log.disable_events(Thread.current)\n    end\n\n    def test_event\n      mock(@logger).event(Fluent::Log::LEVEL_TRACE, { key: \"value\" })\n      @log.event(Fluent::Log::LEVEL_TRACE, { key: \"value\" })\n    end\n\n    def test_caller_line\n      mock(@logger).caller_line(Time.now, 1, Fluent::Log::LEVEL_TRACE)\n      @log.caller_line(Time.now, 1, Fluent::Log::LEVEL_TRACE)\n    end\n\n    def test_puts\n      mock(@logger).puts(\"log\")\n      @log.puts(\"log\")\n    end\n\n    def test_write\n      mock(@logger).write(\"log\")\n      @log.write(\"log\")\n    end\n\n    def test_write_alias\n      assert(@log.respond_to?(:<<))\n      mock(@log.out).write(\"log\")\n      @log << \"log\"\n    end\n\n    def test_out\n      assert_equal(@log.out, @logger.out)\n      @log.out = Object.new\n      assert_equal(@log.out, @logger.out)\n    end\n\n    def test_optional_header\n      assert_equal(@log.optional_header, @logger.optional_header)\n      @log.optional_header = \"optional_header\"\n      assert_equal(@log.optional_header, @logger.optional_header)\n    end\n\n    def test_optional_attrs\n      assert_equal(@log.optional_attrs, @logger.optional_attrs)\n      @log.optional_attrs = \"optional_attrs\"\n      assert_equal(@log.optional_attrs, @logger.optional_attrs)\n    end\n  end\n\n  sub_test_case \"partially delegated\" do\n    def setup\n      super\n      @log = Fluent::PluginLogger.new(@logger)\n    end\n\n    data(\n      text: [:text, \"2016-04-21 02:58:41 +0000 [info]: yaaay\\n\"],\n      json: [:json, %Q({\"time\":\"2016-04-21 02:58:41 +0000\",\"level\":\"info\",\"message\":\"yaaay\"}\\n)],\n    )\n    def test_format(data)\n      fmt, expected_log_line = data\n      with_timezone('utc') {\n        @log.format = fmt\n        @log.info \"yaaay\"\n      }\n      assert{ @log_device.logs.include? expected_log_line }\n    end\n\n    data(\n      text: [:text, \"2016 [info]: yaaay\\n\"],\n      json: [:json, %Q({\"time\":\"2016\",\"level\":\"info\",\"message\":\"yaaay\"}\\n)],\n    )\n    def test_time_format(data)\n      fmt, expected_log_line = data\n      @log.format = fmt\n      @log.time_format = \"%Y\"\n      @log.info \"yaaay\"\n      assert{ @log_device.logs.include? expected_log_line }\n    end\n  end\nend\n\nclass PluginLoggerMixinTest < Test::Unit::TestCase\n  class DummyPlugin < Fluent::Plugin::Input\n  end\n\n  def create_driver(conf)\n    Fluent::Test::Driver::Input.new(DummyPlugin).configure(conf)\n  end\n\n  def setup\n    Fluent::Test.setup\n  end\n\n  def test_default_log\n    plugin = DummyPlugin.new\n    log = plugin.log\n    assert_equal($log, log)\n  end\n\n  def test_log_level\n    d = create_driver(%[log_level fatal])\n    log = d.instance.log\n    assert_not_equal($log.level, log.level)\n    assert_equal(Fluent::Log::LEVEL_FATAL, log.level)\n  end\n\n  def test_optional_header\n    d = create_driver(%[@id myplugin])\n    log = d.instance.log\n    assert_equal(\"[myplugin] \", log.optional_header)\n    assert_equal({}, log.optional_attrs)\n  end\n\n  def test_start\n    plugin = DummyPlugin.new\n    mock(plugin.log).should_receive(:reset).never\n    plugin.start\n  end\n\n  def test_terminate\n    plugin = DummyPlugin.new\n    mock(plugin.log).reset\n    plugin.terminate\n  end\n\n  sub_test_case \"take over the parent logger\" do\n    def setup\n      super\n\n      begin\n        saved_global_logger = $log\n        $log = Fluent::Log.new(\n          ServerEngine::DaemonLogger.new(\n            Fluent::Test::DummyLogDevice.new,\n            log_level: ServerEngine::DaemonLogger::INFO,\n          ),\n          suppress_repeated_stacktrace: true,\n          ignore_repeated_log_interval: 10,\n          ignore_same_log_interval: 10,\n        )\n        $log.force_stacktrace_level(Fluent::Log::LEVEL_INFO)\n        yield\n      ensure\n        $log = saved_global_logger\n      end\n    end\n\n    def test_options\n      plugin = DummyPlugin.new\n      plugin.configure(Fluent::Config::Element.new(\"input\", \"\", {\"@id\" => \"foo\"}, []))\n\n      assert_equal(\n        [\n          true,\n          Fluent::Log::LEVEL_INFO,\n          10,\n          10,\n        ],\n        [\n          plugin.log.instance_variable_get(:@suppress_repeated_stacktrace),\n          plugin.log.instance_variable_get(:@forced_stacktrace_level),\n          plugin.log.instance_variable_get(:@ignore_repeated_log_interval),\n          plugin.log.instance_variable_get(:@ignore_same_log_interval),\n        ]\n      )\n    end\n  end\nend\n\nclass LogDeviceIOTest < Test::Unit::TestCase\n  test 'flush' do\n    io = StringIO.new\n    logdev = Fluent::LogDeviceIO.new(io)\n    assert_equal io, logdev.flush\n\n    io.instance_eval { undef :flush }\n    logdev = Fluent::LogDeviceIO.new(io)\n    assert_raise NoMethodError do\n      logdev.flush\n    end\n  end\n\n  test 'tty?' do\n    io = StringIO.new\n    logdev = Fluent::LogDeviceIO.new(io)\n    assert_equal io.tty?, logdev.tty?\n\n    io.instance_eval { undef :tty? }\n    logdev = Fluent::LogDeviceIO.new(io)\n    assert_raise NoMethodError do\n      logdev.tty?\n    end\n  end\n\n  test 'sync=' do\n    io = StringIO.new\n    logdev = Fluent::LogDeviceIO.new(io)\n    assert_true logdev.sync = true\n\n    io.instance_eval { undef :sync= }\n    logdev = Fluent::LogDeviceIO.new(io)\n    assert_raise NoMethodError do\n      logdev.sync = true\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_match.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/match'\n\nclass MatchTest < Test::Unit::TestCase\n  include Fluent\n\n  def test_simple\n    assert_glob_match('a', 'a')\n    assert_glob_match('a.b', 'a.b')\n    assert_glob_not_match('a', 'b')\n    assert_glob_not_match('a.b', 'aab')\n  end\n\n  def test_wildcard\n    assert_glob_match('a*', 'a')\n    assert_glob_match('a*', 'ab')\n    assert_glob_match('a*', 'abc')\n\n    assert_glob_match('*a', 'a')\n    assert_glob_match('*a', 'ba')\n    assert_glob_match('*a', 'cba')\n\n    assert_glob_match('*a*', 'a')\n    assert_glob_match('*a*', 'ba')\n    assert_glob_match('*a*', 'ac')\n    assert_glob_match('*a*', 'bac')\n\n    assert_glob_not_match('a*', 'a.b')\n    assert_glob_not_match('a*', 'ab.c')\n    assert_glob_not_match('a*', 'ba')\n    assert_glob_not_match('*a', 'ab')\n\n    assert_glob_match('a.*', 'a.b')\n    assert_glob_match('a.*', 'a.c')\n    assert_glob_not_match('a.*', 'ab')\n\n    assert_glob_match('a.*.c', 'a.b.c')\n    assert_glob_match('a.*.c', 'a.c.c')\n    assert_glob_not_match('a.*.c', 'a.c')\n  end\n\n  def test_recursive_wildcard\n    assert_glob_match('a.**', 'a')\n    assert_glob_not_match('a.**', 'ab')\n    assert_glob_not_match('a.**', 'abc')\n    assert_glob_match('a.**', 'a.b')\n    assert_glob_not_match('a.**', 'ab.c')\n    assert_glob_not_match('a.**', 'ab.d.e')\n\n    assert_glob_match('a**', 'a')\n    assert_glob_match('a**', 'ab')\n    assert_glob_match('a**', 'abc')\n    assert_glob_match('a**', 'a.b')\n    assert_glob_match('a**', 'ab.c')\n    assert_glob_match('a**', 'ab.d.e')\n\n    assert_glob_match('**.a', 'a')\n    assert_glob_not_match('**.a', 'ba')\n    assert_glob_not_match('**.a', 'c.ba')\n    assert_glob_match('**.a', 'b.a')\n    assert_glob_match('**.a', 'cb.a')\n    assert_glob_match('**.a', 'd.e.a')\n\n    assert_glob_match('**a', 'a')\n    assert_glob_match('**a', 'ba')\n    assert_glob_match('**a', 'c.ba')\n    assert_glob_match('**a', 'b.a')\n    assert_glob_match('**a', 'cb.a')\n    assert_glob_match('**a', 'd.e.a')\n  end\n\n  def test_or\n    assert_glob_match('a.{b,c}', 'a.b')\n    assert_glob_match('a.{b,c}', 'a.c')\n    assert_glob_not_match('a.{b,c}', 'a.d')\n\n    assert_glob_match('a.{b,c}.**', 'a.b')\n    assert_glob_match('a.{b,c}.**', 'a.c')\n    assert_glob_not_match('a.{b,c}.**', 'a.d')\n    assert_glob_not_match('a.{b,c}.**', 'a.cd')\n\n    assert_glob_match('a.{b.**,c}', 'a.b')\n    assert_glob_match('a.{b.**,c}', 'a.b.c')\n    assert_glob_match('a.{b.**,c}', 'a.c')\n    assert_glob_not_match('a.{b.**,c}', 'a.c.d')\n  end\n\n  def test_multi_pattern_or\n    assert_or_match('a.b a.c', 'a.b')\n    assert_or_match('a.b a.c', 'a.c')\n    assert_or_not_match('a.b a.c', 'a.d')\n\n    assert_or_match('a.b.** a.c.**', 'a.b')\n    assert_or_match('a.b.** a.c.**', 'a.c')\n    assert_or_not_match('a.b.** a.c.**', 'a.d')\n    assert_or_not_match('a.b.** a.c.**', 'a.cd')\n\n    assert_or_match('a.b.** a.c', 'a.b')\n    assert_or_match('a.b.** a.c', 'a.b.c')\n    assert_or_match('a.b.** a.c', 'a.c')\n    assert_or_not_match('a.b.** a.c', 'a.c.d')\n  end\n\n  def test_regex_pattern\n    assert_glob_match('/a/', 'a')\n    assert_glob_not_match('/a/', 'abc')\n    assert_glob_match('/a.*/', 'abc')\n    assert_glob_not_match('/b.*/', 'abc')\n    assert_glob_match('/a\\..*/', 'a.b.c')\n    assert_glob_not_match('/(?!a\\.).*/', 'a.b.c')\n    assert_glob_not_match('/a\\..*/', 'b.b.c')\n    assert_glob_match('/(?!a\\.).*/', 'b.b.c')\n  end\n\n  #def test_character_class\n  #  assert_match('[a]', 'a')\n  #  assert_match('[ab]', 'a')\n  #  assert_match('[ab]', 'b')\n  #  assert_not_match('[ab]', 'c')\n  #\n  #  assert_match('[a-b]', 'a')\n  #  assert_match('[a-b]', 'a')\n  #  assert_match('[a-b]', 'b')\n  #  assert_not_match('[a-b]', 'c')\n  #\n  #  assert_match('[a-b0-9]', 'a')\n  #  assert_match('[a-b0-9]', '0')\n  #  assert_not_match('[a-b0-9]', 'c')\n  #end\n\n  def assert_glob_match(pat, str)\n    assert_true GlobMatchPattern.new(pat).match(str)\n    assert_true EventRouter::Rule.new(pat, nil).match?(str)\n  end\n\n  def assert_glob_not_match(pat, str)\n    assert_false GlobMatchPattern.new(pat).match(str)\n    assert_false EventRouter::Rule.new(pat, nil).match?(str)\n  end\n\n  def assert_or_match(pats, str)\n    assert_true EventRouter::Rule.new(pats, nil).match?(str)\n  end\n\n  def assert_or_not_match(pats, str)\n    assert_false EventRouter::Rule.new(pats, nil).match?(str)\n  end\nend\n"
  },
  {
    "path": "test/test_mixin.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/mixin'\nrequire 'fluent/env'\nrequire 'fluent/plugin'\nrequire 'fluent/config'\nrequire 'fluent/test'\nrequire 'timecop'\n\nmodule MixinTest\n  module Utils\n    def setup\n      super\n      Fluent::Test.setup\n      @time = Time.utc(1,2,3,4,5,2010,nil,nil,nil,nil)\n      Timecop.freeze(@time)\n    end\n\n    def teardown\n      super\n      Timecop.return\n      GC.start\n    end\n\n    module Checker\n      extend self\n      def format_check(tag, time, record); end\n    end\n\n    @@num = 0\n\n    def create_register_output_name\n      @@num += 1\n      \"mixin_text_#{@@num}\"\n    end\n\n    def format_check(hash, tagname = 'test')\n      mock(Checker).format_check(tagname, @time.to_i, hash)\n    end\n\n    def create_driver(include_klass, conf = '', tag = \"test\", &block)\n      register_output_name = create_register_output_name\n      include_klasses = [include_klass].flatten\n\n      klass = Class.new(Fluent::BufferedOutput) {\n        include_klasses.each {|k| include k }\n\n        Fluent::Plugin.register_output(register_output_name, self)\n        def format(tag, time, record)\n          Checker.format_check(tag, time, record)\n          [tag, time, record].to_msgpack\n        end\n\n        def write(chunk); end\n      }\n\n      if block\n        Utils.const_set(\"MixinTestClass#{@@num}\", klass)\n        klass.module_eval(&block)\n      end\n\n      Fluent::Test::BufferedOutputTestDriver.new(klass, tag) {\n      }.configure(\"type #{register_output_name}\" + conf)\n    end\n  end\n\n  class SetTagKeyMixinText < Test::Unit::TestCase\n    include Utils\n\n    def test_tag_key_default\n      format_check({\n        'a' => 1\n      })\n\n      d = create_driver(Fluent::SetTagKeyMixin, %[\n      ])\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    def test_include_tag_key_true\n      format_check({\n        'tag' => 'test',\n        'a' => 1\n      })\n\n      d = create_driver(Fluent::SetTagKeyMixin, %[\n      include_tag_key true\n      ])\n\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    def test_include_tag_key_false\n      format_check({\n        'a' => 1\n      })\n\n      d = create_driver(Fluent::SetTagKeyMixin, %[\n      include_tag_key false\n      ])\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    def test_tag_key_set\n      format_check({\n        'tag_key_changed' => 'test',\n        'a' => 1\n      })\n\n      d = create_driver(Fluent::SetTagKeyMixin, %[\n      include_tag_key true\n      tag_key tag_key_changed\n      ])\n\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    sub_test_case \"mixin\" do\n      data(\n        'true' => true,\n        'false' => false)\n      test 'include_tag_key' do |param|\n        d = create_driver(Fluent::SetTagKeyMixin) {\n          config_set_default :include_tag_key, param\n        }\n\n        assert_equal(param, d.instance.include_tag_key)\n      end\n    end\n  end\n\n  class SetTimeKeyMixinText < Test::Unit::TestCase\n    include Utils\n\n    def test_time_key_default\n      format_check({\n        'a' => 1\n      })\n\n      d = create_driver(Fluent::SetTimeKeyMixin, %[\n      ])\n\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    def test_include_time_key_true\n      format_check({\n        'time' => \"2010-05-04T03:02:01Z\",\n        'a' => 1\n      })\n\n      d = create_driver(Fluent::SetTimeKeyMixin, %[\n      include_time_key true\n      ])\n\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    def test_time_format\n      format_check({\n        'time' => \"20100504\",\n        'a' => 1\n      })\n\n      d = create_driver(Fluent::SetTimeKeyMixin, %[\n      include_time_key true\n      time_format %Y%m%d\n      ])\n\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    def test_timezone_1\n      format_check({\n        'time' => \"2010-05-03T17:02:01-10:00\",\n        'a' => 1\n      })\n\n      d = create_driver(Fluent::SetTimeKeyMixin, %[\n      include_time_key true\n      timezone Pacific/Honolulu\n      ])\n\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    def test_timezone_2\n      format_check({\n        'time' => \"2010-05-04T08:32:01+05:30\",\n        'a' => 1\n      })\n\n      d = create_driver(Fluent::SetTimeKeyMixin, %[\n      include_time_key true\n      timezone +05:30\n      ])\n\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    def test_timezone_invalid\n      assert_raise(Fluent::ConfigError) do\n        create_driver(Fluent::SetTimeKeyMixin, %[\n        include_time_key true\n        timezone Invalid/Invalid\n        ])\n      end\n    end\n\n    sub_test_case \"mixin\" do\n      data(\n        'true' => true,\n        'false' => false)\n      test 'include_time_key' do |param|\n        d = create_driver(Fluent::SetTimeKeyMixin) {\n          config_set_default :include_time_key, param\n        }\n\n        assert_equal(param, d.instance.include_time_key)\n      end\n    end\n  end\n\n  class HandleTagMixinTest < Test::Unit::TestCase\n    include Utils\n\n    def test_add_tag_prefix\n      format_check({\n        'a' => 1\n      }, 'tag_prefix.test')\n      format_check({\n        'a' => 2\n      }, 'tag_prefix.test')\n\n      d = create_driver(Fluent::HandleTagNameMixin, %[\n        add_tag_prefix tag_prefix.\n        include_tag_key true\n      ])\n\n      d.emit({'a' => 1})\n      d.emit({'a' => 2})\n      d.run\n    end\n\n    def test_add_tag_suffix\n      format_check({\n        'a' => 1\n      }, 'test.test_suffix')\n      format_check({\n        'a' => 2\n      }, 'test.test_suffix')\n\n      d = create_driver(Fluent::HandleTagNameMixin, %[\n        add_tag_suffix .test_suffix\n        include_tag_key true\n      ])\n\n      d.emit({'a' => 1})\n      d.emit({'a' => 2})\n      d.run\n    end\n\n    def test_remove_tag_prefix\n      format_check({\n        'a' => 1\n      }, 'test')\n      format_check({\n        'a' => 2\n      }, 'test')\n\n      d = create_driver(Fluent::HandleTagNameMixin, %[\n        remove_tag_prefix te\n        include_tag_key true\n      ], \"tetest\")\n\n      d.emit({'a' => 1})\n      d.emit({'a' => 2})\n      d.run\n    end\n\n    def test_remove_tag_suffix\n      format_check({\n        'a' => 1\n      }, 'test')\n      format_check({\n        'a' => 2\n      }, 'test')\n\n      d = create_driver(Fluent::HandleTagNameMixin, %[\n        remove_tag_suffix st\n        include_tag_key true\n      ], \"testst\")\n\n      d.emit({'a' => 1})\n      d.emit({'a' => 2})\n      d.run\n    end\n\n    def test_mix_tag_handle\n      format_check({\n        'a' => 1\n      }, 'prefix.t')\n\n      d = create_driver(Fluent::HandleTagNameMixin, %[\n        remove_tag_prefix tes\n        add_tag_prefix prefix.\n      ])\n\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    def test_with_set_tag_key_mixin\n      format_check({\n        'tag' => 'tag_prefix.test',\n        'a' => 1\n      }, 'tag_prefix.test')\n\n      d = create_driver([Fluent::SetTagKeyMixin, Fluent::HandleTagNameMixin], %[\n        add_tag_prefix tag_prefix.\n        include_tag_key true\n      ])\n\n      d.emit({'a' => 1})\n      d.run\n    end\n\n    def test_with_set_tag_key_mixin_include_order_reverse\n      format_check({\n        'tag' => 'tag_prefix.test',\n        'a' => 1\n      }, 'tag_prefix.test')\n\n      d = create_driver([Fluent::HandleTagNameMixin, Fluent::SetTagKeyMixin], %[\n        add_tag_prefix tag_prefix.\n        include_tag_key true\n      ])\n\n      d.emit({'a' => 1})\n      d.run\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_msgpack_factory.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/msgpack_factory'\n\nclass MessagePackFactoryTest < Test::Unit::TestCase\n  test 'call log.warn only once' do\n    klass = Class.new do\n      include Fluent::MessagePackFactory::Mixin\n    end\n\n    mp = klass.new\n\n    mock.proxy($log).warn(anything).once\n\n    assert mp.msgpack_factory\n    assert mp.msgpack_factory\n    assert mp.msgpack_factory\n  end\n\n  sub_test_case 'thread_local_msgpack_packer' do\n    test 'packer is cached' do\n      packer1 = Fluent::MessagePackFactory.thread_local_msgpack_packer\n      packer2 = Fluent::MessagePackFactory.thread_local_msgpack_packer\n      assert_equal packer1, packer2\n    end\n  end\n\n  sub_test_case 'thread_local_msgpack_unpacker' do\n    test 'unpacker is cached' do\n      unpacker1 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker\n      unpacker2 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker\n      assert_equal unpacker1, unpacker2\n    end\n\n    # We need to reset the buffer every time so that received incomplete data\n    # must not affect data from other senders.\n    test 'reset the internal buffer of unpacker every time' do\n      unpacker1 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker\n      unpacker1.feed_each(\"\\xA6foo\") do |result|\n        flunk(\"This callback must not be called since the data is uncomplete.\")\n      end\n\n      records = []\n      unpacker2 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker\n      unpacker2.feed_each(\"\\xA3foo\") do |result|\n        records.append(result)\n      end\n      assert_equal [\"foo\"], records\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_oj_options.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/test'\nrequire 'fluent/oj_options'\n\nclass OjOptionsTest < ::Test::Unit::TestCase\n  begin\n    require 'oj'\n    @@oj_is_available = true\n  rescue LoadError\n    @@oj_is_available = false\n  end\n\n  setup do\n    @orig_env = {}\n    ENV.each do |key, value|\n      @orig_env[key] = value if key.start_with?(\"FLUENT_OJ_OPTION_\")\n    end\n  end\n\n  teardown do\n    ENV.delete_if { |key| key.start_with?(\"FLUENT_OJ_OPTION_\") }\n    @orig_env.each { |key, value| ENV[key] = value }\n  end\n\n  test \"available?\" do\n    assert_equal(@@oj_is_available, Fluent::OjOptions.available?)\n  end\n\n  sub_test_case \"set by environment variable\" do\n    test \"when no env vars set, returns default options\" do\n      ENV.delete_if { |key| key.start_with?(\"FLUENT_OJ_OPTION_\") }\n      defaults = Fluent::OjOptions::DEFAULTS\n      assert_equal(defaults, Fluent::OjOptions.load_env)\n      assert_equal(defaults, Oj.default_options.slice(*defaults.keys)) if @@oj_is_available\n    end\n\n    test \"valid env var passed with valid value, default is overridden\" do\n      ENV[\"FLUENT_OJ_OPTION_BIGDECIMAL_LOAD\"] = \":bigdecimal\"\n      assert_equal(:bigdecimal, Fluent::OjOptions.load_env[:bigdecimal_load])\n      assert_equal(:bigdecimal, Oj.default_options[:bigdecimal_load]) if @@oj_is_available\n    end\n\n    test \"valid env var passed with invalid value, default is not overridden\" do\n      ENV[\"FLUENT_OJ_OPTION_BIGDECIMAL_LOAD\"] = \":conor\"\n      assert_equal(:float, Fluent::OjOptions.load_env[:bigdecimal_load])\n      assert_equal(:float, Oj.default_options[:bigdecimal_load]) if @@oj_is_available\n    end\n\n    test \"invalid env var passed, nothing done with it\" do\n      ENV[\"FLUENT_OJ_OPTION_CONOR\"] = \":conor\"\n      assert_equal(nil, Fluent::OjOptions.load_env[:conor])\n      assert_equal(nil, Oj.default_options[:conor]) if @@oj_is_available\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_output.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/test'\nrequire 'fluent/output'\nrequire 'fluent/output_chain'\nrequire 'fluent/plugin/buffer'\nrequire 'timecop'\nrequire 'flexmock/test_unit'\n\nmodule FluentOutputTest\n  include Fluent\n  include FlexMock::TestCase\n\n  class BufferedOutputTest < ::Test::Unit::TestCase\n    include FluentOutputTest\n\n    class << self\n      def startup\n        $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), 'scripts'))\n        require 'fluent/plugin/out_test'\n        require 'fluent/plugin/out_test2'\n      end\n\n      def shutdown\n        $LOAD_PATH.shift\n      end\n    end\n\n    def setup\n      Fluent::Test.setup\n    end\n\n    CONFIG = %[]\n\n    def create_driver(conf=CONFIG)\n      Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do\n        def write(chunk)\n          chunk.read\n        end\n      end.configure(conf)\n    end\n\n    def test_configure\n      # default\n      d = create_driver\n      assert_equal 'memory', d.instance.buffer_type\n      assert_equal 60, d.instance.flush_interval\n      assert_equal false, d.instance.disable_retry_limit\n      assert_equal 17, d.instance.retry_limit\n      assert_equal 1.0, d.instance.retry_wait\n      assert_equal nil, d.instance.max_retry_wait\n      assert_equal 1.0, d.instance.retry_wait\n      assert_equal 1, d.instance.num_threads\n      assert_equal 1, d.instance.queued_chunk_flush_interval\n\n      # max_retry_wait\n      d = create_driver(CONFIG + %[max_retry_wait 4])\n      assert_equal 4, d.instance.max_retry_wait\n\n      # disable_retry_limit\n      d = create_driver(CONFIG + %[disable_retry_limit true])\n      assert_equal true, d.instance.disable_retry_limit\n\n      #### retry_state cares it\n      # # retry_wait is converted to Float for calc_retry_wait\n      # d = create_driver(CONFIG + %[retry_wait 1s])\n      # assert_equal Float, d.instance.retry_wait.class\n    end\n\n    class FormatterInjectTestOutput < Fluent::Output\n      def initialize\n        super\n        @formatter = nil\n      end\n    end\n    def test_start\n      i = FormatterInjectTestOutput.new\n      i.configure(config_element('ROOT', '', {}, [config_element('inject', '', {'hostname_key' => \"host\"})]))\n      assert_nothing_raised do\n        i.start\n      end\n    end\n\n    def create_mock_driver(conf=CONFIG)\n      Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do\n        attr_accessor :submit_flush_threads\n\n        def start_mock\n          @started = false\n          start\n          # ensure OutputThread to start successfully\n          submit_flush\n          sleep 0.5\n          while !@started\n            submit_flush\n            sleep 0.5\n          end\n        end\n\n        def try_flush\n          @started = true\n          @submit_flush_threads ||= {}\n          @submit_flush_threads[Thread.current] ||= 0\n          @submit_flush_threads[Thread.current] += 1\n        end\n\n        def write(chunk)\n          chunk.read\n        end\n      end.configure(conf)\n    end\n\n    def test_secondary\n      d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do\n        def write(chunk)\n          chunk.read\n        end\n      end\n\n      mock(d.instance.log).warn(\"Use different plugin for secondary. Check the plugin works with primary like secondary_file\",\n                                primary: d.instance.class.to_s, secondary: \"Fluent::Plugin::Test2Output\")\n      d.configure(CONFIG + %[\n        <secondary>\n          type test2\n          name c0\n        </secondary>\n      ])\n\n      assert_not_nil d.instance.instance_variable_get(:@secondary).router\n    end\n\n    def test_secondary_with_no_warn_log\n      # ObjectBufferedOutput doesn't implement `custom_filter`\n      d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ObjectBufferedOutput)\n\n      mock(d.instance.log).warn(\"Use different plugin for secondary. Check the plugin works with primary like secondary_file\",\n                                primary: d.instance.class.to_s, secondary: \"Fluent::Plugin::Test2Output\").never\n      d.configure(CONFIG + %[\n        <secondary>\n          type test2\n          name c0\n        </secondary>\n      ])\n\n      assert_not_nil d.instance.instance_variable_get(:@secondary).router\n    end\n\n    test 'BufferQueueLimitError compatibility' do\n      assert_equal Fluent::Plugin::Buffer::BufferOverflowError, Fluent::BufferQueueLimitError\n    end\n  end\n\n  class ObjectBufferedOutputTest < ::Test::Unit::TestCase\n    include FluentOutputTest\n\n    def setup\n      Fluent::Test.setup\n    end\n\n    CONFIG = %[]\n\n    def create_driver(conf=CONFIG)\n      Fluent::Test::OutputTestDriver.new(Fluent::ObjectBufferedOutput).configure(conf, true)\n    end\n\n    def test_configure\n      # default\n      d = create_driver\n      assert_equal true, d.instance.time_as_integer\n    end\n  end\n\n  class TimeSlicedOutputTest < ::Test::Unit::TestCase\n    include FluentOutputTest\n    include FlexMock::TestCase\n\n    def setup\n      Fluent::Test.setup\n      FileUtils.rm_rf(TMP_DIR)\n      FileUtils.mkdir_p(TMP_DIR)\n    end\n\n    TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/tmp/time_sliced_output\")\n\n    CONFIG = %[\n      buffer_path #{TMP_DIR}/foo\n      time_slice_format %Y%m%d%H\n    ]\n\n    class TimeSlicedOutputTestPlugin < Fluent::TimeSlicedOutput\n      attr_reader :written_chunk_keys, :errors_in_write\n      def initialize\n        super\n        @written_chunk_keys = []\n        @errors_in_write = []\n      end\n\n      def configure(conf)\n        super\n\n        @formatter = Fluent::Plugin.new_formatter('out_file')\n        @formatter.configure(conf)\n      end\n\n      def format(tag, time, record)\n        @formatter.format(tag, time, record)\n      end\n      def write(chunk)\n        @written_chunk_keys << chunk.key\n        true\n      rescue => e\n        @errors_in_write << e\n      end\n    end\n\n    def create_driver(conf=CONFIG)\n      Fluent::Test::TimeSlicedOutputTestDriver.new(TimeSlicedOutputTestPlugin).configure(conf, true)\n    end\n\n    data(:none => '',\n         :utc => \"utc\",\n         :localtime => 'localtime',\n         :timezone => 'timezone +0000')\n    test 'configure with timezone related parameters' do |param|\n      assert_nothing_raised {\n        create_driver(CONFIG + param)\n      }\n    end\n\n    sub_test_case \"test emit\" do\n      setup do\n        @time = Time.parse(\"2011-01-02 13:14:15 UTC\")\n        Timecop.freeze(@time)\n        @newline = if Fluent.windows?\n                     \"\\r\\n\"\n                   else\n                     \"\\n\"\n                   end\n      end\n\n      teardown do\n        Timecop.return\n      end\n\n      test \"emit with invalid event\" do\n        d = create_driver\n        d.instance.start\n        d.instance.after_start\n        assert_raise ArgumentError, \"time must be a Fluent::EventTime (or Integer)\" do\n          d.instance.emit_events('test', OneEventStream.new('string', 10))\n        end\n      end\n\n      test \"plugin can get key of chunk in #write\" do\n        d = create_driver\n        d.instance.start\n        d.instance.after_start\n        d.instance.emit_events('test', OneEventStream.new(event_time(\"2016-11-08 17:44:30 +0900\"), {\"message\" => \"yay\"}))\n        d.instance.force_flush\n        waiting(10) do\n          sleep 0.1 until d.instance.written_chunk_keys.size == 1\n        end\n        assert_equal [], d.instance.errors_in_write\n        assert_equal [\"2016110808\"], d.instance.written_chunk_keys # default timezone is UTC\n      end\n\n      test \"check formatted time compatibility with utc. Should Z, not +00:00\" do\n        d = create_driver(CONFIG + %[\n          utc\n          include_time_key\n        ])\n        time = Time.parse(\"2016-11-08 12:00:00 UTC\").to_i\n        d.emit({\"a\" => 1}, time)\n        d.expect_format %[2016-11-08T12:00:00Z\\ttest\\t{\"a\":1,\"time\":\"2016-11-08T12:00:00Z\"}#{@newline}]\n        d.run\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_plugin.rb",
    "content": "require_relative 'helper'\n\nrequire 'fluent/plugin'\nrequire 'fluent/plugin/input'\nrequire 'fluent/plugin/filter'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/buffer'\nrequire 'fluent/plugin/parser'\nrequire 'fluent/plugin/formatter'\nrequire 'fluent/plugin/storage'\n\nclass PluginTest < Test::Unit::TestCase\n  class Dummy1Input < Fluent::Plugin::Input\n    Fluent::Plugin.register_input('plugin_test_dummy1', self)\n  end\n  class Dummy2Input < Fluent::Plugin::Input\n    Fluent::Plugin.register_input('plugin_test_dummy2', self)\n    helpers :storage\n    config_section :storage do\n      config_set_default :@type, 'plugin_test_dummy1'\n    end\n    def multi_workers_ready?\n      true\n    end\n  end\n  class DummyFilter < Fluent::Plugin::Filter\n    Fluent::Plugin.register_filter('plugin_test_dummy', self)\n    helpers :parser, :formatter\n    config_section :parse do\n      config_set_default :@type, 'plugin_test_dummy'\n    end\n    config_section :format do\n      config_set_default :@type, 'plugin_test_dummy'\n    end\n    def filter(tag, time, record)\n      record\n    end\n  end\n  class Dummy1Output < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('plugin_test_dummy1', self)\n    def write(chunk)\n      # drop\n    end\n  end\n  class Dummy2Output < Fluent::Plugin::Output\n    Fluent::Plugin.register_output('plugin_test_dummy2', self)\n    config_section :buffer do\n      config_set_default :@type, 'plugin_test_dummy1'\n    end\n    def multi_workers_ready?\n      true\n    end\n    def write(chunk)\n      # drop\n    end\n  end\n  class Dummy1Buffer < Fluent::Plugin::Buffer\n    Fluent::Plugin.register_buffer('plugin_test_dummy1', self)\n  end\n  class Dummy2Buffer < Fluent::Plugin::Buffer\n    Fluent::Plugin.register_buffer('plugin_test_dummy2', self)\n    def multi_workers_ready?\n      false\n    end\n  end\n  class DummyParser < Fluent::Plugin::Parser\n    Fluent::Plugin.register_parser('plugin_test_dummy', self)\n  end\n  class DummyFormatter < Fluent::Plugin::Formatter\n    Fluent::Plugin.register_formatter('plugin_test_dummy', self)\n  end\n  class Dummy1Storage < Fluent::Plugin::Storage\n    Fluent::Plugin.register_storage('plugin_test_dummy1', self)\n  end\n  class Dummy2Storage < Fluent::Plugin::Storage\n    Fluent::Plugin.register_storage('plugin_test_dummy2', self)\n    def multi_workers_ready?\n      false\n    end\n  end\n  class DummyOwner < Fluent::Plugin::Base\n    include Fluent::PluginId\n    include Fluent::PluginLoggerMixin\n  end\n  class DummyEventRouter\n    def emit(tag, time, record); end\n    def emit_array(tag, array); end\n    def emit_stream(tag, es); end\n    def emit_error_event(tag, time, record, error); end\n  end\n\n  sub_test_case '#new_* methods' do\n    data(\n      input1: ['plugin_test_dummy1', Dummy1Input, :new_input],\n      input2: ['plugin_test_dummy2', Dummy2Input, :new_input],\n      filter: ['plugin_test_dummy', DummyFilter, :new_filter],\n      output1: ['plugin_test_dummy1', Dummy1Output, :new_output],\n      output2: ['plugin_test_dummy2', Dummy2Output, :new_output],\n    )\n    test 'returns plugin instances of registered plugin classes' do |(type, klass, m)|\n      instance = Fluent::Plugin.__send__(m, type)\n      assert_kind_of klass, instance\n    end\n\n    data(\n      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer],\n      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer],\n      parser: ['plugin_test_dummy', DummyParser, :new_parser],\n      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter],\n      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage],\n      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage],\n    )\n    test 'returns plugin instances of registered owned plugin classes' do |(type, klass, m)|\n      owner = DummyOwner.new\n      instance = Fluent::Plugin.__send__(m, type, parent: owner)\n      assert_kind_of klass, instance\n    end\n\n    data(\n      input1: ['plugin_test_dummy1', Dummy1Input, :new_input, nil],\n      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, nil],\n      filter: ['plugin_test_dummy', DummyFilter, :new_filter, nil],\n      output1: ['plugin_test_dummy1', Dummy1Output, :new_output, nil],\n      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, nil],\n      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer, {parent: DummyOwner.new}],\n      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer, {parent: DummyOwner.new}],\n      parser: ['plugin_test_dummy', DummyParser, :new_parser, {parent: DummyOwner.new}],\n      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter, {parent: DummyOwner.new}],\n      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage, {parent: DummyOwner.new}],\n      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage, {parent: DummyOwner.new}],\n    )\n    test 'returns plugin instances which are extended by FeatureAvailabilityChecker module' do |(type, _, m, kwargs)|\n      instance = if kwargs\n                   Fluent::Plugin.__send__(m, type, **kwargs)\n                 else\n                   Fluent::Plugin.__send__(m, type)\n                 end\n      assert_kind_of Fluent::Plugin::FeatureAvailabilityChecker, instance\n    end\n  end\n\n  sub_test_case 'with default system configuration' do\n    data(\n      input1: ['plugin_test_dummy1', Dummy1Input, :new_input, nil],\n      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, nil],\n      filter: ['plugin_test_dummy', DummyFilter, :new_filter, nil],\n      output1: ['plugin_test_dummy1', Dummy1Output, :new_output, nil],\n      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, nil],\n      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer, {parent: DummyOwner.new}],\n      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer, {parent: DummyOwner.new}],\n      parser: ['plugin_test_dummy', DummyParser, :new_parser, {parent: DummyOwner.new}],\n      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter, {parent: DummyOwner.new}],\n      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage, {parent: DummyOwner.new}],\n      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage, {parent: DummyOwner.new}],\n    )\n    test '#configure does not raise anything' do |(type, _, m, kwargs)|\n      instance = if kwargs\n                   Fluent::Plugin.__send__(m, type, **kwargs)\n                 else\n                   Fluent::Plugin.__send__(m, type)\n                 end\n      if instance.respond_to?(:context_router=)\n        instance.context_router = DummyEventRouter.new\n      end\n      assert_nothing_raised do\n        instance.configure(config_element())\n      end\n    end\n  end\n\n  sub_test_case 'with single worker configuration' do\n    data(\n      input1: ['plugin_test_dummy1', Dummy1Input, :new_input, nil],\n      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, nil],\n      filter: ['plugin_test_dummy', DummyFilter, :new_filter, nil],\n      output1: ['plugin_test_dummy1', Dummy1Output, :new_output, nil],\n      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, nil],\n      buffer1: ['plugin_test_dummy1', Dummy1Buffer, :new_buffer, {parent: DummyOwner.new}],\n      buffer2: ['plugin_test_dummy2', Dummy2Buffer, :new_buffer, {parent: DummyOwner.new}],\n      parser: ['plugin_test_dummy', DummyParser, :new_parser, {parent: DummyOwner.new}],\n      formatter: ['plugin_test_dummy', DummyFormatter, :new_formatter, {parent: DummyOwner.new}],\n      storage1: ['plugin_test_dummy1', Dummy1Storage, :new_storage, {parent: DummyOwner.new}],\n      storage2: ['plugin_test_dummy2', Dummy2Storage, :new_storage, {parent: DummyOwner.new}],\n    )\n    test '#configure does not raise anything' do |(type, _, m, kwargs)|\n      instance = if kwargs\n                   Fluent::Plugin.__send__(m, type, **kwargs)\n                 else\n                   Fluent::Plugin.__send__(m, type)\n                 end\n      if instance.respond_to?(:context_router=)\n        instance.context_router = DummyEventRouter.new\n      end\n      assert_nothing_raised do\n        instance.system_config_override('workers' => 1)\n        instance.configure(config_element())\n      end\n    end\n  end\n\n  sub_test_case 'with multi workers configuration' do\n    data(\n      input1: ['plugin_test_dummy1', Dummy1Input, :new_input],\n      output1: ['plugin_test_dummy1', Dummy1Output, :new_output],\n    )\n    test '#configure raise configuration error if plugins are not ready for multi workers' do |(type, klass, new_method)|\n      conf = config_element()\n      instance = Fluent::Plugin.__send__(new_method, type)\n      if instance.respond_to?(:context_router=)\n        instance.context_router = DummyEventRouter.new\n      end\n      assert_raise Fluent::ConfigError.new(\"Plugin '#{type}' does not support multi workers configuration (#{klass})\") do\n        instance.system_config_override('workers' => 3)\n        instance.configure(conf)\n      end\n    end\n\n    data(\n      input2: ['plugin_test_dummy2', Dummy2Input, :new_input], # with Dummy1Storage\n      filter: ['plugin_test_dummy', DummyFilter, :new_filter], # with DummyParser and DummyFormatter\n      output2: ['plugin_test_dummy2', Dummy2Output, :new_output], # with Dummy1Buffer\n    )\n    test '#configure does not raise any errors if plugins and its owned plugins are ready for multi workers' do |(type, _klass, new_method)|\n      conf = config_element()\n      instance = Fluent::Plugin.__send__(new_method, type)\n      if instance.respond_to?(:context_router=)\n        instance.context_router = DummyEventRouter.new\n      end\n      assert_nothing_raised do\n        instance.system_config_override('workers' => 3)\n        instance.configure(conf)\n      end\n    end\n\n    data(\n      input2: ['plugin_test_dummy2', Dummy2Input, :new_input, 'storage', 'plugin_test_dummy2', Dummy2Storage],\n      output2: ['plugin_test_dummy2', Dummy2Output, :new_output, 'buffer', 'plugin_test_dummy2', Dummy2Buffer],\n    )\n    test '#configure raise configuration error if configured owned plugins are not ready for multi workers' do |(type, _klass, new_method, subsection, subsection_type, problematic)|\n      conf = config_element('root', '', {}, [config_element(subsection, '', {'@type' => subsection_type})])\n      instance = Fluent::Plugin.__send__(new_method, type)\n      if instance.respond_to?(:context_router=)\n        instance.context_router = DummyEventRouter.new\n      end\n      assert_raise Fluent::ConfigError.new(\"Plugin '#{subsection_type}' does not support multi workers configuration (#{problematic})\") do\n        instance.system_config_override('workers' => 3)\n        instance.configure(conf)\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_plugin_classes.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/plugin/input'\nrequire 'fluent/plugin/output'\nrequire 'fluent/plugin/bare_output'\nrequire 'fluent/plugin/filter'\n\nmodule FluentTest\n  class FluentTestCounterMetrics < Fluent::Plugin::Metrics\n    Fluent::Plugin.register_metrics('test_counter', self)\n\n    attr_reader :data\n\n    def initialize\n      super\n      @data = 0\n    end\n    def get\n      @data\n    end\n    def inc\n      @data +=1\n    end\n    def add(value)\n      @data += value\n    end\n    def set(value)\n      @data = value\n    end\n    def close\n      @data = 0\n      super\n    end\n  end\n\n  class FluentTestGaugeMetrics < Fluent::Plugin::Metrics\n    Fluent::Plugin.register_metrics('test_gauge', self)\n\n    attr_reader :data\n\n    def initialize\n      super\n      @data = 0\n    end\n    def get\n      @data\n    end\n    def inc\n      @data += 1\n    end\n    def dec\n      @data -=1\n    end\n    def add(value)\n      @data += value\n    end\n    def sub(value)\n      @data -= value\n    end\n    def set(value)\n      @data = value\n    end\n    def close\n      @data = 0\n      super\n    end\n  end\n\n  class FluentTestInput < ::Fluent::Plugin::Input\n    ::Fluent::Plugin.register_input('test_in', self)\n\n    attr_reader :started\n\n    def initialize\n      super\n      # stub metrics instances\n      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new\n      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new\n    end\n\n    def start\n      super\n      @started = true\n    end\n\n    def shutdown\n      @started = false\n      super\n    end\n  end\n\n  class FluentTestGenInput < ::Fluent::Plugin::Input\n    ::Fluent::Plugin.register_input('test_in_gen', self)\n\n    helpers :thread\n\n    attr_reader :started\n\n    config_param :num, :integer, default: 10000\n    config_param :interval_sec, :float, default: nil\n    config_param :async, :bool, default: false\n\n    def initialize\n      super\n      # stub metrics instances\n      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new\n      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new\n    end\n\n    def multi_workers_ready?\n      true\n    end\n\n    def zero_downtime_restart_ready?\n      true\n    end\n\n    def start\n      super\n      @started = true\n\n      if @async\n        thread_create(:test_in_gen, &method(:emit))\n      else\n        emit\n      end\n    end\n\n    def emit\n      @num.times { |i|\n        break if @async and not thread_current_running?\n        router.emit(\"test.event\", Fluent::EventTime.now, {'message' => 'Hello!', 'key' => \"value#{i}\", 'num' => i})\n        sleep @interval_sec if @interval_sec\n      }\n    end\n\n    def shutdown\n      @started = false\n      super\n    end\n  end\n\n  class FluentTestOutput < ::Fluent::Plugin::Output\n    ::Fluent::Plugin.register_output('test_out', self)\n\n    def initialize\n      super\n      @events = Hash.new { |h, k| h[k] = [] }\n      # stub metrics instances\n      @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new\n      @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new\n      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new\n      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new\n      @write_count_metrics = FluentTest::FluentTestCounterMetrics.new\n      @write_secondary_count_metrics = FluentTest::FluentTestCounterMetrics.new\n      @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new\n      @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new\n      @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new\n    end\n\n    attr_reader :events\n    attr_reader :started\n\n    def start\n      super\n      @started = true\n    end\n\n    def shutdown\n      @started = false\n      super\n    end\n\n    def process(tag, es)\n      es.each do |time, record|\n        @events[tag] << record\n      end\n    end\n  end\n\n  class FluentTestDynamicOutput < ::Fluent::Plugin::BareOutput\n    ::Fluent::Plugin.register_output('test_dynamic_out', self)\n\n    attr_reader :child\n    attr_reader :started\n\n    def start\n      super\n      @started = true\n      @child = Fluent::Plugin.new_output('copy')\n      conf = config_element('DYNAMIC', '', {}, [\n          config_element('store', '', {'@type' => 'test_out', '@id' => 'dyn_out1'}),\n          config_element('store', '', {'@type' => 'test_out', '@id' => 'dyn_out2'}),\n      ])\n      @child.configure(conf)\n      @child.start\n    end\n\n    def after_start\n      super\n      @child.after_start\n    end\n\n    def stop\n      super\n      @child.stop\n    end\n\n    def before_shutdown\n      super\n      @child.before_shutdown\n    end\n\n    def shutdown\n      @started = false\n      super\n      @child.shutdown\n    end\n\n    def after_shutdown\n      super\n      @child.after_shutdown\n    end\n\n    def close\n      super\n      @child.close\n    end\n\n    def terminate\n      super\n      @child.terminate\n    end\n\n    def process(tag, es)\n      es.each do |time, record|\n        @events[tag] << record\n      end\n    end\n  end\n\n  class FluentTestBufferedOutput < ::Fluent::Plugin::Output\n    ::Fluent::Plugin.register_output('test_out_buffered', self)\n\n    attr_reader :started\n\n    def start\n      super\n      @started = true\n    end\n\n    def shutdown\n      @started = false\n      super\n    end\n\n    def write(chunk)\n      # drop everything\n    end\n  end\n\n  class FluentTestEmitOutput < ::Fluent::Plugin::Output\n    ::Fluent::Plugin.register_output('test_out_emit', self)\n    helpers :event_emitter\n    def write(chunk)\n      tag = chunk.metadata.tag || 'test'\n      array = []\n      chunk.each do |time, record|\n        array << [time, record]\n      end\n      router.emit_array(tag, array)\n    end\n  end\n\n  class FluentTestErrorOutput < ::Fluent::Plugin::Output\n    ::Fluent::Plugin.register_output('test_out_error', self)\n\n    def initialize\n      super\n      # stub metrics instances\n      @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new\n      @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new\n      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new\n      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new\n      @write_count_metrics = FluentTest::FluentTestCounterMetrics.new\n      @write_secondary_count_metrics = FluentTest::FluentTestCounterMetrics.new\n      @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new\n      @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new\n      @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new\n    end\n\n    def format(tag, time, record)\n      raise \"emit error!\"\n    end\n\n    def write(chunk)\n      raise \"chunk error!\"\n    end\n  end\n\n  class FluentCompatTestFilter < ::Fluent::Filter\n    ::Fluent::Plugin.register_filter('test_compat_filter', self)\n\n    def initialize(field = '__test__')\n      super()\n      @num = 0\n      @field = field\n      # stub metrics instances\n      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new\n      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new\n    end\n\n    attr_reader :num\n    attr_reader :started\n\n    def start\n      super\n      @started = true\n    end\n\n    def shutdown\n      @started = false\n      super\n    end\n\n    def filter(tag, time, record)\n      record[@field] = @num\n      @num += 1\n      record\n    end\n  end\n\n  class FluentTestFilter < ::Fluent::Plugin::Filter\n    ::Fluent::Plugin.register_filter('test_filter', self)\n\n    def initialize(field = '__test__')\n      super()\n      @num = 0\n      @field = field\n      # stub metrics instances\n      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new\n      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new\n    end\n\n    attr_reader :num\n    attr_reader :started\n\n    def start\n      super\n      @started = true\n    end\n\n    def shutdown\n      @started = false\n      super\n    end\n\n    def filter(tag, time, record)\n      record[@field] = @num\n      @num += 1\n      record\n    end\n  end\n\n  class FluentTestBuffer < Fluent::Plugin::Buffer\n    ::Fluent::Plugin.register_buffer('test_buffer', self)\n\n    def resume\n      return {}, []\n    end\n\n    def generate_chunk(metadata)\n    end\n\n    def multi_workers_ready?\n      false\n    end\n  end\n\n  class TestEmitErrorHandler\n    def initialize\n      @events = Hash.new { |h, k| h[k] = [] }\n    end\n\n    attr_reader :events\n\n    def handle_emit_error(tag, time, record, error)\n      @events[tag] << record\n    end\n\n    def handle_emits_error(tag, es, error)\n      es.each { |time,record| handle_emit_error(tag, time, record, error) }\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_plugin_helper.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/plugin_helper'\nrequire 'fluent/plugin/base'\n\nclass PluginHelperTest < Test::Unit::TestCase\n  module FluentTest; end\n\n  sub_test_case 'Fluent::Plugin::Base.helpers method works as shortcut to include helper modules' do\n    class FluentTest::PluginTest1 < Fluent::Plugin::TestBase\n      helpers :event_emitter\n    end\n    class FluentTest::PluginTest2 < Fluent::Plugin::TestBase\n      helpers :thread\n    end\n    class FluentTest::PluginTest3 < Fluent::Plugin::TestBase\n      helpers :event_loop\n    end\n    class FluentTest::PluginTest4 < Fluent::Plugin::TestBase\n      helpers :timer\n    end\n    class FluentTest::PluginTest5 < Fluent::Plugin::TestBase\n      helpers :child_process\n    end\n    class FluentTest::PluginTest6 < Fluent::Plugin::TestBase\n      helpers :retry_state\n    end\n    class FluentTest::PluginTest0 < Fluent::Plugin::TestBase\n      helpers :event_emitter, :thread, :event_loop, :timer, :child_process, :retry_state\n    end\n\n    test 'plugin can include helper event_emitter' do\n      assert FluentTest::PluginTest1.include?(Fluent::PluginHelper::EventEmitter)\n      p1 = FluentTest::PluginTest1.new\n      assert p1.respond_to?(:has_router?)\n      assert p1.has_router?\n    end\n\n    test 'plugin can include helper thread' do\n      assert FluentTest::PluginTest2.include?(Fluent::PluginHelper::Thread)\n      p2 = FluentTest::PluginTest2.new\n      assert p2.respond_to?(:thread_current_running?)\n      assert p2.respond_to?(:thread_create)\n    end\n\n    test 'plugin can include helper event_loop' do\n      assert FluentTest::PluginTest3.include?(Fluent::PluginHelper::EventLoop)\n      p3 = FluentTest::PluginTest3.new\n      assert p3.respond_to?(:event_loop_attach)\n      assert p3.respond_to?(:event_loop_running?)\n    end\n\n    test 'plugin can include helper timer' do\n      assert FluentTest::PluginTest4.include?(Fluent::PluginHelper::Timer)\n      p4 = FluentTest::PluginTest4.new\n      assert p4.respond_to?(:timer_execute)\n    end\n\n    test 'plugin can include helper child_process' do\n      assert FluentTest::PluginTest5.include?(Fluent::PluginHelper::ChildProcess)\n      p5 = FluentTest::PluginTest5.new\n      assert p5.respond_to?(:child_process_execute)\n    end\n\n    test 'plugin can 2 or more helpers at once' do\n      assert FluentTest::PluginTest0.include?(Fluent::PluginHelper::EventEmitter)\n      assert FluentTest::PluginTest0.include?(Fluent::PluginHelper::Thread)\n      assert FluentTest::PluginTest0.include?(Fluent::PluginHelper::EventLoop)\n      assert FluentTest::PluginTest0.include?(Fluent::PluginHelper::Timer)\n      assert FluentTest::PluginTest0.include?(Fluent::PluginHelper::ChildProcess)\n\n      p0 = FluentTest::PluginTest0.new\n      assert p0.respond_to?(:child_process_execute)\n      assert p0.respond_to?(:timer_execute)\n      assert p0.respond_to?(:event_loop_attach)\n      assert p0.respond_to?(:event_loop_running?)\n      assert p0.respond_to?(:thread_current_running?)\n      assert p0.respond_to?(:thread_create)\n      assert p0.respond_to?(:has_router?)\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_plugin_id.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/plugin/base'\nrequire 'fluent/system_config'\nrequire 'fileutils'\n\nclass PluginIdTest < Test::Unit::TestCase\n  TMP_DIR = File.expand_path(File.dirname(__FILE__) + \"/tmp/plugin_id/#{ENV['TEST_ENV_NUMBER']}\")\n\n  class MyPlugin < Fluent::Plugin::Base\n    include Fluent::PluginId\n  end\n\n  setup do\n    @p = MyPlugin.new\n  end\n\n  sub_test_case '#plugin_id_for_test?' do\n    test 'returns true always in test files' do\n      assert @p.plugin_id_for_test?\n    end\n\n    test 'returns false always out of test files' do\n      # TODO: no good way to write this test....\n    end\n  end\n\n  sub_test_case 'configured without @id' do\n    setup do\n      @p.configure(config_element())\n    end\n\n    test '#plugin_id_configured? returns false' do\n      assert_false @p.plugin_id_configured?\n    end\n\n    test '#plugin_id returns object_id based string' do\n      assert_kind_of String, @p.plugin_id\n      assert @p.plugin_id =~ /^object:[0-9a-f]+/\n    end\n\n    test '#plugin_root_dir returns nil' do\n      assert_nil @p.plugin_root_dir\n    end\n  end\n\n  sub_test_case 'configured with @id' do\n    setup do\n      FileUtils.rm_rf(TMP_DIR)\n      FileUtils.mkdir_p(TMP_DIR)\n      @p.configure(config_element('ROOT', '', {'@id' => 'testing_plugin_id'}))\n    end\n\n    test '#plugin_id_configured? returns true' do\n      assert @p.plugin_id_configured?\n    end\n\n    test '#plugin_id returns the configured value' do\n      assert_equal 'testing_plugin_id', @p.plugin_id\n    end\n\n    test '#plugin_root_dir returns nil without system root directory configuration' do\n      assert_nil @p.plugin_root_dir\n    end\n\n    test '#plugin_root_dir returns an existing directory path frozen String' do\n      root_dir = Fluent::SystemConfig.overwrite_system_config('root_dir' => File.join(TMP_DIR, \"myroot\")) do\n        @p.plugin_root_dir\n      end\n      assert_kind_of String, root_dir\n      assert Dir.exist?(root_dir)\n      assert root_dir =~ %r!/worker0/!\n      assert root_dir.frozen?\n    end\n\n    test '#plugin_root_dir returns the same value for 2nd or more call' do\n      root_dir = Fluent::SystemConfig.overwrite_system_config('root_dir' => File.join(TMP_DIR, \"myroot\")) do\n        @p.plugin_root_dir\n      end\n      twice = Fluent::SystemConfig.overwrite_system_config('root_dir' => File.join(TMP_DIR, \"myroot\")) do\n        @p.plugin_root_dir\n      end\n      assert_equal root_dir.object_id, twice.object_id\n    end\n\n    test '#plugin_root_dir refers SERVERENGINE_WORKER_ID environment path to create it' do\n      prev_env_val = ENV['SERVERENGINE_WORKER_ID']\n      begin\n        ENV['SERVERENGINE_WORKER_ID'] = '7'\n        root_dir = Fluent::SystemConfig.overwrite_system_config('root_dir' => File.join(TMP_DIR, \"myroot\")) do\n          @p.plugin_root_dir\n        end\n        assert_kind_of String, root_dir\n        assert Dir.exist?(root_dir)\n        assert root_dir =~ %r!/worker7/!\n        assert root_dir.frozen?\n      ensure\n        ENV['SERVERENGINE_WORKER_ID'] = prev_env_val\n      end\n    end\n\n    test '#plugin_root_dir create directory with specify mode if not exists ' do\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      root_dir = Fluent::SystemConfig.overwrite_system_config({ 'root_dir' => File.join(TMP_DIR, \"myroot\"), 'dir_permission' => '0777' }) do\n        @p.plugin_root_dir\n      end\n\n      assert_equal '777', File.stat(root_dir).mode.to_s(8)[-3, 3]\n    end\n\n    test '#plugin_root_dir create directory with default permission if not exists ' do\n      root_dir = Fluent::SystemConfig.overwrite_system_config({ 'root_dir' => File.join(TMP_DIR, \"myroot\") }) do\n        @p.plugin_root_dir\n      end\n\n      assert_equal '755', File.stat(root_dir).mode.to_s(8)[-3, 3]\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_process.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/process'\n\nclass ProcessCompatibilityTest < ::Test::Unit::TestCase\n  test 'DetachProcessMixin is defined' do\n    assert defined?(::Fluent::DetachProcessMixin)\n    assert_equal ::Fluent::DetachProcessMixin, ::Fluent::Compat::DetachProcessMixin\n  end\n\n  test 'DetachMultiProcessMixin is defined' do\n    assert defined?(::Fluent::DetachMultiProcessMixin)\n    assert_equal ::Fluent::DetachMultiProcessMixin, ::Fluent::Compat::DetachMultiProcessMixin\n  end\nend\n"
  },
  {
    "path": "test/test_root_agent.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/event_router'\nrequire 'fluent/system_config'\nrequire 'timecop'\nrequire_relative 'test_plugin_classes'\n\nclass RootAgentTest < ::Test::Unit::TestCase\n  include Fluent\n  include FluentTest\n\n  def test_initialize\n    ra = RootAgent.new(log: $log)\n    assert_equal 0, ra.instance_variable_get(:@suppress_emit_error_log_interval)\n    assert_nil ra.instance_variable_get(:@next_emit_error_log_time)\n  end\n\n  data(\n    'suppress interval' => [{'emit_error_log_interval' => 30}, {:@suppress_emit_error_log_interval => 30}],\n    'without source' => [{'without_source' => true}, {:@without_source => true}],\n    'enable input metrics' => [{'enable_input_metrics' => true}, {:@enable_input_metrics => true}],\n    'disable input metrics' => [{'enable_input_metrics' => false}, {:@enable_input_metrics => false}],\n    )\n  def test_initialize_with_opt(data)\n    opt, expected = data\n    ra = RootAgent.new(log: $log, system_config: SystemConfig.new(opt))\n    expected.each { |k, v|\n      assert_equal v, ra.instance_variable_get(k)\n    }\n  end\n\n  sub_test_case 'configure' do\n    setup do\n      @ra = RootAgent.new(log: $log)\n      stub(Engine).root_agent { @ra }\n    end\n\n    def configure_ra(conf_str)\n      conf = Config.parse(conf_str, \"(test)\", \"(test_dir)\", true)\n      @ra.configure(conf)\n      @ra\n    end\n\n    test 'empty' do\n      ra = configure_ra('')\n      assert_empty ra.inputs\n      assert_empty ra.labels\n      assert_empty ra.outputs\n      assert_empty ra.filters\n      assert_nil ra.context\n      assert_nil ra.error_collector\n    end\n\n    test 'raises configuration error for missing type of source' do\n      conf = <<-EOC\n<source>\n</source>\nEOC\n      errmsg = \"Missing '@type' parameter on <source> directive\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error for missing type of match' do\n      conf = <<-EOC\n<source>\n  @type test_in\n</source>\n<match *.**>\n</match>\nEOC\n      errmsg = \"Missing '@type' parameter on <match> directive\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error for missing type of filter' do\n      conf = <<-EOC\n<source>\n  @type test_in\n</source>\n<filter *.**>\n</filter>\n<match *.**>\n  @type test_out\n</match>\nEOC\n      errmsg = \"Missing '@type' parameter on <filter> directive\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error if there are two same label section' do\n      conf = <<-EOC\n<source>\n  @type test_in\n  @label @test\n</source>\n<label @test>\n  @type test_out\n</label>\n<label @test>\n  @type test_out\n</label>\nEOC\n      errmsg = \"Section <label @test> appears twice\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error for label without name' do\n      conf = <<-EOC\n<label>\n  @type test_out\n</label>\nEOC\n      errmsg = \"Missing symbol argument on <label> directive\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error for <label @ROOT>' do\n      conf = <<-EOC\n<source>\n  @type test_in\n  @label @ROOT\n</source>\n<label @ROOT>\n  @type test_out\n</label>\nEOC\n      errmsg = \"@ROOT for <label> is not permitted, reserved for getting root router\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error if there are not match sections in label section' do\n      conf = <<-EOC\n<source>\n  @type test_in\n  @label @test\n</source>\n<label @test>\n  @type test_out\n</label>\nEOC\n      errmsg = \"Missing <match> sections in <label @test> section\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        configure_ra(conf)\n      end\n    end\n\n    test 'with plugins' do\n      # check @type and type in one configuration\n      conf = <<-EOC\n<source>\n  @type test_in\n  @id test_in\n</source>\n<filter>\n  type test_filter\n  id test_filter\n</filter>\n<match **>\n  @type relabel\n  @id test_relabel\n  @label @test\n</match>\n<label @test>\n  <match **>\n    type test_out\n    id test_out\n  </match>\n</label>\n<label @ERROR>\n  <match>\n    @type null\n  </match>\n</label>\nEOC\n      ra = configure_ra(conf)\n      assert_kind_of FluentTestInput, ra.inputs.first\n      assert_kind_of Plugin::RelabelOutput, ra.outputs.first\n      assert_kind_of FluentTestFilter, ra.filters.first\n      assert ra.error_collector\n\n      %W(@test @ERROR).each { |label_symbol|\n        assert_include ra.labels, label_symbol\n        assert_kind_of Label, ra.labels[label_symbol]\n      }\n\n      test_label = ra.labels['@test']\n      assert_kind_of FluentTestOutput, test_label.outputs.first\n      assert_equal ra, test_label.root_agent\n\n      error_label = ra.labels['@ERROR']\n      assert_kind_of Fluent::Plugin::NullOutput, error_label.outputs.first\n    end\n  end\n\n  sub_test_case 'start/shutdown' do\n    def setup_root_agent(conf)\n      ra = RootAgent.new(log: $log)\n      stub(Engine).root_agent { ra }\n      ra.configure(Config.parse(conf, \"(test)\", \"(test_dir)\", true))\n      ra\n    end\n\n    test 'plugin status' do\n      ra = setup_root_agent(<<-EOC)\n<source>\n  @type test_in\n  @id test_in\n</source>\n<filter>\n  type test_filter\n  id test_filter\n</filter>\n<match **>\n  @type test_out\n  @id test_out\n</match>\nEOC\n      ra.start\n      assert_true ra.inputs.first.started\n      assert_true ra.filters.first.started\n      assert_true ra.outputs.first.started\n\n      ra.shutdown\n      assert_false ra.inputs.first.started\n      assert_false ra.filters.first.started\n      assert_false ra.outputs.first.started\n    end\n\n    test 'output plugin threads should run before input plugin is blocked with buffer full' do\n      ra = setup_root_agent(<<-EOC)\n<source>\n  @type test_in_gen\n  @id test_in_gen\n</source>\n<match **>\n  @type test_out_buffered\n  @id test_out_buffered\n  <buffer>\n    chunk_limit_size 1k\n    queue_limit_length 2\n    flush_thread_count 2\n    overflow_action block\n  </buffer>\n</match>\nEOC\n      waiting(5) { ra.start }\n      assert_true ra.inputs.first.started\n      assert_true ra.outputs.first.started\n\n      ra.shutdown\n      assert_false ra.inputs.first.started\n      assert_false ra.outputs.first.started\n    end\n  end\n\n  sub_test_case 'configured with label and secondary plugin' do\n    setup do\n      @ra = RootAgent.new(log: $log)\n      stub(Engine).root_agent{ @ra }\n      @ra.configure(Config.parse(<<-EOC, \"(test)\", \"(test_dir)\", true))\n<source>\n  @type test_in\n  @label @route_a\n</source>\n<label @route_a>\n  <match a.**>\n    @type test_out_buffered\n    <secondary>\n      @type test_out_emit\n    </secondary>\n  </match>\n</label>\n<label @route_b>\n  <match b.**>\n    @type test_out\n  </match>\n</label>\nEOC\n    end\n\n    test 'secondary plugin has an event router for the label which the plugin is in' do\n      assert_equal 1, @ra.inputs.size\n      assert_equal 2, @ra.labels.size\n      assert_equal ['@route_a', '@route_b'], @ra.labels.keys\n      assert_equal '@route_a', @ra.labels['@route_a'].context\n      assert_equal '@route_b', @ra.labels['@route_b'].context\n\n      c1 = @ra.labels['@route_a']\n\n      assert_equal 1, c1.outputs.size\n      assert !c1.outputs.first.has_router?\n\n      assert c1.outputs.first.secondary\n      assert c1.outputs.first.secondary.has_router?\n      assert_equal c1.event_router, c1.outputs.first.secondary.router\n    end\n  end\n\n  sub_test_case 'configured with label and secondary plugin with @label specifier' do\n    setup do\n      @ra = RootAgent.new(log: $log)\n      stub(Engine).root_agent{ @ra }\n      @ra.configure(Config.parse(<<-EOC, \"(test)\", \"(test_dir)\", true))\n<source>\n  @type test_in\n  @label @route_a\n</source>\n<label @route_a>\n  <match a.**>\n    @type test_out_buffered\n    <secondary>\n      @type test_out_emit\n      @label @route_b\n    </secondary>\n  </match>\n</label>\n<label @route_b>\n  <match b.**>\n    @type test_out\n  </match>\n</label>\nEOC\n    end\n\n    test 'secondary plugin has an event router for the label specified in secondary section' do\n      assert_equal 1, @ra.inputs.size\n      assert_equal 2, @ra.labels.size\n      assert_equal ['@route_a', '@route_b'], @ra.labels.keys\n      assert_equal '@route_a', @ra.labels['@route_a'].context\n      assert_equal '@route_b', @ra.labels['@route_b'].context\n\n      c1 = @ra.labels['@route_a']\n      c2 = @ra.labels['@route_b']\n\n      assert_equal 1, c1.outputs.size\n      assert !c1.outputs.first.has_router?\n\n      assert c1.outputs.first.secondary\n      assert c1.outputs.first.secondary.has_router?\n      assert_equal c2.event_router, c1.outputs.first.secondary.router\n    end\n  end\n\n  sub_test_case 'configured with label and secondary plugin with @label specifier in primary output' do\n    setup do\n      @ra = RootAgent.new(log: $log)\n      stub(Engine).root_agent{ @ra }\n      @ra.configure(Config.parse(<<-EOC, \"(test)\", \"(test_dir)\", true))\n<source>\n  @type test_in\n  @label @route_a\n</source>\n<label @route_a>\n  <match a.**>\n    @type test_out_emit\n    @label @route_b\n    <secondary>\n      @type test_out_emit\n    </secondary>\n  </match>\n</label>\n<label @route_b>\n  <match b.**>\n    @type test_out\n  </match>\n</label>\nEOC\n    end\n\n    test 'secondary plugin has an event router for the label specified in secondary section' do\n      assert_equal 1, @ra.inputs.size\n      assert_equal 2, @ra.labels.size\n      assert_equal ['@route_a', '@route_b'], @ra.labels.keys\n      assert_equal '@route_a', @ra.labels['@route_a'].context\n      assert_equal '@route_b', @ra.labels['@route_b'].context\n\n      c1 = @ra.labels['@route_a']\n      c2 = @ra.labels['@route_b']\n\n      assert_equal 1, c1.outputs.size\n      assert c1.outputs.first.secondary\n\n      p1 = c1.outputs.first\n      assert p1.has_router?\n      assert_equal c1.event_router, p1.context_router\n      assert_equal c2.event_router, p1.router\n\n      s1 = p1.secondary\n      assert s1.has_router?\n      assert_equal c1.event_router, s1.context_router\n      assert_equal c2.event_router, s1.router\n    end\n  end\n\n  sub_test_case 'configured with MultiOutput plugins' do\n    setup do\n      @ra = RootAgent.new(log: $log)\n      stub(Engine).root_agent { @ra }\n      @ra.configure(Config.parse(<<-EOC, \"(test)\", \"(test_dir)\", true))\n<source>\n  @type test_in\n  @id test_in\n</source>\n<filter>\n  @type test_filter\n  @id test_filter\n</filter>\n<match **>\n  @type copy\n  @id test_copy\n  <store>\n    @type test_out\n    @id test_out1\n  </store>\n  <store>\n    @type test_out\n    @id test_out2\n  </store>\n</match>\nEOC\n      @ra\n    end\n\n    test 'plugin status with multi output' do\n      assert_equal 1, @ra.inputs.size\n      assert_equal 1, @ra.filters.size\n      assert_equal 3, @ra.outputs.size\n\n      @ra.start\n      assert_equal [true], @ra.inputs.map{|i| i.started? }\n      assert_equal [true], @ra.filters.map{|i| i.started? }\n      assert_equal [true, true, true], @ra.outputs.map{|i| i.started? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.after_started? }\n      assert_equal [true], @ra.filters.map{|i| i.after_started? }\n      assert_equal [true, true, true], @ra.outputs.map{|i| i.after_started? }\n\n      @ra.shutdown\n      assert_equal [true], @ra.inputs.map{|i| i.stopped? }\n      assert_equal [true], @ra.filters.map{|i| i.stopped? }\n      assert_equal [true, true, true], @ra.outputs.map{|i| i.stopped? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.before_shutdown? }\n      assert_equal [true], @ra.filters.map{|i| i.before_shutdown? }\n      assert_equal [true, true, true], @ra.outputs.map{|i| i.before_shutdown? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.shutdown? }\n      assert_equal [true], @ra.filters.map{|i| i.shutdown? }\n      assert_equal [true, true, true], @ra.outputs.map{|i| i.shutdown? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.after_shutdown? }\n      assert_equal [true], @ra.filters.map{|i| i.after_shutdown? }\n      assert_equal [true, true, true], @ra.outputs.map{|i| i.after_shutdown? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.closed? }\n      assert_equal [true], @ra.filters.map{|i| i.closed? }\n      assert_equal [true, true, true], @ra.outputs.map{|i| i.closed? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.terminated? }\n      assert_equal [true], @ra.filters.map{|i| i.terminated? }\n      assert_equal [true, true, true], @ra.outputs.map{|i| i.terminated? }\n    end\n  end\n\n  sub_test_case 'configured with MultiOutput plugins and labels' do\n    setup do\n      @ra = RootAgent.new(log: $log)\n      stub(Engine).root_agent { @ra }\n      @ra.configure(Config.parse(<<-EOC, \"(test)\", \"(test_dir)\", true))\n<source>\n  @type test_in\n  @id test_in\n  @label @testing\n</source>\n<label @testing>\n  <filter>\n    @type test_filter\n    @id test_filter\n  </filter>\n  <match **>\n    @type copy\n    @id test_copy\n    <store>\n      @type test_out\n      @id test_out1\n    </store>\n    <store>\n      @type test_out\n      @id test_out2\n    </store>\n  </match>\n</label>\nEOC\n      @ra\n    end\n\n    test 'plugin status with multi output' do\n      assert_equal 1, @ra.inputs.size\n      assert_equal 0, @ra.filters.size\n      assert_equal 0, @ra.outputs.size\n      assert_equal 1, @ra.labels.size\n      assert_equal '@testing', @ra.labels.keys.first\n      assert_equal 1, @ra.labels.values.first.filters.size\n      assert_equal 3, @ra.labels.values.first.outputs.size\n\n      label_filters = @ra.labels.values.first.filters\n      label_outputs = @ra.labels.values.first.outputs\n\n      @ra.start\n      assert_equal [true], @ra.inputs.map{|i| i.started? }\n      assert_equal [true], label_filters.map{|i| i.started? }\n      assert_equal [true, true, true], label_outputs.map{|i| i.started? }\n\n      @ra.shutdown\n      assert_equal [true], @ra.inputs.map{|i| i.stopped? }\n      assert_equal [true], label_filters.map{|i| i.stopped? }\n      assert_equal [true, true, true], label_outputs.map{|i| i.stopped? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.before_shutdown? }\n      assert_equal [true], label_filters.map{|i| i.before_shutdown? }\n      assert_equal [true, true, true], label_outputs.map{|i| i.before_shutdown? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.shutdown? }\n      assert_equal [true], label_filters.map{|i| i.shutdown? }\n      assert_equal [true, true, true], label_outputs.map{|i| i.shutdown? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.after_shutdown? }\n      assert_equal [true], label_filters.map{|i| i.after_shutdown? }\n      assert_equal [true, true, true], label_outputs.map{|i| i.after_shutdown? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.closed? }\n      assert_equal [true], label_filters.map{|i| i.closed? }\n      assert_equal [true, true, true], label_outputs.map{|i| i.closed? }\n\n      assert_equal [true], @ra.inputs.map{|i| i.terminated? }\n      assert_equal [true], label_filters.map{|i| i.terminated? }\n      assert_equal [true, true, true], label_outputs.map{|i| i.terminated? }\n    end\n\n    test 'plugin #shutdown is not called twice' do\n      assert_equal 1, @ra.inputs.size\n      assert_equal 0, @ra.filters.size\n      assert_equal 0, @ra.outputs.size\n      assert_equal 1, @ra.labels.size\n      assert_equal '@testing', @ra.labels.keys.first\n      assert_equal 1, @ra.labels.values.first.filters.size\n      assert_equal 3, @ra.labels.values.first.outputs.size\n\n      @ra.start\n\n      old_level = @ra.log.level\n      begin\n        @ra.log.instance_variable_get(:@logger).level = Fluent::Log::LEVEL_INFO - 1\n        assert_equal Fluent::Log::LEVEL_INFO, @ra.log.level\n\n        @ra.log.out.flush_logs = false\n\n        @ra.shutdown\n\n        test_out1_shutdown_logs = @ra.log.out.logs.select{|line| line =~ /shutting down output plugin type=:test_out plugin_id=\"test_out1\"/ }\n        assert_equal 1, test_out1_shutdown_logs.size\n      ensure\n        @ra.log.out.flush_logs = true\n        @ra.log.out.reset\n        @ra.log.level = old_level\n      end\n    end\n  end\n\n  sub_test_case 'configured with MultiOutput plugin which creates plugin instances dynamically' do\n    setup do\n      @ra = RootAgent.new(log: $log)\n      stub(Engine).root_agent { @ra }\n      @ra.configure(Config.parse(<<-EOC, \"(test)\", \"(test_dir)\", true))\n<source>\n  @type test_in\n  @id test_in\n  @label @testing\n</source>\n<label @testing>\n  <match **>\n    @type test_dynamic_out\n    @id test_dyn\n  </match>\n</label>\nEOC\n      @ra\n    end\n\n    test 'plugin status with multi output' do\n      assert_equal 1, @ra.inputs.size\n      assert_equal 0, @ra.filters.size\n      assert_equal 0, @ra.outputs.size\n      assert_equal 1, @ra.labels.size\n      assert_equal '@testing', @ra.labels.keys.first\n      assert_equal 0, @ra.labels.values.first.filters.size\n      assert_equal 1, @ra.labels.values.first.outputs.size\n\n      dyn_out = @ra.labels.values.first.outputs.first\n      assert_nil dyn_out.child\n\n      @ra.start\n\n      assert_equal 1, @ra.labels.values.first.outputs.size\n\n      assert dyn_out.child\n      assert_false dyn_out.child.outputs_statically_created\n      assert_equal 2, dyn_out.child.outputs.size\n\n      assert_equal true, dyn_out.child.outputs[0].started?\n      assert_equal true, dyn_out.child.outputs[1].started?\n      assert_equal true, dyn_out.child.outputs[0].after_started?\n      assert_equal true, dyn_out.child.outputs[1].after_started?\n\n      @ra.shutdown\n\n      assert_equal 1, @ra.labels.values.first.outputs.size\n\n      assert_false dyn_out.child.outputs_statically_created\n      assert_equal 2, dyn_out.child.outputs.size\n\n      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.stopped? }\n      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.before_shutdown? }\n      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.shutdown? }\n      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.after_shutdown? }\n      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.closed? }\n      assert_equal [true, true], dyn_out.child.outputs.map{|i| i.terminated? }\n    end\n  end\n\n  sub_test_case 'configure emit_error_interval' do\n    setup do\n      system_config = SystemConfig.new\n      system_config.emit_error_log_interval = 30\n      @ra = RootAgent.new(log: $log, system_config: system_config)\n      stub(Engine).root_agent { @ra }\n      @ra.log.out.reset\n      one_minute_ago = Time.now.to_i - 60\n      Timecop.freeze(one_minute_ago)\n    end\n\n    teardown do\n      Timecop.return\n    end\n\n    test 'suppresses errors' do\n      mock(@ra.log).warn_backtrace()\n      e = StandardError.new('standard error')\n      begin\n        @ra.handle_emits_error(\"tag\", nil, e)\n      rescue\n      end\n\n      begin\n        @ra.handle_emits_error(\"tag\", nil, e)\n      rescue\n      end\n\n      assert_equal 1, @ra.log.out.logs.size\n    end\n  end\n\n  sub_test_case 'configured at worker2 with 4 workers environment' do\n    setup do\n      ENV['SERVERENGINE_WORKER_ID'] = '2'\n      @ra = RootAgent.new(log: $log)\n      system_config = SystemConfig.new\n      system_config.workers = 4\n      stub(Engine).worker_id { 2 }\n      stub(Engine).root_agent { @ra }\n      stub(Engine).system_config { system_config }\n      @ra\n    end\n\n    teardown '' do\n      ENV.delete('SERVERENGINE_WORKER_ID')\n    end\n\n    def configure_ra(conf_str)\n      conf = Config.parse(conf_str, \"(test)\", \"(test_dir)\", true)\n      @ra.configure(conf)\n      @ra\n    end\n\n    test 'raises configuration error for missing worker id' do\n      errmsg = 'Missing worker id on <worker> directive'\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        conf = <<-EOC\n<worker>\n</worker>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error for too big worker id' do\n      errmsg = \"worker id 4 specified by <worker> directive is not allowed. Available worker id is between 0 and 3\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        conf = <<-EOC\n<worker 4>\n</worker>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error for too big worker id on multi workers syntax' do\n      errmsg = \"worker id 4 specified by <worker> directive is not allowed. Available worker id is between 0 and 3\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        conf = <<-EOC\n<worker 1-4>\n</worker>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error for worker id collisions on multi workers syntax' do\n      errmsg = \"specified worker_id<2> collisions is detected on <worker> directive. Available worker id(s): [3]\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        conf = <<-EOC\n<worker 0-2>\n</worker>\n<worker 2-4>\n</worker>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error for worker id collisions on multi workers syntax when multi available worker_ids are left' do\n      errmsg = \"specified worker_id<1> collisions is detected on <worker> directive. Available worker id(s): [2, 3]\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        conf = <<-EOC\n<worker 0-1>\n</worker>\n<worker 1-3>\n</worker>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error for too big worker id on invalid reversed multi workers syntax' do\n      errmsg = \"greater first_worker_id<3> than last_worker_id<0> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        conf = <<-EOC\n<worker 3-0>\n</worker>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error for invalid elements as a child of worker section' do\n      errmsg = '<worker> section cannot have <system> directive'\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        conf = <<-EOC\n<worker 2>\n<system>\n</system>\n</worker>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'raises configuration error when configured plugins do not have support multi worker configuration' do\n      errmsg = \"Plugin 'test_out' does not support multi workers configuration (FluentTest::FluentTestOutput)\"\n      assert_raise Fluent::ConfigError.new(errmsg) do\n        conf = <<-EOC\n<match **>\n@type test_out\n</match>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'does not raise configuration error when configured plugins in worker section do not have support multi worker configuration' do\n      assert_nothing_raised do\n        conf = <<-EOC\n<worker 2>\n<match **>\n  @type test_out\n</match>\n</worker>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'does not raise configuration error when configured plugins as a children of MultiOutput in worker section do not have support multi worker configuration' do\n      assert_nothing_raised do\n        conf = <<-EOC\n<worker 2>\n<match **>\n  @type copy\n  <store>\n    @type test_out\n  </store>\n  <store>\n    @type test_out\n  </store>\n</match>\n</worker>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'does not raise configuration error when configured plugins owned by plugin do not have support multi worker configuration' do\n      assert_nothing_raised do\n        conf = <<-EOC\n<worker 2>\n<match **>\n  @type test_out_buffered\n  <buffer>\n    @type test_buffer\n  </buffer>\n</match>\n</worker>\nEOC\n        configure_ra(conf)\n      end\n    end\n\n    test 'with plugins' do\n      conf = <<-EOC\n<worker 2>\n<source>\n  @type test_in\n  @id test_in\n</source>\n<filter>\n  type test_filter\n  id test_filter\n</filter>\n<match **>\n  @type relabel\n  @id test_relabel\n  @label @test\n</match>\n<label @test>\n  <match **>\n    type test_out\n    id test_out\n  </match>\n</label>\n<label @ERROR>\n  <match>\n    @type null\n  </match>\n</label>\n</worker>\nEOC\n      ra = configure_ra(conf)\n      assert_kind_of FluentTestInput, ra.inputs.first\n      assert_kind_of Plugin::RelabelOutput, ra.outputs.first\n      assert_kind_of FluentTestFilter, ra.filters.first\n      assert ra.error_collector\n\n      %W(@test @ERROR).each { |label_symbol|\n        assert_include ra.labels, label_symbol\n        assert_kind_of Label, ra.labels[label_symbol]\n      }\n\n      test_label = ra.labels['@test']\n      assert_kind_of FluentTestOutput, test_label.outputs.first\n      assert_equal ra, test_label.root_agent\n\n      error_label = ra.labels['@ERROR']\n      assert_kind_of Fluent::Plugin::NullOutput, error_label.outputs.first\n    end\n\n    test 'with plugins but for another worker' do\n      conf = <<-EOC\n<worker 0>\n<source>\n  @type test_in\n  @id test_in\n</source>\n<filter>\n  type test_filter\n  id test_filter\n</filter>\n<match **>\n  @type relabel\n  @id test_relabel\n  @label @test\n</match>\n<label @test>\n  <match **>\n    type test_out\n    id test_out\n  </match>\n</label>\n<label @ERROR>\n  <match>\n    @type null\n  </match>\n</label>\n</worker>\nEOC\n      ra = configure_ra(conf)\n      assert_equal 0, ra.inputs.size\n      assert_equal 0, ra.outputs.size\n      assert_equal 0, ra.filters.size\n      assert_equal 0, ra.labels.size\n      refute ra.error_collector\n    end\n\n    test 'with plugins for workers syntax should match worker_id equals to 2' do\n      conf = <<-EOC\n<worker 0-2>\n  <source>\n    @type forward\n  </source>\n  <filter **>\n    @type test_filter\n    @id test_filter\n  </filter>\n  <match pattern>\n    @type stdout\n  </match>\n  <label @ERROR>\n    <match>\n      @type null\n    </match>\n  </label>\n</worker>\nEOC\n\n      ra = configure_ra(conf)\n      assert_kind_of Fluent::Plugin::ForwardInput, ra.inputs.first\n      assert_kind_of Fluent::Plugin::StdoutOutput, ra.outputs.first\n      assert_kind_of FluentTestFilter, ra.filters.first\n      assert ra.error_collector\n    end\n  end\n\n  sub_test_case 'start with-source-only' do\n    def conf\n      <<~EOC\n        <source>\n          @type test_in_gen\n          @id test_in_gen\n          num 20\n          interval_sec 0.1\n          async\n        </source>\n\n        <filter test.**>\n          @type record_transformer\n          @id record_transformer\n          <record>\n            foo foo\n          </record>\n        </filter>\n\n        <match test.**>\n          @type test_out\n          @id test_out\n        </match>\n      EOC\n    end\n\n    def setup\n      omit \"Not supported on Windows\" if Fluent.windows?\n      system_config = SystemConfig.new(\n        Config::Element.new('system', '', {\n          'with_source_only' => true,\n        }, [\n          Config::Element.new('source_only_buffer', '', {\n            'flush_interval' => 1,\n          }, []),\n        ])\n      )\n      @root_agent = RootAgent.new(log: $log, system_config: system_config)\n      stub(Engine).root_agent { @root_agent }\n      stub(Engine).system_config { system_config }\n      @root_agent.configure(Config.parse(conf, \"(test)\", \"(test_dir)\"))\n    end\n\n    test 'only input plugins should start' do\n      @root_agent.start\n\n      assert_equal(\n        {\n          \"input started?\" => [true],\n          \"filter started?\" => [false],\n          \"output started?\" => [false],\n        },\n        {\n          \"input started?\" => @root_agent.inputs.map { |plugin| plugin.started? },\n          \"filter started?\" => @root_agent.filters.map { |plugin| plugin.started? },\n          \"output started?\" => @root_agent.outputs.map { |plugin| plugin.started? },\n        }\n      )\n    ensure\n      @root_agent.shutdown\n      # Buffer files remain because not cancelling source-only.\n      # As a test, they should be clean-up-ed.\n      buf_dir = @root_agent.instance_variable_get(:@source_only_buffer_agent).instance_variable_get(:@base_buffer_dir)\n      FileUtils.remove_dir(buf_dir)\n    end\n\n    test '#cancel_source_only! should start all plugins' do\n      @root_agent.start\n      @root_agent.cancel_source_only!\n\n      assert_equal(\n        {\n          \"input started?\" => [true],\n          \"filter started?\" => [true],\n          \"output started?\" => [true],\n        },\n        {\n          \"input started?\" => @root_agent.inputs.map { |plugin| plugin.started? },\n          \"filter started?\" => @root_agent.filters.map { |plugin| plugin.started? },\n          \"output started?\" => @root_agent.outputs.map { |plugin| plugin.started? },\n        }\n      )\n    ensure\n      @root_agent.shutdown\n    end\n\n    test 'buffer should be loaded after #cancel_source_only!' do\n      @root_agent.start\n      sleep 1\n      @root_agent.cancel_source_only!\n\n      waiting(3) do\n        # Wait buffer loaded after source-only cancelled\n        sleep 1 until @root_agent.outputs[0].events[\"test.event\"].any? { |record| record[\"num\"] == 0 }\n      end\n\n      waiting(3) do\n        # Wait the last data output\n        sleep 1 until @root_agent.outputs[0].events[\"test.event\"].any? { |record| record[\"num\"] == 19 }\n      end\n\n      # all data should be outputted\n      assert { @root_agent.outputs[0].events[\"test.event\"].size == 20 }\n    ensure\n      @root_agent.shutdown\n    end\n  end\n\n  sub_test_case 'start_in_parallel' do\n    def conf\n      <<~EOC\n        <source>\n          @type test_in_gen\n          @id test_in_gen\n          num 20\n          interval_sec 0.1\n          async\n        </source>\n\n        <source>\n          @type test_in\n          @id test_in\n        </source>\n\n        <filter test.**>\n          @type record_transformer\n          @id record_transformer\n          <record>\n            foo foo\n          </record>\n        </filter>\n\n        <match test.**>\n          @type test_out\n          @id test_out\n        </match>\n      EOC\n    end\n\n    def setup\n      omit \"Not supported on Windows\" if Fluent.windows?\n      system_config = SystemConfig.new(\n        Config::Element.new('system', '', {}, [\n          Config::Element.new('source_only_buffer', '', {\n            'flush_interval' => 1,\n          }, []),\n        ])\n      )\n      @root_agent = RootAgent.new(log: $log, system_config: system_config, start_in_parallel: true)\n      stub(Engine).root_agent { @root_agent }\n      stub(Engine).system_config { system_config }\n      @root_agent.configure(Config.parse(conf, \"(test)\", \"(test_dir)\"))\n    end\n\n    test 'only input plugins should start' do\n      @root_agent.start\n\n      assert_equal(\n        {\n          \"input started?\" => [true, false],\n          \"filter started?\" => [false],\n          \"output started?\" => [false],\n        },\n        {\n          \"input started?\" => @root_agent.inputs.map { |plugin| plugin.started? },\n          \"filter started?\" => @root_agent.filters.map { |plugin| plugin.started? },\n          \"output started?\" => @root_agent.outputs.map { |plugin| plugin.started? },\n        }\n      )\n    ensure\n      @root_agent.shutdown\n      # Buffer files remain because not cancelling source-only.\n      # As a test, they should be clean-up-ed.\n      buf_dir = @root_agent.instance_variable_get(:@source_only_buffer_agent).instance_variable_get(:@base_buffer_dir)\n      FileUtils.remove_dir(buf_dir)\n    end\n\n    test '#cancel_source_only! should start all plugins' do\n      @root_agent.start\n      @root_agent.cancel_source_only!\n\n      assert_equal(\n        {\n          \"input started?\" => [true, true],\n          \"filter started?\" => [true],\n          \"output started?\" => [true],\n        },\n        {\n          \"input started?\" => @root_agent.inputs.map { |plugin| plugin.started? },\n          \"filter started?\" => @root_agent.filters.map { |plugin| plugin.started? },\n          \"output started?\" => @root_agent.outputs.map { |plugin| plugin.started? },\n        }\n      )\n    ensure\n      @root_agent.shutdown\n    end\n\n    test 'buffer should be loaded after #cancel_source_only!' do\n      @root_agent.start\n      sleep 1\n      @root_agent.cancel_source_only!\n\n      waiting(3) do\n        # Wait buffer loaded after source-only cancelled\n        sleep 1 until @root_agent.outputs[0].events[\"test.event\"].any? { |record| record[\"num\"] == 0 }\n      end\n\n      waiting(3) do\n        # Wait the last data output\n        sleep 1 until @root_agent.outputs[0].events[\"test.event\"].any? { |record| record[\"num\"] == 19 }\n      end\n\n      # all data should be outputted\n      assert { @root_agent.outputs[0].events[\"test.event\"].size == 20 }\n    ensure\n      @root_agent.shutdown\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_source_only_buffer_agent.rb",
    "content": "require_relative 'helper'\n\nclass SourceOnlyBufferAgentTest < ::Test::Unit::TestCase\n  def log\n    logger = ServerEngine::DaemonLogger.new(\n      Fluent::Test::DummyLogDevice.new,\n      { log_level: ServerEngine::DaemonLogger::INFO }\n    )\n    Fluent::Log.new(logger)\n  end\n\n  def setup\n    omit \"Not supported on Windows\" if Fluent.windows?\n    @log = log\n  end\n\n  sub_test_case \"#configure\" do\n    test \"default\" do\n      system_config = Fluent::SystemConfig.new\n      root_agent = Fluent::RootAgent.new(log: @log, system_config: system_config)\n      stub(Fluent::Engine).root_agent { root_agent }\n      stub(Fluent::Engine).system_config { system_config }\n      root_agent.configure(config_element)\n\n      agent = Fluent::SourceOnlyBufferAgent.new(log: @log, system_config: system_config)\n      agent.configure\n\n      assert_equal(\n        {\n          \"num of filter plugins\" => 0,\n          \"num of output plugins\" => 1,\n          \"base_buffer_dir\" => agent.instance_variable_get(:@default_buffer_path),\n          \"actual_buffer_dir\" => agent.instance_variable_get(:@default_buffer_path),\n          \"EventRouter of BufferOutput\" => root_agent.event_router.object_id,\n          \"flush_thread_count\" => 0,\n          \"flush_at_shutdown\" => false,\n        },\n        {\n          \"num of filter plugins\" => agent.filters.size,\n          \"num of output plugins\" => agent.outputs.size,\n          \"base_buffer_dir\" => agent.instance_variable_get(:@base_buffer_dir),\n          \"actual_buffer_dir\" => agent.instance_variable_get(:@actual_buffer_dir),\n          \"EventRouter of BufferOutput\" => agent.outputs[0].router.object_id,\n          \"flush_thread_count\" => agent.outputs[0].buffer_config.flush_thread_count,\n          \"flush_at_shutdown\" => agent.outputs[0].buffer_config.flush_at_shutdown,\n        }\n      )\n\n      assert do\n        @log.out.logs.any? { |log| log.include? \"the emitted data will be stored in the buffer files\" }\n      end\n    end\n\n    test \"flush: true\" do\n      system_config = Fluent::SystemConfig.new\n      root_agent = Fluent::RootAgent.new(log: @log, system_config: system_config)\n      stub(Fluent::Engine).root_agent { root_agent }\n      stub(Fluent::Engine).system_config { system_config }\n      root_agent.configure(config_element)\n\n      agent = Fluent::SourceOnlyBufferAgent.new(log: @log, system_config: system_config)\n      agent.configure(flush: true)\n\n      assert_equal(\n        {\n          \"num of filter plugins\" => 0,\n          \"num of output plugins\" => 1,\n          \"base_buffer_dir\" => agent.instance_variable_get(:@default_buffer_path),\n          \"actual_buffer_dir\" => agent.instance_variable_get(:@default_buffer_path),\n          \"EventRouter of BufferOutput\" => root_agent.event_router.object_id,\n          \"flush_thread_count\" => 1,\n          \"flush_at_shutdown\" => true,\n        },\n        {\n          \"num of filter plugins\" => agent.filters.size,\n          \"num of output plugins\" => agent.outputs.size,\n          \"base_buffer_dir\" => agent.instance_variable_get(:@base_buffer_dir),\n          \"actual_buffer_dir\" => agent.instance_variable_get(:@actual_buffer_dir),\n          \"EventRouter of BufferOutput\" => agent.outputs[0].router.object_id,\n          \"flush_thread_count\" => agent.outputs[0].buffer_config.flush_thread_count,\n          \"flush_at_shutdown\" => agent.outputs[0].buffer_config.flush_at_shutdown,\n        }\n      )\n\n      assert do\n        not @log.out.logs.any? { |log| log.include? \"the emitted data will be stored in the buffer files\" }\n      end\n    end\n\n    test \"multiple workers\" do\n      system_config = Fluent::SystemConfig.new(config_element(\"system\", \"\", {\"workers\" => 2}))\n      root_agent = Fluent::RootAgent.new(log: @log, system_config: system_config)\n      stub(Fluent::Engine).root_agent { root_agent }\n      stub(Fluent::Engine).system_config { system_config }\n      root_agent.configure(config_element)\n\n      agent = Fluent::SourceOnlyBufferAgent.new(log: @log, system_config: system_config)\n      agent.configure\n\n      assert_equal(\n        {\n          \"num of filter plugins\" => 0,\n          \"num of output plugins\" => 1,\n          \"base_buffer_dir\" => agent.instance_variable_get(:@default_buffer_path),\n          \"actual_buffer_dir\" => \"#{agent.instance_variable_get(:@default_buffer_path)}/worker0\",\n          \"EventRouter of BufferOutput\" => root_agent.event_router.object_id,\n          \"flush_thread_count\" => 0,\n          \"flush_at_shutdown\" => false,\n        },\n        {\n          \"num of filter plugins\" => agent.filters.size,\n          \"num of output plugins\" => agent.outputs.size,\n          \"base_buffer_dir\" => agent.instance_variable_get(:@base_buffer_dir),\n          \"actual_buffer_dir\" => agent.instance_variable_get(:@actual_buffer_dir),\n          \"EventRouter of BufferOutput\" => agent.outputs[0].router.object_id,\n          \"flush_thread_count\" => agent.outputs[0].buffer_config.flush_thread_count,\n          \"flush_at_shutdown\" => agent.outputs[0].buffer_config.flush_at_shutdown,\n        }\n      )\n    end\n\n    test \"full setting with flush:true\" do\n      system_config = Fluent::SystemConfig.new(config_element(\"system\", \"\", {}, [\n        config_element(\"source_only_buffer\", \"\", {\n          \"flush_thread_count\" => 4,\n          \"overflow_action\" => :throw_exception,\n          \"path\" => \"tmp_buffer_path\",\n          \"flush_interval\" => 1,\n          \"chunk_limit_size\" => 100,\n          \"total_limit_size\" => 1000,\n          \"compress\" => :gzip,\n        })\n      ]))\n      root_agent = Fluent::RootAgent.new(log: @log, system_config: system_config)\n      stub(Fluent::Engine).root_agent { root_agent }\n      stub(Fluent::Engine).system_config { system_config }\n      root_agent.configure(config_element)\n\n      agent = Fluent::SourceOnlyBufferAgent.new(log: @log, system_config: system_config)\n      agent.configure(flush: true)\n\n      assert_equal(\n        {\n          \"num of filter plugins\" => 0,\n          \"num of output plugins\" => 1,\n          \"base_buffer_dir\" => \"tmp_buffer_path\",\n          \"actual_buffer_dir\" => \"tmp_buffer_path\",\n          \"EventRouter of BufferOutput\" => root_agent.event_router.object_id,\n          \"flush_thread_count\" => 4,\n          \"flush_at_shutdown\" => true,\n          \"overflow_action\" => :throw_exception,\n          \"flush_interval\" => 1,\n          \"chunk_limit_size\" => 100,\n          \"total_limit_size\" => 1000,\n          \"compress\" => :gzip,\n        },\n        {\n          \"num of filter plugins\" => agent.filters.size,\n          \"num of output plugins\" => agent.outputs.size,\n          \"base_buffer_dir\" => agent.instance_variable_get(:@base_buffer_dir),\n          \"actual_buffer_dir\" => agent.instance_variable_get(:@actual_buffer_dir),\n          \"EventRouter of BufferOutput\" => agent.outputs[0].router.object_id,\n          \"flush_thread_count\" => agent.outputs[0].buffer_config.flush_thread_count,\n          \"flush_at_shutdown\" => agent.outputs[0].buffer_config.flush_at_shutdown,\n          \"overflow_action\" => agent.outputs[0].buffer_config.overflow_action,\n          \"flush_interval\" => agent.outputs[0].buffer_config.flush_interval,\n          \"chunk_limit_size\" => agent.outputs[0].buffer.chunk_limit_size,\n          \"total_limit_size\" => agent.outputs[0].buffer.total_limit_size,\n          \"compress\" => agent.outputs[0].buffer.compress,\n        }\n      )\n    end\n  end\n\n  sub_test_case \"#cleanup\" do\n    test \"do not remove the buffer if it is not empty\" do\n      system_config = Fluent::SystemConfig.new\n      root_agent = Fluent::RootAgent.new(log: @log, system_config: system_config)\n      stub(Fluent::Engine).root_agent { root_agent }\n      stub(Fluent::Engine).system_config { system_config }\n      root_agent.configure(config_element)\n\n      agent = Fluent::SourceOnlyBufferAgent.new(log: @log, system_config: system_config)\n      agent.configure\n\n      stub(Dir).empty?(agent.instance_variable_get(:@actual_buffer_dir)) { false }\n      mock(FileUtils).remove_dir.never\n\n      agent.cleanup\n\n      assert do\n        @log.out.logs.any? { |log| log.include? \"some buffer files remain in\" }\n      end\n    end\n\n    test \"remove the buffer if it is empty\" do\n      system_config = Fluent::SystemConfig.new\n      root_agent = Fluent::RootAgent.new(log: @log, system_config: system_config)\n      stub(Fluent::Engine).root_agent { root_agent }\n      stub(Fluent::Engine).system_config { system_config }\n      root_agent.configure(config_element)\n\n      agent = Fluent::SourceOnlyBufferAgent.new(log: @log, system_config: system_config)\n      agent.configure\n\n      stub(Dir).empty?(agent.instance_variable_get(:@actual_buffer_dir)) { true }\n      mock(FileUtils).remove_dir(agent.instance_variable_get(:@base_buffer_dir)).times(1)\n\n      agent.cleanup\n\n      assert do\n        not @log.out.logs.any? { |log| log.include? \"some buffer files remain in\" }\n      end\n    end\n  end\n\n  sub_test_case \"error\" do\n    test \"#emit_error_event\" do\n      system_config = Fluent::SystemConfig.new\n      root_agent = Fluent::RootAgent.new(log: @log, system_config: system_config)\n      stub(Fluent::Engine).root_agent { root_agent }\n      stub(Fluent::Engine).system_config { system_config }\n      root_agent.configure(config_element)\n\n      agent = Fluent::SourceOnlyBufferAgent.new(log: @log, system_config: system_config)\n      agent.configure\n\n      agent.event_router.emit_error_event(\"tag\", 0, \"hello\", Exception.new)\n\n      assert do\n        @log.out.logs.any? { |log| log.include? \"SourceOnlyBufferAgent: dump an error event\" }\n      end\n    end\n\n    test \"#handle_emits_error\" do\n      system_config = Fluent::SystemConfig.new\n      root_agent = Fluent::RootAgent.new(log: @log, system_config: system_config)\n      stub(Fluent::Engine).root_agent { root_agent }\n      stub(Fluent::Engine).system_config { system_config }\n      root_agent.configure(config_element)\n\n      agent = Fluent::SourceOnlyBufferAgent.new(log: @log, system_config: system_config)\n      agent.configure\n\n      stub(agent.outputs[0]).emit_events { raise \"test error\" }\n\n      agent.event_router.emit(\"foo\", 0, \"hello\")\n\n      assert do\n        @log.out.logs.any? { |log| log.include? \"SourceOnlyBufferAgent: emit transaction failed\" }\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_static_config_analysis.rb",
    "content": "require_relative 'helper'\n\nrequire 'fluent/config'\nrequire 'fluent/static_config_analysis'\nrequire 'fluent/plugin/out_forward'\nrequire 'fluent/plugin/out_stdout'\nrequire 'fluent/plugin/out_exec'\nrequire 'fluent/plugin/in_forward'\nrequire 'fluent/plugin/in_sample'\nrequire 'fluent/plugin/filter_grep'\nrequire 'fluent/plugin/filter_stdout'\nrequire 'fluent/plugin/filter_parser'\n\nclass StaticConfigAnalysisTest < ::Test::Unit::TestCase\n  sub_test_case '.call' do\n    test 'returns outputs, inputs and filters' do\n      conf_data = <<-CONF\n<source>\n  @type forward\n</source>\n<filter>\n  @type grep\n</filter>\n<match>\n  @type forward\n</match>\n      CONF\n\n      c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n      ret = Fluent::StaticConfigAnalysis.call(c)\n      assert_equal 1, ret.outputs.size\n      assert_kind_of Fluent::Plugin::ForwardOutput, ret.outputs[0].plugin\n      assert_equal 1, ret.inputs.size\n      assert_kind_of Fluent::Plugin::ForwardInput, ret.inputs[0].plugin\n      assert_equal 1, ret.filters.size\n      assert_kind_of Fluent::Plugin::GrepFilter, ret.filters[0].plugin\n      assert_empty ret.labels\n\n      assert_equal [Fluent::Plugin::ForwardOutput, Fluent::Plugin::ForwardInput, Fluent::Plugin::GrepFilter], ret.all_plugins.map(&:class)\n    end\n\n    test 'returns wrapped element with worker and label section' do\n      conf_data = <<-CONF\n<source>\n  @type forward\n</source>\n<filter>\n  @type grep\n</filter>\n<match>\n  @type forward\n</match>\n<worker 0>\n  <source>\n    @type dummy\n  </source>\n  <filter>\n    @type parser\n  </filter>\n  <match>\n    @type exec\n  </match>\n</worker>\n<label @test>\n  <filter>\n    @type stdout\n  </filter>\n  <match>\n    @type stdout\n  </match>\n</label>\n      CONF\n\n      c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n      ret = Fluent::StaticConfigAnalysis.call(c)\n      assert_equal [Fluent::Plugin::ExecOutput, Fluent::Plugin::StdoutOutput, Fluent::Plugin::ForwardOutput], ret.outputs.map { |x| x.plugin.class }\n      assert_equal [Fluent::Plugin::SampleInput, Fluent::Plugin::ForwardInput], ret.inputs.map { |x| x.plugin.class }\n      assert_equal [Fluent::Plugin::ParserFilter, Fluent::Plugin::StdoutFilter, Fluent::Plugin::GrepFilter], ret.filters.map { |x| x.plugin.class }\n      assert_equal 1, ret.labels.size\n      assert_equal '@test', ret.labels[0].name\n    end\n\n    sub_test_case 'raises config error' do\n      data(\n        'empty' => ['', 'Missing worker id on <worker> directive'],\n        'invalid number' => ['a', 'worker id should be integer: a'],\n        'worker id is negative' => ['-1', 'worker id should be integer: -1'],\n        'min worker id is less than 0' => ['-1-1', 'worker id should be integer: -1-1'],\n        'max worker id is less than 0' => ['1--1', 'worker id -1 specified by <worker> directive is not allowed. Available worker id is between 0 and 1'],\n        'min worker id is greater than workers' => ['0-2', 'worker id 2 specified by <worker> directive is not allowed. Available worker id is between 0 and 1'],\n        'max worker is less than min worker' => ['1-0', \"greater first_worker_id<1> than last_worker_id<0> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>\"],\n      )\n      test 'when worker number is invalid' do |v|\n        val, msg = v\n        conf_data = <<-CONF\n<worker #{val}>\n</worker>\nCONF\n\n        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n        assert_raise(Fluent::ConfigError.new(msg)) do\n          Fluent::StaticConfigAnalysis.call(c, workers: 2)\n        end\n      end\n\n      test 'when worker number is duplicated' do\n        conf_data = <<-CONF\n<worker 0-1>\n</worker>\n<worker 0-1>\n</worker>\nCONF\n\n        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n        assert_raise(Fluent::ConfigError.new(\"specified worker_id<0> collisions is detected on <worker> directive. Available worker id(s): []\")) do\n          Fluent::StaticConfigAnalysis.call(c, workers: 2)\n        end\n      end\n\n      test 'duplicated label exits' do\n        conf_data = <<-CONF\n<label @dup>\n</label>\n<label @dup>\n</label>\nCONF\n\n        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n        assert_raise(Fluent::ConfigError.new('Section <label @dup> appears twice')) do\n          Fluent::StaticConfigAnalysis.call(c, workers: 2)\n        end\n      end\n\n      test 'empty label' do\n        conf_data = <<-CONF\n<label>\n</label>\nCONF\n\n        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n        assert_raise(Fluent::ConfigError.new('Missing symbol argument on <label> directive')) do\n          Fluent::StaticConfigAnalysis.call(c, workers: 2)\n        end\n      end\n\n      data(\n        'in filter' => 'filter',\n        'in source' => 'source',\n        'in match' => 'match',\n      )\n      test 'when @type is missing' do |name|\n        conf_data = <<-CONF\n<#{name}>\n  @type\n</#{name}>\nCONF\n        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n        assert_raise(Fluent::ConfigError.new(\"Missing '@type' parameter on <#{name}> directive\")) do\n          Fluent::StaticConfigAnalysis.call(c)\n        end\n      end\n\n      test 'when worker has worker section' do\n        conf_data = <<-CONF\n<worker 0>\n  <worker 0>\n  </worker>\n</worker>\nCONF\n        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)\n        assert_raise(Fluent::ConfigError.new(\"<worker> section cannot have <worker> directive\")) do\n          Fluent::StaticConfigAnalysis.call(c)\n        end\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_supervisor.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/event_router'\nrequire 'fluent/system_config'\nrequire 'fluent/supervisor'\nrequire 'fluent/file_wrapper'\nrequire_relative 'test_plugin_classes'\n\nrequire 'net/http'\nrequire 'uri'\nrequire 'fileutils'\nrequire 'tempfile'\nrequire 'securerandom'\nrequire 'pathname'\n\nif Fluent.windows?\n  require 'win32/event'\nend\n\nclass SupervisorTest < ::Test::Unit::TestCase\n  class DummyServer\n    include Fluent::ServerModule\n    attr_accessor :rpc_endpoint, :enable_get_dump, :socket_manager_server\n    def config\n      {}\n    end\n  end\n\n  def tmp_dir\n    File.join(File.dirname(__FILE__), \"tmp\", \"supervisor#{ENV['TEST_ENV_NUMBER']}\", SecureRandom.hex(10))\n  end\n\n  def setup\n    @stored_global_logger = $log\n    @tmp_dir = tmp_dir\n    @tmp_root_dir = File.join(@tmp_dir, 'root')\n    FileUtils.mkdir_p(@tmp_dir)\n    @sigdump_path = \"/tmp/sigdump-#{Process.pid}.log\"\n  end\n\n  def teardown\n    $log = @stored_global_logger\n    begin\n      FileUtils.rm_rf(@tmp_dir)\n    rescue Errno::EACCES\n      # It may occur on Windows because of delete pending state due to delayed GC.\n      # Ruby 3.2 or later doesn't ignore Errno::EACCES:\n      # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81\n    end\n  end\n\n  def write_config(path, data)\n    FileUtils.mkdir_p(File.dirname(path))\n    Fluent::FileWrapper.open(path, \"w\") {|f| f.write data }\n  end\n\n\n  def test_system_config\n    sv = Fluent::Supervisor.new({})\n    conf_data = <<-EOC\n<system>\n  rpc_endpoint 127.0.0.1:24445\n  suppress_repeated_stacktrace false\n  suppress_config_dump true\n  without_source true\n  with_source_only true\n  enable_get_dump true\n  enable_input_metrics false\n  process_name \"process_name\"\n  log_level info\n  root_dir #{@tmp_root_dir}\n  <log>\n    path /tmp/fluentd.log\n    format json\n    time_format %Y\n  </log>\n  <counter_server>\n    bind 127.0.0.1\n    port 24321\n    scope server1\n    backup_path /tmp/backup\n  </counter_server>\n  <counter_client>\n    host 127.0.0.1\n    port 24321\n    timeout 2\n  </counter_client>\n  <source_only_buffer>\n    flush_thread_count 4\n    overflow_action throw_exception\n    path /tmp/source-only-buffer\n    flush_interval 1\n    chunk_limit_size 100\n    total_limit_size 1000\n    compress gzip\n  </source_only_buffer>\n</system>\n    EOC\n    conf = Fluent::Config.parse(conf_data, \"(test)\", \"(test_dir)\", true)\n    sys_conf = sv.__send__(:build_system_config, conf)\n\n    assert_equal '127.0.0.1:24445', sys_conf.rpc_endpoint\n    assert_equal false, sys_conf.suppress_repeated_stacktrace\n    assert_equal true, sys_conf.suppress_config_dump\n    assert_equal true, sys_conf.without_source\n    assert_equal true, sys_conf.with_source_only\n    assert_equal true, sys_conf.enable_get_dump\n    assert_equal false, sys_conf.enable_input_metrics\n    assert_equal \"process_name\", sys_conf.process_name\n    assert_equal 2, sys_conf.log_level\n    assert_equal @tmp_root_dir, sys_conf.root_dir\n    assert_equal \"/tmp/fluentd.log\", sys_conf.log.path\n    assert_equal :json, sys_conf.log.format\n    assert_equal '%Y', sys_conf.log.time_format\n    counter_server = sys_conf.counter_server\n    assert_equal '127.0.0.1', counter_server.bind\n    assert_equal 24321, counter_server.port\n    assert_equal 'server1', counter_server.scope\n    assert_equal '/tmp/backup', counter_server.backup_path\n    counter_client = sys_conf.counter_client\n    assert_equal '127.0.0.1', counter_client.host\n    assert_equal 24321, counter_client.port\n    assert_equal 2, counter_client.timeout\n    source_only_buffer = sys_conf.source_only_buffer\n    assert_equal 4, source_only_buffer.flush_thread_count\n    assert_equal :throw_exception, source_only_buffer.overflow_action\n    assert_equal \"/tmp/source-only-buffer\", source_only_buffer.path\n    assert_equal 1, source_only_buffer.flush_interval\n    assert_equal 100, source_only_buffer.chunk_limit_size\n    assert_equal 1000, source_only_buffer.total_limit_size\n    assert_equal :gzip, source_only_buffer.compress\n  end\n\n  sub_test_case \"yaml config\" do\n    def parse_yaml(yaml)\n      context = Kernel.binding\n\n      config = nil\n      Tempfile.open do |file|\n        file.puts(yaml)\n        file.flush\n        s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(file))\n        config = Fluent::Config::YamlParser::Parser.new(s).build.to_element\n      end\n      config\n    end\n\n    def test_system_config\n      sv = Fluent::Supervisor.new({})\n      conf_data = <<-EOC\n      system:\n        rpc_endpoint: 127.0.0.1:24445\n        suppress_repeated_stacktrace: true\n        suppress_config_dump: true\n        without_source: true\n        with_source_only: true\n        enable_get_dump: true\n        process_name: \"process_name\"\n        log_level: info\n        root_dir: !fluent/s \"#{@tmp_root_dir}\"\n        log:\n          path: /tmp/fluentd.log\n          format: json\n          time_format: \"%Y\"\n        counter_server:\n          bind: 127.0.0.1\n          port: 24321\n          scope: server1\n          backup_path: /tmp/backup\n        counter_client:\n          host: 127.0.0.1\n          port: 24321\n          timeout: 2\n        source_only_buffer:\n          flush_thread_count: 4\n          overflow_action: throw_exception\n          path: /tmp/source-only-buffer\n          flush_interval: 1\n          chunk_limit_size: 100\n          total_limit_size: 1000\n          compress: gzip\n      EOC\n      conf = parse_yaml(conf_data)\n      sys_conf = sv.__send__(:build_system_config, conf)\n\n    counter_client = sys_conf.counter_client\n    counter_server = sys_conf.counter_server\n    source_only_buffer = sys_conf.source_only_buffer\n    assert_equal(\n      [\n        '127.0.0.1:24445',\n        true,\n        true,\n        true,\n        true,\n        true,\n        \"process_name\",\n        2,\n        @tmp_root_dir,\n        \"/tmp/fluentd.log\",\n        :json,\n        '%Y',\n        '127.0.0.1',\n        24321,\n        'server1',\n        '/tmp/backup',\n        '127.0.0.1',\n        24321,\n        2,\n        4,\n        :throw_exception,\n        \"/tmp/source-only-buffer\",\n        1,\n        100,\n        1000,\n        :gzip,\n      ],\n      [\n        sys_conf.rpc_endpoint,\n        sys_conf.suppress_repeated_stacktrace,\n        sys_conf.suppress_config_dump,\n        sys_conf.without_source,\n        sys_conf.with_source_only,\n        sys_conf.enable_get_dump,\n        sys_conf.process_name,\n        sys_conf.log_level,\n        sys_conf.root_dir,\n        sys_conf.log.path,\n        sys_conf.log.format,\n        sys_conf.log.time_format,\n        counter_server.bind,\n        counter_server.port,\n        counter_server.scope,\n        counter_server.backup_path,\n        counter_client.host,\n        counter_client.port,\n        counter_client.timeout,\n        source_only_buffer.flush_thread_count,\n        source_only_buffer.overflow_action,\n        source_only_buffer.path,\n        source_only_buffer.flush_interval,\n        source_only_buffer.chunk_limit_size,\n        source_only_buffer.total_limit_size,\n        source_only_buffer.compress,\n      ])\n    end\n  end\n\n  def test_usr1_in_main_process_signal_handlers\n    omit \"Windows cannot handle signals\" if Fluent.windows?\n\n    create_info_dummy_logger\n\n    sv = Fluent::Supervisor.new({})\n    sv.send(:install_main_process_signal_handlers)\n\n    Process.kill :USR1, Process.pid\n\n    sleep 1\n\n    info_msg = \"[info]: force flushing buffered events\\n\"\n    assert{ $log.out.logs.first.end_with?(info_msg) }\n  ensure\n    $log.out.reset if $log&.out&.respond_to?(:reset)\n  end\n\n  def test_cont_in_main_process_signal_handlers\n    omit \"Windows cannot handle signals\" if Fluent.windows?\n\n    # https://github.com/fluent/fluentd/issues/4063\n    GC.start\n\n    sv = Fluent::Supervisor.new({})\n    sv.send(:install_main_process_signal_handlers)\n\n    Process.kill :CONT, Process.pid\n\n    sleep 1\n\n    assert{ File.exist?(@sigdump_path) }\n  ensure\n    File.delete(@sigdump_path) if File.exist?(@sigdump_path)\n  end\n\n  def test_term_cont_in_main_process_signal_handlers\n    omit \"Windows cannot handle signals\" if Fluent.windows?\n\n    # https://github.com/fluent/fluentd/issues/4063\n    GC.start\n\n    create_debug_dummy_logger\n\n    sv = Fluent::Supervisor.new({})\n    sv.send(:install_main_process_signal_handlers)\n\n    Process.kill :TERM, Process.pid\n    Process.kill :CONT, Process.pid\n\n    sleep 1\n\n    debug_msg = \"[debug]: fluentd main process get SIGTERM\\n\"\n    logs = $log.out.logs\n    assert{ logs.any?{|log| log.include?(debug_msg) } }\n\n    assert{ not File.exist?(@sigdump_path) }\n  ensure\n    $log.out.reset if $log&.out&.respond_to?(:reset)\n    File.delete(@sigdump_path) if File.exist?(@sigdump_path)\n  end\n\n  def test_winch_in_main_process_signal_handlers\n    omit \"Windows cannot handle signals\" if Fluent.windows?\n\n    mock(Fluent::Engine).cancel_source_only!\n    create_info_dummy_logger\n\n    sv = Fluent::Supervisor.new({})\n    sv.send(:install_main_process_signal_handlers)\n\n    Process.kill :WINCH, Process.pid\n\n    sleep 1\n\n    info_msg = \"[info]: try to cancel with-source-only mode\\n\"\n    assert{ $log.out.logs.first.end_with?(info_msg) }\n  ensure\n    $log.out.reset if $log&.out&.respond_to?(:reset)\n  end\n\n  def test_main_process_command_handlers\n    omit \"Only for Windows, alternative to UNIX signals\" unless Fluent.windows?\n\n    create_info_dummy_logger\n\n    sv = Fluent::Supervisor.new({})\n    r, w = IO.pipe\n    $stdin = r\n    sv.send(:install_main_process_signal_handlers)\n\n    begin\n      w.write(\"GRACEFUL_RESTART\\n\")\n      w.flush\n    ensure\n      $stdin = STDIN\n    end\n\n    sleep 1\n\n    info_msg = \"[info]: force flushing buffered events\\n\"\n    assert{ $log.out.logs.first.end_with?(info_msg) }\n  ensure\n    $log.out.reset if $log&.out&.respond_to?(:reset)\n  end\n\n  def test_usr1_in_supervisor_signal_handler\n    omit \"Windows cannot handle signals\" if Fluent.windows?\n\n    create_debug_dummy_logger\n\n    server = DummyServer.new\n    server.install_supervisor_signal_handlers\n\n    Process.kill :USR1, Process.pid\n\n    sleep 1\n\n    debug_msg = '[debug]: fluentd supervisor process get SIGUSR1'\n    logs = $log.out.logs\n    assert{ logs.any?{|log| log.include?(debug_msg) } }\n  ensure\n    $log.out.reset if $log&.out&.respond_to?(:reset)\n  end\n\n  def test_cont_in_supervisor_signal_handler\n    omit \"Windows cannot handle signals\" if Fluent.windows?\n\n    # https://github.com/fluent/fluentd/issues/4063\n    GC.start\n\n    server = DummyServer.new\n    server.install_supervisor_signal_handlers\n\n    Process.kill :CONT, Process.pid\n\n    sleep 1\n\n    assert{ File.exist?(@sigdump_path) }\n  ensure\n    File.delete(@sigdump_path) if File.exist?(@sigdump_path)\n  end\n\n  def test_term_cont_in_supervisor_signal_handler\n    omit \"Windows cannot handle signals\" if Fluent.windows?\n\n    # https://github.com/fluent/fluentd/issues/4063\n    GC.start\n\n    server = DummyServer.new\n    server.install_supervisor_signal_handlers\n\n    Process.kill :TERM, Process.pid\n    Process.kill :CONT, Process.pid\n\n    assert{ not File.exist?(@sigdump_path) }\n  ensure\n    File.delete(@sigdump_path) if File.exist?(@sigdump_path)\n  end\n\n  def test_winch_in_supervisor_signal_handler\n    omit \"Windows cannot handle signals\" if Fluent.windows?\n\n    create_debug_dummy_logger\n\n    server = DummyServer.new\n    server.install_supervisor_signal_handlers\n\n    Process.kill :WINCH, Process.pid\n\n    sleep 1\n\n    debug_msg = '[debug]: fluentd supervisor process got SIGWINCH'\n    logs = $log.out.logs\n    assert{ logs.any?{|log| log.include?(debug_msg) } }\n  ensure\n    $log.out.reset if $log&.out&.respond_to?(:reset)\n  end\n\n  def test_windows_shutdown_event\n    omit \"Only for Windows platform\" unless Fluent.windows?\n\n    create_debug_dummy_logger\n\n    server = DummyServer.new\n    def server.config\n      {:signame => \"TestFluentdEvent\"}\n    end\n\n    mock(server).stop(true)\n    stub(Process).kill.times(0)\n\n    server.install_windows_event_handler\n    begin\n      sleep 0.1 # Wait for starting windows event thread\n      event = Win32::Event.open(\"TestFluentdEvent\")\n      event.set\n      event.close\n      sleep 1.0 # Wait for dumping\n    ensure\n      server.stop_windows_event_thread\n    end\n\n    debug_msg = '[debug]: Got Win32 event \"TestFluentdEvent\"'\n    logs = $log.out.logs\n    assert{ logs.any?{|log| log.include?(debug_msg) } }\n  ensure\n    $log.out.reset if $log&.out&.respond_to?(:reset)\n  end\n\n  def test_supervisor_event_handler\n    omit \"Only for Windows, alternative to UNIX signals\" unless Fluent.windows?\n\n    create_debug_dummy_logger\n\n    server = DummyServer.new\n    def server.config\n      {:signame => \"TestFluentdEvent\"}\n    end\n    server.install_windows_event_handler\n    begin\n      sleep 0.1 # Wait for starting windows event thread\n      event = Win32::Event.open(\"TestFluentdEvent_USR1\")\n      event.set\n      event.close\n      sleep 1.0 # Wait for dumping\n    ensure\n      server.stop_windows_event_thread\n    end\n\n    debug_msg = '[debug]: Got Win32 event \"TestFluentdEvent_USR1\"'\n    logs = $log.out.logs\n    assert{ logs.any?{|log| log.include?(debug_msg) } }\n  ensure\n    $log.out.reset if $log&.out&.respond_to?(:reset)\n  end\n\n  data(\"Normal\", {raw_path: \"C:\\\\Windows\\\\Temp\\\\sigdump.log\", expected: \"C:\\\\Windows\\\\Temp\\\\sigdump-#{Process.pid}.log\"})\n  data(\"UNIX style\", {raw_path: \"/Windows/Temp/sigdump.log\", expected: \"/Windows/Temp/sigdump-#{Process.pid}.log\"})\n  data(\"No extension\", {raw_path: \"C:\\\\Windows\\\\Temp\\\\sigdump\", expected: \"C:\\\\Windows\\\\Temp\\\\sigdump-#{Process.pid}\"})\n  data(\"Multi-extension\", {raw_path: \"C:\\\\Windows\\\\Temp\\\\sig.dump.bk\", expected: \"C:\\\\Windows\\\\Temp\\\\sig.dump-#{Process.pid}.bk\"})\n  def test_fluentsigdump_get_path_with_pid(data)\n    path = Fluent::FluentSigdump.get_path_with_pid(data[:raw_path])\n    assert_equal(data[:expected], path)\n  end\n\n  def test_supervisor_event_dump_windows\n    omit \"Only for Windows, alternative to UNIX signals\" unless Fluent.windows?\n\n    # https://github.com/fluent/fluentd/issues/4063\n    GC.start\n\n    ENV['SIGDUMP_PATH'] = @tmp_dir + \"/sigdump.log\"\n\n    server = DummyServer.new\n    def server.config\n      {:signame => \"TestFluentdEvent\"}\n    end\n    server.install_windows_event_handler\n\n    begin\n      sleep 0.1 # Wait for starting windows event thread\n      event = Win32::Event.open(\"TestFluentdEvent_CONT\")\n      event.set\n      event.close\n      sleep 1.0 # Wait for dumping\n    ensure\n      server.stop_windows_event_thread\n    end\n\n    result_filepaths = Dir.glob(\"#{@tmp_dir}/*\")\n    assert {result_filepaths.length > 0}\n  ensure\n    ENV.delete('SIGDUMP_PATH')\n  end\n\n  data(:ipv4 => [\"0.0.0.0\", \"127.0.0.1\", false],\n       :ipv6 => [\"[::]\", \"[::1]\", true],\n       :localhost_ipv4 => [\"localhost\", \"127.0.0.1\", false])\n  def test_rpc_server(data)\n    omit \"Windows cannot handle signals\" if Fluent.windows?\n\n    bindaddr, localhost, ipv6 = data\n    omit \"IPv6 is not supported on this environment\" if ipv6 && !ipv6_enabled?\n\n    create_info_dummy_logger\n\n    sv = Fluent::Supervisor.new({})\n    conf_data = <<-EOC\n  <system>\n    rpc_endpoint \"#{bindaddr}:24447\"\n  </system>\n    EOC\n    conf = Fluent::Config.parse(conf_data, \"(test)\", \"(test_dir)\", true)\n    sys_conf = sv.__send__(:build_system_config, conf)\n\n    server = DummyServer.new\n    server.rpc_endpoint = sys_conf.rpc_endpoint\n    server.enable_get_dump = sys_conf.enable_get_dump\n\n    server.run_rpc_server\n\n    sv.send(:install_main_process_signal_handlers)\n    response = Net::HTTP.get(URI.parse(\"http://#{localhost}:24447/api/plugins.flushBuffers\"))\n    info_msg = \"[info]: force flushing buffered events\\n\"\n\n    server.stop_rpc_server\n\n    # In TravisCI with OSX(Xcode), it seems that can't use rpc server.\n    # This test will be passed in such environment.\n    pend unless $log.out.logs.first\n\n    assert_equal('{\"ok\":true}', response)\n    assert{ $log.out.logs.first.end_with?(info_msg) }\n  ensure\n    $log.out.reset if $log.out.is_a?(Fluent::Test::DummyLogDevice)\n  end\n\n  data(:no_port => [\"127.0.0.1\"],\n       :invalid_addr => [\"*:24447\"])\n  def test_invalid_rpc_endpoint(data)\n    endpoint = data[0]\n\n    sv = Fluent::Supervisor.new({})\n    conf_data = <<-EOC\n  <system>\n    rpc_endpoint \"#{endpoint}\"\n  </system>\n    EOC\n    conf = Fluent::Config.parse(conf_data, \"(test)\", \"(test_dir)\", true)\n    sys_conf = sv.__send__(:build_system_config, conf)\n\n    server = DummyServer.new\n    server.rpc_endpoint = sys_conf.rpc_endpoint\n\n    assert_raise(Fluent::ConfigError.new(\"Invalid rpc_endpoint: #{endpoint}\")) do\n      server.run_rpc_server\n    end\n  end\n\n  data(:ipv4 => [\"0.0.0.0\", \"127.0.0.1\", false],\n       :ipv6 => [\"[::]\", \"[::1]\", true],\n       :localhost_ipv4 => [\"localhost\", \"127.0.0.1\", true])\n  def test_rpc_server_windows(data)\n    omit \"Only for windows platform\" unless Fluent.windows?\n\n    bindaddr, localhost, ipv6 = data\n    omit \"IPv6 is not supported on this environment\" if ipv6 && !ipv6_enabled?\n\n    create_info_dummy_logger\n\n    sv = Fluent::Supervisor.new({})\n    conf_data = <<-EOC\n  <system>\n    rpc_endpoint \"#{bindaddr}:24447\"\n  </system>\n    EOC\n    conf = Fluent::Config.parse(conf_data, \"(test)\", \"(test_dir)\", true)\n    sys_conf = sv.__send__(:build_system_config, conf)\n\n    server = DummyServer.new\n    def server.config\n      {\n        :signame => \"TestFluentdEvent\",\n        :worker_pid => 5963,\n      }\n    end\n    server.rpc_endpoint = sys_conf.rpc_endpoint\n\n    server.run_rpc_server\n\n    mock(server).restart(true) { nil }\n    response = Net::HTTP.get(URI.parse(\"http://#{localhost}:24447/api/plugins.flushBuffers\"))\n\n    server.stop_rpc_server\n    assert_equal('{\"ok\":true}', response)\n  end\n\n  sub_test_case \"serverengine_config\" do\n    def test_normal\n      params = {}\n      params['workers'] = 1\n      params['fluentd_conf_path'] = \"fluentd.conf\"\n      params['use_v1_config'] = true\n      params['conf_encoding'] = 'utf-8'\n      params['log_level'] = Fluent::Log::LEVEL_INFO\n      load_config_proc =  Proc.new { Fluent::Supervisor.serverengine_config(params) }\n\n      se_config = load_config_proc.call\n      assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]\n      assert_equal 'spawn', se_config[:worker_type]\n      assert_equal 1, se_config[:workers]\n      assert_equal false, se_config[:log_stdin]\n      assert_equal false, se_config[:log_stdout]\n      assert_equal false, se_config[:log_stderr]\n      assert_equal true, se_config[:enable_heartbeat]\n      assert_equal false, se_config[:auto_heartbeat]\n      assert_equal \"fluentd.conf\", se_config[:config_path]\n      assert_equal false, se_config[:daemonize]\n      assert_nil se_config[:pid_path]\n    end\n\n    def test_daemonize\n      params = {}\n      params['workers'] = 1\n      params['fluentd_conf_path'] = \"fluentd.conf\"\n      params['use_v1_config'] = true\n      params['conf_encoding'] = 'utf-8'\n      params['log_level'] = Fluent::Log::LEVEL_INFO\n      params['daemonize'] = './fluentd.pid'\n      load_config_proc = Proc.new { Fluent::Supervisor.serverengine_config(params) }\n\n      se_config = load_config_proc.call\n      assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]\n      assert_equal 'spawn', se_config[:worker_type]\n      assert_equal 1, se_config[:workers]\n      assert_equal false, se_config[:log_stdin]\n      assert_equal false, se_config[:log_stdout]\n      assert_equal false, se_config[:log_stderr]\n      assert_equal true, se_config[:enable_heartbeat]\n      assert_equal false, se_config[:auto_heartbeat]\n      assert_equal \"fluentd.conf\", se_config[:config_path]\n      assert_equal true, se_config[:daemonize]\n      assert_equal './fluentd.pid', se_config[:pid_path]\n    end\n\n    data(\"nil\", [nil, nil])\n    data(\"default\", [\"0\", 0])\n    data(\"000\", [\"000\", 0])\n    data(\"0000\", [\"0000\", 0])\n    data(\"2\", [\"2\", 2])\n    data(\"222\", [\"222\", 146])\n    data(\"0222\", [\"0222\", 146])\n    data(\"0 as integer\", [0, 0])\n    def test_chumask((chumask, expected))\n      params = { \"chumask\" => chumask }\n      load_config_proc = Proc.new { Fluent::Supervisor.serverengine_config(params) }\n\n      se_config = load_config_proc.call\n\n      assert_equal expected, se_config[:chumask]\n    end\n  end\n\n  data(\"default\", [{}, \"0\"])\n  data(\"222\", [{chumask: \"222\"}, \"222\"])\n  def test_chumask_should_be_passed_to_ServerEngine((cl_opt, expected_chumask_value))\n    proxy.mock(Fluent::Supervisor).serverengine_config(hash_including(\"chumask\" => expected_chumask_value))\n    any_instance_of(ServerEngine::Daemon) { |daemon| mock(daemon).run.once }\n\n    supervisor = Fluent::Supervisor.new(cl_opt)\n    stub(Fluent::Config).build { config_element('ROOT') }\n    stub(supervisor).build_spawn_command { \"dummy command line\" }\n    supervisor.configure(supervisor: true)\n\n    supervisor.run_supervisor\n  end\n\n  sub_test_case \"init logger\" do\n    data(supervisor: true)\n    data(worker: false)\n    def test_init_for_logger(supervisor)\n      tmp_conf_path = \"#{@tmp_dir}/dir/test_init_for_logger.conf\"\n      conf_info_str = <<~EOC\n        <system>\n          log_level warn # To suppress logs\n          suppress_repeated_stacktrace false\n          ignore_repeated_log_interval 10s\n          ignore_same_log_interval 20s\n          <log>\n            format json\n            time_format %FT%T.%L%z\n            forced_stacktrace_level info\n          </log>\n        </system>\n      EOC\n      write_config tmp_conf_path, conf_info_str\n\n      s = Fluent::Supervisor.new({config_path: tmp_conf_path})\n      s.configure(supervisor: supervisor)\n\n      assert_equal :json, $log.format\n      assert_equal '%FT%T.%L%z', $log.time_format\n      assert_equal false, $log.suppress_repeated_stacktrace\n      assert_equal 10, $log.ignore_repeated_log_interval\n      assert_equal 20, $log.ignore_same_log_interval\n      assert_equal Fluent::Log::LEVEL_INFO, $log.instance_variable_get(:@forced_stacktrace_level)\n      assert_true $log.force_stacktrace_level?\n    end\n\n    data(\n      daily_age: 'daily',\n      weekly_age: 'weekly',\n      monthly_age: 'monthly',\n      integer_age: 2,\n    )\n    def test_logger_with_rotate_age_and_rotate_size(rotate_age)\n      config_path = \"#{@tmp_dir}/empty.conf\"\n      write_config config_path, \"\"\n\n      sv = Fluent::Supervisor.new(\n        config_path: config_path,\n        log_path: \"#{@tmp_dir}/test\",\n        log_rotate_age: rotate_age,\n        log_rotate_size: 10,\n      )\n      sv.__send__(:setup_global_logger)\n\n      assert_equal Fluent::LogDeviceIO, $log.out.class\n      assert_equal rotate_age, $log.out.instance_variable_get(:@shift_age)\n      assert_equal 10, $log.out.instance_variable_get(:@shift_size)\n    end\n\n    def test_can_start_with_rotate_but_no_log_path\n      config_path = \"#{@tmp_dir}/empty.conf\"\n      write_config config_path, \"\"\n\n      sv = Fluent::Supervisor.new(\n        config_path: config_path,\n        log_rotate_age: 5,\n      )\n      sv.__send__(:setup_global_logger)\n\n      assert_true $log.stdout?\n    end\n\n    sub_test_case \"system log rotation\" do\n      def parse_text(text)\n        basepath = File.expand_path(File.dirname(__FILE__) + '/../../')\n        Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }\n      end\n\n      def test_override_default_log_rotate\n        Tempfile.open do |file|\n          config = parse_text(<<-EOS)\n            <system>\n              <log>\n                rotate_age 3\n                rotate_size 300\n              </log>\n            </system>\n          EOS\n          file.puts(config)\n          file.flush\n          sv = Fluent::Supervisor.new({log_path: \"#{@tmp_dir}/test.log\", config_path: file.path})\n\n          sv.__send__(:setup_global_logger)\n          logger = $log.instance_variable_get(:@logger)\n\n          assert_equal Fluent::LogDeviceIO, $log.out.class\n          assert_equal 3, $log.out.instance_variable_get(:@shift_age)\n          assert_equal 300, $log.out.instance_variable_get(:@shift_size)\n        end\n      end\n\n      def test_override_default_log_rotate_with_yaml_config\n        Tempfile.open do |file|\n          config = <<-EOS\n            system:\n              log:\n                rotate_age: 3\n                rotate_size: 300\n          EOS\n          file.puts(config)\n          file.flush\n          sv = Fluent::Supervisor.new({log_path: \"#{@tmp_dir}/test.log\", config_path: file.path, config_file_type: :yaml})\n\n          sv.__send__(:setup_global_logger)\n          logger = $log.instance_variable_get(:@logger)\n\n          assert_equal Fluent::LogDeviceIO, $log.out.class\n          assert_equal 3, $log.out.instance_variable_get(:@shift_age)\n          assert_equal 300, $log.out.instance_variable_get(:@shift_size)\n        end\n      end\n    end\n\n    def test_log_level_affects\n      sv = Fluent::Supervisor.new({})\n\n      c = Fluent::Config::Element.new('system', '', { 'log_level' => 'error' }, [])\n      stub(Fluent::Config).build { config_element('ROOT', '', {}, [c]) }\n\n      sv.configure\n      assert_equal Fluent::Log::LEVEL_ERROR, $log.level\n    end\n\n    data(supervisor: true)\n    data(worker: false)\n    def test_log_path(supervisor)\n      log_path = Pathname(@tmp_dir) + \"fluentd.log\"\n      config_path = Pathname(@tmp_dir) + \"fluentd.conf\"\n      write_config config_path.to_s, \"\"\n\n      s = Fluent::Supervisor.new(config_path: config_path.to_s, log_path: log_path.to_s)\n      assert_rr do\n        mock.proxy(File).chmod(0o777, log_path.parent.to_s).never\n        s.__send__(:setup_global_logger, supervisor: supervisor)\n      end\n\n      assert { log_path.parent.exist? }\n    ensure\n      $log.out.close\n    end\n\n    data(supervisor: true)\n    data(worker: false)\n    def test_dir_permission(supervisor)\n      omit \"NTFS doesn't support UNIX like permissions\" if Fluent.windows?\n\n      log_path = Pathname(@tmp_dir) + \"fluentd.log\"\n      config_path = Pathname(@tmp_dir) + \"fluentd.conf\"\n      conf = <<~EOC\n        <system>\n          dir_permission 0o777\n        </system>\n      EOC\n      write_config config_path.to_s, conf\n\n      s = Fluent::Supervisor.new(config_path: config_path.to_s, log_path: log_path.to_s)\n      assert_rr do\n        mock.proxy(File).chmod(0o777, log_path.parent.to_s).once\n        s.__send__(:setup_global_logger, supervisor: supervisor)\n      end\n\n      assert { log_path.parent.exist? }\n      assert { (File.stat(log_path.parent).mode & 0xFFF) == 0o777 }\n    ensure\n      $log.out.close\n    end\n\n    def test_files_for_each_process_with_rotate_on_windows\n      omit \"Only for Windows.\" unless Fluent.windows?\n\n      log_path = Pathname(@tmp_dir) + \"log\" + \"fluentd.log\"\n      config_path = Pathname(@tmp_dir) + \"fluentd.conf\"\n      conf = <<~EOC\n        <system>\n          <log>\n            rotate_age 5\n          </log>\n        </system>\n      EOC\n      write_config config_path.to_s, conf\n\n      s = Fluent::Supervisor.new(config_path: config_path.to_s, log_path: log_path.to_s)\n      s.__send__(:setup_global_logger, supervisor: true)\n      $log.out.close\n\n      s = Fluent::Supervisor.new(config_path: config_path.to_s, log_path: log_path.to_s)\n      s.__send__(:setup_global_logger, supervisor: false)\n      $log.out.close\n\n      ENV[\"SERVERENGINE_WORKER_ID\"] = \"1\"\n      s = Fluent::Supervisor.new(config_path: config_path.to_s, log_path: log_path.to_s)\n      s.__send__(:setup_global_logger, supervisor: false)\n      $log.out.close\n\n      assert { log_path.parent.entries.size == 5 } # [\".\", \"..\", \"logfile.log\", ...]\n    ensure\n      ENV.delete(\"SERVERENGINE_WORKER_ID\")\n    end\n  end\n\n  def test_enable_shared_socket\n    server = DummyServer.new\n    begin\n      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')\n      server.before_run\n      sleep 0.1 if Fluent.windows? # Wait for starting windows event thread\n      assert_not_nil(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])\n    ensure\n      server.after_run\n      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')\n    end\n  end\n\n  def test_disable_shared_socket\n    server = DummyServer.new\n    def server.config\n      {\n        :disable_shared_socket => true,\n      }\n    end\n    begin\n      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')\n      server.before_run\n      sleep 0.1 if Fluent.windows? # Wait for starting windows event thread\n      assert_nil(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])\n    ensure\n      server.after_run\n      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')\n    end\n  end\n\n  sub_test_case \"zero_downtime_restart\" do\n    setup do\n      omit \"Not supported on Windows\" if Fluent.windows?\n    end\n\n    data(\n      # When daemonize, exit-status is important. The new spawned process does double-fork and exits soon.\n      \"daemonize and succeeded double-fork of new process\" => [true, true, 0, false],\n      \"daemonize and failed double-fork of new process\" => [true, false, 0, true],\n      # When no daemon, whether the new spawned process is alive is important, not exit-status.\n      \"no daemon and new process alive\" => [false, false, 3, false],\n      \"no daemon and new process dead\" => [false, false, 0, true],\n    )\n    def test_zero_downtime_restart((daemonize, wait_success, wait_sleep, restart_canceled))\n      # == Arrange ==\n      env_spawn = {}\n      pid_wait = nil\n\n      server = DummyServer.new\n\n      stub(server).config do\n        {\n          daemonize: daemonize,\n          pid_path: \"test-pid-file\",\n        }\n      end\n      process_stub = stub(Process)\n      process_stub.spawn do |env, commands|\n        env_spawn = env\n        -1\n      end\n      process_stub.wait2 do |pid|\n        pid_wait = pid\n        sleep wait_sleep\n        if wait_success\n          status = Class.new{def success?; true; end}.new\n        else\n          status = Class.new{def success?; false; end}.new\n        end\n        [pid, status]\n      end\n      stub(File).read(\"test-pid-file\") { -1 }\n\n      # mock to check notify_new_supervisor_that_old_one_has_stopped sends SIGWINCH\n      if restart_canceled\n        mock(Process).kill(:WINCH, -1).never\n      else\n        mock(Process).kill(:WINCH, -1)\n      end\n\n      # == Act and Assert ==\n      server.before_run\n      server.zero_downtime_restart.join\n      sleep 1 # To wait a sub thread for waitpid in zero_downtime_restart\n      server.after_run\n\n      assert_equal(\n        [\n          !restart_canceled,\n          true,\n          Process.pid,\n          -1,\n        ],\n        [\n          server.instance_variable_get(:@starting_new_supervisor_with_zero_downtime),\n          env_spawn.key?(\"SERVERENGINE_SOCKETMANAGER_INTERNAL_TOKEN\"),\n          env_spawn[\"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\"].to_i,\n          pid_wait,\n        ]\n      )\n    ensure\n      Fluent::Supervisor.cleanup_socketmanager_path\n      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')\n    end\n\n    def test_share_sockets\n      server = DummyServer.new\n      server.before_run\n      path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']\n\n      client = ServerEngine::SocketManager::Client.new(path)\n      udp_port = unused_port(protocol: :udp)\n      tcp_port = unused_port(protocol: :tcp)\n      client.listen_udp(\"localhost\", udp_port)\n      client.listen_tcp(\"localhost\", tcp_port)\n\n      ENV['FLUENT_RUNNING_IN_PARALLEL_WITH_OLD'] = \"\"\n      new_server = DummyServer.new\n      stub(new_server).stop_parallel_old_supervisor_after_delay\n      new_server.before_run\n\n      assert_equal(\n        [[udp_port], [tcp_port]],\n        [\n          new_server.socket_manager_server.udp_sockets.values.map { |v| v.addr[1] },\n          new_server.socket_manager_server.tcp_sockets.values.map { |v| v.addr[1] },\n        ]\n      )\n    ensure\n      server&.after_run\n      new_server&.after_run\n      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')\n      ENV.delete(\"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\")\n    end\n\n    def test_stop_parallel_old_supervisor_after_delay\n      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = \"\"\n      ENV['FLUENT_RUNNING_IN_PARALLEL_WITH_OLD'] = \"-1\"\n      stub(ServerEngine::SocketManager::Server).share_sockets_with_another_server\n      mock(Process).kill(:TERM, -1)\n\n      server = DummyServer.new\n      server.before_run\n      sleep 12 # Can't we skip the delay for this test?\n    ensure\n      server&.after_run\n      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')\n      ENV.delete(\"FLUENT_RUNNING_IN_PARALLEL_WITH_OLD\")\n    end\n  end\n\n  sub_test_case \"include additional configuration\" do\n    setup do\n      @config_include_dir = File.join(@tmp_dir, \"conf.d\")\n      FileUtils.mkdir_p(@config_include_dir)\n    end\n\n    test \"no additional configuration\" do\n      c = Fluent::Config::Element.new('system', '', { 'config_include_dir' => '' }, [])\n      stub(Fluent::Config).build { config_element('ROOT', '', {}, [c]) }\n      supervisor = Fluent::Supervisor.new({})\n      supervisor.configure(supervisor: true)\n      assert_equal([c], supervisor.instance_variable_get(:@conf).elements)\n    end\n\n    data(\n      \"single source\" => [\"forward\"],\n      \"multiple sources\" => [\"forward\", \"tcp\"])\n    test \"additional configuration\" do |sources|\n      c = Fluent::Config::Element.new('system', '',\n                                      { 'config_include_dir' => @config_include_dir }, [])\n      config_path = \"#{@config_include_dir}/dummy.conf\"\n      stub.proxy(Fluent::Config).build\n      stub(Fluent::Config).build(config_path: \"/etc/fluent/fluent.conf\", encoding: \"utf-8\",\n                                 additional_config: anything, use_v1_config: anything,\n                                 type: anything,\n                                 on_file_parsed: anything) { config_element('ROOT', '', {}, [c]) }\n      sources.each do |type|\n        config = <<~EOF\n        <source>\n          @type #{type}\n        </source>\n        EOF\n        additional_config_path = \"#{@config_include_dir}/#{type}.conf\"\n        write_config(additional_config_path, config)\n      end\n      supervisor = Fluent::Supervisor.new({})\n      supervisor.configure(supervisor: true)\n      expected = [c].concat(sources.collect { |type| {\"@type\" => type} })\n      assert_equal(expected, supervisor.instance_variable_get(:@conf).elements)\n    end\n\n    data(\n      \"single YAML source\" => [\"forward\"],\n      \"multiple YAML sources\" => [\"forward\", \"tcp\"])\n    test \"additional YAML configuration\" do |sources|\n      c = Fluent::Config::Element.new('system', '',\n                                      { 'config_include_dir' => @config_include_dir }, [])\n      config_path = \"#{@config_include_dir}/dummy.yml\"\n      stub.proxy(Fluent::Config).build\n      stub(Fluent::Config).build(config_path: \"/etc/fluent/fluent.conf\", encoding: \"utf-8\",\n                                 additional_config: anything, use_v1_config: anything,\n                                 type: anything,\n                                 on_file_parsed: anything) { config_element('ROOT', '', {}, [c]) }\n      sources.each do |type|\n        config = <<~EOF\n        config:\n          - source:\n              $type: #{type}\n        EOF\n        additional_config_path = \"#{@config_include_dir}/#{type}.yml\"\n        write_config(additional_config_path, config)\n      end\n      supervisor = Fluent::Supervisor.new({})\n      supervisor.configure(supervisor: true)\n      expected = [c].concat(sources.collect { |type| {\"@type\" => type} })\n      assert_equal(expected, supervisor.instance_variable_get(:@conf).elements)\n    end\n\n    data(\n      \"single source\" => [false, [\"forward\"]],\n      \"multiple sources\" => [false, [\"forward\", \"tcp\"]],\n      \"single YAML source\" => [true, [\"forward\"]],\n      \"multiple YAML sources\" => [true, [\"forward\", \"tcp\"]])\n    test \"reload with additional configuration\" do |(yaml, sources)|\n      c = Fluent::Config::Element.new('system', '',\n                                      { 'config_include_dir' => @config_include_dir }, [])\n      config_path = \"#{@config_include_dir}/dummy.yml\"\n      stub.proxy(Fluent::Config).build\n      stub(Fluent::Config).build(config_path: \"/etc/fluent/fluent.conf\", encoding: \"utf-8\",\n                                 additional_config: anything, use_v1_config: anything,\n                                 type: anything,\n                                 on_file_parsed: anything) { config_element('ROOT', '', {}, [c]) }\n      sources.each do |type|\n        if yaml\n          config = <<~EOF\n          config:\n            - source:\n                $type: #{type}\n          EOF\n          additional_config_path = \"#{@config_include_dir}/#{type}.yml\"\n          write_config(additional_config_path, config)\n        else\n          config = <<~EOF\n          <source>\n            @type #{type}\n          </source>\n          EOF\n          additional_config_path = \"#{@config_include_dir}/#{type}.conf\"\n          write_config(additional_config_path, config)\n        end\n      end\n      supervisor = Fluent::Supervisor.new({})\n      supervisor.configure(supervisor: true)\n      supervisor.__send__(:reload_config)\n      expected = [c].concat(sources.collect { |type| {\"@type\" => type} })\n      assert_equal(expected, supervisor.instance_variable_get(:@conf).elements)\n    end\n\n    test \"prevent duplicate loading\" do\n      write_config(\"#{@config_include_dir}/system.conf\", <<~EOF)\n        <system>\n          config_include_dir #{@config_include_dir}\n        </system>\n      EOF\n      write_config(\"#{@config_include_dir}/forward.conf\", <<~EOF)\n        <match test>\n          @type forward\n          <buffer>\n            @include #{@config_include_dir}/common_param.conf\n          </buffer>\n          <server>\n            host 127.0.0.1\n            port 24224\n          </server>\n        </match>\n      EOF\n      write_config(\"#{@config_include_dir}/common_param.conf\", <<~EOF)\n        flush_interval 5s\n      EOF\n      write_config(\"#{@config_include_dir}/obsolete_plugins.conf\", <<~EOF)\n        <source>\n          @type obsolete_plugins\n        </source>\n      EOF\n\n      write_config(\"#{@tmp_dir}/fluent.conf\", <<~EOF)\n        <match sample.*>\n          @type file\n          <buffer>\n            @include #{@config_include_dir}/common_param.conf\n          </buffer>\n        </match>\n\n        @include #{@config_include_dir}/forward.conf\n        @include #{@config_include_dir}/system.conf\n      EOF\n\n      supervisor = Fluent::Supervisor.new({ config_path: \"#{@tmp_dir}/fluent.conf\" })\n      stub(supervisor).setup_global_logger { create_debug_dummy_logger }\n\n      supervisor.configure(supervisor: true)\n      elements = supervisor.instance_variable_get(:@conf).elements\n      assert_equal(4, elements.size)\n\n      assert_equal('match', elements[0].name)\n      assert_equal('file', elements[0]['@type'])\n      assert_equal('buffer', elements[0].elements[0].name)\n      assert_equal('5s', elements[0].elements[0]['flush_interval'])\n\n      assert_equal('match', elements[1].name)\n      assert_equal('forward', elements[1]['@type'])\n      assert_equal('buffer', elements[1].elements[0].name)\n      assert_equal('5s', elements[1].elements[0]['flush_interval'])\n\n      assert_equal('system', elements[2].name)\n      assert_equal(@config_include_dir, elements[2]['config_include_dir'])\n\n      assert_equal('source', elements[3].name)\n      assert_equal('obsolete_plugins', elements[3]['@type'])\n\n      skipped_files = %W[\n        #{@config_include_dir}/common_param.conf\n        #{@config_include_dir}/forward.conf\n        #{@config_include_dir}/system.conf\n      ]\n      loaded_files = %W[\n        #{@config_include_dir}/obsolete_plugins.conf\n      ]\n\n      logs_line = $log.out.logs.join\n      skipped_files.each do |path|\n        assert { logs_line.include?(\"skip auto loading, it was already loaded path=\\\"#{path}\\\"\") }\n      end\n      loaded_files.each do |path|\n        assert { logs_line.include?(\"loading additional configuration file path=\\\"#{path}\\\"\") }\n      end\n\n      # reload\n      $log.out.reset\n      supervisor.__send__(:reload_config)\n      sleep 0.2 # wait to finish reloading\n\n      reload_elements = supervisor.instance_variable_get(:@conf).elements\n      assert_equal(elements, reload_elements)\n\n      logs_line = $log.out.logs.join\n      skipped_files.each do |path|\n        assert { logs_line.include?(\"skip auto loading, it was already loaded path=\\\"#{path}\\\"\") }\n      end\n      loaded_files.each do |path|\n        assert { logs_line.include?(\"loading additional configuration file path=\\\"#{path}\\\"\") }\n      end\n    ensure\n      $log.out.reset if $log&.out&.respond_to?(:reset)\n    end\n\n    test \"do not load additional configuration when loaded all files with @include\" do\n      write_config(\"#{@config_include_dir}/forward.conf\", <<~EOF)\n        <match test>\n          @type forward\n          <server>\n            host 127.0.0.1\n            port 24224\n          </server>\n        </match>\n      EOF\n      write_config(\"#{@config_include_dir}/obsolete_plugins.conf\", <<~EOF)\n        <source>\n          @type obsolete_plugins\n        </source>\n      EOF\n\n      write_config(\"#{@tmp_dir}/fluent.conf\", <<~EOF)\n        <system>\n          config_include_dir #{@config_include_dir}\n        </system>\n\n        <match sample.*>\n          @type file\n        </match>\n\n        @include #{@config_include_dir}/*.conf\n      EOF\n\n      supervisor = Fluent::Supervisor.new({ config_path: \"#{@tmp_dir}/fluent.conf\" })\n      stub(supervisor).setup_global_logger { create_debug_dummy_logger }\n\n      supervisor.configure(supervisor: true)\n      elements = supervisor.instance_variable_get(:@conf).elements\n      assert_equal(4, elements.size)\n\n      assert_equal('system', elements[0].name)\n      assert_equal(@config_include_dir, elements[0]['config_include_dir'])\n\n      assert_equal('match', elements[1].name)\n      assert_equal('file', elements[1]['@type'])\n\n      assert_equal('match', elements[2].name)\n      assert_equal('forward', elements[2]['@type'])\n\n      assert_equal('source', elements[3].name)\n      assert_equal('obsolete_plugins', elements[3]['@type'])\n\n      # no additional load, all files were skipped\n      skipped_files = %W[\n        #{@config_include_dir}/forward.conf\n        #{@config_include_dir}/obsolete_plugins.conf\n      ]\n\n      logs_line = $log.out.logs.join\n      skipped_files.each do |path|\n        assert { logs_line.include?(\"skip auto loading, it was already loaded path=\\\"#{path}\\\"\") }\n      end\n      assert_not_match(/loading additional configuration file/, logs_line)\n    ensure\n      $log.out.reset if $log&.out&.respond_to?(:reset)\n    end\n\n    test \"can load partial config loaded config_include_dir feature by even if already loaded\" do\n      write_config(\"#{@config_include_dir}/system.conf\", <<~EOF)\n        <system>\n          config_include_dir #{@config_include_dir}\n        </system>\n      EOF\n      write_config(\"#{@config_include_dir}/forward.conf\", <<~EOF)\n        <match test>\n          @type forward\n          <buffer>\n            @include #{@config_include_dir}/common_param.conf\n          </buffer>\n          <server>\n            host 127.0.0.1\n            port 24224\n          </server>\n        </match>\n      EOF\n      write_config(\"#{@config_include_dir}/common_param.conf\", <<~EOF)\n        flush_interval 5s\n      EOF\n      write_config(\"#{@config_include_dir}/obsolete_plugins.conf\", <<~EOF)\n        <source>\n          @type obsolete_plugins\n        </source>\n      EOF\n\n      write_config(\"#{@tmp_dir}/fluent.conf\", <<~EOF)\n        <match sample.*>\n          @type file\n          <buffer>\n            @include #{@config_include_dir}/common_param.conf\n          </buffer>\n        </match>\n\n        @include #{@config_include_dir}/system.conf\n      EOF\n\n      supervisor = Fluent::Supervisor.new({ config_path: \"#{@tmp_dir}/fluent.conf\" })\n      stub(supervisor).setup_global_logger { create_debug_dummy_logger }\n\n      supervisor.configure(supervisor: true)\n      elements = supervisor.instance_variable_get(:@conf).elements\n      assert_equal(4, elements.size)\n\n      assert_equal('match', elements[0].name)\n      assert_equal('file', elements[0]['@type'])\n      assert_equal('buffer', elements[0].elements[0].name)\n      assert_equal('5s', elements[0].elements[0]['flush_interval'])\n\n      assert_equal('system', elements[1].name)\n      assert_equal(@config_include_dir, elements[1]['config_include_dir'])\n\n      # include forward.conf using config_include_dir feature\n      assert_equal('match', elements[2].name)\n      assert_equal('forward', elements[2]['@type'])\n      assert_equal('buffer', elements[2].elements[0].name)\n      assert_equal('5s', elements[2].elements[0]['flush_interval'])\n\n      assert_equal('source', elements[3].name)\n      assert_equal('obsolete_plugins', elements[3]['@type'])\n\n      skipped_files = %W[\n        #{@config_include_dir}/common_param.conf\n        #{@config_include_dir}/system.conf\n      ]\n      loaded_files = %W[\n        #{@config_include_dir}/forward.conf\n        #{@config_include_dir}/obsolete_plugins.conf\n      ]\n\n      logs_line = $log.out.logs.join\n      skipped_files.each do |path|\n        assert { logs_line.include?(\"skip auto loading, it was already loaded path=\\\"#{path}\\\"\") }\n      end\n      loaded_files.each do |path|\n        assert { logs_line.include?(\"loading additional configuration file path=\\\"#{path}\\\"\") }\n      end\n    ensure\n      $log.out.reset if $log&.out&.respond_to?(:reset)\n    end\n\n    test \"can load config files even if disable config_include_dir\" do\n      write_config(\"#{@config_include_dir}/forward.conf\", <<~EOF)\n        <match test>\n          @type forward\n          <server>\n            host 127.0.0.1\n            port 24224\n          </server>\n        </match>\n      EOF\n\n      write_config(\"#{@tmp_dir}/fluent.conf\", <<~EOF)\n        <system>\n          config_include_dir \"\" \n        </system>\n\n        <match sample.*>\n          @type file\n        </match>\n\n        @include #{@config_include_dir}/*.conf\n      EOF\n\n      supervisor = Fluent::Supervisor.new({ config_path: \"#{@tmp_dir}/fluent.conf\" })\n      stub(supervisor).setup_global_logger { create_debug_dummy_logger }\n\n      supervisor.configure(supervisor: true)\n      elements = supervisor.instance_variable_get(:@conf).elements\n      assert_equal(3, elements.size)\n\n      assert_equal('system', elements[0].name)\n      assert_equal(\"\", elements[0]['config_include_dir'])\n\n      assert_equal('match', elements[1].name)\n      assert_equal('file', elements[1]['@type'])\n\n      assert_equal('match', elements[2].name)\n      assert_equal('forward', elements[2]['@type'])\n\n      # There is no logs for additional loading\n      logs_line = $log.out.logs.join\n      assert_not_match(/skip auto loading, it was already loaded/, logs_line)\n      assert_not_match(/loading additional configuration file/, logs_line)\n    ensure\n      $log.out.reset if $log&.out&.respond_to?(:reset)\n    end\n  end\n\n  def create_debug_dummy_logger\n    dl_opts = {}\n    dl_opts[:log_level] = ServerEngine::DaemonLogger::DEBUG\n    logdev = Fluent::Test::DummyLogDevice.new\n    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n    $log = Fluent::Log.new(logger)\n  end\n\n  def create_info_dummy_logger\n    dl_opts = {}\n    dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO\n    logdev = Fluent::Test::DummyLogDevice.new\n    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)\n    $log = Fluent::Log.new(logger)\n  end\nend\n"
  },
  {
    "path": "test/test_test_drivers.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/plugin/input'\nrequire 'fluent/test/driver/input'\nrequire 'fluent/plugin/output'\nrequire 'fluent/test/driver/output'\nrequire 'fluent/plugin/filter'\nrequire 'fluent/test/driver/filter'\nrequire 'fluent/plugin/multi_output'\nrequire 'fluent/test/driver/multi_output'\nrequire 'fluent/plugin/parser'\nrequire 'fluent/test/driver/parser'\nrequire 'fluent/plugin/formatter'\nrequire 'fluent/test/driver/formatter'\n\nrequire 'timecop'\n\nclass TestDriverTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  sub_test_case 'plugin test driver' do\n    data(\n      'input plugin test driver'  => [Fluent::Test::Driver::Input, Fluent::Plugin::Input],\n      'multi_output plugin test driver' => [Fluent::Test::Driver::MultiOutput, Fluent::Plugin::MultiOutput],\n      'parser plugin test driver'       => [Fluent::Test::Driver::Parser, Fluent::Plugin::Parser],\n      'formatter plugin test driver'    => [Fluent::Test::Driver::Formatter, Fluent::Plugin::Formatter],\n    )\n    test 'returns the block value as the return value of #run' do |args|\n      driver_class, plugin_class = args\n      d = driver_class.new(Class.new(plugin_class))\n      v = d.run do\n        x = 1 + 2\n        y = 2 + 4\n        3 || x || y\n      end\n      assert_equal 3, v\n    end\n\n    data(\n      'input plugin test driver'  => [Fluent::Test::Driver::Input, Fluent::Plugin::Input],\n      'multi_output plugin test driver' => [Fluent::Test::Driver::MultiOutput, Fluent::Plugin::MultiOutput],\n      'parser plugin test driver'       => [Fluent::Test::Driver::Parser, Fluent::Plugin::Parser],\n      'formatter plugin test driver'    => [Fluent::Test::Driver::Formatter, Fluent::Plugin::Formatter],\n    )\n    test 'raises error for hard timeout' do |args|\n      driver_class, plugin_class = args\n      d = driver_class.new(Class.new(plugin_class))\n      assert_raise Fluent::Test::Driver::TestTimedOut do\n        d.run(timeout: 0.5) do\n          sleep 2\n        end\n      end\n    end\n\n    data(\n      'input plugin test driver'  => [Fluent::Test::Driver::Input, Fluent::Plugin::Input],\n      'multi_output plugin test driver' => [Fluent::Test::Driver::MultiOutput, Fluent::Plugin::MultiOutput],\n      'parser plugin test driver'       => [Fluent::Test::Driver::Parser, Fluent::Plugin::Parser],\n      'formatter plugin test driver'    => [Fluent::Test::Driver::Formatter, Fluent::Plugin::Formatter],\n    )\n    test 'can stop with soft timeout for blocks never stops, even with Timecop' do |args|\n      Timecop.freeze(Time.parse(\"2016-11-04 18:49:00\"))\n      begin\n        driver_class, plugin_class = args\n        d = driver_class.new(Class.new(plugin_class))\n        assert_nothing_raised do\n          before = Process.clock_gettime(Process::CLOCK_MONOTONIC)\n          d.end_if{ false }\n          d.run(timeout: 1) do\n            sleep 0.1 until d.stop?\n          end\n          after = Process.clock_gettime(Process::CLOCK_MONOTONIC)\n          assert{ after >= before + 1.0 }\n        end\n      ensure\n        Timecop.return\n      end\n    end\n\n    test 'raise errors raised in threads' do\n      d = Fluent::Test::Driver::Input.new(Fluent::Plugin::Input) do\n        helpers :thread\n        def start\n          super\n          thread_create(:input_thread_for_test_driver_test) do\n            sleep 0.5\n            raise \"yaaaaaaaaaay!\"\n          end\n        end\n      end\n\n      assert_raise RuntimeError.new(\"yaaaaaaaaaay!\") do\n        d.end_if{ false }\n        d.run(timeout: 3) do\n          sleep 0.1 until d.stop?\n        end\n      end\n    end\n  end\n\n  sub_test_case 'output plugin test driver' do\n    test 'returns the block value as the return value of #run' do\n      d = Fluent::Test::Driver::Output.new(Fluent::Plugin::Output) do\n        def prefer_buffered_processing\n          false\n        end\n        def process(tag, es)\n          # drop\n        end\n      end\n      v = d.run do\n        x = 1 + 2\n        y = 2 + 4\n        3 || x || y\n      end\n      assert_equal 3, v\n    end\n  end\n\n  sub_test_case 'filter plugin test driver' do\n    test 'returns the block value as the return value of #run' do\n      d = Fluent::Test::Driver::Filter.new(Fluent::Plugin::Filter) do\n        def filter(tag, time, record)\n          record\n        end\n      end\n      v = d.run do\n        x = 1 + 2\n        y = 2 + 4\n        3 || x || y\n      end\n      assert_equal 3, v\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_time_formatter.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/test'\nrequire 'fluent/time'\n\nclass TimeFormatterTest < ::Test::Unit::TestCase\n  setup do\n    @fmt =\"%Y%m%d %H%M%z\"  # YYYYMMDD HHMM[+-]HHMM\n  end\n\n  def format(format, localtime, timezone)\n    formatter = Fluent::TimeFormatter.new(format, localtime, timezone)\n    formatter.format(@time)\n  end\n\n  module TestLists\n    def test_default_utc_nil\n      assert_equal(\"2014-09-27T00:00:00Z\", format(nil, false, nil))\n    end\n\n    def test_default_utc_pHH_MM\n      assert_equal(\"2014-09-27T01:30:00+01:30\", format(nil, false, \"+01:30\"))\n    end\n\n    def test_default_utc_nHH_MM\n      assert_equal(\"2014-09-26T22:30:00-01:30\", format(nil, false, \"-01:30\"))\n    end\n\n    def test_default_utc_pHHMM\n      assert_equal(\"2014-09-27T02:30:00+02:30\", format(nil, false, \"+0230\"))\n    end\n\n    def test_default_utc_nHHMM\n      assert_equal(\"2014-09-26T21:30:00-02:30\", format(nil, false, \"-0230\"))\n    end\n\n    def test_default_utc_pHH\n      assert_equal(\"2014-09-27T03:00:00+03:00\", format(nil, false, \"+03\"))\n    end\n\n    def test_default_utc_nHH\n      assert_equal(\"2014-09-26T21:00:00-03:00\", format(nil, false, \"-03\"))\n    end\n\n    def test_default_utc_timezone_1\n      # Asia/Tokyo (+09:00) does not have daylight saving time.\n      assert_equal(\"2014-09-27T09:00:00+09:00\", format(nil, false, \"Asia/Tokyo\"))\n    end\n\n    def test_default_utc_timezone_2\n      # Pacific/Honolulu (-10:00) does not have daylight saving time.\n      assert_equal(\"2014-09-26T14:00:00-10:00\", format(nil, false, \"Pacific/Honolulu\"))\n    end\n\n    def test_default_utc_timezone_3\n      # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time.\n      assert_equal(\"2014-09-26T21:00:00-03:00\", format(nil, false, \"America/Argentina/Buenos_Aires\"))\n    end\n\n    def test_default_utc_timezone_4\n      # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its\n      # UTC offset in DST is +02:00. In September, Europe/Paris is in DST.\n      assert_equal(\"2014-09-27T02:00:00+02:00\", format(nil, false, \"Europe/Paris\"))\n    end\n\n    def test_default_utc_timezone_5\n      # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its\n      # UTC offset in DST is +02:00. In January, Europe/Paris is not in DST.\n      @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i\n      assert_equal(\"2014-01-24T01:00:00+01:00\", format(nil, false, \"Europe/Paris\"))\n    end\n\n    def test_default_utc_invalid\n      assert_equal(\"2014-09-27T00:00:00Z\", format(nil, false, \"Invalid\"))\n    end\n\n    def test_default_localtime_nil_1\n      with_timezone(\"UTC-04\") do\n        assert_equal(\"2014-09-27T04:00:00+04:00\", format(nil, true, nil))\n      end\n    end\n\n    def test_default_localtime_nil_2\n      with_timezone(\"UTC+05\") do\n        assert_equal(\"2014-09-26T19:00:00-05:00\", format(nil, true, nil))\n      end\n    end\n\n    def test_default_localtime_timezone\n      # 'timezone' takes precedence over 'localtime'.\n      with_timezone(\"UTC-06\") do\n        assert_equal(\"2014-09-27T07:00:00+07:00\", format(nil, true, \"+07\"))\n      end\n    end\n\n    def test_specific_utc_nil\n      assert_equal(\"20140927 0000+0000\", format(@fmt, false, nil))\n    end\n\n    def test_specific_utc_pHH_MM\n      assert_equal(\"20140927 0830+0830\", format(@fmt, false, \"+08:30\"))\n    end\n\n    def test_specific_utc_nHH_MM\n      assert_equal(\"20140926 1430-0930\", format(@fmt, false, \"-09:30\"))\n    end\n\n    def test_specific_utc_pHHMM\n      assert_equal(\"20140927 1030+1030\", format(@fmt, false, \"+1030\"))\n    end\n\n    def test_specific_utc_nHHMM\n      assert_equal(\"20140926 1230-1130\", format(@fmt, false, \"-1130\"))\n    end\n\n    def test_specific_utc_pHH\n      assert_equal(\"20140927 1200+1200\", format(@fmt, false, \"+12\"))\n    end\n\n    def test_specific_utc_nHH\n      assert_equal(\"20140926 1100-1300\", format(@fmt, false, \"-13\"))\n    end\n\n    def test_specific_utc_timezone_1\n      # Europe/Moscow (+04:00) does not have daylight saving time.\n      assert_equal(\"20140927 0400+0400\", format(@fmt, false, \"Europe/Moscow\"))\n    end\n\n    def test_specific_utc_timezone_2\n      # Pacific/Galapagos (-06:00) does not have daylight saving time.\n      assert_equal(\"20140926 1800-0600\", format(@fmt, false, \"Pacific/Galapagos\"))\n    end\n\n    def test_specific_utc_timezone_3\n      # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time.\n      assert_equal(\"20140926 2100-0300\", format(@fmt, false, \"America/Argentina/Buenos_Aires\"))\n    end\n\n    def test_specific_utc_timezone_4\n      # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its\n      # UTC offset in DST is -07:00. In September, America/Los_Angeles is in DST.\n      assert_equal(\"20140926 1700-0700\", format(@fmt, false, \"America/Los_Angeles\"))\n    end\n\n    def test_specific_utc_timezone_5\n      # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its\n      # UTC offset in DST is -07:00. In January, America/Los_Angeles is not in DST.\n      @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i\n      assert_equal(\"20140123 1600-0800\", format(@fmt, false, \"America/Los_Angeles\"))\n    end\n\n    def test_specific_utc_invalid\n      assert_equal(\"20140927 0000+0000\", format(@fmt, false, \"Invalid\"))\n    end\n\n    def test_specific_localtime_nil_1\n      with_timezone(\"UTC-07\") do\n        assert_equal(\"20140927 0700+0700\", format(@fmt, true, nil))\n      end\n    end\n\n    def test_specific_localtime_nil_2\n      with_timezone(\"UTC+08\") do\n        assert_equal(\"20140926 1600-0800\", format(@fmt, true, nil))\n      end\n    end\n\n    def test_specific_localtime_timezone\n      # 'timezone' takes precedence over 'localtime'.\n      with_timezone(\"UTC-09\") do\n        assert_equal(\"20140926 1400-1000\", format(@fmt, true, \"-10\"))\n      end\n    end\n  end\n\n  sub_test_case 'Fluent::EventTime time' do\n    setup do\n      @time = Fluent::EventTime.from_time(Time.new(2014, 9, 27, 0, 0, 0, 0))\n    end\n\n    include TestLists\n  end\n\n  # for v0.12 compatibility\n  sub_test_case 'Integer time' do\n    setup do\n      @time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i\n    end\n\n    include TestLists\n  end\n\n  def test_format_with_subsec\n    time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i\n    time = Fluent::EventTime.new(time)\n    formatter = Fluent::TimeFormatter.new(\"%Y%m%d %H%M.%N\", false, nil)\n    assert_equal(\"20140927 0000.000000000\", formatter.format(time))\n  end\n\n  sub_test_case 'TimeMixin::Formatter' do\n    class DummyForTimeFormatter\n      include Fluent::Configurable\n      include Fluent::TimeMixin::Formatter\n    end\n\n    test 'provides configuration parameters for TimeFormatter with default values for localtime' do\n      str = with_timezone(\"UTC+07\") do\n        i = DummyForTimeFormatter.new\n        i.configure(config_element('format'))\n\n        assert_nil   i.time_format\n        assert_true  i.localtime\n        assert_false i.utc\n        assert_nil   i.timezone\n\n        fmt = i.time_formatter_create\n        fmt.format(event_time(\"2016-09-02 18:42:31.012345678 UTC\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n      end\n      assert_equal \"2016-09-02T11:42:31-07:00\", str\n    end\n\n    test 'provides configuration parameters for TimeFormatter, configurable for any time format' do\n      str = with_timezone(\"UTC+07\") do\n        i = DummyForTimeFormatter.new\n        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z'}))\n\n        fmt = i.time_formatter_create\n        fmt.format(event_time(\"2016-09-02 18:42:31.012345678 UTC\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n      end\n      assert_equal \"2016-09-02 11:42:31.012345678 -0700\", str\n    end\n\n    test 'provides configuration parameters for TimeFormatter, configurable for UTC' do\n      str = with_timezone(\"UTC+07\") do\n        i = DummyForTimeFormatter.new\n        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z', 'utc' => 'true'}))\n\n        fmt = i.time_formatter_create\n        fmt.format(event_time(\"2016-09-02 18:42:31.012345678 UTC\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n      end\n      assert_equal \"2016-09-02 18:42:31.012345678 +0000\", str\n    end\n\n    test 'provides configuration parameters for TimeFormatter, configurable for any timezone' do\n      str = with_timezone(\"UTC+07\") do\n        i = DummyForTimeFormatter.new\n        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z', 'timezone' => '+0900'}))\n\n        fmt = i.time_formatter_create\n        fmt.format(event_time(\"2016-09-02 18:42:31.012345678 UTC\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n      end\n      assert_equal \"2016-09-03 03:42:31.012345678 +0900\", str\n    end\n\n    test '#time_formatter_create returns TimeFormatter with specified time format and timezone' do\n      str = with_timezone(\"UTC+07\") do\n        i = DummyForTimeFormatter.new\n        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z', 'timezone' => '+0900'}))\n\n        fmt = i.time_formatter_create(format: '%m/%d/%Y %H-%M-%S %N', timezone: '+0000')\n        fmt.format(event_time(\"2016-09-02 18:42:31.012345678 UTC\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n      end\n      assert_equal \"09/02/2016 18-42-31 012345678\", str\n    end\n\n    test '#time_formatter_create returns TimeFormatter with localtime besides any configuration parameters' do\n      str = with_timezone(\"UTC+07\") do\n        i = DummyForTimeFormatter.new\n        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z', 'utc' => 'true'}))\n\n        fmt = i.time_formatter_create(format: '%m/%d/%Y %H-%M-%S %N', force_localtime: true)\n        fmt.format(event_time(\"2016-09-02 18:42:31.012345678 UTC\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n      end\n      assert_equal \"09/02/2016 11-42-31 012345678\", str\n\n      str = with_timezone(\"UTC+07\") do\n        i = DummyForTimeFormatter.new\n        i.configure(config_element('format', '', {'time_format' => '%Y-%m-%d %H:%M:%S.%N %z', 'timezone' => '+0900'}))\n\n        fmt = i.time_formatter_create(format: '%m/%d/%Y %H-%M-%S %N', force_localtime: true)\n        fmt.format(event_time(\"2016-09-02 18:42:31.012345678 UTC\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n      end\n      assert_equal \"09/02/2016 11-42-31 012345678\", str\n    end\n  end\n\n  test '#time_formatter_create returns NumericTimeFormatter to format time as unixtime when time_type unixtime specified' do\n    i = DummyForTimeFormatter.new\n    i.configure(config_element('format', '', {'time_type' => 'unixtime'}))\n    fmt = i.time_formatter_create\n    time = event_time(\"2016-10-03 20:08:30.123456789 +0100\", format: '%Y-%m-%d %H:%M:%S.%N %z')\n    assert_equal \"#{time.sec}\", fmt.format(time)\n  end\n\n  test '#time_formatter_create returns NumericTimeFormatter to format time as float when time_type float specified' do\n    i = DummyForTimeFormatter.new\n    i.configure(config_element('format', '', {'time_type' => 'float'}))\n    fmt = i.time_formatter_create\n    time = event_time(\"2016-10-03 20:08:30.123456789 +0100\", format: '%Y-%m-%d %H:%M:%S.%N %z')\n    assert_equal \"#{time.sec}.#{time.nsec}\", fmt.format(time)\n  end\nend\n"
  },
  {
    "path": "test/test_time_parser.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/test'\nrequire 'fluent/time'\n\nclass TimeParserTest < ::Test::Unit::TestCase\n  def setup\n    Fluent::Test.setup\n  end\n\n  def test_call_with_parse\n    parser = Fluent::TimeParser.new\n\n    assert(parser.parse('2013-09-18 12:00:00 +0900').is_a?(Fluent::EventTime))\n\n    time = event_time('2013-09-18 12:00:00 +0900')\n    assert_equal(time, parser.parse('2013-09-18 12:00:00 +0900'))\n  end\n\n  def test_parse_with_strptime\n    parser = Fluent::TimeParser.new('%d/%b/%Y:%H:%M:%S %z')\n\n    assert(parser.parse('28/Feb/2013:12:00:00 +0900').is_a?(Fluent::EventTime))\n\n    time = event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z')\n    assert_equal(time, parser.parse('28/Feb/2013:12:00:00 +0900'))\n  end\n\n  def test_parse_nsec_with_strptime\n    parser = Fluent::TimeParser.new('%d/%b/%Y:%H:%M:%S:%N %z')\n\n    assert(parser.parse('28/Feb/2013:12:00:00:123456789 +0900').is_a?(Fluent::EventTime))\n\n    time = event_time('28/Feb/2013:12:00:00:123456789 +0900', format: '%d/%b/%Y:%H:%M:%S:%N %z')\n    assert_equal_event_time(time, parser.parse('28/Feb/2013:12:00:00:123456789 +0900'))\n  end\n\n  def test_parse_iso8601\n    parser = Fluent::TimeParser.new('%iso8601')\n\n    assert(parser.parse('2017-01-01T12:00:00+09:00').is_a?(Fluent::EventTime))\n\n    time = event_time('2017-01-01T12:00:00+09:00')\n    assert_equal(time, parser.parse('2017-01-01T12:00:00+09:00'))\n\n    time_with_msec = event_time('2017-01-01T12:00:00.123+09:00')\n    assert_equal(time_with_msec, parser.parse('2017-01-01T12:00:00.123+09:00'))\n  end\n\n  def test_parse_with_invalid_argument\n    parser = Fluent::TimeParser.new\n\n    [[], {}, nil, true, 10000, //, ->{}, '', :symbol].each { |v|\n      assert_raise Fluent::TimeParser::TimeParseError do\n        parser.parse(v)\n      end\n    }\n  end\n\n  def test_parse_time_in_localtime\n    time = with_timezone(\"UTC+02\") do\n      parser = Fluent::TimeParser.new(\"%Y-%m-%d %H:%M:%S.%N\", true)\n      parser.parse(\"2016-09-02 18:42:31.123456789\")\n    end\n    assert_equal_event_time(time, event_time(\"2016-09-02 18:42:31.123456789 -02:00\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n  end\n\n  def test_parse_time_in_utc\n    time = with_timezone(\"UTC-09\") do\n      parser = Fluent::TimeParser.new(\"%Y-%m-%d %H:%M:%S.%N\", false)\n      parser.parse(\"2016-09-02 18:42:31.123456789\")\n    end\n    assert_equal_event_time(time, event_time(\"2016-09-02 18:42:31.123456789 UTC\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n  end\n\n  def test_parse_string_with_expected_timezone\n    time = with_timezone(\"UTC-09\") do\n      parser = Fluent::TimeParser.new(\"%Y-%m-%d %H:%M:%S.%N\", nil, \"-07:00\")\n      parser.parse(\"2016-09-02 18:42:31.123456789\")\n    end\n    assert_equal_event_time(time, event_time(\"2016-09-02 18:42:31.123456789 -07:00\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n  end\n\n  def test_parse_time_with_expected_timezone_name\n    time = with_timezone(\"UTC-09\") do\n      parser = Fluent::TimeParser.new(\"%Y-%m-%d %H:%M:%S.%N\", nil, \"Europe/Zurich\")\n      parser.parse(\"2016-12-02 18:42:31.123456789\")\n    end\n    assert_equal_event_time(time, event_time(\"2016-12-02 18:42:31.123456789 +01:00\", format: '%Y-%m-%d %H:%M:%S.%N %z'))\n  end\n\n  sub_test_case 'TimeMixin::Parser' do\n    class DummyForTimeParser\n      include Fluent::Configurable\n      include Fluent::TimeMixin::Parser\n    end\n\n    test 'provides configuration parameters for TimeParser with default values for localtime' do\n      time = with_timezone(\"UTC+07\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse'))\n\n        assert_nil   i.time_format\n        assert_true  i.localtime\n        assert_false i.utc\n        assert_nil   i.timezone\n\n        parser = i.time_parser_create\n        # time_format unspecified\n        # localtime\n        parser.parse(\"2016-09-02 18:42:31.012345678\")\n      end\n      assert_equal_event_time(event_time(\"2016-09-02 18:42:31.012345678 -07:00\", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)\n    end\n\n    test 'provides configuration parameters for TimeParser, configurable for any time format' do\n      time = with_timezone(\"UTC+07\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S %N'}))\n        parser = i.time_parser_create\n        # time_format specified\n        # localtime\n        parser.parse(\"09/02/2016 18-42-31 012345678\")\n      end\n      assert_equal_event_time(event_time(\"2016-09-02 18:42:31.012345678 -07:00\", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)\n    end\n\n    test 'provides configuration parameters for TimeParser, configurable for UTC by localtime=false' do\n      time = with_timezone(\"UTC+07\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S %N', 'localtime' => 'false'}))\n        parser = i.time_parser_create\n        # time_format specified\n        # utc\n        parser.parse(\"09/02/2016 18-42-31 012345678\")\n      end\n      assert_equal_event_time(event_time(\"2016-09-02 18:42:31.012345678 UTC\", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)\n    end\n\n    test 'provides configuration parameters for TimeParser, configurable for UTC by utc=true' do\n      time = with_timezone(\"UTC+07\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S %N', 'utc' => 'true'}))\n        parser = i.time_parser_create\n        # time_format specified\n        # utc\n        parser.parse(\"09/02/2016 18-42-31 012345678\")\n      end\n      assert_equal_event_time(event_time(\"2016-09-02 18:42:31.012345678 UTC\", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)\n    end\n\n    test 'provides configuration parameters for TimeParser, configurable for any timezone' do\n      time = with_timezone(\"UTC+07\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S %N', 'timezone' => '-01:00'}))\n        parser = i.time_parser_create\n        # time_format specified\n        # -01:00\n        parser.parse(\"09/02/2016 18-42-31 012345678\")\n      end\n      assert_equal_event_time(event_time(\"2016-09-02 18:42:31.012345678 -01:00\", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)\n    end\n\n    test 'specifying timezone without time format raises configuration error' do\n      assert_raise Fluent::ConfigError.new(\"specifying timezone requires time format\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse', '', {'utc' => 'true'}))\n        i.time_parser_create\n      end\n      assert_raise Fluent::ConfigError.new(\"specifying timezone requires time format\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse', '', {'localtime' => 'false'}))\n        i.time_parser_create\n      end\n      assert_raise Fluent::ConfigError.new(\"specifying timezone requires time format\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse', '', {'timezone' => '-0700'}))\n        i.time_parser_create\n      end\n    end\n\n    test '#time_parser_create returns TimeParser with specified time format and timezone' do\n      time = with_timezone(\"UTC-09\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S %N'}))\n        assert_equal '%m/%d/%Y %H-%M-%S %N', i.time_format\n        assert_true i.localtime\n        parser = i.time_parser_create(format: '%Y-%m-%d %H:%M:%S.%N %z')\n        parser.parse(\"2016-09-05 17:59:38.987654321 -03:00\")\n      end\n      assert_equal_event_time(event_time(\"2016-09-05 17:59:38.987654321 -03:00\", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)\n    end\n\n    test '#time_parser_create returns TimeParser with localtime when specified it forcedly besides any configuration parameters' do\n      time = with_timezone(\"UTC-09\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S', 'utc' => 'true'}))\n        assert_equal '%m/%d/%Y %H-%M-%S', i.time_format\n        assert_true i.utc\n        parser = i.time_parser_create(format: '%Y-%m-%d %H:%M:%S.%N', force_localtime: true)\n        parser.parse(\"2016-09-05 17:59:38.987654321\")\n      end\n      assert_equal_event_time(event_time(\"2016-09-05 17:59:38.987654321 +09:00\", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)\n\n      time = with_timezone(\"UTC-09\") do\n        i = DummyForTimeParser.new\n        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S', 'timezone' => '+0000'}))\n        assert_equal '%m/%d/%Y %H-%M-%S', i.time_format\n        assert_equal '+0000', i.timezone\n        parser = i.time_parser_create(format: '%Y-%m-%d %H:%M:%S.%N', force_localtime: true)\n        parser.parse(\"2016-09-05 17:59:38.987654321\")\n      end\n      assert_equal_event_time(event_time(\"2016-09-05 17:59:38.987654321 +09:00\", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)\n    end\n\n    test '#time_parser_create returns NumericTimeParser to parse time as unixtime when time_type unixtime specified' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '', {'time_type' => 'unixtime'}))\n      parser = i.time_parser_create\n      time = event_time(\"2016-10-03 20:08:30.123456789 +0100\", format: '%Y-%m-%d %H:%M:%S.%N %z')\n      assert_equal_event_time(Fluent::EventTime.new(time.to_i), parser.parse(\"#{time.sec}\"))\n    end\n\n    test '#time_parser_create returns NumericTimeParser to parse time as float when time_type float specified' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '', {'time_type' => 'float'}))\n      parser = i.time_parser_create\n      time = event_time(\"2016-10-03 20:08:30.123456789 +0100\", format: '%Y-%m-%d %H:%M:%S.%N %z')\n      assert_equal_event_time(time, parser.parse(\"#{time.sec}.#{time.nsec}\"))\n    end\n  end\n\n  sub_test_case 'MixedTimeParser fallback' do\n    class DummyForTimeParser\n      include Fluent::Configurable\n      include Fluent::TimeMixin::Parser\n    end\n\n    test 'no time_format_fallbacks failure' do\n      i = DummyForTimeParser.new\n      assert_raise(Fluent::ConfigError.new(\"time_type is :mixed but time_format and time_format_fallbacks is empty.\")) do\n        i.configure(config_element('parse', '', {'time_type' => 'mixed'}))\n      end\n    end\n\n    test 'fallback time format failure' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '',\n                                 {'time_type' => 'mixed',\n                                  'time_format_fallbacks' => ['%iso8601']}))\n      parser = i.time_parser_create\n      assert_raise(Fluent::TimeParser::TimeParseError.new(\"invalid time format: value = INVALID, even though fallbacks: Fluent::TimeParser\")) do\n        parser.parse(\"INVALID\")\n      end\n    end\n\n    test 'primary format is unixtime, secondary %iso8601 is used' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '',\n                                 {'time_type' => 'mixed',\n                                  'time_format' => 'unixtime',\n                                  'time_format_fallbacks' => ['%iso8601']}))\n      parser = i.time_parser_create\n      time = event_time('2021-01-01T12:00:00+0900')\n      assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))\n    end\n\n    test 'primary format is %iso8601, secondary unixtime is used' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '',\n                                 {'time_type' => 'mixed',\n                                  'time_format' => '%iso8601',\n                                  'time_format_fallbacks' => ['unixtime']}))\n      parser = i.time_parser_create\n      time = event_time('2021-01-01T12:00:00+0900')\n      assert_equal_event_time(time, parser.parse(\"#{time.sec}\"))\n    end\n\n    test 'primary format is %iso8601, no secondary is used' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '',\n                                 {'time_type' => 'mixed',\n                                  'time_format' => '%iso8601'}))\n      parser = i.time_parser_create\n      time = event_time('2021-01-01T12:00:00+0900')\n      assert_equal_event_time(time, parser.parse(\"2021-01-01T12:00:00+0900\"))\n    end\n\n    test 'primary format is unixtime, no secondary is used' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '',\n                                 {'time_type' => 'mixed',\n                                  'time_format' => 'unixtime'}))\n      parser = i.time_parser_create\n      time = event_time('2021-01-01T12:00:00+0900')\n      assert_equal_event_time(time, parser.parse(\"#{time.sec}\"))\n    end\n\n    test 'primary format is %iso8601, raise error because of no appropriate secondary' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '',\n                                 {'time_type' => 'mixed',\n                                  'time_format' => '%iso8601'}))\n      parser = i.time_parser_create\n      time = event_time('2021-01-01T12:00:00+0900')\n      assert_raise(\"Fluent::TimeParser::TimeParseError: invalid time format: value = #{time.sec}, even though fallbacks: Fluent::TimeParser\") do\n        parser.parse(\"#{time.sec}\")\n      end\n    end\n\n    test 'primary format is unixtime, raise error because of no appropriate secondary' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '',\n                                 {'time_type' => 'mixed',\n                                  'time_format' => 'unixtime'}))\n      parser = i.time_parser_create\n      time = event_time('2021-01-01T12:00:00+0900')\n      assert_raise(\"Fluent::TimeParser::TimeParseError: invalid time format: value = #{time}, even though fallbacks: Fluent::NumericTimeParser\") do\n        parser.parse(\"2021-01-01T12:00:00+0900\")\n      end\n    end\n\n    test 'fallback to unixtime' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '', {'time_type' => 'mixed',\n                                               'time_format_fallbacks' => ['%iso8601', 'unixtime']}))\n      parser = i.time_parser_create\n      time = event_time('2021-01-01T12:00:00+0900')\n      assert_equal_event_time(Fluent::EventTime.new(time.to_i), parser.parse(\"#{time.sec}\"))\n    end\n\n    test 'fallback to %iso8601' do\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '', {'time_type' => 'mixed',\n                                               'time_format_fallbacks' => ['unixtime', '%iso8601']}))\n      parser = i.time_parser_create\n      time = event_time('2021-01-01T12:00:00+0900')\n      assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))\n    end\n  end\n\n  # https://github.com/fluent/fluentd/issues/3195\n  test 'change timezone without zone specifier in a format' do\n    expected = 1607457600 # 2020-12-08T20:00:00Z\n    time1 = time2 = nil\n\n    with_timezone(\"UTC-05\") do # EST\n      i = DummyForTimeParser.new\n      i.configure(config_element('parse', '', {'time_type' => 'string',\n                                               'time_format' => '%Y-%m-%dT%H:%M:%SZ',\n                                               'utc' => true}))\n      parser = i.time_parser_create\n\n      time1 = parser.parse('2020-12-08T20:00:00Z').to_i\n      time2 = with_timezone(\"UTC-04\") do # EDT\n        # to avoid using cache, increment 1 sec\n        parser.parse('2020-12-08T20:00:01Z').to_i\n      end\n    end\n\n    assert_equal([expected, expected + 1], [time1, time2])\n  end\nend\n"
  },
  {
    "path": "test/test_tls.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/tls'\n\nclass UniqueIdTest < Test::Unit::TestCase\n  TEST_TLS1_1_CASES = {\n    'New TLS v1.1' => :'TLS1_1',\n    'Old TLS v1.1' => :'TLSv1_1',\n  }\n  TEST_TLS1_2_CASES = {\n    'New TLS v1.2' => :'TLS1_2',\n    'Old TLS v1.2' => :'TLSv1_2'\n  }\n  TEST_TLS1_3_CASES = {\n    'New TLS v1.3' => :'TLS1_3',\n    'Old TLS v1.3' => :'TLSv1_3'\n  } if defined?(OpenSSL::SSL::TLS1_3_VERSION)\n  TEST_TLS_CASES = TEST_TLS1_1_CASES.merge(TEST_TLS1_2_CASES)\n\n  sub_test_case 'constants' do\n    test 'default version' do\n      assert_equal :'TLSv1_2', Fluent::TLS::DEFAULT_VERSION\n    end\n\n    data(TEST_TLS_CASES)\n    test 'supported versions' do |ver|\n      assert_include Fluent::TLS::SUPPORTED_VERSIONS, ver\n    end\n\n    test 'default ciphers' do\n      assert_equal \"ALL:!aNULL:!eNULL:!SSLv2\", Fluent::TLS::CIPHERS_DEFAULT\n    end\n  end\n\n  sub_test_case 'set_version_to_context' do\n    setup do\n      @ctx = OpenSSL::SSL::SSLContext.new\n    end\n\n    # TODO: After openssl module supports min_version/max_version accessor, add assert for it.\n\n    data(TEST_TLS_CASES)\n    test 'with version' do |ver|\n      assert_nothing_raised {\n        Fluent::TLS.set_version_to_context(@ctx, ver, nil, nil)\n      }\n    end\n\n    data(TEST_TLS_CASES)\n    test 'can specify old/new syntax to min_version/max_version' do |ver|\n      omit \"min_version=/max_version= is not supported\" unless Fluent::TLS::MIN_MAX_AVAILABLE\n\n      assert_nothing_raised {\n        Fluent::TLS.set_version_to_context(@ctx, Fluent::TLS::DEFAULT_VERSION, ver, ver)\n      }\n    end\n\n    test 'raise ConfigError when either one of min_version/max_version is not specified' do\n      omit \"min_version=/max_version= is not supported\" unless Fluent::TLS::MIN_MAX_AVAILABLE\n\n      ver = Fluent::TLS::DEFAULT_VERSION\n      assert_raise(Fluent::ConfigError) {\n        Fluent::TLS.set_version_to_context(@ctx, ver, ver, nil)\n      }\n      assert_raise(Fluent::ConfigError) {\n        Fluent::TLS.set_version_to_context(@ctx, ver, nil, ver)\n      }\n    end\n  end\n\n  sub_test_case 'set_version_to_options' do\n    setup do\n      @opt = {}\n    end\n\n    test 'set min_version/max_version when supported' do\n      omit \"min_version=/max_version= is not supported\" unless Fluent::TLS::MIN_MAX_AVAILABLE\n\n      ver = Fluent::TLS::DEFAULT_VERSION\n      assert_raise(Fluent::ConfigError) {\n        Fluent::TLS.set_version_to_options(@opt, ver, ver, nil)\n      }\n      assert_raise(Fluent::ConfigError) {\n        Fluent::TLS.set_version_to_options(@opt, ver, nil, ver)\n      }\n\n      ver = :'TLSv1_3' if defined?(OpenSSL::SSL::TLS1_3_VERSION)\n      assert_equal Fluent::TLS.const_get(:METHODS_MAP)[ver], Fluent::TLS.set_version_to_options(@opt, ver, nil, nil)[:min_version]\n      assert_equal Fluent::TLS.const_get(:METHODS_MAP)[ver], Fluent::TLS.set_version_to_options(@opt, ver, nil, nil)[:max_version]\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_unique_id.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/plugin/base'\nrequire 'fluent/unique_id'\n\nmodule UniqueIdTestEnv\n  class Dummy < Fluent::Plugin::Base\n    include Fluent::UniqueId::Mixin\n  end\nend\n\nclass UniqueIdTest < Test::Unit::TestCase\n  sub_test_case 'module used directly' do\n    test '.generate generates 128bit length unique id (16bytes)' do\n      assert_equal 16, Fluent::UniqueId.generate.bytesize\n      ary = []\n      100_000.times do\n        ary << Fluent::UniqueId.generate\n      end\n      assert_equal 100_000, ary.uniq.size\n    end\n\n    test '.hex dumps 16bytes id into 32 chars' do\n      assert_equal 32, Fluent::UniqueId.hex(Fluent::UniqueId.generate).size\n      assert(Fluent::UniqueId.hex(Fluent::UniqueId.generate) =~ /^[0-9a-z]{32}$/)\n    end\n  end\n\n  sub_test_case 'mixin' do\n    setup do\n      @i = UniqueIdTestEnv::Dummy.new\n    end\n\n    test '#generate_unique_id generates 128bit length id (16bytes)' do\n      assert_equal 16, @i.generate_unique_id.bytesize\n      ary = []\n      100_000.times do\n        ary << @i.generate_unique_id\n      end\n      assert_equal 100_000, ary.uniq.size\n    end\n\n    test '#dump_unique_id_hex dumps 16bytes id into 32 chars' do\n      assert_equal 32, @i.dump_unique_id_hex(@i.generate_unique_id).size\n      assert(@i.dump_unique_id_hex(@i.generate_unique_id) =~ /^[0-9a-z]{32}$/)\n    end\n  end\nend\n"
  },
  {
    "path": "test/test_variable_store.rb",
    "content": "require_relative 'helper'\nrequire 'fluent/variable_store'\n\nclass VariableStoreTest < Test::Unit::TestCase\n  def setup\n  end\n\n  def teardown\n    Fluent::VariableStore.try_to_reset do\n      # nothing\n    end\n  end\n\n  sub_test_case '#fetch_or_build' do\n    test 'fetch same object when the same key is passed' do\n      c1 = Fluent::VariableStore.fetch_or_build(:test)\n      c2 = Fluent::VariableStore.fetch_or_build(:test)\n\n      assert_equal c1, c2\n      assert_equal c1.object_id, c2.object_id\n\n      c3 = Fluent::VariableStore.fetch_or_build(:test2)\n      assert_not_equal c1.object_id, c3.object_id\n    end\n\n    test 'can be passed a default value' do\n      c1 = Fluent::VariableStore.fetch_or_build(:test, default_value: Set.new)\n      c2 = Fluent::VariableStore.fetch_or_build(:test)\n\n      assert_kind_of Set, c1\n      assert_equal c1, c2\n      assert_equal c1.object_id, c2.object_id\n    end\n  end\n\n  sub_test_case '#try_to_reset' do\n    test 'reset all values' do\n      c1 = Fluent::VariableStore.fetch_or_build(:test)\n      c1[:k1] = 1\n      assert_equal 1, c1[:k1]\n\n      Fluent::VariableStore.try_to_reset do\n        # nothing\n      end\n\n      c1 = Fluent::VariableStore.fetch_or_build(:test)\n      assert_nil c1[:k1]\n    end\n\n    test 'rollback resetting if error raised' do\n      c1 = Fluent::VariableStore.fetch_or_build(:test)\n      c1[:k1] = 1\n      assert_equal 1, c1[:k1]\n\n      assert_raise(RuntimeError.new('pass')) do\n        Fluent::VariableStore.try_to_reset do\n          raise 'pass'\n        end\n      end\n\n      c1 = Fluent::VariableStore.fetch_or_build(:test)\n      assert_equal 1, c1[:k1]\n    end\n  end\nend\n"
  }
]