[ { "path": ".bazelrc", "content": "# Copyright 2022 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# Enable layering check features. Useful on Clang only.\nbuild --features=layering_check\n# Enable parse headers features. Enforcing that headers are self-contained.\nbuild --features=parse_headers\n\n# Abseil requires C++17 at minimum.\nbuild --enable_platform_specific_config\nbuild:linux --cxxopt=-std=c++17\nbuild:macos --cxxopt=-std=c++17\nbuild:windows --cxxopt=/std:c++17\n\n# Print test logs for failed tests.\ntest --test_output=errors\n\n# https://bazel.build/configure/best-practices#bazelrc-file\ntry-import %workspace%/user.bazelrc\n" }, { "path": ".bcr/metadata.template.json", "content": "{\n \"homepage\": \"https://github.com/google/re2\",\n \"maintainers\": [\n {\n \"email\": \"rsc@google.com\",\n \"github\": \"rsc\",\n \"github_user_id\": 104030,\n \"name\": \"Russ Cox\"\n }\n ],\n \"repository\": [\n \"github:google/re2\"\n ],\n \"versions\": [],\n \"yanked_versions\": {}\n}\n" }, { "path": ".bcr/presubmit.yml", "content": "matrix:\n platform:\n - rockylinux8\n - debian10\n - ubuntu2004\n - macos\n bazel:\n - 7.x\n - 8.x\n\ntasks:\n unix_build:\n platform: ${{ platform }}\n bazel: ${{ bazel }}\n build_flags:\n - '--cxxopt=-std=c++17'\n build_targets:\n - '@re2//:re2'\n - '@re2//python:re2'\n windows_build:\n platform: windows\n bazel: ${{ bazel }}\n build_flags:\n - '--cxxopt=/std:c++17'\n build_targets:\n - '@re2//:re2'\n - '@re2//python:re2'\n\nbcr_test_module:\n module_path: '.'\n matrix:\n platform:\n - rockylinux8\n - debian10\n - ubuntu2004\n - macos\n - windows\n bazel:\n - 7.x\n - 8.x\n tasks:\n unix_test:\n platform: ${{ platform }}\n bazel: ${{ bazel }}\n test_flags:\n - '--cxxopt=-std=c++17'\n test_targets:\n - '//:small_tests'\n - '//python:all'\n windows_test:\n platform: windows\n bazel: ${{ bazel }}\n test_flags:\n - '--cxxopt=/std:c++17'\n test_targets:\n - '//:small_tests'\n - '//python:all'\n" }, { "path": ".bcr/source.template.json", "content": "{\n \"integrity\": \"\",\n \"strip_prefix\": \"{REPO}-{VERSION}\",\n \"url\": \"https://github.com/{OWNER}/{REPO}/releases/download/{TAG}/{REPO}-{TAG}.zip\"\n}\n" }, { "path": ".github/bazel.sh", "content": "#!/bin/bash\nset -eux\n\n# Disable MSYS/MSYS2 path conversion, which interferes with Bazel.\nexport MSYS_NO_PATHCONV='1'\nexport MSYS2_ARG_CONV_EXCL='*'\n\nfor compilation_mode in dbg opt\ndo\n bazel clean\n bazel build \\\n --extra_toolchains=//python/toolchains:all \\\n --compilation_mode=${compilation_mode} -- \\\n //:re2 \\\n //python:re2\n bazel test \\\n --extra_toolchains=//python/toolchains:all \\\n --compilation_mode=${compilation_mode} -- \\\n //:small_tests \\\n //python:all\ndone\n\nexit 0\n" }, { "path": ".github/cmake.sh", "content": "#!/bin/bash\nset -eux\n\nfor CMAKE_BUILD_TYPE in Debug Release\ndo\n cmake . -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -D RE2_BUILD_TESTING=ON \"$@\"\n cmake --build . --config ${CMAKE_BUILD_TYPE} --clean-first\n ctest -C ${CMAKE_BUILD_TYPE} --output-on-failure -E 'dfa|exhaustive|random'\ndone\n\nexit 0\n" }, { "path": ".github/workflows/ci-bazel.yml", "content": "name: CI (Bazel)\non:\n pull_request:\n branches: [main]\n push:\n branches: [main, rsc-testing]\npermissions:\n contents: read\njobs:\n build:\n runs-on: ${{ matrix.os }}\n strategy:\n fail-fast: false\n matrix:\n os: [macos-latest, ubuntu-latest, windows-latest]\n # Keep in sync with python.yml.\n ver: ['3.10', '3.11', '3.12', '3.13', '3.14']\n steps:\n - uses: actions/checkout@v4.2.2\n - uses: bazel-contrib/setup-bazel@0.15.0\n with:\n bazelisk-version: '1.x'\n - uses: actions/setup-python@v5.6.0\n with:\n python-version: ${{ matrix.ver }}\n - name: Prepare Python ${{ matrix.ver }} environment\n run: |\n python -m pip install --upgrade pip\n python -m pip install --upgrade absl-py mypy\n python python/toolchains/generate.py\n shell: bash\n - run: .github/bazel.sh\n shell: bash\n # TODO(junyer): Run mypy as per https://github.com/google/re2/issues/496.\n" }, { "path": ".github/workflows/ci-cmake.yml", "content": "name: CI (CMake)\non:\n pull_request:\n branches: [main]\n push:\n branches: [main, rsc-testing]\npermissions:\n contents: read\njobs:\n build-linux:\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n matrix:\n build_shared_libs: [OFF, ON]\n steps:\n - uses: actions/checkout@v4.2.2\n - name: Install Abseil, GoogleTest and Benchmark\n run: |\n vcpkg update\n vcpkg install abseil gtest benchmark\n shell: bash\n - run: |\n .github/cmake.sh -D BUILD_SHARED_LIBS=${{ matrix.build_shared_libs }} \\\n -D CMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake\n shell: bash\n build-macos:\n runs-on: macos-latest\n strategy:\n fail-fast: false\n matrix:\n build_shared_libs: [OFF, ON]\n steps:\n - uses: actions/checkout@v4.2.2\n - name: Install Abseil, GoogleTest and Benchmark\n run: |\n brew update\n brew install abseil googletest google-benchmark\n shell: bash\n - run: .github/cmake.sh -D BUILD_SHARED_LIBS=${{ matrix.build_shared_libs }}\n shell: bash\n build-windows:\n runs-on: windows-latest\n strategy:\n fail-fast: false\n matrix:\n build_shared_libs: [OFF, ON]\n steps:\n - uses: actions/checkout@v4.2.2\n - name: Install Abseil, GoogleTest and Benchmark\n run: |\n vcpkg update\n vcpkg install abseil gtest benchmark\n shell: bash\n - run: |\n .github/cmake.sh -D BUILD_SHARED_LIBS=${{ matrix.build_shared_libs }} \\\n -D CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake\n shell: bash\n" }, { "path": ".github/workflows/ci.yml", "content": "name: CI\non:\n pull_request:\n branches: [main]\n push:\n branches: [main, rsc-testing]\npermissions:\n contents: read\njobs:\n build-appleclang:\n runs-on: macos-latest\n strategy:\n fail-fast: false\n matrix:\n ver: [17, 20]\n env:\n CC: clang\n CXX: clang++\n # Unlike GCC and upstream Clang, AppleClang still defaults to `-std=c++98`\n # for some reason. Also, the macOS image on GitHub Actions provides wildly\n # numbered Xcode versions. Thus, rather than varying the compiler version,\n # we set the `-std` flag explicitly in order to vary the language version.\n # (The other two flags are the default provided for CXXFLAGS in Makefile.)\n CXXFLAGS: -O3 -g -std=c++${{ matrix.ver }}\n steps:\n - uses: actions/checkout@v4.2.2\n - name: Install Abseil, GoogleTest and Benchmark\n run: |\n brew update\n brew install abseil googletest google-benchmark\n shell: bash\n - run: make && make test\n shell: bash\n build-clang:\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n matrix:\n ver: [18, 19, 20]\n env:\n CC: clang-${{ matrix.ver }}\n CXX: clang++-${{ matrix.ver }}\n PKG_CONFIG_PATH: /usr/local/share/vcpkg/installed/x64-linux/lib/pkgconfig\n steps:\n - uses: actions/checkout@v4.2.2\n - name: Install Clang ${{ matrix.ver }}\n run: |\n # Avoid `Conflicts: python3-lldb-x.y` between packages.\n sudo apt purge -y python3-lldb-14\n wget https://apt.llvm.org/llvm.sh\n chmod +x ./llvm.sh\n sudo ./llvm.sh ${{ matrix.ver }}\n shell: bash\n - name: Install Abseil, GoogleTest and Benchmark\n run: |\n vcpkg update\n vcpkg install abseil gtest benchmark\n shell: bash\n - run: make && make test\n shell: bash\n build-gcc:\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n matrix:\n ver: [12, 13, 14]\n env:\n CC: gcc-${{ matrix.ver }}\n CXX: g++-${{ matrix.ver }}\n PKG_CONFIG_PATH: /usr/local/share/vcpkg/installed/x64-linux/lib/pkgconfig\n steps:\n - uses: actions/checkout@v4.1.7\n - name: Install Abseil, GoogleTest and Benchmark\n run: |\n vcpkg update\n vcpkg install abseil gtest benchmark\n shell: bash\n - run: make && make test\n shell: bash\n" }, { "path": ".github/workflows/pages.yml", "content": "name: Pages\non:\n workflow_dispatch:\npermissions:\n contents: read\njobs:\n build:\n runs-on: ubuntu-latest\n container:\n image: emscripten/emsdk\n # Don't run as root within the container.\n # Neither Git nor Bazel appreciates that.\n # 1001 is the GitHub Actions runner user.\n options: --init --user 1001\n env:\n BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n # Bazel fails if the username is unknown.\n USER: runner\n steps:\n - uses: actions/checkout@v4.2.2\n - uses: bazel-contrib/setup-bazel@0.15.0\n with:\n bazelisk-version: '1.x'\n - run: app/build.sh\n shell: bash\n - uses: actions/upload-pages-artifact@v3.0.1\n with:\n path: app/deploy\n deploy:\n needs:\n - build\n permissions:\n contents: read\n # Needed for Pages deployment.\n id-token: write\n pages: write\n environment: github-pages\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4.2.2\n - uses: actions/deploy-pages@v4.0.5\n" }, { "path": ".github/workflows/python.yml", "content": "name: Python\non:\n workflow_dispatch:\n inputs:\n build:\n required: true\n type: number\n force-sdist:\n required: false\n type: boolean\n default: false\npermissions:\n contents: read\njobs:\n wheel-linux:\n name: Linux ${{ matrix.os }}, ${{ matrix.arch.name }}, Python ${{ matrix.ver }}\n runs-on: ${{ matrix.arch.runs-on }}\n container:\n image: quay.io/pypa/${{ matrix.os }}_${{ matrix.arch.python-name }}\n # Don't run as root within the container.\n # Neither Git nor Bazel appreciates that.\n # 1001 is the GitHub Actions runner user.\n options: --init --user 1001\n strategy:\n fail-fast: false\n matrix:\n arch:\n - { name: X64, python-name: x86_64, runs-on: [ubuntu-latest] }\n - { name: ARM64, python-name: aarch64, runs-on: [ubuntu-24.04-arm] }\n os: [manylinux_2_28]\n # Keep in sync with ci-bazel.yml and list below.\n # Also, when bumping the minimum version, update ../../python/setup.py.\n ver: ['3.10', '3.11', '3.12', '3.13', '3.14']\n env:\n BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n PYTHON: /usr/local/bin/python${{ matrix.ver }}\n # Bazel fails if the username is unknown.\n USER: runner\n steps:\n - uses: actions/checkout@v4.2.2\n # Stash the timestamp for the commit SHA that triggered the workflow.\n - run: echo \"timestamp=$(git log -1 --pretty=%ct)\" >> \"${GITHUB_ENV}\"\n shell: bash\n - uses: bazel-contrib/setup-bazel@0.15.0\n with:\n bazelisk-version: '1.x'\n - name: Prepare Python ${{ matrix.ver }} environment\n run: |\n \"${PYTHON}\" -m pip install --upgrade pip\n \"${PYTHON}\" -m pip install --upgrade setuptools build wheel auditwheel\n \"${PYTHON}\" -m pip install --upgrade absl-py mypy\n \"${PYTHON}\" python/toolchains/generate.py\n shell: bash\n - name: Build wheel\n env:\n SOURCE_DATE_EPOCH: ${{ env.timestamp }}\n run: |\n \"${PYTHON}\" -m build --wheel\n \"${PYTHON}\" -m auditwheel repair --wheel-dir=. dist/*\n shell: bash\n working-directory: python\n - name: Test wheel\n run: |\n \"${PYTHON}\" -m pip install google_re2-*.whl\n # Pivot out of the repository so that we test the wheel.\n DIR=$(mktemp -d)\n cp re2_test.py \"${DIR}\"\n cd \"${DIR}\"\n \"${PYTHON}\" re2_test.py\n shell: bash\n working-directory: python\n - uses: actions/upload-artifact@v4.6.2\n with:\n name: ${{ hashFiles('python/google_re2-*.whl') }}\n path: python/google_re2-*.whl\n retention-days: 1\n wheel-macos:\n name: macOS ${{ matrix.os }}, ${{ matrix.arch.name }}, Python ${{ matrix.ver }}\n runs-on: macos-${{ matrix.os }}-large\n strategy:\n fail-fast: false\n matrix:\n arch:\n - { name: X64, bazel-name: x86_64, python-name: x86_64 }\n - { name: ARM64, bazel-name: arm64, python-name: arm64 }\n # TODO(rsc): Stop cross-compiling now that we don't use macOS 12.\n # instead, specify `-large` suffix on X64 and `-xlarge` suffix on ARM64.\n os: [13, 14, 15, 26]\n ver: ['3.10', '3.11', '3.12', '3.13', '3.14']\n env:\n BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n BAZEL_CPU: darwin_${{ matrix.arch.bazel-name }}\n PLAT_NAME: macosx-${{ matrix.os }}.0-${{ matrix.arch.python-name }}\n # Force a specific target version of macOS.\n # Otherwise, `delocate` renames the wheels!\n MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os }}.0\n # Stop macOS from reporting the system version as 10.x.\n # Otherwise, Python refuses to install the built wheel!\n SYSTEM_VERSION_COMPAT: 0\n steps:\n - uses: actions/checkout@v4.1.7\n # Stash the timestamp for the commit SHA that triggered the workflow.\n - run: echo \"timestamp=$(git log -1 --pretty=%ct)\" >> \"${GITHUB_ENV}\"\n shell: bash\n - uses: bazel-contrib/setup-bazel@0.15.0\n with:\n bazelisk-version: '1.x'\n - uses: actions/setup-python@v5.6.0\n with:\n python-version: ${{ matrix.ver }}\n - name: Prepare Python ${{ matrix.ver }} environment\n run: |\n python -m pip install --upgrade pip\n python -m pip install --upgrade setuptools build wheel delocate\n python -m pip install --upgrade absl-py mypy\n python python/toolchains/generate.py\n shell: bash\n - name: Build wheel\n env:\n SOURCE_DATE_EPOCH: ${{ env.timestamp }}\n run: |\n python -m build --wheel\n python -m delocate.cmd.delocate_wheel --wheel-dir=. dist/*\n shell: bash\n working-directory: python\n - if: matrix.arch.name == runner.arch\n name: Test wheel\n run: |\n python -m pip install google_re2-*.whl\n # Pivot out of the repository so that we test the wheel.\n DIR=$(mktemp -d)\n cp re2_test.py \"${DIR}\"\n cd \"${DIR}\"\n python re2_test.py\n shell: bash\n working-directory: python\n - uses: actions/upload-artifact@v4.6.2\n with:\n name: ${{ hashFiles('python/google_re2-*.whl') }}\n path: python/google_re2-*.whl\n retention-days: 1\n wheel-windows:\n name: Windows, ${{ matrix.arch.name }}, Python ${{ matrix.ver }}\n runs-on: ${{ matrix.arch.name == 'ARM64' && 'windows-11-arm' || 'windows-latest' }}\n strategy:\n fail-fast: false\n matrix:\n arch:\n - { name: X86, bazel-name: x64_x86, python-name: win32 }\n - { name: X64, bazel-name: x64, python-name: win_amd64 }\n - { name: ARM64, bazel-name: arm64, python-name: win_arm64 }\n ver: ['3.10', '3.11', '3.12', '3.13', '3.14']\n exclude:\n - arch: { name: ARM64, bazel-name: arm64, python-name: win_arm64 }\n ver: '3.9'\n - arch: { name: ARM64, bazel-name: arm64, python-name: win_arm64 }\n ver: '3.10'\n env:\n BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n BAZEL_CPU: ${{ matrix.arch.bazel-name }}_windows\n PLAT_NAME: ${{ matrix.arch.python-name }}\n steps:\n - uses: actions/checkout@v4.2.2\n # Stash the timestamp for the commit SHA that triggered the workflow.\n - run: echo \"timestamp=$(git log -1 --pretty=%ct)\" >> \"${GITHUB_ENV}\"\n shell: bash\n - uses: bazel-contrib/setup-bazel@0.15.0\n with:\n bazelisk-version: '1.x'\n # Lowercase the architecture name for `actions/setup-python`.\n - run: |\n ARCHITECTURE=${{ matrix.arch.name }}\n echo \"architecture=${ARCHITECTURE,,}\" >> \"${GITHUB_ENV}\"\n shell: bash\n - uses: actions/setup-python@v5.6.0\n with:\n python-version: ${{ matrix.ver }}\n architecture: ${{ env.architecture }}\n - name: Prepare Python ${{ matrix.ver }} environment\n run: |\n python -m pip install --upgrade pip\n python -m pip install --upgrade setuptools build wheel delvewheel\n python -m pip install --upgrade absl-py mypy\n python python/toolchains/generate.py\n shell: bash\n - name: Build wheel\n env:\n SOURCE_DATE_EPOCH: ${{ env.timestamp }}\n run: |\n python -m build --wheel\n python -m delvewheel repair --wheel-dir=. dist/*\n shell: bash\n working-directory: python\n - name: Test wheel\n run: |\n python -m pip install google_re2-*.whl\n # Pivot out of the repository so that we test the wheel.\n DIR=$(mktemp -d)\n cp re2_test.py \"${DIR}\"\n cd \"${DIR}\"\n python re2_test.py\n shell: bash\n working-directory: python\n - uses: actions/upload-artifact@v4.6.2\n with:\n name: ${{ hashFiles('python/google_re2-*.whl') }}\n path: python/google_re2-*.whl\n retention-days: 1\n publish:\n needs:\n - wheel-linux\n - wheel-macos\n - wheel-windows\n permissions:\n contents: read\n # Required for PyPI publishing.\n id-token: write\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4.2.2\n # Stash the timestamp for the commit SHA that triggered the workflow.\n - run: echo \"timestamp=$(git log -1 --pretty=%ct)\" >> \"${GITHUB_ENV}\"\n shell: bash\n - uses: actions/setup-python@v5.6.0\n with:\n python-version: '3.x'\n - name: Prepare Python 3.x environment\n run: |\n python -m pip install --upgrade pip\n python -m pip install --upgrade setuptools build wheel\n shell: bash\n - if: inputs.build <= 1 || inputs.force-sdist == true\n name: Build sdist\n env:\n SOURCE_DATE_EPOCH: ${{ env.timestamp }}\n run: |\n python -m build --sdist\n shell: bash\n working-directory: python\n - uses: actions/download-artifact@v4.3.0\n with:\n path: python\n - name: Set build number to ${{ inputs.build }}\n env:\n SOURCE_DATE_EPOCH: ${{ env.timestamp }}\n run: |\n mkdir -p dist\n for WHL in */google_re2-*.whl; do\n python -m wheel unpack \"${WHL}\"\n python -m wheel pack --dest-dir=dist --build-number=${{ inputs.build }} google_re2-*\n rm -rf google_re2-*\n done\n shell: bash\n working-directory: python\n - if: inputs.build >= 1\n uses: pypa/gh-action-pypi-publish@v1.13.0\n with:\n packages-dir: python/dist\n" }, { "path": ".github/workflows/release-bazel.yml", "content": "name: Release (Bazel)\non:\n # Allow manual triggering from GH UI\n workflow_dispatch:\n inputs:\n tag_name:\n required: true\n type: string\n # Automated trigger from the release.yaml workflow\n workflow_call:\n inputs:\n tag_name:\n required: true\n type: string\n secrets:\n BCR_PUBLISH_TOKEN:\n description: 'Token for pushing to re2-machine/bazel-central-registry'\n required: true\njobs:\n release:\n uses: bazel-contrib/publish-to-bcr/.github/workflows/publish.yaml@v0.2.2\n with:\n draft: false\n tag_name: ${{ inputs.tag_name }}\n registry_fork: re2-machine/bazel-central-registry\n # NOTE: To use attest: true, we need a signed intoto.jsonl file,\n # but that appears to require using\n # the release_ruleset support described on\n # https://github.com/bazel-contrib/publish-to-bcr?tab=readme-ov-file#attesation-support\n # but that requires a release_prep.sh file,\n # and an override on the test command,\n # and may insist on doing the release upload of the source zip\n # (which we do ourselves separately),\n # and possibly more problems I didn't hit because I gave up.\n attest: false # too hard to generate the intoto.jsonl file\n permissions:\n contents: write\n id-token: write\n attestations: write\n secrets:\n # Necessary to push to the BCR fork, and to open a pull request against a registry\n publish_token: ${{ secrets.BCR_PUBLISH_TOKEN }}\n" }, { "path": ".github/workflows/release.yml", "content": "name: Release\non:\n push:\n tags: ['2[0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]'] # yyyy-mm-dd\npermissions:\n contents: read\njobs:\n create:\n permissions:\n # Required to create the release\n # and upload the release assets.\n contents: write\n # Required for Sigstore signing.\n id-token: write\n runs-on: ubuntu-latest\n env:\n GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n steps:\n - uses: actions/checkout@v4.2.2\n - run: |\n gh release create \"${GITHUB_REF_NAME}\" \\\n --generate-notes --latest --verify-tag \\\n --repo \"${GITHUB_REPOSITORY}\"\n gh release download \"${GITHUB_REF_NAME}\" \\\n --archive tar.gz \\\n --repo \"${GITHUB_REPOSITORY}\"\n gh release download \"${GITHUB_REF_NAME}\" \\\n --archive zip \\\n --repo \"${GITHUB_REPOSITORY}\"\n shell: bash\n - uses: sigstore/gh-action-sigstore-python@v3.0.1\n with:\n # N.B. This is a whitespace-separated string!\n inputs: '*.tar.gz *.zip'\n - run: |\n gh release upload \"${GITHUB_REF_NAME}\" \\\n *.tar.gz *.zip *.sigstore* \\\n --repo \"${GITHUB_REPOSITORY}\"\n shell: bash\n create-bazel:\n needs: create\n uses: ./.github/workflows/release-bazel.yml\n with:\n tag_name: ${{ github.ref_name }}\n permissions:\n contents: write\n id-token: write\n attestations: write\n secrets:\n BCR_PUBLISH_TOKEN: ${{ secrets.BCR_PUBLISH_TOKEN }}\n" }, { "path": ".gitignore", "content": "*.pyc\n*.orig\ncore\nobj/\nbenchlog.*\nuser.bazelrc\n" }, { "path": "BUILD.bazel", "content": "# Copyright 2009 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# Bazel (http://bazel.build/) BUILD file for RE2.\n\nload(\"@rules_cc//cc:cc_binary.bzl\", \"cc_binary\")\nload(\"@rules_cc//cc:cc_library.bzl\", \"cc_library\")\nload(\"@rules_cc//cc:cc_test.bzl\", \"cc_test\")\n\nlicenses([\"notice\"])\n\nexports_files([\"LICENSE\"])\n\ncc_library(\n name = \"re2\",\n srcs = [\n \"re2/bitmap256.cc\",\n \"re2/bitmap256.h\",\n \"re2/bitstate.cc\",\n \"re2/compile.cc\",\n \"re2/dfa.cc\",\n \"re2/filtered_re2.cc\",\n \"re2/mimics_pcre.cc\",\n \"re2/nfa.cc\",\n \"re2/onepass.cc\",\n \"re2/parse.cc\",\n \"re2/perl_groups.cc\",\n \"re2/pod_array.h\",\n \"re2/prefilter.cc\",\n \"re2/prefilter.h\",\n \"re2/prefilter_tree.cc\",\n \"re2/prefilter_tree.h\",\n \"re2/prog.cc\",\n \"re2/prog.h\",\n \"re2/re2.cc\",\n \"re2/regexp.cc\",\n \"re2/regexp.h\",\n \"re2/set.cc\",\n \"re2/simplify.cc\",\n \"re2/sparse_array.h\",\n \"re2/sparse_set.h\",\n \"re2/tostring.cc\",\n \"re2/unicode_casefold.cc\",\n \"re2/unicode_casefold.h\",\n \"re2/unicode_groups.cc\",\n \"re2/unicode_groups.h\",\n \"re2/walker-inl.h\",\n \"util/rune.cc\",\n \"util/strutil.cc\",\n \"util/strutil.h\",\n \"util/utf.h\",\n ],\n hdrs = [\n \"re2/filtered_re2.h\",\n \"re2/re2.h\",\n \"re2/set.h\",\n \"re2/stringpiece.h\",\n ],\n copts = select({\n # WebAssembly support for threads is... fraught at every level.\n \"@platforms//cpu:wasm32\": [],\n \"@platforms//cpu:wasm64\": [],\n \"@platforms//os:emscripten\": [],\n \"@platforms//os:wasi\": [],\n \"@platforms//os:windows\": [],\n \"//conditions:default\": [\"-pthread\"],\n }),\n linkopts = select({\n # macOS doesn't need `-pthread' when linking and it appears that\n # older versions of Clang will warn about the unused command line\n # argument, so just don't pass it.\n \"@platforms//os:macos\": [],\n # WebAssembly support for threads is... fraught at every level.\n \"@platforms//cpu:wasm32\": [],\n \"@platforms//cpu:wasm64\": [],\n \"@platforms//os:emscripten\": [],\n \"@platforms//os:wasi\": [],\n \"@platforms//os:windows\": [],\n \"//conditions:default\": [\"-pthread\"],\n }),\n visibility = [\"//visibility:public\"],\n deps = [\n \"@abseil-cpp//absl/base\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/container:fixed_array\",\n \"@abseil-cpp//absl/container:flat_hash_map\",\n \"@abseil-cpp//absl/container:flat_hash_set\",\n \"@abseil-cpp//absl/container:inlined_vector\",\n \"@abseil-cpp//absl/hash\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@abseil-cpp//absl/strings\",\n \"@abseil-cpp//absl/strings:str_format\",\n \"@abseil-cpp//absl/synchronization\",\n \"@abseil-cpp//absl/types:optional\",\n \"@abseil-cpp//absl/types:span\",\n ],\n)\n\ncc_library(\n name = \"testing\",\n testonly = 1,\n srcs = [\n \"re2/testing/backtrack.cc\",\n \"re2/testing/dump.cc\",\n \"re2/testing/exhaustive_tester.cc\",\n \"re2/testing/null_walker.cc\",\n \"re2/testing/regexp_generator.cc\",\n \"re2/testing/string_generator.cc\",\n \"re2/testing/tester.cc\",\n \"util/pcre.cc\",\n ],\n hdrs = [\n \"re2/testing/exhaustive_tester.h\",\n \"re2/testing/regexp_generator.h\",\n \"re2/testing/string_generator.h\",\n \"re2/testing/tester.h\",\n \"util/malloc_counter.h\",\n \"util/pcre.h\",\n\n # Exposed for testing only.\n \"re2/bitmap256.h\",\n \"re2/pod_array.h\",\n \"re2/prefilter.h\",\n \"re2/prefilter_tree.h\",\n \"re2/prog.h\",\n \"re2/regexp.h\",\n \"re2/sparse_array.h\",\n \"re2/sparse_set.h\",\n \"re2/unicode_casefold.h\",\n \"re2/unicode_groups.h\",\n \"re2/walker-inl.h\",\n \"util/strutil.h\",\n \"util/utf.h\",\n ],\n visibility = [\":__subpackages__\"],\n deps = [\n \":re2\",\n \"@abseil-cpp//absl/base\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/container:flat_hash_set\",\n \"@abseil-cpp//absl/flags:flag\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@abseil-cpp//absl/strings\",\n \"@abseil-cpp//absl/strings:str_format\",\n \"@googletest//:gtest\",\n ],\n)\n\ncc_test(\n name = \"charclass_test\",\n size = \"small\",\n srcs = [\"re2/testing/charclass_test.cc\"],\n deps = [\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/strings:str_format\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"compile_test\",\n size = \"small\",\n srcs = [\"re2/testing/compile_test.cc\"],\n deps = [\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@abseil-cpp//absl/strings\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"filtered_re2_test\",\n size = \"small\",\n srcs = [\"re2/testing/filtered_re2_test.cc\"],\n deps = [\n \":re2\",\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"mimics_pcre_test\",\n size = \"small\",\n srcs = [\"re2/testing/mimics_pcre_test.cc\"],\n deps = [\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"parse_test\",\n size = \"small\",\n srcs = [\"re2/testing/parse_test.cc\"],\n deps = [\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"possible_match_test\",\n size = \"small\",\n srcs = [\"re2/testing/possible_match_test.cc\"],\n deps = [\n \":re2\",\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@abseil-cpp//absl/strings\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"re2_arg_test\",\n size = \"small\",\n srcs = [\"re2/testing/re2_arg_test.cc\"],\n deps = [\n \":re2\",\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@abseil-cpp//absl/types:optional\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"re2_test\",\n size = \"small\",\n srcs = [\"re2/testing/re2_test.cc\"],\n deps = [\n \":re2\",\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@abseil-cpp//absl/strings\",\n \"@abseil-cpp//absl/strings:str_format\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"regexp_test\",\n size = \"small\",\n srcs = [\"re2/testing/regexp_test.cc\"],\n deps = [\n \":testing\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"required_prefix_test\",\n size = \"small\",\n srcs = [\"re2/testing/required_prefix_test.cc\"],\n deps = [\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"search_test\",\n size = \"small\",\n srcs = [\"re2/testing/search_test.cc\"],\n deps = [\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"set_test\",\n size = \"small\",\n srcs = [\"re2/testing/set_test.cc\"],\n deps = [\n \":re2\",\n \":testing\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"simplify_test\",\n size = \"small\",\n srcs = [\"re2/testing/simplify_test.cc\"],\n deps = [\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"string_generator_test\",\n size = \"small\",\n srcs = [\"re2/testing/string_generator_test.cc\"],\n deps = [\n \":testing\",\n \"@abseil-cpp//absl/strings\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"dfa_test\",\n size = \"large\",\n srcs = [\"re2/testing/dfa_test.cc\"],\n deps = [\n \":re2\",\n \":testing\",\n \"@abseil-cpp//absl/base:core_headers\",\n \"@abseil-cpp//absl/flags:flag\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@abseil-cpp//absl/strings\",\n \"@abseil-cpp//absl/strings:str_format\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"exhaustive1_test\",\n size = \"large\",\n srcs = [\"re2/testing/exhaustive1_test.cc\"],\n deps = [\n \":testing\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"exhaustive2_test\",\n size = \"large\",\n srcs = [\"re2/testing/exhaustive2_test.cc\"],\n deps = [\n \":testing\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"exhaustive3_test\",\n size = \"large\",\n srcs = [\"re2/testing/exhaustive3_test.cc\"],\n deps = [\n \":testing\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"exhaustive_test\",\n size = \"large\",\n srcs = [\"re2/testing/exhaustive_test.cc\"],\n deps = [\n \":testing\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_test(\n name = \"random_test\",\n size = \"large\",\n srcs = [\"re2/testing/random_test.cc\"],\n deps = [\n \":testing\",\n \"@abseil-cpp//absl/flags:flag\",\n \"@abseil-cpp//absl/strings:str_format\",\n \"@googletest//:gtest\",\n \"@googletest//:gtest_main\",\n ],\n)\n\ncc_binary(\n name = \"regexp_benchmark\",\n testonly = 1,\n srcs = [\"re2/testing/regexp_benchmark.cc\"],\n deps = [\n \":re2\",\n \":testing\",\n \"@abseil-cpp//absl/container:flat_hash_map\",\n \"@abseil-cpp//absl/flags:flag\",\n \"@abseil-cpp//absl/log:absl_check\",\n \"@abseil-cpp//absl/log:absl_log\",\n \"@abseil-cpp//absl/strings\",\n \"@abseil-cpp//absl/strings:str_format\",\n \"@abseil-cpp//absl/synchronization\",\n \"@google_benchmark//:benchmark_main\",\n ],\n)\n\ntest_suite(\n name = \"small_tests\",\n tags = [\"small\"],\n tests = [\n \":charclass_test\",\n \":compile_test\",\n \":filtered_re2_test\",\n \":mimics_pcre_test\",\n \":parse_test\",\n \":possible_match_test\",\n \":re2_arg_test\",\n \":re2_test\",\n \":regexp_test\",\n \":required_prefix_test\",\n \":search_test\",\n \":set_test\",\n \":simplify_test\",\n \":string_generator_test\",\n ],\n)\n" }, { "path": "CMakeLists.txt", "content": "# Copyright 2015 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# https://github.com/google/oss-policies-info/blob/main/foundational-cxx-support-matrix.md\ncmake_minimum_required(VERSION 3.22)\n\nproject(RE2 CXX)\ninclude(CMakePackageConfigHelpers)\ninclude(CTest)\ninclude(GNUInstallDirs)\n\nset(RE2_CXX_VERSION cxx_std_17)\n\noption(BUILD_SHARED_LIBS \"build shared libraries\" OFF)\noption(RE2_USE_ICU \"build against ICU for full Unicode properties support\" OFF)\n\n# For historical reasons, this is just \"USEPCRE\", not \"RE2_USE_PCRE\".\noption(USEPCRE \"build against PCRE for testing and benchmarking\" OFF)\n\n# See https://groups.google.com/g/re2-dev/c/P6_NM0YIWvA for details.\n# This has no effect unless RE2 is being built for an Apple platform\n# such as macOS or iOS.\noption(RE2_BUILD_FRAMEWORK \"build RE2 as a framework\" OFF)\n\n# CMake seems to have no way to enable/disable testing per subproject,\n# so we provide an option similar to BUILD_TESTING, but just for RE2.\n# RE2_BUILD_TESTING builds and runs tests, and builds benchmarks\n# RE2_TEST and RE2_BENCHMARK provide more fine-grained control.\noption(RE2_TEST \"build and run RE2 tests\" OFF)\noption(RE2_BENCHMARK \"build RE2 benchmarks\" OFF)\noption(RE2_BUILD_TESTING \"build and run RE2 tests; build RE2 benchmarks\" OFF)\n\n# RE2_INSTALL (which defaults to ON) controls whether the installation\n# rules are generated.\noption(RE2_INSTALL \"install RE2\" ON)\n\n# The pkg-config Requires: field.\nset(REQUIRES)\n\n# ABI version\n# http://tldp.org/HOWTO/Program-Library-HOWTO/shared-libraries.html\nset(SONAME 11)\n\nset(EXTRA_TARGET_LINK_LIBRARIES)\n\nif(MSVC)\n if(MSVC_VERSION LESS 1920)\n message(FATAL_ERROR \"you need Visual Studio 2019 or later\")\n endif()\n if(BUILD_SHARED_LIBS)\n set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)\n endif()\n # CMake defaults to /W3, but some users like /W4 (or /Wall) and /WX,\n # so we disable various warnings that aren't particularly helpful.\n add_compile_options(/wd4100 /wd4201 /wd4456 /wd4457 /wd4702 /wd4815)\n # Without a byte order mark (BOM), Visual Studio assumes that the source\n # file is encoded using the current user code page, so we specify UTF-8.\n add_compile_options(/utf-8)\nendif()\n\nif(WIN32)\n add_definitions(-DUNICODE -D_UNICODE -DSTRICT -DNOMINMAX)\n add_definitions(-D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS)\nendif()\n\nif(UNIX)\n set(THREADS_PREFER_PTHREAD_FLAG ON)\n find_package(Threads REQUIRED)\nendif()\n\nset(ABSL_DEPS\n absl_absl_check\n absl_absl_log\n absl_base\n absl_core_headers\n absl_fixed_array\n absl_flags\n absl_flat_hash_map\n absl_flat_hash_set\n absl_hash\n absl_inlined_vector\n absl_optional\n absl_span\n absl_str_format\n absl_strings\n absl_synchronization\n )\n\n# If a top-level project has called add_directory(abseil-cpp) already (possibly\n# indirectly), let that take precedence over any copy of Abseil that might have\n# been installed on the system. And likewise for ICU, GoogleTest and Benchmark.\nif(NOT TARGET absl::base)\n find_package(absl REQUIRED)\nendif()\nlist(APPEND REQUIRES ${ABSL_DEPS})\n\nif(RE2_USE_ICU)\n if(NOT TARGET ICU::uc)\n find_package(ICU REQUIRED COMPONENTS uc)\n endif()\n add_definitions(-DRE2_USE_ICU)\n list(APPEND REQUIRES icu-uc)\nendif()\n\nif(USEPCRE)\n add_definitions(-DUSEPCRE)\n list(APPEND EXTRA_TARGET_LINK_LIBRARIES pcre)\nendif()\n\nlist(JOIN REQUIRES \" \" REQUIRES)\n\nset(RE2_SOURCES\n re2/bitmap256.cc\n re2/bitstate.cc\n re2/compile.cc\n re2/dfa.cc\n re2/filtered_re2.cc\n re2/mimics_pcre.cc\n re2/nfa.cc\n re2/onepass.cc\n re2/parse.cc\n re2/perl_groups.cc\n re2/prefilter.cc\n re2/prefilter_tree.cc\n re2/prog.cc\n re2/re2.cc\n re2/regexp.cc\n re2/set.cc\n re2/simplify.cc\n re2/tostring.cc\n re2/unicode_casefold.cc\n re2/unicode_groups.cc\n util/rune.cc\n util/strutil.cc\n )\n\nset(RE2_HEADERS\n re2/filtered_re2.h\n re2/re2.h\n re2/set.h\n re2/stringpiece.h\n )\n\nadd_library(re2 ${RE2_SOURCES})\ntarget_compile_features(re2 PUBLIC ${RE2_CXX_VERSION})\ntarget_include_directories(re2 PUBLIC $)\n# CMake gives \"set_target_properties called with incorrect number of arguments.\"\n# errors if we don't quote ${RE2_HEADERS}, so quote it despite prevailing style.\nset_target_properties(re2 PROPERTIES PUBLIC_HEADER \"${RE2_HEADERS}\")\nset_target_properties(re2 PROPERTIES SOVERSION ${SONAME} VERSION ${SONAME}.0.0)\nadd_library(re2::re2 ALIAS re2)\n\nif(APPLE AND RE2_BUILD_FRAMEWORK)\n set_target_properties(re2 PROPERTIES\n FRAMEWORK TRUE\n FRAMEWORK_VERSION A\n MACOSX_FRAMEWORK_IDENTIFIER com.googlesource.code.re2)\nendif()\n\nif(UNIX)\n target_link_libraries(re2 PUBLIC Threads::Threads)\nendif()\n\nforeach(dep ${ABSL_DEPS})\n # Work around https://gitlab.kitware.com/cmake/cmake/-/issues/16899. >:(\n string(PREPEND dep \"^\")\n string(REGEX REPLACE \"\\\\^absl_\" \"absl::\" dep ${dep})\n target_link_libraries(re2 PUBLIC ${dep})\nendforeach()\n\nif(RE2_USE_ICU)\n target_link_libraries(re2 PUBLIC ICU::uc)\nendif()\n\nif(RE2_BUILD_TESTING OR RE2_TEST OR RE2_BENCHMARK)\n set(TESTING_SOURCES\n re2/testing/backtrack.cc\n re2/testing/dump.cc\n re2/testing/exhaustive_tester.cc\n re2/testing/null_walker.cc\n re2/testing/regexp_generator.cc\n re2/testing/string_generator.cc\n re2/testing/tester.cc\n util/pcre.cc\n )\n\n add_library(testing ${TESTING_SOURCES})\n if(BUILD_SHARED_LIBS AND WIN32)\n target_compile_definitions(testing PRIVATE -DRE2_BUILD_TESTING_DLL)\n endif()\n target_compile_features(testing PUBLIC ${RE2_CXX_VERSION})\n target_link_libraries(testing PUBLIC re2 GTest::gtest)\n\n if(RE2_BUILD_TESTING OR RE2_TEST)\n if(NOT TARGET GTest::gtest)\n find_package(GTest REQUIRED)\n endif()\n\n set(TEST_TARGETS\n charclass_test\n compile_test\n filtered_re2_test\n mimics_pcre_test\n parse_test\n possible_match_test\n re2_test\n re2_arg_test\n regexp_test\n required_prefix_test\n search_test\n set_test\n simplify_test\n string_generator_test\n\n dfa_test\n exhaustive1_test\n exhaustive2_test\n exhaustive3_test\n exhaustive_test\n random_test\n )\n\n foreach(target ${TEST_TARGETS})\n add_executable(${target} re2/testing/${target}.cc)\n if(BUILD_SHARED_LIBS AND WIN32)\n target_compile_definitions(${target} PRIVATE -DRE2_CONSUME_TESTING_DLL)\n endif()\n target_compile_features(${target} PUBLIC ${RE2_CXX_VERSION})\n target_link_libraries(${target} PUBLIC re2 testing GTest::gtest_main ${EXTRA_TARGET_LINK_LIBRARIES})\n add_test(NAME ${target} COMMAND ${target})\n endforeach()\n endif()\n\n if(RE2_BUILD_TESTING OR RE2_BENCHMARK)\n if(NOT TARGET benchmark::benchmark)\n find_package(benchmark REQUIRED)\n endif()\n set(BENCHMARK_TARGETS\n regexp_benchmark\n )\n foreach(target ${BENCHMARK_TARGETS})\n add_executable(${target} re2/testing/${target}.cc)\n if(BUILD_SHARED_LIBS AND WIN32)\n target_compile_definitions(${target} PRIVATE -DRE2_CONSUME_TESTING_DLL)\n endif()\n target_compile_features(${target} PUBLIC ${RE2_CXX_VERSION})\n target_link_libraries(${target} PUBLIC testing re2 benchmark::benchmark_main ${EXTRA_TARGET_LINK_LIBRARIES})\n endforeach()\n endif()\nendif()\n\nif(RE2_INSTALL)\n install(TARGETS re2\n EXPORT re2Targets\n ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}\n LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}\n RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}\n FRAMEWORK DESTINATION ${CMAKE_INSTALL_LIBDIR}\n PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/re2\n INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})\n install(EXPORT re2Targets\n DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/re2\n NAMESPACE re2::)\n\n configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/re2Config.cmake.in\n ${CMAKE_CURRENT_BINARY_DIR}/re2Config.cmake\n INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/re2)\n write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/re2ConfigVersion.cmake\n VERSION ${SONAME}.0.0\n COMPATIBILITY SameMajorVersion)\n\n install(FILES ${CMAKE_CURRENT_BINARY_DIR}/re2Config.cmake\n ${CMAKE_CURRENT_BINARY_DIR}/re2ConfigVersion.cmake\n DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/re2)\n\n configure_file(${CMAKE_CURRENT_SOURCE_DIR}/re2.pc.in\n ${CMAKE_CURRENT_BINARY_DIR}/re2.pc\n @ONLY)\n\n install(FILES ${CMAKE_CURRENT_BINARY_DIR}/re2.pc\n DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)\nendif()\n" }, { "path": "CONTRIBUTING.md", "content": "See the [Contribute](https://github.com/google/re2/wiki/Contribute) wiki page.\n" }, { "path": "LICENSE", "content": "// Copyright (c) 2009 The RE2 Authors. All rights reserved.\n//\n// Redistribution and use in source and binary forms, with or without\n// modification, are permitted provided that the following conditions are\n// met:\n//\n// * Redistributions of source code must retain the above copyright\n// notice, this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above\n// copyright notice, this list of conditions and the following disclaimer\n// in the documentation and/or other materials provided with the\n// distribution.\n// * Neither the name of Google Inc. nor the names of its\n// contributors may be used to endorse or promote products derived from\n// this software without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "MODULE.bazel", "content": "# Copyright 2009 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# Bazel (http://bazel.build/) MODULE file for RE2.\n\nmodule(\n name = \"re2\",\n version = \"2025-11-05\",\n compatibility_level = 1,\n)\n\nbazel_dep(name = \"platforms\", version = \"1.0.0\")\nbazel_dep(name = \"apple_support\", version = \"1.24.2\")\nbazel_dep(name = \"rules_cc\", version = \"0.2.14\")\nbazel_dep(name = \"abseil-cpp\", version = \"20250814.1\")\nbazel_dep(name = \"rules_python\", version = \"1.7.0\")\nbazel_dep(name = \"pybind11_bazel\", version = \"3.0.0\")\n\n# This is a temporary hack for `x64_x86_windows`.\n# TODO(junyer): Remove whenever no longer needed.\ncc_configure = use_extension(\"@rules_cc//cc:extensions.bzl\", \"cc_configure_extension\", dev_dependency = True)\nuse_repo(cc_configure, \"local_config_cc\")\n\n# These dependencies will be ignored when the `re2` module is not\n# the root module (or when `--ignore_dev_dependency` is enabled).\nbazel_dep(name = \"google_benchmark\", version = \"1.9.4\", dev_dependency = True)\nbazel_dep(name = \"googletest\", version = \"1.17.0.bcr.2\", dev_dependency = True)\nbazel_dep(name = \"abseil-py\", version = \"2.1.0\", dev_dependency = True)\n" }, { "path": "Makefile", "content": "# Copyright 2009 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# Build against Abseil.\nABSL_DEPS=\\\n\tabsl_absl_check\\\n\tabsl_absl_log\\\n\tabsl_base\\\n\tabsl_core_headers\\\n\tabsl_fixed_array\\\n\tabsl_flags\\\n\tabsl_flat_hash_map\\\n\tabsl_flat_hash_set\\\n\tabsl_hash\\\n\tabsl_inlined_vector\\\n\tabsl_optional\\\n\tabsl_span\\\n\tabsl_str_format\\\n\tabsl_strings\\\n\tabsl_synchronization\\\n\nPKG_CONFIG?=pkg-config\nCCABSL=$(shell $(PKG_CONFIG) $(ABSL_DEPS) --cflags)\n# GCC barfs on `-Wl` whereas Clang doesn't mind, but it's unclear what\n# causes it to manifest on Ubuntu 22.04 LTS, so filter it out for now.\n# Similar is needed for `static-testinstall` and `shared-testinstall`.\nLDABSL=$(shell $(PKG_CONFIG) $(ABSL_DEPS) --libs | sed -e 's/-Wl / /g')\n\n# To build against ICU for full Unicode properties support,\n# uncomment the next two lines:\n# CCICU=$(shell $(PKG_CONFIG) icu-uc --cflags) -DRE2_USE_ICU\n# LDICU=$(shell $(PKG_CONFIG) icu-uc --libs)\n\n# Build against GoogleTest and Benchmark for... testing and benchmarking.\n# Capture only the `-L` flags for now; we will pass the `-l` flags later.\nCCGTEST=$(shell $(PKG_CONFIG) gtest gtest_main --cflags)\nLDGTEST=$(shell $(PKG_CONFIG) gtest gtest_main --libs-only-L)\nCCBENCHMARK=$(shell $(PKG_CONFIG) benchmark --cflags)\nLDBENCHMARK=$(shell $(PKG_CONFIG) benchmark --libs-only-L)\n\n# To build against PCRE for testing and benchmarking,\n# uncomment the next two lines:\n# CCPCRE=-I/usr/local/include -DUSEPCRE\n# LDPCRE=-L/usr/local/lib -lpcre\n\nCXX?=g++\n# can override\nCXXFLAGS?=-O3 -g\nLDFLAGS?=\n# required\nRE2_CXXFLAGS?=-std=c++17 -pthread -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -I. $(CCABSL) $(CCICU) $(CCGTEST) $(CCBENCHMARK) $(CCPCRE)\nRE2_LDFLAGS?=-pthread $(LDABSL) $(LDICU) $(LDGTEST) $(LDBENCHMARK) $(LDPCRE)\nAR?=ar\nARFLAGS?=rsc\nNM?=nm\nNMFLAGS?=-p\n\n# Variables mandated by GNU, the arbiter of all good taste on the internet.\n# http://www.gnu.org/prep/standards/standards.html\nprefix=/usr/local\nexec_prefix=$(prefix)\nincludedir=$(prefix)/include\nlibdir=$(exec_prefix)/lib\nINSTALL=install\nINSTALL_DATA=$(INSTALL) -m 644\n\n# Work around the weirdness of sed(1) on Darwin. :/\nifeq ($(shell uname),Darwin)\nSED_INPLACE=sed -i ''\nelse ifeq ($(shell uname),SunOS)\nSED_INPLACE=sed -i\nelse\nSED_INPLACE=sed -i\nendif\n\n# The pkg-config Requires: field.\nREQUIRES=$(ABSL_DEPS)\nifdef LDICU\nREQUIRES+=icu-uc\nendif\n\n# ABI version\n# http://tldp.org/HOWTO/Program-Library-HOWTO/shared-libraries.html\nSONAME=11\n\n# To rebuild the Tables generated by Perl and Python scripts (requires Internet\n# access for Unicode data), uncomment the following line:\n# REBUILD_TABLES=1\n\n# The SunOS linker does not support wildcards. :(\nifeq ($(shell uname),Darwin)\nSOEXT=dylib\nSOEXTVER=$(SONAME).$(SOEXT)\nSOEXTVER00=$(SONAME).0.0.$(SOEXT)\nMAKE_SHARED_LIBRARY=$(CXX) -dynamiclib -Wl,-compatibility_version,$(SONAME),-current_version,$(SONAME).0.0,-install_name,$(libdir)/libre2.$(SOEXTVER),-exported_symbols_list,libre2.symbols.darwin\nelse ifeq ($(shell uname),SunOS)\nSOEXT=so\nSOEXTVER=$(SOEXT).$(SONAME)\nSOEXTVER00=$(SOEXT).$(SONAME).0.0\nMAKE_SHARED_LIBRARY=$(CXX) -shared -Wl,-soname,libre2.$(SOEXTVER)\nelse\nSOEXT=so\nSOEXTVER=$(SOEXT).$(SONAME)\nSOEXTVER00=$(SOEXT).$(SONAME).0.0\nMAKE_SHARED_LIBRARY=$(CXX) -shared -Wl,-soname,libre2.$(SOEXTVER),--version-script,libre2.symbols\nendif\n\n.PHONY: all\nall: obj/libre2.a obj/so/libre2.$(SOEXT)\n\nINSTALL_HFILES=\\\n\tre2/filtered_re2.h\\\n\tre2/re2.h\\\n\tre2/set.h\\\n\tre2/stringpiece.h\\\n\nHFILES=\\\n\tutil/malloc_counter.h\\\n\tutil/pcre.h\\\n\tutil/strutil.h\\\n\tutil/utf.h\\\n\tre2/bitmap256.h\\\n\tre2/filtered_re2.h\\\n\tre2/pod_array.h\\\n\tre2/prefilter.h\\\n\tre2/prefilter_tree.h\\\n\tre2/prog.h\\\n\tre2/re2.h\\\n\tre2/regexp.h\\\n\tre2/set.h\\\n\tre2/sparse_array.h\\\n\tre2/sparse_set.h\\\n\tre2/stringpiece.h\\\n\tre2/testing/exhaustive_tester.h\\\n\tre2/testing/regexp_generator.h\\\n\tre2/testing/string_generator.h\\\n\tre2/testing/tester.h\\\n\tre2/unicode_casefold.h\\\n\tre2/unicode_groups.h\\\n\tre2/walker-inl.h\\\n\nOFILES=\\\n\tobj/util/rune.o\\\n\tobj/util/strutil.o\\\n\tobj/re2/bitmap256.o\\\n\tobj/re2/bitstate.o\\\n\tobj/re2/compile.o\\\n\tobj/re2/dfa.o\\\n\tobj/re2/filtered_re2.o\\\n\tobj/re2/mimics_pcre.o\\\n\tobj/re2/nfa.o\\\n\tobj/re2/onepass.o\\\n\tobj/re2/parse.o\\\n\tobj/re2/perl_groups.o\\\n\tobj/re2/prefilter.o\\\n\tobj/re2/prefilter_tree.o\\\n\tobj/re2/prog.o\\\n\tobj/re2/re2.o\\\n\tobj/re2/regexp.o\\\n\tobj/re2/set.o\\\n\tobj/re2/simplify.o\\\n\tobj/re2/tostring.o\\\n\tobj/re2/unicode_casefold.o\\\n\tobj/re2/unicode_groups.o\\\n\nTESTOFILES=\\\n\tobj/util/pcre.o\\\n\tobj/re2/testing/backtrack.o\\\n\tobj/re2/testing/dump.o\\\n\tobj/re2/testing/exhaustive_tester.o\\\n\tobj/re2/testing/null_walker.o\\\n\tobj/re2/testing/regexp_generator.o\\\n\tobj/re2/testing/string_generator.o\\\n\tobj/re2/testing/tester.o\\\n\nTESTS=\\\n\tobj/test/charclass_test\\\n\tobj/test/compile_test\\\n\tobj/test/filtered_re2_test\\\n\tobj/test/mimics_pcre_test\\\n\tobj/test/parse_test\\\n\tobj/test/possible_match_test\\\n\tobj/test/re2_test\\\n\tobj/test/re2_arg_test\\\n\tobj/test/regexp_test\\\n\tobj/test/required_prefix_test\\\n\tobj/test/search_test\\\n\tobj/test/set_test\\\n\tobj/test/simplify_test\\\n\tobj/test/string_generator_test\\\n\nBIGTESTS=\\\n\tobj/test/dfa_test\\\n\tobj/test/exhaustive1_test\\\n\tobj/test/exhaustive2_test\\\n\tobj/test/exhaustive3_test\\\n\tobj/test/exhaustive_test\\\n\tobj/test/random_test\\\n\nSOFILES=$(patsubst obj/%,obj/so/%,$(OFILES))\n# We use TESTOFILES for testing the shared lib, only it is built differently.\nSTESTS=$(patsubst obj/%,obj/so/%,$(TESTS))\nSBIGTESTS=$(patsubst obj/%,obj/so/%,$(BIGTESTS))\n\nDOFILES=$(patsubst obj/%,obj/dbg/%,$(OFILES))\nDTESTOFILES=$(patsubst obj/%,obj/dbg/%,$(TESTOFILES))\nDTESTS=$(patsubst obj/%,obj/dbg/%,$(TESTS))\nDBIGTESTS=$(patsubst obj/%,obj/dbg/%,$(BIGTESTS))\n\n.PRECIOUS: obj/%.o\nobj/%.o: %.cc $(HFILES)\n\t@mkdir -p $$(dirname $@)\n\t$(CXX) -c -o $@ $(CPPFLAGS) $(RE2_CXXFLAGS) $(CXXFLAGS) -DNDEBUG $*.cc\n\n.PRECIOUS: obj/dbg/%.o\nobj/dbg/%.o: %.cc $(HFILES)\n\t@mkdir -p $$(dirname $@)\n\t$(CXX) -c -o $@ $(CPPFLAGS) $(RE2_CXXFLAGS) $(CXXFLAGS) $*.cc\n\n.PRECIOUS: obj/so/%.o\nobj/so/%.o: %.cc $(HFILES)\n\t@mkdir -p $$(dirname $@)\n\t$(CXX) -c -o $@ -fPIC $(CPPFLAGS) $(RE2_CXXFLAGS) $(CXXFLAGS) -DNDEBUG $*.cc\n\n.PRECIOUS: obj/libre2.a\nobj/libre2.a: $(OFILES)\n\t@mkdir -p obj\n\t$(AR) $(ARFLAGS) obj/libre2.a $(OFILES)\n\n.PRECIOUS: obj/dbg/libre2.a\nobj/dbg/libre2.a: $(DOFILES)\n\t@mkdir -p obj/dbg\n\t$(AR) $(ARFLAGS) obj/dbg/libre2.a $(DOFILES)\n\n.PRECIOUS: obj/so/libre2.$(SOEXT)\nobj/so/libre2.$(SOEXT): $(SOFILES) libre2.symbols libre2.symbols.darwin\n\t@mkdir -p obj/so\n\t$(MAKE_SHARED_LIBRARY) -o obj/so/libre2.$(SOEXTVER) $(SOFILES) $(RE2_LDFLAGS) $(LDFLAGS)\n\tln -sf libre2.$(SOEXTVER) $@\n\n.PRECIOUS: obj/dbg/test/%\nobj/dbg/test/%: obj/dbg/libre2.a obj/dbg/re2/testing/%.o $(DTESTOFILES)\n\t@mkdir -p obj/dbg/test\n\t$(CXX) -o $@ obj/dbg/re2/testing/$*.o $(DTESTOFILES) obj/dbg/libre2.a $(RE2_LDFLAGS) $(LDFLAGS) -lgtest -lgtest_main\n\n.PRECIOUS: obj/test/%\nobj/test/%: obj/libre2.a obj/re2/testing/%.o $(TESTOFILES)\n\t@mkdir -p obj/test\n\t$(CXX) -o $@ obj/re2/testing/$*.o $(TESTOFILES) obj/libre2.a $(RE2_LDFLAGS) $(LDFLAGS) -lgtest -lgtest_main\n\n# Test the shared lib, falling back to the static lib for private symbols\n.PRECIOUS: obj/so/test/%\nobj/so/test/%: obj/so/libre2.$(SOEXT) obj/libre2.a obj/re2/testing/%.o $(TESTOFILES)\n\t@mkdir -p obj/so/test\n\t$(CXX) -o $@ obj/re2/testing/$*.o $(TESTOFILES) -Lobj/so -lre2 obj/libre2.a $(RE2_LDFLAGS) $(LDFLAGS) -lgtest -lgtest_main\n\nobj/test/regexp_benchmark: obj/libre2.a obj/re2/testing/regexp_benchmark.o $(TESTOFILES)\n\t@mkdir -p obj/test\n\t$(CXX) -o $@ obj/re2/testing/regexp_benchmark.o $(TESTOFILES) obj/libre2.a $(RE2_LDFLAGS) $(LDFLAGS) -lgtest -lbenchmark -lbenchmark_main\n\nobj/test/re2_fuzzer: obj/libre2.a obj/re2/fuzzing/re2_fuzzer.o\n\t@mkdir -p obj/test\n\t$(CXX) -o $@ obj/re2/fuzzing/re2_fuzzer.o obj/libre2.a $(RE2_LDFLAGS) $(LDFLAGS)\n\nifdef REBUILD_TABLES\n.PRECIOUS: re2/perl_groups.cc\nre2/perl_groups.cc: re2/make_perl_groups.pl\n\tperl $< > $@\n\n.PRECIOUS: re2/unicode_%.cc\nre2/unicode_%.cc: re2/make_unicode_%.py re2/unicode.py\n\tpython3 $< > $@\nendif\n\n.PHONY: distclean\ndistclean: clean\n\trm -f re2/perl_groups.cc re2/unicode_casefold.cc re2/unicode_groups.cc\n\n.PHONY: clean\nclean:\n\trm -rf obj\n\trm -f re2/*.pyc\n\n.PHONY: testofiles\ntestofiles: $(TESTOFILES)\n\n.PHONY: test\ntest: $(DTESTS) $(TESTS) $(STESTS) debug-test static-test shared-test\n\n.PHONY: debug-test\ndebug-test: $(DTESTS)\n\t@./runtests $(DTESTS)\n\n.PHONY: static-test\nstatic-test: $(TESTS)\n\t@./runtests $(TESTS)\n\n.PHONY: shared-test\nshared-test: $(STESTS)\n\t@./runtests -shared-library-path obj/so $(STESTS)\n\n.PHONY: debug-bigtest\ndebug-bigtest: $(DTESTS) $(DBIGTESTS)\n\t@./runtests $(DTESTS) $(DBIGTESTS)\n\n.PHONY: static-bigtest\nstatic-bigtest: $(TESTS) $(BIGTESTS)\n\t@./runtests $(TESTS) $(BIGTESTS)\n\n.PHONY: shared-bigtest\nshared-bigtest: $(STESTS) $(SBIGTESTS)\n\t@./runtests -shared-library-path obj/so $(STESTS) $(SBIGTESTS)\n\n.PHONY: benchmark\nbenchmark: obj/test/regexp_benchmark\n\n.PHONY: fuzz\nfuzz: obj/test/re2_fuzzer\n\n.PHONY: install\ninstall: static-install shared-install\n\n.PHONY: static\nstatic: obj/libre2.a\n\n.PHONY: static-install\nstatic-install: obj/libre2.a common-install\n\t$(INSTALL) obj/libre2.a $(DESTDIR)$(libdir)/libre2.a\n\n.PHONY: shared\nshared: obj/so/libre2.$(SOEXT)\n\n.PHONY: shared-install\nshared-install: obj/so/libre2.$(SOEXT) common-install\n\t$(INSTALL) obj/so/libre2.$(SOEXT) $(DESTDIR)$(libdir)/libre2.$(SOEXTVER00)\n\tln -sf libre2.$(SOEXTVER00) $(DESTDIR)$(libdir)/libre2.$(SOEXTVER)\n\tln -sf libre2.$(SOEXTVER00) $(DESTDIR)$(libdir)/libre2.$(SOEXT)\n\n.PHONY: common-install\ncommon-install:\n\tmkdir -p $(DESTDIR)$(includedir)/re2 $(DESTDIR)$(libdir)/pkgconfig\n\t$(INSTALL_DATA) $(INSTALL_HFILES) $(DESTDIR)$(includedir)/re2\n\t$(INSTALL_DATA) re2.pc.in $(DESTDIR)$(libdir)/pkgconfig/re2.pc\n\t$(SED_INPLACE) -e \"s#@CMAKE_INSTALL_FULL_INCLUDEDIR@#$(includedir)#\" $(DESTDIR)$(libdir)/pkgconfig/re2.pc\n\t$(SED_INPLACE) -e \"s#@CMAKE_INSTALL_FULL_LIBDIR@#$(libdir)#\" $(DESTDIR)$(libdir)/pkgconfig/re2.pc\n\t$(SED_INPLACE) -e \"s#@REQUIRES@#$(REQUIRES)#\" $(DESTDIR)$(libdir)/pkgconfig/re2.pc\n\t$(SED_INPLACE) -e \"s#@SONAME@#$(SONAME)#\" $(DESTDIR)$(libdir)/pkgconfig/re2.pc\n\n.PHONY: testinstall\ntestinstall: static-testinstall shared-testinstall\n\t@echo\n\t@echo Install tests passed.\n\t@echo\n\n.PHONY: static-testinstall\nstatic-testinstall:\nifeq ($(shell uname),Darwin)\n\t@echo Skipping test for libre2.a on Darwin.\nelse ifeq ($(shell uname),SunOS)\n\t@echo Skipping test for libre2.a on SunOS.\nelse\n\t@mkdir -p obj\n\t@cp testinstall.cc obj/static-testinstall.cc\n\t(cd obj && export PKG_CONFIG_PATH=$(DESTDIR)$(libdir)/pkgconfig; \\\n\t $(CXX) static-testinstall.cc -o static-testinstall $(CXXFLAGS) $(LDFLAGS) \\\n\t $$($(PKG_CONFIG) re2 --cflags) \\\n\t $$($(PKG_CONFIG) re2 --libs | sed -e 's/-Wl / /g' | sed -e 's/-lre2/-l:libre2.a/'))\n\tobj/static-testinstall\nendif\n\n.PHONY: shared-testinstall\nshared-testinstall:\n\t@mkdir -p obj\n\t@cp testinstall.cc obj/shared-testinstall.cc\n\t(cd obj && export PKG_CONFIG_PATH=$(DESTDIR)$(libdir)/pkgconfig; \\\n\t $(CXX) shared-testinstall.cc -o shared-testinstall $(CXXFLAGS) $(LDFLAGS) \\\n\t $$($(PKG_CONFIG) re2 --cflags) \\\n\t $$($(PKG_CONFIG) re2 --libs | sed -e 's/-Wl / /g'))\nifeq ($(shell uname),Darwin)\n\tDYLD_LIBRARY_PATH=\"$(DESTDIR)$(libdir):$(DYLD_LIBRARY_PATH)\" obj/shared-testinstall\nelse\n\tLD_LIBRARY_PATH=\"$(DESTDIR)$(libdir):$(LD_LIBRARY_PATH)\" obj/shared-testinstall\nendif\n\n.PHONY: benchlog\nbenchlog: obj/test/regexp_benchmark\n\t(echo '==BENCHMARK==' `hostname` `date`; \\\n\t (uname -a; $(CXX) --version; git rev-parse --short HEAD; file obj/test/regexp_benchmark) | sed 's/^/# /'; \\\n\t echo; \\\n\t ./obj/test/regexp_benchmark 'PCRE|RE2') | tee -a benchlog.$$(hostname | sed 's/\\..*//')\n\n.PHONY: log\nlog:\n\t$(MAKE) clean\n\t$(MAKE) CXXFLAGS=\"$(CXXFLAGS) -DLOGGING=1\" \\\n\t\t$(filter obj/test/exhaustive%_test,$(BIGTESTS))\n\techo '#' RE2 exhaustive tests built by make log >re2-exhaustive.txt\n\techo '#' $$(date) >>re2-exhaustive.txt\n\tobj/test/exhaustive_test |grep -v '^PASS$$' >>re2-exhaustive.txt\n\tobj/test/exhaustive1_test |grep -v '^PASS$$' >>re2-exhaustive.txt\n\tobj/test/exhaustive2_test |grep -v '^PASS$$' >>re2-exhaustive.txt\n\tobj/test/exhaustive3_test |grep -v '^PASS$$' >>re2-exhaustive.txt\n\n\t$(MAKE) CXXFLAGS=\"$(CXXFLAGS) -DLOGGING=1\" obj/test/search_test\n\techo '#' RE2 basic search tests built by make $@ >re2-search.txt\n\techo '#' $$(date) >>re2-search.txt\n\tobj/test/search_test |grep -v '^PASS$$' >>re2-search.txt\n" }, { "path": "README.md", "content": "# RE2, a regular expression library\n\nRE2 is an efficient, principled regular expression library\nthat has been used in production at Google and many other places\nsince 2006.\n\n_**Safety is RE2's primary goal.**_\n\nRE2 was designed and implemented with an explicit goal of being able\nto handle regular expressions from untrusted users without risk.\nOne of its primary guarantees is that the match time is linear in the\nlength of the input string. It was also written with production concerns in mind:\nthe parser, the compiler and the execution engines limit their memory usage\nby working within a configurable budget—failing gracefully when exhausted—and\nthey avoid stack overflow by eschewing recursion.\n\nIt is not a goal to be faster than all other engines under all circumstances.\nAlthough RE2 guarantees a running time that is asymptotically linear in\nthe length of the input, more complex expressions may incur larger constant factors;\nlonger expressions increase the overhead required to handle those expressions safely.\nIn a sense, RE2 is pessimistic where a backtracking engine is optimistic:\nA backtracking engine tests each alternative sequentially, making it fast when the first alternative is common.\nBy contrast RE2 evaluates all alternatives in parallel, avoiding the performance penalty for the last alternative,\nat the cost of some overhead. This pessimism is what makes RE2 secure.\n\nIt is also not a goal to implement all of the features offered by Perl, PCRE and other engines.\nAs a matter of principle, RE2 does not support constructs for which only backtracking solutions are known to exist.\nThus, backreferences and look-around assertions are not supported.\n\nFor more information, please refer to Russ Cox's articles on regular expression theory and practice:\n\n* [Regular Expression Matching Can Be Simple And Fast](https://swtch.com/~rsc/regexp/regexp1.html)\n* [Regular Expression Matching: the Virtual Machine Approach](https://swtch.com/~rsc/regexp/regexp2.html)\n* [Regular Expression Matching in the Wild](https://swtch.com/~rsc/regexp/regexp3.html)\n\n### Syntax\n\nIn POSIX mode, RE2 accepts standard POSIX (egrep) syntax regular expressions.\nIn Perl mode, RE2 accepts most Perl operators. The only excluded ones are\nthose that require backtracking (and its potential for exponential runtime)\nto implement. These include backreferences (submatching is still okay)\nand generalized assertions.\nThe [Syntax wiki page](https://github.com/google/re2/wiki/Syntax)\ndocuments the supported Perl-mode syntax in detail.\nThe default is Perl mode.\n\n### C++ API\n\nRE2's native language is C++, although there are [ports and wrappers](#ports-and-wrappers) listed below.\n\n#### Matching Interface\n\nThere are two basic operators:\n`RE2::FullMatch` requires the regexp to match the entire input text, and\n`RE2::PartialMatch` looks for a match for a substring of the input text,\nreturning the leftmost-longest match in POSIX mode and the\nsame match that Perl would have chosen in Perl mode.\n\nExamples:\n\n```cpp\nassert(RE2::FullMatch(\"hello\", \"h.*o\"))\nassert(!RE2::FullMatch(\"hello\", \"e\"))\n\nassert(RE2::PartialMatch(\"hello\", \"h.*o\"))\nassert(RE2::PartialMatch(\"hello\", \"e\"))\n```\n\n#### Submatch Extraction\n\nBoth matching functions take additional arguments in which submatches will be stored.\nThe argument can be a `string*`, or an integer type, or the type `absl::string_view*`.\n(The `absl::string_view` type is very similar to the `std::string_view` type,\nbut for historical reasons, RE2 uses the former.)\nA `string_view` is a pointer to the original input text, along with a count.\nIt behaves like a string but doesn't carry its own storage.\nLike when using a pointer, when using a `string_view`\nyou must be careful not to use it once the original text has been deleted or gone out of scope.\n\nExamples:\n\n```cpp\n// Successful parsing.\nint i;\nstring s;\nassert(RE2::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", &s, &i));\nassert(s == \"ruby\");\nassert(i == 1234);\n\n// Fails: \"ruby\" cannot be parsed as an integer.\nassert(!RE2::FullMatch(\"ruby\", \"(.+)\", &i));\n\n// Success; does not extract the number.\nassert(RE2::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", &s));\n\n// Success; skips NULL argument.\nassert(RE2::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", (void*)NULL, &i));\n\n// Fails: integer overflow keeps value from being stored in i.\nassert(!RE2::FullMatch(\"ruby:123456789123\", \"(\\\\w+):(\\\\d+)\", &s, &i));\n```\n\n#### Pre-Compiled Regular Expressions\n\nThe examples above all recompile the regular expression on each call.\nInstead, you can compile it once to an RE2 object and reuse that object for each call.\n\nExample:\n```cpp\nRE2 re(\"(\\\\w+):(\\\\d+)\");\nassert(re.ok()); // compiled; if not, see re.error();\n\nassert(RE2::FullMatch(\"ruby:1234\", re, &s, &i));\nassert(RE2::FullMatch(\"ruby:1234\", re, &s));\nassert(RE2::FullMatch(\"ruby:1234\", re, (void*)NULL, &i));\nassert(!RE2::FullMatch(\"ruby:123456789123\", re, &s, &i));\n```\n\n#### Options\n\nThe constructor takes an optional second argument that can\nbe used to change RE2's default options.\nFor example, `RE2::Quiet` silences the error messages that are\nusually printed when a regular expression fails to parse:\n\n```cpp\nRE2 re(\"(ab\", RE2::Quiet); // don't write to stderr for parser failure\nassert(!re.ok()); // can check re.error() for details\n```\n\nOther useful predefined options are `Latin1` (disable UTF-8) and `POSIX`\n(use POSIX syntax and leftmost longest matching).\n\nYou can also declare your own `RE2::Options` object and then configure it as you like.\nSee the [header](https://github.com/google/re2/blob/main/re2/re2.h) for the full set of options.\n\n#### Unicode Normalization\n\nRE2 operates on Unicode code points: it makes no attempt at normalization.\nFor example, the regular expression /ü/ (U+00FC, u with diaeresis)\ndoes not match the input \"ü\" (U+0075 U+0308, u followed by combining diaeresis).\nNormalization is a long, involved topic.\nThe simplest solution, if you need such matches, is to normalize both the regular expressions\nand the input in a preprocessing step before using RE2.\nFor more details on the general topic, see .\n\n#### Additional Tips and Tricks\n\nFor advanced usage, like constructing your own argument lists,\nor using RE2 as a lexer, or parsing hex, octal, and C-radix numbers,\nsee [re2.h](https://github.com/google/re2/blob/main/re2/re2.h).\n\n### Installation\n\nRE2 can be built and installed using GNU make, CMake, or Bazel.\nThe simplest installation instructions are:\n\n\tmake\n\tmake test\n\tmake benchmark\n\tmake install\n\tmake testinstall\n\nBuilding RE2 requires a C++17 compiler and the [Abseil](https://github.com/abseil/abseil-cpp) library.\nBuilding the tests and benchmarks requires\n[GoogleTest](https://github.com/google/googletest)\nand [Benchmark](https://github.com/google/benchmark).\nTo obtain those:\n\n- Linux: `apt install libabsl-dev libgtest-dev libbenchmark-dev`\n- macOS: `brew install abseil googletest google-benchmark pkg-config-wrapper`\n- Windows: `vcpkg install abseil gtest benchmark` \\\n or `vcpkg add port abseil gtest benchmark`\n\nOnce those are installed, the build has to be able to find them.\nIf the standard Makefile has trouble, then switching to CMake can help:\n\n\trm -rf build\n\tcmake -DRE2_TEST=ON -DRE2_BENCHMARK=ON -S . -B build\n\tcd build\n\tmake\n\tmake test\n\tmake install\n\nWhen using CMake, with benchmarks enabled, `make test` builds and runs test binaries\nand builds a `regexp_benchmark` binary but does not run it.\nIf you don't need the tests or benchmarks at all, you can omit the corresponding `-D` arguments,\nand then you don't need the GoogleTest or Benchmark dependencies either.\n\nAnother useful option is `-DRE2_USE_ICU=ON`, which adds a dependency on the\nICU Unicode library but also extends the list of property names available in the `\\p` and `\\P` patterns.\n\nCMake can also be used to generate Visual Studio and Xcode projects, as well as\nCygwin, MinGW, and MSYS makefiles.\n\n - Visual Studio users: You need Visual Studio 2019 or later.\n - Cygwin users: You must run CMake from the Cygwin command line, not the Windows command line.\n\nIf you are adding RE2 to your own CMake project,\nCMake has two ways to use a dependency: `add_subdirectory()`,\nwhich is when the dependency's **_sources_** are in a subdirectory of your project;\nand `find_package()`, which is when the dependency's\n**_binaries_** have been built and installed somewhere on your system.\nThe Abseil documentation walks through the former [here](https://abseil.io/docs/cpp/quickstart-cmake)\nversus the latter [here](https://abseil.io/docs/cpp/tools/cmake-installs).\nOnce you get Abseil working, getting RE2 working will be a very similar process and,\neither way, `target_link_libraries(… re2::re2)` should Just Work™.\n\nIf you are using [Bazel](https://bazel.io), it will handle the dependencies for you,\nalthough you still need to download Bazel,\nwhich you can do with [Bazelisk](https://github.com/bazelbuild/bazelisk).\n\n\tgo install github.com/bazelbuild/bazelisk@latest\n\t# or on mac: brew install bazelisk\n\n\tbazelisk build :all\n\tbazelisk test :all\n\nIf you are using RE2 from another project, you need to make sure you are\nusing at least C++17.\nSee the RE2 [.bazelrc](https://github.com/google/re2/blob/main/.bazelrc) file for an example.\n\n### Ports and Wrappers\n\nRE2 is implemented in C++.\n\nThe official Python wrapper is [in the `python` directory](https://github.com/google/re2/tree/main/python)\nand [published on PyPI as `google-re2`](https://pypi.org/project/google-re2/).\nNote that there is also a PyPI `re2` but it is not by the RE2 authors and is unmaintained. Use `google-re2`.\n\nThere are also other unofficial wrappers:\n\n- A C wrapper is at .\n- A D wrapper is at and [on DUB](https://code.dlang.org/packages/re2d).\n- An Erlang wrapper is at and [on Hex](https://hex.pm/packages/re2).\n- An Inferno wrapper is at .\n- A Node.js wrapper is at and [on NPM](https://www.npmjs.com/package/re2).\n- An OCaml wrapper is at and [on OPAM](https://opam.ocaml.org/packages/re2/).\n- A Perl wrapper is at and [on CPAN](https://metacpan.org/pod/re::engine::RE2).\n- An R wrapper is at and [on CRAN](https://cran.r-project.org/web/packages/re2/index.html).\n- A Ruby wrapper is at and on RubyGems (rubygems.org).\n- A WebAssembly wrapper is at and on NPM (npmjs.com).\n\n[RE2J](https://github.com/google/re2j) is a port of the RE2 C++ code to pure Java,\nand [RE2JS](https://github.com/le0pard/re2js) is a port of RE2J to JavaScript.\n\nThe [Go `regexp` package](https://go.dev/pkg/regexp)\nand [Rust `regex` crate](https://docs.rs/regex)\ndo not share code with RE2, but they follow the same principles,\naccept the same syntax, and provide the same efficiency guarantees.\n\n### Contact\n\nThe [issue tracker](https://github.com/google/re2/issues) is the best place for discussions.\n\nThere is a [mailing list](https://groups.google.com/group/re2-dev) for keeping up with code changes.\n\nPlease read the [contribution guide](https://github.com/google/re2/wiki/Contribute) before sending changes.\nIn particular, note that RE2 does not use GitHub pull requests.\n" }, { "path": "SECURITY.md", "content": "To report a security issue, please use https://g.co/vulnz. We use\nhttps://g.co/vulnz for our intake, and do coordination and disclosure here on\nGitHub (including using GitHub Security Advisory). The Google Security Team will\nrespond within 5 working days of your report on https://g.co/vulnz.\n" }, { "path": "WORKSPACE.bazel", "content": "# Copyright 2009 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# Bazel (http://bazel.build/) WORKSPACE file for RE2.\n\nworkspace(name = \"com_googlesource_code_re2\")\n" }, { "path": "WORKSPACE.bzlmod", "content": "# Copyright 2009 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# Bazel (http://bazel.build/) WORKSPACE file for RE2.\n\nworkspace(name = \"com_googlesource_code_re2\")\n" }, { "path": "app/BUILD.bazel", "content": "# Copyright 2009 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# Bazel (http://bazel.build/) BUILD file for RE2 app.\n\ncc_binary(\n name = \"_re2.js\",\n testonly = 1,\n srcs = [\"_re2.cc\"],\n linkopts = [\n \"--bind\",\n \"-sENVIRONMENT=web\",\n \"-sSINGLE_FILE=1\",\n \"-sMODULARIZE=1\",\n \"-sEXPORT_ES6=1\",\n \"-sEXPORT_NAME=loadModule\",\n \"-sUSE_PTHREADS=0\",\n ],\n deps = [\n \"//:re2\",\n \"//:testing\",\n ],\n)\n" }, { "path": "app/_re2.cc", "content": "// Copyright 2022 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \n#include \n\n#include \n#include \"re2/prog.h\"\n#include \"re2/re2.h\"\n#include \"re2/regexp.h\"\n\nnamespace re2_app {\n\nstruct Info {\n std::string pattern;\n std::string error;\n std::string prefix;\n bool prefix_foldcase = false;\n std::string accel_prefix;\n bool accel_prefix_foldcase = false;\n int num_captures;\n bool is_one_pass;\n bool can_bit_state;\n std::string bytecode;\n std::string bytemap;\n};\n\nInfo GetInfo(const std::string& pattern) {\n Info info;\n info.pattern = pattern;\n\n RE2::Options options;\n re2::RegexpStatus status;\n re2::Regexp* regexp = re2::Regexp::Parse(\n pattern, static_cast(options.ParseFlags()),\n &status);\n if (regexp == nullptr) {\n info.error = \"failed to parse pattern: \" + status.Text();\n return info;\n }\n\n std::string prefix;\n bool prefix_foldcase;\n re2::Regexp* suffix;\n if (regexp->RequiredPrefix(&prefix, &prefix_foldcase, &suffix)) {\n info.prefix = prefix;\n info.prefix_foldcase = prefix_foldcase;\n } else {\n suffix = regexp->Incref();\n }\n\n std::unique_ptr prog(suffix->CompileToProg(options.max_mem()));\n if (prog == nullptr) {\n info.error = \"failed to compile forward Prog\";\n suffix->Decref();\n regexp->Decref();\n return info;\n }\n\n if (regexp->RequiredPrefixForAccel(&prefix, &prefix_foldcase)) {\n info.accel_prefix = prefix;\n info.accel_prefix_foldcase = prefix_foldcase;\n }\n\n info.num_captures = suffix->NumCaptures();\n info.is_one_pass = prog->IsOnePass();\n info.can_bit_state = prog->CanBitState();\n info.bytecode = prog->Dump();\n info.bytemap = prog->DumpByteMap();\n\n suffix->Decref();\n regexp->Decref();\n return info;\n}\n\nEMSCRIPTEN_BINDINGS(_re2) {\n emscripten::value_object(\"Info\")\n .field(\"pattern\", &Info::pattern)\n .field(\"error\", &Info::error)\n .field(\"prefix\", &Info::prefix)\n .field(\"prefix_foldcase\", &Info::prefix_foldcase)\n .field(\"accel_prefix\", &Info::accel_prefix)\n .field(\"accel_prefix_foldcase\", &Info::accel_prefix_foldcase)\n .field(\"num_captures\", &Info::num_captures)\n .field(\"is_one_pass\", &Info::is_one_pass)\n .field(\"can_bit_state\", &Info::can_bit_state)\n .field(\"bytecode\", &Info::bytecode)\n .field(\"bytemap\", &Info::bytemap);\n\n emscripten::function(\"getInfo\", &GetInfo);\n}\n\n} // namespace re2_app\n" }, { "path": "app/_re2.d.ts", "content": "// Copyright 2022 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\nexport type Info = {\n pattern: ArrayBuffer|Uint8Array|Uint8ClampedArray|Int8Array|string,\n error: ArrayBuffer|Uint8Array|Uint8ClampedArray|Int8Array|string,\n prefix: ArrayBuffer|Uint8Array|Uint8ClampedArray|Int8Array|string,\n prefix_foldcase: boolean,\n accel_prefix: ArrayBuffer|Uint8Array|Uint8ClampedArray|Int8Array|string,\n accel_prefix_foldcase: boolean,\n num_captures: number,\n is_one_pass: boolean,\n can_bit_state: boolean,\n bytecode: ArrayBuffer|Uint8Array|Uint8ClampedArray|Int8Array|string,\n bytemap: ArrayBuffer|Uint8Array|Uint8ClampedArray|Int8Array|string,\n};\n\nexport interface MainModule {\n getInfo(pattern: ArrayBuffer|Uint8Array|Uint8ClampedArray|Int8Array|string): Info;\n}\n\nexport default function loadModule(): Promise;\n" }, { "path": "app/app.ts", "content": "// Copyright 2022 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\nimport {css, html, LitElement, render} from 'lit';\nimport {customElement} from 'lit/decorators.js';\n\nimport /*default*/ loadModule from './_re2';\nimport {Info, MainModule} from './_re2';\n\nvar _re2: MainModule;\nloadModule().then((module: MainModule) => {\n _re2 = module;\n render(html`re2-dev`, document.body);\n});\n\n@customElement('re2-dev')\nexport class RE2Dev extends LitElement {\n private _pattern: string = '';\n private _info: Info|null = null;\n\n constructor() {\n super();\n this._pattern = decodeURIComponent(window.location.hash.slice(1));\n this._info = this._pattern ? _re2.getInfo(this._pattern) : null;\n this.requestUpdate();\n }\n\n private _onChange = (e: Event) => {\n this._pattern = (e.target as HTMLInputElement).value;\n this._info = this._pattern ? _re2.getInfo(this._pattern) : null;\n this.requestUpdate();\n window.location.hash = '#' + encodeURIComponent(this._pattern);\n };\n\n static override styles = css`\n.code {\n font-family: monospace;\n white-space: pre-line;\n}\n`;\n\n override render() {\n var fragments = [];\n fragments.push(html`\n
\n \n
\n`);\n\n if (this._info === null) {\n return html`${fragments}`;\n }\n\n if (this._info.error) {\n fragments.push(html`\n
\n
\n error:\n ${this._info.error}\n
\n`);\n return html`${fragments}`;\n }\n\n fragments.push(html`\n
\n
\n pattern:\n ${this._info.pattern}\n
\n prefix:\n ${this._info.prefix}\n ·\n _foldcase:\n ${this._info.prefix_foldcase}\n
\n accel_prefix:\n ${this._info.accel_prefix}\n ·\n _foldcase:\n ${this._info.accel_prefix_foldcase}\n
\n num_captures:\n ${this._info.num_captures}\n
\n is_one_pass:\n ${this._info.is_one_pass}\n
\n can_bit_state:\n ${this._info.can_bit_state}\n
\n
\n bytecode:\n
\n ${this._info.bytecode}\n
\n bytemap:\n
\n ${this._info.bytemap}\n
\n`);\n return html`${fragments}`;\n }\n}\n\ndeclare global {\n interface HTMLElementTagNameMap {\n 're2-dev': RE2Dev;\n }\n}\n" }, { "path": "app/build.sh", "content": "#!/bin/bash\nset -eux\n\nSRCDIR=$(readlink --canonicalize $(dirname $0))\nDSTDIR=$(mktemp --directory --tmpdir $(basename $0).XXXXXXXXXX)\n\ncd ${SRCDIR}\n# Emscripten doesn't support `-fstack-protector`.\nAR=emar CC=emcc \\\n bazel build \\\n --copt=-fno-stack-protector \\\n --compilation_mode=opt -- :all\ncp ../bazel-bin/app/_re2.js ${DSTDIR}\nbazel clean --expunge\ncp app.ts index.html _re2.d.ts ${DSTDIR}\ncp package.json rollup.config.js tsconfig.json ${DSTDIR}\n\ncd ${DSTDIR}\nnpm install\nnpx tsc\nnpx rollup -c rollup.config.js -d deploy\n\ncd ${SRCDIR}\nmkdir deploy\ncat >deploy/index.html <\nEOF\nmkdir deploy/app\ncp ${DSTDIR}/deploy/* deploy/app\nls -lR deploy\n\nexit 0\n" }, { "path": "app/index.html", "content": "\n\n\n\n\n" }, { "path": "app/package.json", "content": "{\n \"dependencies\": {\n \"lit\": \"*\"\n },\n \"devDependencies\": {\n \"@rollup/plugin-node-resolve\": \"*\",\n \"@rollup/plugin-terser\": \"*\",\n \"@web/rollup-plugin-html\": \"*\",\n \"@web/rollup-plugin-import-meta-assets\": \"*\",\n \"rollup\": \"~2\",\n \"tslib\": \"*\",\n \"typescript\": \"*\"\n }\n}\n" }, { "path": "app/rollup.config.js", "content": "// Copyright 2022 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\nimport nodeResolve from '@rollup/plugin-node-resolve';\nimport terser from '@rollup/plugin-terser';\nimport html from '@web/rollup-plugin-html';\nimport {importMetaAssets} from '@web/rollup-plugin-import-meta-assets';\n\nexport default {\n input: 'index.html',\n output: {\n entryFileNames: '[hash].js',\n chunkFileNames: '[hash].js',\n assetFileNames: '[hash][extname]',\n format: 'es',\n },\n preserveEntrySignatures: false,\n plugins:\n [\n html({\n minify: true,\n }),\n nodeResolve(),\n terser(),\n importMetaAssets(),\n ],\n};\n" }, { "path": "app/tsconfig.json", "content": "{\n \"compilerOptions\": {\n \"target\": \"esnext\",\n \"module\": \"esnext\",\n \"moduleResolution\": \"node\",\n \"noEmitOnError\": true,\n \"lib\": [\"esnext\", \"dom\"],\n \"strict\": true,\n \"esModuleInterop\": false,\n \"allowSyntheticDefaultImports\": true,\n \"experimentalDecorators\": true,\n \"importHelpers\": true,\n \"sourceMap\": true,\n \"inlineSources\": true,\n \"incremental\": true\n }\n}\n" }, { "path": "benchlog/benchplot.py", "content": "#!/usr/bin/env python\n\nimport argparse # for ArgumentParser\nimport subprocess # for Popen\nimport tempfile # for NamedTemporaryFile\nimport os # for remove\n\nclass gnuplot(object):\n\n output = \"result.png\"\n\n script = \"\"\"\n set terminal png size 1024, 768\n set output \"{}.png\"\n set title \"re2 benchlog\"\n set datafile separator \";\"\n set grid x y\n set ylabel \"MB/s\"\n set autoscale\n plot \"\"\"\n\n template = \"\"\"'{}' using 1:5:xticlabels(2) with linespoints linewidth 3 title \"{}\",\\\\\\n\"\"\"\n\n benchdata = dict()\n tempfiles = []\n\n def __enter__(self):\n return self\n\n def __exit__(self, type, value, traceback):\n \"\"\"\n remove all temporary files\n \"\"\"\n\n for filename in self.tempfiles:\n os.remove(filename)\n\n def parse_re2_benchlog(self, filename):\n \"\"\"\n parse the input benchlog and return a dictionary contain bench data\n \"\"\"\n\n benchdata = self.benchdata\n\n with open(filename) as f:\n\n for raw in f.readlines():\n\n data = raw.split('\\t')\n\n if len(data) == 4:\n\n data = data[0].split('/') + data[1:]\n data = list(map(str.strip, data))\n\n if not benchdata.get(data[0]):\n benchdata[data[0]] = [ data[1:] ]\n else:\n benchdata[data[0]].append(data[1:])\n\n def gen_csv(self):\n \"\"\"\n generate temporary csv files\n \"\"\"\n\n for name, data in self.benchdata.items():\n\n with tempfile.NamedTemporaryFile(delete=False) as f:\n\n for index, line in enumerate(data):\n f.write('{};{}\\n'.format(index, ';'.join(line)).encode())\n\n self.tempfiles.append(f.name)\n self.script = self.script + self.template.format(f.name, name)\n\n def run(self):\n self.gen_csv()\n script = self.script[:-3].format(self.output)\n command = subprocess.Popen(['gnuplot'], stdin=subprocess.PIPE)\n command.communicate(script.encode())\n\n\nif __name__ == '__main__':\n\n parser = argparse.ArgumentParser(description='generate plots for benchlog')\n parser.add_argument('benchlog', type=str, help='benchlog generated by re2')\n args = parser.parse_args()\n\n try:\n subprocess.Popen(['gnuplot'], stdin=subprocess.PIPE)\n except FileNotFoundError:\n print('you can install \"gnuplot\" to generate plots automatically')\n exit(1)\n\n with gnuplot() as plot:\n plot.output = args.benchlog\n plot.parse_re2_benchlog(args.benchlog)\n plot.run()\n" }, { "path": "benchlog/mktable", "content": "#!/usr/bin/perl\n# XXX\n\nsub table() {\n\tmy ($name) = @_;\n\tprint <<'EOF';\n\n\nEOF\n\tforeach my $sys (@sys) {\n\t\tmy $ns_pcre = $data{$sys}->{sprintf($name, \"PCRE\")}->{'ns/op'};\n\t\tmy $ns_re2 = $data{$sys}->{sprintf($name, \"RE2\")}->{'ns/op'};\n\t\tprintf \"\\n\", $sysname{$sys}, $ns_pcre/1000., $ns_re2/1000.;\n\t}\n\tprint <<'EOF';\n\n
SystemPCRERE2
%s%.1f µs%.1f µs
\nEOF\n}\n\n@sizes = (\n\t\"8\", \"16\", \"32\", \"64\", \"128\", \"256\", \"512\",\n\t\"1K\", \"2K\", \"4K\", \"8K\", \"16K\", \"32K\", \"64K\", \"128K\", \"256K\", \"512K\",\n\t\"1M\", \"2M\", \"4M\", \"8M\", \"16M\"\n);\n\n%color = (\n\t\"PCRE\" => \"0.7 0 0\",\n\t\"RE2\" => \"0 0 1\",\n);\n\n$ngraph = 0;\n\nsub graph() {\n\tmy ($name) = @_;\n\t\n\tmy $sys = \"wreck\";\n\tmy $base = sprintf(\"regexp3g%d\", ++$ngraph);\n\n\topen(JGR, \">$base.jgr\") || die \"open >$base.jgr: $!\";\n\tprintf JGR \"bbox -20 -12 392 95\\n\";\n\tprintf JGR \"newgraph clip x_translate 0.25 y_translate 0.25\\n\";\n\t$ymax = 0;\n\t%lastx = ();\n\t%lasty = ();\n\tforeach my $who (\"PCRE\", \"RE2\") {\n\t\tprintf JGR \"newcurve pts\\n\";\n\t\tfor(my $i=0; $i<@sizes; $i++) {\n\t\t\tmy $key = sprintf(\"%s%s/%s\", $name, $who, $sizes[$i]);\n\t\t\tmy $val = $data{$sys}->{$key}->{'MB/s'};\n\t\t\tnext if !defined($val);\n\t\t\tif($val > $ymax) {\n\t\t\t\t$ymax = $val;\n\t\t\t}\n\t\t\t$lastx{$who} = $i;\n\t\t\t$lasty{$who} = $val;\n\t\t\tprintf JGR \"$i %f (* %s *)\\n\", $val, $key;\n\t\t}\n\t\tmy $color = $color{$who};\n\t\tprintf JGR \"marktype none color $color linethickness 2 linetype solid label : $who\\n\";\n\t}\n\tmy $n = @sizes;\n\tprintf JGR \"xaxis min -1 max $n size 5 label : text size (bytes)\\n\";\n\tprintf JGR \" no_auto_hash_marks hash_labels fontsize 9\\n\";\n\tfor($i=0; $i<@sizes; $i+=3) {\n\t\tprintf JGR \" hash_at $i hash_label at $i : $sizes[$i]\\n\";\n\t}\n\tmy $y = 1;\n\twhile(10*$y <= $ymax) {\n\t\t$y = 10*$y;\n\t}\n\tfor($i=2; $i<=10; $i++) {\n\t\tif($i*$y > $ymax) {\n\t\t\t$y = $i*$y;\n\t\t\tlast;\n\t\t}\n\t}\n\tforeach my $who (\"PCRE\", \"RE2\") {\n\t\t$x1 = $lastx{$who};\n\t\t$y1 = $lasty{$who};\n\t\t$x1 *= 1.01;\n\t\tmy $v = \"vjc\";\n\t\tif($y1 < 0.05 * $y) {\n\t\t\t$v = \"vjb\";\n\t\t\t$y1 = 0.05 * $y;\n\t\t}\n\t\tprintf JGR \"newstring x $x1 y $y1 hjl $v : $who\\n\";\n\t}\n\tprintf JGR \"yaxis min 0 max $y size 1 label : speed (MB/s)\\n\";\n\tprintf JGR \" hash_labels fontsize 9\\n\";\n\t# printf JGR \"legend defaults font Times-Roman fontsize 10 x 0 y $y hjl vjt\\n\";\n\n\tsystem(\"jgraph $base.jgr >$base.eps\"); # die \"system: $!\";\n\tsystem(\"gs -dTextAlphaBits=4 -dGraphicsAlphaBits=4 -dEPSCrop -sDEVICE=png16m -r100 -sOutputFile=$base.png -dBATCH -dQUIT -dQUIET -dNOPAUSE $base.eps\");\n\t\n\tprintf \"\\n\"\n\t\n}\n\nsub skip() {\n\twhile(<>) {\n\t\tif(/^/) {\n\t\t\tprint;\n\t\t\tlast;\n\t\t}\n\t}\n}\n\n@sys = (\"r70\", \"c2\", \"wreck\", \"mini\");\n%sysname = (\n\t\"r70\" => \"AMD Opteron 8214 HE, 2.2 GHz\",\n\t\"c2\" => \"Intel Core2 Duo E7200, 2.53 GHz\",\n\t\"wreck\" => \"Intel Xeon 5150, 2.66 GHz (Mac Pro)\",\n\t\"mini\" => \"Intel Core2 T5600, 1.83 GHz (Mac Mini)\",\n);\n\n%func = (\n\t\"table\" => \\&table,\n\t\"graph\" => \\&graph,\n\t\n);\n\nforeach my $sys (@sys) {\n\topen(F, \"benchlog.$sys\") || die \"open benchlog.$sys: $!\";\n\tmy %sysdat;\n\twhile() {\n\t\tif(/^([A-Za-z0-9_\\/]+)\\s+(\\d+)\\s+(\\d+) ns\\/op/) {\n\t\t\tmy %row;\n\t\t\t$row{\"name\"} = $1;\n\t\t\t$row{\"iter\"} = $2;\n\t\t\t$row{\"ns/op\"} = $3;\n\t\t\tif(/([\\d.]+) MB\\/s/){\n\t\t\t\t$row{\"MB/s\"} = $1;\n\t\t\t}\n\t\t\t$sysdat{$row{\"name\"}} = \\%row;\n\t\t}\n\t}\n\tclose F;\t\n\t$data{$sys} = \\%sysdat;\n}\n\nwhile(<>) {\n\tprint;\n\tif(/^/) {\n\t\t$func{$1}();\n\t\tskip();\n\t\tnext;\n\t}\n\tif(/^/) {\n\t\t$func{$1}($2);\n\t\tskip();\n\t\tnext;\n\t}\n}\n\n" }, { "path": "doc/mksyntaxgo", "content": "#!/bin/sh\n\nset -e\nout=$GOROOT/src/regexp/syntax/doc.go\ncp syntax.txt $out\nsam -d $out <<'!'\n,x g/NOT SUPPORTED/d\n/^Unicode character class/,$d\n,s/[«»]//g\n,x g/^Possessive repetitions:/d\n,x g/\\\\C/d\n,x g/Flag syntax/d\n,s/.=(true|false)/flag &/g\n,s/^Flags:/ Flag syntax is xyz (set) or -xyz (clear) or xy-z (set xy, clear z). The flags are:\\n/\n,s/\\n\\n\\n+/\\n\\n/g\n,x/(^.*\t.*\\n)+/ | awk -F'\t' '{printf(\" %-14s %s\\n\", $1, $2)}'\n1,2c\n// Copyright 2012 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Code generated by mksyntaxgo from the RE2 distribution. DO NOT EDIT.\n\n/*\nPackage syntax parses regular expressions into parse trees and compiles\nparse trees into programs. Most clients of regular expressions will use the\nfacilities of package [regexp] (such as [regexp.Compile] and [regexp.Match]) instead of this package.\n\n# Syntax\n\nThe regular expression syntax understood by this package when parsing with the [Perl] flag is as follows.\nParts of the syntax can be disabled by passing alternate flags to [Parse].\n\n.\n$a\nUnicode character classes are those in [unicode.Categories] and [unicode.Scripts].\n*/\npackage syntax\n.\nw\nq\n!\n" }, { "path": "doc/mksyntaxhtml", "content": "#!/bin/sh\n\ncp syntax.txt syntax.html\nsam -d syntax.html <<'!'\n,s/\\&/\\&/g\n,s//\\>/g\n,s!== (([^()]|\\([^()]*\\))*)!≡ \\1!g\n,s!«!!g\n,s!»!!g\n,s! vim$! VIM!g\n,s! pcre$! PCRE!g\n,s! perl$! PERL!g\n,x g/NOT SUPPORTED/ s!^[^\t]+!&!\n,s!NOT SUPPORTED!!g\n,s!(^[^\t]+)\t(.*)\\n!\\1\\2\\n!g\n,s!.*:$!&!g\n,s!^$!!g\n,x v// s!.*!&!\n1,2c\n\n\n\n\nRE2 regular expression syntax reference\n\n\n

RE2 regular expression syntax reference

\n\n\n\n\n\n.\n$a\n
This page lists the regular expression syntax accepted by RE2.
It also lists syntax accepted by PCRE, PERL, and VIM.
Grayed out expressions are not supported by RE2.
\n\n\n.\nw\nq\n!\n" }, { "path": "doc/mksyntaxwiki", "content": "#!/bin/sh\n\ncp syntax.txt syntax.wiki\nsam -d syntax.wiki <<'!'\n,s!`!`````!g\n,s!== (([^()]|\\([^()]*\\))*)!≡ `\\1`!g\n,s!«!`!g\n,s!»!`!g\n,s! vim$! VIM!g\n,s! pcre$! PCRE!g\n,s! perl$! PERL!g\n,s!(^[^\t]+)\t(.*)\\n!`\\1`\t\\2\\n!g\n,x g/NOT SUPPORTED/ s!^[^\t]+!&!\n,s!NOT SUPPORTED!(&)!g\n,s!(^[^\t]+)\t(.*)\\n!\\1\\2\\n!g\n,s!.*:$!&!g\n,s!^$!!g\n,x v// s!.*!&!\n1,2c\n#summary I define UNIX as “30 definitions of regular expressions living under one roof.” —Don Knuth\n\n\nGENERATED BY mksyntaxwiki. DO NOT EDIT\n\n\n\n\n\n\n.\n$a\n
This page lists the regular expression syntax accepted by RE2.
It also lists syntax accepted by PCRE, PERL, and VIM.
Grayed out expressions are not supported by RE2.
\n.\nw\nq\n!\n" }, { "path": "doc/syntax.html", "content": "\n\n\n\nRE2 regular expression syntax reference\n\n\n

RE2 regular expression syntax reference

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
This page lists the regular expression syntax accepted by RE2.
It also lists syntax accepted by PCRE, PERL, and VIM.
Grayed out expressions are not supported by RE2.
Single characters:
.any character, possibly including newline (s=true)
[xyz]character class
[^xyz]negated character class
\\dPerl character class
\\Dnegated Perl character class
[[:alpha:]]ASCII character class
[[:^alpha:]]negated ASCII character class
\\pNUnicode character class (one-letter name)
\\p{Greek}Unicode character class
\\PNnegated Unicode character class (one-letter name)
\\P{Greek}negated Unicode character class
Composites:
xyx followed by y
x|yx or y (prefer x)
Repetitions:
x*zero or more x, prefer more
x+one or more x, prefer more
x?zero or one x, prefer one
x{n,m}n or n+1 or ... or m x, prefer more
x{n,}n or more x, prefer more
x{n}exactly n x
x*?zero or more x, prefer fewer
x+?one or more x, prefer fewer
x??zero or one x, prefer zero
x{n,m}?n or n+1 or ... or m x, prefer fewer
x{n,}?n or more x, prefer fewer
x{n}?exactly n x
x{}(≡ x*) VIM
x{-}(≡ x*?) VIM
x{-n}(≡ x{n}?) VIM
x=(≡ x?) VIM
Implementation restriction: The counting forms x{n,m}, x{n,}, and x{n}
reject forms that create a minimum or maximum repetition count above 1000.
Unlimited repetitions are not subject to this restriction.
Possessive repetitions:
x*+zero or more x, possessive
x++one or more x, possessive
x?+zero or one x, possessive
x{n,m}+n or ... or m x, possessive
x{n,}+n or more x, possessive
x{n}+exactly n x, possessive
Grouping:
(re)numbered capturing group (submatch)
(?P<name>re)named & numbered capturing group (submatch)
(?<name>re)named & numbered capturing group (submatch)
(?'name're)named & numbered capturing group (submatch)
(?:re)non-capturing group
(?flags)set flags within current group; non-capturing
(?flags:re)set flags during re; non-capturing
(?#text)comment
(?|x|y|z)branch numbering reset
(?>re)possessive match of re
re@>possessive match of re VIM
%(re)non-capturing group VIM
Flags:
icase-insensitive (default false)
mmulti-line mode: ^ and $ match begin/end line in addition to begin/end text (default false)
slet . match \\n (default false)
Uungreedy: swap meaning of x* and x*?, x+ and x+?, etc (default false)
Flag syntax is xyz (set) or -xyz (clear) or xy-z (set xy, clear z).
Empty strings:
^at beginning of text or line (m=true)
$at end of text (like \\z not \\Z) or line (m=true)
\\Aat beginning of text
\\bat ASCII word boundary (\\w on one side and \\W, \\A, or \\z on the other)
\\Bnot at ASCII word boundary
\\Gat beginning of subtext being searched PCRE
\\Gat end of last match PERL
\\Zat end of text, or before newline at end of text
\\zat end of text
(?=re)before text matching re
(?!re)before text not matching re
(?<=re)after text matching re
(?<!re)after text not matching re
re&before text matching re VIM
re@=before text matching re VIM
re@!before text not matching re VIM
re@<=after text matching re VIM
re@<!after text not matching re VIM
\\zssets start of match (= \\K) VIM
\\zesets end of match VIM
\\%^beginning of file VIM
\\%$end of file VIM
\\%Von screen VIM
\\%#cursor position VIM
\\%'mmark m position VIM
\\%23lin line 23 VIM
\\%23cin column 23 VIM
\\%23vin virtual column 23 VIM
Escape sequences:
\\abell (≡ \\007)
\\fform feed (≡ \\014)
\\thorizontal tab (≡ \\011)
\\nnewline (≡ \\012)
\\rcarriage return (≡ \\015)
\\vvertical tab character (≡ \\013)
\\*literal *, for any punctuation character *
\\123octal character code (up to three digits)
\\x7Fhex character code (exactly two digits)
\\x{10FFFF}hex character code
\\Cmatch a single byte even in UTF-8 mode
\\Q...\\Eliteral text ... even if ... has punctuation
\\1backreference
\\bbackspace (use \\010)
\\cKcontrol char ^K (use \\001 etc)
\\eescape (use \\033)
\\g1backreference
\\g{1}backreference
\\g{+1}backreference
\\g{-1}backreference
\\g{name}named backreference
\\g<name>subroutine call
\\g'name'subroutine call
\\k<name>named backreference
\\k'name'named backreference
\\lXlowercase X
\\uxuppercase x
\\L...\\Elowercase text ...
\\Kreset beginning of $0
\\N{name}named Unicode character
\\Rline break
\\U...\\Eupper case text ...
\\Xextended Unicode sequence
\\%d123decimal character 123 VIM
\\%xFFhex character FF VIM
\\%o123octal character 123 VIM
\\%u1234Unicode character 0x1234 VIM
\\%U12345678Unicode character 0x12345678 VIM
Character class elements:
xsingle character
A-Zcharacter range (inclusive)
\\dPerl character class
[:foo:]ASCII character class foo
\\p{Foo}Unicode character class Foo
\\pFUnicode character class F (one-letter name)
Named character classes as character class elements:
[\\d]digits (≡ \\d)
[^\\d]not digits (≡ \\D)
[\\D]not digits (≡ \\D)
[^\\D]not not digits (≡ \\d)
[[:name:]]named ASCII class inside character class (≡ [:name:])
[^[:name:]]named ASCII class inside negated character class (≡ [:^name:])
[\\p{Name}]named Unicode property inside character class (≡ \\p{Name})
[^\\p{Name}]named Unicode property inside negated character class (≡ \\P{Name})
Perl character classes (all ASCII-only):
\\ddigits (≡ [0-9])
\\Dnot digits (≡ [^0-9])
\\swhitespace (≡ [\\t\\n\\f\\r ])
\\Snot whitespace (≡ [^\\t\\n\\f\\r ])
\\wword characters (≡ [0-9A-Za-z_])
\\Wnot word characters (≡ [^0-9A-Za-z_])
\\hhorizontal space
\\Hnot horizontal space
\\vvertical space
\\Vnot vertical space
ASCII character classes:
[[:alnum:]]alphanumeric (≡ [0-9A-Za-z])
[[:alpha:]]alphabetic (≡ [A-Za-z])
[[:ascii:]]ASCII (≡ [\\x00-\\x7F])
[[:blank:]]blank (≡ [\\t ])
[[:cntrl:]]control (≡ [\\x00-\\x1F\\x7F])
[[:digit:]]digits (≡ [0-9])
[[:graph:]]graphical (≡ [!-~] == [A-Za-z0-9!\"#$%&'()*+,\\-./:;<=>?@[\\\\\\]^_`{|}~])
[[:lower:]]lower case (≡ [a-z])
[[:print:]]printable (≡ [ -~] == [ [:graph:]])
[[:punct:]]punctuation (≡ [!-/:-@[-`{-~])
[[:space:]]whitespace (≡ [\\t\\n\\v\\f\\r ])
[[:upper:]]upper case (≡ [A-Z])
[[:word:]]word characters (≡ [0-9A-Za-z_])
[[:xdigit:]]hex digit (≡ [0-9A-Fa-f])
Unicode character class names--general category:
Cother
Cccontrol
Cfformat
Cnunassigned code points
Coprivate use
Cssurrogate
Lletter
LCcased letter
L&cased letter
Lllowercase letter
Lmmodifier letter
Loother letter
Lttitlecase letter
Luuppercase letter
Mmark
Mcspacing mark
Meenclosing mark
Mnnon-spacing mark
Nnumber
Nddecimal number
Nlletter number
Noother number
Ppunctuation
Pcconnector punctuation
Pddash punctuation
Peclose punctuation
Pffinal punctuation
Piinitial punctuation
Poother punctuation
Psopen punctuation
Ssymbol
Sccurrency symbol
Skmodifier symbol
Smmath symbol
Soother symbol
Zseparator
Zlline separator
Zpparagraph separator
Zsspace separator
Unicode character class names--scripts:
Adlam
Ahom
Anatolian_Hieroglyphs
Arabic
Armenian
Avestan
Balinese
Bamum
Bassa_Vah
Batak
Bengali
Bhaiksuki
Bopomofo
Brahmi
Braille
Buginese
Buhid
Canadian_Aboriginal
Carian
Caucasian_Albanian
Chakma
Cham
Cherokee
Chorasmian
Common
Coptic
Cuneiform
Cypriot
Cypro_Minoan
Cyrillic
Deseret
Devanagari
Dives_Akuru
Dogra
Duployan
Egyptian_Hieroglyphs
Elbasan
Elymaic
Ethiopic
Georgian
Glagolitic
Gothic
Grantha
Greek
Gujarati
Gunjala_Gondi
Gurmukhi
Han
Hangul
Hanifi_Rohingya
Hanunoo
Hatran
Hebrew
Hiragana
Imperial_Aramaic
Inherited
Inscriptional_Pahlavi
Inscriptional_Parthian
Javanese
Kaithi
Kannada
Katakana
Kawi
Kayah_Li
Kharoshthi
Khitan_Small_Script
Khmer
Khojki
Khudawadi
Lao
Latin
Lepcha
Limbu
Linear_A
Linear_B
Lisu
Lycian
Lydian
Mahajani
Makasar
Malayalam
Mandaic
Manichaean
Marchen
Masaram_Gondi
Medefaidrin
Meetei_Mayek
Mende_Kikakui
Meroitic_Cursive
Meroitic_Hieroglyphs
Miao
Modi
Mongolian
Mro
Multani
Myanmar
Nabataean
Nag_Mundari
Nandinagari
New_Tai_Lue
Newa
Nko
Nushu
Nyiakeng_Puachue_Hmong
Ogham
Ol_Chiki
Old_Hungarian
Old_Italic
Old_North_Arabian
Old_Permic
Old_Persian
Old_Sogdian
Old_South_Arabian
Old_Turkic
Old_Uyghur
Oriya
Osage
Osmanya
Pahawh_Hmong
Palmyrene
Pau_Cin_Hau
Phags_Pa
Phoenician
Psalter_Pahlavi
Rejang
Runic
Samaritan
Saurashtra
Sharada
Shavian
Siddham
SignWriting
Sinhala
Sogdian
Sora_Sompeng
Soyombo
Sundanese
Syloti_Nagri
Syriac
Tagalog
Tagbanwa
Tai_Le
Tai_Tham
Tai_Viet
Takri
Tamil
Tangsa
Tangut
Telugu
Thaana
Thai
Tibetan
Tifinagh
Tirhuta
Toto
Ugaritic
Vai
Vithkuqi
Wancho
Warang_Citi
Yezidi
Yi
Zanabazar_Square
Vim character classes:
\\iidentifier character VIM
\\I\\i except digits VIM
\\kkeyword character VIM
\\K\\k except digits VIM
\\ffile name character VIM
\\F\\f except digits VIM
\\pprintable character VIM
\\P\\p except digits VIM
\\swhitespace character (≡ [ \\t]) VIM
\\Snon-white space character (≡ [^ \\t]) VIM
\\ddigits (≡ [0-9]) VIM
\\Dnot \\d VIM
\\xhex digits (≡ [0-9A-Fa-f]) VIM
\\Xnot \\x VIM
\\ooctal digits (≡ [0-7]) VIM
\\Onot \\o VIM
\\wword character VIM
\\Wnot \\w VIM
\\hhead of word character VIM
\\Hnot \\h VIM
\\aalphabetic VIM
\\Anot \\a VIM
\\llowercase VIM
\\Lnot lowercase VIM
\\uuppercase VIM
\\Unot uppercase VIM
\\_x\\x plus newline, for any x VIM
Vim flags:
\\cignore case VIM
\\Cmatch case VIM
\\mmagic VIM
\\Mnomagic VIM
\\vverymagic VIM
\\Vverynomagic VIM
\\Zignore differences in Unicode combining characters VIM
Magic:
(?{code})arbitrary Perl code PERL
(??{code})postponed arbitrary Perl code PERL
(?n)recursive call to regexp capturing group n
(?+n)recursive call to relative group +n
(?-n)recursive call to relative group -n
(?C)PCRE callout PCRE
(?R)recursive call to entire regexp (≡ (?0))
(?&name)recursive call to named group
(?P=name)named backreference
(?P>name)recursive call to named group
(?(cond)true|false)conditional branch
(?(cond)true)conditional branch
(*ACCEPT)make regexps more like Prolog
(*COMMIT)
(*F)
(*FAIL)
(*MARK)
(*PRUNE)
(*SKIP)
(*THEN)
(*ANY)set newline convention
(*ANYCRLF)
(*CR)
(*CRLF)
(*LF)
(*BSR_ANYCRLF)set \\R convention PCRE
(*BSR_UNICODE) PCRE
\n\n\n" }, { "path": "doc/syntax.txt", "content": "RE2 regular expression syntax reference\n-------------------------­-------­-----\n\nSingle characters:\n.\tany character, possibly including newline (s=true)\n[xyz]\tcharacter class\n[^xyz]\tnegated character class\n\\d\tPerl character class\n\\D\tnegated Perl character class\n[[:alpha:]]\tASCII character class\n[[:^alpha:]]\tnegated ASCII character class\n\\pN\tUnicode character class (one-letter name)\n\\p{Greek}\tUnicode character class\n\\PN\tnegated Unicode character class (one-letter name)\n\\P{Greek}\tnegated Unicode character class\n\nComposites:\nxy\t«x» followed by «y»\nx|y\t«x» or «y» (prefer «x»)\n\nRepetitions:\nx*\tzero or more «x», prefer more\nx+\tone or more «x», prefer more\nx?\tzero or one «x», prefer one\nx{n,m}\t«n» or «n»+1 or ... or «m» «x», prefer more\nx{n,}\t«n» or more «x», prefer more\nx{n}\texactly «n» «x»\nx*?\tzero or more «x», prefer fewer\nx+?\tone or more «x», prefer fewer\nx??\tzero or one «x», prefer zero\nx{n,m}?\t«n» or «n»+1 or ... or «m» «x», prefer fewer\nx{n,}?\t«n» or more «x», prefer fewer\nx{n}?\texactly «n» «x»\nx{}\t(== x*) NOT SUPPORTED vim\nx{-}\t(== x*?) NOT SUPPORTED vim\nx{-n}\t(== x{n}?) NOT SUPPORTED vim\nx=\t(== x?) NOT SUPPORTED vim\n\nImplementation restriction: The counting forms «x{n,m}», «x{n,}», and «x{n}»\nreject forms that create a minimum or maximum repetition count above 1000.\nUnlimited repetitions are not subject to this restriction.\n\nPossessive repetitions:\nx*+\tzero or more «x», possessive NOT SUPPORTED\nx++\tone or more «x», possessive NOT SUPPORTED\nx?+\tzero or one «x», possessive NOT SUPPORTED\nx{n,m}+\t«n» or ... or «m» «x», possessive NOT SUPPORTED\nx{n,}+\t«n» or more «x», possessive NOT SUPPORTED\nx{n}+\texactly «n» «x», possessive NOT SUPPORTED\n\nGrouping:\n(re)\tnumbered capturing group (submatch)\n(?Pre)\tnamed & numbered capturing group (submatch)\n(?re)\tnamed & numbered capturing group (submatch)\n(?'name're)\tnamed & numbered capturing group (submatch) NOT SUPPORTED\n(?:re)\tnon-capturing group\n(?flags)\tset flags within current group; non-capturing\n(?flags:re)\tset flags during re; non-capturing\n(?#text)\tcomment NOT SUPPORTED\n(?|x|y|z)\tbranch numbering reset NOT SUPPORTED\n(?>re)\tpossessive match of «re» NOT SUPPORTED\nre@>\tpossessive match of «re» NOT SUPPORTED vim\n%(re)\tnon-capturing group NOT SUPPORTED vim\n\nFlags:\ni\tcase-insensitive (default false)\nm\tmulti-line mode: «^» and «$» match begin/end line in addition to begin/end text (default false)\ns\tlet «.» match «\\n» (default false)\nU\tungreedy: swap meaning of «x*» and «x*?», «x+» and «x+?», etc (default false)\nFlag syntax is «xyz» (set) or «-xyz» (clear) or «xy-z» (set «xy», clear «z»).\n\nEmpty strings:\n^\tat beginning of text or line («m»=true)\n$\tat end of text (like «\\z» not «\\Z») or line («m»=true)\n\\A\tat beginning of text\n\\b\tat ASCII word boundary («\\w» on one side and «\\W», «\\A», or «\\z» on the other)\n\\B\tnot at ASCII word boundary\n\\G\tat beginning of subtext being searched NOT SUPPORTED pcre\n\\G\tat end of last match NOT SUPPORTED perl\n\\Z\tat end of text, or before newline at end of text NOT SUPPORTED\n\\z\tat end of text\n(?=re)\tbefore text matching «re» NOT SUPPORTED\n(?!re)\tbefore text not matching «re» NOT SUPPORTED\n(?<=re)\tafter text matching «re» NOT SUPPORTED\n(?\tsubroutine call NOT SUPPORTED\n\\g'name'\tsubroutine call NOT SUPPORTED\n\\k\tnamed backreference NOT SUPPORTED\n\\k'name'\tnamed backreference NOT SUPPORTED\n\\lX\tlowercase «X» NOT SUPPORTED\n\\ux\tuppercase «x» NOT SUPPORTED\n\\L...\\E\tlowercase text «...» NOT SUPPORTED\n\\K\treset beginning of «$0» NOT SUPPORTED\n\\N{name}\tnamed Unicode character NOT SUPPORTED\n\\R\tline break NOT SUPPORTED\n\\U...\\E\tupper case text «...» NOT SUPPORTED\n\\X\textended Unicode sequence NOT SUPPORTED\n\n\\%d123\tdecimal character 123 NOT SUPPORTED vim\n\\%xFF\thex character FF NOT SUPPORTED vim\n\\%o123\toctal character 123 NOT SUPPORTED vim\n\\%u1234\tUnicode character 0x1234 NOT SUPPORTED vim\n\\%U12345678\tUnicode character 0x12345678 NOT SUPPORTED vim\n\nCharacter class elements:\nx\tsingle character\nA-Z\tcharacter range (inclusive)\n\\d\tPerl character class\n[:foo:]\tASCII character class «foo»\n\\p{Foo}\tUnicode character class «Foo»\n\\pF\tUnicode character class «F» (one-letter name)\n\nNamed character classes as character class elements:\n[\\d]\tdigits (== \\d)\n[^\\d]\tnot digits (== \\D)\n[\\D]\tnot digits (== \\D)\n[^\\D]\tnot not digits (== \\d)\n[[:name:]]\tnamed ASCII class inside character class (== [:name:])\n[^[:name:]]\tnamed ASCII class inside negated character class (== [:^name:])\n[\\p{Name}]\tnamed Unicode property inside character class (== \\p{Name})\n[^\\p{Name}]\tnamed Unicode property inside negated character class (== \\P{Name})\n\nPerl character classes (all ASCII-only):\n\\d\tdigits (== [0-9])\n\\D\tnot digits (== [^0-9])\n\\s\twhitespace (== [\\t\\n\\f\\r ])\n\\S\tnot whitespace (== [^\\t\\n\\f\\r ])\n\\w\tword characters (== [0-9A-Za-z_])\n\\W\tnot word characters (== [^0-9A-Za-z_])\n\n\\h\thorizontal space NOT SUPPORTED\n\\H\tnot horizontal space NOT SUPPORTED\n\\v\tvertical space NOT SUPPORTED\n\\V\tnot vertical space NOT SUPPORTED\n\nASCII character classes:\n[[:alnum:]]\talphanumeric (== [0-9A-Za-z])\n[[:alpha:]]\talphabetic (== [A-Za-z])\n[[:ascii:]]\tASCII (== [\\x00-\\x7F])\n[[:blank:]]\tblank (== [\\t ])\n[[:cntrl:]]\tcontrol (== [\\x00-\\x1F\\x7F])\n[[:digit:]]\tdigits (== [0-9])\n[[:graph:]]\tgraphical (== [!-~] == [A-Za-z0-9!\"#$%&'()*+,\\-./:;<=>?@[\\\\\\]^_`{|}~])\n[[:lower:]]\tlower case (== [a-z])\n[[:print:]]\tprintable (== [ -~] == [ [:graph:]])\n[[:punct:]]\tpunctuation (== [!-/:-@[-`{-~])\n[[:space:]]\twhitespace (== [\\t\\n\\v\\f\\r ])\n[[:upper:]]\tupper case (== [A-Z])\n[[:word:]]\tword characters (== [0-9A-Za-z_])\n[[:xdigit:]]\thex digit (== [0-9A-Fa-f])\n\nUnicode character class names--general category:\nC\tother\nCc\tcontrol\nCf\tformat\nCn\tunassigned code points NOT SUPPORTED\nCo\tprivate use\nCs\tsurrogate\nL\tletter\nLC\tcased letter NOT SUPPORTED\nL&\tcased letter NOT SUPPORTED\nLl\tlowercase letter\nLm\tmodifier letter\nLo\tother letter\nLt\ttitlecase letter\nLu\tuppercase letter\nM\tmark\nMc\tspacing mark\nMe\tenclosing mark\nMn\tnon-spacing mark\nN\tnumber\nNd\tdecimal number\nNl\tletter number\nNo\tother number\nP\tpunctuation\nPc\tconnector punctuation\nPd\tdash punctuation\nPe\tclose punctuation\nPf\tfinal punctuation\nPi\tinitial punctuation\nPo\tother punctuation\nPs\topen punctuation\nS\tsymbol\nSc\tcurrency symbol\nSk\tmodifier symbol\nSm\tmath symbol\nSo\tother symbol\nZ\tseparator\nZl\tline separator\nZp\tparagraph separator\nZs\tspace separator\n\nUnicode character class names--scripts:\nAdlam\nAhom\nAnatolian_Hieroglyphs\nArabic\nArmenian\nAvestan\nBalinese\nBamum\nBassa_Vah\nBatak\nBengali\nBhaiksuki\nBopomofo\nBrahmi\nBraille\nBuginese\nBuhid\nCanadian_Aboriginal\nCarian\nCaucasian_Albanian\nChakma\nCham\nCherokee\nChorasmian\nCommon\nCoptic\nCuneiform\nCypriot\nCypro_Minoan\nCyrillic\nDeseret\nDevanagari\nDives_Akuru\nDogra\nDuployan\nEgyptian_Hieroglyphs\nElbasan\nElymaic\nEthiopic\nGeorgian\nGlagolitic\nGothic\nGrantha\nGreek\nGujarati\nGunjala_Gondi\nGurmukhi\nHan\nHangul\nHanifi_Rohingya\nHanunoo\nHatran\nHebrew\nHiragana\nImperial_Aramaic\nInherited\nInscriptional_Pahlavi\nInscriptional_Parthian\nJavanese\nKaithi\nKannada\nKatakana\nKawi\nKayah_Li\nKharoshthi\nKhitan_Small_Script\nKhmer\nKhojki\nKhudawadi\nLao\nLatin\nLepcha\nLimbu\nLinear_A\nLinear_B\nLisu\nLycian\nLydian\nMahajani\nMakasar\nMalayalam\nMandaic\nManichaean\nMarchen\nMasaram_Gondi\nMedefaidrin\nMeetei_Mayek\nMende_Kikakui\nMeroitic_Cursive\nMeroitic_Hieroglyphs\nMiao\nModi\nMongolian\nMro\nMultani\nMyanmar\nNabataean\nNag_Mundari\nNandinagari\nNew_Tai_Lue\nNewa\nNko\nNushu\nNyiakeng_Puachue_Hmong\nOgham\nOl_Chiki\nOld_Hungarian\nOld_Italic\nOld_North_Arabian\nOld_Permic\nOld_Persian\nOld_Sogdian\nOld_South_Arabian\nOld_Turkic\nOld_Uyghur\nOriya\nOsage\nOsmanya\nPahawh_Hmong\nPalmyrene\nPau_Cin_Hau\nPhags_Pa\nPhoenician\nPsalter_Pahlavi\nRejang\nRunic\nSamaritan\nSaurashtra\nSharada\nShavian\nSiddham\nSignWriting\nSinhala\nSogdian\nSora_Sompeng\nSoyombo\nSundanese\nSyloti_Nagri\nSyriac\nTagalog\nTagbanwa\nTai_Le\nTai_Tham\nTai_Viet\nTakri\nTamil\nTangsa\nTangut\nTelugu\nThaana\nThai\nTibetan\nTifinagh\nTirhuta\nToto\nUgaritic\nVai\nVithkuqi\nWancho\nWarang_Citi\nYezidi\nYi\nZanabazar_Square\n\nVim character classes:\n\\i\tidentifier character NOT SUPPORTED vim\n\\I\t«\\i» except digits NOT SUPPORTED vim\n\\k\tkeyword character NOT SUPPORTED vim\n\\K\t«\\k» except digits NOT SUPPORTED vim\n\\f\tfile name character NOT SUPPORTED vim\n\\F\t«\\f» except digits NOT SUPPORTED vim\n\\p\tprintable character NOT SUPPORTED vim\n\\P\t«\\p» except digits NOT SUPPORTED vim\n\\s\twhitespace character (== [ \\t]) NOT SUPPORTED vim\n\\S\tnon-white space character (== [^ \\t]) NOT SUPPORTED vim\n\\d\tdigits (== [0-9]) vim\n\\D\tnot «\\d» vim\n\\x\thex digits (== [0-9A-Fa-f]) NOT SUPPORTED vim\n\\X\tnot «\\x» NOT SUPPORTED vim\n\\o\toctal digits (== [0-7]) NOT SUPPORTED vim\n\\O\tnot «\\o» NOT SUPPORTED vim\n\\w\tword character vim\n\\W\tnot «\\w» vim\n\\h\thead of word character NOT SUPPORTED vim\n\\H\tnot «\\h» NOT SUPPORTED vim\n\\a\talphabetic NOT SUPPORTED vim\n\\A\tnot «\\a» NOT SUPPORTED vim\n\\l\tlowercase NOT SUPPORTED vim\n\\L\tnot lowercase NOT SUPPORTED vim\n\\u\tuppercase NOT SUPPORTED vim\n\\U\tnot uppercase NOT SUPPORTED vim\n\\_x\t«\\x» plus newline, for any «x» NOT SUPPORTED vim\n\nVim flags:\n\\c\tignore case NOT SUPPORTED vim\n\\C\tmatch case NOT SUPPORTED vim\n\\m\tmagic NOT SUPPORTED vim\n\\M\tnomagic NOT SUPPORTED vim\n\\v\tverymagic NOT SUPPORTED vim\n\\V\tverynomagic NOT SUPPORTED vim\n\\Z\tignore differences in Unicode combining characters NOT SUPPORTED vim\n\nMagic:\n(?{code})\tarbitrary Perl code NOT SUPPORTED perl\n(??{code})\tpostponed arbitrary Perl code NOT SUPPORTED perl\n(?n)\trecursive call to regexp capturing group «n» NOT SUPPORTED\n(?+n)\trecursive call to relative group «+n» NOT SUPPORTED\n(?-n)\trecursive call to relative group «-n» NOT SUPPORTED\n(?C)\tPCRE callout NOT SUPPORTED pcre\n(?R)\trecursive call to entire regexp (== (?0)) NOT SUPPORTED\n(?&name)\trecursive call to named group NOT SUPPORTED\n(?P=name)\tnamed backreference NOT SUPPORTED\n(?P>name)\trecursive call to named group NOT SUPPORTED\n(?(cond)true|false)\tconditional branch NOT SUPPORTED\n(?(cond)true)\tconditional branch NOT SUPPORTED\n(*ACCEPT)\tmake regexps more like Prolog NOT SUPPORTED\n(*COMMIT)\tNOT SUPPORTED\n(*F)\tNOT SUPPORTED\n(*FAIL)\tNOT SUPPORTED\n(*MARK)\tNOT SUPPORTED\n(*PRUNE)\tNOT SUPPORTED\n(*SKIP)\tNOT SUPPORTED\n(*THEN)\tNOT SUPPORTED\n(*ANY)\tset newline convention NOT SUPPORTED\n(*ANYCRLF)\tNOT SUPPORTED\n(*CR)\tNOT SUPPORTED\n(*CRLF)\tNOT SUPPORTED\n(*LF)\tNOT SUPPORTED\n(*BSR_ANYCRLF)\tset \\R convention NOT SUPPORTED pcre\n(*BSR_UNICODE)\tNOT SUPPORTED pcre\n\n" }, { "path": "lib/git/commit-msg.hook", "content": "#!/bin/sh\n# From Gerrit Code Review 2.2.1\n#\n# Part of Gerrit Code Review (http://code.google.com/p/gerrit/)\n#\n# Copyright (C) 2009 The Android Open Source Project\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nCHANGE_ID_AFTER=\"Bug|Issue\"\nMSG=\"$1\"\n\n# Check for, and add if missing, a unique Change-Id\n#\nadd_ChangeId() {\n\tclean_message=`sed -e '\n\t\t/^diff --git a\\/.*/{\n\t\t\ts///\n\t\t\tq\n\t\t}\n\t\t/^Signed-off-by:/d\n\t\t/^#/d\n\t' \"$MSG\" | git stripspace`\n\tif test -z \"$clean_message\"\n\tthen\n\t\treturn\n\tfi\n\n\tif grep -i '^Change-Id:' \"$MSG\" >/dev/null\n\tthen\n\t\treturn\n\tfi\n\n\tid=`_gen_ChangeId`\n\tperl -e '\n\t\t$MSG = shift;\n\t\t$id = shift;\n\t\t$CHANGE_ID_AFTER = shift;\n\n\t\tundef $/;\n\t\topen(I, $MSG); $_ = ; close I;\n\t\ts|^diff --git a/.*||ms;\n\t\ts|^#.*$||mg;\n\t\texit unless $_;\n\n\t\t@message = split /\\n/;\n\t\t$haveFooter = 0;\n\t\t$startFooter = @message;\n\t\tfor($line = @message - 1; $line >= 0; $line--) {\n\t\t\t$_ = $message[$line];\n\n\t\t\tif (/^[a-zA-Z0-9-]+:/ && !m,^[a-z0-9-]+://,) {\n\t\t\t\t$haveFooter++;\n\t\t\t\tnext;\n\t\t\t}\n\t\t\tnext if /^[ []/;\n\t\t\t$startFooter = $line if ($haveFooter && /^\\r?$/);\n\t\t\tlast;\n\t\t}\n\n\t\t@footer = @message[$startFooter+1..@message];\n\t\t@message = @message[0..$startFooter];\n\t\tpush(@footer, \"\") unless @footer;\n\n\t\tfor ($line = 0; $line < @footer; $line++) {\n\t\t\t$_ = $footer[$line];\n\t\t\tnext if /^($CHANGE_ID_AFTER):/i;\n\t\t\tlast;\n\t\t}\n\t\tsplice(@footer, $line, 0, \"Change-Id: I$id\");\n\n\t\t$_ = join(\"\\n\", @message, @footer);\n\t\topen(O, \">$MSG\"); print O; close O;\n\t' \"$MSG\" \"$id\" \"$CHANGE_ID_AFTER\"\n}\n_gen_ChangeIdInput() {\n\techo \"tree `git write-tree`\"\n\tif parent=`git rev-parse HEAD^0 2>/dev/null`\n\tthen\n\t\techo \"parent $parent\"\n\tfi\n\techo \"author `git var GIT_AUTHOR_IDENT`\"\n\techo \"committer `git var GIT_COMMITTER_IDENT`\"\n\techo\n\tprintf '%s' \"$clean_message\"\n}\n_gen_ChangeId() {\n\t_gen_ChangeIdInput |\n\tgit hash-object -t commit --stdin\n}\n\n\nadd_ChangeId\n" }, { "path": "libre2.symbols", "content": "{\n\tglobal:\n\t\t# re2::RE2*\n\t\t_ZN3re23RE2*;\n\t\t_ZNK3re23RE2*;\n\t\t# re2::operator<<*\n\t\t_ZN3re2ls*;\n\t\t# re2::FilteredRE2*\n\t\t_ZN3re211FilteredRE2*;\n\t\t_ZNK3re211FilteredRE2*;\n\t\t# re2::re2_internal*\n\t\t_ZN3re212re2_internal*;\n\t\t_ZNK3re212re2_internal*;\n\tlocal:\n\t\t*;\n};\n" }, { "path": "libre2.symbols.darwin", "content": "# Linker doesn't like these unmangled:\n# re2::RE2*\n__ZN3re23RE2*\n__ZNK3re23RE2*\n# re2::operator<<*\n__ZN3re2ls*\n# re2::FilteredRE2*\n__ZN3re211FilteredRE2*\n__ZNK3re211FilteredRE2*\n# re2::re2_internal*\n__ZN3re212re2_internal*\n__ZNK3re212re2_internal*\n" }, { "path": "python/BUILD.bazel", "content": "# Copyright 2009 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# Bazel (http://bazel.build/) BUILD file for RE2 Python.\n\nload(\"@pybind11_bazel//:build_defs.bzl\", \"pybind_extension\")\nload(\"@rules_python//python:defs.bzl\", \"py_library\", \"py_test\")\n\npybind_extension(\n name = \"_re2\",\n srcs = [\"_re2.cc\"],\n deps = [\n \"//:re2\",\n \"@abseil-cpp//absl/strings\",\n ],\n)\n\npy_library(\n name = \"re2\",\n srcs = [\"re2.py\"],\n data = [\":_re2\"],\n imports = [\".\"],\n visibility = [\"//visibility:public\"],\n)\n\npy_test(\n name = \"re2_test\",\n size = \"small\",\n srcs = [\"re2_test.py\"],\n deps = [\n \":re2\",\n \"@abseil-py//absl/testing:absltest\",\n \"@abseil-py//absl/testing:parameterized\",\n ],\n)\n\n# These are implementation details for `setup.py`, so they can be\n# named however we want. For now, they are named to be consistent\n# with the `--cpu` flag values that they will eventually replace.\n\nplatform(\n name = \"darwin_x86_64\",\n constraint_values = [\n \"@platforms//cpu:x86_64\",\n \"@platforms//os:macos\",\n ],\n)\n\nplatform(\n name = \"darwin_arm64\",\n constraint_values = [\n \"@platforms//cpu:arm64\",\n \"@platforms//os:macos\",\n ],\n)\n\nplatform(\n name = \"x64_x86_windows\",\n constraint_values = [\n \"@platforms//cpu:x86_32\",\n \"@platforms//os:windows\",\n ],\n)\n\nplatform(\n name = \"x64_windows\",\n constraint_values = [\n \"@platforms//cpu:x86_64\",\n \"@platforms//os:windows\",\n ],\n)\n\nplatform(\n name = \"arm64_windows\",\n constraint_values = [\n \"@platforms//cpu:arm64\",\n \"@platforms//os:windows\",\n ],\n)\n" }, { "path": "python/README", "content": "Building requires Python 3 and pybind11 to be installed on your system.\n" }, { "path": "python/_re2.cc", "content": "// Copyright 2019 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"absl/strings/string_view.h\"\n#include \"pybind11/buffer_info.h\"\n#include \"pybind11/gil.h\"\n#include \"pybind11/pybind11.h\"\n#include \"pybind11/pytypes.h\"\n#include \"pybind11/stl.h\" // IWYU pragma: keep\n#include \"re2/filtered_re2.h\"\n#include \"re2/re2.h\"\n#include \"re2/set.h\"\n\n#ifdef _WIN32\n#include \n#define ssize_t SSIZE_T\n#endif\n\nnamespace re2_python {\n\n// This is conventional.\nnamespace py = pybind11;\n\n// In terms of the pybind11 API, a py::buffer is merely a py::object that\n// supports the buffer interface/protocol and you must explicitly request\n// a py::buffer_info in order to access the actual bytes. Under the hood,\n// the py::buffer_info manages a reference count to the py::buffer, so it\n// must be constructed and subsequently destructed while holding the GIL.\nstatic inline absl::string_view FromBytes(const py::buffer_info& bytes) {\n char* data = reinterpret_cast(bytes.ptr);\n ssize_t size = bytes.size;\n return absl::string_view(data, size);\n}\n\nstatic inline int OneCharLen(const char* ptr) {\n return \"\\1\\1\\1\\1\\1\\1\\1\\1\\1\\1\\1\\1\\2\\2\\3\\4\"[(*ptr & 0xFF) >> 4];\n}\n\n// Helper function for when Python encodes str to bytes and then needs to\n// convert str offsets to bytes offsets. Assumes that text is valid UTF-8.\nssize_t CharLenToBytes(py::buffer buffer, ssize_t pos, ssize_t len) {\n auto bytes = buffer.request();\n auto text = FromBytes(bytes);\n auto ptr = text.data() + pos;\n auto end = text.data() + text.size();\n while (ptr < end && len > 0) {\n ptr += OneCharLen(ptr);\n --len;\n }\n return ptr - (text.data() + pos);\n}\n\n// Helper function for when Python decodes bytes to str and then needs to\n// convert bytes offsets to str offsets. Assumes that text is valid UTF-8.\nssize_t BytesToCharLen(py::buffer buffer, ssize_t pos, ssize_t endpos) {\n auto bytes = buffer.request();\n auto text = FromBytes(bytes);\n auto ptr = text.data() + pos;\n auto end = text.data() + endpos;\n ssize_t len = 0;\n while (ptr < end) {\n ptr += OneCharLen(ptr);\n ++len;\n }\n return len;\n}\n\nstd::unique_ptr RE2InitShim(py::buffer buffer,\n const RE2::Options& options) {\n auto bytes = buffer.request();\n auto pattern = FromBytes(bytes);\n return std::make_unique(pattern, options);\n}\n\npy::bytes RE2ErrorShim(const RE2& self) {\n // Return std::string as bytes. That is, without decoding to str.\n return self.error();\n}\n\nstd::vector> RE2NamedCapturingGroupsShim(\n const RE2& self) {\n const int num_groups = self.NumberOfCapturingGroups();\n std::vector> groups;\n groups.reserve(num_groups);\n for (const auto& it : self.NamedCapturingGroups()) {\n groups.emplace_back(it.first, it.second);\n }\n return groups;\n}\n\nstd::vector RE2ProgramFanoutShim(const RE2& self) {\n std::vector histogram;\n self.ProgramFanout(&histogram);\n return histogram;\n}\n\nstd::vector RE2ReverseProgramFanoutShim(const RE2& self) {\n std::vector histogram;\n self.ReverseProgramFanout(&histogram);\n return histogram;\n}\n\nstd::tuple RE2PossibleMatchRangeShim(\n const RE2& self, int maxlen) {\n std::string min, max;\n // Return std::string as bytes. That is, without decoding to str.\n return {self.PossibleMatchRange(&min, &max, maxlen), min, max};\n}\n\nstd::vector> RE2MatchShim(const RE2& self,\n RE2::Anchor anchor,\n py::buffer buffer,\n ssize_t pos,\n ssize_t endpos) {\n auto bytes = buffer.request();\n auto text = FromBytes(bytes);\n const int num_groups = self.NumberOfCapturingGroups() + 1; // need $0\n std::vector groups;\n groups.resize(num_groups);\n py::gil_scoped_release release_gil;\n if (!self.Match(text, pos, endpos, anchor, groups.data(), groups.size())) {\n // Ensure that groups are null before converting to spans!\n for (auto& it : groups) {\n it = absl::string_view();\n }\n }\n std::vector> spans;\n spans.reserve(num_groups);\n for (const auto& it : groups) {\n if (it.data() == NULL) {\n spans.emplace_back(-1, -1);\n } else {\n spans.emplace_back(it.data() - text.data(),\n it.data() - text.data() + it.size());\n }\n }\n return spans;\n}\n\npy::bytes RE2QuoteMetaShim(py::buffer buffer) {\n auto bytes = buffer.request();\n auto pattern = FromBytes(bytes);\n // Return std::string as bytes. That is, without decoding to str.\n return RE2::QuoteMeta(pattern);\n}\n\nclass Set {\n public:\n Set(RE2::Anchor anchor, const RE2::Options& options)\n : set_(options, anchor) {}\n\n ~Set() = default;\n\n // Not copyable or movable.\n Set(const Set&) = delete;\n Set& operator=(const Set&) = delete;\n\n int Add(py::buffer buffer) {\n auto bytes = buffer.request();\n auto pattern = FromBytes(bytes);\n int index = set_.Add(pattern, /*error=*/NULL); // -1 on error\n return index;\n }\n\n bool Compile() {\n // Compiling can fail.\n return set_.Compile();\n }\n\n std::vector Match(py::buffer buffer) const {\n auto bytes = buffer.request();\n auto text = FromBytes(bytes);\n std::vector matches;\n py::gil_scoped_release release_gil;\n set_.Match(text, &matches);\n return matches;\n }\n\n private:\n RE2::Set set_;\n};\n\nclass Filter {\n public:\n Filter() = default;\n ~Filter() = default;\n\n // Not copyable or movable.\n Filter(const Filter&) = delete;\n Filter& operator=(const Filter&) = delete;\n\n int Add(py::buffer buffer, const RE2::Options& options) {\n auto bytes = buffer.request();\n auto pattern = FromBytes(bytes);\n int index = -1; // not clobbered on error\n filter_.Add(pattern, options, &index);\n return index;\n }\n\n bool Compile() {\n std::vector atoms;\n filter_.Compile(&atoms);\n RE2::Options options;\n options.set_literal(true);\n options.set_case_sensitive(false);\n set_ = std::make_unique(options, RE2::UNANCHORED);\n for (int i = 0; i < static_cast(atoms.size()); ++i) {\n if (set_->Add(atoms[i], /*error=*/NULL) != i) {\n // Should never happen: the atom is a literal!\n py::pybind11_fail(\"set_->Add() failed\");\n }\n }\n // Compiling can fail.\n return set_->Compile();\n }\n\n std::vector Match(py::buffer buffer, bool potential) const {\n if (set_ == nullptr) {\n py::pybind11_fail(\"Match() called before compiling\");\n }\n\n auto bytes = buffer.request();\n auto text = FromBytes(bytes);\n std::vector atoms;\n py::gil_scoped_release release_gil;\n set_->Match(text, &atoms);\n std::vector matches;\n if (potential) {\n filter_.AllPotentials(atoms, &matches);\n } else {\n filter_.AllMatches(text, atoms, &matches);\n }\n return matches;\n }\n\n const RE2& GetRE2(int index) const {\n return filter_.GetRE2(index);\n }\n\n private:\n re2::FilteredRE2 filter_;\n std::unique_ptr set_;\n};\n\nPYBIND11_MODULE(_re2, module) {\n // Translate exceptions thrown by py::pybind11_fail() into Python.\n py::register_local_exception(module, \"Error\");\n\n module.def(\"CharLenToBytes\", &CharLenToBytes);\n module.def(\"BytesToCharLen\", &BytesToCharLen);\n\n // CLASSES\n // class RE2\n // enum Anchor\n // class Options\n // enum Encoding\n // class Set\n // class Filter\n py::class_ re2(module, \"RE2\");\n py::enum_ anchor(re2, \"Anchor\");\n py::class_ options(re2, \"Options\");\n py::enum_ encoding(options, \"Encoding\");\n py::class_ set(module, \"Set\");\n py::class_ filter(module, \"Filter\");\n\n anchor.value(\"UNANCHORED\", RE2::Anchor::UNANCHORED);\n anchor.value(\"ANCHOR_START\", RE2::Anchor::ANCHOR_START);\n anchor.value(\"ANCHOR_BOTH\", RE2::Anchor::ANCHOR_BOTH);\n\n encoding.value(\"UTF8\", RE2::Options::Encoding::EncodingUTF8);\n encoding.value(\"LATIN1\", RE2::Options::Encoding::EncodingLatin1);\n\n options.def(py::init<>())\n .def_property(\"max_mem\", //\n &RE2::Options::max_mem, //\n &RE2::Options::set_max_mem) //\n .def_property(\"encoding\", //\n &RE2::Options::encoding, //\n &RE2::Options::set_encoding) //\n .def_property(\"posix_syntax\", //\n &RE2::Options::posix_syntax, //\n &RE2::Options::set_posix_syntax) //\n .def_property(\"longest_match\", //\n &RE2::Options::longest_match, //\n &RE2::Options::set_longest_match) //\n .def_property(\"log_errors\", //\n &RE2::Options::log_errors, //\n &RE2::Options::set_log_errors) //\n .def_property(\"literal\", //\n &RE2::Options::literal, //\n &RE2::Options::set_literal) //\n .def_property(\"never_nl\", //\n &RE2::Options::never_nl, //\n &RE2::Options::set_never_nl) //\n .def_property(\"dot_nl\", //\n &RE2::Options::dot_nl, //\n &RE2::Options::set_dot_nl) //\n .def_property(\"never_capture\", //\n &RE2::Options::never_capture, //\n &RE2::Options::set_never_capture) //\n .def_property(\"case_sensitive\", //\n &RE2::Options::case_sensitive, //\n &RE2::Options::set_case_sensitive) //\n .def_property(\"perl_classes\", //\n &RE2::Options::perl_classes, //\n &RE2::Options::set_perl_classes) //\n .def_property(\"word_boundary\", //\n &RE2::Options::word_boundary, //\n &RE2::Options::set_word_boundary) //\n .def_property(\"one_line\", //\n &RE2::Options::one_line, //\n &RE2::Options::set_one_line); //\n\n re2.def(py::init(&RE2InitShim))\n .def(\"ok\", &RE2::ok)\n .def(\"error\", &RE2ErrorShim)\n .def(\"options\", &RE2::options)\n .def(\"NumberOfCapturingGroups\", &RE2::NumberOfCapturingGroups)\n .def(\"NamedCapturingGroups\", &RE2NamedCapturingGroupsShim)\n .def(\"ProgramSize\", &RE2::ProgramSize)\n .def(\"ReverseProgramSize\", &RE2::ReverseProgramSize)\n .def(\"ProgramFanout\", &RE2ProgramFanoutShim)\n .def(\"ReverseProgramFanout\", &RE2ReverseProgramFanoutShim)\n .def(\"PossibleMatchRange\", &RE2PossibleMatchRangeShim)\n .def(\"Match\", &RE2MatchShim)\n .def_static(\"QuoteMeta\", &RE2QuoteMetaShim);\n\n set.def(py::init())\n .def(\"Add\", &Set::Add)\n .def(\"Compile\", &Set::Compile)\n .def(\"Match\", &Set::Match);\n\n filter.def(py::init<>())\n .def(\"Add\", &Filter::Add)\n .def(\"Compile\", &Filter::Compile)\n .def(\"Match\", &Filter::Match)\n .def(\"GetRE2\", &Filter::GetRE2,\n py::return_value_policy::reference_internal);\n}\n\n} // namespace re2_python\n" }, { "path": "python/re2.py", "content": "# Copyright 2019 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\nr\"\"\"A drop-in replacement for the re module.\n\nIt uses RE2 under the hood, of course, so various PCRE features\n(e.g. backreferences, look-around assertions) are not supported.\nSee https://github.com/google/re2/wiki/Syntax for the canonical\nreference, but known syntactic \"gotchas\" relative to Python are:\n\n * PCRE supports \\Z and \\z; RE2 supports \\z; Python supports \\z,\n but calls it \\Z. You must rewrite \\Z to \\z in pattern strings.\n\nKnown differences between this module's API and the re module's API:\n\n * The error class does not provide any error information as attributes.\n * The Options class replaces the re module's flags with RE2's options as\n gettable/settable properties. Please see re2.h for their documentation.\n * The pattern string and the input string do not have to be the same type.\n Any str will be encoded to UTF-8.\n * The pattern string cannot be str if the options specify Latin-1 encoding.\n\nThis module's LRU cache contains a maximum of 128 regular expression objects.\nEach regular expression object's underlying RE2 object uses a maximum of 8MiB\nof memory (by default). Hence, this module's LRU cache uses a maximum of 1GiB\nof memory (by default), but in most cases, it should use much less than that.\n\"\"\"\n\nimport codecs\nimport functools\nimport itertools\n\nimport _re2\n\n\n# pybind11 translates C++ exceptions to Python exceptions.\n# We use that same Python exception class for consistency.\nerror = _re2.Error\n\n\nclass Options(_re2.RE2.Options):\n\n __slots__ = ()\n\n NAMES = (\n 'max_mem',\n 'encoding',\n 'posix_syntax',\n 'longest_match',\n 'log_errors',\n 'literal',\n 'never_nl',\n 'dot_nl',\n 'never_capture',\n 'case_sensitive',\n 'perl_classes',\n 'word_boundary',\n 'one_line',\n )\n\n\ndef compile(pattern, options=None):\n if isinstance(pattern, _Regexp):\n if options:\n raise error('pattern is already compiled, so '\n 'options may not be specified')\n return pattern\n options = options or Options()\n values = tuple(getattr(options, name) for name in Options.NAMES)\n return _Regexp._make(pattern, values)\n\n\ndef search(pattern, text, options=None):\n return compile(pattern, options=options).search(text)\n\n\ndef match(pattern, text, options=None):\n return compile(pattern, options=options).match(text)\n\n\ndef fullmatch(pattern, text, options=None):\n return compile(pattern, options=options).fullmatch(text)\n\n\ndef finditer(pattern, text, options=None):\n return compile(pattern, options=options).finditer(text)\n\n\ndef findall(pattern, text, options=None):\n return compile(pattern, options=options).findall(text)\n\n\ndef split(pattern, text, maxsplit=0, options=None):\n return compile(pattern, options=options).split(text, maxsplit)\n\n\ndef subn(pattern, repl, text, count=0, options=None):\n return compile(pattern, options=options).subn(repl, text, count)\n\n\ndef sub(pattern, repl, text, count=0, options=None):\n return compile(pattern, options=options).sub(repl, text, count)\n\n\ndef _encode(t):\n return t.encode(encoding='utf-8')\n\n\ndef _decode(b):\n return b.decode(encoding='utf-8')\n\n\ndef escape(pattern):\n if isinstance(pattern, str):\n encoded_pattern = _encode(pattern)\n escaped = _re2.RE2.QuoteMeta(encoded_pattern)\n decoded_escaped = _decode(escaped)\n return decoded_escaped\n else:\n escaped = _re2.RE2.QuoteMeta(pattern)\n return escaped\n\n\ndef purge():\n return _Regexp._make.cache_clear()\n\n\n_Anchor = _re2.RE2.Anchor\n_NULL_SPAN = (-1, -1)\n\n\nclass _Regexp(object):\n\n __slots__ = ('_pattern', '_regexp')\n\n @classmethod\n @functools.lru_cache(typed=True)\n def _make(cls, pattern, values):\n options = Options()\n for name, value in zip(Options.NAMES, values):\n setattr(options, name, value)\n return cls(pattern, options)\n\n def __init__(self, pattern, options):\n self._pattern = pattern\n if isinstance(self._pattern, str):\n if options.encoding == Options.Encoding.LATIN1:\n raise error('string type of pattern is str, but '\n 'encoding specified in options is LATIN1')\n encoded_pattern = _encode(self._pattern)\n self._regexp = _re2.RE2(encoded_pattern, options)\n else:\n self._regexp = _re2.RE2(self._pattern, options)\n if not self._regexp.ok():\n raise error(self._regexp.error())\n\n def __getstate__(self):\n options = {name: getattr(self.options, name) for name in Options.NAMES}\n return self._pattern, options\n\n def __setstate__(self, state):\n pattern, options = state\n values = tuple(options[name] for name in Options.NAMES)\n other = _Regexp._make(pattern, values)\n self._pattern = other._pattern\n self._regexp = other._regexp\n\n def _match(self, anchor, text, pos=None, endpos=None):\n pos = 0 if pos is None else max(0, min(pos, len(text)))\n endpos = len(text) if endpos is None else max(0, min(endpos, len(text)))\n if pos > endpos:\n return\n if isinstance(text, str):\n encoded_text = _encode(text)\n encoded_pos = _re2.CharLenToBytes(encoded_text, 0, pos)\n if endpos == len(text):\n # This is the common case.\n encoded_endpos = len(encoded_text)\n else:\n encoded_endpos = encoded_pos + _re2.CharLenToBytes(\n encoded_text, encoded_pos, endpos - pos)\n decoded_offsets = {0: 0}\n last_offset = 0\n while True:\n spans = self._regexp.Match(anchor, encoded_text, encoded_pos,\n encoded_endpos)\n if spans[0] == _NULL_SPAN:\n break\n\n # This algorithm is linear in the length of encoded_text. Specifically,\n # no matter how many groups there are for a given regular expression or\n # how many iterations through the loop there are for a given generator,\n # this algorithm uses a single, straightforward pass over encoded_text.\n offsets = sorted(set(itertools.chain(*spans)))\n if offsets[0] == -1:\n offsets = offsets[1:]\n # Discard the rest of the items because they are useless now - and we\n # could accumulate one item per str offset in the pathological case!\n decoded_offsets = {last_offset: decoded_offsets[last_offset]}\n for offset in offsets:\n decoded_offsets[offset] = (\n decoded_offsets[last_offset] +\n _re2.BytesToCharLen(encoded_text, last_offset, offset))\n last_offset = offset\n\n def decode(span):\n if span == _NULL_SPAN:\n return span\n return decoded_offsets[span[0]], decoded_offsets[span[1]]\n\n decoded_spans = [decode(span) for span in spans]\n yield _Match(self, text, pos, endpos, decoded_spans)\n if encoded_pos == encoded_endpos:\n break\n elif encoded_pos == spans[0][1]:\n # We matched the empty string at encoded_pos and would be stuck, so\n # in order to make forward progress, increment the str offset.\n encoded_pos += _re2.CharLenToBytes(encoded_text, encoded_pos, 1)\n else:\n encoded_pos = spans[0][1]\n else:\n while True:\n spans = self._regexp.Match(anchor, text, pos, endpos)\n if spans[0] == _NULL_SPAN:\n break\n yield _Match(self, text, pos, endpos, spans)\n if pos == endpos:\n break\n elif pos == spans[0][1]:\n # We matched the empty string at pos and would be stuck, so in order\n # to make forward progress, increment the bytes offset.\n pos += 1\n else:\n pos = spans[0][1]\n\n def search(self, text, pos=None, endpos=None):\n return next(self._match(_Anchor.UNANCHORED, text, pos, endpos), None)\n\n def match(self, text, pos=None, endpos=None):\n return next(self._match(_Anchor.ANCHOR_START, text, pos, endpos), None)\n\n def fullmatch(self, text, pos=None, endpos=None):\n return next(self._match(_Anchor.ANCHOR_BOTH, text, pos, endpos), None)\n\n def finditer(self, text, pos=None, endpos=None):\n return self._match(_Anchor.UNANCHORED, text, pos, endpos)\n\n def findall(self, text, pos=None, endpos=None):\n empty = type(text)()\n items = []\n for match in self.finditer(text, pos, endpos):\n if not self.groups:\n item = match.group()\n elif self.groups == 1:\n item = match.groups(default=empty)[0]\n else:\n item = match.groups(default=empty)\n items.append(item)\n return items\n\n def _split(self, cb, text, maxsplit=0):\n if maxsplit < 0:\n return [text], 0\n elif maxsplit > 0:\n matchiter = itertools.islice(self.finditer(text), maxsplit)\n else:\n matchiter = self.finditer(text)\n pieces = []\n end = 0\n numsplit = 0\n for match in matchiter:\n pieces.append(text[end:match.start()])\n pieces.extend(cb(match))\n end = match.end()\n numsplit += 1\n pieces.append(text[end:])\n return pieces, numsplit\n\n def split(self, text, maxsplit=0):\n cb = lambda match: [match[group] for group in range(1, self.groups + 1)]\n pieces, _ = self._split(cb, text, maxsplit)\n return pieces\n\n def subn(self, repl, text, count=0):\n cb = lambda match: [repl(match) if callable(repl) else match.expand(repl)]\n empty = type(text)()\n pieces, numsplit = self._split(cb, text, count)\n joined_pieces = empty.join(pieces)\n return joined_pieces, numsplit\n\n def sub(self, repl, text, count=0):\n joined_pieces, _ = self.subn(repl, text, count)\n return joined_pieces\n\n @property\n def pattern(self):\n return self._pattern\n\n @property\n def options(self):\n return self._regexp.options()\n\n @property\n def groups(self):\n return self._regexp.NumberOfCapturingGroups()\n\n @property\n def groupindex(self):\n groups = self._regexp.NamedCapturingGroups()\n if isinstance(self._pattern, str):\n decoded_groups = [(_decode(group), index) for group, index in groups]\n return dict(decoded_groups)\n else:\n return dict(groups)\n\n @property\n def programsize(self):\n return self._regexp.ProgramSize()\n\n @property\n def reverseprogramsize(self):\n return self._regexp.ReverseProgramSize()\n\n @property\n def programfanout(self):\n return self._regexp.ProgramFanout()\n\n @property\n def reverseprogramfanout(self):\n return self._regexp.ReverseProgramFanout()\n\n def possiblematchrange(self, maxlen):\n ok, min, max = self._regexp.PossibleMatchRange(maxlen)\n if not ok:\n raise error('failed to compute match range')\n return min, max\n\n\nclass _Match(object):\n\n __slots__ = ('_regexp', '_text', '_pos', '_endpos', '_spans')\n\n def __init__(self, regexp, text, pos, endpos, spans):\n self._regexp = regexp\n self._text = text\n self._pos = pos\n self._endpos = endpos\n self._spans = spans\n\n # Python prioritises three-digit octal numbers over group escapes.\n # For example, \\100 should not be handled the same way as \\g<10>0.\n _OCTAL_RE = compile('\\\\\\\\[0-7][0-7][0-7]')\n\n # Python supports \\1 through \\99 (inclusive) and \\g<...> syntax.\n _GROUP_RE = compile('\\\\\\\\[1-9][0-9]?|\\\\\\\\g<\\\\w+>')\n\n @classmethod\n @functools.lru_cache(typed=True)\n def _split(cls, template):\n if isinstance(template, str):\n backslash = '\\\\'\n else:\n backslash = b'\\\\'\n empty = type(template)()\n pieces = [empty]\n index = template.find(backslash)\n while index != -1:\n piece, template = template[:index], template[index:]\n pieces[-1] += piece\n octal_match = cls._OCTAL_RE.match(template)\n group_match = cls._GROUP_RE.match(template)\n if (not octal_match) and group_match:\n index = group_match.end()\n piece, template = template[:index], template[index:]\n pieces.extend((piece, empty))\n else:\n # 2 isn't enough for \\o, \\x, \\N, \\u and \\U escapes, but none of those\n # should contain backslashes, so break them here and then fix them at\n # the beginning of the next loop iteration or right before returning.\n index = 2\n piece, template = template[:index], template[index:]\n pieces[-1] += piece\n index = template.find(backslash)\n pieces[-1] += template\n return pieces\n\n def expand(self, template):\n if isinstance(template, str):\n unescape = codecs.unicode_escape_decode\n else:\n unescape = codecs.escape_decode\n empty = type(template)()\n # Make a copy so that we don't clobber the cached pieces!\n pieces = list(self._split(template))\n for index, piece in enumerate(pieces):\n if not index % 2:\n pieces[index], _ = unescape(piece)\n else:\n if len(piece) <= 3: # \\1 through \\99 (inclusive)\n group = int(piece[1:])\n else: # \\g<...>\n group = piece[3:-1]\n try:\n group = int(group)\n except ValueError:\n pass\n pieces[index] = self.__getitem__(group) or empty\n joined_pieces = empty.join(pieces)\n return joined_pieces\n\n def __getitem__(self, group):\n if not isinstance(group, int):\n try:\n group = self._regexp.groupindex[group]\n except KeyError:\n raise IndexError('bad group name')\n if not 0 <= group <= self._regexp.groups:\n raise IndexError('bad group index')\n span = self._spans[group]\n if span == _NULL_SPAN:\n return None\n return self._text[span[0]:span[1]]\n\n def group(self, *groups):\n if not groups:\n groups = (0,)\n items = (self.__getitem__(group) for group in groups)\n return next(items) if len(groups) == 1 else tuple(items)\n\n def groups(self, default=None):\n items = []\n for group in range(1, self._regexp.groups + 1):\n item = self.__getitem__(group)\n items.append(default if item is None else item)\n return tuple(items)\n\n def groupdict(self, default=None):\n items = []\n for group, index in self._regexp.groupindex.items():\n item = self.__getitem__(index)\n items.append((group, default) if item is None else (group, item))\n return dict(items)\n\n def start(self, group=0):\n if not 0 <= group <= self._regexp.groups:\n raise IndexError('bad group index')\n return self._spans[group][0]\n\n def end(self, group=0):\n if not 0 <= group <= self._regexp.groups:\n raise IndexError('bad group index')\n return self._spans[group][1]\n\n def span(self, group=0):\n if not 0 <= group <= self._regexp.groups:\n raise IndexError('bad group index')\n return self._spans[group]\n\n @property\n def re(self):\n return self._regexp\n\n @property\n def string(self):\n return self._text\n\n @property\n def pos(self):\n return self._pos\n\n @property\n def endpos(self):\n return self._endpos\n\n @property\n def lastindex(self):\n max_end = -1\n max_group = None\n # We look for the rightmost right parenthesis by keeping the first group\n # that ends at max_end because that is the leftmost/outermost group when\n # there are nested groups!\n for group in range(1, self._regexp.groups + 1):\n end = self._spans[group][1]\n if max_end < end:\n max_end = end\n max_group = group\n return max_group\n\n @property\n def lastgroup(self):\n max_group = self.lastindex\n if not max_group:\n return None\n for group, index in self._regexp.groupindex.items():\n if max_group == index:\n return group\n return None\n\n\nclass Set(object):\n \"\"\"A Pythonic wrapper around RE2::Set.\"\"\"\n\n __slots__ = ('_set')\n\n def __init__(self, anchor, options=None):\n options = options or Options()\n self._set = _re2.Set(anchor, options)\n\n @classmethod\n def SearchSet(cls, options=None):\n return cls(_Anchor.UNANCHORED, options=options)\n\n @classmethod\n def MatchSet(cls, options=None):\n return cls(_Anchor.ANCHOR_START, options=options)\n\n @classmethod\n def FullMatchSet(cls, options=None):\n return cls(_Anchor.ANCHOR_BOTH, options=options)\n\n def Add(self, pattern):\n if isinstance(pattern, str):\n encoded_pattern = _encode(pattern)\n index = self._set.Add(encoded_pattern)\n else:\n index = self._set.Add(pattern)\n if index == -1:\n raise error('failed to add %r to Set' % pattern)\n return index\n\n def Compile(self):\n if not self._set.Compile():\n raise error('failed to compile Set')\n\n def Match(self, text):\n if isinstance(text, str):\n encoded_text = _encode(text)\n matches = self._set.Match(encoded_text)\n else:\n matches = self._set.Match(text)\n return matches or None\n\n\nclass Filter(object):\n \"\"\"A Pythonic wrapper around FilteredRE2.\"\"\"\n\n __slots__ = ('_filter', '_patterns')\n\n def __init__(self):\n self._filter = _re2.Filter()\n self._patterns = []\n\n def Add(self, pattern, options=None):\n options = options or Options()\n if isinstance(pattern, str):\n encoded_pattern = _encode(pattern)\n index = self._filter.Add(encoded_pattern, options)\n else:\n index = self._filter.Add(pattern, options)\n if index == -1:\n raise error('failed to add %r to Filter' % pattern)\n self._patterns.append(pattern)\n return index\n\n def Compile(self):\n if not self._filter.Compile():\n raise error('failed to compile Filter')\n\n def Match(self, text, potential=False):\n if isinstance(text, str):\n encoded_text = _encode(text)\n matches = self._filter.Match(encoded_text, potential)\n else:\n matches = self._filter.Match(text, potential)\n return matches or None\n\n def re(self, index):\n if not 0 <= index < len(self._patterns):\n raise IndexError('bad index')\n proxy = object.__new__(_Regexp)\n proxy._pattern = self._patterns[index]\n proxy._regexp = self._filter.GetRE2(index)\n return proxy\n" }, { "path": "python/re2_test.py", "content": "# Copyright 2019 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\"\"\"Tests for google3.third_party.re2.python.re2.\"\"\"\n\nimport collections\nimport pickle\nimport re\n\nfrom absl.testing import absltest\nfrom absl.testing import parameterized\nimport re2\n\n\nclass OptionsTest(parameterized.TestCase):\n\n @parameterized.parameters(*re2.Options.NAMES)\n def test_option(self, name):\n options = re2.Options()\n value = getattr(options, name)\n if isinstance(value, re2.Options.Encoding):\n value = next(v for v in type(value).__members__.values() if v != value)\n elif isinstance(value, bool):\n value = not value\n elif isinstance(value, int):\n value = value + 1\n else:\n raise TypeError('option {!r}: {!r} {!r}'.format(name, type(value), value))\n setattr(options, name, value)\n self.assertEqual(value, getattr(options, name))\n\n\nclass Re2CompileTest(parameterized.TestCase):\n \"\"\"Contains tests that apply to the re2 module only.\n\n We disagree with Python on the string types of group names,\n so there is no point attempting to verify consistency.\n \"\"\"\n\n @parameterized.parameters(\n (u'(foo*)(?Pqux+)', 2, [(u'bar', 2)]),\n (b'(foo*)(?Pqux+)', 2, [(b'bar', 2)]),\n (u'(foo*)(?P<中文>qux+)', 2, [(u'中文', 2)]),\n )\n def test_compile(self, pattern, expected_groups, expected_groupindex):\n regexp = re2.compile(pattern)\n self.assertIs(regexp, re2.compile(pattern)) # cached\n self.assertIs(regexp, re2.compile(regexp)) # cached\n with self.assertRaisesRegex(re2.error,\n ('pattern is already compiled, so '\n 'options may not be specified')):\n options = re2.Options()\n options.log_errors = not options.log_errors\n re2.compile(regexp, options=options)\n self.assertIsNotNone(regexp.options)\n self.assertEqual(expected_groups, regexp.groups)\n self.assertDictEqual(dict(expected_groupindex), regexp.groupindex)\n\n def test_compile_with_options(self):\n options = re2.Options()\n options.max_mem = 100\n with self.assertRaisesRegex(re2.error, 'pattern too large'):\n re2.compile('.{1000}', options=options)\n\n def test_programsize_reverseprogramsize(self):\n regexp = re2.compile('a+b')\n self.assertEqual(7, regexp.programsize)\n self.assertEqual(7, regexp.reverseprogramsize)\n\n def test_programfanout_reverseprogramfanout(self):\n regexp = re2.compile('a+b')\n self.assertListEqual([1, 1], regexp.programfanout)\n self.assertListEqual([3], regexp.reverseprogramfanout)\n\n @parameterized.parameters(\n (u'abc', 0, None),\n (b'abc', 0, None),\n (u'abc', 10, (b'abc', b'abc')),\n (b'abc', 10, (b'abc', b'abc')),\n (u'ab*c', 10, (b'ab', b'ac')),\n (b'ab*c', 10, (b'ab', b'ac')),\n (u'ab+c', 10, (b'abb', b'abc')),\n (b'ab+c', 10, (b'abb', b'abc')),\n (u'ab?c', 10, (b'abc', b'ac')),\n (b'ab?c', 10, (b'abc', b'ac')),\n (u'.*', 10, (b'', b'\\xf4\\xbf\\xbf\\xc0')),\n (b'.*', 10, None),\n (u'\\\\C*', 10, None),\n (b'\\\\C*', 10, None),\n )\n def test_possiblematchrange(self, pattern, maxlen, expected_min_max):\n # For brevity, the string type of pattern determines the encoding.\n # It would otherwise be possible to have bytes with UTF8, but as per\n # the module docstring, it isn't permitted to have str with LATIN1.\n options = re2.Options()\n if isinstance(pattern, str):\n options.encoding = re2.Options.Encoding.UTF8\n else:\n options.encoding = re2.Options.Encoding.LATIN1\n regexp = re2.compile(pattern, options=options)\n if expected_min_max:\n self.assertEqual(expected_min_max, regexp.possiblematchrange(maxlen))\n else:\n with self.assertRaisesRegex(re2.error, 'failed to compute match range'):\n regexp.possiblematchrange(maxlen)\n\n\nParams = collections.namedtuple(\n 'Params', ('pattern', 'text', 'spans', 'search', 'match', 'fullmatch'))\n\nPARAMS = [\n Params(u'\\\\d+', u'Hello, world.', None, False, False, False),\n Params(b'\\\\d+', b'Hello, world.', None, False, False, False),\n Params(u'\\\\s+', u'Hello, world.', [(6, 7)], True, False, False),\n Params(b'\\\\s+', b'Hello, world.', [(6, 7)], True, False, False),\n Params(u'\\\\w+', u'Hello, world.', [(0, 5)], True, True, False),\n Params(b'\\\\w+', b'Hello, world.', [(0, 5)], True, True, False),\n Params(u'(\\\\d+)?', u'Hello, world.', [(0, 0), (-1, -1)], True, True, False),\n Params(b'(\\\\d+)?', b'Hello, world.', [(0, 0), (-1, -1)], True, True, False),\n Params(u'youtube(_device|_md|_gaia|_multiday|_multiday_gaia)?',\n u'youtube_ads', [(0, 7), (-1, -1)], True, True, False),\n Params(b'youtube(_device|_md|_gaia|_multiday|_multiday_gaia)?',\n b'youtube_ads', [(0, 7), (-1, -1)], True, True, False),\n]\n\n\ndef upper(match):\n return match.group().upper()\n\n\nclass ReRegexpTest(parameterized.TestCase):\n \"\"\"Contains tests that apply to the re and re2 modules.\"\"\"\n\n MODULE = re\n\n @parameterized.parameters((p.pattern,) for p in PARAMS)\n def test_pickle(self, pattern):\n regexp = self.MODULE.compile(pattern)\n rick = pickle.loads(pickle.dumps(regexp))\n self.assertEqual(regexp.pattern, rick.pattern)\n\n @parameterized.parameters(\n (p.pattern, p.text, (p.spans if p.search else None)) for p in PARAMS)\n def test_search(self, pattern, text, expected_spans):\n match = self.MODULE.search(pattern, text)\n if expected_spans is None:\n self.assertIsNone(match)\n else:\n spans = [match.span(group) for group in range(match.re.groups + 1)]\n self.assertListEqual(expected_spans, spans)\n\n def test_search_with_pos_and_endpos(self):\n regexp = self.MODULE.compile(u'.+') # empty string NOT allowed\n text = u'I \\u2665 RE2!'\n # Note that len(text) is the position of the empty string at the end of\n # text, so range() stops at len(text) + 1 in order to include len(text).\n for pos in range(len(text) + 1):\n for endpos in range(pos, len(text) + 1):\n match = regexp.search(text, pos=pos, endpos=endpos)\n if pos == endpos:\n self.assertIsNone(match)\n else:\n self.assertEqual(pos, match.pos)\n self.assertEqual(endpos, match.endpos)\n self.assertEqual(pos, match.start())\n self.assertEqual(endpos, match.end())\n self.assertTupleEqual((pos, endpos), match.span())\n\n def test_search_with_bogus_pos_and_endpos(self):\n regexp = self.MODULE.compile(u'.*') # empty string allowed\n text = u'I \\u2665 RE2!'\n\n match = regexp.search(text, pos=-100)\n self.assertEqual(0, match.pos)\n match = regexp.search(text, pos=100)\n self.assertEqual(8, match.pos)\n\n match = regexp.search(text, endpos=-100)\n self.assertEqual(0, match.endpos)\n match = regexp.search(text, endpos=100)\n self.assertEqual(8, match.endpos)\n\n match = regexp.search(text, pos=100, endpos=-100)\n self.assertIsNone(match)\n\n @parameterized.parameters(\n (p.pattern, p.text, (p.spans if p.match else None)) for p in PARAMS)\n def test_match(self, pattern, text, expected_spans):\n match = self.MODULE.match(pattern, text)\n if expected_spans is None:\n self.assertIsNone(match)\n else:\n spans = [match.span(group) for group in range(match.re.groups + 1)]\n self.assertListEqual(expected_spans, spans)\n\n @parameterized.parameters(\n (p.pattern, p.text, (p.spans if p.fullmatch else None)) for p in PARAMS)\n def test_fullmatch(self, pattern, text, expected_spans):\n match = self.MODULE.fullmatch(pattern, text)\n if expected_spans is None:\n self.assertIsNone(match)\n else:\n spans = [match.span(group) for group in range(match.re.groups + 1)]\n self.assertListEqual(expected_spans, spans)\n\n @parameterized.parameters(\n (u'', u'', [(0, 0)]),\n (b'', b'', [(0, 0)]),\n (u'', u'x', [(0, 0), (1, 1)]),\n (b'', b'x', [(0, 0), (1, 1)]),\n (u'', u'xy', [(0, 0), (1, 1), (2, 2)]),\n (b'', b'xy', [(0, 0), (1, 1), (2, 2)]),\n (u'.', u'xy', [(0, 1), (1, 2)]),\n (b'.', b'xy', [(0, 1), (1, 2)]),\n (u'x', u'xy', [(0, 1)]),\n (b'x', b'xy', [(0, 1)]),\n (u'y', u'xy', [(1, 2)]),\n (b'y', b'xy', [(1, 2)]),\n (u'z', u'xy', []),\n (b'z', b'xy', []),\n (u'\\\\w*', u'Hello, world.', [(0, 5), (5, 5), (6, 6), (7, 12), (12, 12),\n (13, 13)]),\n (b'\\\\w*', b'Hello, world.', [(0, 5), (5, 5), (6, 6), (7, 12), (12, 12),\n (13, 13)]),\n )\n def test_finditer(self, pattern, text, expected_matches):\n matches = [match.span() for match in self.MODULE.finditer(pattern, text)]\n self.assertListEqual(expected_matches, matches)\n\n @parameterized.parameters(\n (u'\\\\w\\\\w+', u'Hello, world.', [u'Hello', u'world']),\n (b'\\\\w\\\\w+', b'Hello, world.', [b'Hello', b'world']),\n (u'(\\\\w)\\\\w+', u'Hello, world.', [u'H', u'w']),\n (b'(\\\\w)\\\\w+', b'Hello, world.', [b'H', b'w']),\n (u'(\\\\w)(\\\\w+)', u'Hello, world.', [(u'H', u'ello'), (u'w', u'orld')]),\n (b'(\\\\w)(\\\\w+)', b'Hello, world.', [(b'H', b'ello'), (b'w', b'orld')]),\n (u'(\\\\w)(\\\\w+)?', u'Hello, w.', [(u'H', u'ello'), (u'w', u'')]),\n (b'(\\\\w)(\\\\w+)?', b'Hello, w.', [(b'H', b'ello'), (b'w', b'')]),\n )\n def test_findall(self, pattern, text, expected_matches):\n matches = self.MODULE.findall(pattern, text)\n self.assertListEqual(expected_matches, matches)\n\n @parameterized.parameters(\n (u'\\\\W+', u'Hello, world.', -1, [u'Hello, world.']),\n (b'\\\\W+', b'Hello, world.', -1, [b'Hello, world.']),\n (u'\\\\W+', u'Hello, world.', 0, [u'Hello', u'world', u'']),\n (b'\\\\W+', b'Hello, world.', 0, [b'Hello', b'world', b'']),\n (u'\\\\W+', u'Hello, world.', 1, [u'Hello', u'world.']),\n (b'\\\\W+', b'Hello, world.', 1, [b'Hello', b'world.']),\n (u'(\\\\W+)', u'Hello, world.', -1, [u'Hello, world.']),\n (b'(\\\\W+)', b'Hello, world.', -1, [b'Hello, world.']),\n (u'(\\\\W+)', u'Hello, world.', 0, [u'Hello', u', ', u'world', u'.', u'']),\n (b'(\\\\W+)', b'Hello, world.', 0, [b'Hello', b', ', b'world', b'.', b'']),\n (u'(\\\\W+)', u'Hello, world.', 1, [u'Hello', u', ', u'world.']),\n (b'(\\\\W+)', b'Hello, world.', 1, [b'Hello', b', ', b'world.']),\n )\n def test_split(self, pattern, text, maxsplit, expected_pieces):\n pieces = self.MODULE.split(pattern, text, maxsplit)\n self.assertListEqual(expected_pieces, pieces)\n\n @parameterized.parameters(\n (u'\\\\w+', upper, u'Hello, world.', -1, u'Hello, world.', 0),\n (b'\\\\w+', upper, b'Hello, world.', -1, b'Hello, world.', 0),\n (u'\\\\w+', upper, u'Hello, world.', 0, u'HELLO, WORLD.', 2),\n (b'\\\\w+', upper, b'Hello, world.', 0, b'HELLO, WORLD.', 2),\n (u'\\\\w+', upper, u'Hello, world.', 1, u'HELLO, world.', 1),\n (b'\\\\w+', upper, b'Hello, world.', 1, b'HELLO, world.', 1),\n (u'\\\\w+', u'MEEP', u'Hello, world.', -1, u'Hello, world.', 0),\n (b'\\\\w+', b'MEEP', b'Hello, world.', -1, b'Hello, world.', 0),\n (u'\\\\w+', u'MEEP', u'Hello, world.', 0, u'MEEP, MEEP.', 2),\n (b'\\\\w+', b'MEEP', b'Hello, world.', 0, b'MEEP, MEEP.', 2),\n (u'\\\\w+', u'MEEP', u'Hello, world.', 1, u'MEEP, world.', 1),\n (b'\\\\w+', b'MEEP', b'Hello, world.', 1, b'MEEP, world.', 1),\n (u'\\\\\\\\', u'\\\\\\\\\\\\\\\\', u'Hello,\\\\world.', 0, u'Hello,\\\\\\\\world.', 1),\n (b'\\\\\\\\', b'\\\\\\\\\\\\\\\\', b'Hello,\\\\world.', 0, b'Hello,\\\\\\\\world.', 1),\n )\n def test_subn_sub(self, pattern, repl, text, count, expected_joined_pieces,\n expected_numsplit):\n joined_pieces, numsplit = self.MODULE.subn(pattern, repl, text, count)\n self.assertEqual(expected_joined_pieces, joined_pieces)\n self.assertEqual(expected_numsplit, numsplit)\n\n joined_pieces = self.MODULE.sub(pattern, repl, text, count)\n self.assertEqual(expected_joined_pieces, joined_pieces)\n\n\nclass Re2RegexpTest(ReRegexpTest):\n \"\"\"Contains tests that apply to the re2 module only.\"\"\"\n\n MODULE = re2\n\n def test_compile_with_latin1_encoding(self):\n options = re2.Options()\n options.encoding = re2.Options.Encoding.LATIN1\n with self.assertRaisesRegex(re2.error,\n ('string type of pattern is str, but '\n 'encoding specified in options is LATIN1')):\n re2.compile(u'.?', options=options)\n\n # ... whereas this is fine, of course.\n re2.compile(b'.?', options=options)\n\n @parameterized.parameters(\n (u'\\\\p{Lo}', u'\\u0ca0_\\u0ca0', [(0, 1), (2, 3)]),\n (b'\\\\p{Lo}', b'\\xe0\\xb2\\xa0_\\xe0\\xb2\\xa0', [(0, 3), (4, 7)]),\n )\n def test_finditer_with_utf8(self, pattern, text, expected_matches):\n matches = [match.span() for match in self.MODULE.finditer(pattern, text)]\n self.assertListEqual(expected_matches, matches)\n\n def test_purge(self):\n re2.compile('Goodbye, world.')\n self.assertGreater(re2._Regexp._make.cache_info().currsize, 0)\n re2.purge()\n self.assertEqual(re2._Regexp._make.cache_info().currsize, 0)\n\n def test_options(self):\n opt = re2.Options()\n opt.case_sensitive = False\n r = re2.compile('test', opt)\n self.assertIsNotNone(r.search('TEST'))\n self.assertIsNotNone(re2.search(r, 'TEST'))\n\nclass Re2EscapeTest(parameterized.TestCase):\n \"\"\"Contains tests that apply to the re2 module only.\n\n We disagree with Python on the escaping of some characters,\n so there is no point attempting to verify consistency.\n \"\"\"\n\n @parameterized.parameters(\n (u'a*b+c?', u'a\\\\*b\\\\+c\\\\?'),\n (b'a*b+c?', b'a\\\\*b\\\\+c\\\\?'),\n )\n def test_escape(self, pattern, expected_escaped):\n escaped = re2.escape(pattern)\n self.assertEqual(expected_escaped, escaped)\n\n\nclass ReMatchTest(parameterized.TestCase):\n \"\"\"Contains tests that apply to the re and re2 modules.\"\"\"\n\n MODULE = re\n\n def test_expand(self):\n pattern = u'(?P[\\u2600-\\u26ff]+).*?(?P

[^\\\\s\\\\w]+)'\n text = u'I \\u2665 RE2!\\n'\n match = self.MODULE.search(pattern, text)\n\n self.assertEqual(u'\\u2665\\n!', match.expand(u'\\\\1\\\\n\\\\2'))\n self.assertEqual(u'\\u2665\\n!', match.expand(u'\\\\g<1>\\\\n\\\\g<2>'))\n self.assertEqual(u'\\u2665\\n!', match.expand(u'\\\\g\\\\n\\\\g

'))\n self.assertEqual(u'\\\\1\\\\2\\n\\u2665!', match.expand(u'\\\\\\\\1\\\\\\\\2\\\\n\\\\1\\\\2'))\n\n def test_expand_with_octal(self):\n pattern = u'()()()()()()()()()(\\\\w+)'\n text = u'Hello, world.'\n match = self.MODULE.search(pattern, text)\n\n self.assertEqual(u'Hello\\n', match.expand(u'\\\\g<0>\\\\n'))\n self.assertEqual(u'Hello\\n', match.expand(u'\\\\g<10>\\\\n'))\n\n self.assertEqual(u'\\x00\\n', match.expand(u'\\\\0\\\\n'))\n self.assertEqual(u'\\x00\\n', match.expand(u'\\\\00\\\\n'))\n self.assertEqual(u'\\x00\\n', match.expand(u'\\\\000\\\\n'))\n self.assertEqual(u'\\x000\\n', match.expand(u'\\\\0000\\\\n'))\n\n self.assertEqual(u'\\n', match.expand(u'\\\\1\\\\n'))\n self.assertEqual(u'Hello\\n', match.expand(u'\\\\10\\\\n'))\n self.assertEqual(u'@\\n', match.expand(u'\\\\100\\\\n'))\n self.assertEqual(u'@0\\n', match.expand(u'\\\\1000\\\\n'))\n\n def test_getitem_group_groups_groupdict(self):\n pattern = u'(?P[\\u2600-\\u26ff]+).*?(?P

[^\\\\s\\\\w]+)'\n text = u'Hello, world.\\nI \\u2665 RE2!\\nGoodbye, world.\\n'\n match = self.MODULE.search(pattern, text)\n\n self.assertEqual(u'\\u2665 RE2!', match[0])\n self.assertEqual(u'\\u2665', match[1])\n self.assertEqual(u'!', match[2])\n self.assertEqual(u'\\u2665', match[u'S'])\n self.assertEqual(u'!', match[u'P'])\n\n self.assertEqual(u'\\u2665 RE2!', match.group())\n self.assertEqual(u'\\u2665 RE2!', match.group(0))\n self.assertEqual(u'\\u2665', match.group(1))\n self.assertEqual(u'!', match.group(2))\n self.assertEqual(u'\\u2665', match.group(u'S'))\n self.assertEqual(u'!', match.group(u'P'))\n\n self.assertTupleEqual((u'\\u2665', u'!'), match.group(1, 2))\n self.assertTupleEqual((u'\\u2665', u'!'), match.group(u'S', u'P'))\n self.assertTupleEqual((u'\\u2665', u'!'), match.groups())\n self.assertDictEqual({u'S': u'\\u2665', u'P': u'!'}, match.groupdict())\n\n def test_bogus_group_start_end_and_span(self):\n pattern = u'(?P[\\u2600-\\u26ff]+).*?(?P

[^\\\\s\\\\w]+)'\n text = u'I \\u2665 RE2!\\n'\n match = self.MODULE.search(pattern, text)\n\n self.assertRaises(IndexError, match.group, -1)\n self.assertRaises(IndexError, match.group, 3)\n self.assertRaises(IndexError, match.group, 'X')\n\n self.assertRaises(IndexError, match.start, -1)\n self.assertRaises(IndexError, match.start, 3)\n\n self.assertRaises(IndexError, match.end, -1)\n self.assertRaises(IndexError, match.end, 3)\n\n self.assertRaises(IndexError, match.span, -1)\n self.assertRaises(IndexError, match.span, 3)\n\n @parameterized.parameters(\n (u'((a)(b))((c)(d))', u'foo bar qux', None, None),\n (u'(?P(a)(b))((c)(d))', u'foo abcd qux', 4, None),\n (u'(?P(a)(b))(?P(c)(d))', u'foo abcd qux', 4, 'four'),\n )\n def test_lastindex_lastgroup(self, pattern, text, expected_lastindex,\n expected_lastgroup):\n match = self.MODULE.search(pattern, text)\n if expected_lastindex is None:\n self.assertIsNone(match)\n else:\n self.assertEqual(expected_lastindex, match.lastindex)\n self.assertEqual(expected_lastgroup, match.lastgroup)\n\n\nclass Re2MatchTest(ReMatchTest):\n \"\"\"Contains tests that apply to the re2 module only.\"\"\"\n\n MODULE = re2\n\n\nclass SetTest(absltest.TestCase):\n\n def test_search(self):\n s = re2.Set.SearchSet()\n self.assertEqual(0, s.Add('\\\\d+'))\n self.assertEqual(1, s.Add('\\\\s+'))\n self.assertEqual(2, s.Add('\\\\w+'))\n self.assertRaises(re2.error, s.Add, '(MEEP')\n s.Compile()\n self.assertItemsEqual([1, 2], s.Match('Hello, world.'))\n\n def test_match(self):\n s = re2.Set.MatchSet()\n self.assertEqual(0, s.Add('\\\\d+'))\n self.assertEqual(1, s.Add('\\\\s+'))\n self.assertEqual(2, s.Add('\\\\w+'))\n self.assertRaises(re2.error, s.Add, '(MEEP')\n s.Compile()\n self.assertItemsEqual([2], s.Match('Hello, world.'))\n\n def test_fullmatch(self):\n s = re2.Set.FullMatchSet()\n self.assertEqual(0, s.Add('\\\\d+'))\n self.assertEqual(1, s.Add('\\\\s+'))\n self.assertEqual(2, s.Add('\\\\w+'))\n self.assertRaises(re2.error, s.Add, '(MEEP')\n s.Compile()\n self.assertIsNone(s.Match('Hello, world.'))\n\n\nclass FilterTest(absltest.TestCase):\n\n def test_match(self):\n f = re2.Filter()\n self.assertEqual(0, f.Add('Hello, \\\\w+\\\\.'))\n self.assertEqual(1, f.Add('\\\\w+, world\\\\.'))\n self.assertEqual(2, f.Add('Goodbye, \\\\w+\\\\.'))\n self.assertRaises(re2.error, f.Add, '(MEEP')\n f.Compile()\n self.assertItemsEqual([0, 1], f.Match('Hello, world.', potential=True))\n self.assertItemsEqual([0, 1], f.Match('HELLO, WORLD.', potential=True))\n self.assertItemsEqual([0, 1], f.Match('Hello, world.'))\n self.assertIsNone(f.Match('HELLO, WORLD.'))\n\n self.assertRaises(IndexError, f.re, -1)\n self.assertRaises(IndexError, f.re, 3)\n self.assertEqual('Goodbye, \\\\w+\\\\.', f.re(2).pattern)\n # Verify whether the underlying RE2 object is usable.\n self.assertEqual(0, f.re(2).groups)\n\n def test_issue_484(self):\n # Previously, the shim would dereference a null pointer and crash.\n f = re2.Filter()\n with self.assertRaisesRegex(re2.error,\n r'Match\\(\\) called before compiling'):\n f.Match('')\n\n\nif __name__ == '__main__':\n absltest.main()\n" }, { "path": "python/setup.py", "content": "# Copyright 2019 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\nimport os\nimport re\nimport setuptools\nimport setuptools.command.build_ext\nimport shutil\n\nlong_description = r\"\"\"A drop-in replacement for the re module.\n\nIt uses RE2 under the hood, of course, so various PCRE features\n(e.g. backreferences, look-around assertions) are not supported.\nSee https://github.com/google/re2/wiki/Syntax for the canonical\nreference, but known syntactic \"gotchas\" relative to Python are:\n\n * PCRE supports \\Z and \\z; RE2 supports \\z; Python supports \\z,\n but calls it \\Z. You must rewrite \\Z to \\z in pattern strings.\n\nKnown differences between this module's API and the re module's API:\n\n * The error class does not provide any error information as attributes.\n * The Options class replaces the re module's flags with RE2's options as\n gettable/settable properties. Please see re2.h for their documentation.\n * The pattern string and the input string do not have to be the same type.\n Any str will be encoded to UTF-8.\n * The pattern string cannot be str if the options specify Latin-1 encoding.\n\nKnown issues with regard to building the C++ extension:\n\n * Building requires RE2 to be installed on your system.\n On Debian, for example, install the libre2-dev package.\n * Building requires pybind11 to be installed on your system OR venv.\n On Debian, for example, install the pybind11-dev package.\n For a venv, install the pybind11 package from PyPI.\n * Building on macOS is known to work, but has been known to fail.\n For example, the system Python may not know which compiler flags\n to set when building bindings for software installed by Homebrew;\n see https://docs.brew.sh/Homebrew-and-Python#brewed-python-modules.\n * Building on Windows has not been tested yet and will probably fail.\n\"\"\"\n\n\nclass BuildExt(setuptools.command.build_ext.build_ext):\n\n def build_extension(self, ext):\n if 'GITHUB_ACTIONS' not in os.environ:\n return super().build_extension(ext)\n\n cmd = ['bazel', 'build']\n try:\n cpu = os.environ['BAZEL_CPU']\n cmd.append(f'--cpu={cpu}')\n cmd.append(f'--platforms=//python:{cpu}')\n if cpu == 'x64_x86_windows':\n # Register the local 32-bit C++ toolchain with highest priority.\n # (This is likely to break in some release of Bazel after 7.0.0,\n # but this special case can hopefully be entirely removed then.)\n cmd.append(f'--extra_toolchains=@local_config_cc//:cc-toolchain-{cpu}')\n except KeyError:\n pass\n try:\n ver = os.environ['MACOSX_DEPLOYMENT_TARGET']\n cmd.append(f'--macos_minimum_os={ver}')\n except KeyError:\n pass\n # Register the local Python toolchains with highest priority.\n cmd.append('--extra_toolchains=//python/toolchains:all')\n cmd += ['--compilation_mode=opt', '--', ':all']\n self.spawn(cmd)\n\n # This ensures that f'_re2.{importlib.machinery.EXTENSION_SUFFIXES[0]}'\n # is the filename in the destination directory, which is what's needed.\n shutil.copyfile('../bazel-bin/python/_re2.so',\n self.get_ext_fullpath(ext.name))\n\n cmd = ['bazel', 'clean', '--expunge']\n self.spawn(cmd)\n\n\ndef options():\n bdist_wheel = {}\n try:\n bdist_wheel['plat_name'] = os.environ['PLAT_NAME']\n except KeyError:\n pass\n return {'bdist_wheel': bdist_wheel}\n\n\ndef include_dirs():\n try:\n import pybind11\n yield pybind11.get_include()\n except ModuleNotFoundError:\n pass\n\n\next_module = setuptools.Extension(\n name='_re2',\n sources=['_re2.cc'],\n include_dirs=list(include_dirs()),\n libraries=['re2'],\n extra_compile_args=['-fvisibility=hidden'],\n)\n\n# We need `re2` to be a package, not a module, because it appears that\n# modules can't have `.pyi` files, so munge the module into a package.\nPACKAGE = 're2'\ntry:\n # If we are building from the sdist, we are already in package form.\n if not os.path.exists('PKG-INFO'):\n os.makedirs(PACKAGE)\n for filename in (\n 're2.py',\n # TODO(junyer): Populate as per https://github.com/google/re2/issues/496.\n # 're2.pyi',\n # '_re2.pyi',\n ):\n with open(filename, 'r') as file:\n contents = file.read()\n filename = re.sub(r'^re2(?=\\.py)', '__init__', filename)\n contents = re.sub(r'^(?=import _)', 'from . ', contents, flags=re.MULTILINE)\n with open(f'{PACKAGE}/{filename}', 'x') as file:\n file.write(contents)\n # TODO(junyer): Populate as per https://github.com/google/re2/issues/496.\n # with open(f'{PACKAGE}/py.typed', 'x') as file:\n # pass\n\n setuptools.setup(\n name='google-re2',\n version='1.1.20251105',\n description='RE2 Python bindings',\n long_description=long_description,\n long_description_content_type='text/plain',\n author='The RE2 Authors',\n author_email='re2-dev@googlegroups.com',\n url='https://github.com/google/re2',\n packages=[PACKAGE],\n ext_package=PACKAGE,\n ext_modules=[ext_module],\n # Note: Keep the minimum Python version, which appears twice below, in sync with ../.github/workflows/python.yml.\n classifiers=[\n 'Development Status :: 5 - Production/Stable',\n 'Intended Audience :: Developers',\n 'License :: OSI Approved :: BSD License',\n 'Programming Language :: C++',\n 'Programming Language :: Python :: 3.10',\n ],\n options=options(),\n cmdclass={'build_ext': BuildExt},\n python_requires='~=3.10',\n )\nexcept:\n raise\nelse:\n # If we are building from the sdist, we are already in package form.\n if not os.path.exists('PKG-INFO'):\n shutil.rmtree(PACKAGE)\n" }, { "path": "python/toolchains/generate.py", "content": "# Copyright 2019 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\nimport os\nimport shutil\nimport sys\nimport sysconfig\n\n\ndef generate():\n include = sysconfig.get_path('include')\n libs = os.path.join(include, '../libs')\n\n mydir = os.path.dirname(sys.argv[0]) or '.'\n shutil.copytree(include, f'{mydir}/include')\n try:\n shutil.copytree(libs, f'{mydir}/libs')\n except FileNotFoundError:\n # We must not be running on Windows. :)\n pass\n\n with open(f'{mydir}/BUILD.bazel', 'x') as file:\n file.write(\n \"\"\"\\\nload(\"@rules_cc//cc:cc_import.bzl\", \"cc_import\")\nload(\"@rules_cc//cc:cc_library.bzl\", \"cc_library\")\nload(\"@rules_python//python/cc:py_cc_toolchain.bzl\", \"py_cc_toolchain\")\nload(\"@rules_python//python:py_runtime.bzl\", \"py_runtime\")\nload(\"@rules_python//python:py_runtime_pair.bzl\", \"py_runtime_pair\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ntoolchain(\n name = \"py\",\n toolchain = \":py_toolchain\",\n toolchain_type = \"@rules_python//python:toolchain_type\",\n)\n\npy_runtime_pair(\n name = \"py_toolchain\",\n py3_runtime = \":interpreter\",\n)\n\npy_runtime(\n name = \"interpreter\",\n interpreter_path = \"{interpreter_path}\",\n interpreter_version_info = {{\n \"major\": \"{major}\",\n \"minor\": \"{minor}\",\n }},\n python_version = \"PY3\",\n)\n\ntoolchain(\n name = \"py_cc\",\n toolchain = \":py_cc_toolchain\",\n toolchain_type = \"@rules_python//python/cc:toolchain_type\",\n)\n\npy_cc_toolchain(\n name = \"py_cc_toolchain\",\n headers = \":headers\",\n libs = \":libraries\",\n python_version = \"{major}.{minor}\",\n)\n\ncc_library(\n name = \"headers\",\n hdrs = glob([\"include/**/*.h\"]),\n includes = [\"include\"],\n deps = select({{\n \"@platforms//os:windows\": [\":interface_library\"],\n \"//conditions:default\": [],\n }}),\n)\n\ncc_import(\n name = \"interface_library\",\n interface_library = select({{\n \"@platforms//os:windows\": \"libs/python{major}{minor}.lib\",\n \"//conditions:default\": None,\n }}),\n system_provided = True,\n)\n\n# Not actually necessary for our purposes. :)\ncc_library(\n name = \"libraries\",\n)\n\"\"\".format(\n interpreter_path=sys.executable.replace('\\\\', '/'),\n major=sys.version_info.major,\n minor=sys.version_info.minor,\n )\n )\n\n\nif __name__ == '__main__':\n generate()\n" }, { "path": "re2/bitmap256.cc", "content": "// Copyright 2023 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \"re2/bitmap256.h\"\n\n#include \n\n#include \"absl/log/absl_check.h\"\n\nnamespace re2 {\n\nint Bitmap256::FindNextSetBit(int c) const {\n ABSL_DCHECK_GE(c, 0);\n ABSL_DCHECK_LE(c, 255);\n\n // Check the word that contains the bit. Mask out any lower bits.\n int i = c / 64;\n uint64_t word = words_[i] & (~uint64_t{0} << (c % 64));\n if (word != 0)\n return (i * 64) + FindLSBSet(word);\n\n // Check any following words.\n i++;\n switch (i) {\n case 1:\n if (words_[1] != 0)\n return (1 * 64) + FindLSBSet(words_[1]);\n [[fallthrough]];\n case 2:\n if (words_[2] != 0)\n return (2 * 64) + FindLSBSet(words_[2]);\n [[fallthrough]];\n case 3:\n if (words_[3] != 0)\n return (3 * 64) + FindLSBSet(words_[3]);\n [[fallthrough]];\n default:\n return -1;\n }\n}\n\n} // namespace re2\n" }, { "path": "re2/bitmap256.h", "content": "// Copyright 2016 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_BITMAP256_H_\n#define RE2_BITMAP256_H_\n\n#include \n#include \n\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n\n#ifdef _MSC_VER\n#include \n#endif\n\nnamespace re2 {\n\nclass Bitmap256 {\n public:\n Bitmap256() {\n Clear();\n }\n\n // Clears all of the bits.\n void Clear() {\n memset(words_, 0, sizeof words_);\n }\n\n // Tests the bit with index c.\n bool Test(int c) const {\n ABSL_DCHECK_GE(c, 0);\n ABSL_DCHECK_LE(c, 255);\n\n return (words_[c / 64] & (uint64_t{1} << (c % 64))) != 0;\n }\n\n // Sets the bit with index c.\n void Set(int c) {\n ABSL_DCHECK_GE(c, 0);\n ABSL_DCHECK_LE(c, 255);\n\n words_[c / 64] |= (uint64_t{1} << (c % 64));\n }\n\n // Finds the next non-zero bit with index >= c.\n // Returns -1 if no such bit exists.\n int FindNextSetBit(int c) const;\n\n private:\n // Finds the least significant non-zero bit in n.\n static int FindLSBSet(uint64_t n) {\n ABSL_DCHECK_NE(n, uint64_t{0});\n#if defined(__GNUC__)\n return __builtin_ctzll(n);\n#elif defined(_MSC_VER) && defined(_M_X64)\n unsigned long c;\n _BitScanForward64(&c, n);\n return static_cast(c);\n#elif defined(_MSC_VER) && defined(_M_IX86)\n unsigned long c;\n if (static_cast(n) != 0) {\n _BitScanForward(&c, static_cast(n));\n return static_cast(c);\n } else {\n _BitScanForward(&c, static_cast(n >> 32));\n return static_cast(c) + 32;\n }\n#else\n int c = 63;\n for (int shift = 1 << 5; shift != 0; shift >>= 1) {\n uint64_t word = n << shift;\n if (word != 0) {\n n = word;\n c -= shift;\n }\n }\n return c;\n#endif\n }\n\n uint64_t words_[4];\n};\n\n} // namespace re2\n\n#endif // RE2_BITMAP256_H_\n" }, { "path": "re2/bitstate.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Tested by search_test.cc, exhaustive_test.cc, tester.cc\n\n// Prog::SearchBitState is a regular expression search with submatch\n// tracking for small regular expressions and texts. Similarly to\n// testing/backtrack.cc, it allocates a bitmap with (count of\n// lists) * (length of text) bits to make sure it never explores the\n// same (instruction list, character position) multiple times. This\n// limits the search to run in time linear in the length of the text.\n//\n// Unlike testing/backtrack.cc, SearchBitState is not recursive\n// on the text.\n//\n// SearchBitState is a fast replacement for the NFA code on small\n// regexps and texts when SearchOnePass cannot be used.\n\n#include \n#include \n#include \n\n#include \n#include \n\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/prog.h\"\n#include \"re2/regexp.h\"\n\nnamespace re2 {\n\nstruct Job {\n int id;\n int rle; // run length encoding\n const char* p;\n};\n\nclass BitState {\n public:\n explicit BitState(Prog* prog);\n\n // The usual Search prototype.\n // Can only call Search once per BitState.\n bool Search(absl::string_view text, absl::string_view context, bool anchored,\n bool longest, absl::string_view* submatch, int nsubmatch);\n\n private:\n static inline bool ShouldVisit(absl::string_view text, uint64_t* visited, uint16_t id, const char* p);\n void Push(int id, const char* p);\n void GrowStack();\n bool TrySearch(int id, const char* p);\n\n // Search parameters\n Prog* prog_; // program being run\n absl::string_view text_; // text being searched\n absl::string_view context_; // greater context of text being searched\n bool anchored_; // whether search is anchored at text.begin()\n bool longest_; // whether search wants leftmost-longest match\n bool endmatch_; // whether match must end at text.end()\n absl::string_view* submatch_; // submatches to fill in\n int nsubmatch_; // # of submatches to fill in\n\n // Search state\n static constexpr int kVisitedBits = 64;\n PODArray visited_; // bitmap: (list ID, char*) pairs visited\n PODArray cap_; // capture registers\n PODArray job_; // stack of text positions to explore\n int njob_; // stack size\n\n BitState(const BitState&) = delete;\n BitState& operator=(const BitState&) = delete;\n};\n\nBitState::BitState(Prog* prog)\n : prog_(prog),\n anchored_(false),\n longest_(false),\n endmatch_(false),\n submatch_(NULL),\n nsubmatch_(0),\n njob_(0) {\n}\n\n// Given the text being searched and current visited state,\n// as well as a list ID, should the search visit the (list ID, p) pair?\n// If so, remember that it was visited so that the next time,\n// we don't repeat the visit.\n// We pass text and visited to this as a static method so that the\n// caller can do those loads once instead of this code dereferencing\n// them multiple times.\nbool BitState::ShouldVisit(absl::string_view text, uint64_t* visited, uint16_t list_id, const char* p) {\n int n = list_id * static_cast(text.size()+1) +\n static_cast(p-text.data());\n if (visited[n/kVisitedBits] & (uint64_t{1} << (n & (kVisitedBits-1))))\n return false;\n visited[n/kVisitedBits] |= uint64_t{1} << (n & (kVisitedBits-1));\n return true;\n}\n\n// Grow the stack.\nvoid BitState::GrowStack() {\n PODArray tmp(2*job_.size());\n memmove(tmp.data(), job_.data(), njob_*sizeof job_[0]);\n job_ = std::move(tmp);\n}\n\n// Push (id, p) onto the stack, growing it if necessary.\nvoid BitState::Push(int id, const char* p) {\n if (njob_ >= job_.size()) {\n GrowStack();\n if (njob_ >= job_.size()) {\n ABSL_LOG(DFATAL) << \"GrowStack() failed: \"\n << \"njob_ = \" << njob_ << \", \"\n << \"job_.size() = \" << job_.size();\n return;\n }\n }\n\n // If id < 0, it's undoing a Capture,\n // so we mustn't interfere with that.\n if (id >= 0 && njob_ > 0) {\n Job* top = &job_[njob_-1];\n if (id == top->id &&\n p == top->p + top->rle + 1 &&\n top->rle < std::numeric_limits::max()) {\n ++top->rle;\n return;\n }\n }\n\n Job* top = &job_[njob_++];\n top->id = id;\n top->rle = 0;\n top->p = p;\n}\n\n// Try a search from instruction id0 in state p0.\n// Return whether it succeeded.\nbool BitState::TrySearch(int id0, const char* p0) {\n bool matched = false;\n const char* end = text_.data() + text_.size();\n uint16_t* list_heads = prog_->list_heads();\n uint64_t* visited = visited_.data();\n njob_ = 0;\n // Push() no longer checks ShouldVisit(),\n // so we must perform the check ourselves.\n if (ShouldVisit(text_, visited, list_heads[id0], p0))\n Push(id0, p0);\n while (njob_ > 0) {\n // Pop job off stack.\n --njob_;\n int id = job_[njob_].id;\n int& rle = job_[njob_].rle;\n const char* p = job_[njob_].p;\n\n if (id < 0) {\n // Undo the Capture.\n cap_[prog_->inst(-id)->cap()] = p;\n continue;\n }\n\n if (rle > 0) {\n p += rle;\n // Revivify job on stack.\n --rle;\n ++njob_;\n }\n\n Loop:\n // Visit id, p.\n Prog::Inst* ip = prog_->inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"Unexpected opcode: \" << ip->opcode();\n return false;\n\n case kInstFail:\n break;\n\n case kInstAltMatch:\n if (ip->greedy(prog_)) {\n // out1 is the Match instruction.\n id = ip->out1();\n p = end;\n goto Loop;\n }\n if (longest_) {\n // ip must be non-greedy...\n // out is the Match instruction.\n id = ip->out();\n p = end;\n goto Loop;\n }\n goto Next;\n\n case kInstByteRange: {\n int c = -1;\n if (p < end)\n c = *p & 0xFF;\n if (!ip->Matches(c))\n goto Next;\n\n if (ip->hint() != 0)\n Push(id+ip->hint(), p); // try the next when we're done\n id = ip->out();\n p++;\n goto CheckAndLoop;\n }\n\n case kInstCapture:\n if (!ip->last())\n Push(id+1, p); // try the next when we're done\n\n if (0 <= ip->cap() && ip->cap() < cap_.size()) {\n // Capture p to register, but save old value first.\n Push(-id, cap_[ip->cap()]); // undo when we're done\n cap_[ip->cap()] = p;\n }\n\n id = ip->out();\n goto CheckAndLoop;\n\n case kInstEmptyWidth:\n if (ip->empty() & ~Prog::EmptyFlags(context_, p))\n goto Next;\n\n if (!ip->last())\n Push(id+1, p); // try the next when we're done\n id = ip->out();\n goto CheckAndLoop;\n\n case kInstNop:\n if (!ip->last())\n Push(id+1, p); // try the next when we're done\n id = ip->out();\n\n CheckAndLoop:\n // Sanity check: id is the head of its list, which must\n // be the case if id-1 is the last of *its* list. :)\n ABSL_DCHECK(id == 0 || prog_->inst(id-1)->last());\n if (ShouldVisit(text_, visited, list_heads[id], p))\n goto Loop;\n break;\n\n case kInstMatch: {\n if (endmatch_ && p != end)\n goto Next;\n\n // We found a match. If the caller doesn't care\n // where the match is, no point going further.\n if (nsubmatch_ == 0)\n return true;\n\n // Record best match so far.\n // Only need to check end point, because this entire\n // call is only considering one start position.\n matched = true;\n cap_[1] = p;\n if (submatch_[0].data() == NULL ||\n (longest_ && p > submatch_[0].data() + submatch_[0].size())) {\n for (int i = 0; i < nsubmatch_; i++)\n submatch_[i] = absl::string_view(\n cap_[2 * i],\n static_cast(cap_[2 * i + 1] - cap_[2 * i]));\n }\n\n // If going for first match, we're done.\n if (!longest_)\n return true;\n\n // If we used the entire text, no longer match is possible.\n if (p == end)\n return true;\n\n // Otherwise, continue on in hope of a longer match.\n // Note the absence of the ShouldVisit() check here\n // due to execution remaining in the same list.\n Next:\n if (!ip->last()) {\n id++;\n goto Loop;\n }\n break;\n }\n }\n }\n return matched;\n}\n\n// Search text (within context) for prog_.\nbool BitState::Search(absl::string_view text, absl::string_view context,\n bool anchored, bool longest, absl::string_view* submatch,\n int nsubmatch) {\n // Search parameters.\n text_ = text;\n context_ = context;\n if (context_.data() == NULL)\n context_ = text;\n if (prog_->anchor_start() && BeginPtr(context_) != BeginPtr(text))\n return false;\n if (prog_->anchor_end() && EndPtr(context_) != EndPtr(text))\n return false;\n anchored_ = anchored || prog_->anchor_start();\n longest_ = longest || prog_->anchor_end();\n endmatch_ = prog_->anchor_end();\n submatch_ = submatch;\n nsubmatch_ = nsubmatch;\n for (int i = 0; i < nsubmatch_; i++)\n submatch_[i] = absl::string_view();\n\n // Allocate scratch space.\n int nvisited = prog_->list_count() * static_cast(text.size()+1);\n nvisited = (nvisited + kVisitedBits-1) / kVisitedBits;\n visited_ = PODArray(nvisited);\n memset(visited_.data(), 0, nvisited*sizeof visited_[0]);\n\n int ncap = 2*nsubmatch;\n if (ncap < 2)\n ncap = 2;\n cap_ = PODArray(ncap);\n memset(cap_.data(), 0, ncap*sizeof cap_[0]);\n\n // When sizeof(Job) == 16, we start with a nice round 1KiB. :)\n job_ = PODArray(64);\n\n // Anchored search must start at text.begin().\n if (anchored_) {\n cap_[0] = text.data();\n return TrySearch(prog_->start(), text.data());\n }\n\n // Unanchored search, starting from each possible text position.\n // Notice that we have to try the empty string at the end of\n // the text, so the loop condition is p <= text.end(), not p < text.end().\n // This looks like it's quadratic in the size of the text,\n // but we are not clearing visited_ between calls to TrySearch,\n // so no work is duplicated and it ends up still being linear.\n const char* etext = text.data() + text.size();\n for (const char* p = text.data(); p <= etext; p++) {\n // Try to use prefix accel (e.g. memchr) to skip ahead.\n if (p < etext && prog_->can_prefix_accel()) {\n p = reinterpret_cast(prog_->PrefixAccel(p, etext - p));\n if (p == NULL)\n p = etext;\n }\n\n cap_[0] = p;\n if (TrySearch(prog_->start(), p)) // Match must be leftmost; done.\n return true;\n // Avoid invoking undefined behavior (arithmetic on a null pointer)\n // by simply not continuing the loop.\n if (p == NULL)\n break;\n }\n return false;\n}\n\n// Bit-state search.\nbool Prog::SearchBitState(absl::string_view text, absl::string_view context,\n Anchor anchor, MatchKind kind,\n absl::string_view* match, int nmatch) {\n // If full match, we ask for an anchored longest match\n // and then check that match[0] == text.\n // So make sure match[0] exists.\n absl::string_view sp0;\n if (kind == kFullMatch) {\n anchor = kAnchored;\n if (nmatch < 1) {\n match = &sp0;\n nmatch = 1;\n }\n }\n\n // Run the search.\n BitState b(this);\n bool anchored = anchor == kAnchored;\n bool longest = kind != kFirstMatch;\n if (!b.Search(text, context, anchored, longest, match, nmatch))\n return false;\n if (kind == kFullMatch && EndPtr(match[0]) != EndPtr(text))\n return false;\n return true;\n}\n\n} // namespace re2\n" }, { "path": "re2/compile.cc", "content": "// Copyright 2007 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Compile regular expression to Prog.\n//\n// Prog and Inst are defined in prog.h.\n// This file's external interface is just Regexp::CompileToProg.\n// The Compiler class defined in this file is private.\n\n#include \n#include \n\n#include \n#include \n\n#include \"absl/container/flat_hash_map.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/prog.h\"\n#include \"re2/re2.h\"\n#include \"re2/regexp.h\"\n#include \"re2/walker-inl.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\n// List of pointers to Inst* that need to be filled in (patched).\n// Because the Inst* haven't been filled in yet,\n// we can use the Inst* word to hold the list's \"next\" pointer.\n// It's kind of sleazy, but it works well in practice.\n// See http://swtch.com/~rsc/regexp/regexp1.html for inspiration.\n//\n// Because the out and out1 fields in Inst are no longer pointers,\n// we can't use pointers directly here either. Instead, head refers\n// to inst_[head>>1].out (head&1 == 0) or inst_[head>>1].out1 (head&1 == 1).\n// head == 0 represents the NULL list. This is okay because instruction #0\n// is always the fail instruction, which never appears on a list.\nstruct PatchList {\n // Returns patch list containing just p.\n static PatchList Mk(uint32_t p) {\n return {p, p};\n }\n\n // Patches all the entries on l to have value p.\n // Caller must not ever use patch list again.\n static void Patch(Prog::Inst* inst0, PatchList l, uint32_t p) {\n while (l.head != 0) {\n Prog::Inst* ip = &inst0[l.head>>1];\n if (l.head&1) {\n l.head = ip->out1();\n ip->out1_ = p;\n } else {\n l.head = ip->out();\n ip->set_out(p);\n }\n }\n }\n\n // Appends two patch lists and returns result.\n static PatchList Append(Prog::Inst* inst0, PatchList l1, PatchList l2) {\n if (l1.head == 0)\n return l2;\n if (l2.head == 0)\n return l1;\n Prog::Inst* ip = &inst0[l1.tail>>1];\n if (l1.tail&1)\n ip->out1_ = l2.head;\n else\n ip->set_out(l2.head);\n return {l1.head, l2.tail};\n }\n\n uint32_t head;\n uint32_t tail; // for constant-time append\n};\n\nstatic const PatchList kNullPatchList = {0, 0};\n\n// Compiled program fragment.\nstruct Frag {\n uint32_t begin;\n PatchList end;\n bool nullable;\n\n Frag() : begin(0), end(kNullPatchList), nullable(false) {}\n Frag(uint32_t begin, PatchList end, bool nullable)\n : begin(begin), end(end), nullable(nullable) {}\n};\n\n// Input encodings.\nenum Encoding {\n kEncodingUTF8 = 1, // UTF-8 (0-10FFFF)\n kEncodingLatin1, // Latin-1 (0-FF)\n};\n\nclass Compiler : public Regexp::Walker {\n public:\n explicit Compiler();\n ~Compiler();\n\n // Compiles Regexp to a new Prog.\n // Caller is responsible for deleting Prog when finished with it.\n // If reversed is true, compiles for walking over the input\n // string backward (reverses all concatenations).\n static Prog *Compile(Regexp* re, bool reversed, int64_t max_mem);\n\n // Compiles alternation of all the re to a new Prog.\n // Each re has a match with an id equal to its index in the vector.\n static Prog* CompileSet(Regexp* re, RE2::Anchor anchor, int64_t max_mem);\n\n // Interface for Regexp::Walker, which helps traverse the Regexp.\n // The walk is purely post-recursive: given the machines for the\n // children, PostVisit combines them to create the machine for\n // the current node. The child_args are Frags.\n // The Compiler traverses the Regexp parse tree, visiting\n // each node in depth-first order. It invokes PreVisit before\n // visiting the node's children and PostVisit after visiting\n // the children.\n Frag PreVisit(Regexp* re, Frag parent_arg, bool* stop);\n Frag PostVisit(Regexp* re, Frag parent_arg, Frag pre_arg, Frag* child_args,\n int nchild_args);\n Frag ShortVisit(Regexp* re, Frag parent_arg);\n Frag Copy(Frag arg);\n\n // Given fragment a, returns a+ or a+?; a* or a*?; a? or a??\n Frag Plus(Frag a, bool nongreedy);\n Frag Star(Frag a, bool nongreedy);\n Frag Quest(Frag a, bool nongreedy);\n\n // Given fragment a, returns (a) capturing as \\n.\n Frag Capture(Frag a, int n);\n\n // Given fragments a and b, returns ab; a|b\n Frag Cat(Frag a, Frag b);\n Frag Alt(Frag a, Frag b);\n\n // Returns a fragment that can't match anything.\n Frag NoMatch();\n\n // Returns a fragment that matches the empty string.\n Frag Match(int32_t id);\n\n // Returns a no-op fragment.\n Frag Nop();\n\n // Returns a fragment matching the byte range lo-hi.\n Frag ByteRange(int lo, int hi, bool foldcase);\n\n // Returns a fragment matching an empty-width special op.\n Frag EmptyWidth(EmptyOp op);\n\n // Adds n instructions to the program.\n // Returns the index of the first one.\n // Returns -1 if no more instructions are available.\n int AllocInst(int n);\n\n // Rune range compiler.\n\n // Begins a new alternation.\n void BeginRange();\n\n // Adds a fragment matching the rune range lo-hi.\n void AddRuneRange(Rune lo, Rune hi, bool foldcase);\n void AddRuneRangeLatin1(Rune lo, Rune hi, bool foldcase);\n void AddRuneRangeUTF8(Rune lo, Rune hi, bool foldcase);\n void Add_80_10ffff();\n\n // New suffix that matches the byte range lo-hi, then goes to next.\n int UncachedRuneByteSuffix(uint8_t lo, uint8_t hi, bool foldcase, int next);\n int CachedRuneByteSuffix(uint8_t lo, uint8_t hi, bool foldcase, int next);\n\n // Returns true iff the suffix is cached.\n bool IsCachedRuneByteSuffix(int id);\n\n // Adds a suffix to alternation.\n void AddSuffix(int id);\n\n // Adds a suffix to the trie starting from the given root node.\n // Returns zero iff allocating an instruction fails. Otherwise, returns\n // the current root node, which might be different from what was given.\n int AddSuffixRecursive(int root, int id);\n\n // Finds the trie node for the given suffix. Returns a Frag in order to\n // distinguish between pointing at the root node directly (end.head == 0)\n // and pointing at an Alt's out1 or out (end.head&1 == 1 or 0, respectively).\n Frag FindByteRange(int root, int id);\n\n // Compares two ByteRanges and returns true iff they are equal.\n bool ByteRangeEqual(int id1, int id2);\n\n // Returns the alternation of all the added suffixes.\n Frag EndRange();\n\n // Single rune.\n Frag Literal(Rune r, bool foldcase);\n\n void Setup(Regexp::ParseFlags flags, int64_t max_mem, RE2::Anchor anchor);\n Prog* Finish(Regexp* re);\n\n // Returns .* where dot = any byte\n Frag DotStar();\n\n private:\n Prog* prog_; // Program being built.\n bool failed_; // Did we give up compiling?\n Encoding encoding_; // Input encoding\n bool reversed_; // Should program run backward over text?\n\n PODArray inst_;\n int ninst_; // Number of instructions used.\n int max_ninst_; // Maximum number of instructions.\n\n int64_t max_mem_; // Total memory budget.\n\n absl::flat_hash_map rune_cache_;\n Frag rune_range_;\n\n RE2::Anchor anchor_; // anchor mode for RE2::Set\n\n Compiler(const Compiler&) = delete;\n Compiler& operator=(const Compiler&) = delete;\n};\n\nCompiler::Compiler() {\n prog_ = new Prog();\n failed_ = false;\n encoding_ = kEncodingUTF8;\n reversed_ = false;\n ninst_ = 0;\n max_ninst_ = 1; // make AllocInst for fail instruction okay\n max_mem_ = 0;\n int fail = AllocInst(1);\n inst_[fail].InitFail();\n max_ninst_ = 0; // Caller must change\n}\n\nCompiler::~Compiler() {\n delete prog_;\n}\n\nint Compiler::AllocInst(int n) {\n if (failed_ || ninst_ + n > max_ninst_) {\n failed_ = true;\n return -1;\n }\n\n if (ninst_ + n > inst_.size()) {\n int cap = inst_.size();\n if (cap == 0)\n cap = 8;\n while (ninst_ + n > cap)\n cap *= 2;\n PODArray inst(cap);\n if (inst_.data() != NULL)\n memmove(inst.data(), inst_.data(), ninst_*sizeof inst_[0]);\n memset(inst.data() + ninst_, 0, (cap - ninst_)*sizeof inst_[0]);\n inst_ = std::move(inst);\n }\n int id = ninst_;\n ninst_ += n;\n return id;\n}\n\n// These routines are somewhat hard to visualize in text --\n// see http://swtch.com/~rsc/regexp/regexp1.html for\n// pictures explaining what is going on here.\n\n// Returns an unmatchable fragment.\nFrag Compiler::NoMatch() {\n return Frag();\n}\n\n// Is a an unmatchable fragment?\nstatic bool IsNoMatch(Frag a) {\n return a.begin == 0;\n}\n\n// Given fragments a and b, returns fragment for ab.\nFrag Compiler::Cat(Frag a, Frag b) {\n if (IsNoMatch(a) || IsNoMatch(b))\n return NoMatch();\n\n // Elide no-op.\n Prog::Inst* begin = &inst_[a.begin];\n if (begin->opcode() == kInstNop &&\n a.end.head == (a.begin << 1) &&\n begin->out() == 0) {\n // in case refs to a somewhere\n PatchList::Patch(inst_.data(), a.end, b.begin);\n return b;\n }\n\n // To run backward over string, reverse all concatenations.\n if (reversed_) {\n PatchList::Patch(inst_.data(), b.end, a.begin);\n return Frag(b.begin, a.end, b.nullable && a.nullable);\n }\n\n PatchList::Patch(inst_.data(), a.end, b.begin);\n return Frag(a.begin, b.end, a.nullable && b.nullable);\n}\n\n// Given fragments for a and b, returns fragment for a|b.\nFrag Compiler::Alt(Frag a, Frag b) {\n // Special case for convenience in loops.\n if (IsNoMatch(a))\n return b;\n if (IsNoMatch(b))\n return a;\n\n int id = AllocInst(1);\n if (id < 0)\n return NoMatch();\n\n inst_[id].InitAlt(a.begin, b.begin);\n return Frag(id, PatchList::Append(inst_.data(), a.end, b.end),\n a.nullable || b.nullable);\n}\n\n// When capturing submatches in like-Perl mode, a kOpAlt Inst\n// treats out_ as the first choice, out1_ as the second.\n//\n// For *, +, and ?, if out_ causes another repetition,\n// then the operator is greedy. If out1_ is the repetition\n// (and out_ moves forward), then the operator is non-greedy.\n\n// Given a fragment for a, returns a fragment for a+ or a+? (if nongreedy)\nFrag Compiler::Plus(Frag a, bool nongreedy) {\n int id = AllocInst(1);\n if (id < 0)\n return NoMatch();\n PatchList pl;\n if (nongreedy) {\n inst_[id].InitAlt(0, a.begin);\n pl = PatchList::Mk(id << 1);\n } else {\n inst_[id].InitAlt(a.begin, 0);\n pl = PatchList::Mk((id << 1) | 1);\n }\n PatchList::Patch(inst_.data(), a.end, id);\n return Frag(a.begin, pl, a.nullable);\n}\n\n// Given a fragment for a, returns a fragment for a* or a*? (if nongreedy)\nFrag Compiler::Star(Frag a, bool nongreedy) {\n // When the subexpression is nullable, one Alt isn't enough to guarantee\n // correct priority ordering within the transitive closure. The simplest\n // solution is to handle it as (a+)? instead, which adds the second Alt.\n if (a.nullable)\n return Quest(Plus(a, nongreedy), nongreedy);\n\n int id = AllocInst(1);\n if (id < 0)\n return NoMatch();\n PatchList pl;\n if (nongreedy) {\n inst_[id].InitAlt(0, a.begin);\n pl = PatchList::Mk(id << 1);\n } else {\n inst_[id].InitAlt(a.begin, 0);\n pl = PatchList::Mk((id << 1) | 1);\n }\n PatchList::Patch(inst_.data(), a.end, id);\n return Frag(id, pl, true);\n}\n\n// Given a fragment for a, returns a fragment for a? or a?? (if nongreedy)\nFrag Compiler::Quest(Frag a, bool nongreedy) {\n if (IsNoMatch(a))\n return Nop();\n int id = AllocInst(1);\n if (id < 0)\n return NoMatch();\n PatchList pl;\n if (nongreedy) {\n inst_[id].InitAlt(0, a.begin);\n pl = PatchList::Mk(id << 1);\n } else {\n inst_[id].InitAlt(a.begin, 0);\n pl = PatchList::Mk((id << 1) | 1);\n }\n return Frag(id, PatchList::Append(inst_.data(), pl, a.end), true);\n}\n\n// Returns a fragment for the byte range lo-hi.\nFrag Compiler::ByteRange(int lo, int hi, bool foldcase) {\n int id = AllocInst(1);\n if (id < 0)\n return NoMatch();\n inst_[id].InitByteRange(lo, hi, foldcase, 0);\n return Frag(id, PatchList::Mk(id << 1), false);\n}\n\n// Returns a no-op fragment. Sometimes unavoidable.\nFrag Compiler::Nop() {\n int id = AllocInst(1);\n if (id < 0)\n return NoMatch();\n inst_[id].InitNop(0);\n return Frag(id, PatchList::Mk(id << 1), true);\n}\n\n// Returns a fragment that signals a match.\nFrag Compiler::Match(int32_t match_id) {\n int id = AllocInst(1);\n if (id < 0)\n return NoMatch();\n inst_[id].InitMatch(match_id);\n return Frag(id, kNullPatchList, false);\n}\n\n// Returns a fragment matching a particular empty-width op (like ^ or $)\nFrag Compiler::EmptyWidth(EmptyOp empty) {\n int id = AllocInst(1);\n if (id < 0)\n return NoMatch();\n inst_[id].InitEmptyWidth(empty, 0);\n return Frag(id, PatchList::Mk(id << 1), true);\n}\n\n// Given a fragment a, returns a fragment with capturing parens around a.\nFrag Compiler::Capture(Frag a, int n) {\n if (IsNoMatch(a))\n return NoMatch();\n int id = AllocInst(2);\n if (id < 0)\n return NoMatch();\n inst_[id].InitCapture(2*n, a.begin);\n inst_[id+1].InitCapture(2*n+1, 0);\n PatchList::Patch(inst_.data(), a.end, id+1);\n\n return Frag(id, PatchList::Mk((id+1) << 1), a.nullable);\n}\n\n// A Rune is a name for a Unicode code point.\n// Returns maximum rune encoded by UTF-8 sequence of length len.\nstatic int MaxRune(int len) {\n int b; // number of Rune bits in len-byte UTF-8 sequence (len < UTFmax)\n if (len == 1)\n b = 7;\n else\n b = 8-(len+1) + 6*(len-1);\n return (1<::const_iterator it = rune_cache_.find(key);\n if (it != rune_cache_.end())\n return it->second;\n int id = UncachedRuneByteSuffix(lo, hi, foldcase, next);\n rune_cache_[key] = id;\n return id;\n}\n\nbool Compiler::IsCachedRuneByteSuffix(int id) {\n uint8_t lo = inst_[id].lo_;\n uint8_t hi = inst_[id].hi_;\n bool foldcase = inst_[id].foldcase() != 0;\n int next = inst_[id].out();\n\n uint64_t key = MakeRuneCacheKey(lo, hi, foldcase, next);\n return rune_cache_.find(key) != rune_cache_.end();\n}\n\nvoid Compiler::AddSuffix(int id) {\n if (failed_)\n return;\n\n if (rune_range_.begin == 0) {\n rune_range_.begin = id;\n return;\n }\n\n if (encoding_ == kEncodingUTF8) {\n // Build a trie in order to reduce fanout.\n rune_range_.begin = AddSuffixRecursive(rune_range_.begin, id);\n return;\n }\n\n int alt = AllocInst(1);\n if (alt < 0) {\n rune_range_.begin = 0;\n return;\n }\n inst_[alt].InitAlt(rune_range_.begin, id);\n rune_range_.begin = alt;\n}\n\nint Compiler::AddSuffixRecursive(int root, int id) {\n ABSL_DCHECK(inst_[root].opcode() == kInstAlt ||\n inst_[root].opcode() == kInstByteRange);\n\n Frag f = FindByteRange(root, id);\n if (IsNoMatch(f)) {\n int alt = AllocInst(1);\n if (alt < 0)\n return 0;\n inst_[alt].InitAlt(root, id);\n return alt;\n }\n\n int br;\n if (f.end.head == 0)\n br = root;\n else if (f.end.head&1)\n br = inst_[f.begin].out1();\n else\n br = inst_[f.begin].out();\n\n if (IsCachedRuneByteSuffix(br)) {\n // We can't fiddle with cached suffixes, so make a clone of the head.\n int byterange = AllocInst(1);\n if (byterange < 0)\n return 0;\n inst_[byterange].InitByteRange(inst_[br].lo(), inst_[br].hi(),\n inst_[br].foldcase(), inst_[br].out());\n\n // Ensure that the parent points to the clone, not to the original.\n // Note that this could leave the head unreachable except via the cache.\n br = byterange;\n if (f.end.head == 0)\n root = br;\n else if (f.end.head&1)\n inst_[f.begin].out1_ = br;\n else\n inst_[f.begin].set_out(br);\n }\n\n int out = inst_[id].out();\n if (!IsCachedRuneByteSuffix(id)) {\n // The head should be the instruction most recently allocated, so free it\n // instead of leaving it unreachable.\n ABSL_DCHECK_EQ(id, ninst_-1);\n inst_[id].out_opcode_ = 0;\n inst_[id].out1_ = 0;\n ninst_--;\n }\n\n out = AddSuffixRecursive(inst_[br].out(), out);\n if (out == 0)\n return 0;\n\n inst_[br].set_out(out);\n return root;\n}\n\nbool Compiler::ByteRangeEqual(int id1, int id2) {\n return inst_[id1].lo() == inst_[id2].lo() &&\n inst_[id1].hi() == inst_[id2].hi() &&\n inst_[id1].foldcase() == inst_[id2].foldcase();\n}\n\nFrag Compiler::FindByteRange(int root, int id) {\n if (inst_[root].opcode() == kInstByteRange) {\n if (ByteRangeEqual(root, id))\n return Frag(root, kNullPatchList, false);\n else\n return NoMatch();\n }\n\n while (inst_[root].opcode() == kInstAlt) {\n int out1 = inst_[root].out1();\n if (ByteRangeEqual(out1, id))\n return Frag(root, PatchList::Mk((root << 1) | 1), false);\n\n // CharClass is a sorted list of ranges, so if out1 of the root Alt wasn't\n // what we're looking for, then we can stop immediately. Unfortunately, we\n // can't short-circuit the search in reverse mode.\n if (!reversed_)\n return NoMatch();\n\n int out = inst_[root].out();\n if (inst_[out].opcode() == kInstAlt)\n root = out;\n else if (ByteRangeEqual(out, id))\n return Frag(root, PatchList::Mk(root << 1), false);\n else\n return NoMatch();\n }\n\n ABSL_LOG(DFATAL) << \"should never happen\";\n return NoMatch();\n}\n\nFrag Compiler::EndRange() {\n return rune_range_;\n}\n\n// Converts rune range lo-hi into a fragment that recognizes\n// the bytes that would make up those runes in the current\n// encoding (Latin 1 or UTF-8).\n// This lets the machine work byte-by-byte even when\n// using multibyte encodings.\n\nvoid Compiler::AddRuneRange(Rune lo, Rune hi, bool foldcase) {\n switch (encoding_) {\n default:\n case kEncodingUTF8:\n AddRuneRangeUTF8(lo, hi, foldcase);\n break;\n case kEncodingLatin1:\n AddRuneRangeLatin1(lo, hi, foldcase);\n break;\n }\n}\n\nvoid Compiler::AddRuneRangeLatin1(Rune lo, Rune hi, bool foldcase) {\n // Latin-1 is easy: runes *are* bytes.\n if (lo > hi || lo > 0xFF)\n return;\n if (hi > 0xFF)\n hi = 0xFF;\n AddSuffix(UncachedRuneByteSuffix(static_cast(lo),\n static_cast(hi), foldcase, 0));\n}\n\nvoid Compiler::Add_80_10ffff() {\n // The 80-10FFFF (Runeself-Runemax) rune range occurs frequently enough\n // (for example, for /./ and /[^a-z]/) that it is worth simplifying: by\n // permitting overlong encodings in E0 and F0 sequences and code points\n // over 10FFFF in F4 sequences, the size of the bytecode and the number\n // of equivalence classes are reduced significantly.\n int id;\n if (reversed_) {\n // Prefix factoring matters, but we don't have to handle it here\n // because the rune range trie logic takes care of that already.\n id = UncachedRuneByteSuffix(0xC2, 0xDF, false, 0);\n id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);\n AddSuffix(id);\n\n id = UncachedRuneByteSuffix(0xE0, 0xEF, false, 0);\n id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);\n id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);\n AddSuffix(id);\n\n id = UncachedRuneByteSuffix(0xF0, 0xF4, false, 0);\n id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);\n id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);\n id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);\n AddSuffix(id);\n } else {\n // Suffix factoring matters - and we do have to handle it here.\n int cont1 = UncachedRuneByteSuffix(0x80, 0xBF, false, 0);\n id = UncachedRuneByteSuffix(0xC2, 0xDF, false, cont1);\n AddSuffix(id);\n\n int cont2 = UncachedRuneByteSuffix(0x80, 0xBF, false, cont1);\n id = UncachedRuneByteSuffix(0xE0, 0xEF, false, cont2);\n AddSuffix(id);\n\n int cont3 = UncachedRuneByteSuffix(0x80, 0xBF, false, cont2);\n id = UncachedRuneByteSuffix(0xF0, 0xF4, false, cont3);\n AddSuffix(id);\n }\n}\n\nvoid Compiler::AddRuneRangeUTF8(Rune lo, Rune hi, bool foldcase) {\n if (lo > hi)\n return;\n\n // Pick off 80-10FFFF as a common special case.\n if (lo == 0x80 && hi == 0x10ffff) {\n Add_80_10ffff();\n return;\n }\n\n // Split range into same-length sized ranges.\n for (int i = 1; i < UTFmax; i++) {\n Rune max = MaxRune(i);\n if (lo <= max && max < hi) {\n AddRuneRangeUTF8(lo, max, foldcase);\n AddRuneRangeUTF8(max+1, hi, foldcase);\n return;\n }\n }\n\n // ASCII range is always a special case.\n if (hi < Runeself) {\n AddSuffix(UncachedRuneByteSuffix(static_cast(lo),\n static_cast(hi), foldcase, 0));\n return;\n }\n\n // Split range into sections that agree on leading bytes.\n for (int i = 1; i < UTFmax; i++) {\n uint32_t m = (1<<(6*i)) - 1; // last i bytes of a UTF-8 sequence\n if ((lo & ~m) != (hi & ~m)) {\n if ((lo & m) != 0) {\n AddRuneRangeUTF8(lo, lo|m, foldcase);\n AddRuneRangeUTF8((lo|m)+1, hi, foldcase);\n return;\n }\n if ((hi & m) != m) {\n AddRuneRangeUTF8(lo, (hi&~m)-1, foldcase);\n AddRuneRangeUTF8(hi&~m, hi, foldcase);\n return;\n }\n }\n }\n\n // Finally. Generate byte matching equivalent for lo-hi.\n uint8_t ulo[UTFmax], uhi[UTFmax];\n int n = runetochar(reinterpret_cast(ulo), &lo);\n int m = runetochar(reinterpret_cast(uhi), &hi);\n (void)m; // USED(m)\n ABSL_DCHECK_EQ(n, m);\n\n // The logic below encodes this thinking:\n //\n // 1. When we have built the whole suffix, we know that it cannot\n // possibly be a suffix of anything longer: in forward mode, nothing\n // else can occur before the leading byte; in reverse mode, nothing\n // else can occur after the last continuation byte or else the leading\n // byte would have to change. Thus, there is no benefit to caching\n // the first byte of the suffix whereas there is a cost involved in\n // cloning it if it begins a common prefix, which is fairly likely.\n //\n // 2. Conversely, the last byte of the suffix cannot possibly be a\n // prefix of anything because next == 0, so we will never want to\n // clone it, but it is fairly likely to be a common suffix. Perhaps\n // more so in reverse mode than in forward mode because the former is\n // \"converging\" towards lower entropy, but caching is still worthwhile\n // for the latter in cases such as 80-BF.\n //\n // 3. Handling the bytes between the first and the last is less\n // straightforward and, again, the approach depends on whether we are\n // \"converging\" towards lower entropy: in forward mode, a single byte\n // is unlikely to be part of a common suffix whereas a byte range\n // is more likely so; in reverse mode, a byte range is unlikely to\n // be part of a common suffix whereas a single byte is more likely\n // so. The same benefit versus cost argument applies here.\n int id = 0;\n if (reversed_) {\n for (int i = 0; i < n; i++) {\n // In reverse UTF-8 mode: cache the leading byte; don't cache the last\n // continuation byte; cache anything else iff it's a single byte (XX-XX).\n if (i == 0 || (ulo[i] == uhi[i] && i != n-1))\n id = CachedRuneByteSuffix(ulo[i], uhi[i], false, id);\n else\n id = UncachedRuneByteSuffix(ulo[i], uhi[i], false, id);\n }\n } else {\n for (int i = n-1; i >= 0; i--) {\n // In forward UTF-8 mode: don't cache the leading byte; cache the last\n // continuation byte; cache anything else iff it's a byte range (XX-YY).\n if (i == n-1 || (ulo[i] < uhi[i] && i != 0))\n id = CachedRuneByteSuffix(ulo[i], uhi[i], false, id);\n else\n id = UncachedRuneByteSuffix(ulo[i], uhi[i], false, id);\n }\n }\n AddSuffix(id);\n}\n\n// Should not be called.\nFrag Compiler::Copy(Frag arg) {\n // We're using WalkExponential; there should be no copying.\n failed_ = true;\n ABSL_LOG(DFATAL) << \"Compiler::Copy called!\";\n return NoMatch();\n}\n\n// Visits a node quickly; called once WalkExponential has\n// decided to cut this walk short.\nFrag Compiler::ShortVisit(Regexp* re, Frag) {\n failed_ = true;\n return NoMatch();\n}\n\n// Called before traversing a node's children during the walk.\nFrag Compiler::PreVisit(Regexp* re, Frag, bool* stop) {\n // Cut off walk if we've already failed.\n if (failed_)\n *stop = true;\n\n return Frag(); // not used by caller\n}\n\nFrag Compiler::Literal(Rune r, bool foldcase) {\n switch (encoding_) {\n default:\n return Frag();\n\n case kEncodingLatin1:\n return ByteRange(r, r, foldcase);\n\n case kEncodingUTF8: {\n if (r < Runeself) // Make common case fast.\n return ByteRange(r, r, foldcase);\n uint8_t buf[UTFmax];\n int n = runetochar(reinterpret_cast(buf), &r);\n Frag f = ByteRange((uint8_t)buf[0], buf[0], false);\n for (int i = 1; i < n; i++)\n f = Cat(f, ByteRange((uint8_t)buf[i], buf[i], false));\n return f;\n }\n }\n}\n\n// Called after traversing the node's children during the walk.\n// Given their frags, build and return the frag for this re.\nFrag Compiler::PostVisit(Regexp* re, Frag, Frag, Frag* child_frags,\n int nchild_frags) {\n // If a child failed, don't bother going forward, especially\n // since the child_frags might contain Frags with NULLs in them.\n if (failed_)\n return NoMatch();\n\n // Given the child fragments, return the fragment for this node.\n switch (re->op()) {\n case kRegexpRepeat:\n // Should not see; code at bottom of function will print error\n break;\n\n case kRegexpNoMatch:\n return NoMatch();\n\n case kRegexpEmptyMatch:\n return Nop();\n\n case kRegexpHaveMatch: {\n Frag f = Match(re->match_id());\n if (anchor_ == RE2::ANCHOR_BOTH) {\n // Append \\z or else the subexpression will effectively be unanchored.\n // Complemented by the UNANCHORED case in CompileSet().\n f = Cat(EmptyWidth(kEmptyEndText), f);\n }\n return f;\n }\n\n case kRegexpConcat: {\n Frag f = child_frags[0];\n for (int i = 1; i < nchild_frags; i++)\n f = Cat(f, child_frags[i]);\n return f;\n }\n\n case kRegexpAlternate: {\n Frag f = child_frags[0];\n for (int i = 1; i < nchild_frags; i++)\n f = Alt(f, child_frags[i]);\n return f;\n }\n\n case kRegexpStar:\n return Star(child_frags[0], (re->parse_flags()&Regexp::NonGreedy) != 0);\n\n case kRegexpPlus:\n return Plus(child_frags[0], (re->parse_flags()&Regexp::NonGreedy) != 0);\n\n case kRegexpQuest:\n return Quest(child_frags[0], (re->parse_flags()&Regexp::NonGreedy) != 0);\n\n case kRegexpLiteral:\n return Literal(re->rune(), (re->parse_flags()&Regexp::FoldCase) != 0);\n\n case kRegexpLiteralString: {\n // Concatenation of literals.\n if (re->nrunes() == 0)\n return Nop();\n Frag f;\n for (int i = 0; i < re->nrunes(); i++) {\n Frag f1 = Literal(re->runes()[i],\n (re->parse_flags()&Regexp::FoldCase) != 0);\n if (i == 0)\n f = f1;\n else\n f = Cat(f, f1);\n }\n return f;\n }\n\n case kRegexpAnyChar:\n BeginRange();\n AddRuneRange(0, Runemax, false);\n return EndRange();\n\n case kRegexpAnyByte:\n return ByteRange(0x00, 0xFF, false);\n\n case kRegexpCharClass: {\n CharClass* cc = re->cc();\n if (cc->empty()) {\n // This can't happen.\n failed_ = true;\n ABSL_LOG(DFATAL) << \"No ranges in char class\";\n return NoMatch();\n }\n\n // ASCII case-folding optimization: if the char class\n // behaves the same on A-Z as it does on a-z,\n // discard any ranges wholly contained in A-Z\n // and mark the other ranges as foldascii.\n // This reduces the size of a program for\n // (?i)abc from 3 insts per letter to 1 per letter.\n bool foldascii = cc->FoldsASCII();\n\n // Character class is just a big OR of the different\n // character ranges in the class.\n BeginRange();\n for (CharClass::iterator i = cc->begin(); i != cc->end(); ++i) {\n // ASCII case-folding optimization (see above).\n if (foldascii && 'A' <= i->lo && i->hi <= 'Z')\n continue;\n\n // If this range contains all of A-Za-z or none of it,\n // the fold flag is unnecessary; don't bother.\n bool fold = foldascii;\n if ((i->lo <= 'A' && 'z' <= i->hi) || i->hi < 'A' || 'z' < i->lo ||\n ('Z' < i->lo && i->hi < 'a'))\n fold = false;\n\n AddRuneRange(i->lo, i->hi, fold);\n }\n return EndRange();\n }\n\n case kRegexpCapture:\n // If this is a non-capturing parenthesis -- (?:foo) --\n // just use the inner expression.\n if (re->cap() < 0)\n return child_frags[0];\n return Capture(child_frags[0], re->cap());\n\n case kRegexpBeginLine:\n return EmptyWidth(reversed_ ? kEmptyEndLine : kEmptyBeginLine);\n\n case kRegexpEndLine:\n return EmptyWidth(reversed_ ? kEmptyBeginLine : kEmptyEndLine);\n\n case kRegexpBeginText:\n return EmptyWidth(reversed_ ? kEmptyEndText : kEmptyBeginText);\n\n case kRegexpEndText:\n return EmptyWidth(reversed_ ? kEmptyBeginText : kEmptyEndText);\n\n case kRegexpWordBoundary:\n return EmptyWidth(kEmptyWordBoundary);\n\n case kRegexpNoWordBoundary:\n return EmptyWidth(kEmptyNonWordBoundary);\n }\n failed_ = true;\n ABSL_LOG(DFATAL) << \"Missing case in Compiler: \" << re->op();\n return NoMatch();\n}\n\n// Is this regexp required to start at the beginning of the text?\n// Only approximate; can return false for complicated regexps like (\\Aa|\\Ab),\n// but handles (\\A(a|b)). Could use the Walker to write a more exact one.\nstatic bool IsAnchorStart(Regexp** pre, int depth) {\n Regexp* re = *pre;\n Regexp* sub;\n // The depth limit makes sure that we don't overflow\n // the stack on a deeply nested regexp. As the comment\n // above says, IsAnchorStart is conservative, so returning\n // a false negative is okay. The exact limit is somewhat arbitrary.\n if (re == NULL || depth >= 4)\n return false;\n switch (re->op()) {\n default:\n break;\n case kRegexpConcat:\n if (re->nsub() > 0) {\n sub = re->sub()[0]->Incref();\n if (IsAnchorStart(&sub, depth+1)) {\n PODArray subcopy(re->nsub());\n subcopy[0] = sub; // already have reference\n for (int i = 1; i < re->nsub(); i++)\n subcopy[i] = re->sub()[i]->Incref();\n *pre = Regexp::Concat(subcopy.data(), re->nsub(), re->parse_flags());\n re->Decref();\n return true;\n }\n sub->Decref();\n }\n break;\n case kRegexpCapture:\n sub = re->sub()[0]->Incref();\n if (IsAnchorStart(&sub, depth+1)) {\n *pre = Regexp::Capture(sub, re->parse_flags(), re->cap());\n re->Decref();\n return true;\n }\n sub->Decref();\n break;\n case kRegexpBeginText:\n *pre = Regexp::LiteralString(NULL, 0, re->parse_flags());\n re->Decref();\n return true;\n }\n return false;\n}\n\n// Is this regexp required to start at the end of the text?\n// Only approximate; can return false for complicated regexps like (a\\z|b\\z),\n// but handles ((a|b)\\z). Could use the Walker to write a more exact one.\nstatic bool IsAnchorEnd(Regexp** pre, int depth) {\n Regexp* re = *pre;\n Regexp* sub;\n // The depth limit makes sure that we don't overflow\n // the stack on a deeply nested regexp. As the comment\n // above says, IsAnchorEnd is conservative, so returning\n // a false negative is okay. The exact limit is somewhat arbitrary.\n if (re == NULL || depth >= 4)\n return false;\n switch (re->op()) {\n default:\n break;\n case kRegexpConcat:\n if (re->nsub() > 0) {\n sub = re->sub()[re->nsub() - 1]->Incref();\n if (IsAnchorEnd(&sub, depth+1)) {\n PODArray subcopy(re->nsub());\n subcopy[re->nsub() - 1] = sub; // already have reference\n for (int i = 0; i < re->nsub() - 1; i++)\n subcopy[i] = re->sub()[i]->Incref();\n *pre = Regexp::Concat(subcopy.data(), re->nsub(), re->parse_flags());\n re->Decref();\n return true;\n }\n sub->Decref();\n }\n break;\n case kRegexpCapture:\n sub = re->sub()[0]->Incref();\n if (IsAnchorEnd(&sub, depth+1)) {\n *pre = Regexp::Capture(sub, re->parse_flags(), re->cap());\n re->Decref();\n return true;\n }\n sub->Decref();\n break;\n case kRegexpEndText:\n *pre = Regexp::LiteralString(NULL, 0, re->parse_flags());\n re->Decref();\n return true;\n }\n return false;\n}\n\nvoid Compiler::Setup(Regexp::ParseFlags flags, int64_t max_mem,\n RE2::Anchor anchor) {\n if (flags & Regexp::Latin1)\n encoding_ = kEncodingLatin1;\n max_mem_ = max_mem;\n if (max_mem <= 0) {\n max_ninst_ = 100000; // more than enough\n } else if (static_cast(max_mem) <= sizeof(Prog)) {\n // No room for anything.\n max_ninst_ = 0;\n } else {\n int64_t m = (max_mem - sizeof(Prog)) / sizeof(Prog::Inst);\n // Limit instruction count so that inst->id() fits nicely in an int.\n // SparseArray also assumes that the indices (inst->id()) are ints.\n // The call to WalkExponential uses 2*max_ninst_ below,\n // and other places in the code use 2 or 3 * prog->size().\n // Limiting to 2^24 should avoid overflow in those places.\n // (The point of allowing more than 32 bits of memory is to\n // have plenty of room for the DFA states, not to use it up\n // on the program.)\n if (m >= 1<<24)\n m = 1<<24;\n // Inst imposes its own limit (currently bigger than 2^24 but be safe).\n if (m > Prog::Inst::kMaxInst)\n m = Prog::Inst::kMaxInst;\n max_ninst_ = static_cast(m);\n }\n anchor_ = anchor;\n}\n\n// Compiles re, returning program.\n// Caller is responsible for deleting prog_.\n// If reversed is true, compiles a program that expects\n// to run over the input string backward (reverses all concatenations).\n// The reversed flag is also recorded in the returned program.\nProg* Compiler::Compile(Regexp* re, bool reversed, int64_t max_mem) {\n Compiler c;\n c.Setup(re->parse_flags(), max_mem, RE2::UNANCHORED /* unused */);\n c.reversed_ = reversed;\n\n // Simplify to remove things like counted repetitions\n // and character classes like \\d.\n Regexp* sre = re->Simplify();\n if (sre == NULL)\n return NULL;\n\n // Record whether prog is anchored, removing the anchors.\n // (They get in the way of other optimizations.)\n bool is_anchor_start = IsAnchorStart(&sre, 0);\n bool is_anchor_end = IsAnchorEnd(&sre, 0);\n\n // Generate fragment for entire regexp.\n Frag all = c.WalkExponential(sre, Frag(), 2*c.max_ninst_);\n sre->Decref();\n if (c.failed_)\n return NULL;\n\n // Success! Finish by putting Match node at end, and record start.\n // Turn off c.reversed_ (if it is set) to force the remaining concatenations\n // to behave normally.\n c.reversed_ = false;\n all = c.Cat(all, c.Match(0));\n\n c.prog_->set_reversed(reversed);\n if (c.prog_->reversed()) {\n c.prog_->set_anchor_start(is_anchor_end);\n c.prog_->set_anchor_end(is_anchor_start);\n } else {\n c.prog_->set_anchor_start(is_anchor_start);\n c.prog_->set_anchor_end(is_anchor_end);\n }\n\n c.prog_->set_start(all.begin);\n if (!c.prog_->anchor_start()) {\n // Also create unanchored version, which starts with a .*? loop.\n all = c.Cat(c.DotStar(), all);\n }\n c.prog_->set_start_unanchored(all.begin);\n\n // Hand ownership of prog_ to caller.\n return c.Finish(re);\n}\n\nProg* Compiler::Finish(Regexp* re) {\n if (failed_)\n return NULL;\n\n if (prog_->start() == 0 && prog_->start_unanchored() == 0) {\n // No possible matches; keep Fail instruction only.\n ninst_ = 1;\n }\n\n // Hand off the array to Prog.\n prog_->inst_ = std::move(inst_);\n prog_->size_ = ninst_;\n\n prog_->Optimize();\n prog_->Flatten();\n prog_->ComputeByteMap();\n\n if (!prog_->reversed()) {\n std::string prefix;\n bool prefix_foldcase;\n if (re->RequiredPrefixForAccel(&prefix, &prefix_foldcase))\n prog_->ConfigurePrefixAccel(prefix, prefix_foldcase);\n }\n\n // Record remaining memory for DFA.\n if (max_mem_ <= 0) {\n prog_->set_dfa_mem(1<<20);\n } else {\n int64_t m = max_mem_ - sizeof(Prog);\n m -= prog_->size_*sizeof(Prog::Inst); // account for inst_\n if (prog_->CanBitState())\n m -= prog_->size_*sizeof(uint16_t); // account for list_heads_\n if (m < 0)\n m = 0;\n prog_->set_dfa_mem(m);\n }\n\n Prog* p = prog_;\n prog_ = NULL;\n return p;\n}\n\n// Converts Regexp to Prog.\nProg* Regexp::CompileToProg(int64_t max_mem) {\n return Compiler::Compile(this, false, max_mem);\n}\n\nProg* Regexp::CompileToReverseProg(int64_t max_mem) {\n return Compiler::Compile(this, true, max_mem);\n}\n\nFrag Compiler::DotStar() {\n return Star(ByteRange(0x00, 0xff, false), true);\n}\n\n// Compiles RE set to Prog.\nProg* Compiler::CompileSet(Regexp* re, RE2::Anchor anchor, int64_t max_mem) {\n Compiler c;\n c.Setup(re->parse_flags(), max_mem, anchor);\n\n Regexp* sre = re->Simplify();\n if (sre == NULL)\n return NULL;\n\n Frag all = c.WalkExponential(sre, Frag(), 2*c.max_ninst_);\n sre->Decref();\n if (c.failed_)\n return NULL;\n\n c.prog_->set_anchor_start(true);\n c.prog_->set_anchor_end(true);\n\n if (anchor == RE2::UNANCHORED) {\n // Prepend .* or else the expression will effectively be anchored.\n // Complemented by the ANCHOR_BOTH case in PostVisit().\n all = c.Cat(c.DotStar(), all);\n }\n c.prog_->set_start(all.begin);\n c.prog_->set_start_unanchored(all.begin);\n\n Prog* prog = c.Finish(re);\n if (prog == NULL)\n return NULL;\n\n // Make sure DFA has enough memory to operate,\n // since we're not going to fall back to the NFA.\n bool dfa_failed = false;\n absl::string_view sp = \"hello, world\";\n prog->SearchDFA(sp, sp, Prog::kAnchored, Prog::kManyMatch,\n NULL, &dfa_failed, NULL);\n if (dfa_failed) {\n delete prog;\n return NULL;\n }\n\n return prog;\n}\n\nProg* Prog::CompileSet(Regexp* re, RE2::Anchor anchor, int64_t max_mem) {\n return Compiler::CompileSet(re, anchor, max_mem);\n}\n\n} // namespace re2\n" }, { "path": "re2/dfa.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// A DFA (deterministic finite automaton)-based regular expression search.\n//\n// The DFA search has two main parts: the construction of the automaton,\n// which is represented by a graph of State structures, and the execution\n// of the automaton over a given input string.\n//\n// The basic idea is that the State graph is constructed so that the\n// execution can simply start with a state s, and then for each byte c in\n// the input string, execute \"s = s->next[c]\", checking at each point whether\n// the current s represents a matching state.\n//\n// The simple explanation just given does convey the essence of this code,\n// but it omits the details of how the State graph gets constructed as well\n// as some performance-driven optimizations to the execution of the automaton.\n// All these details are explained in the comments for the code following\n// the definition of class DFA.\n//\n// See http://swtch.com/~rsc/regexp/ for a very bare-bones equivalent.\n\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"absl/base/call_once.h\"\n#include \"absl/base/thread_annotations.h\"\n#include \"absl/container/flat_hash_map.h\"\n#include \"absl/container/flat_hash_set.h\"\n#include \"absl/hash/hash.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"absl/synchronization/mutex.h\"\n#include \"absl/types/span.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/prog.h\"\n#include \"re2/re2.h\"\n#include \"re2/sparse_set.h\"\n#include \"util/strutil.h\"\n\n// Silence \"zero-sized array in struct/union\" warning for DFA::State::next_.\n#ifdef _MSC_VER\n#pragma warning(disable: 4200)\n#endif\n\nnamespace re2 {\n\n// Controls whether the DFA should bail out early if the NFA would be faster.\nstatic bool dfa_should_bail_when_slow = true;\n\nvoid Prog::TESTING_ONLY_set_dfa_should_bail_when_slow(bool b) {\n dfa_should_bail_when_slow = b;\n}\n\n// Changing this to true compiles in prints that trace execution of the DFA.\n// Generates a lot of output -- only useful for debugging.\nstatic const bool ExtraDebug = false;\n\n// A DFA implementation of a regular expression program.\n// Since this is entirely a forward declaration mandated by C++,\n// some of the comments here are better understood after reading\n// the comments in the sections that follow the DFA definition.\nclass DFA {\n public:\n DFA(Prog* prog, Prog::MatchKind kind, int64_t max_mem);\n ~DFA();\n bool ok() const { return !init_failed_; }\n Prog::MatchKind kind() { return kind_; }\n\n // Searches for the regular expression in text, which is considered\n // as a subsection of context for the purposes of interpreting flags\n // like ^ and $ and \\A and \\z.\n // Returns whether a match was found.\n // If a match is found, sets *ep to the end point of the best match in text.\n // If \"anchored\", the match must begin at the start of text.\n // If \"want_earliest_match\", the match that ends first is used, not\n // necessarily the best one.\n // If \"run_forward\" is true, the DFA runs from text.begin() to text.end().\n // If it is false, the DFA runs from text.end() to text.begin(),\n // returning the leftmost end of the match instead of the rightmost one.\n // If the DFA cannot complete the search (for example, if it is out of\n // memory), it sets *failed and returns false.\n bool Search(absl::string_view text, absl::string_view context, bool anchored,\n bool want_earliest_match, bool run_forward, bool* failed,\n const char** ep, SparseSet* matches);\n\n // Builds out all states for the entire DFA.\n // If cb is not empty, it receives one callback per state built.\n // Returns the number of states built.\n // FOR TESTING OR EXPERIMENTAL PURPOSES ONLY.\n int BuildAllStates(const Prog::DFAStateCallback& cb);\n\n // Computes min and max for matching strings. Won't return strings\n // bigger than maxlen.\n bool PossibleMatchRange(std::string* min, std::string* max, int maxlen);\n\n // These data structures are logically private, but C++ makes it too\n // difficult to mark them as such.\n class RWLocker;\n class StateSaver;\n class Workq;\n\n // A single DFA state. The DFA is represented as a graph of these\n // States, linked by the next_ pointers. If in state s and reading\n // byte c, the next state should be s->next_[c].\n struct State {\n inline bool IsMatch() const { return (flag_ & kFlagMatch) != 0; }\n\n template \n friend H AbslHashValue(H h, const State& a) {\n const absl::Span ainst(a.inst_, a.ninst_);\n return H::combine(std::move(h), a.flag_, ainst);\n }\n\n friend bool operator==(const State& a, const State& b) {\n const absl::Span ainst(a.inst_, a.ninst_);\n const absl::Span binst(b.inst_, b.ninst_);\n return &a == &b || (a.flag_ == b.flag_ && ainst == binst);\n }\n\n int* inst_; // Instruction pointers in the state.\n int ninst_; // # of inst_ pointers.\n uint32_t flag_; // Empty string bitfield flags in effect on the way\n // into this state, along with kFlagMatch if this\n // is a matching state.\n\n std::atomic next_[]; // Outgoing arrows from State,\n // one per input byte class\n };\n\n enum {\n kByteEndText = 256, // imaginary byte at end of text\n\n kFlagEmptyMask = 0xFF, // State.flag_: bits holding kEmptyXXX flags\n kFlagMatch = 0x0100, // State.flag_: this is a matching state\n kFlagLastWord = 0x0200, // State.flag_: last byte was a word char\n kFlagNeedShift = 16, // needed kEmpty bits are or'ed in shifted left\n };\n\n struct StateHash {\n size_t operator()(const State* a) const {\n ABSL_DCHECK(a != NULL);\n return absl::Hash()(*a);\n }\n };\n\n struct StateEqual {\n bool operator()(const State* a, const State* b) const {\n ABSL_DCHECK(a != NULL);\n ABSL_DCHECK(b != NULL);\n return *a == *b;\n }\n };\n\n typedef absl::flat_hash_set StateSet;\n\n private:\n // Make it easier to swap in a scalable reader-writer mutex.\n using CacheMutex = absl::Mutex;\n\n enum {\n // Indices into start_ for unanchored searches.\n // Add kStartAnchored for anchored searches.\n kStartBeginText = 0, // text at beginning of context\n kStartBeginLine = 2, // text at beginning of line\n kStartAfterWordChar = 4, // text follows a word character\n kStartAfterNonWordChar = 6, // text follows non-word character\n kMaxStart = 8,\n\n kStartAnchored = 1,\n };\n\n // Resets the DFA State cache, flushing all saved State* information.\n // Releases and reacquires cache_mutex_ via cache_lock, so any\n // State* existing before the call are not valid after the call.\n // Use a StateSaver to preserve important states across the call.\n // cache_mutex_.r <= L < mutex_\n // After: cache_mutex_.w <= L < mutex_\n void ResetCache(RWLocker* cache_lock);\n\n // Looks up and returns the State corresponding to a Workq.\n // L >= mutex_\n State* WorkqToCachedState(Workq* q, Workq* mq, uint32_t flag);\n\n // Looks up and returns a State matching the inst, ninst, and flag.\n // L >= mutex_\n State* CachedState(int* inst, int ninst, uint32_t flag);\n\n // Clear the cache entirely.\n // Must hold cache_mutex_.w or be in destructor.\n void ClearCache();\n\n // Converts a State into a Workq: the opposite of WorkqToCachedState.\n // L >= mutex_\n void StateToWorkq(State* s, Workq* q);\n\n // Runs a State on a given byte, returning the next state.\n State* RunStateOnByteUnlocked(State*, int); // cache_mutex_.r <= L < mutex_\n State* RunStateOnByte(State*, int); // L >= mutex_\n\n // Runs a Workq on a given byte followed by a set of empty-string flags,\n // producing a new Workq in nq. If a match instruction is encountered,\n // sets *ismatch to true.\n // L >= mutex_\n void RunWorkqOnByte(Workq* q, Workq* nq,\n int c, uint32_t flag, bool* ismatch);\n\n // Runs a Workq on a set of empty-string flags, producing a new Workq in nq.\n // L >= mutex_\n void RunWorkqOnEmptyString(Workq* q, Workq* nq, uint32_t flag);\n\n // Adds the instruction id to the Workq, following empty arrows\n // according to flag.\n // L >= mutex_\n void AddToQueue(Workq* q, int id, uint32_t flag);\n\n // For debugging, returns a text representation of State.\n static std::string DumpState(State* state);\n\n // For debugging, returns a text representation of a Workq.\n static std::string DumpWorkq(Workq* q);\n\n // Search parameters\n struct SearchParams {\n SearchParams(absl::string_view text, absl::string_view context,\n RWLocker* cache_lock)\n : text(text),\n context(context),\n anchored(false),\n can_prefix_accel(false),\n want_earliest_match(false),\n run_forward(false),\n start(NULL),\n cache_lock(cache_lock),\n failed(false),\n ep(NULL),\n matches(NULL) {}\n\n absl::string_view text;\n absl::string_view context;\n bool anchored;\n bool can_prefix_accel;\n bool want_earliest_match;\n bool run_forward;\n State* start;\n RWLocker* cache_lock;\n bool failed; // \"out\" parameter: whether search gave up\n const char* ep; // \"out\" parameter: end pointer for match\n SparseSet* matches;\n\n private:\n SearchParams(const SearchParams&) = delete;\n SearchParams& operator=(const SearchParams&) = delete;\n };\n\n // Before each search, the parameters to Search are analyzed by\n // AnalyzeSearch to determine the state in which to start.\n struct StartInfo {\n StartInfo() : start(NULL) {}\n std::atomic start;\n };\n\n // Fills in params->start and params->can_prefix_accel using\n // the other search parameters. Returns true on success,\n // false on failure.\n // cache_mutex_.r <= L < mutex_\n bool AnalyzeSearch(SearchParams* params);\n bool AnalyzeSearchHelper(SearchParams* params, StartInfo* info,\n uint32_t flags);\n\n // The generic search loop, inlined to create specialized versions.\n // cache_mutex_.r <= L < mutex_\n // Might unlock and relock cache_mutex_ via params->cache_lock.\n template \n inline bool InlinedSearchLoop(SearchParams* params);\n\n // The specialized versions of InlinedSearchLoop. The three letters\n // at the ends of the name denote the true/false values used as the\n // last three parameters of InlinedSearchLoop.\n // cache_mutex_.r <= L < mutex_\n // Might unlock and relock cache_mutex_ via params->cache_lock.\n bool SearchFFF(SearchParams* params);\n bool SearchFFT(SearchParams* params);\n bool SearchFTF(SearchParams* params);\n bool SearchFTT(SearchParams* params);\n bool SearchTFF(SearchParams* params);\n bool SearchTFT(SearchParams* params);\n bool SearchTTF(SearchParams* params);\n bool SearchTTT(SearchParams* params);\n\n // The main search loop: calls an appropriate specialized version of\n // InlinedSearchLoop.\n // cache_mutex_.r <= L < mutex_\n // Might unlock and relock cache_mutex_ via params->cache_lock.\n bool FastSearchLoop(SearchParams* params);\n\n\n // Looks up bytes in bytemap_ but handles case c == kByteEndText too.\n int ByteMap(int c) {\n if (c == kByteEndText)\n return prog_->bytemap_range();\n return prog_->bytemap()[c];\n }\n\n // Constant after initialization.\n Prog* prog_; // The regular expression program to run.\n Prog::MatchKind kind_; // The kind of DFA.\n bool init_failed_; // initialization failed (out of memory)\n\n absl::Mutex mutex_; // mutex_ >= cache_mutex_.r\n\n // Scratch areas, protected by mutex_.\n Workq* q0_; // Two pre-allocated work queues.\n Workq* q1_;\n PODArray stack_; // Pre-allocated stack for AddToQueue\n\n // State* cache. Many threads use and add to the cache simultaneously,\n // holding cache_mutex_ for reading and mutex_ (above) when adding.\n // If the cache fills and needs to be discarded, the discarding is done\n // while holding cache_mutex_ for writing, to avoid interrupting other\n // readers. Any State* pointers are only valid while cache_mutex_\n // is held.\n CacheMutex cache_mutex_;\n int64_t mem_budget_; // Total memory budget for all States.\n int64_t state_budget_; // Amount of memory remaining for new States.\n StateSet state_cache_; // All States computed so far.\n StartInfo start_[kMaxStart];\n\n DFA(const DFA&) = delete;\n DFA& operator=(const DFA&) = delete;\n};\n\n// Shorthand for casting to uint8_t*.\nstatic inline const uint8_t* BytePtr(const void* v) {\n return reinterpret_cast(v);\n}\n\n// Work queues\n\n// Marks separate thread groups of different priority\n// in the work queue when in leftmost-longest matching mode.\n#define Mark (-1)\n\n// Separates the match IDs from the instructions in inst_.\n// Used only for \"many match\" DFA states.\n#define MatchSep (-2)\n\n// Internally, the DFA uses a sparse array of\n// program instruction pointers as a work queue.\n// In leftmost longest mode, marks separate sections\n// of workq that started executing at different\n// locations in the string (earlier locations first).\nclass DFA::Workq : public SparseSet {\n public:\n // Constructor: n is number of normal slots, maxmark number of mark slots.\n Workq(int n, int maxmark) :\n SparseSet(n+maxmark),\n n_(n),\n maxmark_(maxmark),\n nextmark_(n),\n last_was_mark_(true) {\n }\n\n bool is_mark(int i) { return i >= n_; }\n\n int maxmark() { return maxmark_; }\n\n void clear() {\n SparseSet::clear();\n nextmark_ = n_;\n }\n\n void mark() {\n if (last_was_mark_)\n return;\n last_was_mark_ = false;\n SparseSet::insert_new(nextmark_++);\n }\n\n int size() {\n return n_ + maxmark_;\n }\n\n void insert(int id) {\n if (contains(id))\n return;\n insert_new(id);\n }\n\n void insert_new(int id) {\n last_was_mark_ = false;\n SparseSet::insert_new(id);\n }\n\n private:\n int n_; // size excluding marks\n int maxmark_; // maximum number of marks\n int nextmark_; // id of next mark\n bool last_was_mark_; // last inserted was mark\n\n Workq(const Workq&) = delete;\n Workq& operator=(const Workq&) = delete;\n};\n\nDFA::DFA(Prog* prog, Prog::MatchKind kind, int64_t max_mem)\n : prog_(prog),\n kind_(kind),\n init_failed_(false),\n q0_(NULL),\n q1_(NULL),\n mem_budget_(max_mem) {\n if (ExtraDebug)\n absl::FPrintF(stderr, \"\\nkind %d\\n%s\\n\", kind_, prog_->DumpUnanchored());\n int nmark = 0;\n if (kind_ == Prog::kLongestMatch)\n nmark = prog_->size();\n // See DFA::AddToQueue() for why this is so.\n int nstack = prog_->inst_count(kInstCapture) +\n prog_->inst_count(kInstEmptyWidth) +\n prog_->inst_count(kInstNop) +\n nmark + 1; // + 1 for start inst\n\n // Account for space needed for DFA, q0, q1, stack.\n mem_budget_ -= sizeof(DFA);\n mem_budget_ -= (prog_->size() + nmark) *\n (sizeof(int)+sizeof(int)) * 2; // q0, q1\n mem_budget_ -= nstack * sizeof(int); // stack\n if (mem_budget_ < 0) {\n init_failed_ = true;\n return;\n }\n\n state_budget_ = mem_budget_;\n\n // Make sure there is a reasonable amount of working room left.\n // At minimum, the search requires room for two states in order\n // to limp along, restarting frequently. We'll get better performance\n // if there is room for a larger number of states, say 20.\n // Note that a state stores list heads only, so we use the program\n // list count for the upper bound, not the program size.\n int nnext = prog_->bytemap_range() + 1; // + 1 for kByteEndText slot\n int64_t one_state = sizeof(State) + nnext*sizeof(std::atomic) +\n (prog_->list_count()+nmark)*sizeof(int);\n if (state_budget_ < 20*one_state) {\n init_failed_ = true;\n return;\n }\n\n q0_ = new Workq(prog_->size(), nmark);\n q1_ = new Workq(prog_->size(), nmark);\n stack_ = PODArray(nstack);\n}\n\nDFA::~DFA() {\n delete q0_;\n delete q1_;\n ClearCache();\n}\n\n// In the DFA state graph, s->next[c] == NULL means that the\n// state has not yet been computed and needs to be. We need\n// a different special value to signal that s->next[c] is a\n// state that can never lead to a match (and thus the search\n// can be called off). Hence DeadState.\n#define DeadState reinterpret_cast(1)\n\n// Signals that the rest of the string matches no matter what it is.\n#define FullMatchState reinterpret_cast(2)\n\n#define SpecialStateMax FullMatchState\n\n// Debugging printouts\n\n// For debugging, returns a string representation of the work queue.\nstd::string DFA::DumpWorkq(Workq* q) {\n std::string s;\n const char* sep = \"\";\n for (Workq::iterator it = q->begin(); it != q->end(); ++it) {\n if (q->is_mark(*it)) {\n s += \"|\";\n sep = \"\";\n } else {\n s += absl::StrFormat(\"%s%d\", sep, *it);\n sep = \",\";\n }\n }\n return s;\n}\n\n// For debugging, returns a string representation of the state.\nstd::string DFA::DumpState(State* state) {\n if (state == NULL)\n return \"_\";\n if (state == DeadState)\n return \"X\";\n if (state == FullMatchState)\n return \"*\";\n std::string s;\n const char* sep = \"\";\n s += absl::StrFormat(\"(%p)\", state);\n for (int i = 0; i < state->ninst_; i++) {\n if (state->inst_[i] == Mark) {\n s += \"|\";\n sep = \"\";\n } else if (state->inst_[i] == MatchSep) {\n s += \"||\";\n sep = \"\";\n } else {\n s += absl::StrFormat(\"%s%d\", sep, state->inst_[i]);\n sep = \",\";\n }\n }\n s += absl::StrFormat(\" flag=%#x\", state->flag_);\n return s;\n}\n\n//////////////////////////////////////////////////////////////////////\n//\n// DFA state graph construction.\n//\n// The DFA state graph is a heavily-linked collection of State* structures.\n// The state_cache_ is a set of all the State structures ever allocated,\n// so that if the same state is reached by two different paths,\n// the same State structure can be used. This reduces allocation\n// requirements and also avoids duplication of effort across the two\n// identical states.\n//\n// A State is defined by an ordered list of instruction ids and a flag word.\n//\n// The choice of an ordered list of instructions differs from a typical\n// textbook DFA implementation, which would use an unordered set.\n// Textbook descriptions, however, only care about whether\n// the DFA matches, not where it matches in the text. To decide where the\n// DFA matches, we need to mimic the behavior of the dominant backtracking\n// implementations like PCRE, which try one possible regular expression\n// execution, then another, then another, stopping when one of them succeeds.\n// The DFA execution tries these many executions in parallel, representing\n// each by an instruction id. These pointers are ordered in the State.inst_\n// list in the same order that the executions would happen in a backtracking\n// search: if a match is found during execution of inst_[2], inst_[i] for i>=3\n// can be discarded.\n//\n// Textbooks also typically do not consider context-aware empty string operators\n// like ^ or $. These are handled by the flag word, which specifies the set\n// of empty-string operators that should be matched when executing at the\n// current text position. These flag bits are defined in prog.h.\n// The flag word also contains two DFA-specific bits: kFlagMatch if the state\n// is a matching state (one that reached a kInstMatch in the program)\n// and kFlagLastWord if the last processed byte was a word character, for the\n// implementation of \\B and \\b.\n//\n// The flag word also contains, shifted up 16 bits, the bits looked for by\n// any kInstEmptyWidth instructions in the state. These provide a useful\n// summary indicating when new flags might be useful.\n//\n// The permanent representation of a State's instruction ids is just an array,\n// but while a state is being analyzed, these instruction ids are represented\n// as a Workq, which is an array that allows iteration in insertion order.\n\n// NOTE(rsc): The choice of State construction determines whether the DFA\n// mimics backtracking implementations (so-called leftmost first matching) or\n// traditional DFA implementations (so-called leftmost longest matching as\n// prescribed by POSIX). This implementation chooses to mimic the\n// backtracking implementations, because we want to replace PCRE. To get\n// POSIX behavior, the states would need to be considered not as a simple\n// ordered list of instruction ids, but as a list of unordered sets of instruction\n// ids. A match by a state in one set would inhibit the running of sets\n// farther down the list but not other instruction ids in the same set. Each\n// set would correspond to matches beginning at a given point in the string.\n// This is implemented by separating different sets with Mark pointers.\n\n// Looks in the State cache for a State matching q, flag.\n// If one is found, returns it. If one is not found, allocates one,\n// inserts it in the cache, and returns it.\n// If mq is not null, MatchSep and the match IDs in mq will be appended\n// to the State.\nDFA::State* DFA::WorkqToCachedState(Workq* q, Workq* mq, uint32_t flag) {\n //mutex_.AssertHeld();\n\n // Construct array of instruction ids for the new state.\n // In some cases, kInstAltMatch may trigger an upgrade to FullMatchState.\n // Otherwise, \"compress\" q down to list heads for storage; StateToWorkq()\n // will \"decompress\" it for computation by exploring from each list head.\n //\n // Historically, only kInstByteRange, kInstEmptyWidth and kInstMatch were\n // useful to keep, but it turned out that kInstAlt was necessary to keep:\n //\n // > [*] kInstAlt would seem useless to record in a state, since\n // > we've already followed both its arrows and saved all the\n // > interesting states we can reach from there. The problem\n // > is that one of the empty-width instructions might lead\n // > back to the same kInstAlt (if an empty-width operator is starred),\n // > producing a different evaluation order depending on whether\n // > we keep the kInstAlt to begin with. Sigh.\n // > A specific case that this affects is /(^|a)+/ matching \"a\".\n // > If we don't save the kInstAlt, we will match the whole \"a\" (0,1)\n // > but in fact the correct leftmost-first match is the leading \"\" (0,0).\n //\n // Recall that flattening transformed the Prog from \"tree\" form to \"list\"\n // form: in the former, kInstAlt existed explicitly... and abundantly; in\n // the latter, it's implied between the instructions that compose a list.\n // Thus, because the information wasn't lost, the bug doesn't remanifest.\n PODArray inst(q->size());\n int n = 0;\n uint32_t needflags = 0; // flags needed by kInstEmptyWidth instructions\n bool sawmatch = false; // whether queue contains guaranteed kInstMatch\n bool sawmark = false; // whether queue contains a Mark\n if (ExtraDebug)\n absl::FPrintF(stderr, \"WorkqToCachedState %s [%#x]\", DumpWorkq(q), flag);\n for (Workq::iterator it = q->begin(); it != q->end(); ++it) {\n int id = *it;\n if (sawmatch && (kind_ == Prog::kFirstMatch || q->is_mark(id)))\n break;\n if (q->is_mark(id)) {\n if (n > 0 && inst[n-1] != Mark) {\n sawmark = true;\n inst[n++] = Mark;\n }\n continue;\n }\n Prog::Inst* ip = prog_->inst(id);\n switch (ip->opcode()) {\n case kInstAltMatch:\n // This state will continue to a match no matter what\n // the rest of the input is. If it is the highest priority match\n // being considered, return the special FullMatchState\n // to indicate that it's all matches from here out.\n if (kind_ != Prog::kManyMatch &&\n (kind_ != Prog::kFirstMatch ||\n (it == q->begin() && ip->greedy(prog_))) &&\n (kind_ != Prog::kLongestMatch || !sawmark) &&\n (flag & kFlagMatch)) {\n if (ExtraDebug)\n absl::FPrintF(stderr, \" -> FullMatchState\\n\");\n return FullMatchState;\n }\n [[fallthrough]];\n default:\n // Record iff id is the head of its list, which must\n // be the case if id-1 is the last of *its* list. :)\n if (prog_->inst(id-1)->last())\n inst[n++] = *it;\n if (ip->opcode() == kInstEmptyWidth)\n needflags |= ip->empty();\n if (ip->opcode() == kInstMatch && !prog_->anchor_end())\n sawmatch = true;\n break;\n }\n }\n ABSL_DCHECK_LE(n, q->size());\n if (n > 0 && inst[n-1] == Mark)\n n--;\n\n // If there are no empty-width instructions waiting to execute,\n // then the extra flag bits will not be used, so there is no\n // point in saving them. (Discarding them reduces the number\n // of distinct states.)\n if (needflags == 0)\n flag &= kFlagMatch;\n\n // NOTE(rsc): The code above cannot do flag &= needflags,\n // because if the right flags were present to pass the current\n // kInstEmptyWidth instructions, new kInstEmptyWidth instructions\n // might be reached that in turn need different flags.\n // The only sure thing is that if there are no kInstEmptyWidth\n // instructions at all, no flags will be needed.\n // We could do the extra work to figure out the full set of\n // possibly needed flags by exploring past the kInstEmptyWidth\n // instructions, but the check above -- are any flags needed\n // at all? -- handles the most common case. More fine-grained\n // analysis can only be justified by measurements showing that\n // too many redundant states are being allocated.\n\n // If there are no Insts in the list, it's a dead state,\n // which is useful to signal with a special pointer so that\n // the execution loop can stop early. This is only okay\n // if the state is *not* a matching state.\n if (n == 0 && flag == 0) {\n if (ExtraDebug)\n absl::FPrintF(stderr, \" -> DeadState\\n\");\n return DeadState;\n }\n\n // If we're in longest match mode, the state is a sequence of\n // unordered state sets separated by Marks. Sort each set\n // to canonicalize, to reduce the number of distinct sets stored.\n if (kind_ == Prog::kLongestMatch) {\n int* ip = inst.data();\n int* ep = ip + n;\n while (ip < ep) {\n int* markp = ip;\n while (markp < ep && *markp != Mark)\n markp++;\n std::sort(ip, markp);\n if (markp < ep)\n markp++;\n ip = markp;\n }\n }\n\n // If we're in many match mode, canonicalize for similar reasons:\n // we have an unordered set of states (i.e. we don't have Marks)\n // and sorting will reduce the number of distinct sets stored.\n if (kind_ == Prog::kManyMatch) {\n int* ip = inst.data();\n int* ep = ip + n;\n std::sort(ip, ep);\n }\n\n // Append MatchSep and the match IDs in mq if necessary.\n if (mq != NULL) {\n inst[n++] = MatchSep;\n for (Workq::iterator i = mq->begin(); i != mq->end(); ++i) {\n int id = *i;\n Prog::Inst* ip = prog_->inst(id);\n if (ip->opcode() == kInstMatch)\n inst[n++] = ip->match_id();\n }\n }\n\n // Save the needed empty-width flags in the top bits for use later.\n flag |= needflags << kFlagNeedShift;\n\n State* state = CachedState(inst.data(), n, flag);\n return state;\n}\n\n// Looks in the State cache for a State matching inst, ninst, flag.\n// If one is found, returns it. If one is not found, allocates one,\n// inserts it in the cache, and returns it.\nDFA::State* DFA::CachedState(int* inst, int ninst, uint32_t flag) {\n //mutex_.AssertHeld();\n\n // Look in the cache for a pre-existing state.\n // We have to initialise the struct like this because otherwise\n // MSVC will complain about the flexible array member. :(\n State state;\n state.inst_ = inst;\n state.ninst_ = ninst;\n state.flag_ = flag;\n StateSet::iterator it = state_cache_.find(&state);\n if (it != state_cache_.end()) {\n if (ExtraDebug)\n absl::FPrintF(stderr, \" -cached-> %s\\n\", DumpState(*it));\n return *it;\n }\n\n // Must have enough memory for new state.\n // In addition to what we're going to allocate,\n // the state cache hash table seems to incur about 18 bytes per\n // State*. Worst case for non-small sets is it being half full, where each\n // value present takes up 1 byte hash sample plus the pointer itself.\n const int kStateCacheOverhead = 18;\n int nnext = prog_->bytemap_range() + 1; // + 1 for kByteEndText slot\n int mem = sizeof(State) + nnext*sizeof(std::atomic);\n int instmem = ninst*sizeof(int);\n if (mem_budget_ < mem + instmem + kStateCacheOverhead) {\n mem_budget_ = -1;\n return NULL;\n }\n mem_budget_ -= mem + instmem + kStateCacheOverhead;\n\n // Allocate new state along with room for next_ and inst_.\n // inst_ is stored separately since it's colder; this also\n // means that the States for a given DFA are the same size\n // class, so the allocator can hopefully pack them better.\n char* space = std::allocator().allocate(mem);\n State* s = new (space) State;\n (void) new (s->next_) std::atomic[nnext];\n // Work around a unfortunate bug in older versions of libstdc++.\n // (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64658)\n for (int i = 0; i < nnext; i++)\n (void) new (s->next_ + i) std::atomic(NULL);\n s->inst_ = std::allocator().allocate(ninst);\n (void) new (s->inst_) int[ninst];\n memmove(s->inst_, inst, instmem);\n s->ninst_ = ninst;\n s->flag_ = flag;\n if (ExtraDebug)\n absl::FPrintF(stderr, \" -> %s\\n\", DumpState(s));\n\n // Put state in cache and return it.\n state_cache_.insert(s);\n return s;\n}\n\n// Clear the cache. Must hold cache_mutex_.w or be in destructor.\nvoid DFA::ClearCache() {\n StateSet::iterator begin = state_cache_.begin();\n StateSet::iterator end = state_cache_.end();\n while (begin != end) {\n StateSet::iterator tmp = begin;\n ++begin;\n // Deallocate the instruction array, which is stored separately as above.\n std::allocator().deallocate((*tmp)->inst_, (*tmp)->ninst_);\n // Deallocate the blob of memory that we allocated in DFA::CachedState().\n // We recompute mem in order to benefit from sized delete where possible.\n int nnext = prog_->bytemap_range() + 1; // + 1 for kByteEndText slot\n int mem = sizeof(State) + nnext*sizeof(std::atomic);\n std::allocator().deallocate(reinterpret_cast(*tmp), mem);\n }\n state_cache_.clear();\n}\n\n// Copies insts in state s to the work queue q.\nvoid DFA::StateToWorkq(State* s, Workq* q) {\n q->clear();\n for (int i = 0; i < s->ninst_; i++) {\n if (s->inst_[i] == Mark) {\n q->mark();\n } else if (s->inst_[i] == MatchSep) {\n // Nothing after this is an instruction!\n break;\n } else {\n // Explore from the head of the list.\n AddToQueue(q, s->inst_[i], s->flag_ & kFlagEmptyMask);\n }\n }\n}\n\n// Adds ip to the work queue, following empty arrows according to flag.\nvoid DFA::AddToQueue(Workq* q, int id, uint32_t flag) {\n\n // Use stack_ to hold our stack of instructions yet to process.\n // It was preallocated as follows:\n // one entry per Capture;\n // one entry per EmptyWidth; and\n // one entry per Nop.\n // This reflects the maximum number of stack pushes that each can\n // perform. (Each instruction can be processed at most once.)\n // When using marks, we also added nmark == prog_->size().\n // (Otherwise, nmark == 0.)\n int* stk = stack_.data();\n int nstk = 0;\n\n stk[nstk++] = id;\n while (nstk > 0) {\n ABSL_DCHECK_LE(nstk, stack_.size());\n id = stk[--nstk];\n\n Loop:\n if (id == Mark) {\n q->mark();\n continue;\n }\n\n if (id == 0)\n continue;\n\n // If ip is already on the queue, nothing to do.\n // Otherwise add it. We don't actually keep all the\n // ones that get added, but adding all of them here\n // increases the likelihood of q->contains(id),\n // reducing the amount of duplicated work.\n if (q->contains(id))\n continue;\n q->insert_new(id);\n\n // Process instruction.\n Prog::Inst* ip = prog_->inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"unhandled opcode: \" << ip->opcode();\n break;\n\n case kInstByteRange: // just save these on the queue\n case kInstMatch:\n if (ip->last())\n break;\n id = id+1;\n goto Loop;\n\n case kInstCapture: // DFA treats captures as no-ops.\n case kInstNop:\n if (!ip->last())\n stk[nstk++] = id+1;\n\n // If this instruction is the [00-FF]* loop at the beginning of\n // a leftmost-longest unanchored search, separate with a Mark so\n // that future threads (which will start farther to the right in\n // the input string) are lower priority than current threads.\n if (ip->opcode() == kInstNop && q->maxmark() > 0 &&\n id == prog_->start_unanchored() && id != prog_->start())\n stk[nstk++] = Mark;\n id = ip->out();\n goto Loop;\n\n case kInstAltMatch:\n ABSL_DCHECK(!ip->last());\n id = id+1;\n goto Loop;\n\n case kInstEmptyWidth:\n if (!ip->last())\n stk[nstk++] = id+1;\n\n // Continue on if we have all the right flag bits.\n if (ip->empty() & ~flag)\n break;\n id = ip->out();\n goto Loop;\n }\n }\n}\n\n// Running of work queues. In the work queue, order matters:\n// the queue is sorted in priority order. If instruction i comes before j,\n// then the instructions that i produces during the run must come before\n// the ones that j produces. In order to keep this invariant, all the\n// work queue runners have to take an old queue to process and then\n// also a new queue to fill in. It's not acceptable to add to the end of\n// an existing queue, because new instructions will not end up in the\n// correct position.\n\n// Runs the work queue, processing the empty strings indicated by flag.\n// For example, flag == kEmptyBeginLine|kEmptyEndLine means to match\n// both ^ and $. It is important that callers pass all flags at once:\n// processing both ^ and $ is not the same as first processing only ^\n// and then processing only $. Doing the two-step sequence won't match\n// ^$^$^$ but processing ^ and $ simultaneously will (and is the behavior\n// exhibited by existing implementations).\nvoid DFA::RunWorkqOnEmptyString(Workq* oldq, Workq* newq, uint32_t flag) {\n newq->clear();\n for (Workq::iterator i = oldq->begin(); i != oldq->end(); ++i) {\n if (oldq->is_mark(*i))\n AddToQueue(newq, Mark, flag);\n else\n AddToQueue(newq, *i, flag);\n }\n}\n\n// Runs the work queue, processing the single byte c followed by any empty\n// strings indicated by flag. For example, c == 'a' and flag == kEmptyEndLine,\n// means to match c$. Sets the bool *ismatch to true if the end of the\n// regular expression program has been reached (the regexp has matched).\nvoid DFA::RunWorkqOnByte(Workq* oldq, Workq* newq,\n int c, uint32_t flag, bool* ismatch) {\n //mutex_.AssertHeld();\n\n newq->clear();\n for (Workq::iterator i = oldq->begin(); i != oldq->end(); ++i) {\n if (oldq->is_mark(*i)) {\n if (*ismatch)\n return;\n newq->mark();\n continue;\n }\n int id = *i;\n Prog::Inst* ip = prog_->inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"unhandled opcode: \" << ip->opcode();\n break;\n\n case kInstFail: // never succeeds\n case kInstCapture: // already followed\n case kInstNop: // already followed\n case kInstAltMatch: // already followed\n case kInstEmptyWidth: // already followed\n break;\n\n case kInstByteRange: // can follow if c is in range\n if (!ip->Matches(c))\n break;\n AddToQueue(newq, ip->out(), flag);\n if (ip->hint() != 0) {\n // We have a hint, but we must cancel out the\n // increment that will occur after the break.\n i += ip->hint() - 1;\n } else {\n // We have no hint, so we must find the end\n // of the current list and then skip to it.\n Prog::Inst* ip0 = ip;\n while (!ip->last())\n ++ip;\n i += ip - ip0;\n }\n break;\n\n case kInstMatch:\n if (prog_->anchor_end() && c != kByteEndText &&\n kind_ != Prog::kManyMatch)\n break;\n *ismatch = true;\n if (kind_ == Prog::kFirstMatch) {\n // Can stop processing work queue since we found a match.\n return;\n }\n break;\n }\n }\n\n if (ExtraDebug)\n absl::FPrintF(stderr, \"%s on %d[%#x] -> %s [%d]\\n\",\n DumpWorkq(oldq), c, flag, DumpWorkq(newq), *ismatch);\n}\n\n// Processes input byte c in state, returning new state.\n// Caller does not hold mutex.\nDFA::State* DFA::RunStateOnByteUnlocked(State* state, int c) {\n // Keep only one RunStateOnByte going\n // even if the DFA is being run by multiple threads.\n absl::MutexLock l(mutex_);\n return RunStateOnByte(state, c);\n}\n\n// Processes input byte c in state, returning new state.\nDFA::State* DFA::RunStateOnByte(State* state, int c) {\n //mutex_.AssertHeld();\n\n if (state <= SpecialStateMax) {\n if (state == FullMatchState) {\n // It is convenient for routines like PossibleMatchRange\n // if we implement RunStateOnByte for FullMatchState:\n // once you get into this state you never get out,\n // so it's pretty easy.\n return FullMatchState;\n }\n if (state == DeadState) {\n ABSL_LOG(DFATAL) << \"DeadState in RunStateOnByte\";\n return NULL;\n }\n if (state == NULL) {\n ABSL_LOG(DFATAL) << \"NULL state in RunStateOnByte\";\n return NULL;\n }\n ABSL_LOG(DFATAL) << \"Unexpected special state in RunStateOnByte\";\n return NULL;\n }\n\n // If someone else already computed this, return it.\n State* ns = state->next_[ByteMap(c)].load(std::memory_order_relaxed);\n if (ns != NULL)\n return ns;\n\n // Convert state into Workq.\n StateToWorkq(state, q0_);\n\n // Flags marking the kinds of empty-width things (^ $ etc)\n // around this byte. Before the byte we have the flags recorded\n // in the State structure itself. After the byte we have\n // nothing yet (but that will change: read on).\n uint32_t needflag = state->flag_ >> kFlagNeedShift;\n uint32_t beforeflag = state->flag_ & kFlagEmptyMask;\n uint32_t oldbeforeflag = beforeflag;\n uint32_t afterflag = 0;\n\n if (c == '\\n') {\n // Insert implicit $ and ^ around \\n\n beforeflag |= kEmptyEndLine;\n afterflag |= kEmptyBeginLine;\n }\n\n if (c == kByteEndText) {\n // Insert implicit $ and \\z before the fake \"end text\" byte.\n beforeflag |= kEmptyEndLine | kEmptyEndText;\n }\n\n // The state flag kFlagLastWord says whether the last\n // byte processed was a word character. Use that info to\n // insert empty-width (non-)word boundaries.\n bool islastword = (state->flag_ & kFlagLastWord) != 0;\n bool isword = c != kByteEndText && Prog::IsWordChar(static_cast(c));\n if (isword == islastword)\n beforeflag |= kEmptyNonWordBoundary;\n else\n beforeflag |= kEmptyWordBoundary;\n\n // Okay, finally ready to run.\n // Only useful to rerun on empty string if there are new, useful flags.\n if (beforeflag & ~oldbeforeflag & needflag) {\n RunWorkqOnEmptyString(q0_, q1_, beforeflag);\n using std::swap;\n swap(q0_, q1_);\n }\n bool ismatch = false;\n RunWorkqOnByte(q0_, q1_, c, afterflag, &ismatch);\n using std::swap;\n swap(q0_, q1_);\n\n // Save afterflag along with ismatch and isword in new state.\n uint32_t flag = afterflag;\n if (ismatch)\n flag |= kFlagMatch;\n if (isword)\n flag |= kFlagLastWord;\n\n if (ismatch && kind_ == Prog::kManyMatch)\n ns = WorkqToCachedState(q0_, q1_, flag);\n else\n ns = WorkqToCachedState(q0_, NULL, flag);\n\n // Flush ns before linking to it.\n // Write barrier before updating state->next_ so that the\n // main search loop can proceed without any locking, for speed.\n // (Otherwise it would need one mutex operation per input byte.)\n state->next_[ByteMap(c)].store(ns, std::memory_order_release);\n return ns;\n}\n\n\n//////////////////////////////////////////////////////////////////////\n// DFA cache reset.\n\n// Reader-writer lock helper.\n//\n// The DFA uses a reader-writer mutex to protect the state graph itself.\n// Traversing the state graph requires holding the mutex for reading,\n// and discarding the state graph and starting over requires holding the\n// lock for writing. If a search needs to expand the graph but is out\n// of memory, it will need to drop its read lock and then acquire the\n// write lock. Since it cannot then atomically downgrade from write lock\n// to read lock, it runs the rest of the search holding the write lock.\n// (This probably helps avoid repeated contention, but really the decision\n// is forced by the Mutex interface.) It's a bit complicated to keep\n// track of whether the lock is held for reading or writing and thread\n// that through the search, so instead we encapsulate it in the RWLocker\n// and pass that around.\n\nclass DFA::RWLocker {\n public:\n explicit RWLocker(CacheMutex* mu);\n ~RWLocker();\n\n // If the lock is only held for reading right now,\n // drop the read lock and re-acquire for writing.\n // Subsequent calls to LockForWriting are no-ops.\n // Notice that the lock is *released* temporarily.\n void LockForWriting();\n\n private:\n CacheMutex* mu_;\n bool writing_;\n\n RWLocker(const RWLocker&) = delete;\n RWLocker& operator=(const RWLocker&) = delete;\n};\n\nDFA::RWLocker::RWLocker(CacheMutex* mu) : mu_(mu), writing_(false) {\n mu_->ReaderLock();\n}\n\n// This function is marked as ABSL_NO_THREAD_SAFETY_ANALYSIS because\n// the annotations don't support lock upgrade.\nvoid DFA::RWLocker::LockForWriting() ABSL_NO_THREAD_SAFETY_ANALYSIS {\n if (!writing_) {\n mu_->ReaderUnlock();\n mu_->WriterLock();\n writing_ = true;\n }\n}\n\nDFA::RWLocker::~RWLocker() {\n if (!writing_)\n mu_->ReaderUnlock();\n else\n mu_->WriterUnlock();\n}\n\n\n// When the DFA's State cache fills, we discard all the states in the\n// cache and start over. Many threads can be using and adding to the\n// cache at the same time, so we synchronize using the cache_mutex_\n// to keep from stepping on other threads. Specifically, all the\n// threads using the current cache hold cache_mutex_ for reading.\n// When a thread decides to flush the cache, it drops cache_mutex_\n// and then re-acquires it for writing. That ensures there are no\n// other threads accessing the cache anymore. The rest of the search\n// runs holding cache_mutex_ for writing, avoiding any contention\n// with or cache pollution caused by other threads.\n\nvoid DFA::ResetCache(RWLocker* cache_lock) {\n // Re-acquire the cache_mutex_ for writing (exclusive use).\n cache_lock->LockForWriting();\n\n hooks::GetDFAStateCacheResetHook()({\n state_budget_,\n state_cache_.size(),\n });\n\n // Clear the cache, reset the memory budget.\n for (int i = 0; i < kMaxStart; i++)\n start_[i].start.store(NULL, std::memory_order_relaxed);\n ClearCache();\n mem_budget_ = state_budget_;\n}\n\n// Typically, a couple States do need to be preserved across a cache\n// reset, like the State at the current point in the search.\n// The StateSaver class helps keep States across cache resets.\n// It makes a copy of the state's guts outside the cache (before the reset)\n// and then can be asked, after the reset, to recreate the State\n// in the new cache. For example, in a DFA method (\"this\" is a DFA):\n//\n// StateSaver saver(this, s);\n// ResetCache(cache_lock);\n// s = saver.Restore();\n//\n// The saver should always have room in the cache to re-create the state,\n// because resetting the cache locks out all other threads, and the cache\n// is known to have room for at least a couple states (otherwise the DFA\n// constructor fails).\n\nclass DFA::StateSaver {\n public:\n explicit StateSaver(DFA* dfa, State* state);\n ~StateSaver();\n\n // Recreates and returns a state equivalent to the\n // original state passed to the constructor.\n // Returns NULL if the cache has filled, but\n // since the DFA guarantees to have room in the cache\n // for a couple states, should never return NULL\n // if used right after ResetCache.\n State* Restore();\n\n private:\n DFA* dfa_; // the DFA to use\n int* inst_; // saved info from State\n int ninst_;\n uint32_t flag_;\n bool is_special_; // whether original state was special\n State* special_; // if is_special_, the original state\n\n StateSaver(const StateSaver&) = delete;\n StateSaver& operator=(const StateSaver&) = delete;\n};\n\nDFA::StateSaver::StateSaver(DFA* dfa, State* state) {\n dfa_ = dfa;\n if (state <= SpecialStateMax) {\n inst_ = NULL;\n ninst_ = 0;\n flag_ = 0;\n is_special_ = true;\n special_ = state;\n return;\n }\n is_special_ = false;\n special_ = NULL;\n flag_ = state->flag_;\n ninst_ = state->ninst_;\n inst_ = new int[ninst_];\n memmove(inst_, state->inst_, ninst_*sizeof inst_[0]);\n}\n\nDFA::StateSaver::~StateSaver() {\n if (!is_special_)\n delete[] inst_;\n}\n\nDFA::State* DFA::StateSaver::Restore() {\n if (is_special_)\n return special_;\n absl::MutexLock l(dfa_->mutex_);\n State* s = dfa_->CachedState(inst_, ninst_, flag_);\n if (s == NULL)\n ABSL_LOG(DFATAL) << \"StateSaver failed to restore state.\";\n return s;\n}\n\n\n//////////////////////////////////////////////////////////////////////\n//\n// DFA execution.\n//\n// The basic search loop is easy: start in a state s and then for each\n// byte c in the input, s = s->next[c].\n//\n// This simple description omits a few efficiency-driven complications.\n//\n// First, the State graph is constructed incrementally: it is possible\n// that s->next[c] is null, indicating that that state has not been\n// fully explored. In this case, RunStateOnByte must be invoked to\n// determine the next state, which is cached in s->next[c] to save\n// future effort. An alternative reason for s->next[c] to be null is\n// that the DFA has reached a so-called \"dead state\", in which any match\n// is no longer possible. In this case RunStateOnByte will return NULL\n// and the processing of the string can stop early.\n//\n// Second, a 256-element pointer array for s->next_ makes each State\n// quite large (2kB on 64-bit machines). Instead, dfa->bytemap_[]\n// maps from bytes to \"byte classes\" and then next_ only needs to have\n// as many pointers as there are byte classes. A byte class is simply a\n// range of bytes that the regexp never distinguishes between.\n// A regexp looking for a[abc] would have four byte ranges -- 0 to 'a'-1,\n// 'a', 'b' to 'c', and 'c' to 0xFF. The bytemap slows us a little bit\n// but in exchange we typically cut the size of a State (and thus our\n// memory footprint) by about 5-10x. The comments still refer to\n// s->next[c] for simplicity, but code should refer to s->next_[bytemap_[c]].\n//\n// Third, it is common for a DFA for an unanchored match to begin in a\n// state in which only one particular byte value can take the DFA to a\n// different state. That is, s->next[c] != s for only one c. In this\n// situation, the DFA can do better than executing the simple loop.\n// Instead, it can call memchr to search very quickly for the byte c.\n// Whether the start state has this property is determined during a\n// pre-compilation pass and the \"can_prefix_accel\" argument is set.\n//\n// Fourth, the desired behavior is to search for the leftmost-best match\n// (approximately, the same one that Perl would find), which is not\n// necessarily the match ending earliest in the string. Each time a\n// match is found, it must be noted, but the DFA must continue on in\n// hope of finding a higher-priority match. In some cases, the caller only\n// cares whether there is any match at all, not which one is found.\n// The \"want_earliest_match\" flag causes the search to stop at the first\n// match found.\n//\n// Fifth, one algorithm that uses the DFA needs it to run over the\n// input string backward, beginning at the end and ending at the beginning.\n// Passing false for the \"run_forward\" flag causes the DFA to run backward.\n//\n// The checks for these last three cases, which in a naive implementation\n// would be performed once per input byte, slow the general loop enough\n// to merit specialized versions of the search loop for each of the\n// eight possible settings of the three booleans. Rather than write\n// eight different functions, we write one general implementation and then\n// inline it to create the specialized ones.\n//\n// Note that matches are delayed by one byte, to make it easier to\n// accomodate match conditions depending on the next input byte (like $ and \\b).\n// When s->next[c]->IsMatch(), it means that there is a match ending just\n// *before* byte c.\n\n// The generic search loop. Searches text for a match, returning\n// the pointer to the end of the chosen match, or NULL if no match.\n// The bools are equal to the same-named variables in params, but\n// making them function arguments lets the inliner specialize\n// this function to each combination (see two paragraphs above).\ntemplate \ninline bool DFA::InlinedSearchLoop(SearchParams* params) {\n State* start = params->start;\n const uint8_t* bp = BytePtr(params->text.data()); // start of text\n const uint8_t* p = bp; // text scanning point\n const uint8_t* ep = BytePtr(params->text.data() +\n params->text.size()); // end of text\n const uint8_t* resetp = NULL; // p at last cache reset\n if (!run_forward) {\n using std::swap;\n swap(p, ep);\n }\n\n const uint8_t* bytemap = prog_->bytemap();\n const uint8_t* lastmatch = NULL; // most recent matching position in text\n bool matched = false;\n\n State* s = start;\n if (ExtraDebug)\n absl::FPrintF(stderr, \"@stx: %s\\n\", DumpState(s));\n\n if (s->IsMatch()) {\n matched = true;\n lastmatch = p;\n if (ExtraDebug)\n absl::FPrintF(stderr, \"match @stx! [%s]\\n\", DumpState(s));\n if (params->matches != NULL) {\n for (int i = s->ninst_ - 1; i >= 0; i--) {\n int id = s->inst_[i];\n if (id == MatchSep)\n break;\n params->matches->insert(id);\n }\n }\n if (want_earliest_match) {\n params->ep = reinterpret_cast(lastmatch);\n return true;\n }\n }\n\n while (p != ep) {\n if (ExtraDebug)\n absl::FPrintF(stderr, \"@%d: %s\\n\", p - bp, DumpState(s));\n\n if (can_prefix_accel && s == start) {\n // In start state, only way out is to find the prefix,\n // so we use prefix accel (e.g. memchr) to skip ahead.\n // If not found, we can skip to the end of the string.\n p = BytePtr(prog_->PrefixAccel(p, ep - p));\n if (p == NULL) {\n p = ep;\n break;\n }\n }\n\n int c;\n if (run_forward)\n c = *p++;\n else\n c = *--p;\n\n // Note that multiple threads might be consulting\n // s->next_[bytemap[c]] simultaneously.\n // RunStateOnByte takes care of the appropriate locking,\n // including a memory barrier so that the unlocked access\n // (sometimes known as \"double-checked locking\") is safe.\n // The alternative would be either one DFA per thread\n // or one mutex operation per input byte.\n //\n // ns == DeadState means the state is known to be dead\n // (no more matches are possible).\n // ns == NULL means the state has not yet been computed\n // (need to call RunStateOnByteUnlocked).\n // RunStateOnByte returns ns == NULL if it is out of memory.\n // ns == FullMatchState means the rest of the string matches.\n //\n // Okay to use bytemap[] not ByteMap() here, because\n // c is known to be an actual byte and not kByteEndText.\n\n State* ns = s->next_[bytemap[c]].load(std::memory_order_acquire);\n if (ns == NULL) {\n ns = RunStateOnByteUnlocked(s, c);\n if (ns == NULL) {\n // After we reset the cache, we hold cache_mutex exclusively,\n // so if resetp != NULL, it means we filled the DFA state\n // cache with this search alone (without any other threads).\n // Benchmarks show that doing a state computation on every\n // byte runs at about 0.2 MB/s, while the NFA (nfa.cc) can do the\n // same at about 2 MB/s. Unless we're processing an average\n // of 10 bytes per state computation, fail so that RE2 can\n // fall back to the NFA. However, RE2::Set cannot fall back,\n // so we just have to keep on keeping on in that case.\n if (dfa_should_bail_when_slow && resetp != NULL &&\n static_cast(p - resetp) < 10*state_cache_.size() &&\n kind_ != Prog::kManyMatch) {\n params->failed = true;\n return false;\n }\n resetp = p;\n\n // Prepare to save start and s across the reset.\n StateSaver save_start(this, start);\n StateSaver save_s(this, s);\n\n // Discard all the States in the cache.\n ResetCache(params->cache_lock);\n\n // Restore start and s so we can continue.\n if ((start = save_start.Restore()) == NULL ||\n (s = save_s.Restore()) == NULL) {\n // Restore already did ABSL_LOG(DFATAL).\n params->failed = true;\n return false;\n }\n ns = RunStateOnByteUnlocked(s, c);\n if (ns == NULL) {\n ABSL_LOG(DFATAL) << \"RunStateOnByteUnlocked failed after ResetCache\";\n params->failed = true;\n return false;\n }\n }\n }\n if (ns <= SpecialStateMax) {\n if (ns == DeadState) {\n params->ep = reinterpret_cast(lastmatch);\n return matched;\n }\n // FullMatchState\n params->ep = reinterpret_cast(ep);\n return true;\n }\n\n s = ns;\n if (s->IsMatch()) {\n matched = true;\n // The DFA notices the match one byte late,\n // so adjust p before using it in the match.\n if (run_forward)\n lastmatch = p - 1;\n else\n lastmatch = p + 1;\n if (ExtraDebug)\n absl::FPrintF(stderr, \"match @%d! [%s]\\n\", lastmatch - bp, DumpState(s));\n if (params->matches != NULL) {\n for (int i = s->ninst_ - 1; i >= 0; i--) {\n int id = s->inst_[i];\n if (id == MatchSep)\n break;\n params->matches->insert(id);\n }\n }\n if (want_earliest_match) {\n params->ep = reinterpret_cast(lastmatch);\n return true;\n }\n }\n }\n\n // Process one more byte to see if it triggers a match.\n // (Remember, matches are delayed one byte.)\n if (ExtraDebug)\n absl::FPrintF(stderr, \"@etx: %s\\n\", DumpState(s));\n\n int lastbyte;\n if (run_forward) {\n if (EndPtr(params->text) == EndPtr(params->context))\n lastbyte = kByteEndText;\n else\n lastbyte = EndPtr(params->text)[0] & 0xFF;\n } else {\n if (BeginPtr(params->text) == BeginPtr(params->context))\n lastbyte = kByteEndText;\n else\n lastbyte = BeginPtr(params->text)[-1] & 0xFF;\n }\n\n State* ns = s->next_[ByteMap(lastbyte)].load(std::memory_order_acquire);\n if (ns == NULL) {\n ns = RunStateOnByteUnlocked(s, lastbyte);\n if (ns == NULL) {\n StateSaver save_s(this, s);\n ResetCache(params->cache_lock);\n if ((s = save_s.Restore()) == NULL) {\n params->failed = true;\n return false;\n }\n ns = RunStateOnByteUnlocked(s, lastbyte);\n if (ns == NULL) {\n ABSL_LOG(DFATAL) << \"RunStateOnByteUnlocked failed after Reset\";\n params->failed = true;\n return false;\n }\n }\n }\n if (ns <= SpecialStateMax) {\n if (ns == DeadState) {\n params->ep = reinterpret_cast(lastmatch);\n return matched;\n }\n // FullMatchState\n params->ep = reinterpret_cast(ep);\n return true;\n }\n\n s = ns;\n if (s->IsMatch()) {\n matched = true;\n lastmatch = p;\n if (ExtraDebug)\n absl::FPrintF(stderr, \"match @etx! [%s]\\n\", DumpState(s));\n if (params->matches != NULL) {\n for (int i = s->ninst_ - 1; i >= 0; i--) {\n int id = s->inst_[i];\n if (id == MatchSep)\n break;\n params->matches->insert(id);\n }\n }\n }\n\n params->ep = reinterpret_cast(lastmatch);\n return matched;\n}\n\n// Inline specializations of the general loop.\nbool DFA::SearchFFF(SearchParams* params) {\n return InlinedSearchLoop(params);\n}\nbool DFA::SearchFFT(SearchParams* params) {\n return InlinedSearchLoop(params);\n}\nbool DFA::SearchFTF(SearchParams* params) {\n return InlinedSearchLoop(params);\n}\nbool DFA::SearchFTT(SearchParams* params) {\n return InlinedSearchLoop(params);\n}\nbool DFA::SearchTFF(SearchParams* params) {\n return InlinedSearchLoop(params);\n}\nbool DFA::SearchTFT(SearchParams* params) {\n return InlinedSearchLoop(params);\n}\nbool DFA::SearchTTF(SearchParams* params) {\n return InlinedSearchLoop(params);\n}\nbool DFA::SearchTTT(SearchParams* params) {\n return InlinedSearchLoop(params);\n}\n\n// For performance, calls the appropriate specialized version\n// of InlinedSearchLoop.\nbool DFA::FastSearchLoop(SearchParams* params) {\n // Because the methods are private, the Searches array\n // cannot be declared at top level.\n static bool (DFA::*Searches[])(SearchParams*) = {\n &DFA::SearchFFF,\n &DFA::SearchFFT,\n &DFA::SearchFTF,\n &DFA::SearchFTT,\n &DFA::SearchTFF,\n &DFA::SearchTFT,\n &DFA::SearchTTF,\n &DFA::SearchTTT,\n };\n\n int index = 4 * params->can_prefix_accel +\n 2 * params->want_earliest_match +\n 1 * params->run_forward;\n return (this->*Searches[index])(params);\n}\n\n\n// The discussion of DFA execution above ignored the question of how\n// to determine the initial state for the search loop. There are two\n// factors that influence the choice of start state.\n//\n// The first factor is whether the search is anchored or not.\n// The regexp program (Prog*) itself has\n// two different entry points: one for anchored searches and one for\n// unanchored searches. (The unanchored version starts with a leading \".*?\"\n// and then jumps to the anchored one.)\n//\n// The second factor is where text appears in the larger context, which\n// determines which empty-string operators can be matched at the beginning\n// of execution. If text is at the very beginning of context, \\A and ^ match.\n// Otherwise if text is at the beginning of a line, then ^ matches.\n// Otherwise it matters whether the character before text is a word character\n// or a non-word character.\n//\n// The two cases (unanchored vs not) and four cases (empty-string flags)\n// combine to make the eight cases recorded in the DFA's begin_text_[2],\n// begin_line_[2], after_wordchar_[2], and after_nonwordchar_[2] cached\n// StartInfos. The start state for each is filled in the first time it\n// is used for an actual search.\n\n// Examines text, context, and anchored to determine the right start\n// state for the DFA search loop. Fills in params and returns true on success.\n// Returns false on failure.\nbool DFA::AnalyzeSearch(SearchParams* params) {\n absl::string_view text = params->text;\n absl::string_view context = params->context;\n\n // Sanity check: make sure that text lies within context.\n if (BeginPtr(text) < BeginPtr(context) || EndPtr(text) > EndPtr(context)) {\n ABSL_LOG(DFATAL) << \"context does not contain text\";\n params->start = DeadState;\n return true;\n }\n\n // Determine correct search type.\n int start;\n uint32_t flags;\n if (params->run_forward) {\n if (BeginPtr(text) == BeginPtr(context)) {\n start = kStartBeginText;\n flags = kEmptyBeginText|kEmptyBeginLine;\n } else if (BeginPtr(text)[-1] == '\\n') {\n start = kStartBeginLine;\n flags = kEmptyBeginLine;\n } else if (Prog::IsWordChar(BeginPtr(text)[-1] & 0xFF)) {\n start = kStartAfterWordChar;\n flags = kFlagLastWord;\n } else {\n start = kStartAfterNonWordChar;\n flags = 0;\n }\n } else {\n if (EndPtr(text) == EndPtr(context)) {\n start = kStartBeginText;\n flags = kEmptyBeginText|kEmptyBeginLine;\n } else if (EndPtr(text)[0] == '\\n') {\n start = kStartBeginLine;\n flags = kEmptyBeginLine;\n } else if (Prog::IsWordChar(EndPtr(text)[0] & 0xFF)) {\n start = kStartAfterWordChar;\n flags = kFlagLastWord;\n } else {\n start = kStartAfterNonWordChar;\n flags = 0;\n }\n }\n if (params->anchored)\n start |= kStartAnchored;\n StartInfo* info = &start_[start];\n\n // Try once without cache_lock for writing.\n // Try again after resetting the cache\n // (ResetCache will relock cache_lock for writing).\n if (!AnalyzeSearchHelper(params, info, flags)) {\n ResetCache(params->cache_lock);\n if (!AnalyzeSearchHelper(params, info, flags)) {\n params->failed = true;\n ABSL_LOG(DFATAL) << \"Failed to analyze start state.\";\n return false;\n }\n }\n\n params->start = info->start.load(std::memory_order_acquire);\n\n // Even if we could prefix accel, we cannot do so when anchored and,\n // less obviously, we cannot do so when we are going to need flags.\n // This trick works only when there is a single byte that leads to a\n // different state!\n if (prog_->can_prefix_accel() &&\n !params->anchored &&\n params->start > SpecialStateMax &&\n params->start->flag_ >> kFlagNeedShift == 0)\n params->can_prefix_accel = true;\n\n if (ExtraDebug)\n absl::FPrintF(stderr, \"anchored=%d fwd=%d flags=%#x state=%s can_prefix_accel=%d\\n\",\n params->anchored, params->run_forward, flags,\n DumpState(params->start), params->can_prefix_accel);\n\n return true;\n}\n\n// Fills in info if needed. Returns true on success, false on failure.\nbool DFA::AnalyzeSearchHelper(SearchParams* params, StartInfo* info,\n uint32_t flags) {\n // Quick check.\n State* start = info->start.load(std::memory_order_acquire);\n if (start != NULL)\n return true;\n\n absl::MutexLock l(mutex_);\n start = info->start.load(std::memory_order_relaxed);\n if (start != NULL)\n return true;\n\n q0_->clear();\n AddToQueue(q0_,\n params->anchored ? prog_->start() : prog_->start_unanchored(),\n flags);\n start = WorkqToCachedState(q0_, NULL, flags);\n if (start == NULL)\n return false;\n\n // Synchronize with \"quick check\" above.\n info->start.store(start, std::memory_order_release);\n return true;\n}\n\n// The actual DFA search: calls AnalyzeSearch and then FastSearchLoop.\nbool DFA::Search(absl::string_view text, absl::string_view context,\n bool anchored, bool want_earliest_match, bool run_forward,\n bool* failed, const char** epp, SparseSet* matches) {\n *epp = NULL;\n if (!ok()) {\n *failed = true;\n return false;\n }\n *failed = false;\n\n if (ExtraDebug) {\n absl::FPrintF(stderr, \"\\nprogram:\\n%s\\n\", prog_->DumpUnanchored());\n absl::FPrintF(stderr, \"text %s anchored=%d earliest=%d fwd=%d kind %d\\n\",\n text, anchored, want_earliest_match, run_forward, kind_);\n }\n\n RWLocker l(&cache_mutex_);\n SearchParams params(text, context, &l);\n params.anchored = anchored;\n params.want_earliest_match = want_earliest_match;\n params.run_forward = run_forward;\n // matches should be null except when using RE2::Set.\n ABSL_DCHECK(matches == NULL || kind_ == Prog::kManyMatch);\n params.matches = matches;\n\n if (!AnalyzeSearch(¶ms)) {\n *failed = true;\n return false;\n }\n if (params.start == DeadState)\n return false;\n if (params.start == FullMatchState) {\n if (run_forward == want_earliest_match)\n *epp = text.data();\n else\n *epp = text.data() + text.size();\n return true;\n }\n if (ExtraDebug)\n absl::FPrintF(stderr, \"start %s\\n\", DumpState(params.start));\n bool ret = FastSearchLoop(¶ms);\n if (params.failed) {\n *failed = true;\n return false;\n }\n *epp = params.ep;\n return ret;\n}\n\nDFA* Prog::GetDFA(MatchKind kind) {\n // For a forward DFA, half the memory goes to each DFA.\n // However, if it is a \"many match\" DFA, then there is\n // no counterpart with which the memory must be shared.\n //\n // For a reverse DFA, all the memory goes to the\n // \"longest match\" DFA, because RE2 never does reverse\n // \"first match\" searches.\n if (kind == kFirstMatch) {\n absl::call_once(dfa_first_once_, [](Prog* prog) {\n prog->dfa_first_ = new DFA(prog, kFirstMatch, prog->dfa_mem_ / 2);\n }, this);\n return dfa_first_;\n } else if (kind == kManyMatch) {\n absl::call_once(dfa_first_once_, [](Prog* prog) {\n prog->dfa_first_ = new DFA(prog, kManyMatch, prog->dfa_mem_);\n }, this);\n return dfa_first_;\n } else {\n absl::call_once(dfa_longest_once_, [](Prog* prog) {\n if (!prog->reversed_)\n prog->dfa_longest_ = new DFA(prog, kLongestMatch, prog->dfa_mem_ / 2);\n else\n prog->dfa_longest_ = new DFA(prog, kLongestMatch, prog->dfa_mem_);\n }, this);\n return dfa_longest_;\n }\n}\n\nvoid Prog::DeleteDFA(DFA* dfa) {\n delete dfa;\n}\n\n// Executes the regexp program to search in text,\n// which itself is inside the larger context. (As a convenience,\n// passing a NULL context is equivalent to passing text.)\n// Returns true if a match is found, false if not.\n// If a match is found, fills in match0->end() to point at the end of the match\n// and sets match0->begin() to text.begin(), since the DFA can't track\n// where the match actually began.\n//\n// This is the only external interface (class DFA only exists in this file).\n//\nbool Prog::SearchDFA(absl::string_view text, absl::string_view context,\n Anchor anchor, MatchKind kind, absl::string_view* match0,\n bool* failed, SparseSet* matches) {\n *failed = false;\n\n if (context.data() == NULL)\n context = text;\n bool caret = anchor_start();\n bool dollar = anchor_end();\n if (reversed_) {\n using std::swap;\n swap(caret, dollar);\n }\n if (caret && BeginPtr(context) != BeginPtr(text))\n return false;\n if (dollar && EndPtr(context) != EndPtr(text))\n return false;\n\n // Handle full match by running an anchored longest match\n // and then checking if it covers all of text.\n bool anchored = anchor == kAnchored || anchor_start() || kind == kFullMatch;\n bool endmatch = false;\n if (kind == kManyMatch) {\n // This is split out in order to avoid clobbering kind.\n } else if (kind == kFullMatch || anchor_end()) {\n endmatch = true;\n kind = kLongestMatch;\n }\n\n // If the caller doesn't care where the match is (just whether one exists),\n // then we can stop at the very first match we find, the so-called\n // \"earliest match\".\n bool want_earliest_match = false;\n if (kind == kManyMatch) {\n // This is split out in order to avoid clobbering kind.\n if (matches == NULL) {\n want_earliest_match = true;\n }\n } else if (match0 == NULL && !endmatch) {\n want_earliest_match = true;\n kind = kLongestMatch;\n }\n\n DFA* dfa = GetDFA(kind);\n const char* ep;\n bool matched = dfa->Search(text, context, anchored,\n want_earliest_match, !reversed_,\n failed, &ep, matches);\n if (*failed) {\n hooks::GetDFASearchFailureHook()({\n // Nothing yet...\n });\n return false;\n }\n if (!matched)\n return false;\n if (endmatch && ep != (reversed_ ? text.data() : text.data() + text.size()))\n return false;\n\n // If caller cares, record the boundary of the match.\n // We only know where it ends, so use the boundary of text\n // as the beginning.\n if (match0) {\n if (reversed_)\n *match0 =\n absl::string_view(ep, static_cast(text.data() + text.size() - ep));\n else\n *match0 =\n absl::string_view(text.data(), static_cast(ep - text.data()));\n }\n return true;\n}\n\n// Build out all states in DFA. Returns number of states.\nint DFA::BuildAllStates(const Prog::DFAStateCallback& cb) {\n if (!ok())\n return 0;\n\n // Pick out start state for unanchored search\n // at beginning of text.\n RWLocker l(&cache_mutex_);\n SearchParams params(absl::string_view(), absl::string_view(), &l);\n params.anchored = false;\n if (!AnalyzeSearch(¶ms) ||\n params.start == NULL ||\n params.start == DeadState)\n return 0;\n\n // Add start state to work queue.\n // Note that any State* that we handle here must point into the cache,\n // so we can simply depend on pointer-as-a-number hashing and equality.\n absl::flat_hash_map m;\n std::deque q;\n m.emplace(params.start, static_cast(m.size()));\n q.push_back(params.start);\n\n // Compute the input bytes needed to cover all of the next pointers.\n int nnext = prog_->bytemap_range() + 1; // + 1 for kByteEndText slot\n std::vector input(nnext);\n for (int c = 0; c < 256; c++) {\n int b = prog_->bytemap()[c];\n while (c < 256-1 && prog_->bytemap()[c+1] == b)\n c++;\n input[b] = c;\n }\n input[prog_->bytemap_range()] = kByteEndText;\n\n // Scratch space for the output.\n std::vector output(nnext);\n\n // Flood to expand every state.\n bool oom = false;\n while (!q.empty()) {\n State* s = q.front();\n q.pop_front();\n for (int c : input) {\n State* ns = RunStateOnByteUnlocked(s, c);\n if (ns == NULL) {\n oom = true;\n break;\n }\n if (ns == DeadState) {\n output[ByteMap(c)] = -1;\n continue;\n }\n if (m.find(ns) == m.end()) {\n m.emplace(ns, static_cast(m.size()));\n q.push_back(ns);\n }\n output[ByteMap(c)] = m[ns];\n }\n if (cb)\n cb(oom ? NULL : output.data(),\n s == FullMatchState || s->IsMatch());\n if (oom)\n break;\n }\n\n return static_cast(m.size());\n}\n\n// Build out all states in DFA for kind. Returns number of states.\nint Prog::BuildEntireDFA(MatchKind kind, const DFAStateCallback& cb) {\n return GetDFA(kind)->BuildAllStates(cb);\n}\n\n// Computes min and max for matching string.\n// Won't return strings bigger than maxlen.\nbool DFA::PossibleMatchRange(std::string* min, std::string* max, int maxlen) {\n if (!ok())\n return false;\n\n // NOTE: if future users of PossibleMatchRange want more precision when\n // presented with infinitely repeated elements, consider making this a\n // parameter to PossibleMatchRange.\n static int kMaxEltRepetitions = 0;\n\n // Keep track of the number of times we've visited states previously. We only\n // revisit a given state if it's part of a repeated group, so if the value\n // portion of the map tuple exceeds kMaxEltRepetitions we bail out and set\n // |*max| to |PrefixSuccessor(*max)|.\n //\n // Also note that previously_visited_states[UnseenStatePtr] will, in the STL\n // tradition, implicitly insert a '0' value at first use. We take advantage\n // of that property below.\n absl::flat_hash_map previously_visited_states;\n\n // Pick out start state for anchored search at beginning of text.\n RWLocker l(&cache_mutex_);\n SearchParams params(absl::string_view(), absl::string_view(), &l);\n params.anchored = true;\n if (!AnalyzeSearch(¶ms))\n return false;\n if (params.start == DeadState) { // No matching strings\n *min = \"\";\n *max = \"\";\n return true;\n }\n if (params.start == FullMatchState) // Every string matches: no max\n return false;\n\n // The DFA is essentially a big graph rooted at params.start,\n // and paths in the graph correspond to accepted strings.\n // Each node in the graph has potentially 256+1 arrows\n // coming out, one for each byte plus the magic end of\n // text character kByteEndText.\n\n // To find the smallest possible prefix of an accepted\n // string, we just walk the graph preferring to follow\n // arrows with the lowest bytes possible. To find the\n // largest possible prefix, we follow the largest bytes\n // possible.\n\n // The test for whether there is an arrow from s on byte j is\n // ns = RunStateOnByteUnlocked(s, j);\n // if (ns == NULL)\n // return false;\n // if (ns != DeadState && ns->ninst > 0)\n // The RunStateOnByteUnlocked call asks the DFA to build out the graph.\n // It returns NULL only if the DFA has run out of memory,\n // in which case we can't be sure of anything.\n // The second check sees whether there was graph built\n // and whether it is interesting graph. Nodes might have\n // ns->ninst == 0 if they exist only to represent the fact\n // that a match was found on the previous byte.\n\n // Build minimum prefix.\n State* s = params.start;\n min->clear();\n absl::MutexLock lock(mutex_);\n for (int i = 0; i < maxlen; i++) {\n if (previously_visited_states[s] > kMaxEltRepetitions)\n break;\n previously_visited_states[s]++;\n\n // Stop if min is a match.\n State* ns = RunStateOnByte(s, kByteEndText);\n if (ns == NULL) // DFA out of memory\n return false;\n if (ns != DeadState && (ns == FullMatchState || ns->IsMatch()))\n break;\n\n // Try to extend the string with low bytes.\n bool extended = false;\n for (int j = 0; j < 256; j++) {\n ns = RunStateOnByte(s, j);\n if (ns == NULL) // DFA out of memory\n return false;\n if (ns == FullMatchState ||\n (ns > SpecialStateMax && ns->ninst_ > 0)) {\n extended = true;\n min->append(1, static_cast(j));\n s = ns;\n break;\n }\n }\n if (!extended)\n break;\n }\n\n // Build maximum prefix.\n previously_visited_states.clear();\n s = params.start;\n max->clear();\n for (int i = 0; i < maxlen; i++) {\n if (previously_visited_states[s] > kMaxEltRepetitions)\n break;\n previously_visited_states[s] += 1;\n\n // Try to extend the string with high bytes.\n bool extended = false;\n for (int j = 255; j >= 0; j--) {\n State* ns = RunStateOnByte(s, j);\n if (ns == NULL)\n return false;\n if (ns == FullMatchState ||\n (ns > SpecialStateMax && ns->ninst_ > 0)) {\n extended = true;\n max->append(1, static_cast(j));\n s = ns;\n break;\n }\n }\n if (!extended) {\n // Done, no need for PrefixSuccessor.\n return true;\n }\n }\n\n // Stopped while still adding to *max - round aaaaaaaaaa... to aaaa...b\n PrefixSuccessor(max);\n\n // If there are no bytes left, we have no way to say \"there is no maximum\n // string\". We could make the interface more complicated and be able to\n // return \"there is no maximum but here is a minimum\", but that seems like\n // overkill -- the most common no-max case is all possible strings, so not\n // telling the caller that the empty string is the minimum match isn't a\n // great loss.\n if (max->empty())\n return false;\n\n return true;\n}\n\n// PossibleMatchRange for a Prog.\nbool Prog::PossibleMatchRange(std::string* min, std::string* max, int maxlen) {\n // Have to use dfa_longest_ to get all strings for full matches.\n // For example, (a|aa) never matches aa in first-match mode.\n return GetDFA(kLongestMatch)->PossibleMatchRange(min, max, maxlen);\n}\n\n} // namespace re2\n" }, { "path": "re2/filtered_re2.cc", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \"re2/filtered_re2.h\"\n\n#include \n\n#include \n#include \n#include \n\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/prefilter.h\"\n#include \"re2/prefilter_tree.h\"\n\nnamespace re2 {\n\nFilteredRE2::FilteredRE2()\n : compiled_(false),\n prefilter_tree_(new PrefilterTree()) {\n}\n\nFilteredRE2::FilteredRE2(int min_atom_len)\n : compiled_(false),\n prefilter_tree_(new PrefilterTree(min_atom_len)) {\n}\n\nFilteredRE2::~FilteredRE2() {\n for (size_t i = 0; i < re2_vec_.size(); i++)\n delete re2_vec_[i];\n}\n\nFilteredRE2::FilteredRE2(FilteredRE2&& other)\n : re2_vec_(std::move(other.re2_vec_)),\n compiled_(other.compiled_),\n prefilter_tree_(std::move(other.prefilter_tree_)) {\n other.re2_vec_.clear();\n other.re2_vec_.shrink_to_fit();\n other.compiled_ = false;\n other.prefilter_tree_.reset(new PrefilterTree());\n}\n\nFilteredRE2& FilteredRE2::operator=(FilteredRE2&& other) {\n this->~FilteredRE2();\n (void) new (this) FilteredRE2(std::move(other));\n return *this;\n}\n\nRE2::ErrorCode FilteredRE2::Add(absl::string_view pattern,\n const RE2::Options& options, int* id) {\n RE2* re = new RE2(pattern, options);\n RE2::ErrorCode code = re->error_code();\n\n if (!re->ok()) {\n if (options.log_errors()) {\n ABSL_LOG(ERROR) << \"Couldn't compile regular expression, skipping: \"\n << pattern << \" due to error \" << re->error();\n }\n delete re;\n } else {\n *id = static_cast(re2_vec_.size());\n re2_vec_.push_back(re);\n }\n\n return code;\n}\n\nvoid FilteredRE2::Compile(std::vector* atoms) {\n if (compiled_) {\n ABSL_LOG(ERROR) << \"Compile called already.\";\n return;\n }\n\n // Similarly to PrefilterTree::Compile(), make compiling\n // a no-op if it's attempted before adding any patterns.\n if (re2_vec_.empty()) {\n return;\n }\n\n for (size_t i = 0; i < re2_vec_.size(); i++) {\n Prefilter* prefilter = Prefilter::FromRE2(re2_vec_[i]);\n prefilter_tree_->Add(prefilter);\n }\n atoms->clear();\n prefilter_tree_->Compile(atoms);\n compiled_ = true;\n}\n\nint FilteredRE2::SlowFirstMatch(absl::string_view text) const {\n for (size_t i = 0; i < re2_vec_.size(); i++)\n if (RE2::PartialMatch(text, *re2_vec_[i]))\n return static_cast(i);\n return -1;\n}\n\nint FilteredRE2::FirstMatch(absl::string_view text,\n const std::vector& atoms) const {\n if (!compiled_) {\n ABSL_LOG(DFATAL) << \"FirstMatch called before Compile.\";\n return -1;\n }\n std::vector regexps;\n prefilter_tree_->RegexpsGivenStrings(atoms, ®exps);\n for (size_t i = 0; i < regexps.size(); i++)\n if (RE2::PartialMatch(text, *re2_vec_[regexps[i]]))\n return regexps[i];\n return -1;\n}\n\nbool FilteredRE2::AllMatches(absl::string_view text,\n const std::vector& atoms,\n std::vector* matching_regexps) const {\n matching_regexps->clear();\n std::vector regexps;\n prefilter_tree_->RegexpsGivenStrings(atoms, ®exps);\n for (size_t i = 0; i < regexps.size(); i++)\n if (RE2::PartialMatch(text, *re2_vec_[regexps[i]]))\n matching_regexps->push_back(regexps[i]);\n return !matching_regexps->empty();\n}\n\nvoid FilteredRE2::AllPotentials(const std::vector& atoms,\n std::vector* potential_regexps) const {\n prefilter_tree_->RegexpsGivenStrings(atoms, potential_regexps);\n}\n\nvoid FilteredRE2::RegexpsGivenStrings(const std::vector& matched_atoms,\n std::vector* passed_regexps) {\n prefilter_tree_->RegexpsGivenStrings(matched_atoms, passed_regexps);\n}\n\nvoid FilteredRE2::PrintPrefilter(int regexpid) {\n prefilter_tree_->PrintPrefilter(regexpid);\n}\n\n} // namespace re2\n" }, { "path": "re2/filtered_re2.h", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_FILTERED_RE2_H_\n#define RE2_FILTERED_RE2_H_\n\n// The class FilteredRE2 is used as a wrapper to multiple RE2 regexps.\n// It provides a prefilter mechanism that helps in cutting down the\n// number of regexps that need to be actually searched.\n//\n// By design, it does not include a string matching engine. This is to\n// allow the user of the class to use their favorite string matching\n// engine. The overall flow is: Add all the regexps using Add, then\n// Compile the FilteredRE2. Compile returns strings that need to be\n// matched. Note that the returned strings are lowercased and distinct.\n// For applying regexps to a search text, the caller does the string\n// matching using the returned strings. When doing the string match,\n// note that the caller has to do that in a case-insensitive way or\n// on a lowercased version of the search text. Then call FirstMatch\n// or AllMatches with a vector of indices of strings that were found\n// in the text to get the actual regexp matches.\n\n#include \n#include \n#include \n\n#include \"absl/strings/string_view.h\"\n#include \"re2/re2.h\"\n\nnamespace re2 {\n\nclass PrefilterTree;\n\nclass FilteredRE2 {\n public:\n FilteredRE2();\n explicit FilteredRE2(int min_atom_len);\n ~FilteredRE2();\n\n // Not copyable.\n FilteredRE2(const FilteredRE2&) = delete;\n FilteredRE2& operator=(const FilteredRE2&) = delete;\n // Movable.\n FilteredRE2(FilteredRE2&& other);\n FilteredRE2& operator=(FilteredRE2&& other);\n\n // Uses RE2 constructor to create a RE2 object (re). Returns\n // re->error_code(). If error_code is other than NoError, then re is\n // deleted and not added to re2_vec_.\n RE2::ErrorCode Add(absl::string_view pattern,\n const RE2::Options& options,\n int* id);\n\n // Prepares the regexps added by Add for filtering. Returns a set\n // of strings that the caller should check for in candidate texts.\n // The returned strings are lowercased and distinct. When doing\n // string matching, it should be performed in a case-insensitive\n // way or the search text should be lowercased first. Call after\n // all Add calls are done.\n void Compile(std::vector* strings_to_match);\n\n // Returns the index of the first matching regexp.\n // Returns -1 on no match. Can be called prior to Compile.\n // Does not do any filtering: simply tries to Match the\n // regexps in a loop.\n int SlowFirstMatch(absl::string_view text) const;\n\n // Returns the index of the first matching regexp.\n // Returns -1 on no match. Compile has to be called before\n // calling this.\n int FirstMatch(absl::string_view text,\n const std::vector& atoms) const;\n\n // Returns the indices of all matching regexps, after first clearing\n // matched_regexps.\n bool AllMatches(absl::string_view text,\n const std::vector& atoms,\n std::vector* matching_regexps) const;\n\n // Returns the indices of all potentially matching regexps after first\n // clearing potential_regexps.\n // A regexp is potentially matching if it passes the filter.\n // If a regexp passes the filter it may still not match.\n // A regexp that does not pass the filter is guaranteed to not match.\n void AllPotentials(const std::vector& atoms,\n std::vector* potential_regexps) const;\n\n // The number of regexps added.\n int NumRegexps() const { return static_cast(re2_vec_.size()); }\n\n // Get the individual RE2 objects.\n const RE2& GetRE2(int regexpid) const { return *re2_vec_[regexpid]; }\n\n private:\n // Print prefilter.\n void PrintPrefilter(int regexpid);\n\n // Useful for testing and debugging.\n void RegexpsGivenStrings(const std::vector& matched_atoms,\n std::vector* passed_regexps);\n\n // All the regexps in the FilteredRE2.\n std::vector re2_vec_;\n\n // Has the FilteredRE2 been compiled using Compile()\n bool compiled_;\n\n // An AND-OR tree of string atoms used for filtering regexps.\n std::unique_ptr prefilter_tree_;\n};\n\n} // namespace re2\n\n#endif // RE2_FILTERED_RE2_H_\n" }, { "path": "re2/fuzzing/re2_fuzzer.cc", "content": "// Copyright 2016 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \"absl/strings/string_view.h\"\n#include \"re2/filtered_re2.h\"\n#include \"re2/re2.h\"\n#include \"re2/regexp.h\"\n#include \"re2/set.h\"\n#include \"re2/walker-inl.h\"\n\n// NOT static, NOT signed.\nuint8_t dummy = 0;\n\n// Walks kRegexpConcat and kRegexpAlternate subexpressions\n// to determine their maximum length.\nclass SubexpressionWalker : public re2::Regexp::Walker {\n public:\n SubexpressionWalker() = default;\n ~SubexpressionWalker() override = default;\n\n int PostVisit(re2::Regexp* re, int parent_arg, int pre_arg,\n int* child_args, int nchild_args) override {\n switch (re->op()) {\n case re2::kRegexpConcat:\n case re2::kRegexpAlternate: {\n int max = nchild_args;\n for (int i = 0; i < nchild_args; i++)\n max = std::max(max, child_args[i]);\n return max;\n }\n\n default:\n break;\n }\n return -1;\n }\n\n // Should never be called: we use Walk(), not WalkExponential().\n int ShortVisit(re2::Regexp* re, int parent_arg) override {\n return parent_arg;\n }\n\n private:\n SubexpressionWalker(const SubexpressionWalker&) = delete;\n SubexpressionWalker& operator=(const SubexpressionWalker&) = delete;\n};\n\n// Walks substrings (i.e. kRegexpLiteralString subexpressions)\n// to determine their maximum length... in runes, but avoiding\n// overheads due to UTF-8 encoding is worthwhile when fuzzing.\nclass SubstringWalker : public re2::Regexp::Walker {\n public:\n SubstringWalker() = default;\n ~SubstringWalker() override = default;\n\n int PostVisit(re2::Regexp* re, int parent_arg, int pre_arg,\n int* child_args, int nchild_args) override {\n switch (re->op()) {\n case re2::kRegexpConcat:\n case re2::kRegexpAlternate:\n case re2::kRegexpStar:\n case re2::kRegexpPlus:\n case re2::kRegexpQuest:\n case re2::kRegexpRepeat:\n case re2::kRegexpCapture: {\n int max = -1;\n for (int i = 0; i < nchild_args; i++)\n max = std::max(max, child_args[i]);\n return max;\n }\n\n case re2::kRegexpLiteralString:\n return re->nrunes();\n\n default:\n break;\n }\n return -1;\n }\n\n // Should never be called: we use Walk(), not WalkExponential().\n int ShortVisit(re2::Regexp* re, int parent_arg) override {\n return parent_arg;\n }\n\n private:\n SubstringWalker(const SubstringWalker&) = delete;\n SubstringWalker& operator=(const SubstringWalker&) = delete;\n};\n\nvoid TestOneInput(absl::string_view pattern, const RE2::Options& options,\n RE2::Anchor anchor, absl::string_view text) {\n // Crudely limit the use of ., \\p, \\P, \\d, \\D, \\s, \\S, \\w and \\W.\n // Otherwise, we will waste time on inputs that have long runs of various\n // character classes. The fuzzer has shown itself to be easily capable of\n // generating such patterns that fall within the other limits, but result\n // in timeouts nonetheless. The marginal cost is high - even more so when\n // counted repetition is involved - whereas the marginal benefit is zero.\n // Crudely limit the use of 'k', 'K', 's' and 'S' too because they become\n // three-element character classes when case-insensitive and using UTF-8.\n // TODO(junyer): Handle [[:alnum:]] et al. when they start to cause pain.\n int char_class = 0;\n int backslash_p = 0; // very expensive, so handle specially\n for (size_t i = 0; i < pattern.size(); i++) {\n if (pattern[i] == '.' ||\n pattern[i] == 'k' || pattern[i] == 'K' ||\n pattern[i] == 's' || pattern[i] == 'S')\n char_class++;\n if (pattern[i] != '\\\\')\n continue;\n i++;\n if (i >= pattern.size())\n break;\n if (pattern[i] == 'p' || pattern[i] == 'P' ||\n pattern[i] == 'd' || pattern[i] == 'D' ||\n pattern[i] == 's' || pattern[i] == 'S' ||\n pattern[i] == 'w' || pattern[i] == 'W')\n char_class++;\n if (pattern[i] == 'p' || pattern[i] == 'P')\n backslash_p++;\n }\n if (char_class > 9)\n return;\n if (backslash_p > 1)\n return;\n\n // Iterate just once when fuzzing. Otherwise, we easily get bogged down\n // and coverage is unlikely to improve despite significant expense.\n RE2::FUZZING_ONLY_set_maximum_global_replace_count(1);\n // The default is 1000. Even 100 turned out to be too generous\n // for fuzzing, empirically speaking, so let's try 10 instead.\n re2::Regexp::FUZZING_ONLY_set_maximum_repeat_count(10);\n\n RE2 re(pattern, options);\n if (!re.ok())\n return;\n\n // Don't waste time fuzzing programs with large subexpressions.\n // They can cause bug reports due to fuzzer timeouts. And they\n // aren't interesting for fuzzing purposes.\n if (SubexpressionWalker().Walk(re.Regexp(), -1) > 9)\n return;\n\n // Don't waste time fuzzing programs with large substrings.\n // They can cause bug reports due to fuzzer timeouts when they\n // are repetitions (e.g. hundreds of NUL bytes) and matching is\n // unanchored. And they aren't interesting for fuzzing purposes.\n if (SubstringWalker().Walk(re.Regexp(), -1) > 9)\n return;\n\n // Don't waste time fuzzing high-size programs.\n // They can cause bug reports due to fuzzer timeouts.\n int size = re.ProgramSize();\n if (size > 9999)\n return;\n int rsize = re.ReverseProgramSize();\n if (rsize > 9999)\n return;\n\n // Don't waste time fuzzing high-fanout programs.\n // They can cause bug reports due to fuzzer timeouts.\n std::vector histogram;\n int fanout = re.ProgramFanout(&histogram);\n if (fanout > 9)\n return;\n int rfanout = re.ReverseProgramFanout(&histogram);\n if (rfanout > 9)\n return;\n\n if (re.NumberOfCapturingGroups() == 0) {\n // Avoid early return due to too many arguments.\n absl::string_view sp = text;\n RE2::FullMatch(sp, re);\n RE2::PartialMatch(sp, re);\n RE2::Consume(&sp, re);\n sp = text; // Reset.\n RE2::FindAndConsume(&sp, re);\n } else {\n // Okay, we have at least one capturing group...\n // Try conversion for variously typed arguments.\n absl::string_view sp = text;\n short s;\n RE2::FullMatch(sp, re, &s);\n long l;\n RE2::PartialMatch(sp, re, &l);\n float f;\n RE2::Consume(&sp, re, &f);\n sp = text; // Reset.\n double d;\n RE2::FindAndConsume(&sp, re, &d);\n }\n\n std::string s = std::string(text);\n RE2::Replace(&s, re, \"\");\n s = std::string(text); // Reset.\n RE2::GlobalReplace(&s, re, \"\");\n\n std::string min, max;\n re.PossibleMatchRange(&min, &max, /*maxlen=*/9);\n\n // Exercise some other API functionality.\n dummy += re.NamedCapturingGroups().size();\n dummy += re.CapturingGroupNames().size();\n dummy += RE2::QuoteMeta(pattern).size();\n dummy += re.Regexp()->ToString().size();\n\n RE2::Set set(options, anchor);\n int index = set.Add(pattern, /*error=*/NULL); // -1 on error\n if (index != -1 && set.Compile()) {\n std::vector matches;\n set.Match(text, &matches);\n }\n\n re2::FilteredRE2 filter;\n index = -1; // not clobbered on error\n filter.Add(pattern, options, &index);\n if (index != -1) {\n std::vector atoms;\n filter.Compile(&atoms);\n // Pretend that all atoms match, which\n // triggers the AND-OR tree maximally.\n std::vector matched_atoms;\n matched_atoms.reserve(atoms.size());\n for (size_t i = 0; i < atoms.size(); ++i)\n matched_atoms.push_back(static_cast(i));\n std::vector matches;\n filter.AllMatches(text, matched_atoms, &matches);\n }\n}\n\n// Entry point for libFuzzer.\nextern \"C\" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {\n // An input larger than 4 KiB probably isn't interesting. (This limit\n // allows for fdp.ConsumeRandomLengthString()'s backslash behaviour.)\n if (size == 0 || size > 4096)\n return 0;\n\n FuzzedDataProvider fdp(data, size);\n\n // The convention here is that fdp.ConsumeBool() returning false sets\n // the default value whereas returning true sets the alternate value:\n // most options default to false and so can be set directly; encoding\n // defaults to UTF-8; case_sensitive defaults to true. We do NOT want\n // to log errors. max_mem is 64 MiB because we can afford to use more\n // RAM in exchange for (hopefully) faster fuzzing.\n RE2::Options options;\n options.set_encoding(fdp.ConsumeBool() ? RE2::Options::EncodingLatin1\n : RE2::Options::EncodingUTF8);\n options.set_posix_syntax(fdp.ConsumeBool());\n options.set_longest_match(fdp.ConsumeBool());\n options.set_log_errors(false);\n options.set_max_mem(64 << 20);\n options.set_literal(fdp.ConsumeBool());\n options.set_never_nl(fdp.ConsumeBool());\n options.set_dot_nl(fdp.ConsumeBool());\n options.set_never_capture(fdp.ConsumeBool());\n options.set_case_sensitive(!fdp.ConsumeBool());\n options.set_perl_classes(fdp.ConsumeBool());\n options.set_word_boundary(fdp.ConsumeBool());\n options.set_one_line(fdp.ConsumeBool());\n\n // ConsumeEnum() would require RE2::Anchor to specify\n // kMaxValue, so just use PickValueInArray() instead.\n RE2::Anchor anchor = fdp.PickValueInArray({\n RE2::UNANCHORED,\n RE2::ANCHOR_START,\n RE2::ANCHOR_BOTH,\n });\n\n std::string pattern = fdp.ConsumeRandomLengthString(999);\n std::string text = fdp.ConsumeRandomLengthString(999);\n\n TestOneInput(pattern, options, anchor, text);\n return 0;\n}\n" }, { "path": "re2/make_perl_groups.pl", "content": "#!/usr/bin/perl\n# Copyright 2008 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# Generate table entries giving character ranges\n# for POSIX/Perl character classes. Rather than\n# figure out what the definition is, it is easier to ask\n# Perl about each letter from 0-128 and write down\n# its answer.\n\n@posixclasses = (\n\t\"[:alnum:]\",\n\t\"[:alpha:]\",\n\t\"[:ascii:]\",\n\t\"[:blank:]\",\n\t\"[:cntrl:]\",\n\t\"[:digit:]\",\n\t\"[:graph:]\",\n\t\"[:lower:]\",\n\t\"[:print:]\",\n\t\"[:punct:]\",\n\t\"[:space:]\",\n\t\"[:upper:]\",\n\t\"[:word:]\",\n\t\"[:xdigit:]\",\n);\n\n@perlclasses = (\n\t\"\\\\d\",\n\t\"\\\\s\",\n\t\"\\\\w\",\n);\n\n%overrides = (\n\t# Prior to Perl 5.18, \\s did not match vertical tab.\n\t# RE2 preserves that original behaviour.\n\t\"\\\\s:11\" => 0,\n);\n\nsub ComputeClass($) {\n my ($cname) = @_;\n my @ranges;\n my $regexp = qr/[$cname]/;\n my $start = -1;\n for (my $i=0; $i<=129; $i++) {\n if ($i == 129) { $i = 256; }\n if ($i <= 128 && ($overrides{\"$cname:$i\"} // chr($i) =~ $regexp)) {\n if ($start < 0) {\n $start = $i;\n }\n } else {\n if ($start >= 0) {\n push @ranges, [$start, $i-1];\n }\n $start = -1;\n }\n }\n return @ranges;\n}\n\nsub PrintClass($$@) {\n my ($cnum, $cname, @ranges) = @_;\n print \"static const URange16 code${cnum}[] = { /* $cname */\\n\";\n for (my $i=0; $i<@ranges; $i++) {\n my @a = @{$ranges[$i]};\n printf \"\\t{ 0x%x, 0x%x },\\n\", $a[0], $a[1];\n }\n print \"};\\n\";\n my $n = @ranges;\n my $escname = $cname;\n $escname =~ s/\\\\/\\\\\\\\/g;\n $negname = $escname;\n if ($negname =~ /:/) {\n $negname =~ s/:/:^/;\n } else {\n $negname =~ y/a-z/A-Z/;\n }\n return \"{ \\\"$escname\\\", +1, code$cnum, $n, 0, 0 }\", \"{ \\\"$negname\\\", -1, code$cnum, $n, 0, 0 }\";\n}\n\nmy $cnum = 0;\n\nsub PrintClasses($@) {\n my ($pname, @classes) = @_;\n my @entries;\n foreach my $cname (@classes) {\n my @ranges = ComputeClass($cname);\n push @entries, PrintClass(++$cnum, $cname, @ranges);\n }\n print \"const UGroup ${pname}_groups[] = {\\n\";\n foreach my $e (@entries) {\n print \"\\t$e,\\n\";\n }\n print \"};\\n\";\n my $count = @entries;\n print \"const int num_${pname}_groups = $count;\\n\";\n}\n\nprint <perl_groups.cc\n\n#include \"re2/unicode_groups.h\"\n\nnamespace re2 {\n\nEOF\n\nPrintClasses(\"perl\", @perlclasses);\nPrintClasses(\"posix\", @posixclasses);\n\nprint <unicode_casefold.cc\n\n#include \"re2/unicode_casefold.h\"\n\nnamespace re2 {\n\n\"\"\"\n\n_trailer = \"\"\"\n\n} // namespace re2\n\n\"\"\"\n\ndef _Delta(a, b):\n \"\"\"Compute the delta for b - a. Even/odd and odd/even\n are handled specially, as described above.\"\"\"\n if a+1 == b:\n if a%2 == 0:\n return 'EvenOdd'\n else:\n return 'OddEven'\n if a == b+1:\n if a%2 == 0:\n return 'OddEven'\n else:\n return 'EvenOdd'\n return b - a\n\ndef _AddDelta(a, delta):\n \"\"\"Return a + delta, handling EvenOdd and OddEven specially.\"\"\"\n if type(delta) == int:\n return a+delta\n if delta == 'EvenOdd':\n if a%2 == 0:\n return a+1\n else:\n return a-1\n if delta == 'OddEven':\n if a%2 == 1:\n return a+1\n else:\n return a-1\n print(\"Bad Delta:\", delta, file=sys.stderr)\n raise unicode.Error(\"Bad Delta\")\n\ndef _MakeRanges(pairs):\n \"\"\"Turn a list like [(65,97), (66, 98), ..., (90,122)]\n into [(65, 90, +32)].\"\"\"\n ranges = []\n last = -100\n\n def evenodd(last, a, b, r):\n if a != last+1 or b != _AddDelta(a, r[2]):\n return False\n r[1] = a\n return True\n\n def evenoddpair(last, a, b, r):\n if a != last+2:\n return False\n delta = r[2]\n d = delta\n if type(delta) is not str:\n return False\n if delta.endswith('Skip'):\n d = delta[:-4]\n else:\n delta = d + 'Skip'\n if b != _AddDelta(a, d):\n return False\n r[1] = a\n r[2] = delta\n return True\n\n for a, b in pairs:\n if ranges and evenodd(last, a, b, ranges[-1]):\n pass\n elif ranges and evenoddpair(last, a, b, ranges[-1]):\n pass\n else:\n ranges.append([a, a, _Delta(a, b)])\n last = a\n return ranges\n\n# The maximum size of a case-folding group.\n# Case folding is implemented in parse.cc by a recursive process\n# with a recursion depth equal to the size of the largest\n# case-folding group, so it is important that this bound be small.\n# The current tables have no group bigger than 4.\n# If there are ever groups bigger than 10 or so, it will be\n# time to rework the code in parse.cc.\nMaxCasefoldGroup = 4\n\ndef main():\n lowergroups, casegroups = unicode.CaseGroups()\n foldpairs = []\n seen = {}\n for c in casegroups:\n if len(c) > MaxCasefoldGroup:\n raise unicode.Error(\"casefold group too long: %s\" % (c,))\n for i in range(len(c)):\n if c[i-1] in seen:\n raise unicode.Error(\"bad casegroups %d -> %d\" % (c[i-1], c[i]))\n seen[c[i-1]] = True\n foldpairs.append([c[i-1], c[i]])\n\n lowerpairs = []\n for lower, group in lowergroups.items():\n for g in group:\n if g != lower:\n lowerpairs.append([g, lower])\n\n def printpairs(name, foldpairs):\n foldpairs.sort()\n foldranges = _MakeRanges(foldpairs)\n print(\"// %d groups, %d pairs, %d ranges\" % (len(casegroups), len(foldpairs), len(foldranges)))\n print(\"const CaseFold unicode_%s[] = {\" % (name,))\n for lo, hi, delta in foldranges:\n print(\"\\t{ %d, %d, %s },\" % (lo, hi, delta))\n print(\"};\")\n print(\"const int num_unicode_%s = %d;\" % (name, len(foldranges)))\n print(\"\")\n\n print(_header)\n printpairs(\"casefold\", foldpairs)\n printpairs(\"tolower\", lowerpairs)\n print(_trailer)\n\nif __name__ == '__main__':\n main()\n" }, { "path": "re2/make_unicode_groups.py", "content": "#!/usr/bin/python3\n# Copyright 2008 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n\"\"\"Generate C++ tables for Unicode Script and Category groups.\"\"\"\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport sys\nimport unicode\n\n_header = \"\"\"\n// GENERATED BY make_unicode_groups.py; DO NOT EDIT.\n// make_unicode_groups.py >unicode_groups.cc\n\n#include \"re2/unicode_groups.h\"\n\nnamespace re2 {\n\n\"\"\"\n\n_trailer = \"\"\"\n\n} // namespace re2\n\n\"\"\"\n\nn16 = 0\nn32 = 0\n\ndef MakeRanges(codes):\n \"\"\"Turn a list like [1,2,3,7,8,9] into a range list [[1,3], [7,9]]\"\"\"\n ranges = []\n last = -100\n for c in codes:\n if c == last+1:\n ranges[-1][1] = c\n else:\n ranges.append([c, c])\n last = c\n return ranges\n\ndef PrintRanges(type, name, ranges):\n \"\"\"Print the ranges as an array of type named name.\"\"\"\n print(\"static const %s %s[] = {\" % (type, name))\n for lo, hi in ranges:\n print(\"\\t{ %d, %d },\" % (lo, hi))\n print(\"};\")\n\n# def PrintCodes(type, name, codes):\n# \"\"\"Print the codes as an array of type named name.\"\"\"\n# print(\"static %s %s[] = {\" % (type, name))\n# for c in codes:\n# print(\"\\t%d,\" % (c,))\n# print(\"};\")\n\ndef PrintGroup(name, codes):\n \"\"\"Print the data structures for the group of codes.\n Return a UGroup literal for the group.\"\"\"\n\n # See unicode_groups.h for a description of the data structure.\n\n # Split codes into 16-bit ranges and 32-bit ranges.\n range16 = MakeRanges([c for c in codes if c < 65536])\n range32 = MakeRanges([c for c in codes if c >= 65536])\n\n # Pull singleton ranges out of range16.\n # code16 = [lo for lo, hi in range16 if lo == hi]\n # range16 = [[lo, hi] for lo, hi in range16 if lo != hi]\n\n global n16\n global n32\n n16 += len(range16)\n n32 += len(range32)\n\n ugroup = \"{ \\\"%s\\\", +1\" % (name,)\n # if len(code16) > 0:\n # PrintCodes(\"uint16_t\", name+\"_code16\", code16)\n # ugroup += \", %s_code16, %d\" % (name, len(code16))\n # else:\n # ugroup += \", 0, 0\"\n if len(range16) > 0:\n PrintRanges(\"URange16\", name+\"_range16\", range16)\n ugroup += \", %s_range16, %d\" % (name, len(range16))\n else:\n ugroup += \", 0, 0\"\n if len(range32) > 0:\n PrintRanges(\"URange32\", name+\"_range32\", range32)\n ugroup += \", %s_range32, %d\" % (name, len(range32))\n else:\n ugroup += \", 0, 0\"\n ugroup += \" }\"\n return ugroup\n\ndef main():\n categories = unicode.Categories()\n scripts = unicode.Scripts()\n print(_header)\n ugroups = []\n for name in sorted(categories):\n ugroups.append(PrintGroup(name, categories[name]))\n for name in sorted(scripts):\n ugroups.append(PrintGroup(name, scripts[name]))\n print(\"// %d 16-bit ranges, %d 32-bit ranges\" % (n16, n32))\n print(\"const UGroup unicode_groups[] = {\")\n ugroups.sort()\n for ug in ugroups:\n print(\"\\t%s,\" % (ug,))\n print(\"};\")\n print(\"const int num_unicode_groups = %d;\" % (len(ugroups),))\n print(_trailer)\n\nif __name__ == '__main__':\n main()\n" }, { "path": "re2/mimics_pcre.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Determine whether this library should match PCRE exactly\n// for a particular Regexp. (If so, the testing framework can\n// check that it does.)\n//\n// This library matches PCRE except in these cases:\n// * the regexp contains a repetition of an empty string,\n// like (a*)* or (a*)+. In this case, PCRE will treat\n// the repetition sequence as ending with an empty string,\n// while this library does not.\n// * Perl and PCRE differ on whether \\v matches \\n.\n// For historical reasons, this library implements the Perl behavior.\n// * Perl and PCRE allow $ in one-line mode to match either the very\n// end of the text or just before a \\n at the end of the text.\n// This library requires it to match only the end of the text.\n// * Similarly, Perl and PCRE do not allow ^ in multi-line mode to\n// match the end of the text if the last character is a \\n.\n// This library does allow it.\n//\n// Regexp::MimicsPCRE checks for any of these conditions.\n\n#include \"absl/log/absl_log.h\"\n#include \"re2/regexp.h\"\n#include \"re2/walker-inl.h\"\n\nnamespace re2 {\n\n// Returns whether re might match an empty string.\nstatic bool CanBeEmptyString(Regexp *re);\n\n// Walker class to compute whether library handles a regexp\n// exactly as PCRE would. See comment at top for conditions.\n\nclass PCREWalker : public Regexp::Walker {\n public:\n PCREWalker() {}\n\n virtual bool PostVisit(Regexp* re, bool parent_arg, bool pre_arg,\n bool* child_args, int nchild_args);\n\n virtual bool ShortVisit(Regexp* re, bool a) {\n // Should never be called: we use Walk(), not WalkExponential().\n#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION\n ABSL_LOG(DFATAL) << \"PCREWalker::ShortVisit called\";\n#endif\n return a;\n }\n\n private:\n PCREWalker(const PCREWalker&) = delete;\n PCREWalker& operator=(const PCREWalker&) = delete;\n};\n\n// Called after visiting each of re's children and accumulating\n// the return values in child_args. So child_args contains whether\n// this library mimics PCRE for those subexpressions.\nbool PCREWalker::PostVisit(Regexp* re, bool parent_arg, bool pre_arg,\n bool* child_args, int nchild_args) {\n // If children failed, so do we.\n for (int i = 0; i < nchild_args; i++)\n if (!child_args[i])\n return false;\n\n // Otherwise look for other reasons to fail.\n switch (re->op()) {\n // Look for repeated empty string.\n case kRegexpStar:\n case kRegexpPlus:\n case kRegexpQuest:\n if (CanBeEmptyString(re->sub()[0]))\n return false;\n break;\n case kRegexpRepeat:\n if (re->max() == -1 && CanBeEmptyString(re->sub()[0]))\n return false;\n break;\n\n // Look for \\v\n case kRegexpLiteral:\n if (re->rune() == '\\v')\n return false;\n break;\n\n // Look for $ in single-line mode.\n case kRegexpEndText:\n case kRegexpEmptyMatch:\n if (re->parse_flags() & Regexp::WasDollar)\n return false;\n break;\n\n // Look for ^ in multi-line mode.\n case kRegexpBeginLine:\n // No condition: in single-line mode ^ becomes kRegexpBeginText.\n return false;\n\n default:\n break;\n }\n\n // Not proven guilty.\n return true;\n}\n\n// Returns whether this regexp's behavior will mimic PCRE's exactly.\nbool Regexp::MimicsPCRE() {\n PCREWalker w;\n return w.Walk(this, true);\n}\n\n\n// Walker class to compute whether a Regexp can match an empty string.\n// It is okay to overestimate. For example, \\b\\B cannot match an empty\n// string, because \\b and \\B are mutually exclusive, but this isn't\n// that smart and will say it can. Spurious empty strings\n// will reduce the number of regexps we sanity check against PCRE,\n// but they won't break anything.\n\nclass EmptyStringWalker : public Regexp::Walker {\n public:\n EmptyStringWalker() {}\n\n virtual bool PostVisit(Regexp* re, bool parent_arg, bool pre_arg,\n bool* child_args, int nchild_args);\n\n virtual bool ShortVisit(Regexp* re, bool a) {\n // Should never be called: we use Walk(), not WalkExponential().\n#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION\n ABSL_LOG(DFATAL) << \"EmptyStringWalker::ShortVisit called\";\n#endif\n return a;\n }\n\n private:\n EmptyStringWalker(const EmptyStringWalker&) = delete;\n EmptyStringWalker& operator=(const EmptyStringWalker&) = delete;\n};\n\n// Called after visiting re's children. child_args contains the return\n// value from each of the children's PostVisits (i.e., whether each child\n// can match an empty string). Returns whether this clause can match an\n// empty string.\nbool EmptyStringWalker::PostVisit(Regexp* re, bool parent_arg, bool pre_arg,\n bool* child_args, int nchild_args) {\n switch (re->op()) {\n case kRegexpNoMatch: // never empty\n case kRegexpLiteral:\n case kRegexpAnyChar:\n case kRegexpAnyByte:\n case kRegexpCharClass:\n case kRegexpLiteralString:\n return false;\n\n case kRegexpEmptyMatch: // always empty\n case kRegexpBeginLine: // always empty, when they match\n case kRegexpEndLine:\n case kRegexpNoWordBoundary:\n case kRegexpWordBoundary:\n case kRegexpBeginText:\n case kRegexpEndText:\n case kRegexpStar: // can always be empty\n case kRegexpQuest:\n case kRegexpHaveMatch:\n return true;\n\n case kRegexpConcat: // can be empty if all children can\n for (int i = 0; i < nchild_args; i++)\n if (!child_args[i])\n return false;\n return true;\n\n case kRegexpAlternate: // can be empty if any child can\n for (int i = 0; i < nchild_args; i++)\n if (child_args[i])\n return true;\n return false;\n\n case kRegexpPlus: // can be empty if the child can\n case kRegexpCapture:\n return child_args[0];\n\n case kRegexpRepeat: // can be empty if child can or is x{0}\n return child_args[0] || re->min() == 0;\n }\n return false;\n}\n\n// Returns whether re can match an empty string.\nstatic bool CanBeEmptyString(Regexp* re) {\n EmptyStringWalker w;\n return w.Walk(re, true);\n}\n\n} // namespace re2\n" }, { "path": "re2/nfa.cc", "content": "// Copyright 2006-2007 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Tested by search_test.cc.\n//\n// Prog::SearchNFA, an NFA search.\n// This is an actual NFA like the theorists talk about,\n// not the pseudo-NFA found in backtracking regexp implementations.\n//\n// IMPLEMENTATION\n//\n// This algorithm is a variant of one that appeared in Rob Pike's sam editor,\n// which is a variant of the one described in Thompson's 1968 CACM paper.\n// See http://swtch.com/~rsc/regexp/ for various history. The main feature\n// over the DFA implementation is that it tracks submatch boundaries.\n//\n// When the choice of submatch boundaries is ambiguous, this particular\n// implementation makes the same choices that traditional backtracking\n// implementations (in particular, Perl and PCRE) do.\n// Note that unlike in Perl and PCRE, this algorithm *cannot* take exponential\n// time in the length of the input.\n//\n// Like Thompson's original machine and like the DFA implementation, this\n// implementation notices a match only once it is one byte past it.\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/prog.h\"\n#include \"re2/regexp.h\"\n#include \"re2/sparse_array.h\"\n#include \"re2/sparse_set.h\"\n\nnamespace re2 {\n\nstatic const bool ExtraDebug = false;\n\nclass NFA {\n public:\n NFA(Prog* prog);\n ~NFA();\n\n // Searches for a matching string.\n // * If anchored is true, only considers matches starting at offset.\n // Otherwise finds lefmost match at or after offset.\n // * If longest is true, returns the longest match starting\n // at the chosen start point. Otherwise returns the so-called\n // left-biased match, the one traditional backtracking engines\n // (like Perl and PCRE) find.\n // Records submatch boundaries in submatch[1..nsubmatch-1].\n // Submatch[0] is the entire match. When there is a choice in\n // which text matches each subexpression, the submatch boundaries\n // are chosen to match what a backtracking implementation would choose.\n bool Search(absl::string_view text, absl::string_view context, bool anchored,\n bool longest, absl::string_view* submatch, int nsubmatch);\n\n private:\n struct Thread {\n union {\n int ref;\n Thread* next; // when on free list\n };\n const char** capture;\n };\n\n // State for explicit stack in AddToThreadq.\n struct AddState {\n int id; // Inst to process\n Thread* t; // if not null, set t0 = t before processing id\n };\n\n // Threadq is a list of threads. The list is sorted by the order\n // in which Perl would explore that particular state -- the earlier\n // choices appear earlier in the list.\n typedef SparseArray Threadq;\n\n inline Thread* AllocThread();\n inline Thread* Incref(Thread* t);\n inline void Decref(Thread* t);\n\n // Follows all empty arrows from id0 and enqueues all the states reached.\n // Enqueues only the ByteRange instructions that match byte c.\n // context is used (with p) for evaluating empty-width specials.\n // p is the current input position, and t0 is the current thread.\n void AddToThreadq(Threadq* q, int id0, int c, absl::string_view context,\n const char* p, Thread* t0);\n\n // Run runq on byte c, appending new states to nextq.\n // Updates matched_ and match_ as new, better matches are found.\n // context is used (with p) for evaluating empty-width specials.\n // p is the position of byte c in the input string for AddToThreadq;\n // p-1 will be used when processing Match instructions.\n // Frees all the threads on runq.\n // If there is a shortcut to the end, returns that shortcut.\n int Step(Threadq* runq, Threadq* nextq, int c, absl::string_view context,\n const char* p);\n\n // Returns text version of capture information, for debugging.\n std::string FormatCapture(const char** capture);\n\n void CopyCapture(const char** dst, const char** src) {\n memmove(dst, src, ncapture_*sizeof src[0]);\n }\n\n Prog* prog_; // underlying program\n int start_; // start instruction in program\n int ncapture_; // number of submatches to track\n bool longest_; // whether searching for longest match\n bool endmatch_; // whether match must end at text.end()\n const char* btext_; // beginning of text (for FormatSubmatch)\n const char* etext_; // end of text (for endmatch_)\n Threadq q0_, q1_; // pre-allocated for Search.\n PODArray stack_; // pre-allocated for AddToThreadq\n std::deque arena_; // thread arena\n Thread* freelist_; // thread freelist\n const char** match_; // best match so far\n bool matched_; // any match so far?\n\n NFA(const NFA&) = delete;\n NFA& operator=(const NFA&) = delete;\n};\n\nNFA::NFA(Prog* prog) {\n prog_ = prog;\n start_ = prog_->start();\n ncapture_ = 0;\n longest_ = false;\n endmatch_ = false;\n btext_ = NULL;\n etext_ = NULL;\n q0_.resize(prog_->size());\n q1_.resize(prog_->size());\n // See NFA::AddToThreadq() for why this is so.\n int nstack = 2*prog_->inst_count(kInstCapture) +\n prog_->inst_count(kInstEmptyWidth) +\n prog_->inst_count(kInstNop) + 1; // + 1 for start inst\n stack_ = PODArray(nstack);\n freelist_ = NULL;\n match_ = NULL;\n matched_ = false;\n}\n\nNFA::~NFA() {\n delete[] match_;\n for (const Thread& t : arena_)\n delete[] t.capture;\n}\n\nNFA::Thread* NFA::AllocThread() {\n Thread* t = freelist_;\n if (t != NULL) {\n freelist_ = t->next;\n t->ref = 1;\n // We don't need to touch t->capture because\n // the caller will immediately overwrite it.\n return t;\n }\n arena_.emplace_back();\n t = &arena_.back();\n t->ref = 1;\n t->capture = new const char*[ncapture_];\n return t;\n}\n\nNFA::Thread* NFA::Incref(Thread* t) {\n ABSL_DCHECK(t != NULL);\n t->ref++;\n return t;\n}\n\nvoid NFA::Decref(Thread* t) {\n ABSL_DCHECK(t != NULL);\n t->ref--;\n if (t->ref > 0)\n return;\n ABSL_DCHECK_EQ(t->ref, 0);\n t->next = freelist_;\n freelist_ = t;\n}\n\n// Follows all empty arrows from id0 and enqueues all the states reached.\n// Enqueues only the ByteRange instructions that match byte c.\n// context is used (with p) for evaluating empty-width specials.\n// p is the current input position, and t0 is the current thread.\nvoid NFA::AddToThreadq(Threadq* q, int id0, int c, absl::string_view context,\n const char* p, Thread* t0) {\n if (id0 == 0)\n return;\n\n // Use stack_ to hold our stack of instructions yet to process.\n // It was preallocated as follows:\n // two entries per Capture;\n // one entry per EmptyWidth; and\n // one entry per Nop.\n // This reflects the maximum number of stack pushes that each can\n // perform. (Each instruction can be processed at most once.)\n AddState* stk = stack_.data();\n int nstk = 0;\n\n stk[nstk++] = {id0, NULL};\n while (nstk > 0) {\n ABSL_DCHECK_LE(nstk, stack_.size());\n AddState a = stk[--nstk];\n\n Loop:\n if (a.t != NULL) {\n // t0 was a thread that we allocated and copied in order to\n // record the capture, so we must now decref it.\n Decref(t0);\n t0 = a.t;\n }\n\n int id = a.id;\n if (id == 0)\n continue;\n if (q->has_index(id)) {\n if (ExtraDebug)\n absl::FPrintF(stderr, \" [%d%s]\\n\", id, FormatCapture(t0->capture));\n continue;\n }\n\n // Create entry in q no matter what. We might fill it in below,\n // or we might not. Even if not, it is necessary to have it,\n // so that we don't revisit id0 during the recursion.\n q->set_new(id, NULL);\n Thread** tp = &q->get_existing(id);\n int j;\n Thread* t;\n Prog::Inst* ip = prog_->inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"unhandled \" << ip->opcode() << \" in AddToThreadq\";\n break;\n\n case kInstFail:\n break;\n\n case kInstAltMatch:\n // Save state; will pick up at next byte.\n t = Incref(t0);\n *tp = t;\n\n ABSL_DCHECK(!ip->last());\n a = {id+1, NULL};\n goto Loop;\n\n case kInstNop:\n if (!ip->last())\n stk[nstk++] = {id+1, NULL};\n\n // Continue on.\n a = {ip->out(), NULL};\n goto Loop;\n\n case kInstCapture:\n if (!ip->last())\n stk[nstk++] = {id+1, NULL};\n\n if ((j=ip->cap()) < ncapture_) {\n // Push a dummy whose only job is to restore t0\n // once we finish exploring this possibility.\n stk[nstk++] = {0, t0};\n\n // Record capture.\n t = AllocThread();\n CopyCapture(t->capture, t0->capture);\n t->capture[j] = p;\n t0 = t;\n }\n a = {ip->out(), NULL};\n goto Loop;\n\n case kInstByteRange:\n if (!ip->Matches(c))\n goto Next;\n\n // Save state; will pick up at next byte.\n t = Incref(t0);\n *tp = t;\n if (ExtraDebug)\n absl::FPrintF(stderr, \" + %d%s\\n\", id, FormatCapture(t0->capture));\n\n if (ip->hint() == 0)\n break;\n a = {id+ip->hint(), NULL};\n goto Loop;\n\n case kInstMatch:\n // Save state; will pick up at next byte.\n t = Incref(t0);\n *tp = t;\n if (ExtraDebug)\n absl::FPrintF(stderr, \" ! %d%s\\n\", id, FormatCapture(t0->capture));\n\n Next:\n if (ip->last())\n break;\n a = {id+1, NULL};\n goto Loop;\n\n case kInstEmptyWidth:\n if (!ip->last())\n stk[nstk++] = {id+1, NULL};\n\n // Continue on if we have all the right flag bits.\n if (ip->empty() & ~Prog::EmptyFlags(context, p))\n break;\n a = {ip->out(), NULL};\n goto Loop;\n }\n }\n}\n\n// Run runq on byte c, appending new states to nextq.\n// Updates matched_ and match_ as new, better matches are found.\n// context is used (with p) for evaluating empty-width specials.\n// p is the position of byte c in the input string for AddToThreadq;\n// p-1 will be used when processing Match instructions.\n// Frees all the threads on runq.\n// If there is a shortcut to the end, returns that shortcut.\nint NFA::Step(Threadq* runq, Threadq* nextq, int c, absl::string_view context,\n const char* p) {\n nextq->clear();\n\n for (Threadq::iterator i = runq->begin(); i != runq->end(); ++i) {\n Thread* t = i->value();\n if (t == NULL)\n continue;\n\n if (longest_) {\n // Can skip any threads started after our current best match.\n if (matched_ && match_[0] < t->capture[0]) {\n Decref(t);\n continue;\n }\n }\n\n int id = i->index();\n Prog::Inst* ip = prog_->inst(id);\n\n switch (ip->opcode()) {\n default:\n // Should only see the values handled below.\n ABSL_LOG(DFATAL) << \"Unhandled \" << ip->opcode() << \" in step\";\n break;\n\n case kInstByteRange:\n AddToThreadq(nextq, ip->out(), c, context, p, t);\n break;\n\n case kInstAltMatch:\n if (i != runq->begin())\n break;\n // The match is ours if we want it.\n if (ip->greedy(prog_) || longest_) {\n CopyCapture(match_, t->capture);\n matched_ = true;\n\n Decref(t);\n for (++i; i != runq->end(); ++i) {\n if (i->value() != NULL)\n Decref(i->value());\n }\n runq->clear();\n if (ip->greedy(prog_))\n return ip->out1();\n return ip->out();\n }\n break;\n\n case kInstMatch: {\n // Avoid invoking undefined behavior (arithmetic on a null pointer)\n // by storing p instead of p-1. (What would the latter even mean?!)\n // This complements the special case in NFA::Search().\n if (p == NULL) {\n CopyCapture(match_, t->capture);\n match_[1] = p;\n matched_ = true;\n break;\n }\n\n if (endmatch_ && p-1 != etext_)\n break;\n\n if (longest_) {\n // Leftmost-longest mode: save this match only if\n // it is either farther to the left or at the same\n // point but longer than an existing match.\n if (!matched_ || t->capture[0] < match_[0] ||\n (t->capture[0] == match_[0] && p-1 > match_[1])) {\n CopyCapture(match_, t->capture);\n match_[1] = p-1;\n matched_ = true;\n }\n } else {\n // Leftmost-biased mode: this match is by definition\n // better than what we've already found (see next line).\n CopyCapture(match_, t->capture);\n match_[1] = p-1;\n matched_ = true;\n\n // Cut off the threads that can only find matches\n // worse than the one we just found: don't run the\n // rest of the current Threadq.\n Decref(t);\n for (++i; i != runq->end(); ++i) {\n if (i->value() != NULL)\n Decref(i->value());\n }\n runq->clear();\n return 0;\n }\n break;\n }\n }\n Decref(t);\n }\n runq->clear();\n return 0;\n}\n\nstd::string NFA::FormatCapture(const char** capture) {\n std::string s;\n for (int i = 0; i < ncapture_; i+=2) {\n if (capture[i] == NULL)\n s += \"(?,?)\";\n else if (capture[i+1] == NULL)\n s += absl::StrFormat(\"(%d,?)\",\n capture[i] - btext_);\n else\n s += absl::StrFormat(\"(%d,%d)\",\n capture[i] - btext_,\n capture[i+1] - btext_);\n }\n return s;\n}\n\nbool NFA::Search(absl::string_view text, absl::string_view context,\n bool anchored, bool longest, absl::string_view* submatch,\n int nsubmatch) {\n if (start_ == 0)\n return false;\n\n if (context.data() == NULL)\n context = text;\n\n // Sanity check: make sure that text lies within context.\n if (BeginPtr(text) < BeginPtr(context) || EndPtr(text) > EndPtr(context)) {\n ABSL_LOG(DFATAL) << \"context does not contain text\";\n return false;\n }\n\n if (prog_->anchor_start() && BeginPtr(context) != BeginPtr(text))\n return false;\n if (prog_->anchor_end() && EndPtr(context) != EndPtr(text))\n return false;\n anchored |= prog_->anchor_start();\n if (prog_->anchor_end()) {\n longest = true;\n endmatch_ = true;\n }\n\n if (nsubmatch < 0) {\n ABSL_LOG(DFATAL) << \"Bad args: nsubmatch=\" << nsubmatch;\n return false;\n }\n\n // Save search parameters.\n ncapture_ = 2*nsubmatch;\n longest_ = longest;\n\n if (nsubmatch == 0) {\n // We need to maintain match[0], both to distinguish the\n // longest match (if longest is true) and also to tell\n // whether we've seen any matches at all.\n ncapture_ = 2;\n }\n\n match_ = new const char*[ncapture_];\n memset(match_, 0, ncapture_*sizeof match_[0]);\n matched_ = false;\n\n // For debugging prints.\n btext_ = context.data();\n // For convenience.\n etext_ = text.data() + text.size();\n\n if (ExtraDebug)\n absl::FPrintF(stderr, \"NFA::Search %s (context: %s) anchored=%d longest=%d\\n\",\n text, context, anchored, longest);\n\n // Set up search.\n Threadq* runq = &q0_;\n Threadq* nextq = &q1_;\n runq->clear();\n nextq->clear();\n\n // Loop over the text, stepping the machine.\n for (const char* p = text.data();; p++) {\n if (ExtraDebug) {\n int c = 0;\n if (p == btext_)\n c = '^';\n else if (p > etext_)\n c = '$';\n else if (p < etext_)\n c = p[0] & 0xFF;\n\n absl::FPrintF(stderr, \"%c:\", c);\n for (Threadq::iterator i = runq->begin(); i != runq->end(); ++i) {\n Thread* t = i->value();\n if (t == NULL)\n continue;\n absl::FPrintF(stderr, \" %d%s\", i->index(), FormatCapture(t->capture));\n }\n absl::FPrintF(stderr, \"\\n\");\n }\n\n // This is a no-op the first time around the loop because runq is empty.\n int id = Step(runq, nextq, p < etext_ ? p[0] & 0xFF : -1, context, p);\n ABSL_DCHECK_EQ(runq->size(), 0);\n using std::swap;\n swap(nextq, runq);\n nextq->clear();\n if (id != 0) {\n // We're done: full match ahead.\n p = etext_;\n for (;;) {\n Prog::Inst* ip = prog_->inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"Unexpected opcode in short circuit: \"\n << ip->opcode();\n break;\n\n case kInstCapture:\n if (ip->cap() < ncapture_)\n match_[ip->cap()] = p;\n id = ip->out();\n continue;\n\n case kInstNop:\n id = ip->out();\n continue;\n\n case kInstMatch:\n match_[1] = p;\n matched_ = true;\n break;\n }\n break;\n }\n break;\n }\n\n if (p > etext_)\n break;\n\n // Start a new thread if there have not been any matches.\n // (No point in starting a new thread if there have been\n // matches, since it would be to the right of the match\n // we already found.)\n if (!matched_ && (!anchored || p == text.data())) {\n // Try to use prefix accel (e.g. memchr) to skip ahead.\n // The search must be unanchored and there must be zero\n // possible matches already.\n if (!anchored && runq->size() == 0 &&\n p < etext_ && prog_->can_prefix_accel()) {\n p = reinterpret_cast(prog_->PrefixAccel(p, etext_ - p));\n if (p == NULL)\n p = etext_;\n }\n\n Thread* t = AllocThread();\n CopyCapture(t->capture, match_);\n t->capture[0] = p;\n AddToThreadq(runq, start_, p < etext_ ? p[0] & 0xFF : -1, context, p,\n t);\n Decref(t);\n }\n\n // If all the threads have died, stop early.\n if (runq->size() == 0) {\n if (ExtraDebug)\n absl::FPrintF(stderr, \"dead\\n\");\n break;\n }\n\n // Avoid invoking undefined behavior (arithmetic on a null pointer)\n // by simply not continuing the loop.\n // This complements the special case in NFA::Step().\n if (p == NULL) {\n (void) Step(runq, nextq, -1, context, p);\n ABSL_DCHECK_EQ(runq->size(), 0);\n using std::swap;\n swap(nextq, runq);\n nextq->clear();\n break;\n }\n }\n\n for (Threadq::iterator i = runq->begin(); i != runq->end(); ++i) {\n if (i->value() != NULL)\n Decref(i->value());\n }\n\n if (matched_) {\n for (int i = 0; i < nsubmatch; i++)\n submatch[i] = absl::string_view(\n match_[2 * i],\n static_cast(match_[2 * i + 1] - match_[2 * i]));\n if (ExtraDebug)\n absl::FPrintF(stderr, \"match (%d,%d)\\n\",\n match_[0] - btext_,\n match_[1] - btext_);\n return true;\n }\n return false;\n}\n\nbool Prog::SearchNFA(absl::string_view text, absl::string_view context,\n Anchor anchor, MatchKind kind, absl::string_view* match,\n int nmatch) {\n if (ExtraDebug)\n Dump();\n\n NFA nfa(this);\n absl::string_view sp;\n if (kind == kFullMatch) {\n anchor = kAnchored;\n if (nmatch == 0) {\n match = &sp;\n nmatch = 1;\n }\n }\n if (!nfa.Search(text, context, anchor == kAnchored, kind != kFirstMatch, match, nmatch))\n return false;\n if (kind == kFullMatch && EndPtr(match[0]) != EndPtr(text))\n return false;\n return true;\n}\n\n// For each instruction i in the program reachable from the start, compute the\n// number of instructions reachable from i by following only empty transitions\n// and record that count as fanout[i].\n//\n// fanout holds the results and is also the work queue for the outer iteration.\n// reachable holds the reached nodes for the inner iteration.\nvoid Prog::Fanout(SparseArray* fanout) {\n ABSL_DCHECK_EQ(fanout->max_size(), size());\n SparseSet reachable(size());\n fanout->clear();\n fanout->set_new(start(), 0);\n for (SparseArray::iterator i = fanout->begin(); i != fanout->end(); ++i) {\n int* count = &i->value();\n reachable.clear();\n reachable.insert(i->index());\n for (SparseSet::iterator j = reachable.begin(); j != reachable.end(); ++j) {\n int id = *j;\n Prog::Inst* ip = inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"unhandled \" << ip->opcode()\n << \" in Prog::Fanout()\";\n break;\n\n case kInstByteRange:\n if (!ip->last())\n reachable.insert(id+1);\n\n (*count)++;\n if (!fanout->has_index(ip->out())) {\n fanout->set_new(ip->out(), 0);\n }\n break;\n\n case kInstAltMatch:\n ABSL_DCHECK(!ip->last());\n reachable.insert(id+1);\n break;\n\n case kInstCapture:\n case kInstEmptyWidth:\n case kInstNop:\n if (!ip->last())\n reachable.insert(id+1);\n\n reachable.insert(ip->out());\n break;\n\n case kInstMatch:\n if (!ip->last())\n reachable.insert(id+1);\n break;\n\n case kInstFail:\n break;\n }\n }\n }\n}\n\n} // namespace re2\n" }, { "path": "re2/onepass.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Tested by search_test.cc.\n//\n// Prog::SearchOnePass is an efficient implementation of\n// regular expression search with submatch tracking for\n// what I call \"one-pass regular expressions\". (An alternate\n// name might be \"backtracking-free regular expressions\".)\n//\n// One-pass regular expressions have the property that\n// at each input byte during an anchored match, there may be\n// multiple alternatives but only one can proceed for any\n// given input byte.\n//\n// For example, the regexp /x*yx*/ is one-pass: you read\n// x's until a y, then you read the y, then you keep reading x's.\n// At no point do you have to guess what to do or back up\n// and try a different guess.\n//\n// On the other hand, /x*x/ is not one-pass: when you're\n// looking at an input \"x\", it's not clear whether you should\n// use it to extend the x* or as the final x.\n//\n// More examples: /([^ ]*) (.*)/ is one-pass; /(.*) (.*)/ is not.\n// /(\\d+)-(\\d+)/ is one-pass; /(\\d+).(\\d+)/ is not.\n//\n// A simple intuition for identifying one-pass regular expressions\n// is that it's always immediately obvious when a repetition ends.\n// It must also be immediately obvious which branch of an | to take:\n//\n// /x(y|z)/ is one-pass, but /(xy|xz)/ is not.\n//\n// The NFA-based search in nfa.cc does some bookkeeping to\n// avoid the need for backtracking and its associated exponential blowup.\n// But if we have a one-pass regular expression, there is no\n// possibility of backtracking, so there is no need for the\n// extra bookkeeping. Hence, this code.\n//\n// On a one-pass regular expression, the NFA code in nfa.cc\n// runs at about 1/20 of the backtracking-based PCRE speed.\n// In contrast, the code in this file runs at about the same\n// speed as PCRE.\n//\n// One-pass regular expressions get used a lot when RE is\n// used for parsing simple strings, so it pays off to\n// notice them and handle them efficiently.\n//\n// See also Anne Brüggemann-Klein and Derick Wood,\n// \"One-unambiguous regular languages\", Information and Computation 142(2).\n\n#include \n#include \n\n#include \n#include \n#include \n\n#include \"absl/container/fixed_array.h\"\n#include \"absl/container/inlined_vector.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/prog.h\"\n#include \"re2/sparse_set.h\"\n#include \"util/utf.h\"\n\n// Silence \"zero-sized array in struct/union\" warning for OneState::action.\n#ifdef _MSC_VER\n#pragma warning(disable: 4200)\n#endif\n\nnamespace re2 {\n\nstatic const bool ExtraDebug = false;\n\n// The key insight behind this implementation is that the\n// non-determinism in an NFA for a one-pass regular expression\n// is contained. To explain what that means, first a\n// refresher about what regular expression programs look like\n// and how the usual NFA execution runs.\n//\n// In a regular expression program, only the kInstByteRange\n// instruction processes an input byte c and moves on to the\n// next byte in the string (it does so if c is in the given range).\n// The kInstByteRange instructions correspond to literal characters\n// and character classes in the regular expression.\n//\n// The kInstAlt instructions are used as wiring to connect the\n// kInstByteRange instructions together in interesting ways when\n// implementing | + and *.\n// The kInstAlt instruction forks execution, like a goto that\n// jumps to ip->out() and ip->out1() in parallel. Each of the\n// resulting computation paths is called a thread.\n//\n// The other instructions -- kInstEmptyWidth, kInstMatch, kInstCapture --\n// are interesting in their own right but like kInstAlt they don't\n// advance the input pointer. Only kInstByteRange does.\n//\n// The automaton execution in nfa.cc runs all the possible\n// threads of execution in lock-step over the input. To process\n// a particular byte, each thread gets run until it either dies\n// or finds a kInstByteRange instruction matching the byte.\n// If the latter happens, the thread stops just past the\n// kInstByteRange instruction (at ip->out()) and waits for\n// the other threads to finish processing the input byte.\n// Then, once all the threads have processed that input byte,\n// the whole process repeats. The kInstAlt state instruction\n// might create new threads during input processing, but no\n// matter what, all the threads stop after a kInstByteRange\n// and wait for the other threads to \"catch up\".\n// Running in lock step like this ensures that the NFA reads\n// the input string only once.\n//\n// Each thread maintains its own set of capture registers\n// (the string positions at which it executed the kInstCapture\n// instructions corresponding to capturing parentheses in the\n// regular expression). Repeated copying of the capture registers\n// is the main performance bottleneck in the NFA implementation.\n//\n// A regular expression program is \"one-pass\" if, no matter what\n// the input string, there is only one thread that makes it\n// past a kInstByteRange instruction at each input byte. This means\n// that there is in some sense only one active thread throughout\n// the execution. Other threads might be created during the\n// processing of an input byte, but they are ephemeral: only one\n// thread is left to start processing the next input byte.\n// This is what I meant above when I said the non-determinism\n// was \"contained\".\n//\n// To execute a one-pass regular expression program, we can build\n// a DFA (no non-determinism) that has at most as many states as\n// the NFA (compare this to the possibly exponential number of states\n// in the general case). Each state records, for each possible\n// input byte, the next state along with the conditions required\n// before entering that state -- empty-width flags that must be true\n// and capture operations that must be performed. It also records\n// whether a set of conditions required to finish a match at that\n// point in the input rather than process the next byte.\n\n// A state in the one-pass NFA - just an array of actions indexed\n// by the bytemap_[] of the next input byte. (The bytemap\n// maps next input bytes into equivalence classes, to reduce\n// the memory footprint.)\nstruct OneState {\n uint32_t matchcond; // conditions to match right now.\n uint32_t action[];\n};\n\n// The uint32_t conditions in the action are a combination of\n// condition and capture bits and the next state. The bottom 16 bits\n// are the condition and capture bits, and the top 16 are the index of\n// the next state.\n//\n// Bits 0-5 are the empty-width flags from prog.h.\n// Bit 6 is kMatchWins, which means the match takes\n// priority over moving to next in a first-match search.\n// The remaining bits mark capture registers that should\n// be set to the current input position. The capture bits\n// start at index 2, since the search loop can take care of\n// cap[0], cap[1] (the overall match position).\n// That means we can handle up to 5 capturing parens: $1 through $4, plus $0.\n// No input position can satisfy both kEmptyWordBoundary\n// and kEmptyNonWordBoundary, so we can use that as a sentinel\n// instead of needing an extra bit.\n\nstatic const int kIndexShift = 16; // number of bits below index\nstatic const int kEmptyShift = 6; // number of empty flags in prog.h\nstatic const int kRealCapShift = kEmptyShift + 1;\nstatic const int kRealMaxCap = (kIndexShift - kRealCapShift) / 2 * 2;\n\n// Parameters used to skip over cap[0], cap[1].\nstatic const int kCapShift = kRealCapShift - 2;\nstatic const int kMaxCap = kRealMaxCap + 2;\n\nstatic const uint32_t kMatchWins = 1 << kEmptyShift;\nstatic const uint32_t kCapMask = ((1 << kRealMaxCap) - 1) << kRealCapShift;\n\nstatic const uint32_t kImpossible = kEmptyWordBoundary | kEmptyNonWordBoundary;\n\n// Check, at compile time, that prog.h agrees with math above.\n// This function is never called.\nvoid OnePass_Checks() {\n static_assert((1<(nodes + statesize*nodeindex);\n}\n\nbool Prog::SearchOnePass(absl::string_view text, absl::string_view context,\n Anchor anchor, MatchKind kind,\n absl::string_view* match, int nmatch) {\n if (anchor != kAnchored && kind != kFullMatch) {\n ABSL_LOG(DFATAL) << \"Cannot use SearchOnePass for unanchored matches.\";\n return false;\n }\n\n // Make sure we have at least cap[1],\n // because we use it to tell if we matched.\n int ncap = 2*nmatch;\n if (ncap < 2)\n ncap = 2;\n\n const char* cap[kMaxCap];\n for (int i = 0; i < ncap; i++)\n cap[i] = NULL;\n\n const char* matchcap[kMaxCap];\n for (int i = 0; i < ncap; i++)\n matchcap[i] = NULL;\n\n if (context.data() == NULL)\n context = text;\n if (anchor_start() && BeginPtr(context) != BeginPtr(text))\n return false;\n if (anchor_end() && EndPtr(context) != EndPtr(text))\n return false;\n if (anchor_end())\n kind = kFullMatch;\n\n uint8_t* nodes = onepass_nodes_.data();\n int statesize = sizeof(OneState) + bytemap_range()*sizeof(uint32_t);\n // start() is always mapped to the zeroth OneState.\n OneState* state = IndexToNode(nodes, statesize, 0);\n uint8_t* bytemap = bytemap_;\n const char* bp = text.data();\n const char* ep = text.data() + text.size();\n const char* p;\n bool matched = false;\n matchcap[0] = bp;\n cap[0] = bp;\n uint32_t nextmatchcond = state->matchcond;\n for (p = bp; p < ep; p++) {\n int c = bytemap[*p & 0xFF];\n uint32_t matchcond = nextmatchcond;\n uint32_t cond = state->action[c];\n\n // Determine whether we can reach act->next.\n // If so, advance state and nextmatchcond.\n if ((cond & kEmptyAllFlags) == 0 || Satisfy(cond, context, p)) {\n uint32_t nextindex = cond >> kIndexShift;\n state = IndexToNode(nodes, statesize, nextindex);\n nextmatchcond = state->matchcond;\n } else {\n state = NULL;\n nextmatchcond = kImpossible;\n }\n\n // This code section is carefully tuned.\n // The goto sequence is about 10% faster than the\n // obvious rewrite as a large if statement in the\n // ASCIIMatchRE2 and DotMatchRE2 benchmarks.\n\n // Saving the match capture registers is expensive.\n // Is this intermediate match worth thinking about?\n\n // Not if we want a full match.\n if (kind == kFullMatch)\n goto skipmatch;\n\n // Not if it's impossible.\n if (matchcond == kImpossible)\n goto skipmatch;\n\n // Not if the possible match is beaten by the certain\n // match at the next byte. When this test is useless\n // (e.g., HTTPPartialMatchRE2) it slows the loop by\n // about 10%, but when it avoids work (e.g., DotMatchRE2),\n // it cuts the loop execution by about 45%.\n if ((cond & kMatchWins) == 0 && (nextmatchcond & kEmptyAllFlags) == 0)\n goto skipmatch;\n\n // Finally, the match conditions must be satisfied.\n if ((matchcond & kEmptyAllFlags) == 0 || Satisfy(matchcond, context, p)) {\n for (int i = 2; i < 2*nmatch; i++)\n matchcap[i] = cap[i];\n if (nmatch > 1 && (matchcond & kCapMask))\n ApplyCaptures(matchcond, p, matchcap, ncap);\n matchcap[1] = p;\n matched = true;\n\n // If we're in longest match mode, we have to keep\n // going and see if we find a longer match.\n // In first match mode, we can stop if the match\n // takes priority over the next state for this input byte.\n // That bit is per-input byte and thus in cond, not matchcond.\n if (kind == kFirstMatch && (cond & kMatchWins))\n goto done;\n }\n\n skipmatch:\n if (state == NULL)\n goto done;\n if ((cond & kCapMask) && nmatch > 1)\n ApplyCaptures(cond, p, cap, ncap);\n }\n\n // Look for match at end of input.\n {\n uint32_t matchcond = state->matchcond;\n if (matchcond != kImpossible &&\n ((matchcond & kEmptyAllFlags) == 0 || Satisfy(matchcond, context, p))) {\n if (nmatch > 1 && (matchcond & kCapMask))\n ApplyCaptures(matchcond, p, cap, ncap);\n for (int i = 2; i < ncap; i++)\n matchcap[i] = cap[i];\n matchcap[1] = p;\n matched = true;\n }\n }\n\ndone:\n if (!matched)\n return false;\n for (int i = 0; i < nmatch; i++)\n match[i] = absl::string_view(\n matchcap[2 * i],\n static_cast(matchcap[2 * i + 1] - matchcap[2 * i]));\n return true;\n}\n\n// Analysis to determine whether a given regexp program is one-pass.\n\n// If ip is not on workq, adds ip to work queue and returns true.\n// If ip is already on work queue, does nothing and returns false.\n// If ip is NULL, does nothing and returns true (pretends to add it).\ntypedef SparseSet Instq;\nstatic bool AddQ(Instq *q, int id) {\n if (id == 0)\n return true;\n if (q->contains(id))\n return false;\n q->insert(id);\n return true;\n}\n\nstruct InstCond {\n int id;\n uint32_t cond;\n};\n\n// Returns whether this is a one-pass program; that is,\n// returns whether it is safe to use SearchOnePass on this program.\n// These conditions must be true for any instruction ip:\n//\n// (1) for any other Inst nip, there is at most one input-free\n// path from ip to nip.\n// (2) there is at most one kInstByte instruction reachable from\n// ip that matches any particular byte c.\n// (3) there is at most one input-free path from ip to a kInstMatch\n// instruction.\n//\n// This is actually just a conservative approximation: it might\n// return false when the answer is true, when kInstEmptyWidth\n// instructions are involved.\n// Constructs and saves corresponding one-pass NFA on success.\nbool Prog::IsOnePass() {\n if (did_onepass_)\n return onepass_nodes_.data() != NULL;\n did_onepass_ = true;\n\n if (start() == 0) // no match\n return false;\n\n // Steal memory for the one-pass NFA from the overall DFA budget.\n // Willing to use at most 1/4 of the DFA budget (heuristic).\n // Limit max node count to 65000 as a conservative estimate to\n // avoid overflowing 16-bit node index in encoding.\n int maxnodes = 2 + inst_count(kInstByteRange);\n int statesize = sizeof(OneState) + bytemap_range()*sizeof(uint32_t);\n if (maxnodes >= 65000 || dfa_mem_ / 4 / statesize < maxnodes)\n return false;\n\n // Flood the graph starting at the start state, and check\n // that in each reachable state, each possible byte leads\n // to a unique next state.\n int stacksize = inst_count(kInstCapture) +\n inst_count(kInstEmptyWidth) +\n inst_count(kInstNop) + 1; // + 1 for start inst\n absl::FixedArray stack_storage(stacksize);\n InstCond* stack = stack_storage.data();\n\n int size = this->size();\n absl::FixedArray nodebyid_storage(size, -1); // indexed by ip\n int* nodebyid = nodebyid_storage.data();\n\n // Originally, nodes was a uint8_t[maxnodes*statesize], but that was\n // unnecessarily optimistic: why allocate a large amount of memory\n // upfront for a large program when it is unlikely to be one-pass?\n absl::InlinedVector nodes;\n\n Instq tovisit(size), workq(size);\n AddQ(&tovisit, start());\n nodebyid[start()] = 0;\n int nalloc = 1;\n nodes.insert(nodes.end(), statesize, 0);\n for (Instq::iterator it = tovisit.begin(); it != tovisit.end(); ++it) {\n int id = *it;\n int nodeindex = nodebyid[id];\n OneState* node = IndexToNode(nodes.data(), statesize, nodeindex);\n\n // Flood graph using manual stack, filling in actions as found.\n // Default is none.\n for (int b = 0; b < bytemap_range_; b++)\n node->action[b] = kImpossible;\n node->matchcond = kImpossible;\n\n workq.clear();\n bool matched = false;\n int nstack = 0;\n stack[nstack].id = id;\n stack[nstack++].cond = 0;\n while (nstack > 0) {\n int id = stack[--nstack].id;\n uint32_t cond = stack[nstack].cond;\n\n Loop:\n Prog::Inst* ip = inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"unhandled opcode: \" << ip->opcode();\n break;\n\n case kInstAltMatch:\n // TODO(rsc): Ignoring kInstAltMatch optimization.\n // Should implement it in this engine, but it's subtle.\n ABSL_DCHECK(!ip->last());\n // If already on work queue, (1) is violated: bail out.\n if (!AddQ(&workq, id+1))\n goto fail;\n id = id+1;\n goto Loop;\n\n case kInstByteRange: {\n int nextindex = nodebyid[ip->out()];\n if (nextindex == -1) {\n if (nalloc >= maxnodes) {\n if (ExtraDebug)\n ABSL_LOG(ERROR) << absl::StrFormat(\n \"Not OnePass: hit node limit %d >= %d\", nalloc, maxnodes);\n goto fail;\n }\n nextindex = nalloc;\n AddQ(&tovisit, ip->out());\n nodebyid[ip->out()] = nalloc;\n nalloc++;\n nodes.insert(nodes.end(), statesize, 0);\n // Update node because it might have been invalidated.\n node = IndexToNode(nodes.data(), statesize, nodeindex);\n }\n for (int c = ip->lo(); c <= ip->hi(); c++) {\n int b = bytemap_[c];\n // Skip any bytes immediately after c that are also in b.\n while (c < 256-1 && bytemap_[c+1] == b)\n c++;\n uint32_t act = node->action[b];\n uint32_t newact = (nextindex << kIndexShift) | cond;\n if (matched)\n newact |= kMatchWins;\n if ((act & kImpossible) == kImpossible) {\n node->action[b] = newact;\n } else if (act != newact) {\n if (ExtraDebug)\n ABSL_LOG(ERROR) << absl::StrFormat(\n \"Not OnePass: conflict on byte %#x at state %d\", c, *it);\n goto fail;\n }\n }\n if (ip->foldcase()) {\n Rune lo = std::max(ip->lo(), 'a') + 'A' - 'a';\n Rune hi = std::min(ip->hi(), 'z') + 'A' - 'a';\n for (int c = lo; c <= hi; c++) {\n int b = bytemap_[c];\n // Skip any bytes immediately after c that are also in b.\n while (c < 256-1 && bytemap_[c+1] == b)\n c++;\n uint32_t act = node->action[b];\n uint32_t newact = (nextindex << kIndexShift) | cond;\n if (matched)\n newact |= kMatchWins;\n if ((act & kImpossible) == kImpossible) {\n node->action[b] = newact;\n } else if (act != newact) {\n if (ExtraDebug)\n ABSL_LOG(ERROR) << absl::StrFormat(\n \"Not OnePass: conflict on byte %#x at state %d\", c, *it);\n goto fail;\n }\n }\n }\n\n if (ip->last())\n break;\n // If already on work queue, (1) is violated: bail out.\n if (!AddQ(&workq, id+1))\n goto fail;\n id = id+1;\n goto Loop;\n }\n\n case kInstCapture:\n case kInstEmptyWidth:\n case kInstNop:\n if (!ip->last()) {\n // If already on work queue, (1) is violated: bail out.\n if (!AddQ(&workq, id+1))\n goto fail;\n stack[nstack].id = id+1;\n stack[nstack++].cond = cond;\n }\n\n if (ip->opcode() == kInstCapture && ip->cap() < kMaxCap)\n cond |= (1 << kCapShift) << ip->cap();\n if (ip->opcode() == kInstEmptyWidth)\n cond |= ip->empty();\n\n // kInstCapture and kInstNop always proceed to ip->out().\n // kInstEmptyWidth only sometimes proceeds to ip->out(),\n // but as a conservative approximation we assume it always does.\n // We could be a little more precise by looking at what c\n // is, but that seems like overkill.\n\n // If already on work queue, (1) is violated: bail out.\n if (!AddQ(&workq, ip->out())) {\n if (ExtraDebug)\n ABSL_LOG(ERROR) << absl::StrFormat(\n \"Not OnePass: multiple paths %d -> %d\", *it, ip->out());\n goto fail;\n }\n id = ip->out();\n goto Loop;\n\n case kInstMatch:\n if (matched) {\n // (3) is violated\n if (ExtraDebug)\n ABSL_LOG(ERROR) << absl::StrFormat(\n \"Not OnePass: multiple matches from %d\", *it);\n goto fail;\n }\n matched = true;\n node->matchcond = cond;\n\n if (ip->last())\n break;\n // If already on work queue, (1) is violated: bail out.\n if (!AddQ(&workq, id+1))\n goto fail;\n id = id+1;\n goto Loop;\n\n case kInstFail:\n break;\n }\n }\n }\n\n if (ExtraDebug) { // For debugging, dump one-pass NFA to ABSL_LOG(ERROR).\n ABSL_LOG(ERROR) << \"bytemap:\\n\" << DumpByteMap();\n ABSL_LOG(ERROR) << \"prog:\\n\" << Dump();\n\n std::map idmap;\n for (int i = 0; i < size; i++)\n if (nodebyid[i] != -1)\n idmap[nodebyid[i]] = i;\n\n std::string dump;\n for (Instq::iterator it = tovisit.begin(); it != tovisit.end(); ++it) {\n int id = *it;\n int nodeindex = nodebyid[id];\n if (nodeindex == -1)\n continue;\n OneState* node = IndexToNode(nodes.data(), statesize, nodeindex);\n dump += absl::StrFormat(\"node %d id=%d: matchcond=%#x\\n\",\n nodeindex, id, node->matchcond);\n for (int i = 0; i < bytemap_range_; i++) {\n if ((node->action[i] & kImpossible) == kImpossible)\n continue;\n dump += absl::StrFormat(\" %d cond %#x -> %d id=%d\\n\",\n i, node->action[i] & 0xFFFF,\n node->action[i] >> kIndexShift,\n idmap[node->action[i] >> kIndexShift]);\n }\n }\n ABSL_LOG(ERROR) << \"nodes:\\n\" << dump;\n }\n\n dfa_mem_ -= nalloc*statesize;\n onepass_nodes_ = PODArray(nalloc*statesize);\n memmove(onepass_nodes_.data(), nodes.data(), nalloc*statesize);\n return true;\n\nfail:\n return false;\n}\n\n} // namespace re2\n" }, { "path": "re2/parse.cc", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Regular expression parser.\n\n// The parser is a simple precedence-based parser with a\n// manual stack. The parsing work is done by the methods\n// of the ParseState class. The Regexp::Parse function is\n// essentially just a lexer that calls the ParseState method\n// for each token.\n\n// The parser recognizes POSIX extended regular expressions\n// excluding backreferences, collating elements, and collating\n// classes. It also allows the empty string as a regular expression\n// and recognizes the Perl escape sequences \\d, \\s, \\w, \\D, \\S, and \\W.\n// See regexp.h for rationale.\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/ascii.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/regexp.h\"\n#include \"re2/unicode_casefold.h\"\n#include \"re2/unicode_groups.h\"\n#include \"re2/walker-inl.h\"\n#include \"util/utf.h\"\n\n#if defined(RE2_USE_ICU)\n#include \"unicode/uniset.h\"\n#include \"unicode/unistr.h\"\n#include \"unicode/utypes.h\"\n#endif\n\nnamespace re2 {\n\n// Controls the maximum repeat count permitted by the parser.\nstatic int maximum_repeat_count = 1000;\n\nvoid Regexp::FUZZING_ONLY_set_maximum_repeat_count(int i) {\n maximum_repeat_count = i;\n}\n\n// Regular expression parse state.\n// The list of parsed regexps so far is maintained as a vector of\n// Regexp pointers called the stack. Left parenthesis and vertical\n// bar markers are also placed on the stack, as Regexps with\n// non-standard opcodes.\n// Scanning a left parenthesis causes the parser to push a left parenthesis\n// marker on the stack.\n// Scanning a vertical bar causes the parser to pop the stack until it finds a\n// vertical bar or left parenthesis marker (not popping the marker),\n// concatenate all the popped results, and push them back on\n// the stack (DoConcatenation).\n// Scanning a right parenthesis causes the parser to act as though it\n// has seen a vertical bar, which then leaves the top of the stack in the\n// form LeftParen regexp VerticalBar regexp VerticalBar ... regexp VerticalBar.\n// The parser pops all this off the stack and creates an alternation of the\n// regexps (DoAlternation).\n\nclass Regexp::ParseState {\n public:\n ParseState(ParseFlags flags, absl::string_view whole_regexp,\n RegexpStatus* status);\n ~ParseState();\n\n ParseFlags flags() { return flags_; }\n int rune_max() { return rune_max_; }\n\n // Parse methods. All public methods return a bool saying\n // whether parsing should continue. If a method returns\n // false, it has set fields in *status_, and the parser\n // should return NULL.\n\n // Pushes the given regular expression onto the stack.\n // Could check for too much memory used here.\n bool PushRegexp(Regexp* re);\n\n // Pushes the literal rune r onto the stack.\n bool PushLiteral(Rune r);\n\n // Pushes a regexp with the given op (and no args) onto the stack.\n bool PushSimpleOp(RegexpOp op);\n\n // Pushes a ^ onto the stack.\n bool PushCaret();\n\n // Pushes a \\b (word == true) or \\B (word == false) onto the stack.\n bool PushWordBoundary(bool word);\n\n // Pushes a $ onto the stack.\n bool PushDollar();\n\n // Pushes a . onto the stack\n bool PushDot();\n\n // Pushes a repeat operator regexp onto the stack.\n // A valid argument for the operator must already be on the stack.\n // s is the name of the operator, for use in error messages.\n bool PushRepeatOp(RegexpOp op, absl::string_view s, bool nongreedy);\n\n // Pushes a repetition regexp onto the stack.\n // A valid argument for the operator must already be on the stack.\n bool PushRepetition(int min, int max, absl::string_view s, bool nongreedy);\n\n // Checks whether a particular regexp op is a marker.\n bool IsMarker(RegexpOp op);\n\n // Processes a left parenthesis in the input.\n // Pushes a marker onto the stack.\n bool DoLeftParen(absl::string_view name);\n bool DoLeftParenNoCapture();\n\n // Processes a vertical bar in the input.\n bool DoVerticalBar();\n\n // Processes a right parenthesis in the input.\n bool DoRightParen();\n\n // Processes the end of input, returning the final regexp.\n Regexp* DoFinish();\n\n // Finishes the regexp if necessary, preparing it for use\n // in a more complicated expression.\n // If it is a CharClassBuilder, converts into a CharClass.\n Regexp* FinishRegexp(Regexp*);\n\n // These routines don't manipulate the parse stack\n // directly, but they do need to look at flags_.\n // ParseCharClass also manipulates the internals of Regexp\n // while creating *out_re.\n\n // Parse a character class into *out_re.\n // Removes parsed text from s.\n bool ParseCharClass(absl::string_view* s, Regexp** out_re,\n RegexpStatus* status);\n\n // Parse a character class character into *rp.\n // Removes parsed text from s.\n bool ParseCCCharacter(absl::string_view* s, Rune* rp,\n absl::string_view whole_class,\n RegexpStatus* status);\n\n // Parse a character class range into rr.\n // Removes parsed text from s.\n bool ParseCCRange(absl::string_view* s, RuneRange* rr,\n absl::string_view whole_class,\n RegexpStatus* status);\n\n // Parse a Perl flag set or non-capturing group from s.\n bool ParsePerlFlags(absl::string_view* s);\n\n // Finishes the current concatenation,\n // collapsing it into a single regexp on the stack.\n void DoConcatenation();\n\n // Finishes the current alternation,\n // collapsing it to a single regexp on the stack.\n void DoAlternation();\n\n // Generalized DoAlternation/DoConcatenation.\n void DoCollapse(RegexpOp op);\n\n // Maybe concatenate Literals into LiteralString.\n bool MaybeConcatString(int r, ParseFlags flags);\n\nprivate:\n ParseFlags flags_;\n absl::string_view whole_regexp_;\n RegexpStatus* status_;\n Regexp* stacktop_;\n int ncap_; // number of capturing parens seen\n int rune_max_; // maximum char value for this encoding\n\n ParseState(const ParseState&) = delete;\n ParseState& operator=(const ParseState&) = delete;\n};\n\n// Pseudo-operators - only on parse stack.\nconst RegexpOp kLeftParen = static_cast(kMaxRegexpOp+1);\nconst RegexpOp kVerticalBar = static_cast(kMaxRegexpOp+2);\n\nRegexp::ParseState::ParseState(ParseFlags flags,\n absl::string_view whole_regexp,\n RegexpStatus* status)\n : flags_(flags), whole_regexp_(whole_regexp),\n status_(status), stacktop_(NULL), ncap_(0) {\n if (flags_ & Latin1)\n rune_max_ = 0xFF;\n else\n rune_max_ = Runemax;\n}\n\n// Cleans up by freeing all the regexps on the stack.\nRegexp::ParseState::~ParseState() {\n Regexp* next;\n for (Regexp* re = stacktop_; re != NULL; re = next) {\n next = re->down_;\n re->down_ = NULL;\n if (re->op() == kLeftParen)\n delete re->name_;\n re->Decref();\n }\n}\n\n// Finishes the regexp if necessary, preparing it for use in\n// a more complex expression.\n// If it is a CharClassBuilder, converts into a CharClass.\nRegexp* Regexp::ParseState::FinishRegexp(Regexp* re) {\n if (re == NULL)\n return NULL;\n re->down_ = NULL;\n\n if (re->op_ == kRegexpCharClass && re->ccb_ != NULL) {\n CharClassBuilder* ccb = re->ccb_;\n re->ccb_ = NULL;\n re->cc_ = ccb->GetCharClass();\n delete ccb;\n }\n\n return re;\n}\n\n// Pushes the given regular expression onto the stack.\n// Could check for too much memory used here.\nbool Regexp::ParseState::PushRegexp(Regexp* re) {\n MaybeConcatString(-1, NoParseFlags);\n\n // Special case: a character class of one character is just\n // a literal. This is a common idiom for escaping\n // single characters (e.g., [.] instead of \\.), and some\n // analysis does better with fewer character classes.\n // Similarly, [Aa] can be rewritten as a literal A with ASCII case folding.\n if (re->op_ == kRegexpCharClass && re->ccb_ != NULL) {\n re->ccb_->RemoveAbove(rune_max_);\n if (re->ccb_->size() == 1) {\n Rune r = re->ccb_->begin()->lo;\n re->Decref();\n re = new Regexp(kRegexpLiteral, flags_);\n re->rune_ = r;\n } else if (re->ccb_->size() == 2) {\n Rune r = re->ccb_->begin()->lo;\n if ('A' <= r && r <= 'Z' && re->ccb_->Contains(r + 'a' - 'A')) {\n re->Decref();\n re = new Regexp(kRegexpLiteral, flags_ | FoldCase);\n re->rune_ = r + 'a' - 'A';\n }\n }\n }\n\n if (!IsMarker(re->op()))\n re->simple_ = re->ComputeSimple();\n re->down_ = stacktop_;\n stacktop_ = re;\n return true;\n}\n\n// Searches the case folding tables and returns the CaseFold* that contains r.\n// If there isn't one, returns the CaseFold* with smallest f->lo bigger than r.\n// If there isn't one, returns NULL.\nconst CaseFold* LookupCaseFold(const CaseFold* f, int n, Rune r) {\n const CaseFold* ef = f + n;\n\n // Binary search for entry containing r.\n while (n > 0) {\n int m = n/2;\n if (f[m].lo <= r && r <= f[m].hi)\n return &f[m];\n if (r < f[m].lo) {\n n = m;\n } else {\n f += m+1;\n n -= m+1;\n }\n }\n\n // There is no entry that contains r, but f points\n // where it would have been. Unless f points at\n // the end of the array, it points at the next entry\n // after r.\n if (f < ef)\n return f;\n\n // No entry contains r; no entry contains runes > r.\n return NULL;\n}\n\n// Returns the result of applying the fold f to the rune r.\nRune ApplyFold(const CaseFold* f, Rune r) {\n switch (f->delta) {\n default:\n return r + f->delta;\n\n case EvenOddSkip: // even <-> odd but only applies to every other\n if ((r - f->lo) % 2)\n return r;\n [[fallthrough]];\n case EvenOdd: // even <-> odd\n if (r%2 == 0)\n return r + 1;\n return r - 1;\n\n case OddEvenSkip: // odd <-> even but only applies to every other\n if ((r - f->lo) % 2)\n return r;\n [[fallthrough]];\n case OddEven: // odd <-> even\n if (r%2 == 1)\n return r + 1;\n return r - 1;\n }\n}\n\n// Returns the next Rune in r's folding cycle (see unicode_casefold.h).\n// Examples:\n// CycleFoldRune('A') = 'a'\n// CycleFoldRune('a') = 'A'\n//\n// CycleFoldRune('K') = 'k'\n// CycleFoldRune('k') = 0x212A (Kelvin)\n// CycleFoldRune(0x212A) = 'K'\n//\n// CycleFoldRune('?') = '?'\nRune CycleFoldRune(Rune r) {\n const CaseFold* f = LookupCaseFold(unicode_casefold, num_unicode_casefold, r);\n if (f == NULL || r < f->lo)\n return r;\n return ApplyFold(f, r);\n}\n\n// Add lo-hi to the class, along with their fold-equivalent characters.\nstatic void AddFoldedRangeLatin1(CharClassBuilder* cc, Rune lo, Rune hi) {\n while (lo <= hi) {\n cc->AddRange(lo, lo);\n if ('A' <= lo && lo <= 'Z') {\n cc->AddRange(lo - 'A' + 'a', lo - 'A' + 'a');\n }\n if ('a' <= lo && lo <= 'z') {\n cc->AddRange(lo - 'a' + 'A', lo - 'a' + 'A');\n }\n lo++;\n }\n}\n\n// Add lo-hi to the class, along with their fold-equivalent characters.\n// If lo-hi is already in the class, assume that the fold-equivalent\n// chars are there too, so there's no work to do.\nstatic void AddFoldedRange(CharClassBuilder* cc, Rune lo, Rune hi, int depth) {\n // AddFoldedRange calls itself recursively for each rune in the fold cycle.\n // Most folding cycles are small: there aren't any bigger than four in the\n // current Unicode tables. make_unicode_casefold.py checks that\n // the cycles are not too long, and we double-check here using depth.\n if (depth > 10) {\n ABSL_LOG(DFATAL) << \"AddFoldedRange recurses too much.\";\n return;\n }\n\n if (!cc->AddRange(lo, hi)) // lo-hi was already there? we're done\n return;\n\n while (lo <= hi) {\n const CaseFold* f = LookupCaseFold(unicode_casefold, num_unicode_casefold, lo);\n if (f == NULL) // lo has no fold, nor does anything above lo\n break;\n if (lo < f->lo) { // lo has no fold; next rune with a fold is f->lo\n lo = f->lo;\n continue;\n }\n\n // Add in the result of folding the range lo - f->hi\n // and that range's fold, recursively.\n Rune lo1 = lo;\n Rune hi1 = std::min(hi, f->hi);\n switch (f->delta) {\n default:\n lo1 += f->delta;\n hi1 += f->delta;\n break;\n case EvenOdd:\n if (lo1%2 == 1)\n lo1--;\n if (hi1%2 == 0)\n hi1++;\n break;\n case OddEven:\n if (lo1%2 == 0)\n lo1--;\n if (hi1%2 == 1)\n hi1++;\n break;\n }\n AddFoldedRange(cc, lo1, hi1, depth+1);\n\n // Pick up where this fold left off.\n lo = f->hi + 1;\n }\n}\n\n// Pushes the literal rune r onto the stack.\nbool Regexp::ParseState::PushLiteral(Rune r) {\n // Do case folding if needed.\n if (flags_ & FoldCase) {\n if (flags_ & Latin1 && (('A' <= r && r <= 'Z') ||\n ('a' <= r && r <= 'z'))) {\n Regexp* re = new Regexp(kRegexpCharClass, flags_ & ~FoldCase);\n re->ccb_ = new CharClassBuilder;\n AddFoldedRangeLatin1(re->ccb_, r, r);\n return PushRegexp(re);\n }\n if (!(flags_ & Latin1) && CycleFoldRune(r) != r) {\n Regexp* re = new Regexp(kRegexpCharClass, flags_ & ~FoldCase);\n re->ccb_ = new CharClassBuilder;\n Rune r1 = r;\n do {\n if (!(flags_ & NeverNL) || r != '\\n') {\n re->ccb_->AddRange(r, r);\n }\n r = CycleFoldRune(r);\n } while (r != r1);\n return PushRegexp(re);\n }\n }\n\n // Exclude newline if applicable.\n if ((flags_ & NeverNL) && r == '\\n')\n return PushRegexp(new Regexp(kRegexpNoMatch, flags_));\n\n // No fancy stuff worked. Ordinary literal.\n if (MaybeConcatString(r, flags_))\n return true;\n\n Regexp* re = new Regexp(kRegexpLiteral, flags_);\n re->rune_ = r;\n return PushRegexp(re);\n}\n\n// Pushes a ^ onto the stack.\nbool Regexp::ParseState::PushCaret() {\n if (flags_ & OneLine) {\n return PushSimpleOp(kRegexpBeginText);\n }\n return PushSimpleOp(kRegexpBeginLine);\n}\n\n// Pushes a \\b or \\B onto the stack.\nbool Regexp::ParseState::PushWordBoundary(bool word) {\n if (word)\n return PushSimpleOp(kRegexpWordBoundary);\n return PushSimpleOp(kRegexpNoWordBoundary);\n}\n\n// Pushes a $ onto the stack.\nbool Regexp::ParseState::PushDollar() {\n if (flags_ & OneLine) {\n // Clumsy marker so that MimicsPCRE() can tell whether\n // this kRegexpEndText was a $ and not a \\z.\n Regexp::ParseFlags oflags = flags_;\n flags_ = flags_ | WasDollar;\n bool ret = PushSimpleOp(kRegexpEndText);\n flags_ = oflags;\n return ret;\n }\n return PushSimpleOp(kRegexpEndLine);\n}\n\n// Pushes a . onto the stack.\nbool Regexp::ParseState::PushDot() {\n if ((flags_ & DotNL) && !(flags_ & NeverNL))\n return PushSimpleOp(kRegexpAnyChar);\n // Rewrite . into [^\\n]\n Regexp* re = new Regexp(kRegexpCharClass, flags_ & ~FoldCase);\n re->ccb_ = new CharClassBuilder;\n re->ccb_->AddRange(0, '\\n' - 1);\n re->ccb_->AddRange('\\n' + 1, rune_max_);\n return PushRegexp(re);\n}\n\n// Pushes a regexp with the given op (and no args) onto the stack.\nbool Regexp::ParseState::PushSimpleOp(RegexpOp op) {\n Regexp* re = new Regexp(op, flags_);\n return PushRegexp(re);\n}\n\n// Pushes a repeat operator regexp onto the stack.\n// A valid argument for the operator must already be on the stack.\n// The char c is the name of the operator, for use in error messages.\nbool Regexp::ParseState::PushRepeatOp(RegexpOp op, absl::string_view s,\n bool nongreedy) {\n if (stacktop_ == NULL || IsMarker(stacktop_->op())) {\n status_->set_code(kRegexpRepeatArgument);\n status_->set_error_arg(s);\n return false;\n }\n Regexp::ParseFlags fl = flags_;\n if (nongreedy)\n fl = fl ^ NonGreedy;\n\n // Squash **, ++ and ??. Regexp::Star() et al. handle this too, but\n // they're mostly for use during simplification, not during parsing.\n if (op == stacktop_->op() && fl == stacktop_->parse_flags())\n return true;\n\n // Squash *+, *?, +*, +?, ?* and ?+. They all squash to *, so because\n // op is a repeat, we just have to check that stacktop_->op() is too,\n // then adjust stacktop_.\n if ((stacktop_->op() == kRegexpStar ||\n stacktop_->op() == kRegexpPlus ||\n stacktop_->op() == kRegexpQuest) &&\n fl == stacktop_->parse_flags()) {\n stacktop_->op_ = kRegexpStar;\n return true;\n }\n\n Regexp* re = new Regexp(op, fl);\n re->AllocSub(1);\n re->down_ = stacktop_->down_;\n re->sub()[0] = FinishRegexp(stacktop_);\n re->simple_ = re->ComputeSimple();\n stacktop_ = re;\n return true;\n}\n\n// RepetitionWalker reports whether the repetition regexp is valid.\n// Valid means that the combination of the top-level repetition\n// and any inner repetitions does not exceed n copies of the\n// innermost thing.\n// This rewalks the regexp tree and is called for every repetition,\n// so we have to worry about inducing quadratic behavior in the parser.\n// We avoid this by only using RepetitionWalker when min or max >= 2.\n// In that case the depth of any >= 2 nesting can only get to 9 without\n// triggering a parse error, so each subtree can only be rewalked 9 times.\nclass RepetitionWalker : public Regexp::Walker {\n public:\n RepetitionWalker() {}\n virtual int PreVisit(Regexp* re, int parent_arg, bool* stop);\n virtual int PostVisit(Regexp* re, int parent_arg, int pre_arg,\n int* child_args, int nchild_args);\n virtual int ShortVisit(Regexp* re, int parent_arg);\n\n private:\n RepetitionWalker(const RepetitionWalker&) = delete;\n RepetitionWalker& operator=(const RepetitionWalker&) = delete;\n};\n\nint RepetitionWalker::PreVisit(Regexp* re, int parent_arg, bool* stop) {\n int arg = parent_arg;\n if (re->op() == kRegexpRepeat) {\n int m = re->max();\n if (m < 0) {\n m = re->min();\n }\n if (m > 0) {\n arg /= m;\n }\n }\n return arg;\n}\n\nint RepetitionWalker::PostVisit(Regexp* re, int parent_arg, int pre_arg,\n int* child_args, int nchild_args) {\n int arg = pre_arg;\n for (int i = 0; i < nchild_args; i++) {\n if (child_args[i] < arg) {\n arg = child_args[i];\n }\n }\n return arg;\n}\n\nint RepetitionWalker::ShortVisit(Regexp* re, int parent_arg) {\n // Should never be called: we use Walk(), not WalkExponential().\n#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION\n ABSL_LOG(DFATAL) << \"RepetitionWalker::ShortVisit called\";\n#endif\n return 0;\n}\n\n// Pushes a repetition regexp onto the stack.\n// A valid argument for the operator must already be on the stack.\nbool Regexp::ParseState::PushRepetition(int min, int max, absl::string_view s,\n bool nongreedy) {\n if ((max != -1 && max < min) ||\n min > maximum_repeat_count ||\n max > maximum_repeat_count) {\n status_->set_code(kRegexpRepeatSize);\n status_->set_error_arg(s);\n return false;\n }\n if (stacktop_ == NULL || IsMarker(stacktop_->op())) {\n status_->set_code(kRegexpRepeatArgument);\n status_->set_error_arg(s);\n return false;\n }\n Regexp::ParseFlags fl = flags_;\n if (nongreedy)\n fl = fl ^ NonGreedy;\n Regexp* re = new Regexp(kRegexpRepeat, fl);\n re->min_ = min;\n re->max_ = max;\n re->AllocSub(1);\n re->down_ = stacktop_->down_;\n re->sub()[0] = FinishRegexp(stacktop_);\n re->simple_ = re->ComputeSimple();\n stacktop_ = re;\n if (min >= 2 || max >= 2) {\n RepetitionWalker w;\n if (w.Walk(stacktop_, maximum_repeat_count) == 0) {\n status_->set_code(kRegexpRepeatSize);\n status_->set_error_arg(s);\n return false;\n }\n }\n return true;\n}\n\n// Checks whether a particular regexp op is a marker.\nbool Regexp::ParseState::IsMarker(RegexpOp op) {\n return op >= kLeftParen;\n}\n\n// Processes a left parenthesis in the input.\n// Pushes a marker onto the stack.\nbool Regexp::ParseState::DoLeftParen(absl::string_view name) {\n Regexp* re = new Regexp(kLeftParen, flags_);\n re->cap_ = ++ncap_;\n if (name.data() != NULL)\n re->name_ = new std::string(name);\n return PushRegexp(re);\n}\n\n// Pushes a non-capturing marker onto the stack.\nbool Regexp::ParseState::DoLeftParenNoCapture() {\n Regexp* re = new Regexp(kLeftParen, flags_);\n re->cap_ = -1;\n return PushRegexp(re);\n}\n\n// Processes a vertical bar in the input.\nbool Regexp::ParseState::DoVerticalBar() {\n MaybeConcatString(-1, NoParseFlags);\n DoConcatenation();\n\n // Below the vertical bar is a list to alternate.\n // Above the vertical bar is a list to concatenate.\n // We just did the concatenation, so either swap\n // the result below the vertical bar or push a new\n // vertical bar on the stack.\n Regexp* r1;\n Regexp* r2;\n if ((r1 = stacktop_) != NULL &&\n (r2 = r1->down_) != NULL &&\n r2->op() == kVerticalBar) {\n Regexp* r3;\n if ((r3 = r2->down_) != NULL &&\n (r1->op() == kRegexpAnyChar || r3->op() == kRegexpAnyChar)) {\n // AnyChar is above or below the vertical bar. Let it subsume\n // the other when the other is Literal, CharClass or AnyChar.\n if (r3->op() == kRegexpAnyChar &&\n (r1->op() == kRegexpLiteral ||\n r1->op() == kRegexpCharClass ||\n r1->op() == kRegexpAnyChar)) {\n // Discard r1.\n stacktop_ = r2;\n r1->Decref();\n return true;\n }\n if (r1->op() == kRegexpAnyChar &&\n (r3->op() == kRegexpLiteral ||\n r3->op() == kRegexpCharClass ||\n r3->op() == kRegexpAnyChar)) {\n // Rearrange the stack and discard r3.\n r1->down_ = r3->down_;\n r2->down_ = r1;\n stacktop_ = r2;\n r3->Decref();\n return true;\n }\n }\n // Swap r1 below vertical bar (r2).\n r1->down_ = r2->down_;\n r2->down_ = r1;\n stacktop_ = r2;\n return true;\n }\n return PushSimpleOp(kVerticalBar);\n}\n\n// Processes a right parenthesis in the input.\nbool Regexp::ParseState::DoRightParen() {\n // Finish the current concatenation and alternation.\n DoAlternation();\n\n // The stack should be: LeftParen regexp\n // Remove the LeftParen, leaving the regexp,\n // parenthesized.\n Regexp* r1;\n Regexp* r2;\n if ((r1 = stacktop_) == NULL ||\n (r2 = r1->down_) == NULL ||\n r2->op() != kLeftParen) {\n status_->set_code(kRegexpUnexpectedParen);\n status_->set_error_arg(whole_regexp_);\n return false;\n }\n\n // Pop off r1, r2. Will Decref or reuse below.\n stacktop_ = r2->down_;\n\n // Restore flags from when paren opened.\n Regexp* re = r2;\n flags_ = re->parse_flags();\n\n // Rewrite LeftParen as capture if needed.\n if (re->cap_ > 0) {\n re->op_ = kRegexpCapture;\n // re->cap_ is already set\n re->AllocSub(1);\n re->sub()[0] = FinishRegexp(r1);\n re->simple_ = re->ComputeSimple();\n } else {\n re->Decref();\n re = r1;\n }\n return PushRegexp(re);\n}\n\n// Processes the end of input, returning the final regexp.\nRegexp* Regexp::ParseState::DoFinish() {\n DoAlternation();\n Regexp* re = stacktop_;\n if (re != NULL && re->down_ != NULL) {\n status_->set_code(kRegexpMissingParen);\n status_->set_error_arg(whole_regexp_);\n return NULL;\n }\n stacktop_ = NULL;\n return FinishRegexp(re);\n}\n\n// Returns the leading regexp that re starts with.\n// The returned Regexp* points into a piece of re,\n// so it must not be used after the caller calls re->Decref().\nRegexp* Regexp::LeadingRegexp(Regexp* re) {\n if (re->op() == kRegexpEmptyMatch)\n return NULL;\n if (re->op() == kRegexpConcat && re->nsub() >= 2) {\n Regexp** sub = re->sub();\n if (sub[0]->op() == kRegexpEmptyMatch)\n return NULL;\n return sub[0];\n }\n return re;\n}\n\n// Removes LeadingRegexp(re) from re and returns what's left.\n// Consumes the reference to re and may edit it in place.\n// If caller wants to hold on to LeadingRegexp(re),\n// must have already Incref'ed it.\nRegexp* Regexp::RemoveLeadingRegexp(Regexp* re) {\n if (re->op() == kRegexpEmptyMatch)\n return re;\n if (re->op() == kRegexpConcat && re->nsub() >= 2) {\n Regexp** sub = re->sub();\n if (sub[0]->op() == kRegexpEmptyMatch)\n return re;\n sub[0]->Decref();\n sub[0] = NULL;\n if (re->nsub() == 2) {\n // Collapse concatenation to single regexp.\n Regexp* nre = sub[1];\n sub[1] = NULL;\n re->Decref();\n return nre;\n }\n // 3 or more -> 2 or more.\n re->nsub_--;\n memmove(sub, sub + 1, re->nsub_ * sizeof sub[0]);\n return re;\n }\n Regexp::ParseFlags pf = re->parse_flags();\n re->Decref();\n return new Regexp(kRegexpEmptyMatch, pf);\n}\n\n// Returns the leading string that re starts with.\n// The returned Rune* points into a piece of re,\n// so it must not be used after the caller calls re->Decref().\nRune* Regexp::LeadingString(Regexp* re, int* nrune,\n Regexp::ParseFlags* flags) {\n while (re->op() == kRegexpConcat && re->nsub() > 0)\n re = re->sub()[0];\n\n *flags = static_cast(re->parse_flags_ &\n (Regexp::FoldCase | Regexp::Latin1));\n\n if (re->op() == kRegexpLiteral) {\n *nrune = 1;\n return &re->rune_;\n }\n\n if (re->op() == kRegexpLiteralString) {\n *nrune = re->nrunes_;\n return re->runes_;\n }\n\n *nrune = 0;\n return NULL;\n}\n\n// Removes the first n leading runes from the beginning of re.\n// Edits re in place.\nvoid Regexp::RemoveLeadingString(Regexp* re, int n) {\n // Chase down concats to find first string.\n // For regexps generated by parser, nested concats are\n // flattened except when doing so would overflow the 16-bit\n // limit on the size of a concatenation, so we should never\n // see more than two here.\n Regexp* stk[4];\n size_t d = 0;\n while (re->op() == kRegexpConcat) {\n if (d < ABSL_ARRAYSIZE(stk))\n stk[d++] = re;\n re = re->sub()[0];\n }\n\n // Remove leading string from re.\n if (re->op() == kRegexpLiteral) {\n re->rune_ = 0;\n re->op_ = kRegexpEmptyMatch;\n } else if (re->op() == kRegexpLiteralString) {\n if (n >= re->nrunes_) {\n delete[] re->runes_;\n re->runes_ = NULL;\n re->nrunes_ = 0;\n re->op_ = kRegexpEmptyMatch;\n } else if (n == re->nrunes_ - 1) {\n Rune rune = re->runes_[re->nrunes_ - 1];\n delete[] re->runes_;\n re->runes_ = NULL;\n re->nrunes_ = 0;\n re->rune_ = rune;\n re->op_ = kRegexpLiteral;\n } else {\n re->nrunes_ -= n;\n memmove(re->runes_, re->runes_ + n, re->nrunes_ * sizeof re->runes_[0]);\n }\n }\n\n // If re is now empty, concatenations might simplify too.\n while (d > 0) {\n re = stk[--d];\n Regexp** sub = re->sub();\n if (sub[0]->op() == kRegexpEmptyMatch) {\n sub[0]->Decref();\n sub[0] = NULL;\n // Delete first element of concat.\n switch (re->nsub()) {\n case 0:\n case 1:\n // Impossible.\n ABSL_LOG(DFATAL) << \"Concat of \" << re->nsub();\n re->submany_ = NULL;\n re->op_ = kRegexpEmptyMatch;\n break;\n\n case 2: {\n // Replace re with sub[1].\n Regexp* old = sub[1];\n sub[1] = NULL;\n re->Swap(old);\n old->Decref();\n break;\n }\n\n default:\n // Slide down.\n re->nsub_--;\n memmove(sub, sub + 1, re->nsub_ * sizeof sub[0]);\n break;\n }\n }\n }\n}\n\n// In the context of factoring alternations, a Splice is: a factored prefix or\n// merged character class computed by one iteration of one round of factoring;\n// the span of subexpressions of the alternation to be \"spliced\" (i.e. removed\n// and replaced); and, for a factored prefix, the number of suffixes after any\n// factoring that might have subsequently been performed on them. For a merged\n// character class, there are no suffixes, of course, so the field is ignored.\nstruct Splice {\n Splice(Regexp* prefix, Regexp** sub, int nsub)\n : prefix(prefix),\n sub(sub),\n nsub(nsub),\n nsuffix(-1) {}\n\n Regexp* prefix;\n Regexp** sub;\n int nsub;\n int nsuffix;\n};\n\n// Named so because it is used to implement an explicit stack, a Frame is: the\n// span of subexpressions of the alternation to be factored; the current round\n// of factoring; any Splices computed; and, for a factored prefix, an iterator\n// to the next Splice to be factored (i.e. in another Frame) because suffixes.\nstruct Frame {\n Frame(Regexp** sub, int nsub)\n : sub(sub),\n nsub(nsub),\n round(0) {}\n\n Regexp** sub;\n int nsub;\n int round;\n std::vector splices;\n int spliceidx;\n};\n\n// Bundled into a class for friend access to Regexp without needing to declare\n// (or define) Splice in regexp.h.\nclass FactorAlternationImpl {\n public:\n static void Round1(Regexp** sub, int nsub,\n Regexp::ParseFlags flags,\n std::vector* splices);\n static void Round2(Regexp** sub, int nsub,\n Regexp::ParseFlags flags,\n std::vector* splices);\n static void Round3(Regexp** sub, int nsub,\n Regexp::ParseFlags flags,\n std::vector* splices);\n};\n\n// Factors common prefixes from alternation.\n// For example,\n// ABC|ABD|AEF|BCX|BCY\n// simplifies to\n// A(B(C|D)|EF)|BC(X|Y)\n// and thence to\n// A(B[CD]|EF)|BC[XY]\n//\n// Rewrites sub to contain simplified list to alternate and returns\n// the new length of sub. Adjusts reference counts accordingly\n// (incoming sub[i] decremented, outgoing sub[i] incremented).\nint Regexp::FactorAlternation(Regexp** sub, int nsub, ParseFlags flags) {\n std::vector stk;\n stk.emplace_back(sub, nsub);\n\n for (;;) {\n auto& sub = stk.back().sub;\n auto& nsub = stk.back().nsub;\n auto& round = stk.back().round;\n auto& splices = stk.back().splices;\n auto& spliceidx = stk.back().spliceidx;\n\n if (splices.empty()) {\n // Advance to the next round of factoring. Note that this covers\n // the initialised state: when splices is empty and round is 0.\n round++;\n } else if (spliceidx < static_cast(splices.size())) {\n // We have at least one more Splice to factor. Recurse logically.\n stk.emplace_back(splices[spliceidx].sub, splices[spliceidx].nsub);\n continue;\n } else {\n // We have no more Splices to factor. Apply them.\n auto iter = splices.begin();\n int out = 0;\n for (int i = 0; i < nsub; ) {\n // Copy until we reach where the next Splice begins.\n while (sub + i < iter->sub)\n sub[out++] = sub[i++];\n switch (round) {\n case 1:\n case 2: {\n // Assemble the Splice prefix and the suffixes.\n Regexp* re[2];\n re[0] = iter->prefix;\n re[1] = Regexp::AlternateNoFactor(iter->sub, iter->nsuffix, flags);\n sub[out++] = Regexp::Concat(re, 2, flags);\n i += iter->nsub;\n break;\n }\n case 3:\n // Just use the Splice prefix.\n sub[out++] = iter->prefix;\n i += iter->nsub;\n break;\n default:\n ABSL_LOG(DFATAL) << \"unknown round: \" << round;\n break;\n }\n // If we are done, copy until the end of sub.\n if (++iter == splices.end()) {\n while (i < nsub)\n sub[out++] = sub[i++];\n }\n }\n splices.clear();\n nsub = out;\n // Advance to the next round of factoring.\n round++;\n }\n\n switch (round) {\n case 1:\n FactorAlternationImpl::Round1(sub, nsub, flags, &splices);\n break;\n case 2:\n FactorAlternationImpl::Round2(sub, nsub, flags, &splices);\n break;\n case 3:\n FactorAlternationImpl::Round3(sub, nsub, flags, &splices);\n break;\n case 4:\n if (stk.size() == 1) {\n // We are at the top of the stack. Just return.\n return nsub;\n } else {\n // Pop the stack and set the number of suffixes.\n // (Note that references will be invalidated!)\n int nsuffix = nsub;\n stk.pop_back();\n stk.back().splices[stk.back().spliceidx].nsuffix = nsuffix;\n ++stk.back().spliceidx;\n continue;\n }\n default:\n ABSL_LOG(DFATAL) << \"unknown round: \" << round;\n break;\n }\n\n // Set spliceidx depending on whether we have Splices to factor.\n if (splices.empty() || round == 3) {\n spliceidx = static_cast(splices.size());\n } else {\n spliceidx = 0;\n }\n }\n}\n\nvoid FactorAlternationImpl::Round1(Regexp** sub, int nsub,\n Regexp::ParseFlags flags,\n std::vector* splices) {\n // Round 1: Factor out common literal prefixes.\n int start = 0;\n Rune* rune = NULL;\n int nrune = 0;\n Regexp::ParseFlags runeflags = Regexp::NoParseFlags;\n for (int i = 0; i <= nsub; i++) {\n // Invariant: sub[start:i] consists of regexps that all\n // begin with rune[0:nrune].\n Rune* rune_i = NULL;\n int nrune_i = 0;\n Regexp::ParseFlags runeflags_i = Regexp::NoParseFlags;\n if (i < nsub) {\n rune_i = Regexp::LeadingString(sub[i], &nrune_i, &runeflags_i);\n if (runeflags_i == runeflags) {\n int same = 0;\n while (same < nrune && same < nrune_i && rune[same] == rune_i[same])\n same++;\n if (same > 0) {\n // Matches at least one rune in current range. Keep going around.\n nrune = same;\n continue;\n }\n }\n }\n\n // Found end of a run with common leading literal string:\n // sub[start:i] all begin with rune[0:nrune],\n // but sub[i] does not even begin with rune[0].\n if (i == start) {\n // Nothing to do - first iteration.\n } else if (i == start+1) {\n // Just one: don't bother factoring.\n } else {\n Regexp* prefix = Regexp::LiteralString(rune, nrune, runeflags);\n for (int j = start; j < i; j++)\n Regexp::RemoveLeadingString(sub[j], nrune);\n splices->emplace_back(prefix, sub + start, i - start);\n }\n\n // Prepare for next iteration (if there is one).\n if (i < nsub) {\n start = i;\n rune = rune_i;\n nrune = nrune_i;\n runeflags = runeflags_i;\n }\n }\n}\n\nvoid FactorAlternationImpl::Round2(Regexp** sub, int nsub,\n Regexp::ParseFlags flags,\n std::vector* splices) {\n // Round 2: Factor out common simple prefixes,\n // just the first piece of each concatenation.\n // This will be good enough a lot of the time.\n //\n // Complex subexpressions (e.g. involving quantifiers)\n // are not safe to factor because that collapses their\n // distinct paths through the automaton, which affects\n // correctness in some cases.\n int start = 0;\n Regexp* first = NULL;\n for (int i = 0; i <= nsub; i++) {\n // Invariant: sub[start:i] consists of regexps that all\n // begin with first.\n Regexp* first_i = NULL;\n if (i < nsub) {\n first_i = Regexp::LeadingRegexp(sub[i]);\n if (first != NULL &&\n // first must be an empty-width op\n // OR a char class, any char or any byte\n // OR a fixed repeat of a literal, char class, any char or any byte.\n (first->op() == kRegexpBeginLine ||\n first->op() == kRegexpEndLine ||\n first->op() == kRegexpWordBoundary ||\n first->op() == kRegexpNoWordBoundary ||\n first->op() == kRegexpBeginText ||\n first->op() == kRegexpEndText ||\n first->op() == kRegexpCharClass ||\n first->op() == kRegexpAnyChar ||\n first->op() == kRegexpAnyByte ||\n (first->op() == kRegexpRepeat &&\n first->min() == first->max() &&\n (first->sub()[0]->op() == kRegexpLiteral ||\n first->sub()[0]->op() == kRegexpCharClass ||\n first->sub()[0]->op() == kRegexpAnyChar ||\n first->sub()[0]->op() == kRegexpAnyByte))) &&\n Regexp::Equal(first, first_i))\n continue;\n }\n\n // Found end of a run with common leading regexp:\n // sub[start:i] all begin with first,\n // but sub[i] does not.\n if (i == start) {\n // Nothing to do - first iteration.\n } else if (i == start+1) {\n // Just one: don't bother factoring.\n } else {\n Regexp* prefix = first->Incref();\n for (int j = start; j < i; j++)\n sub[j] = Regexp::RemoveLeadingRegexp(sub[j]);\n splices->emplace_back(prefix, sub + start, i - start);\n }\n\n // Prepare for next iteration (if there is one).\n if (i < nsub) {\n start = i;\n first = first_i;\n }\n }\n}\n\nvoid FactorAlternationImpl::Round3(Regexp** sub, int nsub,\n Regexp::ParseFlags flags,\n std::vector* splices) {\n // Round 3: Merge runs of literals and/or character classes.\n int start = 0;\n Regexp* first = NULL;\n for (int i = 0; i <= nsub; i++) {\n // Invariant: sub[start:i] consists of regexps that all\n // are either literals (i.e. runes) or character classes.\n Regexp* first_i = NULL;\n if (i < nsub) {\n first_i = sub[i];\n if (first != NULL &&\n (first->op() == kRegexpLiteral ||\n first->op() == kRegexpCharClass) &&\n (first_i->op() == kRegexpLiteral ||\n first_i->op() == kRegexpCharClass))\n continue;\n }\n\n // Found end of a run of Literal/CharClass:\n // sub[start:i] all are either one or the other,\n // but sub[i] is not.\n if (i == start) {\n // Nothing to do - first iteration.\n } else if (i == start+1) {\n // Just one: don't bother factoring.\n } else {\n CharClassBuilder ccb;\n for (int j = start; j < i; j++) {\n Regexp* re = sub[j];\n if (re->op() == kRegexpCharClass) {\n CharClass* cc = re->cc();\n for (CharClass::iterator it = cc->begin(); it != cc->end(); ++it)\n ccb.AddRangeFlags(it->lo, it->hi, re->parse_flags());\n } else if (re->op() == kRegexpLiteral) {\n if (re->parse_flags() & Regexp::FoldCase) {\n // AddFoldedRange() can terminate prematurely if the character class\n // already contains the rune. For example, if it contains 'a' and we\n // want to add folded 'a', it sees 'a' and stops without adding 'A'.\n // To avoid that, we use an empty character class and then merge it.\n CharClassBuilder tmp;\n tmp.AddRangeFlags(re->rune(), re->rune(), re->parse_flags());\n ccb.AddCharClass(&tmp);\n } else {\n ccb.AddRangeFlags(re->rune(), re->rune(), re->parse_flags());\n }\n } else {\n ABSL_LOG(DFATAL) << \"RE2: unexpected op: \" << re->op() << \" \"\n << re->ToString();\n }\n re->Decref();\n }\n Regexp* re = Regexp::NewCharClass(ccb.GetCharClass(), flags & ~Regexp::FoldCase);\n splices->emplace_back(re, sub + start, i - start);\n }\n\n // Prepare for next iteration (if there is one).\n if (i < nsub) {\n start = i;\n first = first_i;\n }\n }\n}\n\n// Collapse the regexps on top of the stack, down to the\n// first marker, into a new op node (op == kRegexpAlternate\n// or op == kRegexpConcat).\nvoid Regexp::ParseState::DoCollapse(RegexpOp op) {\n // Scan backward to marker, counting children of composite.\n int n = 0;\n Regexp* next = NULL;\n Regexp* sub;\n for (sub = stacktop_; sub != NULL && !IsMarker(sub->op()); sub = next) {\n next = sub->down_;\n if (sub->op_ == op)\n n += sub->nsub_;\n else\n n++;\n }\n\n // If there's just one child, leave it alone.\n // (Concat of one thing is that one thing; alternate of one thing is same.)\n if (stacktop_ != NULL && stacktop_->down_ == next)\n return;\n\n // Construct op (alternation or concatenation), flattening op of op.\n PODArray subs(n);\n next = NULL;\n int i = n;\n for (sub = stacktop_; sub != NULL && !IsMarker(sub->op()); sub = next) {\n next = sub->down_;\n if (sub->op_ == op) {\n Regexp** sub_subs = sub->sub();\n for (int k = sub->nsub_ - 1; k >= 0; k--)\n subs[--i] = sub_subs[k]->Incref();\n sub->Decref();\n } else {\n subs[--i] = FinishRegexp(sub);\n }\n }\n\n Regexp* re = ConcatOrAlternate(op, subs.data(), n, flags_, true);\n re->simple_ = re->ComputeSimple();\n re->down_ = next;\n stacktop_ = re;\n}\n\n// Finishes the current concatenation,\n// collapsing it into a single regexp on the stack.\nvoid Regexp::ParseState::DoConcatenation() {\n Regexp* r1 = stacktop_;\n if (r1 == NULL || IsMarker(r1->op())) {\n // empty concatenation is special case\n Regexp* re = new Regexp(kRegexpEmptyMatch, flags_);\n PushRegexp(re);\n }\n DoCollapse(kRegexpConcat);\n}\n\n// Finishes the current alternation,\n// collapsing it to a single regexp on the stack.\nvoid Regexp::ParseState::DoAlternation() {\n DoVerticalBar();\n // Now stack top is kVerticalBar.\n Regexp* r1 = stacktop_;\n stacktop_ = r1->down_;\n r1->Decref();\n DoCollapse(kRegexpAlternate);\n}\n\n// Incremental conversion of concatenated literals into strings.\n// If top two elements on stack are both literal or string,\n// collapse into single string.\n// Don't walk down the stack -- the parser calls this frequently\n// enough that below the bottom two is known to be collapsed.\n// Only called when another regexp is about to be pushed\n// on the stack, so that the topmost literal is not being considered.\n// (Otherwise ab* would turn into (ab)*.)\n// If r >= 0, consider pushing a literal r on the stack.\n// Return whether that happened.\nbool Regexp::ParseState::MaybeConcatString(int r, ParseFlags flags) {\n Regexp* re1;\n Regexp* re2;\n if ((re1 = stacktop_) == NULL || (re2 = re1->down_) == NULL)\n return false;\n\n if (re1->op_ != kRegexpLiteral && re1->op_ != kRegexpLiteralString)\n return false;\n if (re2->op_ != kRegexpLiteral && re2->op_ != kRegexpLiteralString)\n return false;\n if ((re1->parse_flags_ & FoldCase) != (re2->parse_flags_ & FoldCase))\n return false;\n\n if (re2->op_ == kRegexpLiteral) {\n // convert into string\n Rune rune = re2->rune_;\n re2->op_ = kRegexpLiteralString;\n re2->nrunes_ = 0;\n re2->runes_ = NULL;\n re2->AddRuneToString(rune);\n }\n\n // push re1 into re2.\n if (re1->op_ == kRegexpLiteral) {\n re2->AddRuneToString(re1->rune_);\n } else {\n for (int i = 0; i < re1->nrunes_; i++)\n re2->AddRuneToString(re1->runes_[i]);\n re1->nrunes_ = 0;\n delete[] re1->runes_;\n re1->runes_ = NULL;\n }\n\n // reuse re1 if possible\n if (r >= 0) {\n re1->op_ = kRegexpLiteral;\n re1->rune_ = r;\n re1->parse_flags_ = static_cast(flags);\n return true;\n }\n\n stacktop_ = re2;\n re1->Decref();\n return false;\n}\n\n// Lexing routines.\n\n// Parses a decimal integer, storing it in *np.\n// Sets *s to span the remainder of the string.\nstatic bool ParseInteger(absl::string_view* s, int* np) {\n if (s->empty() || !absl::ascii_isdigit((*s)[0] & 0xFF))\n return false;\n // Disallow leading zeros.\n if (s->size() >= 2 && (*s)[0] == '0' && absl::ascii_isdigit((*s)[1] & 0xFF))\n return false;\n int n = 0;\n int c;\n while (!s->empty() && absl::ascii_isdigit(c = (*s)[0] & 0xFF)) {\n // Avoid overflow.\n if (n >= 100000000)\n return false;\n n = n*10 + c - '0';\n s->remove_prefix(1); // digit\n }\n *np = n;\n return true;\n}\n\n// Parses a repetition suffix like {1,2} or {2} or {2,}.\n// Sets *s to span the remainder of the string on success.\n// Sets *lo and *hi to the given range.\n// In the case of {2,}, the high number is unbounded;\n// sets *hi to -1 to signify this.\n// {,2} is NOT a valid suffix.\n// The Maybe in the name signifies that the regexp parse\n// doesn't fail even if ParseRepetition does, so the string_view\n// s must NOT be edited unless MaybeParseRepetition returns true.\nstatic bool MaybeParseRepetition(absl::string_view* sp, int* lo, int* hi) {\n absl::string_view s = *sp;\n if (s.empty() || s[0] != '{')\n return false;\n s.remove_prefix(1); // '{'\n if (!ParseInteger(&s, lo))\n return false;\n if (s.empty())\n return false;\n if (s[0] == ',') {\n s.remove_prefix(1); // ','\n if (s.empty())\n return false;\n if (s[0] == '}') {\n // {2,} means at least 2\n *hi = -1;\n } else {\n // {2,4} means 2, 3, or 4.\n if (!ParseInteger(&s, hi))\n return false;\n }\n } else {\n // {2} means exactly two\n *hi = *lo;\n }\n if (s.empty() || s[0] != '}')\n return false;\n s.remove_prefix(1); // '}'\n *sp = s;\n return true;\n}\n\n// Removes the next Rune from the string_view and stores it in *r.\n// Returns number of bytes removed from sp.\n// Behaves as though there is a terminating NUL at the end of sp.\n// Argument order is backwards from usual Google style\n// but consistent with chartorune.\nstatic int StringViewToRune(Rune* r, absl::string_view* sp,\n RegexpStatus* status) {\n // fullrune() takes int, not size_t. However, it just looks\n // at the leading byte and treats any length >= 4 the same.\n if (fullrune(sp->data(), static_cast(std::min(size_t{4}, sp->size())))) {\n int n = chartorune(r, sp->data());\n // Some copies of chartorune have a bug that accepts\n // encodings of values in (10FFFF, 1FFFFF] as valid.\n // Those values break the character class algorithm,\n // which assumes Runemax is the largest rune.\n if (*r > Runemax) {\n n = 1;\n *r = Runeerror;\n }\n if (!(n == 1 && *r == Runeerror)) { // no decoding error\n sp->remove_prefix(n);\n return n;\n }\n }\n\n if (status != NULL) {\n status->set_code(kRegexpBadUTF8);\n status->set_error_arg(absl::string_view());\n }\n return -1;\n}\n\n// Returns whether name is valid UTF-8.\n// If not, sets status to kRegexpBadUTF8.\nstatic bool IsValidUTF8(absl::string_view s, RegexpStatus* status) {\n absl::string_view t = s;\n Rune r;\n while (!t.empty()) {\n if (StringViewToRune(&r, &t, status) < 0)\n return false;\n }\n return true;\n}\n\n// Is c a hex digit?\nstatic int IsHex(int c) {\n return ('0' <= c && c <= '9') ||\n ('A' <= c && c <= 'F') ||\n ('a' <= c && c <= 'f');\n}\n\n// Convert hex digit to value.\nstatic int UnHex(int c) {\n if ('0' <= c && c <= '9')\n return c - '0';\n if ('A' <= c && c <= 'F')\n return c - 'A' + 10;\n if ('a' <= c && c <= 'f')\n return c - 'a' + 10;\n ABSL_LOG(DFATAL) << \"Bad hex digit \" << c;\n return 0;\n}\n\n// Parse an escape sequence (e.g., \\n, \\{).\n// Sets *s to span the remainder of the string.\n// Sets *rp to the named character.\nstatic bool ParseEscape(absl::string_view* s, Rune* rp,\n RegexpStatus* status, int rune_max) {\n const char* begin = s->data();\n if (s->empty() || (*s)[0] != '\\\\') {\n // Should not happen - caller always checks.\n status->set_code(kRegexpInternalError);\n status->set_error_arg(absl::string_view());\n return false;\n }\n if (s->size() == 1) {\n status->set_code(kRegexpTrailingBackslash);\n status->set_error_arg(absl::string_view());\n return false;\n }\n Rune c, c1;\n s->remove_prefix(1); // backslash\n if (StringViewToRune(&c, s, status) < 0)\n return false;\n int code;\n switch (c) {\n default:\n if (c < Runeself && !absl::ascii_isalnum(c)) {\n // Escaped non-word characters are always themselves.\n // PCRE is not quite so rigorous: it accepts things like\n // \\q, but we don't. We once rejected \\_, but too many\n // programs and people insist on using it, so allow \\_.\n *rp = c;\n return true;\n }\n goto BadEscape;\n\n // Octal escapes.\n case '1':\n case '2':\n case '3':\n case '4':\n case '5':\n case '6':\n case '7':\n // Single non-zero octal digit is a backreference; not supported.\n if (s->empty() || (*s)[0] < '0' || (*s)[0] > '7')\n goto BadEscape;\n [[fallthrough]];\n case '0':\n // consume up to three octal digits; already have one.\n code = c - '0';\n if (!s->empty() && '0' <= (c = (*s)[0]) && c <= '7') {\n code = code * 8 + c - '0';\n s->remove_prefix(1); // digit\n if (!s->empty()) {\n c = (*s)[0];\n if ('0' <= c && c <= '7') {\n code = code * 8 + c - '0';\n s->remove_prefix(1); // digit\n }\n }\n }\n if (code > rune_max)\n goto BadEscape;\n *rp = code;\n return true;\n\n // Hexadecimal escapes\n case 'x':\n if (s->empty())\n goto BadEscape;\n if (StringViewToRune(&c, s, status) < 0)\n return false;\n if (c == '{') {\n // Any number of digits in braces.\n // Update n as we consume the string, so that\n // the whole thing gets shown in the error message.\n // Perl accepts any text at all; it ignores all text\n // after the first non-hex digit. We require only hex digits,\n // and at least one.\n if (StringViewToRune(&c, s, status) < 0)\n return false;\n int nhex = 0;\n code = 0;\n while (IsHex(c)) {\n nhex++;\n code = code * 16 + UnHex(c);\n if (code > rune_max)\n goto BadEscape;\n if (s->empty())\n goto BadEscape;\n if (StringViewToRune(&c, s, status) < 0)\n return false;\n }\n if (c != '}' || nhex == 0)\n goto BadEscape;\n *rp = code;\n return true;\n }\n // Easy case: two hex digits.\n if (s->empty())\n goto BadEscape;\n if (StringViewToRune(&c1, s, status) < 0)\n return false;\n if (!IsHex(c) || !IsHex(c1))\n goto BadEscape;\n *rp = UnHex(c) * 16 + UnHex(c1);\n return true;\n\n // C escapes.\n case 'n':\n *rp = '\\n';\n return true;\n case 'r':\n *rp = '\\r';\n return true;\n case 't':\n *rp = '\\t';\n return true;\n\n // Less common C escapes.\n case 'a':\n *rp = '\\a';\n return true;\n case 'f':\n *rp = '\\f';\n return true;\n case 'v':\n *rp = '\\v';\n return true;\n\n // This code is disabled to avoid misparsing\n // the Perl word-boundary \\b as a backspace\n // when in POSIX regexp mode. Surprisingly,\n // in Perl, \\b means word-boundary but [\\b]\n // means backspace. We don't support that:\n // if you want a backspace embed a literal\n // backspace character or use \\x08.\n //\n // case 'b':\n // *rp = '\\b';\n // return true;\n }\n\nBadEscape:\n // Unrecognized escape sequence.\n status->set_code(kRegexpBadEscape);\n status->set_error_arg(\n absl::string_view(begin, static_cast(s->data() - begin)));\n return false;\n}\n\n// Add a range to the character class, but exclude newline if asked.\n// Also handle case folding.\nvoid CharClassBuilder::AddRangeFlags(\n Rune lo, Rune hi, Regexp::ParseFlags parse_flags) {\n\n // Take out \\n if the flags say so.\n bool cutnl = !(parse_flags & Regexp::ClassNL) ||\n (parse_flags & Regexp::NeverNL);\n if (cutnl && lo <= '\\n' && '\\n' <= hi) {\n if (lo < '\\n')\n AddRangeFlags(lo, '\\n' - 1, parse_flags);\n if (hi > '\\n')\n AddRangeFlags('\\n' + 1, hi, parse_flags);\n return;\n }\n\n // If folding case, add fold-equivalent characters too.\n if (parse_flags & Regexp::FoldCase) {\n if (parse_flags & Regexp::Latin1) {\n AddFoldedRangeLatin1(this, lo, hi);\n } else {\n AddFoldedRange(this, lo, hi, 0);\n }\n } else {\n AddRange(lo, hi);\n }\n}\n\n// Look for a group with the given name.\nstatic const UGroup* LookupGroup(absl::string_view name,\n const UGroup* groups, int ngroups) {\n // Simple name lookup.\n for (int i = 0; i < ngroups; i++)\n if (absl::string_view(groups[i].name) == name)\n return &groups[i];\n return NULL;\n}\n\n// Look for a POSIX group with the given name (e.g., \"[:^alpha:]\")\nstatic const UGroup* LookupPosixGroup(absl::string_view name) {\n return LookupGroup(name, posix_groups, num_posix_groups);\n}\n\nstatic const UGroup* LookupPerlGroup(absl::string_view name) {\n return LookupGroup(name, perl_groups, num_perl_groups);\n}\n\n#if !defined(RE2_USE_ICU)\n// Fake UGroup containing all Runes\nstatic URange16 any16[] = { { 0, 65535 } };\nstatic URange32 any32[] = { { 65536, Runemax } };\nstatic UGroup anygroup = { \"Any\", +1, any16, 1, any32, 1 };\n\n// Look for a Unicode group with the given name (e.g., \"Han\")\nstatic const UGroup* LookupUnicodeGroup(absl::string_view name) {\n // Special case: \"Any\" means any.\n if (name == absl::string_view(\"Any\"))\n return &anygroup;\n return LookupGroup(name, unicode_groups, num_unicode_groups);\n}\n#endif\n\n// Add a UGroup or its negation to the character class.\nstatic void AddUGroup(CharClassBuilder* cc, const UGroup* g, int sign,\n Regexp::ParseFlags parse_flags) {\n if (sign == +1) {\n for (int i = 0; i < g->nr16; i++) {\n cc->AddRangeFlags(g->r16[i].lo, g->r16[i].hi, parse_flags);\n }\n for (int i = 0; i < g->nr32; i++) {\n cc->AddRangeFlags(g->r32[i].lo, g->r32[i].hi, parse_flags);\n }\n } else {\n if (parse_flags & Regexp::FoldCase) {\n // Normally adding a case-folded group means\n // adding all the extra fold-equivalent runes too.\n // But if we're adding the negation of the group,\n // we have to exclude all the runes that are fold-equivalent\n // to what's already missing. Too hard, so do in two steps.\n CharClassBuilder ccb1;\n AddUGroup(&ccb1, g, +1, parse_flags);\n // If the flags say to take out \\n, put it in, so that negating will take it out.\n // Normally AddRangeFlags does this, but we're bypassing AddRangeFlags.\n bool cutnl = !(parse_flags & Regexp::ClassNL) ||\n (parse_flags & Regexp::NeverNL);\n if (cutnl) {\n ccb1.AddRange('\\n', '\\n');\n }\n ccb1.Negate();\n cc->AddCharClass(&ccb1);\n return;\n }\n int next = 0;\n for (int i = 0; i < g->nr16; i++) {\n if (next < g->r16[i].lo)\n cc->AddRangeFlags(next, g->r16[i].lo - 1, parse_flags);\n next = g->r16[i].hi + 1;\n }\n for (int i = 0; i < g->nr32; i++) {\n if (next < g->r32[i].lo)\n cc->AddRangeFlags(next, g->r32[i].lo - 1, parse_flags);\n next = g->r32[i].hi + 1;\n }\n if (next <= Runemax)\n cc->AddRangeFlags(next, Runemax, parse_flags);\n }\n}\n\n// Maybe parse a Perl character class escape sequence.\n// Only recognizes the Perl character classes (\\d \\s \\w \\D \\S \\W),\n// not the Perl empty-string classes (\\b \\B \\A \\Z \\z).\n// On success, sets *s to span the remainder of the string\n// and returns the corresponding UGroup.\n// The string_view must *NOT* be edited unless the call succeeds.\nconst UGroup* MaybeParsePerlCCEscape(absl::string_view* s,\n Regexp::ParseFlags parse_flags) {\n if (!(parse_flags & Regexp::PerlClasses))\n return NULL;\n if (s->size() < 2 || (*s)[0] != '\\\\')\n return NULL;\n // Could use StringViewToRune, but there aren't\n // any non-ASCII Perl group names.\n absl::string_view name(s->data(), 2);\n const UGroup* g = LookupPerlGroup(name);\n if (g == NULL)\n return NULL;\n s->remove_prefix(name.size());\n return g;\n}\n\nenum ParseStatus {\n kParseOk, // Did some parsing.\n kParseError, // Found an error.\n kParseNothing, // Decided not to parse.\n};\n\n// Maybe parses a Unicode character group like \\p{Han} or \\P{Han}\n// (the latter is a negated group).\nParseStatus ParseUnicodeGroup(absl::string_view* s,\n Regexp::ParseFlags parse_flags,\n CharClassBuilder* cc, RegexpStatus* status) {\n // Decide whether to parse.\n if (!(parse_flags & Regexp::UnicodeGroups))\n return kParseNothing;\n if (s->size() < 2 || (*s)[0] != '\\\\')\n return kParseNothing;\n Rune c = (*s)[1];\n if (c != 'p' && c != 'P')\n return kParseNothing;\n\n // Committed to parse. Results:\n int sign = +1; // -1 = negated char class\n if (c == 'P')\n sign = -sign;\n absl::string_view seq = *s; // \\p{Han} or \\pL\n absl::string_view name; // Han or L\n s->remove_prefix(2); // '\\\\', 'p'\n\n if (!StringViewToRune(&c, s, status))\n return kParseError;\n if (c != '{') {\n // Name is the bit of string we just skipped over for c.\n const char* p = seq.data() + 2;\n name = absl::string_view(p, static_cast(s->data() - p));\n } else {\n // Name is in braces. Look for closing }\n size_t end = s->find('}', 0);\n if (end == absl::string_view::npos) {\n if (!IsValidUTF8(seq, status))\n return kParseError;\n status->set_code(kRegexpBadCharRange);\n status->set_error_arg(seq);\n return kParseError;\n }\n name = absl::string_view(s->data(), end); // without '}'\n s->remove_prefix(end + 1); // with '}'\n if (!IsValidUTF8(name, status))\n return kParseError;\n }\n\n // Chop seq where s now begins.\n seq = absl::string_view(seq.data(), static_cast(s->data() - seq.data()));\n\n if (!name.empty() && name[0] == '^') {\n sign = -sign;\n name.remove_prefix(1); // '^'\n }\n\n#if !defined(RE2_USE_ICU)\n // Look up the group in the RE2 Unicode data.\n const UGroup* g = LookupUnicodeGroup(name);\n if (g == NULL) {\n status->set_code(kRegexpBadCharRange);\n status->set_error_arg(seq);\n return kParseError;\n }\n\n AddUGroup(cc, g, sign, parse_flags);\n#else\n // Look up the group in the ICU Unicode data. Because ICU provides full\n // Unicode properties support, this could be more than a lookup by name.\n ::icu::UnicodeString ustr = ::icu::UnicodeString::fromUTF8(\n std::string(\"\\\\p{\") + std::string(name) + std::string(\"}\"));\n UErrorCode uerr = U_ZERO_ERROR;\n ::icu::UnicodeSet uset(ustr, uerr);\n if (U_FAILURE(uerr)) {\n status->set_code(kRegexpBadCharRange);\n status->set_error_arg(seq);\n return kParseError;\n }\n\n // Convert the UnicodeSet to a URange32 and UGroup that we can add.\n int nr = uset.getRangeCount();\n PODArray r(nr);\n for (int i = 0; i < nr; i++) {\n r[i].lo = uset.getRangeStart(i);\n r[i].hi = uset.getRangeEnd(i);\n }\n UGroup g = {\"\", +1, 0, 0, r.data(), nr};\n AddUGroup(cc, &g, sign, parse_flags);\n#endif\n\n return kParseOk;\n}\n\n// Parses a character class name like [:alnum:].\n// Sets *s to span the remainder of the string.\n// Adds the ranges corresponding to the class to ranges.\nstatic ParseStatus ParseCCName(absl::string_view* s,\n Regexp::ParseFlags parse_flags,\n CharClassBuilder* cc, RegexpStatus* status) {\n // Check begins with [:\n const char* p = s->data();\n const char* ep = s->data() + s->size();\n if (ep - p < 2 || p[0] != '[' || p[1] != ':')\n return kParseNothing;\n\n // Look for closing :].\n const char* q;\n for (q = p+2; q <= ep-2 && (*q != ':' || *(q+1) != ']'); q++)\n ;\n\n // If no closing :], then ignore.\n if (q > ep-2)\n return kParseNothing;\n\n // Got it. Check that it's valid.\n q += 2;\n absl::string_view name(p, static_cast(q - p));\n\n const UGroup* g = LookupPosixGroup(name);\n if (g == NULL) {\n status->set_code(kRegexpBadCharRange);\n status->set_error_arg(name);\n return kParseError;\n }\n\n s->remove_prefix(name.size());\n AddUGroup(cc, g, g->sign, parse_flags);\n return kParseOk;\n}\n\n// Parses a character inside a character class.\n// There are fewer special characters here than in the rest of the regexp.\n// Sets *s to span the remainder of the string.\n// Sets *rp to the character.\nbool Regexp::ParseState::ParseCCCharacter(absl::string_view* s, Rune* rp,\n absl::string_view whole_class,\n RegexpStatus* status) {\n if (s->empty()) {\n status->set_code(kRegexpMissingBracket);\n status->set_error_arg(whole_class);\n return false;\n }\n\n // Allow regular escape sequences even though\n // many need not be escaped in this context.\n if ((*s)[0] == '\\\\')\n return ParseEscape(s, rp, status, rune_max_);\n\n // Otherwise take the next rune.\n return StringViewToRune(rp, s, status) >= 0;\n}\n\n// Parses a character class character, or, if the character\n// is followed by a hyphen, parses a character class range.\n// For single characters, rr->lo == rr->hi.\n// Sets *s to span the remainder of the string.\n// Sets *rp to the character.\nbool Regexp::ParseState::ParseCCRange(absl::string_view* s, RuneRange* rr,\n absl::string_view whole_class,\n RegexpStatus* status) {\n absl::string_view os = *s;\n if (!ParseCCCharacter(s, &rr->lo, whole_class, status))\n return false;\n // [a-] means (a|-), so check for final ].\n if (s->size() >= 2 && (*s)[0] == '-' && (*s)[1] != ']') {\n s->remove_prefix(1); // '-'\n if (!ParseCCCharacter(s, &rr->hi, whole_class, status))\n return false;\n if (rr->hi < rr->lo) {\n status->set_code(kRegexpBadCharRange);\n status->set_error_arg(absl::string_view(\n os.data(), static_cast(s->data() - os.data())));\n return false;\n }\n } else {\n rr->hi = rr->lo;\n }\n return true;\n}\n\n// Parses a possibly-negated character class expression like [^abx-z[:digit:]].\n// Sets *s to span the remainder of the string.\n// Sets *out_re to the regexp for the class.\nbool Regexp::ParseState::ParseCharClass(absl::string_view* s, Regexp** out_re,\n RegexpStatus* status) {\n absl::string_view whole_class = *s;\n if (s->empty() || (*s)[0] != '[') {\n // Caller checked this.\n status->set_code(kRegexpInternalError);\n status->set_error_arg(absl::string_view());\n return false;\n }\n bool negated = false;\n Regexp* re = new Regexp(kRegexpCharClass, flags_ & ~FoldCase);\n re->ccb_ = new CharClassBuilder;\n s->remove_prefix(1); // '['\n if (!s->empty() && (*s)[0] == '^') {\n s->remove_prefix(1); // '^'\n negated = true;\n if (!(flags_ & ClassNL) || (flags_ & NeverNL)) {\n // If NL can't match implicitly, then pretend\n // negated classes include a leading \\n.\n re->ccb_->AddRange('\\n', '\\n');\n }\n }\n bool first = true; // ] is okay as first char in class\n while (!s->empty() && ((*s)[0] != ']' || first)) {\n // - is only okay unescaped as first or last in class.\n // Except that Perl allows - anywhere.\n if ((*s)[0] == '-' && !first && !(flags_&PerlX) &&\n (s->size() == 1 || (*s)[1] != ']')) {\n absl::string_view t = *s;\n t.remove_prefix(1); // '-'\n Rune r;\n int n = StringViewToRune(&r, &t, status);\n if (n < 0) {\n re->Decref();\n return false;\n }\n status->set_code(kRegexpBadCharRange);\n status->set_error_arg(absl::string_view(s->data(), 1+n));\n re->Decref();\n return false;\n }\n first = false;\n\n // Look for [:alnum:] etc.\n if (s->size() > 2 && (*s)[0] == '[' && (*s)[1] == ':') {\n switch (ParseCCName(s, flags_, re->ccb_, status)) {\n case kParseOk:\n continue;\n case kParseError:\n re->Decref();\n return false;\n case kParseNothing:\n break;\n }\n }\n\n // Look for Unicode character group like \\p{Han}\n if (s->size() > 2 &&\n (*s)[0] == '\\\\' &&\n ((*s)[1] == 'p' || (*s)[1] == 'P')) {\n switch (ParseUnicodeGroup(s, flags_, re->ccb_, status)) {\n case kParseOk:\n continue;\n case kParseError:\n re->Decref();\n return false;\n case kParseNothing:\n break;\n }\n }\n\n // Look for Perl character class symbols (extension).\n const UGroup* g = MaybeParsePerlCCEscape(s, flags_);\n if (g != NULL) {\n AddUGroup(re->ccb_, g, g->sign, flags_);\n continue;\n }\n\n // Otherwise assume single character or simple range.\n RuneRange rr;\n if (!ParseCCRange(s, &rr, whole_class, status)) {\n re->Decref();\n return false;\n }\n // AddRangeFlags is usually called in response to a class like\n // \\p{Foo} or [[:foo:]]; for those, it filters \\n out unless\n // Regexp::ClassNL is set. In an explicit range or singleton\n // like we just parsed, we do not filter \\n out, so set ClassNL\n // in the flags.\n re->ccb_->AddRangeFlags(rr.lo, rr.hi, flags_ | Regexp::ClassNL);\n }\n if (s->empty()) {\n status->set_code(kRegexpMissingBracket);\n status->set_error_arg(whole_class);\n re->Decref();\n return false;\n }\n s->remove_prefix(1); // ']'\n\n if (negated)\n re->ccb_->Negate();\n\n *out_re = re;\n return true;\n}\n\n// Returns whether name is a valid capture name.\nstatic bool IsValidCaptureName(absl::string_view name) {\n if (name.empty())\n return false;\n\n // Historically, we effectively used [0-9A-Za-z_]+ to validate; that\n // followed Python 2 except for not restricting the first character.\n // As of Python 3, Unicode characters beyond ASCII are also allowed;\n // accordingly, we permit the Lu, Ll, Lt, Lm, Lo, Nl, Mn, Mc, Nd and\n // Pc categories, but again without restricting the first character.\n // Also, Unicode normalization (e.g. NFKC) isn't performed: Python 3\n // performs it for identifiers, but seemingly not for capture names;\n // if they start doing that for capture names, we won't follow suit.\n static const CharClass* const cc = []() {\n CharClassBuilder ccb;\n for (absl::string_view group :\n {\"Lu\", \"Ll\", \"Lt\", \"Lm\", \"Lo\", \"Nl\", \"Mn\", \"Mc\", \"Nd\", \"Pc\"})\n AddUGroup(&ccb, LookupGroup(group, unicode_groups, num_unicode_groups),\n +1, Regexp::NoParseFlags);\n return ccb.GetCharClass();\n }();\n\n absl::string_view t = name;\n Rune r;\n while (!t.empty()) {\n if (StringViewToRune(&r, &t, NULL) < 0)\n return false;\n if (cc->Contains(r))\n continue;\n return false;\n }\n return true;\n}\n\n// Parses a Perl flag setting or non-capturing group or both,\n// like (?i) or (?: or (?i:. Removes from s, updates parse state.\n// The caller must check that s begins with \"(?\".\n// Returns true on success. If the Perl flag is not\n// well-formed or not supported, sets status_ and returns false.\nbool Regexp::ParseState::ParsePerlFlags(absl::string_view* s) {\n absl::string_view t = *s;\n\n // Caller is supposed to check this.\n if (!(flags_ & PerlX) || t.size() < 2 || t[0] != '(' || t[1] != '?') {\n status_->set_code(kRegexpInternalError);\n ABSL_LOG(DFATAL) << \"Bad call to ParseState::ParsePerlFlags\";\n return false;\n }\n\n // Check for look-around assertions. This is NOT because we support them! ;)\n // As per https://github.com/google/re2/issues/468, we really want to report\n // kRegexpBadPerlOp (not kRegexpBadNamedCapture) for look-behind assertions.\n // Additionally, it would be nice to report not \"(?<\", but \"(?<=\" or \"(? 3 && (t[2] == '=' || t[2] == '!')) ||\n (t.size() > 4 && t[2] == '<' && (t[3] == '=' || t[3] == '!'))) {\n status_->set_code(kRegexpBadPerlOp);\n status_->set_error_arg(absl::string_view(t.data(), t[2] == '<' ? 4 : 3));\n return false;\n }\n\n // Check for named captures, first introduced in Python's regexp library.\n // As usual, there are three slightly different syntaxes:\n //\n // (?Pexpr) the original, introduced by Python\n // (?expr) the .NET alteration, adopted by Perl 5.10\n // (?'name'expr) another .NET alteration, adopted by Perl 5.10\n //\n // Perl 5.10 gave in and implemented the Python version too,\n // but they claim that the last two are the preferred forms.\n // PCRE and languages based on it (specifically, PHP and Ruby)\n // support all three as well. EcmaScript 4 uses only the Python form.\n //\n // In both the open source world (via Code Search) and the\n // Google source tree, (?Pexpr) and (?expr) are the\n // dominant forms of named captures and both are supported.\n if ((t.size() > 4 && t[2] == 'P' && t[3] == '<') ||\n (t.size() > 3 && t[2] == '<')) {\n // Pull out name.\n size_t begin = t[2] == 'P' ? 4 : 3;\n size_t end = t.find('>', begin);\n if (end == absl::string_view::npos) {\n if (!IsValidUTF8(t, status_))\n return false;\n status_->set_code(kRegexpBadNamedCapture);\n status_->set_error_arg(t);\n return false;\n }\n\n absl::string_view capture(t.data(), end+1);\n absl::string_view name(t.data()+begin, end-begin);\n if (!IsValidUTF8(name, status_))\n return false;\n if (!IsValidCaptureName(name)) {\n status_->set_code(kRegexpBadNamedCapture);\n status_->set_error_arg(capture);\n return false;\n }\n\n if (!DoLeftParen(name)) {\n // DoLeftParen's failure set status_.\n return false;\n }\n\n s->remove_prefix(capture.size());\n return true;\n }\n\n t.remove_prefix(2); // \"(?\"\n\n bool negated = false;\n bool sawflags = false;\n int nflags = flags_;\n Rune c;\n for (bool done = false; !done; ) {\n if (t.empty())\n goto BadPerlOp;\n if (StringViewToRune(&c, &t, status_) < 0)\n return false;\n switch (c) {\n default:\n goto BadPerlOp;\n\n // Parse flags.\n case 'i':\n sawflags = true;\n if (negated)\n nflags &= ~FoldCase;\n else\n nflags |= FoldCase;\n break;\n\n case 'm': // opposite of our OneLine\n sawflags = true;\n if (negated)\n nflags |= OneLine;\n else\n nflags &= ~OneLine;\n break;\n\n case 's':\n sawflags = true;\n if (negated)\n nflags &= ~DotNL;\n else\n nflags |= DotNL;\n break;\n\n case 'U':\n sawflags = true;\n if (negated)\n nflags &= ~NonGreedy;\n else\n nflags |= NonGreedy;\n break;\n\n // Negation\n case '-':\n if (negated)\n goto BadPerlOp;\n negated = true;\n sawflags = false;\n break;\n\n // Open new group.\n case ':':\n if (!DoLeftParenNoCapture()) {\n // DoLeftParenNoCapture's failure set status_.\n return false;\n }\n done = true;\n break;\n\n // Finish flags.\n case ')':\n done = true;\n break;\n }\n }\n\n if (negated && !sawflags)\n goto BadPerlOp;\n\n flags_ = static_cast(nflags);\n *s = t;\n return true;\n\nBadPerlOp:\n status_->set_code(kRegexpBadPerlOp);\n status_->set_error_arg(\n absl::string_view(s->data(), static_cast(t.data() - s->data())));\n return false;\n}\n\n// Converts latin1 (assumed to be encoded as Latin1 bytes)\n// into UTF8 encoding in string.\n// Can't use EncodingUtils::EncodeLatin1AsUTF8 because it is\n// deprecated and because it rejects code points 0x80-0x9F.\nvoid ConvertLatin1ToUTF8(absl::string_view latin1, std::string* utf) {\n char buf[UTFmax];\n\n utf->clear();\n for (size_t i = 0; i < latin1.size(); i++) {\n Rune r = latin1[i] & 0xFF;\n int n = runetochar(buf, &r);\n utf->append(buf, n);\n }\n}\n\n// Parses the regular expression given by s,\n// returning the corresponding Regexp tree.\n// The caller must Decref the return value when done with it.\n// Returns NULL on error.\nRegexp* Regexp::Parse(absl::string_view s, ParseFlags global_flags,\n RegexpStatus* status) {\n // Make status non-NULL (easier on everyone else).\n RegexpStatus xstatus;\n if (status == NULL)\n status = &xstatus;\n\n ParseState ps(global_flags, s, status);\n absl::string_view t = s;\n\n // Convert regexp to UTF-8 (easier on the rest of the parser).\n if (global_flags & Latin1) {\n std::string* tmp = new std::string;\n ConvertLatin1ToUTF8(t, tmp);\n status->set_tmp(tmp);\n t = *tmp;\n }\n\n if (global_flags & Literal) {\n // Special parse loop for literal string.\n while (!t.empty()) {\n Rune r;\n if (StringViewToRune(&r, &t, status) < 0)\n return NULL;\n if (!ps.PushLiteral(r))\n return NULL;\n }\n return ps.DoFinish();\n }\n\n absl::string_view lastunary = absl::string_view();\n while (!t.empty()) {\n absl::string_view isunary = absl::string_view();\n switch (t[0]) {\n default: {\n Rune r;\n if (StringViewToRune(&r, &t, status) < 0)\n return NULL;\n if (!ps.PushLiteral(r))\n return NULL;\n break;\n }\n\n case '(':\n // \"(?\" introduces Perl escape.\n if ((ps.flags() & PerlX) && (t.size() >= 2 && t[1] == '?')) {\n // Flag changes and non-capturing groups.\n if (!ps.ParsePerlFlags(&t))\n return NULL;\n break;\n }\n if (ps.flags() & NeverCapture) {\n if (!ps.DoLeftParenNoCapture())\n return NULL;\n } else {\n if (!ps.DoLeftParen(absl::string_view()))\n return NULL;\n }\n t.remove_prefix(1); // '('\n break;\n\n case '|':\n if (!ps.DoVerticalBar())\n return NULL;\n t.remove_prefix(1); // '|'\n break;\n\n case ')':\n if (!ps.DoRightParen())\n return NULL;\n t.remove_prefix(1); // ')'\n break;\n\n case '^': // Beginning of line.\n if (!ps.PushCaret())\n return NULL;\n t.remove_prefix(1); // '^'\n break;\n\n case '$': // End of line.\n if (!ps.PushDollar())\n return NULL;\n t.remove_prefix(1); // '$'\n break;\n\n case '.': // Any character (possibly except newline).\n if (!ps.PushDot())\n return NULL;\n t.remove_prefix(1); // '.'\n break;\n\n case '[': { // Character class.\n Regexp* re;\n if (!ps.ParseCharClass(&t, &re, status))\n return NULL;\n if (!ps.PushRegexp(re))\n return NULL;\n break;\n }\n\n case '*': { // Zero or more.\n RegexpOp op;\n op = kRegexpStar;\n goto Rep;\n case '+': // One or more.\n op = kRegexpPlus;\n goto Rep;\n case '?': // Zero or one.\n op = kRegexpQuest;\n goto Rep;\n Rep:\n absl::string_view opstr = t;\n bool nongreedy = false;\n t.remove_prefix(1); // '*' or '+' or '?'\n if (ps.flags() & PerlX) {\n if (!t.empty() && t[0] == '?') {\n nongreedy = true;\n t.remove_prefix(1); // '?'\n }\n if (!lastunary.empty()) {\n // In Perl it is not allowed to stack repetition operators:\n // a** is a syntax error, not a double-star.\n // (and a++ means something else entirely, which we don't support!)\n status->set_code(kRegexpRepeatOp);\n status->set_error_arg(absl::string_view(\n lastunary.data(),\n static_cast(t.data() - lastunary.data())));\n return NULL;\n }\n }\n opstr = absl::string_view(opstr.data(),\n static_cast(t.data() - opstr.data()));\n if (!ps.PushRepeatOp(op, opstr, nongreedy))\n return NULL;\n isunary = opstr;\n break;\n }\n\n case '{': { // Counted repetition.\n int lo, hi;\n absl::string_view opstr = t;\n if (!MaybeParseRepetition(&t, &lo, &hi)) {\n // Treat like a literal.\n if (!ps.PushLiteral('{'))\n return NULL;\n t.remove_prefix(1); // '{'\n break;\n }\n bool nongreedy = false;\n if (ps.flags() & PerlX) {\n if (!t.empty() && t[0] == '?') {\n nongreedy = true;\n t.remove_prefix(1); // '?'\n }\n if (!lastunary.empty()) {\n // Not allowed to stack repetition operators.\n status->set_code(kRegexpRepeatOp);\n status->set_error_arg(absl::string_view(\n lastunary.data(),\n static_cast(t.data() - lastunary.data())));\n return NULL;\n }\n }\n opstr = absl::string_view(opstr.data(),\n static_cast(t.data() - opstr.data()));\n if (!ps.PushRepetition(lo, hi, opstr, nongreedy))\n return NULL;\n isunary = opstr;\n break;\n }\n\n case '\\\\': { // Escaped character or Perl sequence.\n // \\b and \\B: word boundary or not\n if ((ps.flags() & Regexp::PerlB) &&\n t.size() >= 2 && (t[1] == 'b' || t[1] == 'B')) {\n if (!ps.PushWordBoundary(t[1] == 'b'))\n return NULL;\n t.remove_prefix(2); // '\\\\', 'b'\n break;\n }\n\n if ((ps.flags() & Regexp::PerlX) && t.size() >= 2) {\n if (t[1] == 'A') {\n if (!ps.PushSimpleOp(kRegexpBeginText))\n return NULL;\n t.remove_prefix(2); // '\\\\', 'A'\n break;\n }\n if (t[1] == 'z') {\n if (!ps.PushSimpleOp(kRegexpEndText))\n return NULL;\n t.remove_prefix(2); // '\\\\', 'z'\n break;\n }\n // Do not recognize \\Z, because this library can't\n // implement the exact Perl/PCRE semantics.\n // (This library treats \"(?-m)$\" as \\z, even though\n // in Perl and PCRE it is equivalent to \\Z.)\n\n if (t[1] == 'C') { // \\C: any byte [sic]\n if (!ps.PushSimpleOp(kRegexpAnyByte))\n return NULL;\n t.remove_prefix(2); // '\\\\', 'C'\n break;\n }\n\n if (t[1] == 'Q') { // \\Q ... \\E: the ... is always literals\n t.remove_prefix(2); // '\\\\', 'Q'\n while (!t.empty()) {\n if (t.size() >= 2 && t[0] == '\\\\' && t[1] == 'E') {\n t.remove_prefix(2); // '\\\\', 'E'\n break;\n }\n Rune r;\n if (StringViewToRune(&r, &t, status) < 0)\n return NULL;\n if (!ps.PushLiteral(r))\n return NULL;\n }\n break;\n }\n }\n\n if (t.size() >= 2 && (t[1] == 'p' || t[1] == 'P')) {\n Regexp* re = new Regexp(kRegexpCharClass, ps.flags() & ~FoldCase);\n re->ccb_ = new CharClassBuilder;\n switch (ParseUnicodeGroup(&t, ps.flags(), re->ccb_, status)) {\n case kParseOk:\n if (!ps.PushRegexp(re))\n return NULL;\n goto Break2;\n case kParseError:\n re->Decref();\n return NULL;\n case kParseNothing:\n re->Decref();\n break;\n }\n }\n\n const UGroup* g = MaybeParsePerlCCEscape(&t, ps.flags());\n if (g != NULL) {\n Regexp* re = new Regexp(kRegexpCharClass, ps.flags() & ~FoldCase);\n re->ccb_ = new CharClassBuilder;\n AddUGroup(re->ccb_, g, g->sign, ps.flags());\n if (!ps.PushRegexp(re))\n return NULL;\n break;\n }\n\n Rune r;\n if (!ParseEscape(&t, &r, status, ps.rune_max()))\n return NULL;\n if (!ps.PushLiteral(r))\n return NULL;\n break;\n }\n }\n Break2:\n lastunary = isunary;\n }\n return ps.DoFinish();\n}\n\n} // namespace re2\n" }, { "path": "re2/perl_groups.cc", "content": "// GENERATED BY make_perl_groups.pl; DO NOT EDIT.\n// make_perl_groups.pl >perl_groups.cc\n\n#include \"re2/unicode_groups.h\"\n\nnamespace re2 {\n\nstatic const URange16 code1[] = { /* \\d */\n\t{ 0x30, 0x39 },\n};\nstatic const URange16 code2[] = { /* \\s */\n\t{ 0x9, 0xa },\n\t{ 0xc, 0xd },\n\t{ 0x20, 0x20 },\n};\nstatic const URange16 code3[] = { /* \\w */\n\t{ 0x30, 0x39 },\n\t{ 0x41, 0x5a },\n\t{ 0x5f, 0x5f },\n\t{ 0x61, 0x7a },\n};\nconst UGroup perl_groups[] = {\n\t{ \"\\\\d\", +1, code1, 1, 0, 0 },\n\t{ \"\\\\D\", -1, code1, 1, 0, 0 },\n\t{ \"\\\\s\", +1, code2, 3, 0, 0 },\n\t{ \"\\\\S\", -1, code2, 3, 0, 0 },\n\t{ \"\\\\w\", +1, code3, 4, 0, 0 },\n\t{ \"\\\\W\", -1, code3, 4, 0, 0 },\n};\nconst int num_perl_groups = 6;\nstatic const URange16 code4[] = { /* [:alnum:] */\n\t{ 0x30, 0x39 },\n\t{ 0x41, 0x5a },\n\t{ 0x61, 0x7a },\n};\nstatic const URange16 code5[] = { /* [:alpha:] */\n\t{ 0x41, 0x5a },\n\t{ 0x61, 0x7a },\n};\nstatic const URange16 code6[] = { /* [:ascii:] */\n\t{ 0x0, 0x7f },\n};\nstatic const URange16 code7[] = { /* [:blank:] */\n\t{ 0x9, 0x9 },\n\t{ 0x20, 0x20 },\n};\nstatic const URange16 code8[] = { /* [:cntrl:] */\n\t{ 0x0, 0x1f },\n\t{ 0x7f, 0x7f },\n};\nstatic const URange16 code9[] = { /* [:digit:] */\n\t{ 0x30, 0x39 },\n};\nstatic const URange16 code10[] = { /* [:graph:] */\n\t{ 0x21, 0x7e },\n};\nstatic const URange16 code11[] = { /* [:lower:] */\n\t{ 0x61, 0x7a },\n};\nstatic const URange16 code12[] = { /* [:print:] */\n\t{ 0x20, 0x7e },\n};\nstatic const URange16 code13[] = { /* [:punct:] */\n\t{ 0x21, 0x2f },\n\t{ 0x3a, 0x40 },\n\t{ 0x5b, 0x60 },\n\t{ 0x7b, 0x7e },\n};\nstatic const URange16 code14[] = { /* [:space:] */\n\t{ 0x9, 0xd },\n\t{ 0x20, 0x20 },\n};\nstatic const URange16 code15[] = { /* [:upper:] */\n\t{ 0x41, 0x5a },\n};\nstatic const URange16 code16[] = { /* [:word:] */\n\t{ 0x30, 0x39 },\n\t{ 0x41, 0x5a },\n\t{ 0x5f, 0x5f },\n\t{ 0x61, 0x7a },\n};\nstatic const URange16 code17[] = { /* [:xdigit:] */\n\t{ 0x30, 0x39 },\n\t{ 0x41, 0x46 },\n\t{ 0x61, 0x66 },\n};\nconst UGroup posix_groups[] = {\n\t{ \"[:alnum:]\", +1, code4, 3, 0, 0 },\n\t{ \"[:^alnum:]\", -1, code4, 3, 0, 0 },\n\t{ \"[:alpha:]\", +1, code5, 2, 0, 0 },\n\t{ \"[:^alpha:]\", -1, code5, 2, 0, 0 },\n\t{ \"[:ascii:]\", +1, code6, 1, 0, 0 },\n\t{ \"[:^ascii:]\", -1, code6, 1, 0, 0 },\n\t{ \"[:blank:]\", +1, code7, 2, 0, 0 },\n\t{ \"[:^blank:]\", -1, code7, 2, 0, 0 },\n\t{ \"[:cntrl:]\", +1, code8, 2, 0, 0 },\n\t{ \"[:^cntrl:]\", -1, code8, 2, 0, 0 },\n\t{ \"[:digit:]\", +1, code9, 1, 0, 0 },\n\t{ \"[:^digit:]\", -1, code9, 1, 0, 0 },\n\t{ \"[:graph:]\", +1, code10, 1, 0, 0 },\n\t{ \"[:^graph:]\", -1, code10, 1, 0, 0 },\n\t{ \"[:lower:]\", +1, code11, 1, 0, 0 },\n\t{ \"[:^lower:]\", -1, code11, 1, 0, 0 },\n\t{ \"[:print:]\", +1, code12, 1, 0, 0 },\n\t{ \"[:^print:]\", -1, code12, 1, 0, 0 },\n\t{ \"[:punct:]\", +1, code13, 4, 0, 0 },\n\t{ \"[:^punct:]\", -1, code13, 4, 0, 0 },\n\t{ \"[:space:]\", +1, code14, 2, 0, 0 },\n\t{ \"[:^space:]\", -1, code14, 2, 0, 0 },\n\t{ \"[:upper:]\", +1, code15, 1, 0, 0 },\n\t{ \"[:^upper:]\", -1, code15, 1, 0, 0 },\n\t{ \"[:word:]\", +1, code16, 4, 0, 0 },\n\t{ \"[:^word:]\", -1, code16, 4, 0, 0 },\n\t{ \"[:xdigit:]\", +1, code17, 3, 0, 0 },\n\t{ \"[:^xdigit:]\", -1, code17, 3, 0, 0 },\n};\nconst int num_posix_groups = 28;\n\n} // namespace re2\n" }, { "path": "re2/pod_array.h", "content": "// Copyright 2018 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_POD_ARRAY_H_\n#define RE2_POD_ARRAY_H_\n\n#include \n#include \n\nnamespace re2 {\n\ntemplate \nclass PODArray {\n public:\n static_assert(std::is_trivial::value && std::is_standard_layout::value,\n \"T must be POD\");\n\n PODArray()\n : ptr_() {}\n explicit PODArray(int len)\n : ptr_(std::allocator().allocate(len), Deleter(len)) {}\n\n T* data() const {\n return ptr_.get();\n }\n\n int size() const {\n return ptr_.get_deleter().len_;\n }\n\n T& operator[](int pos) const {\n return ptr_[pos];\n }\n\n private:\n struct Deleter {\n Deleter()\n : len_(0) {}\n explicit Deleter(int len)\n : len_(len) {}\n\n void operator()(T* ptr) const {\n std::allocator().deallocate(ptr, len_);\n }\n\n int len_;\n };\n\n std::unique_ptr ptr_;\n};\n\n} // namespace re2\n\n#endif // RE2_POD_ARRAY_H_\n" }, { "path": "re2/prefilter.cc", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \"re2/prefilter.h\"\n\n#include \n\n#include \n#include \n#include \n\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"re2/re2.h\"\n#include \"re2/regexp.h\"\n#include \"re2/unicode_casefold.h\"\n#include \"re2/walker-inl.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\nstatic const bool ExtraDebug = false;\n\n// Initializes a Prefilter, allocating subs_ as necessary.\nPrefilter::Prefilter(Op op) {\n op_ = op;\n subs_ = NULL;\n if (op_ == AND || op_ == OR)\n subs_ = new std::vector;\n}\n\n// Destroys a Prefilter.\nPrefilter::~Prefilter() {\n if (subs_) {\n for (size_t i = 0; i < subs_->size(); i++)\n delete (*subs_)[i];\n delete subs_;\n subs_ = NULL;\n }\n}\n\n// Simplify if the node is an empty Or or And.\nPrefilter* Prefilter::Simplify() {\n if (op_ != AND && op_ != OR) {\n return this;\n }\n\n // Nothing left in the AND/OR.\n if (subs_->empty()) {\n if (op_ == AND)\n op_ = ALL; // AND of nothing is true\n else\n op_ = NONE; // OR of nothing is false\n\n return this;\n }\n\n // Just one subnode: throw away wrapper.\n if (subs_->size() == 1) {\n Prefilter* a = (*subs_)[0];\n subs_->clear();\n delete this;\n return a->Simplify();\n }\n\n return this;\n}\n\n// Combines two Prefilters together to create an \"op\" (AND or OR).\n// The passed Prefilters will be part of the returned Prefilter or deleted.\n// Does lots of work to avoid creating unnecessarily complicated structures.\nPrefilter* Prefilter::AndOr(Op op, Prefilter* a, Prefilter* b) {\n // If a, b can be rewritten as op, do so.\n a = a->Simplify();\n b = b->Simplify();\n\n // Canonicalize: a->op <= b->op.\n if (a->op() > b->op()) {\n Prefilter* t = a;\n a = b;\n b = t;\n }\n\n // Trivial cases.\n // ALL AND b = b\n // NONE OR b = b\n // ALL OR b = ALL\n // NONE AND b = NONE\n // Don't need to look at b, because of canonicalization above.\n // ALL and NONE are smallest opcodes.\n if (a->op() == ALL || a->op() == NONE) {\n if ((a->op() == ALL && op == AND) ||\n (a->op() == NONE && op == OR)) {\n delete a;\n return b;\n } else {\n delete b;\n return a;\n }\n }\n\n // If a and b match op, merge their contents.\n if (a->op() == op && b->op() == op) {\n for (size_t i = 0; i < b->subs()->size(); i++) {\n Prefilter* bb = (*b->subs())[i];\n a->subs()->push_back(bb);\n }\n b->subs()->clear();\n delete b;\n return a;\n }\n\n // If a already has the same op as the op that is under construction\n // add in b (similarly if b already has the same op, add in a).\n if (b->op() == op) {\n Prefilter* t = a;\n a = b;\n b = t;\n }\n if (a->op() == op) {\n a->subs()->push_back(b);\n return a;\n }\n\n // Otherwise just return the op.\n Prefilter* c = new Prefilter(op);\n c->subs()->push_back(a);\n c->subs()->push_back(b);\n return c;\n}\n\nPrefilter* Prefilter::And(Prefilter* a, Prefilter* b) {\n return AndOr(AND, a, b);\n}\n\nPrefilter* Prefilter::Or(Prefilter* a, Prefilter* b) {\n return AndOr(OR, a, b);\n}\n\nvoid Prefilter::SimplifyStringSet(SSet* ss) {\n // Now make sure that the strings aren't redundant. For example, if\n // we know \"ab\" is a required string, then it doesn't help at all to\n // know that \"abc\" is also a required string, so delete \"abc\". This\n // is because, when we are performing a string search to filter\n // regexps, matching \"ab\" will already allow this regexp to be a\n // candidate for match, so further matching \"abc\" is redundant.\n // Note that we must ignore \"\" because find() would find it at the\n // start of everything and thus we would end up erasing everything.\n //\n // The SSet sorts strings by length, then lexicographically. Note that\n // smaller strings appear first and all strings must be unique. These\n // observations let us skip string comparisons when possible.\n SSIter i = ss->begin();\n if (i != ss->end() && i->empty()) {\n ++i;\n }\n for (; i != ss->end(); ++i) {\n SSIter j = i;\n ++j;\n while (j != ss->end()) {\n if (j->size() > i->size() && j->find(*i) != std::string::npos) {\n j = ss->erase(j);\n continue;\n }\n ++j;\n }\n }\n}\n\nPrefilter* Prefilter::OrStrings(SSet* ss) {\n Prefilter* or_prefilter = new Prefilter(NONE);\n SimplifyStringSet(ss);\n for (SSIter i = ss->begin(); i != ss->end(); ++i)\n or_prefilter = Or(or_prefilter, FromString(*i));\n return or_prefilter;\n}\n\nstatic Rune ToLowerRune(Rune r) {\n if (r < Runeself) {\n if ('A' <= r && r <= 'Z')\n r += 'a' - 'A';\n return r;\n }\n\n const CaseFold *f = LookupCaseFold(unicode_tolower, num_unicode_tolower, r);\n if (f == NULL || r < f->lo)\n return r;\n return ApplyFold(f, r);\n}\n\nstatic Rune ToLowerRuneLatin1(Rune r) {\n if ('A' <= r && r <= 'Z')\n r += 'a' - 'A';\n return r;\n}\n\nPrefilter* Prefilter::FromString(const std::string& str) {\n Prefilter* m = new Prefilter(Prefilter::ATOM);\n m->atom_ = str;\n return m;\n}\n\n// Information about a regexp used during computation of Prefilter.\n// Can be thought of as information about the set of strings matching\n// the given regular expression.\nclass Prefilter::Info {\n public:\n Info();\n ~Info();\n\n // More constructors. They delete their Info* arguments.\n static Info* Alt(Info* a, Info* b);\n static Info* Concat(Info* a, Info* b);\n static Info* And(Info* a, Info* b);\n static Info* Star(Info* a);\n static Info* Plus(Info* a);\n static Info* Quest(Info* a);\n static Info* EmptyString();\n static Info* NoMatch();\n static Info* AnyCharOrAnyByte();\n static Info* CClass(CharClass* cc, bool latin1);\n static Info* Literal(Rune r);\n static Info* LiteralLatin1(Rune r);\n static Info* AnyMatch();\n\n // Format Info as a string.\n std::string ToString();\n\n // Caller takes ownership of the Prefilter.\n Prefilter* TakeMatch();\n\n SSet& exact() { return exact_; }\n\n bool is_exact() const { return is_exact_; }\n\n class Walker;\n\n private:\n SSet exact_;\n\n // When is_exact_ is true, the strings that match\n // are placed in exact_. When it is no longer an exact\n // set of strings that match this RE, then is_exact_\n // is false and the match_ contains the required match\n // criteria.\n bool is_exact_;\n\n // Accumulated Prefilter query that any\n // match for this regexp is guaranteed to match.\n Prefilter* match_;\n};\n\n\nPrefilter::Info::Info()\n : is_exact_(false),\n match_(NULL) {\n}\n\nPrefilter::Info::~Info() {\n delete match_;\n}\n\nPrefilter* Prefilter::Info::TakeMatch() {\n if (is_exact_) {\n match_ = Prefilter::OrStrings(&exact_);\n is_exact_ = false;\n }\n Prefilter* m = match_;\n match_ = NULL;\n return m;\n}\n\n// Format a Info in string form.\nstd::string Prefilter::Info::ToString() {\n if (is_exact_) {\n int n = 0;\n std::string s;\n for (SSIter i = exact_.begin(); i != exact_.end(); ++i) {\n if (n++ > 0)\n s += \",\";\n s += *i;\n }\n return s;\n }\n\n if (match_)\n return match_->DebugString();\n\n return \"\";\n}\n\nvoid Prefilter::CrossProduct(const SSet& a, const SSet& b, SSet* dst) {\n for (ConstSSIter i = a.begin(); i != a.end(); ++i)\n for (ConstSSIter j = b.begin(); j != b.end(); ++j)\n dst->insert(*i + *j);\n}\n\n// Concats a and b. Requires that both are exact sets.\n// Forms an exact set that is a crossproduct of a and b.\nPrefilter::Info* Prefilter::Info::Concat(Info* a, Info* b) {\n if (a == NULL)\n return b;\n ABSL_DCHECK(a->is_exact_);\n ABSL_DCHECK(b && b->is_exact_);\n Info *ab = new Info();\n\n CrossProduct(a->exact_, b->exact_, &ab->exact_);\n ab->is_exact_ = true;\n\n delete a;\n delete b;\n return ab;\n}\n\n// Constructs an inexact Info for ab given a and b.\n// Used only when a or b is not exact or when the\n// exact cross product is likely to be too big.\nPrefilter::Info* Prefilter::Info::And(Info* a, Info* b) {\n if (a == NULL)\n return b;\n if (b == NULL)\n return a;\n\n Info *ab = new Info();\n\n ab->match_ = Prefilter::And(a->TakeMatch(), b->TakeMatch());\n ab->is_exact_ = false;\n delete a;\n delete b;\n return ab;\n}\n\n// Constructs Info for a|b given a and b.\nPrefilter::Info* Prefilter::Info::Alt(Info* a, Info* b) {\n Info *ab = new Info();\n\n if (a->is_exact_ && b->is_exact_) {\n // Avoid string copies by moving the larger exact_ set into\n // ab directly, then merge in the smaller set.\n if (a->exact_.size() < b->exact_.size()) {\n using std::swap;\n swap(a, b);\n }\n ab->exact_ = std::move(a->exact_);\n ab->exact_.insert(b->exact_.begin(), b->exact_.end());\n ab->is_exact_ = true;\n } else {\n // Either a or b has is_exact_ = false. If the other\n // one has is_exact_ = true, we move it to match_ and\n // then create a OR of a,b. The resulting Info has\n // is_exact_ = false.\n ab->match_ = Prefilter::Or(a->TakeMatch(), b->TakeMatch());\n ab->is_exact_ = false;\n }\n\n delete a;\n delete b;\n return ab;\n}\n\n// Constructs Info for a? given a.\nPrefilter::Info* Prefilter::Info::Quest(Info *a) {\n Info *ab = new Info();\n\n ab->is_exact_ = false;\n ab->match_ = new Prefilter(ALL);\n delete a;\n return ab;\n}\n\n// Constructs Info for a* given a.\n// Same as a? -- not much to do.\nPrefilter::Info* Prefilter::Info::Star(Info *a) {\n return Quest(a);\n}\n\n// Constructs Info for a+ given a. If a was exact set, it isn't\n// anymore.\nPrefilter::Info* Prefilter::Info::Plus(Info *a) {\n Info *ab = new Info();\n\n ab->match_ = a->TakeMatch();\n ab->is_exact_ = false;\n\n delete a;\n return ab;\n}\n\nstatic std::string RuneToString(Rune r) {\n char buf[UTFmax];\n int n = runetochar(buf, &r);\n return std::string(buf, n);\n}\n\nstatic std::string RuneToStringLatin1(Rune r) {\n char c = r & 0xff;\n return std::string(&c, 1);\n}\n\n// Constructs Info for literal rune.\nPrefilter::Info* Prefilter::Info::Literal(Rune r) {\n Info* info = new Info();\n info->exact_.insert(RuneToString(ToLowerRune(r)));\n info->is_exact_ = true;\n return info;\n}\n\n// Constructs Info for literal rune for Latin1 encoded string.\nPrefilter::Info* Prefilter::Info::LiteralLatin1(Rune r) {\n Info* info = new Info();\n info->exact_.insert(RuneToStringLatin1(ToLowerRuneLatin1(r)));\n info->is_exact_ = true;\n return info;\n}\n\n// Constructs Info for dot (any character) or \\C (any byte).\nPrefilter::Info* Prefilter::Info::AnyCharOrAnyByte() {\n Prefilter::Info* info = new Prefilter::Info();\n info->match_ = new Prefilter(ALL);\n return info;\n}\n\n// Constructs Prefilter::Info for no possible match.\nPrefilter::Info* Prefilter::Info::NoMatch() {\n Prefilter::Info* info = new Prefilter::Info();\n info->match_ = new Prefilter(NONE);\n return info;\n}\n\n// Constructs Prefilter::Info for any possible match.\n// This Prefilter::Info is valid for any regular expression,\n// since it makes no assertions whatsoever about the\n// strings being matched.\nPrefilter::Info* Prefilter::Info::AnyMatch() {\n Prefilter::Info *info = new Prefilter::Info();\n info->match_ = new Prefilter(ALL);\n return info;\n}\n\n// Constructs Prefilter::Info for just the empty string.\nPrefilter::Info* Prefilter::Info::EmptyString() {\n Prefilter::Info* info = new Prefilter::Info();\n info->is_exact_ = true;\n info->exact_.insert(\"\");\n return info;\n}\n\n// Constructs Prefilter::Info for a character class.\ntypedef CharClass::iterator CCIter;\nPrefilter::Info* Prefilter::Info::CClass(CharClass *cc,\n bool latin1) {\n if (ExtraDebug) {\n ABSL_LOG(ERROR) << \"CharClassInfo:\";\n for (CCIter i = cc->begin(); i != cc->end(); ++i)\n ABSL_LOG(ERROR) << \" \" << i->lo << \"-\" << i->hi;\n }\n\n // If the class is too large, it's okay to overestimate.\n if (cc->size() > 10)\n return AnyCharOrAnyByte();\n\n Prefilter::Info *a = new Prefilter::Info();\n for (CCIter i = cc->begin(); i != cc->end(); ++i)\n for (Rune r = i->lo; r <= i->hi; r++) {\n if (latin1) {\n a->exact_.insert(RuneToStringLatin1(ToLowerRuneLatin1(r)));\n } else {\n a->exact_.insert(RuneToString(ToLowerRune(r)));\n }\n }\n\n\n a->is_exact_ = true;\n\n if (ExtraDebug)\n ABSL_LOG(ERROR) << \" = \" << a->ToString();\n\n return a;\n}\n\nclass Prefilter::Info::Walker : public Regexp::Walker {\n public:\n Walker(bool latin1) : latin1_(latin1) {}\n\n virtual Info* PostVisit(\n Regexp* re, Info* parent_arg,\n Info* pre_arg,\n Info** child_args, int nchild_args);\n\n virtual Info* ShortVisit(\n Regexp* re,\n Info* parent_arg);\n\n bool latin1() { return latin1_; }\n private:\n bool latin1_;\n\n Walker(const Walker&) = delete;\n Walker& operator=(const Walker&) = delete;\n};\n\nPrefilter::Info* Prefilter::BuildInfo(Regexp* re) {\n if (ExtraDebug)\n ABSL_LOG(ERROR) << \"BuildPrefilter::Info: \" << re->ToString();\n\n bool latin1 = (re->parse_flags() & Regexp::Latin1) != 0;\n Prefilter::Info::Walker w(latin1);\n Prefilter::Info* info = w.WalkExponential(re, NULL, 100000);\n\n if (w.stopped_early()) {\n delete info;\n return NULL;\n }\n\n return info;\n}\n\nPrefilter::Info* Prefilter::Info::Walker::ShortVisit(\n Regexp* re, Prefilter::Info* parent_arg) {\n return AnyMatch();\n}\n\n// Constructs the Prefilter::Info for the given regular expression.\n// Assumes re is simplified.\nPrefilter::Info* Prefilter::Info::Walker::PostVisit(\n Regexp* re, Prefilter::Info* parent_arg,\n Prefilter::Info* pre_arg, Prefilter::Info** child_args,\n int nchild_args) {\n Prefilter::Info *info;\n switch (re->op()) {\n default:\n case kRegexpRepeat:\n info = EmptyString();\n ABSL_LOG(DFATAL) << \"Bad regexp op \" << re->op();\n break;\n\n case kRegexpNoMatch:\n info = NoMatch();\n break;\n\n // These ops match the empty string:\n case kRegexpEmptyMatch: // anywhere\n case kRegexpBeginLine: // at beginning of line\n case kRegexpEndLine: // at end of line\n case kRegexpBeginText: // at beginning of text\n case kRegexpEndText: // at end of text\n case kRegexpWordBoundary: // at word boundary\n case kRegexpNoWordBoundary: // not at word boundary\n info = EmptyString();\n break;\n\n case kRegexpLiteral:\n if (latin1()) {\n info = LiteralLatin1(re->rune());\n }\n else {\n info = Literal(re->rune());\n }\n break;\n\n case kRegexpLiteralString:\n if (re->nrunes() == 0) {\n info = NoMatch();\n break;\n }\n if (latin1()) {\n info = LiteralLatin1(re->runes()[0]);\n for (int i = 1; i < re->nrunes(); i++) {\n info = Concat(info, LiteralLatin1(re->runes()[i]));\n }\n } else {\n info = Literal(re->runes()[0]);\n for (int i = 1; i < re->nrunes(); i++) {\n info = Concat(info, Literal(re->runes()[i]));\n }\n }\n break;\n\n case kRegexpConcat: {\n // Accumulate in info.\n // Exact is concat of recent contiguous exact nodes.\n info = NULL;\n Info* exact = NULL;\n for (int i = 0; i < nchild_args; i++) {\n Info* ci = child_args[i]; // child info\n if (!ci->is_exact() ||\n (exact && ci->exact().size() * exact->exact().size() > 16)) {\n // Exact run is over.\n info = And(info, exact);\n exact = NULL;\n // Add this child's info.\n info = And(info, ci);\n } else {\n // Append to exact run.\n exact = Concat(exact, ci);\n }\n }\n info = And(info, exact);\n }\n break;\n\n case kRegexpAlternate:\n info = child_args[0];\n for (int i = 1; i < nchild_args; i++)\n info = Alt(info, child_args[i]);\n break;\n\n case kRegexpStar:\n info = Star(child_args[0]);\n break;\n\n case kRegexpQuest:\n info = Quest(child_args[0]);\n break;\n\n case kRegexpPlus:\n info = Plus(child_args[0]);\n break;\n\n case kRegexpAnyChar:\n case kRegexpAnyByte:\n // Claim nothing, except that it's not empty.\n info = AnyCharOrAnyByte();\n break;\n\n case kRegexpCharClass:\n info = CClass(re->cc(), latin1());\n break;\n\n case kRegexpCapture:\n // These don't affect the set of matching strings.\n info = child_args[0];\n break;\n }\n\n if (ExtraDebug)\n ABSL_LOG(ERROR) << \"BuildInfo \" << re->ToString()\n << \": \" << (info ? info->ToString() : \"\");\n\n return info;\n}\n\n\nPrefilter* Prefilter::FromRegexp(Regexp* re) {\n if (re == NULL)\n return NULL;\n\n Regexp* simple = re->Simplify();\n if (simple == NULL)\n return NULL;\n\n Prefilter::Info* info = BuildInfo(simple);\n simple->Decref();\n if (info == NULL)\n return NULL;\n\n Prefilter* m = info->TakeMatch();\n delete info;\n return m;\n}\n\nstd::string Prefilter::DebugString() const {\n switch (op_) {\n default:\n ABSL_LOG(DFATAL) << \"Bad op in Prefilter::DebugString: \" << op_;\n return absl::StrFormat(\"op%d\", op_);\n case NONE:\n return \"*no-matches*\";\n case ATOM:\n return atom_;\n case ALL:\n return \"\";\n case AND: {\n std::string s = \"\";\n for (size_t i = 0; i < subs_->size(); i++) {\n if (i > 0)\n s += \" \";\n Prefilter* sub = (*subs_)[i];\n s += sub ? sub->DebugString() : \"\";\n }\n return s;\n }\n case OR: {\n std::string s = \"(\";\n for (size_t i = 0; i < subs_->size(); i++) {\n if (i > 0)\n s += \"|\";\n Prefilter* sub = (*subs_)[i];\n s += sub ? sub->DebugString() : \"\";\n }\n s += \")\";\n return s;\n }\n }\n}\n\nPrefilter* Prefilter::FromRE2(const RE2* re2) {\n if (re2 == NULL)\n return NULL;\n\n Regexp* regexp = re2->Regexp();\n if (regexp == NULL)\n return NULL;\n\n return FromRegexp(regexp);\n}\n\n\n} // namespace re2\n" }, { "path": "re2/prefilter.h", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_PREFILTER_H_\n#define RE2_PREFILTER_H_\n\n// Prefilter is the class used to extract string guards from regexps.\n// Rather than using Prefilter class directly, use FilteredRE2.\n// See filtered_re2.h\n\n#include \n#include \n#include \n\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n\nnamespace re2 {\n\nclass RE2;\n\nclass Regexp;\n\nclass Prefilter {\n // Instead of using Prefilter directly, use FilteredRE2; see filtered_re2.h\n public:\n enum Op {\n ALL = 0, // Everything matches\n NONE, // Nothing matches\n ATOM, // The string atom() must match\n AND, // All in subs() must match\n OR, // One of subs() must match\n };\n\n explicit Prefilter(Op op);\n ~Prefilter();\n\n Op op() { return op_; }\n const std::string& atom() const { return atom_; }\n void set_unique_id(int id) { unique_id_ = id; }\n int unique_id() const { return unique_id_; }\n\n // The children of the Prefilter node.\n std::vector* subs() {\n ABSL_DCHECK(op_ == AND || op_ == OR);\n return subs_;\n }\n\n // Set the children vector. Prefilter takes ownership of subs and\n // subs_ will be deleted when Prefilter is deleted.\n void set_subs(std::vector* subs) { subs_ = subs; }\n\n // Given a RE2, return a Prefilter. The caller takes ownership of\n // the Prefilter and should deallocate it. Returns NULL if Prefilter\n // cannot be formed.\n static Prefilter* FromRE2(const RE2* re2);\n\n // Returns a readable debug string of the prefilter.\n std::string DebugString() const;\n\n private:\n template \n friend H AbslHashValue(H h, const Prefilter& a) {\n h = H::combine(std::move(h), a.op_);\n if (a.op_ == ATOM) {\n h = H::combine(std::move(h), a.atom_);\n } else if (a.op_ == AND || a.op_ == OR) {\n h = H::combine(std::move(h), a.subs_->size());\n for (size_t i = 0; i < a.subs_->size(); ++i) {\n h = H::combine(std::move(h), (*a.subs_)[i]->unique_id_);\n }\n }\n return h;\n }\n\n friend bool operator==(const Prefilter& a, const Prefilter& b) {\n if (&a == &b) {\n return true;\n }\n if (a.op_ != b.op_) {\n return false;\n }\n if (a.op_ == ATOM) {\n if (a.atom_ != b.atom_) {\n return false;\n }\n } else if (a.op_ == AND || a.op_ == OR) {\n if (a.subs_->size() != b.subs_->size()) {\n return false;\n }\n for (size_t i = 0; i < a.subs_->size(); ++i) {\n if ((*a.subs_)[i]->unique_id_ != (*b.subs_)[i]->unique_id_) {\n return false;\n }\n }\n }\n return true;\n }\n\n // A comparator used to store exact strings. We compare by length,\n // then lexicographically. This ordering makes it easier to reduce the\n // set of strings in SimplifyStringSet.\n struct LengthThenLex {\n bool operator()(const std::string& a, const std::string& b) const {\n return (a.size() < b.size()) || (a.size() == b.size() && a < b);\n }\n };\n\n class Info;\n\n using SSet = std::set;\n using SSIter = SSet::iterator;\n using ConstSSIter = SSet::const_iterator;\n\n // Combines two prefilters together to create an AND. The passed\n // Prefilters will be part of the returned Prefilter or deleted.\n static Prefilter* And(Prefilter* a, Prefilter* b);\n\n // Combines two prefilters together to create an OR. The passed\n // Prefilters will be part of the returned Prefilter or deleted.\n static Prefilter* Or(Prefilter* a, Prefilter* b);\n\n // Generalized And/Or\n static Prefilter* AndOr(Op op, Prefilter* a, Prefilter* b);\n\n static Prefilter* FromRegexp(Regexp* a);\n\n static Prefilter* FromString(const std::string& str);\n\n static Prefilter* OrStrings(SSet* ss);\n\n static Info* BuildInfo(Regexp* re);\n\n Prefilter* Simplify();\n\n // Removes redundant strings from the set. A string is redundant if\n // any of the other strings appear as a substring. The empty string\n // is a special case, which is ignored.\n static void SimplifyStringSet(SSet* ss);\n\n // Adds the cross-product of a and b to dst.\n // (For each string i in a and j in b, add i+j.)\n static void CrossProduct(const SSet& a, const SSet& b, SSet* dst);\n\n // Kind of Prefilter.\n Op op_;\n\n // Sub-matches for AND or OR Prefilter.\n std::vector* subs_;\n\n // Actual string to match in leaf node.\n std::string atom_;\n\n // If different prefilters have the same string atom, or if they are\n // structurally the same (e.g., OR of same atom strings) they are\n // considered the same unique nodes. This is the id for each unique\n // node. This field is populated with a unique id for every node,\n // and -1 for duplicate nodes.\n int unique_id_;\n\n Prefilter(const Prefilter&) = delete;\n Prefilter& operator=(const Prefilter&) = delete;\n};\n\n} // namespace re2\n\n#endif // RE2_PREFILTER_H_\n" }, { "path": "re2/prefilter_tree.cc", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \"re2/prefilter_tree.h\"\n\n#include \n\n#include \n#include \n#include \n#include \n#include \n\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"re2/prefilter.h\"\n\nnamespace re2 {\n\nstatic const bool ExtraDebug = false;\n\nPrefilterTree::PrefilterTree()\n : compiled_(false),\n min_atom_len_(3) {\n}\n\nPrefilterTree::PrefilterTree(int min_atom_len)\n : compiled_(false),\n min_atom_len_(min_atom_len) {\n}\n\nPrefilterTree::~PrefilterTree() {\n for (size_t i = 0; i < prefilter_vec_.size(); i++)\n delete prefilter_vec_[i];\n}\n\nvoid PrefilterTree::Add(Prefilter* prefilter) {\n if (compiled_) {\n ABSL_LOG(DFATAL) << \"Add called after Compile.\";\n return;\n }\n if (prefilter != NULL && !KeepNode(prefilter)) {\n delete prefilter;\n prefilter = NULL;\n }\n\n prefilter_vec_.push_back(prefilter);\n}\n\nvoid PrefilterTree::Compile(std::vector* atom_vec) {\n if (compiled_) {\n ABSL_LOG(DFATAL) << \"Compile called already.\";\n return;\n }\n\n // Some legacy users of PrefilterTree call Compile() before\n // adding any regexps and expect Compile() to have no effect.\n if (prefilter_vec_.empty()) {\n return;\n }\n\n compiled_ = true;\n\n NodeSet nodes;\n AssignUniqueIds(&nodes, atom_vec);\n if (ExtraDebug)\n PrintDebugInfo(&nodes);\n}\n\nPrefilter* PrefilterTree::CanonicalNode(NodeSet* nodes, Prefilter* node) {\n NodeSet::const_iterator iter = nodes->find(node);\n if (iter != nodes->end()) {\n return *iter;\n }\n return NULL;\n}\n\nbool PrefilterTree::KeepNode(Prefilter* node) const {\n if (node == NULL)\n return false;\n\n switch (node->op()) {\n default:\n ABSL_LOG(DFATAL) << \"Unexpected op in KeepNode: \" << node->op();\n return false;\n\n case Prefilter::ALL:\n case Prefilter::NONE:\n return false;\n\n case Prefilter::ATOM:\n return node->atom().size() >= static_cast(min_atom_len_);\n\n case Prefilter::AND: {\n int j = 0;\n std::vector* subs = node->subs();\n for (size_t i = 0; i < subs->size(); i++)\n if (KeepNode((*subs)[i]))\n (*subs)[j++] = (*subs)[i];\n else\n delete (*subs)[i];\n\n subs->resize(j);\n return j > 0;\n }\n\n case Prefilter::OR:\n for (size_t i = 0; i < node->subs()->size(); i++)\n if (!KeepNode((*node->subs())[i]))\n return false;\n return true;\n }\n}\n\nvoid PrefilterTree::AssignUniqueIds(NodeSet* nodes,\n std::vector* atom_vec) {\n atom_vec->clear();\n\n // Build vector of all filter nodes, sorted topologically\n // from top to bottom in v.\n std::vector v;\n\n // Add the top level nodes of each regexp prefilter.\n for (size_t i = 0; i < prefilter_vec_.size(); i++) {\n Prefilter* f = prefilter_vec_[i];\n if (f == NULL)\n unfiltered_.push_back(static_cast(i));\n\n // We push NULL also on to v, so that we maintain the\n // mapping of index==regexpid for level=0 prefilter nodes.\n v.push_back(f);\n }\n\n // Now add all the descendant nodes.\n for (size_t i = 0; i < v.size(); i++) {\n Prefilter* f = v[i];\n if (f == NULL)\n continue;\n if (f->op() == Prefilter::AND || f->op() == Prefilter::OR) {\n const std::vector& subs = *f->subs();\n for (size_t j = 0; j < subs.size(); j++)\n v.push_back(subs[j]);\n }\n }\n\n // Identify unique nodes.\n int unique_id = 0;\n for (int i = static_cast(v.size()) - 1; i >= 0; i--) {\n Prefilter *node = v[i];\n if (node == NULL)\n continue;\n node->set_unique_id(-1);\n Prefilter* canonical = CanonicalNode(nodes, node);\n if (canonical == NULL) {\n // Any further nodes that have the same atom/subs\n // will find this node as the canonical node.\n nodes->emplace(node);\n if (node->op() == Prefilter::ATOM) {\n atom_vec->push_back(node->atom());\n atom_index_to_id_.push_back(unique_id);\n }\n node->set_unique_id(unique_id++);\n } else {\n node->set_unique_id(canonical->unique_id());\n }\n }\n entries_.resize(unique_id);\n\n // Fill the entries.\n for (int i = static_cast(v.size()) - 1; i >= 0; i--) {\n Prefilter* prefilter = v[i];\n if (prefilter == NULL)\n continue;\n if (CanonicalNode(nodes, prefilter) != prefilter)\n continue;\n int id = prefilter->unique_id();\n switch (prefilter->op()) {\n default:\n ABSL_LOG(DFATAL) << \"Unexpected op: \" << prefilter->op();\n return;\n\n case Prefilter::ATOM:\n entries_[id].propagate_up_at_count = 1;\n break;\n\n case Prefilter::OR:\n case Prefilter::AND: {\n // For each child, we append our id to the child's list of\n // parent ids... unless we happen to have done so already.\n // The number of appends is the number of unique children,\n // which allows correct upward propagation from AND nodes.\n int up_count = 0;\n for (size_t j = 0; j < prefilter->subs()->size(); j++) {\n int child_id = (*prefilter->subs())[j]->unique_id();\n std::vector& parents = entries_[child_id].parents;\n if (parents.empty() || parents.back() != id) {\n parents.push_back(id);\n up_count++;\n }\n }\n entries_[id].propagate_up_at_count =\n prefilter->op() == Prefilter::AND ? up_count : 1;\n break;\n }\n }\n }\n\n // For top level nodes, populate regexp id.\n for (size_t i = 0; i < prefilter_vec_.size(); i++) {\n if (prefilter_vec_[i] == NULL)\n continue;\n int id = CanonicalNode(nodes, prefilter_vec_[i])->unique_id();\n ABSL_DCHECK_LE(0, id);\n Entry* entry = &entries_[id];\n entry->regexps.push_back(static_cast(i));\n }\n\n // Lastly, using probability-based heuristics, we identify nodes\n // that trigger too many parents and then we try to prune edges.\n // We use logarithms below to avoid the likelihood of underflow.\n double log_num_regexps = std::log(prefilter_vec_.size() - unfiltered_.size());\n // Hoisted this above the loop so that we don't thrash the heap.\n std::vector> entries_by_num_edges;\n for (int i = static_cast(v.size()) - 1; i >= 0; i--) {\n Prefilter* prefilter = v[i];\n // Pruning applies only to AND nodes because it \"just\" reduces\n // precision; applied to OR nodes, it would break correctness.\n if (prefilter == NULL || prefilter->op() != Prefilter::AND)\n continue;\n if (CanonicalNode(nodes, prefilter) != prefilter)\n continue;\n int id = prefilter->unique_id();\n\n // Sort the current node's children by the numbers of parents.\n entries_by_num_edges.clear();\n for (size_t j = 0; j < prefilter->subs()->size(); j++) {\n int child_id = (*prefilter->subs())[j]->unique_id();\n const std::vector& parents = entries_[child_id].parents;\n entries_by_num_edges.emplace_back(parents.size(), child_id);\n }\n std::stable_sort(entries_by_num_edges.begin(), entries_by_num_edges.end());\n\n // A running estimate of how many regexps will be triggered by\n // pruning the remaining children's edges to the current node.\n // Our nominal target is one, so the threshold is log(1) == 0;\n // pruning occurs iff the child has more than nine edges left.\n double log_num_triggered = log_num_regexps;\n for (const auto& pair : entries_by_num_edges) {\n int child_id = pair.second;\n std::vector& parents = entries_[child_id].parents;\n if (log_num_triggered > 0.) {\n log_num_triggered += std::log(parents.size());\n log_num_triggered -= log_num_regexps;\n } else if (parents.size() > 9) {\n auto it = std::find(parents.begin(), parents.end(), id);\n if (it != parents.end()) {\n parents.erase(it);\n entries_[id].propagate_up_at_count--;\n }\n }\n }\n }\n}\n\n// Functions for triggering during search.\nvoid PrefilterTree::RegexpsGivenStrings(\n const std::vector& matched_atoms,\n std::vector* regexps) const {\n regexps->clear();\n if (!compiled_) {\n // Some legacy users of PrefilterTree call Compile() before\n // adding any regexps and expect Compile() to have no effect.\n // This kludge is a counterpart to that kludge.\n if (prefilter_vec_.empty()) {\n return;\n }\n\n ABSL_LOG(ERROR) << \"RegexpsGivenStrings called before Compile.\";\n for (size_t i = 0; i < prefilter_vec_.size(); i++)\n regexps->push_back(static_cast(i));\n } else {\n IntMap regexps_map(static_cast(prefilter_vec_.size()));\n std::vector matched_atom_ids;\n for (size_t j = 0; j < matched_atoms.size(); j++)\n matched_atom_ids.push_back(atom_index_to_id_[matched_atoms[j]]);\n PropagateMatch(matched_atom_ids, ®exps_map);\n for (IntMap::const_iterator it = regexps_map.begin();\n it != regexps_map.end();\n ++it)\n regexps->push_back(it->index());\n\n regexps->insert(regexps->end(), unfiltered_.begin(), unfiltered_.end());\n }\n std::sort(regexps->begin(), regexps->end());\n}\n\nvoid PrefilterTree::PropagateMatch(const std::vector& atom_ids,\n IntMap* regexps) const {\n IntMap count(static_cast(entries_.size()));\n IntMap work(static_cast(entries_.size()));\n for (size_t i = 0; i < atom_ids.size(); i++)\n work.set(atom_ids[i], 1);\n for (IntMap::const_iterator it = work.begin(); it != work.end(); ++it) {\n const Entry& entry = entries_[it->index()];\n // Record regexps triggered.\n for (size_t i = 0; i < entry.regexps.size(); i++)\n regexps->set(entry.regexps[i], 1);\n int c;\n // Pass trigger up to parents.\n for (int j : entry.parents) {\n const Entry& parent = entries_[j];\n // Delay until all the children have succeeded.\n if (parent.propagate_up_at_count > 1) {\n if (count.has_index(j)) {\n c = count.get_existing(j) + 1;\n count.set_existing(j, c);\n } else {\n c = 1;\n count.set_new(j, c);\n }\n if (c < parent.propagate_up_at_count)\n continue;\n }\n // Trigger the parent.\n work.set(j, 1);\n }\n }\n}\n\n// Debugging help.\nvoid PrefilterTree::PrintPrefilter(int regexpid) {\n ABSL_LOG(ERROR) << DebugNodeString(prefilter_vec_[regexpid]);\n}\n\nvoid PrefilterTree::PrintDebugInfo(NodeSet* nodes) {\n ABSL_LOG(ERROR) << \"#Unique Atoms: \" << atom_index_to_id_.size();\n ABSL_LOG(ERROR) << \"#Unique Nodes: \" << entries_.size();\n\n for (size_t i = 0; i < entries_.size(); i++) {\n const std::vector& parents = entries_[i].parents;\n const std::vector& regexps = entries_[i].regexps;\n ABSL_LOG(ERROR) << \"EntryId: \" << i\n << \" N: \" << parents.size() << \" R: \" << regexps.size();\n for (int parent : parents)\n ABSL_LOG(ERROR) << parent;\n }\n ABSL_LOG(ERROR) << \"Set:\";\n for (NodeSet::const_iterator iter = nodes->begin();\n iter != nodes->end(); ++iter)\n ABSL_LOG(ERROR) << \"NodeId: \" << (*iter)->unique_id();\n}\n\nstd::string PrefilterTree::DebugNodeString(Prefilter* node) const {\n std::string node_string = \"\";\n if (node->op() == Prefilter::ATOM) {\n ABSL_DCHECK(!node->atom().empty());\n node_string += node->atom();\n } else {\n // Adding the operation disambiguates AND and OR nodes.\n node_string += node->op() == Prefilter::AND ? \"AND\" : \"OR\";\n node_string += \"(\";\n for (size_t i = 0; i < node->subs()->size(); i++) {\n if (i > 0)\n node_string += ',';\n node_string += absl::StrFormat(\"%d\", (*node->subs())[i]->unique_id());\n node_string += \":\";\n node_string += DebugNodeString((*node->subs())[i]);\n }\n node_string += \")\";\n }\n return node_string;\n}\n\n} // namespace re2\n" }, { "path": "re2/prefilter_tree.h", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_PREFILTER_TREE_H_\n#define RE2_PREFILTER_TREE_H_\n\n// The PrefilterTree class is used to form an AND-OR tree of strings\n// that would trigger each regexp. The 'prefilter' of each regexp is\n// added to PrefilterTree, and then PrefilterTree is used to find all\n// the unique strings across the prefilters. During search, by using\n// matches from a string matching engine, PrefilterTree deduces the\n// set of regexps that are to be triggered. The 'string matching\n// engine' itself is outside of this class, and the caller can use any\n// favorite engine. PrefilterTree provides a set of strings (called\n// atoms) that the user of this class should use to do the string\n// matching.\n\n#include \n#include \n\n#include \"absl/container/flat_hash_set.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"re2/prefilter.h\"\n#include \"re2/sparse_array.h\"\n\nnamespace re2 {\n\nclass PrefilterTree {\n public:\n PrefilterTree();\n explicit PrefilterTree(int min_atom_len);\n ~PrefilterTree();\n\n // Adds the prefilter for the next regexp. Note that we assume that\n // Add called sequentially for all regexps. All Add calls\n // must precede Compile.\n void Add(Prefilter* prefilter);\n\n // The Compile returns a vector of string in atom_vec.\n // Call this after all the prefilters are added through Add.\n // No calls to Add after Compile are allowed.\n // The caller should use the returned set of strings to do string matching.\n // Each time a string matches, the corresponding index then has to be\n // and passed to RegexpsGivenStrings below.\n void Compile(std::vector* atom_vec);\n\n // Given the indices of the atoms that matched, returns the indexes\n // of regexps that should be searched. The matched_atoms should\n // contain all the ids of string atoms that were found to match the\n // content. The caller can use any string match engine to perform\n // this function. This function is thread safe.\n void RegexpsGivenStrings(const std::vector& matched_atoms,\n std::vector* regexps) const;\n\n // Print debug prefilter. Also prints unique ids associated with\n // nodes of the prefilter of the regexp.\n void PrintPrefilter(int regexpid);\n\n private:\n using IntMap = SparseArray;\n\n struct PrefilterHash {\n size_t operator()(const Prefilter* a) const {\n ABSL_DCHECK(a != NULL);\n return absl::Hash()(*a);\n }\n };\n\n struct PrefilterEqual {\n bool operator()(const Prefilter* a, const Prefilter* b) const {\n ABSL_DCHECK(a != NULL);\n ABSL_DCHECK(b != NULL);\n return *a == *b;\n }\n };\n\n using NodeSet =\n absl::flat_hash_set;\n\n // Each unique node has a corresponding Entry that helps in\n // passing the matching trigger information along the tree.\n struct Entry {\n public:\n // How many children should match before this node triggers the\n // parent. For an atom and an OR node, this is 1 and for an AND\n // node, it is the number of unique children.\n int propagate_up_at_count;\n\n // When this node is ready to trigger the parent, what are the indices\n // of the parent nodes to trigger. The reason there may be more than\n // one is because of sharing. For example (abc | def) and (xyz | def)\n // are two different nodes, but they share the atom 'def'. So when\n // 'def' matches, it triggers two parents, corresponding to the two\n // different OR nodes.\n std::vector parents;\n\n // When this node is ready to trigger the parent, what are the\n // regexps that are triggered.\n std::vector regexps;\n };\n\n // Returns true if the prefilter node should be kept.\n bool KeepNode(Prefilter* node) const;\n\n // This function assigns unique ids to various parts of the\n // prefilter, by looking at if these nodes are already in the\n // PrefilterTree.\n void AssignUniqueIds(NodeSet* nodes, std::vector* atom_vec);\n\n // Given the matching atoms, find the regexps to be triggered.\n void PropagateMatch(const std::vector& atom_ids,\n IntMap* regexps) const;\n\n // Returns the prefilter node that has the same atom/subs as this\n // node. For the canonical node, returns node. Assumes that the\n // children of node have already been assigned unique ids.\n Prefilter* CanonicalNode(NodeSet* nodes, Prefilter* node);\n\n // Recursively constructs a readable prefilter string.\n std::string DebugNodeString(Prefilter* node) const;\n\n // Used for debugging.\n void PrintDebugInfo(NodeSet* nodes);\n\n // These are all the nodes formed by Compile. Essentially, there is\n // one node for each unique atom and each unique AND/OR node.\n std::vector entries_;\n\n // indices of regexps that always pass through the filter (since we\n // found no required literals in these regexps).\n std::vector unfiltered_;\n\n // vector of Prefilter for all regexps.\n std::vector prefilter_vec_;\n\n // Atom index in returned strings to entry id mapping.\n std::vector atom_index_to_id_;\n\n // Has the prefilter tree been compiled.\n bool compiled_;\n\n // Strings less than this length are not stored as atoms.\n const int min_atom_len_;\n\n PrefilterTree(const PrefilterTree&) = delete;\n PrefilterTree& operator=(const PrefilterTree&) = delete;\n};\n\n} // namespace\n\n#endif // RE2_PREFILTER_TREE_H_\n" }, { "path": "re2/prog.cc", "content": "// Copyright 2007 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Compiled regular expression representation.\n// Tested by compile_test.cc\n\n#include \"re2/prog.h\"\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/bitmap256.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/sparse_array.h\"\n#include \"re2/sparse_set.h\"\n\n#if defined(__AVX2__)\n#include \n#ifdef _MSC_VER\n#include \n#endif\n#endif\n\nnamespace re2 {\n\n// Constructors per Inst opcode\n\nvoid Prog::Inst::InitAlt(uint32_t out, uint32_t out1) {\n ABSL_DCHECK_EQ(out_opcode_, uint32_t{0});\n set_out_opcode(out, kInstAlt);\n out1_ = out1;\n}\n\nvoid Prog::Inst::InitByteRange(int lo, int hi, int foldcase, uint32_t out) {\n ABSL_DCHECK_EQ(out_opcode_, uint32_t{0});\n set_out_opcode(out, kInstByteRange);\n lo_ = lo & 0xFF;\n hi_ = hi & 0xFF;\n hint_foldcase_ = foldcase&1;\n}\n\nvoid Prog::Inst::InitCapture(int cap, uint32_t out) {\n ABSL_DCHECK_EQ(out_opcode_, uint32_t{0});\n set_out_opcode(out, kInstCapture);\n cap_ = cap;\n}\n\nvoid Prog::Inst::InitEmptyWidth(EmptyOp empty, uint32_t out) {\n ABSL_DCHECK_EQ(out_opcode_, uint32_t{0});\n set_out_opcode(out, kInstEmptyWidth);\n empty_ = empty;\n}\n\nvoid Prog::Inst::InitMatch(int32_t id) {\n ABSL_DCHECK_EQ(out_opcode_, uint32_t{0});\n set_opcode(kInstMatch);\n match_id_ = id;\n}\n\nvoid Prog::Inst::InitNop(uint32_t out) {\n ABSL_DCHECK_EQ(out_opcode_, uint32_t{0});\n set_opcode(kInstNop);\n}\n\nvoid Prog::Inst::InitFail() {\n ABSL_DCHECK_EQ(out_opcode_, uint32_t{0});\n set_opcode(kInstFail);\n}\n\nstd::string Prog::Inst::Dump() {\n switch (opcode()) {\n default:\n return absl::StrFormat(\"opcode %d\", static_cast(opcode()));\n\n case kInstAlt:\n return absl::StrFormat(\"alt -> %d | %d\", out(), out1_);\n\n case kInstAltMatch:\n return absl::StrFormat(\"altmatch -> %d | %d\", out(), out1_);\n\n case kInstByteRange:\n return absl::StrFormat(\"byte%s [%02x-%02x] %d -> %d\",\n foldcase() ? \"/i\" : \"\",\n lo_, hi_, hint(), out());\n\n case kInstCapture:\n return absl::StrFormat(\"capture %d -> %d\", cap_, out());\n\n case kInstEmptyWidth:\n return absl::StrFormat(\"emptywidth %#x -> %d\",\n static_cast(empty_), out());\n\n case kInstMatch:\n return absl::StrFormat(\"match! %d\", match_id());\n\n case kInstNop:\n return absl::StrFormat(\"nop -> %d\", out());\n\n case kInstFail:\n return absl::StrFormat(\"fail\");\n }\n}\n\nProg::Prog()\n : anchor_start_(false),\n anchor_end_(false),\n reversed_(false),\n did_flatten_(false),\n did_onepass_(false),\n start_(0),\n start_unanchored_(0),\n size_(0),\n bytemap_range_(0),\n prefix_foldcase_(false),\n prefix_size_(0),\n list_count_(0),\n bit_state_text_max_size_(0),\n dfa_mem_(0),\n dfa_first_(NULL),\n dfa_longest_(NULL) {\n}\n\nProg::~Prog() {\n DeleteDFA(dfa_longest_);\n DeleteDFA(dfa_first_);\n if (prefix_foldcase_)\n delete[] prefix_dfa_;\n}\n\ntypedef SparseSet Workq;\n\nstatic inline void AddToQueue(Workq* q, int id) {\n if (id != 0)\n q->insert(id);\n}\n\nstatic std::string ProgToString(Prog* prog, Workq* q) {\n std::string s;\n for (Workq::iterator i = q->begin(); i != q->end(); ++i) {\n int id = *i;\n Prog::Inst* ip = prog->inst(id);\n s += absl::StrFormat(\"%d. %s\\n\", id, ip->Dump());\n AddToQueue(q, ip->out());\n if (ip->opcode() == kInstAlt || ip->opcode() == kInstAltMatch)\n AddToQueue(q, ip->out1());\n }\n return s;\n}\n\nstatic std::string FlattenedProgToString(Prog* prog, int start) {\n std::string s;\n for (int id = start; id < prog->size(); id++) {\n Prog::Inst* ip = prog->inst(id);\n if (ip->last())\n s += absl::StrFormat(\"%d. %s\\n\", id, ip->Dump());\n else\n s += absl::StrFormat(\"%d+ %s\\n\", id, ip->Dump());\n }\n return s;\n}\n\nstd::string Prog::Dump() {\n if (did_flatten_)\n return FlattenedProgToString(this, start_);\n\n Workq q(size_);\n AddToQueue(&q, start_);\n return ProgToString(this, &q);\n}\n\nstd::string Prog::DumpUnanchored() {\n if (did_flatten_)\n return FlattenedProgToString(this, start_unanchored_);\n\n Workq q(size_);\n AddToQueue(&q, start_unanchored_);\n return ProgToString(this, &q);\n}\n\nstd::string Prog::DumpByteMap() {\n std::string map;\n for (int c = 0; c < 256; c++) {\n int b = bytemap_[c];\n int lo = c;\n while (c < 256-1 && bytemap_[c+1] == b)\n c++;\n int hi = c;\n map += absl::StrFormat(\"[%02x-%02x] -> %d\\n\", lo, hi, b);\n }\n return map;\n}\n\n// Is ip a guaranteed match at end of text, perhaps after some capturing?\nstatic bool IsMatch(Prog* prog, Prog::Inst* ip) {\n for (;;) {\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"Unexpected opcode in IsMatch: \" << ip->opcode();\n return false;\n\n case kInstAlt:\n case kInstAltMatch:\n case kInstByteRange:\n case kInstFail:\n case kInstEmptyWidth:\n return false;\n\n case kInstCapture:\n case kInstNop:\n ip = prog->inst(ip->out());\n break;\n\n case kInstMatch:\n return true;\n }\n }\n}\n\n// Peep-hole optimizer.\nvoid Prog::Optimize() {\n Workq q(size_);\n\n // Eliminate nops. Most are taken out during compilation\n // but a few are hard to avoid.\n q.clear();\n AddToQueue(&q, start_);\n for (Workq::iterator i = q.begin(); i != q.end(); ++i) {\n int id = *i;\n\n Inst* ip = inst(id);\n int j = ip->out();\n Inst* jp;\n while (j != 0 && (jp=inst(j))->opcode() == kInstNop) {\n j = jp->out();\n }\n ip->set_out(j);\n AddToQueue(&q, ip->out());\n\n if (ip->opcode() == kInstAlt) {\n j = ip->out1();\n while (j != 0 && (jp=inst(j))->opcode() == kInstNop) {\n j = jp->out();\n }\n ip->out1_ = j;\n AddToQueue(&q, ip->out1());\n }\n }\n\n // Insert kInstAltMatch instructions\n // Look for\n // ip: Alt -> j | k\n //\t j: ByteRange [00-FF] -> ip\n // k: Match\n // or the reverse (the above is the greedy one).\n // Rewrite Alt to AltMatch.\n q.clear();\n AddToQueue(&q, start_);\n for (Workq::iterator i = q.begin(); i != q.end(); ++i) {\n int id = *i;\n Inst* ip = inst(id);\n AddToQueue(&q, ip->out());\n if (ip->opcode() == kInstAlt)\n AddToQueue(&q, ip->out1());\n\n if (ip->opcode() == kInstAlt) {\n Inst* j = inst(ip->out());\n Inst* k = inst(ip->out1());\n if (j->opcode() == kInstByteRange && j->out() == id &&\n j->lo() == 0x00 && j->hi() == 0xFF &&\n IsMatch(this, k)) {\n ip->set_opcode(kInstAltMatch);\n continue;\n }\n if (IsMatch(this, j) &&\n k->opcode() == kInstByteRange && k->out() == id &&\n k->lo() == 0x00 && k->hi() == 0xFF) {\n ip->set_opcode(kInstAltMatch);\n }\n }\n }\n}\n\nuint32_t Prog::EmptyFlags(absl::string_view text, const char* p) {\n int flags = 0;\n\n // ^ and \\A\n if (p == text.data())\n flags |= kEmptyBeginText | kEmptyBeginLine;\n else if (p[-1] == '\\n')\n flags |= kEmptyBeginLine;\n\n // $ and \\z\n if (p == text.data() + text.size())\n flags |= kEmptyEndText | kEmptyEndLine;\n else if (p < text.data() + text.size() && p[0] == '\\n')\n flags |= kEmptyEndLine;\n\n // \\b and \\B\n if (p == text.data() && p == text.data() + text.size()) {\n // no word boundary here\n } else if (p == text.data()) {\n if (IsWordChar(p[0]))\n flags |= kEmptyWordBoundary;\n } else if (p == text.data() + text.size()) {\n if (IsWordChar(p[-1]))\n flags |= kEmptyWordBoundary;\n } else {\n if (IsWordChar(p[-1]) != IsWordChar(p[0]))\n flags |= kEmptyWordBoundary;\n }\n if (!(flags & kEmptyWordBoundary))\n flags |= kEmptyNonWordBoundary;\n\n return flags;\n}\n\n// ByteMapBuilder implements a coloring algorithm.\n//\n// The first phase is a series of \"mark and merge\" batches: we mark one or more\n// [lo-hi] ranges, then merge them into our internal state. Batching is not for\n// performance; rather, it means that the ranges are treated indistinguishably.\n//\n// Internally, the ranges are represented using a bitmap that stores the splits\n// and a vector that stores the colors; both of them are indexed by the ranges'\n// last bytes. Thus, in order to merge a [lo-hi] range, we split at lo-1 and at\n// hi (if not already split), then recolor each range in between. The color map\n// (i.e. from the old color to the new color) is maintained for the lifetime of\n// the batch and so underpins this somewhat obscure approach to set operations.\n//\n// The second phase builds the bytemap from our internal state: we recolor each\n// range, then store the new color (which is now the byte class) in each of the\n// corresponding array elements. Finally, we output the number of byte classes.\nclass ByteMapBuilder {\n public:\n ByteMapBuilder() {\n // Initial state: the [0-255] range has color 256.\n // This will avoid problems during the second phase,\n // in which we assign byte classes numbered from 0.\n splits_.Set(255);\n colors_[255] = 256;\n nextcolor_ = 257;\n }\n\n void Mark(int lo, int hi);\n void Merge();\n void Build(uint8_t* bytemap, int* bytemap_range);\n\n private:\n int Recolor(int oldcolor);\n\n Bitmap256 splits_;\n int colors_[256];\n int nextcolor_;\n std::vector> colormap_;\n std::vector> ranges_;\n\n ByteMapBuilder(const ByteMapBuilder&) = delete;\n ByteMapBuilder& operator=(const ByteMapBuilder&) = delete;\n};\n\nvoid ByteMapBuilder::Mark(int lo, int hi) {\n ABSL_DCHECK_GE(lo, 0);\n ABSL_DCHECK_GE(hi, 0);\n ABSL_DCHECK_LE(lo, 255);\n ABSL_DCHECK_LE(hi, 255);\n ABSL_DCHECK_LE(lo, hi);\n\n // Ignore any [0-255] ranges. They cause us to recolor every range, which\n // has no effect on the eventual result and is therefore a waste of time.\n if (lo == 0 && hi == 255)\n return;\n\n ranges_.emplace_back(lo, hi);\n}\n\nvoid ByteMapBuilder::Merge() {\n for (std::vector>::const_iterator it = ranges_.begin();\n it != ranges_.end();\n ++it) {\n int lo = it->first-1;\n int hi = it->second;\n\n if (0 <= lo && !splits_.Test(lo)) {\n splits_.Set(lo);\n int next = splits_.FindNextSetBit(lo+1);\n colors_[lo] = colors_[next];\n }\n if (!splits_.Test(hi)) {\n splits_.Set(hi);\n int next = splits_.FindNextSetBit(hi+1);\n colors_[hi] = colors_[next];\n }\n\n int c = lo+1;\n while (c < 256) {\n int next = splits_.FindNextSetBit(c);\n colors_[next] = Recolor(colors_[next]);\n if (next == hi)\n break;\n c = next+1;\n }\n }\n colormap_.clear();\n ranges_.clear();\n}\n\nvoid ByteMapBuilder::Build(uint8_t* bytemap, int* bytemap_range) {\n // Assign byte classes numbered from 0.\n nextcolor_ = 0;\n\n int c = 0;\n while (c < 256) {\n int next = splits_.FindNextSetBit(c);\n uint8_t b = static_cast(Recolor(colors_[next]));\n while (c <= next) {\n bytemap[c] = b;\n c++;\n }\n }\n\n *bytemap_range = nextcolor_;\n}\n\nint ByteMapBuilder::Recolor(int oldcolor) {\n // Yes, this is a linear search. There can be at most 256\n // colors and there will typically be far fewer than that.\n // Also, we need to consider keys *and* values in order to\n // avoid recoloring a given range more than once per batch.\n std::vector>::const_iterator it =\n std::find_if(colormap_.begin(), colormap_.end(),\n [=](const std::pair& kv) -> bool {\n return kv.first == oldcolor || kv.second == oldcolor;\n });\n if (it != colormap_.end())\n return it->second;\n int newcolor = nextcolor_;\n nextcolor_++;\n colormap_.emplace_back(oldcolor, newcolor);\n return newcolor;\n}\n\nvoid Prog::ComputeByteMap() {\n // Fill in bytemap with byte classes for the program.\n // Ranges of bytes that are treated indistinguishably\n // will be mapped to a single byte class.\n ByteMapBuilder builder;\n\n // Don't repeat the work for ^ and $.\n bool marked_line_boundaries = false;\n // Don't repeat the work for \\b and \\B.\n bool marked_word_boundaries = false;\n\n for (int id = 0; id < size(); id++) {\n Inst* ip = inst(id);\n if (ip->opcode() == kInstByteRange) {\n int lo = ip->lo();\n int hi = ip->hi();\n builder.Mark(lo, hi);\n if (ip->foldcase() && lo <= 'z' && hi >= 'a') {\n int foldlo = lo;\n int foldhi = hi;\n if (foldlo < 'a')\n foldlo = 'a';\n if (foldhi > 'z')\n foldhi = 'z';\n if (foldlo <= foldhi) {\n foldlo += 'A' - 'a';\n foldhi += 'A' - 'a';\n builder.Mark(foldlo, foldhi);\n }\n }\n // If this Inst is not the last Inst in its list AND the next Inst is\n // also a ByteRange AND the Insts have the same out, defer the merge.\n if (!ip->last() &&\n inst(id+1)->opcode() == kInstByteRange &&\n ip->out() == inst(id+1)->out())\n continue;\n builder.Merge();\n } else if (ip->opcode() == kInstEmptyWidth) {\n if (ip->empty() & (kEmptyBeginLine|kEmptyEndLine) &&\n !marked_line_boundaries) {\n builder.Mark('\\n', '\\n');\n builder.Merge();\n marked_line_boundaries = true;\n }\n if (ip->empty() & (kEmptyWordBoundary|kEmptyNonWordBoundary) &&\n !marked_word_boundaries) {\n // We require two batches here: the first for ranges that are word\n // characters, the second for ranges that are not word characters.\n for (bool isword : {true, false}) {\n int j;\n for (int i = 0; i < 256; i = j) {\n for (j = i + 1; j < 256 &&\n Prog::IsWordChar(static_cast(i)) ==\n Prog::IsWordChar(static_cast(j));\n j++)\n ;\n if (Prog::IsWordChar(static_cast(i)) == isword)\n builder.Mark(i, j - 1);\n }\n builder.Merge();\n }\n marked_word_boundaries = true;\n }\n }\n }\n\n builder.Build(bytemap_, &bytemap_range_);\n\n if ((0)) { // For debugging, use trivial bytemap.\n ABSL_LOG(ERROR) << \"Using trivial bytemap.\";\n for (int i = 0; i < 256; i++)\n bytemap_[i] = static_cast(i);\n bytemap_range_ = 256;\n }\n}\n\n// Prog::Flatten() implements a graph rewriting algorithm.\n//\n// The overall process is similar to epsilon removal, but retains some epsilon\n// transitions: those from Capture and EmptyWidth instructions; and those from\n// nullable subexpressions. (The latter avoids quadratic blowup in transitions\n// in the worst case.) It might be best thought of as Alt instruction elision.\n//\n// In conceptual terms, it divides the Prog into \"trees\" of instructions, then\n// traverses the \"trees\" in order to produce \"lists\" of instructions. A \"tree\"\n// is one or more instructions that grow from one \"root\" instruction to one or\n// more \"leaf\" instructions; if a \"tree\" has exactly one instruction, then the\n// \"root\" is also the \"leaf\". In most cases, a \"root\" is the successor of some\n// \"leaf\" (i.e. the \"leaf\" instruction's out() returns the \"root\" instruction)\n// and is considered a \"successor root\". A \"leaf\" can be a ByteRange, Capture,\n// EmptyWidth or Match instruction. However, this is insufficient for handling\n// nested nullable subexpressions correctly, so in some cases, a \"root\" is the\n// dominator of the instructions reachable from some \"successor root\" (i.e. it\n// has an unreachable predecessor) and is considered a \"dominator root\". Since\n// only Alt instructions can be \"dominator roots\" (other instructions would be\n// \"leaves\"), only Alt instructions are required to be marked as predecessors.\n//\n// Dividing the Prog into \"trees\" comprises two passes: marking the \"successor\n// roots\" and the predecessors; and marking the \"dominator roots\". Sorting the\n// \"successor roots\" by their bytecode offsets enables iteration in order from\n// greatest to least during the second pass; by working backwards in this case\n// and flooding the graph no further than \"leaves\" and already marked \"roots\",\n// it becomes possible to mark \"dominator roots\" without doing excessive work.\n//\n// Traversing the \"trees\" is just iterating over the \"roots\" in order of their\n// marking and flooding the graph no further than \"leaves\" and \"roots\". When a\n// \"leaf\" is reached, the instruction is copied with its successor remapped to\n// its \"root\" number. When a \"root\" is reached, a Nop instruction is generated\n// with its successor remapped similarly. As each \"list\" is produced, its last\n// instruction is marked as such. After all of the \"lists\" have been produced,\n// a pass over their instructions remaps their successors to bytecode offsets.\nvoid Prog::Flatten() {\n if (did_flatten_)\n return;\n did_flatten_ = true;\n\n // Scratch structures. It's important that these are reused by functions\n // that we call in loops because they would thrash the heap otherwise.\n SparseSet reachable(size());\n std::vector stk;\n stk.reserve(size());\n\n // First pass: Marks \"successor roots\" and predecessors.\n // Builds the mapping from inst-ids to root-ids.\n SparseArray rootmap(size());\n SparseArray predmap(size());\n std::vector> predvec;\n MarkSuccessors(&rootmap, &predmap, &predvec, &reachable, &stk);\n\n // Second pass: Marks \"dominator roots\".\n SparseArray sorted(rootmap);\n std::sort(sorted.begin(), sorted.end(), sorted.less);\n for (SparseArray::const_iterator i = sorted.end() - 1;\n i != sorted.begin();\n --i) {\n if (i->index() != start_unanchored() && i->index() != start())\n MarkDominator(i->index(), &rootmap, &predmap, &predvec, &reachable, &stk);\n }\n\n // Third pass: Emits \"lists\". Remaps outs to root-ids.\n // Builds the mapping from root-ids to flat-ids.\n std::vector flatmap(rootmap.size());\n std::vector flat;\n flat.reserve(size());\n for (SparseArray::const_iterator i = rootmap.begin();\n i != rootmap.end();\n ++i) {\n flatmap[i->value()] = static_cast(flat.size());\n EmitList(i->index(), &rootmap, &flat, &reachable, &stk);\n flat.back().set_last();\n // We have the bounds of the \"list\", so this is the\n // most convenient point at which to compute hints.\n ComputeHints(&flat, flatmap[i->value()], static_cast(flat.size()));\n }\n\n list_count_ = static_cast(flatmap.size());\n for (int i = 0; i < kNumInst; i++)\n inst_count_[i] = 0;\n\n // Fourth pass: Remaps outs to flat-ids.\n // Counts instructions by opcode.\n for (int id = 0; id < static_cast(flat.size()); id++) {\n Inst* ip = &flat[id];\n if (ip->opcode() != kInstAltMatch) // handled in EmitList()\n ip->set_out(flatmap[ip->out()]);\n inst_count_[ip->opcode()]++;\n }\n\n#if !defined(NDEBUG)\n // Address a `-Wunused-but-set-variable' warning from Clang 13.x.\n size_t total = 0;\n for (int i = 0; i < kNumInst; i++)\n total += inst_count_[i];\n ABSL_CHECK_EQ(total, flat.size());\n#endif\n\n // Remap start_unanchored and start.\n if (start_unanchored() == 0) {\n ABSL_DCHECK_EQ(start(), 0);\n } else if (start_unanchored() == start()) {\n set_start_unanchored(flatmap[1]);\n set_start(flatmap[1]);\n } else {\n set_start_unanchored(flatmap[1]);\n set_start(flatmap[2]);\n }\n\n // Finally, replace the old instructions with the new instructions.\n size_ = static_cast(flat.size());\n inst_ = PODArray(size_);\n memmove(inst_.data(), flat.data(), size_*sizeof inst_[0]);\n\n // Populate the list heads for BitState.\n // 512 instructions limits the memory footprint to 1KiB.\n if (size_ <= 512) {\n list_heads_ = PODArray(size_);\n // 0xFF makes it more obvious if we try to look up a non-head.\n memset(list_heads_.data(), 0xFF, size_*sizeof list_heads_[0]);\n for (int i = 0; i < list_count_; ++i)\n list_heads_[flatmap[i]] = i;\n }\n\n // BitState allocates a bitmap of size list_count_ * (text.size()+1)\n // for tracking pairs of possibilities that it has already explored.\n const size_t kBitStateBitmapMaxSize = 256*1024; // max size in bits\n bit_state_text_max_size_ = kBitStateBitmapMaxSize / list_count_ - 1;\n}\n\nvoid Prog::MarkSuccessors(SparseArray* rootmap,\n SparseArray* predmap,\n std::vector>* predvec,\n SparseSet* reachable, std::vector* stk) {\n // Mark the kInstFail instruction.\n rootmap->set_new(0, rootmap->size());\n\n // Mark the start_unanchored and start instructions.\n if (!rootmap->has_index(start_unanchored()))\n rootmap->set_new(start_unanchored(), rootmap->size());\n if (!rootmap->has_index(start()))\n rootmap->set_new(start(), rootmap->size());\n\n reachable->clear();\n stk->clear();\n stk->push_back(start_unanchored());\n while (!stk->empty()) {\n int id = stk->back();\n stk->pop_back();\n Loop:\n if (reachable->contains(id))\n continue;\n reachable->insert_new(id);\n\n Inst* ip = inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"unhandled opcode: \" << ip->opcode();\n break;\n\n case kInstAltMatch:\n case kInstAlt:\n // Mark this instruction as a predecessor of each out.\n for (int out : {ip->out(), ip->out1()}) {\n if (!predmap->has_index(out)) {\n predmap->set_new(out, static_cast(predvec->size()));\n predvec->emplace_back();\n }\n (*predvec)[predmap->get_existing(out)].emplace_back(id);\n }\n stk->push_back(ip->out1());\n id = ip->out();\n goto Loop;\n\n case kInstByteRange:\n case kInstCapture:\n case kInstEmptyWidth:\n // Mark the out of this instruction as a \"root\".\n if (!rootmap->has_index(ip->out()))\n rootmap->set_new(ip->out(), rootmap->size());\n id = ip->out();\n goto Loop;\n\n case kInstNop:\n id = ip->out();\n goto Loop;\n\n case kInstMatch:\n case kInstFail:\n break;\n }\n }\n}\n\nvoid Prog::MarkDominator(int root, SparseArray* rootmap,\n SparseArray* predmap,\n std::vector>* predvec,\n SparseSet* reachable, std::vector* stk) {\n reachable->clear();\n stk->clear();\n stk->push_back(root);\n while (!stk->empty()) {\n int id = stk->back();\n stk->pop_back();\n Loop:\n if (reachable->contains(id))\n continue;\n reachable->insert_new(id);\n\n if (id != root && rootmap->has_index(id)) {\n // We reached another \"tree\" via epsilon transition.\n continue;\n }\n\n Inst* ip = inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"unhandled opcode: \" << ip->opcode();\n break;\n\n case kInstAltMatch:\n case kInstAlt:\n stk->push_back(ip->out1());\n id = ip->out();\n goto Loop;\n\n case kInstByteRange:\n case kInstCapture:\n case kInstEmptyWidth:\n break;\n\n case kInstNop:\n id = ip->out();\n goto Loop;\n\n case kInstMatch:\n case kInstFail:\n break;\n }\n }\n\n for (SparseSet::const_iterator i = reachable->begin();\n i != reachable->end();\n ++i) {\n int id = *i;\n if (predmap->has_index(id)) {\n for (int pred : (*predvec)[predmap->get_existing(id)]) {\n if (!reachable->contains(pred)) {\n // id has a predecessor that cannot be reached from root!\n // Therefore, id must be a \"root\" too - mark it as such.\n if (!rootmap->has_index(id))\n rootmap->set_new(id, rootmap->size());\n }\n }\n }\n }\n}\n\nvoid Prog::EmitList(int root, SparseArray* rootmap,\n std::vector* flat,\n SparseSet* reachable, std::vector* stk) {\n reachable->clear();\n stk->clear();\n stk->push_back(root);\n while (!stk->empty()) {\n int id = stk->back();\n stk->pop_back();\n Loop:\n if (reachable->contains(id))\n continue;\n reachable->insert_new(id);\n\n if (id != root && rootmap->has_index(id)) {\n // We reached another \"tree\" via epsilon transition. Emit a kInstNop\n // instruction so that the Prog does not become quadratically larger.\n flat->emplace_back();\n flat->back().set_opcode(kInstNop);\n flat->back().set_out(rootmap->get_existing(id));\n continue;\n }\n\n Inst* ip = inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(DFATAL) << \"unhandled opcode: \" << ip->opcode();\n break;\n\n case kInstAltMatch:\n flat->emplace_back();\n flat->back().set_opcode(kInstAltMatch);\n flat->back().set_out(static_cast(flat->size()));\n flat->back().out1_ = static_cast(flat->size())+1;\n [[fallthrough]];\n\n case kInstAlt:\n stk->push_back(ip->out1());\n id = ip->out();\n goto Loop;\n\n case kInstByteRange:\n case kInstCapture:\n case kInstEmptyWidth:\n flat->emplace_back();\n memmove(&flat->back(), ip, sizeof *ip);\n flat->back().set_out(rootmap->get_existing(ip->out()));\n break;\n\n case kInstNop:\n id = ip->out();\n goto Loop;\n\n case kInstMatch:\n case kInstFail:\n flat->emplace_back();\n memmove(&flat->back(), ip, sizeof *ip);\n break;\n }\n }\n}\n\n// For each ByteRange instruction in [begin, end), computes a hint to execution\n// engines: the delta to the next instruction (in flat) worth exploring iff the\n// current instruction matched.\n//\n// Implements a coloring algorithm related to ByteMapBuilder, but in this case,\n// colors are instructions and recoloring ranges precisely identifies conflicts\n// between instructions. Iterating backwards over [begin, end) is guaranteed to\n// identify the nearest conflict (if any) with only linear complexity.\nvoid Prog::ComputeHints(std::vector* flat, int begin, int end) {\n Bitmap256 splits;\n int colors[256];\n\n bool dirty = false;\n for (int id = end; id >= begin; --id) {\n if (id == end ||\n (*flat)[id].opcode() != kInstByteRange) {\n if (dirty) {\n dirty = false;\n splits.Clear();\n }\n splits.Set(255);\n colors[255] = id;\n // At this point, the [0-255] range is colored with id.\n // Thus, hints cannot point beyond id; and if id == end,\n // hints that would have pointed to id will be 0 instead.\n continue;\n }\n dirty = true;\n\n // We recolor the [lo-hi] range with id. Note that first ratchets backwards\n // from end to the nearest conflict (if any) during recoloring.\n int first = end;\n auto Recolor = [&](int lo, int hi) {\n // Like ByteMapBuilder, we split at lo-1 and at hi.\n --lo;\n\n if (0 <= lo && !splits.Test(lo)) {\n splits.Set(lo);\n int next = splits.FindNextSetBit(lo+1);\n colors[lo] = colors[next];\n }\n if (!splits.Test(hi)) {\n splits.Set(hi);\n int next = splits.FindNextSetBit(hi+1);\n colors[hi] = colors[next];\n }\n\n int c = lo+1;\n while (c < 256) {\n int next = splits.FindNextSetBit(c);\n // Ratchet backwards...\n first = std::min(first, colors[next]);\n // Recolor with id - because it's the new nearest conflict!\n colors[next] = id;\n if (next == hi)\n break;\n c = next+1;\n }\n };\n\n Inst* ip = &(*flat)[id];\n int lo = ip->lo();\n int hi = ip->hi();\n Recolor(lo, hi);\n if (ip->foldcase() && lo <= 'z' && hi >= 'a') {\n int foldlo = lo;\n int foldhi = hi;\n if (foldlo < 'a')\n foldlo = 'a';\n if (foldhi > 'z')\n foldhi = 'z';\n if (foldlo <= foldhi) {\n foldlo += 'A' - 'a';\n foldhi += 'A' - 'a';\n Recolor(foldlo, foldhi);\n }\n }\n\n if (first != end) {\n uint16_t hint = static_cast(std::min(first - id, 32767));\n ip->hint_foldcase_ |= hint<<1;\n }\n }\n}\n\n// The final state will always be this, which frees up a register for the hot\n// loop and thus avoids the spilling that can occur when building with Clang.\nstatic const size_t kShiftDFAFinal = 9;\n\n// This function takes the prefix as std::string (i.e. not const std::string&\n// as normal) because it's going to clobber it, so a temporary is convenient.\nstatic uint64_t* BuildShiftDFA(std::string prefix) {\n // This constant is for convenience now and also for correctness later when\n // we clobber the prefix, but still need to know how long it was initially.\n const size_t size = prefix.size();\n\n // Construct the NFA.\n // The table is indexed by input byte; each element is a bitfield of states\n // reachable by the input byte. Given a bitfield of the current states, the\n // bitfield of states reachable from those is - for this specific purpose -\n // always ((ncurr << 1) | 1). Intersecting the reachability bitfields gives\n // the bitfield of the next states reached by stepping over the input byte.\n // Credits for this technique: the Hyperscan paper by Geoff Langdale et al.\n uint16_t nfa[256]{};\n for (size_t i = 0; i < size; ++i) {\n uint8_t b = prefix[i];\n nfa[b] |= 1 << (i+1);\n }\n // This is the `\\C*?` for unanchored search.\n for (int b = 0; b < 256; ++b)\n nfa[b] |= 1;\n\n // This maps from DFA state to NFA states; the reverse mapping is used when\n // recording transitions and gets implemented with plain old linear search.\n // The \"Shift DFA\" technique limits this to ten states when using uint64_t;\n // to allow for the initial state, we use at most nine bytes of the prefix.\n // That same limit is also why uint16_t is sufficient for the NFA bitfield.\n uint16_t states[kShiftDFAFinal+1]{};\n states[0] = 1;\n for (size_t dcurr = 0; dcurr < size; ++dcurr) {\n uint8_t b = prefix[dcurr];\n uint16_t ncurr = states[dcurr];\n uint16_t nnext = nfa[b] & ((ncurr << 1) | 1);\n size_t dnext = dcurr+1;\n if (dnext == size)\n dnext = kShiftDFAFinal;\n states[dnext] = nnext;\n }\n\n // Sort and unique the bytes of the prefix to avoid repeating work while we\n // record transitions. This clobbers the prefix, but it's no longer needed.\n std::sort(prefix.begin(), prefix.end());\n prefix.erase(std::unique(prefix.begin(), prefix.end()), prefix.end());\n\n // Construct the DFA.\n // The table is indexed by input byte; each element is effectively a packed\n // array of uint6_t; each array value will be multiplied by six in order to\n // avoid having to do so later in the hot loop as well as masking/shifting.\n // Credits for this technique: \"Shift-based DFAs\" on GitHub by Per Vognsen.\n uint64_t* dfa = new uint64_t[256]{};\n // Record a transition from each state for each of the bytes of the prefix.\n // Note that all other input bytes go back to the initial state by default.\n for (size_t dcurr = 0; dcurr < size; ++dcurr) {\n for (uint8_t b : prefix) {\n uint16_t ncurr = states[dcurr];\n uint16_t nnext = nfa[b] & ((ncurr << 1) | 1);\n size_t dnext = 0;\n while (states[dnext] != nnext)\n ++dnext;\n dfa[b] |= static_cast(dnext * 6) << (dcurr * 6);\n // Convert ASCII letters to uppercase and record the extra transitions.\n // Note that ASCII letters are guaranteed to be lowercase at this point\n // because that's how the parser normalises them. #FunFact: 'k' and 's'\n // match U+212A and U+017F, respectively, so they won't occur here when\n // using UTF-8 encoding because the parser will emit character classes.\n if ('a' <= b && b <= 'z') {\n b -= 'a' - 'A';\n dfa[b] |= static_cast(dnext * 6) << (dcurr * 6);\n }\n }\n }\n // This lets the final state \"saturate\", which will matter for performance:\n // in the hot loop, we check for a match only at the end of each iteration,\n // so we must keep signalling the match until we get around to checking it.\n for (int b = 0; b < 256; ++b)\n dfa[b] |= static_cast(kShiftDFAFinal * 6) << (kShiftDFAFinal * 6);\n\n return dfa;\n}\n\nvoid Prog::ConfigurePrefixAccel(const std::string& prefix,\n bool prefix_foldcase) {\n prefix_foldcase_ = prefix_foldcase;\n prefix_size_ = prefix.size();\n if (prefix_foldcase_) {\n // Use PrefixAccel_ShiftDFA().\n // ... and no more than nine bytes of the prefix. (See above for details.)\n prefix_size_ = std::min(prefix_size_, kShiftDFAFinal);\n prefix_dfa_ = BuildShiftDFA(prefix.substr(0, prefix_size_));\n } else if (prefix_size_ != 1) {\n // Use PrefixAccel_FrontAndBack().\n prefix_front_ = prefix.front();\n prefix_back_ = prefix.back();\n } else {\n // Use memchr(3).\n prefix_front_ = prefix.front();\n }\n}\n\nconst void* Prog::PrefixAccel_ShiftDFA(const void* data, size_t size) {\n if (size < prefix_size_)\n return NULL;\n\n uint64_t curr = 0;\n\n // At the time of writing, rough benchmarks on a Broadwell machine showed\n // that this unroll factor (i.e. eight) achieves a speedup factor of two.\n if (size >= 8) {\n const uint8_t* p = reinterpret_cast(data);\n const uint8_t* endp = p + (size&~7);\n do {\n uint8_t b0 = p[0];\n uint8_t b1 = p[1];\n uint8_t b2 = p[2];\n uint8_t b3 = p[3];\n uint8_t b4 = p[4];\n uint8_t b5 = p[5];\n uint8_t b6 = p[6];\n uint8_t b7 = p[7];\n\n uint64_t next0 = prefix_dfa_[b0];\n uint64_t next1 = prefix_dfa_[b1];\n uint64_t next2 = prefix_dfa_[b2];\n uint64_t next3 = prefix_dfa_[b3];\n uint64_t next4 = prefix_dfa_[b4];\n uint64_t next5 = prefix_dfa_[b5];\n uint64_t next6 = prefix_dfa_[b6];\n uint64_t next7 = prefix_dfa_[b7];\n\n uint64_t curr0 = next0 >> (curr & 63);\n uint64_t curr1 = next1 >> (curr0 & 63);\n uint64_t curr2 = next2 >> (curr1 & 63);\n uint64_t curr3 = next3 >> (curr2 & 63);\n uint64_t curr4 = next4 >> (curr3 & 63);\n uint64_t curr5 = next5 >> (curr4 & 63);\n uint64_t curr6 = next6 >> (curr5 & 63);\n uint64_t curr7 = next7 >> (curr6 & 63);\n\n if ((curr7 & 63) == kShiftDFAFinal * 6) {\n // At the time of writing, using the same masking subexpressions from\n // the preceding lines caused Clang to clutter the hot loop computing\n // them - even though they aren't actually needed for shifting! Hence\n // these rewritten conditions, which achieve a speedup factor of two.\n if (((curr7-curr0) & 63) == 0) return p+1-prefix_size_;\n if (((curr7-curr1) & 63) == 0) return p+2-prefix_size_;\n if (((curr7-curr2) & 63) == 0) return p+3-prefix_size_;\n if (((curr7-curr3) & 63) == 0) return p+4-prefix_size_;\n if (((curr7-curr4) & 63) == 0) return p+5-prefix_size_;\n if (((curr7-curr5) & 63) == 0) return p+6-prefix_size_;\n if (((curr7-curr6) & 63) == 0) return p+7-prefix_size_;\n if (((curr7-curr7) & 63) == 0) return p+8-prefix_size_;\n }\n\n curr = curr7;\n p += 8;\n } while (p != endp);\n data = p;\n size = size&7;\n }\n\n const uint8_t* p = reinterpret_cast(data);\n const uint8_t* endp = p + size;\n while (p != endp) {\n uint8_t b = *p++;\n uint64_t next = prefix_dfa_[b];\n curr = next >> (curr & 63);\n if ((curr & 63) == kShiftDFAFinal * 6)\n return p-prefix_size_;\n }\n return NULL;\n}\n\n#if defined(__AVX2__)\n// Finds the least significant non-zero bit in n.\nstatic int FindLSBSet(uint32_t n) {\n ABSL_DCHECK_NE(n, uint32_t{0});\n#if defined(__GNUC__)\n return __builtin_ctz(n);\n#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))\n unsigned long c;\n _BitScanForward(&c, n);\n return static_cast(c);\n#else\n int c = 31;\n for (int shift = 1 << 4; shift != 0; shift >>= 1) {\n uint32_t word = n << shift;\n if (word != 0) {\n n = word;\n c -= shift;\n }\n }\n return c;\n#endif\n}\n#endif\n\nconst void* Prog::PrefixAccel_FrontAndBack(const void* data, size_t size) {\n ABSL_DCHECK_GE(prefix_size_, size_t{2});\n if (size < prefix_size_)\n return NULL;\n // Don't bother searching the last prefix_size_-1 bytes for prefix_front_.\n // This also means that probing for prefix_back_ doesn't go out of bounds.\n size -= prefix_size_-1;\n\n#if defined(__AVX2__)\n // Use AVX2 to look for prefix_front_ and prefix_back_ 32 bytes at a time.\n if (size >= sizeof(__m256i)) {\n const __m256i* fp = reinterpret_cast(\n reinterpret_cast(data));\n const __m256i* bp = reinterpret_cast(\n reinterpret_cast(data) + prefix_size_-1);\n const __m256i* endfp = fp + size/sizeof(__m256i);\n const __m256i f_set1 = _mm256_set1_epi8(prefix_front_);\n const __m256i b_set1 = _mm256_set1_epi8(prefix_back_);\n do {\n const __m256i f_loadu = _mm256_loadu_si256(fp++);\n const __m256i b_loadu = _mm256_loadu_si256(bp++);\n const __m256i f_cmpeq = _mm256_cmpeq_epi8(f_set1, f_loadu);\n const __m256i b_cmpeq = _mm256_cmpeq_epi8(b_set1, b_loadu);\n const int fb_testz = _mm256_testz_si256(f_cmpeq, b_cmpeq);\n if (fb_testz == 0) { // ZF: 1 means zero, 0 means non-zero.\n const __m256i fb_and = _mm256_and_si256(f_cmpeq, b_cmpeq);\n const int fb_movemask = _mm256_movemask_epi8(fb_and);\n const int fb_ctz = FindLSBSet(fb_movemask);\n return reinterpret_cast(fp-1) + fb_ctz;\n }\n } while (fp != endfp);\n data = fp;\n size = size%sizeof(__m256i);\n }\n#endif\n\n const char* p0 = reinterpret_cast(data);\n for (const char* p = p0;; p++) {\n ABSL_DCHECK_GE(size, static_cast(p-p0));\n p = reinterpret_cast(memchr(p, prefix_front_, size - (p-p0)));\n if (p == NULL || p[prefix_size_-1] == prefix_back_)\n return p;\n }\n}\n\n} // namespace re2\n" }, { "path": "re2/prog.h", "content": "// Copyright 2007 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_PROG_H_\n#define RE2_PROG_H_\n\n// Compiled representation of regular expressions.\n// See regexp.h for the Regexp class, which represents a regular\n// expression symbolically.\n\n#include \n\n#include \n#include \n#include \n#include \n#include \n\n#include \"absl/base/call_once.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/re2.h\"\n#include \"re2/sparse_array.h\"\n#include \"re2/sparse_set.h\"\n\nnamespace re2 {\n\n// Opcodes for Inst\nenum InstOp {\n kInstAlt = 0, // choose between out_ and out1_\n kInstAltMatch, // Alt: out_ is [00-FF] and back, out1_ is match; or vice versa.\n kInstByteRange, // next (possible case-folded) byte must be in [lo_, hi_]\n kInstCapture, // capturing parenthesis number cap_\n kInstEmptyWidth, // empty-width special (^ $ ...); bit(s) set in empty_\n kInstMatch, // found a match!\n kInstNop, // no-op; occasionally unavoidable\n kInstFail, // never match; occasionally unavoidable\n kNumInst,\n};\n\n// Bit flags for empty-width specials\nenum EmptyOp {\n kEmptyBeginLine = 1<<0, // ^ - beginning of line\n kEmptyEndLine = 1<<1, // $ - end of line\n kEmptyBeginText = 1<<2, // \\A - beginning of text\n kEmptyEndText = 1<<3, // \\z - end of text\n kEmptyWordBoundary = 1<<4, // \\b - word boundary\n kEmptyNonWordBoundary = 1<<5, // \\B - not \\b\n kEmptyAllFlags = (1<<6)-1,\n};\n\nclass DFA;\nclass Regexp;\n\n// Compiled form of regexp program.\nclass Prog {\n public:\n Prog();\n ~Prog();\n\n // Single instruction in regexp program.\n class Inst {\n public:\n // See the assertion below for why this is so.\n Inst() = default;\n\n // Copyable.\n Inst(const Inst&) = default;\n Inst& operator=(const Inst&) = default;\n\n // Constructors per opcode\n void InitAlt(uint32_t out, uint32_t out1);\n void InitByteRange(int lo, int hi, int foldcase, uint32_t out);\n void InitCapture(int cap, uint32_t out);\n void InitEmptyWidth(EmptyOp empty, uint32_t out);\n void InitMatch(int id);\n void InitNop(uint32_t out);\n void InitFail();\n\n // Getters\n int id(Prog* p) { return static_cast(this - p->inst_.data()); }\n InstOp opcode() { return static_cast(out_opcode_ & 7); }\n int last() { return (out_opcode_ >> 3) & 1; }\n int out() { return out_opcode_ >> 4; }\n int out1() {\n ABSL_DCHECK(opcode() == kInstAlt || opcode() == kInstAltMatch);\n return out1_;\n }\n int cap() {\n ABSL_DCHECK_EQ(opcode(), kInstCapture);\n return cap_;\n }\n int lo() {\n ABSL_DCHECK_EQ(opcode(), kInstByteRange);\n return lo_;\n }\n int hi() {\n ABSL_DCHECK_EQ(opcode(), kInstByteRange);\n return hi_;\n }\n int foldcase() {\n ABSL_DCHECK_EQ(opcode(), kInstByteRange);\n return hint_foldcase_ & 1;\n }\n int hint() {\n ABSL_DCHECK_EQ(opcode(), kInstByteRange);\n return hint_foldcase_ >> 1;\n }\n int match_id() {\n ABSL_DCHECK_EQ(opcode(), kInstMatch);\n return match_id_;\n }\n EmptyOp empty() {\n ABSL_DCHECK_EQ(opcode(), kInstEmptyWidth);\n return empty_;\n }\n\n bool greedy(Prog* p) {\n ABSL_DCHECK_EQ(opcode(), kInstAltMatch);\n return p->inst(out())->opcode() == kInstByteRange ||\n (p->inst(out())->opcode() == kInstNop &&\n p->inst(p->inst(out())->out())->opcode() == kInstByteRange);\n }\n\n // Does this inst (an kInstByteRange) match c?\n inline bool Matches(int c) {\n ABSL_DCHECK_EQ(opcode(), kInstByteRange);\n if (foldcase() && 'A' <= c && c <= 'Z')\n c += 'a' - 'A';\n return lo_ <= c && c <= hi_;\n }\n\n // Returns string representation for debugging.\n std::string Dump();\n\n // Maximum instruction id.\n // (Must fit in out_opcode_. PatchList/last steal another bit.)\n static const int kMaxInst = (1<<28) - 1;\n\n private:\n void set_opcode(InstOp opcode) {\n out_opcode_ = (out()<<4) | (last()<<3) | opcode;\n }\n\n void set_last() {\n out_opcode_ = (out()<<4) | (1<<3) | opcode();\n }\n\n void set_out(int out) {\n out_opcode_ = (out<<4) | (last()<<3) | opcode();\n }\n\n void set_out_opcode(int out, InstOp opcode) {\n out_opcode_ = (out<<4) | (last()<<3) | opcode;\n }\n\n uint32_t out_opcode_; // 28 bits: out, 1 bit: last, 3 (low) bits: opcode\n union { // additional instruction arguments:\n uint32_t out1_; // opcode == kInstAlt\n // alternate next instruction\n\n int32_t cap_; // opcode == kInstCapture\n // Index of capture register (holds text\n // position recorded by capturing parentheses).\n // For \\n (the submatch for the nth parentheses),\n // the left parenthesis captures into register 2*n\n // and the right one captures into register 2*n+1.\n\n int32_t match_id_; // opcode == kInstMatch\n // Match ID to identify this match (for re2::Set).\n\n struct { // opcode == kInstByteRange\n uint8_t lo_; // byte range is lo_-hi_ inclusive\n uint8_t hi_; //\n uint16_t hint_foldcase_; // 15 bits: hint, 1 (low) bit: foldcase\n // hint to execution engines: the delta to the\n // next instruction (in the current list) worth\n // exploring iff this instruction matched; 0\n // means there are no remaining possibilities,\n // which is most likely for character classes.\n // foldcase: A-Z -> a-z before checking range.\n };\n\n EmptyOp empty_; // opcode == kInstEmptyWidth\n // empty_ is bitwise OR of kEmpty* flags above.\n };\n\n friend class Compiler;\n friend struct PatchList;\n friend class Prog;\n };\n\n // Inst must be trivial so that we can freely clear it with memset(3).\n // Arrays of Inst are initialised by copying the initial elements with\n // memmove(3) and then clearing any remaining elements with memset(3).\n static_assert(std::is_trivial::value, \"Inst must be trivial\");\n\n // Whether to anchor the search.\n enum Anchor {\n kUnanchored, // match anywhere\n kAnchored, // match only starting at beginning of text\n };\n\n // Kind of match to look for (for anchor != kFullMatch)\n //\n // kLongestMatch mode finds the overall longest\n // match but still makes its submatch choices the way\n // Perl would, not in the way prescribed by POSIX.\n // The POSIX rules are much more expensive to implement,\n // and no one has needed them.\n //\n // kFullMatch is not strictly necessary -- we could use\n // kLongestMatch and then check the length of the match -- but\n // the matching code can run faster if it knows to consider only\n // full matches.\n enum MatchKind {\n kFirstMatch, // like Perl, PCRE\n kLongestMatch, // like egrep or POSIX\n kFullMatch, // match only entire text; implies anchor==kAnchored\n kManyMatch // for SearchDFA, records set of matches\n };\n\n Inst *inst(int id) { return &inst_[id]; }\n int start() { return start_; }\n void set_start(int start) { start_ = start; }\n int start_unanchored() { return start_unanchored_; }\n void set_start_unanchored(int start) { start_unanchored_ = start; }\n int size() { return size_; }\n bool reversed() { return reversed_; }\n void set_reversed(bool reversed) { reversed_ = reversed; }\n int list_count() { return list_count_; }\n int inst_count(InstOp op) { return inst_count_[op]; }\n uint16_t* list_heads() { return list_heads_.data(); }\n size_t bit_state_text_max_size() { return bit_state_text_max_size_; }\n int64_t dfa_mem() { return dfa_mem_; }\n void set_dfa_mem(int64_t dfa_mem) { dfa_mem_ = dfa_mem; }\n bool anchor_start() { return anchor_start_; }\n void set_anchor_start(bool b) { anchor_start_ = b; }\n bool anchor_end() { return anchor_end_; }\n void set_anchor_end(bool b) { anchor_end_ = b; }\n int bytemap_range() { return bytemap_range_; }\n const uint8_t* bytemap() { return bytemap_; }\n bool can_prefix_accel() { return prefix_size_ != 0; }\n\n // Accelerates to the first likely occurrence of the prefix.\n // Returns a pointer to the first byte or NULL if not found.\n const void* PrefixAccel(const void* data, size_t size) {\n ABSL_DCHECK(can_prefix_accel());\n if (prefix_foldcase_) {\n return PrefixAccel_ShiftDFA(data, size);\n } else if (prefix_size_ != 1) {\n return PrefixAccel_FrontAndBack(data, size);\n } else {\n return memchr(data, prefix_front_, size);\n }\n }\n\n // Configures prefix accel using the analysis performed during compilation.\n void ConfigurePrefixAccel(const std::string& prefix, bool prefix_foldcase);\n\n // An implementation of prefix accel that uses prefix_dfa_ to perform\n // case-insensitive search.\n const void* PrefixAccel_ShiftDFA(const void* data, size_t size);\n\n // An implementation of prefix accel that looks for prefix_front_ and\n // prefix_back_ to return fewer false positives than memchr(3) alone.\n const void* PrefixAccel_FrontAndBack(const void* data, size_t size);\n\n // Returns string representation of program for debugging.\n std::string Dump();\n std::string DumpUnanchored();\n std::string DumpByteMap();\n\n // Returns the set of kEmpty flags that are in effect at\n // position p within context.\n static uint32_t EmptyFlags(absl::string_view context, const char* p);\n\n // Returns whether byte c is a word character: ASCII only.\n // Used by the implementation of \\b and \\B.\n // This is not right for Unicode, but:\n // - it's hard to get right in a byte-at-a-time matching world\n // (the DFA has only one-byte lookahead).\n // - even if the lookahead were possible, the Progs would be huge.\n // This crude approximation is the same one PCRE uses.\n static bool IsWordChar(uint8_t c) {\n return ('A' <= c && c <= 'Z') ||\n ('a' <= c && c <= 'z') ||\n ('0' <= c && c <= '9') ||\n c == '_';\n }\n\n // Execution engines. They all search for the regexp (run the prog)\n // in text, which is in the larger context (used for ^ $ \\b etc).\n // Anchor and kind control the kind of search.\n // Returns true if match found, false if not.\n // If match found, fills match[0..nmatch-1] with submatch info.\n // match[0] is overall match, match[1] is first set of parens, etc.\n // If a particular submatch is not matched during the regexp match,\n // it is set to NULL.\n //\n // Matching text == absl::string_view() is treated as any other empty\n // string, but note that on return, it will not be possible to distinguish\n // submatches that matched that empty string from submatches that didn't\n // match anything. Either way, match[i] == NULL.\n\n // Search using NFA: can find submatches but kind of slow.\n bool SearchNFA(absl::string_view text, absl::string_view context,\n Anchor anchor, MatchKind kind, absl::string_view* match,\n int nmatch);\n\n // Search using DFA: much faster than NFA but only finds\n // end of match and can use a lot more memory.\n // Returns whether a match was found.\n // If the DFA runs out of memory, sets *failed to true and returns false.\n // If matches != NULL and kind == kManyMatch and there is a match,\n // SearchDFA fills matches with the match IDs of the final matching state.\n bool SearchDFA(absl::string_view text, absl::string_view context,\n Anchor anchor, MatchKind kind, absl::string_view* match0,\n bool* failed, SparseSet* matches);\n\n // The callback issued after building each DFA state with BuildEntireDFA().\n // If next is null, then the memory budget has been exhausted and building\n // will halt. Otherwise, the state has been built and next points to an array\n // of bytemap_range()+1 slots holding the next states as per the bytemap and\n // kByteEndText. The number of the state is implied by the callback sequence:\n // the first callback is for state 0, the second callback is for state 1, ...\n // match indicates whether the state is a matching state.\n using DFAStateCallback = std::function;\n\n // Build the entire DFA for the given match kind.\n // Usually the DFA is built out incrementally, as needed, which\n // avoids lots of unnecessary work.\n // If cb is not empty, it receives one callback per state built.\n // Returns the number of states built.\n // FOR TESTING OR EXPERIMENTAL PURPOSES ONLY.\n int BuildEntireDFA(MatchKind kind, const DFAStateCallback& cb);\n\n // Compute bytemap.\n void ComputeByteMap();\n\n // Run peep-hole optimizer on program.\n void Optimize();\n\n // One-pass NFA: only correct if IsOnePass() is true,\n // but much faster than NFA (competitive with PCRE)\n // for those expressions.\n bool IsOnePass();\n bool SearchOnePass(absl::string_view text, absl::string_view context,\n Anchor anchor, MatchKind kind, absl::string_view* match,\n int nmatch);\n\n // Bit-state backtracking. Fast on small cases but uses memory\n // proportional to the product of the list count and the text size.\n bool CanBitState() { return list_heads_.data() != NULL; }\n bool SearchBitState(absl::string_view text, absl::string_view context,\n Anchor anchor, MatchKind kind, absl::string_view* match,\n int nmatch);\n\n static const int kMaxOnePassCapture = 5; // $0 through $4\n\n // Backtracking search: the gold standard against which the other\n // implementations are checked. FOR TESTING ONLY.\n // It allocates a ton of memory to avoid running forever.\n // It is also recursive, so can't use in production (will overflow stacks).\n // The name \"Unsafe\" here is supposed to be a flag that\n // you should not be using this function.\n bool UnsafeSearchBacktrack(absl::string_view text, absl::string_view context,\n Anchor anchor, MatchKind kind,\n absl::string_view* match, int nmatch);\n\n // Computes range for any strings matching regexp. The min and max can in\n // some cases be arbitrarily precise, so the caller gets to specify the\n // maximum desired length of string returned.\n //\n // Assuming PossibleMatchRange(&min, &max, N) returns successfully, any\n // string s that is an anchored match for this regexp satisfies\n // min <= s && s <= max.\n //\n // Note that PossibleMatchRange() will only consider the first copy of an\n // infinitely repeated element (i.e., any regexp element followed by a '*' or\n // '+' operator). Regexps with \"{N}\" constructions are not affected, as those\n // do not compile down to infinite repetitions.\n //\n // Returns true on success, false on error.\n bool PossibleMatchRange(std::string* min, std::string* max, int maxlen);\n\n // Outputs the program fanout into the given sparse array.\n void Fanout(SparseArray* fanout);\n\n // Compiles a collection of regexps to Prog. Each regexp will have\n // its own Match instruction recording the index in the output vector.\n static Prog* CompileSet(Regexp* re, RE2::Anchor anchor, int64_t max_mem);\n\n // Flattens the Prog from \"tree\" form to \"list\" form. This is an in-place\n // operation in the sense that the old instructions are lost.\n void Flatten();\n\n // Walks the Prog; the \"successor roots\" or predecessors of the reachable\n // instructions are marked in rootmap or predmap/predvec, respectively.\n // reachable and stk are preallocated scratch structures.\n void MarkSuccessors(SparseArray* rootmap,\n SparseArray* predmap,\n std::vector>* predvec,\n SparseSet* reachable, std::vector* stk);\n\n // Walks the Prog from the given \"root\" instruction; the \"dominator root\"\n // of the reachable instructions (if such exists) is marked in rootmap.\n // reachable and stk are preallocated scratch structures.\n void MarkDominator(int root, SparseArray* rootmap,\n SparseArray* predmap,\n std::vector>* predvec,\n SparseSet* reachable, std::vector* stk);\n\n // Walks the Prog from the given \"root\" instruction; the reachable\n // instructions are emitted in \"list\" form and appended to flat.\n // reachable and stk are preallocated scratch structures.\n void EmitList(int root, SparseArray* rootmap,\n std::vector* flat,\n SparseSet* reachable, std::vector* stk);\n\n // Computes hints for ByteRange instructions in [begin, end).\n void ComputeHints(std::vector* flat, int begin, int end);\n\n // Controls whether the DFA should bail out early if the NFA would be faster.\n // FOR TESTING ONLY.\n static void TESTING_ONLY_set_dfa_should_bail_when_slow(bool b);\n\n private:\n friend class Compiler;\n\n DFA* GetDFA(MatchKind kind);\n void DeleteDFA(DFA* dfa);\n\n bool anchor_start_; // regexp has explicit start anchor\n bool anchor_end_; // regexp has explicit end anchor\n bool reversed_; // whether program runs backward over input\n bool did_flatten_; // has Flatten been called?\n bool did_onepass_; // has IsOnePass been called?\n\n int start_; // entry point for program\n int start_unanchored_; // unanchored entry point for program\n int size_; // number of instructions\n int bytemap_range_; // bytemap_[x] < bytemap_range_\n\n bool prefix_foldcase_; // whether prefix is case-insensitive\n size_t prefix_size_; // size of prefix (0 if no prefix)\n union {\n uint64_t* prefix_dfa_; // \"Shift DFA\" for prefix\n struct {\n int prefix_front_; // first byte of prefix\n int prefix_back_; // last byte of prefix\n };\n };\n\n int list_count_; // count of lists (see above)\n int inst_count_[kNumInst]; // count of instructions by opcode\n PODArray list_heads_; // sparse array enumerating list heads\n // not populated if size_ is overly large\n size_t bit_state_text_max_size_; // upper bound (inclusive) on text.size()\n\n PODArray inst_; // pointer to instruction array\n PODArray onepass_nodes_; // data for OnePass nodes\n\n int64_t dfa_mem_; // Maximum memory for DFAs.\n DFA* dfa_first_; // DFA cached for kFirstMatch/kManyMatch\n DFA* dfa_longest_; // DFA cached for kLongestMatch/kFullMatch\n\n uint8_t bytemap_[256]; // map from input bytes to byte classes\n\n absl::once_flag dfa_first_once_;\n absl::once_flag dfa_longest_once_;\n\n Prog(const Prog&) = delete;\n Prog& operator=(const Prog&) = delete;\n};\n\n// std::string_view in MSVC has iterators that aren't just pointers and\n// that don't allow comparisons between different objects - not even if\n// those objects are views into the same string! Thus, we provide these\n// conversion functions for convenience.\nstatic inline const char* BeginPtr(absl::string_view s) {\n return s.data();\n}\nstatic inline const char* EndPtr(absl::string_view s) {\n return s.data() + s.size();\n}\n\n} // namespace re2\n\n#endif // RE2_PROG_H_\n" }, { "path": "re2/re2.cc", "content": "// Copyright 2003-2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Regular expression interface RE2.\n//\n// Originally the PCRE C++ wrapper, but adapted to use\n// the new automata-based regular expression engines.\n\n#include \"re2/re2.h\"\n\n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"absl/base/call_once.h\"\n#include \"absl/base/macros.h\"\n#include \"absl/container/fixed_array.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/ascii.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/prog.h\"\n#include \"re2/regexp.h\"\n#include \"re2/sparse_array.h\"\n#include \"util/strutil.h\"\n#include \"util/utf.h\"\n\n#ifdef _MSC_VER\n#include \n#endif\n\nnamespace re2 {\n\n// Controls the maximum count permitted by GlobalReplace(); -1 is unlimited.\nstatic int maximum_global_replace_count = -1;\n\nvoid RE2::FUZZING_ONLY_set_maximum_global_replace_count(int i) {\n maximum_global_replace_count = i;\n}\n\n// Maximum number of args we can set\nstatic const int kMaxArgs = 16;\nstatic const int kVecSize = 1+kMaxArgs;\n\nconst int RE2::Options::kDefaultMaxMem; // initialized in re2.h\n\nRE2::Options::Options(RE2::CannedOptions opt)\n : max_mem_(kDefaultMaxMem),\n encoding_(opt == RE2::Latin1 ? EncodingLatin1 : EncodingUTF8),\n posix_syntax_(opt == RE2::POSIX),\n longest_match_(opt == RE2::POSIX),\n log_errors_(opt != RE2::Quiet),\n literal_(false),\n never_nl_(false),\n dot_nl_(false),\n never_capture_(false),\n case_sensitive_(true),\n perl_classes_(false),\n word_boundary_(false),\n one_line_(false) {\n}\n\n// Empty objects for use as const references.\n// Statically allocating the storage and then\n// lazily constructing the objects (in a once\n// in RE2::Init()) avoids global constructors\n// and the false positives (thanks, Valgrind)\n// about memory leaks at program termination.\nstruct EmptyStorage {\n std::string empty_string;\n std::map empty_named_groups;\n std::map empty_group_names;\n};\nalignas(EmptyStorage) static char empty_storage[sizeof(EmptyStorage)];\n\nstatic inline std::string* empty_string() {\n return &reinterpret_cast(empty_storage)->empty_string;\n}\n\nstatic inline std::map* empty_named_groups() {\n return &reinterpret_cast(empty_storage)->empty_named_groups;\n}\n\nstatic inline std::map* empty_group_names() {\n return &reinterpret_cast(empty_storage)->empty_group_names;\n}\n\n// Converts from Regexp error code to RE2 error code.\n// Maybe some day they will diverge. In any event, this\n// hides the existence of Regexp from RE2 users.\nstatic RE2::ErrorCode RegexpErrorToRE2(re2::RegexpStatusCode code) {\n switch (code) {\n case re2::kRegexpSuccess:\n return RE2::NoError;\n case re2::kRegexpInternalError:\n return RE2::ErrorInternal;\n case re2::kRegexpBadEscape:\n return RE2::ErrorBadEscape;\n case re2::kRegexpBadCharClass:\n return RE2::ErrorBadCharClass;\n case re2::kRegexpBadCharRange:\n return RE2::ErrorBadCharRange;\n case re2::kRegexpMissingBracket:\n return RE2::ErrorMissingBracket;\n case re2::kRegexpMissingParen:\n return RE2::ErrorMissingParen;\n case re2::kRegexpUnexpectedParen:\n return RE2::ErrorUnexpectedParen;\n case re2::kRegexpTrailingBackslash:\n return RE2::ErrorTrailingBackslash;\n case re2::kRegexpRepeatArgument:\n return RE2::ErrorRepeatArgument;\n case re2::kRegexpRepeatSize:\n return RE2::ErrorRepeatSize;\n case re2::kRegexpRepeatOp:\n return RE2::ErrorRepeatOp;\n case re2::kRegexpBadPerlOp:\n return RE2::ErrorBadPerlOp;\n case re2::kRegexpBadUTF8:\n return RE2::ErrorBadUTF8;\n case re2::kRegexpBadNamedCapture:\n return RE2::ErrorBadNamedCapture;\n }\n return RE2::ErrorInternal;\n}\n\nstatic std::string trunc(absl::string_view pattern) {\n if (pattern.size() < 100)\n return std::string(pattern);\n return std::string(pattern.substr(0, 100)) + \"...\";\n}\n\n\nRE2::RE2(const char* pattern) {\n // If absl::string_view becomes an alias for std::string_view,\n // it will stop allowing NULL to be converted.\n // Handle NULL explicitly to keep callers working no matter what.\n if (pattern == NULL)\n pattern = \"\";\n Init(pattern, DefaultOptions);\n}\n\nRE2::RE2(const std::string& pattern) {\n Init(pattern, DefaultOptions);\n}\n\nRE2::RE2(absl::string_view pattern) {\n Init(pattern, DefaultOptions);\n}\n\nRE2::RE2(absl::string_view pattern, const Options& options) {\n Init(pattern, options);\n}\n\nint RE2::Options::ParseFlags() const {\n int flags = Regexp::ClassNL;\n switch (encoding()) {\n default:\n if (log_errors())\n ABSL_LOG(ERROR) << \"Unknown encoding \" << encoding();\n break;\n case RE2::Options::EncodingUTF8:\n break;\n case RE2::Options::EncodingLatin1:\n flags |= Regexp::Latin1;\n break;\n }\n\n if (!posix_syntax())\n flags |= Regexp::LikePerl;\n\n if (literal())\n flags |= Regexp::Literal;\n\n if (never_nl())\n flags |= Regexp::NeverNL;\n\n if (dot_nl())\n flags |= Regexp::DotNL;\n\n if (never_capture())\n flags |= Regexp::NeverCapture;\n\n if (!case_sensitive())\n flags |= Regexp::FoldCase;\n\n if (perl_classes())\n flags |= Regexp::PerlClasses;\n\n if (word_boundary())\n flags |= Regexp::PerlB;\n\n if (one_line())\n flags |= Regexp::OneLine;\n\n return flags;\n}\n\nvoid RE2::Init(absl::string_view pattern, const Options& options) {\n static absl::once_flag empty_once;\n absl::call_once(empty_once, []() {\n (void) new (empty_storage) EmptyStorage;\n });\n\n pattern_ = new std::string(pattern);\n options_.Copy(options);\n entire_regexp_ = NULL;\n suffix_regexp_ = NULL;\n error_ = empty_string();\n error_arg_ = empty_string();\n\n num_captures_ = -1;\n error_code_ = NoError;\n longest_match_ = options_.longest_match();\n is_one_pass_ = false;\n prefix_foldcase_ = false;\n prefix_.clear();\n prog_ = NULL;\n\n rprog_ = NULL;\n named_groups_ = NULL;\n group_names_ = NULL;\n\n RegexpStatus status;\n entire_regexp_ = Regexp::Parse(\n *pattern_,\n static_cast(options_.ParseFlags()),\n &status);\n if (entire_regexp_ == NULL) {\n if (options_.log_errors()) {\n ABSL_LOG(ERROR) << \"Error parsing '\" << trunc(*pattern_) << \"': \"\n << status.Text();\n }\n error_ = new std::string(status.Text());\n error_code_ = RegexpErrorToRE2(status.code());\n error_arg_ = new std::string(status.error_arg());\n return;\n }\n\n bool foldcase;\n re2::Regexp* suffix;\n if (entire_regexp_->RequiredPrefix(&prefix_, &foldcase, &suffix)) {\n prefix_foldcase_ = foldcase;\n suffix_regexp_ = suffix;\n }\n else {\n suffix_regexp_ = entire_regexp_->Incref();\n }\n\n // Two thirds of the memory goes to the forward Prog,\n // one third to the reverse prog, because the forward\n // Prog has two DFAs but the reverse prog has one.\n prog_ = suffix_regexp_->CompileToProg(options_.max_mem()*2/3);\n if (prog_ == NULL) {\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"Error compiling '\" << trunc(*pattern_) << \"'\";\n error_ = new std::string(\"pattern too large - compile failed\");\n error_code_ = RE2::ErrorPatternTooLarge;\n return;\n }\n\n // We used to compute this lazily, but it's used during the\n // typical control flow for a match call, so we now compute\n // it eagerly, which avoids the overhead of absl::once_flag.\n num_captures_ = suffix_regexp_->NumCaptures();\n\n // Could delay this until the first match call that\n // cares about submatch information, but the one-pass\n // machine's memory gets cut from the DFA memory budget,\n // and that is harder to do if the DFA has already\n // been built.\n is_one_pass_ = prog_->IsOnePass();\n}\n\n// Returns rprog_, computing it if needed.\nre2::Prog* RE2::ReverseProg() const {\n absl::call_once(rprog_once_, [](const RE2* re) {\n re->rprog_ =\n re->suffix_regexp_->CompileToReverseProg(re->options_.max_mem() / 3);\n if (re->rprog_ == NULL) {\n if (re->options_.log_errors())\n ABSL_LOG(ERROR) << \"Error reverse compiling '\" << trunc(*re->pattern_)\n << \"'\";\n // We no longer touch error_ and error_code_ because failing to compile\n // the reverse Prog is not a showstopper: falling back to NFA execution\n // is fine. More importantly, an RE2 object is supposed to be logically\n // immutable: whatever ok() would have returned after Init() completed,\n // it should continue to return that no matter what ReverseProg() does.\n }\n }, this);\n return rprog_;\n}\n\nRE2::~RE2() {\n if (group_names_ != empty_group_names())\n delete group_names_;\n if (named_groups_ != empty_named_groups())\n delete named_groups_;\n delete rprog_;\n delete prog_;\n if (error_arg_ != empty_string())\n delete error_arg_;\n if (error_ != empty_string())\n delete error_;\n if (suffix_regexp_)\n suffix_regexp_->Decref();\n if (entire_regexp_)\n entire_regexp_->Decref();\n delete pattern_;\n}\n\nint RE2::ProgramSize() const {\n if (prog_ == NULL)\n return -1;\n return prog_->size();\n}\n\nint RE2::ReverseProgramSize() const {\n if (prog_ == NULL)\n return -1;\n Prog* prog = ReverseProg();\n if (prog == NULL)\n return -1;\n return prog->size();\n}\n\n// Finds the most significant non-zero bit in n.\nstatic int FindMSBSet(uint32_t n) {\n ABSL_DCHECK_NE(n, uint32_t{0});\n#if defined(__GNUC__)\n return 31 ^ __builtin_clz(n);\n#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))\n unsigned long c;\n _BitScanReverse(&c, n);\n return static_cast(c);\n#else\n int c = 0;\n for (int shift = 1 << 4; shift != 0; shift >>= 1) {\n uint32_t word = n >> shift;\n if (word != 0) {\n n = word;\n c += shift;\n }\n }\n return c;\n#endif\n}\n\nstatic int Fanout(Prog* prog, std::vector* histogram) {\n SparseArray fanout(prog->size());\n prog->Fanout(&fanout);\n int data[32] = {};\n int size = 0;\n for (SparseArray::iterator i = fanout.begin(); i != fanout.end(); ++i) {\n if (i->value() == 0)\n continue;\n uint32_t value = i->value();\n int bucket = FindMSBSet(value);\n bucket += value & (value-1) ? 1 : 0;\n ++data[bucket];\n size = std::max(size, bucket+1);\n }\n if (histogram != NULL)\n histogram->assign(data, data+size);\n return size-1;\n}\n\nint RE2::ProgramFanout(std::vector* histogram) const {\n if (prog_ == NULL)\n return -1;\n return Fanout(prog_, histogram);\n}\n\nint RE2::ReverseProgramFanout(std::vector* histogram) const {\n if (prog_ == NULL)\n return -1;\n Prog* prog = ReverseProg();\n if (prog == NULL)\n return -1;\n return Fanout(prog, histogram);\n}\n\n// Returns named_groups_, computing it if needed.\nconst std::map& RE2::NamedCapturingGroups() const {\n absl::call_once(named_groups_once_, [](const RE2* re) {\n if (re->suffix_regexp_ != NULL)\n re->named_groups_ = re->suffix_regexp_->NamedCaptures();\n if (re->named_groups_ == NULL)\n re->named_groups_ = empty_named_groups();\n }, this);\n return *named_groups_;\n}\n\n// Returns group_names_, computing it if needed.\nconst std::map& RE2::CapturingGroupNames() const {\n absl::call_once(group_names_once_, [](const RE2* re) {\n if (re->suffix_regexp_ != NULL)\n re->group_names_ = re->suffix_regexp_->CaptureNames();\n if (re->group_names_ == NULL)\n re->group_names_ = empty_group_names();\n }, this);\n return *group_names_;\n}\n\n/***** Convenience interfaces *****/\n\nbool RE2::FullMatchN(absl::string_view text, const RE2& re,\n const Arg* const args[], int n) {\n return re.DoMatch(text, ANCHOR_BOTH, NULL, args, n);\n}\n\nbool RE2::PartialMatchN(absl::string_view text, const RE2& re,\n const Arg* const args[], int n) {\n return re.DoMatch(text, UNANCHORED, NULL, args, n);\n}\n\nbool RE2::ConsumeN(absl::string_view* input, const RE2& re,\n const Arg* const args[], int n) {\n size_t consumed;\n if (re.DoMatch(*input, ANCHOR_START, &consumed, args, n)) {\n input->remove_prefix(consumed);\n return true;\n } else {\n return false;\n }\n}\n\nbool RE2::FindAndConsumeN(absl::string_view* input, const RE2& re,\n const Arg* const args[], int n) {\n size_t consumed;\n if (re.DoMatch(*input, UNANCHORED, &consumed, args, n)) {\n input->remove_prefix(consumed);\n return true;\n } else {\n return false;\n }\n}\n\nbool RE2::Replace(std::string* str,\n const RE2& re,\n absl::string_view rewrite) {\n absl::string_view vec[kVecSize];\n int nvec = 1 + MaxSubmatch(rewrite);\n if (nvec > 1 + re.NumberOfCapturingGroups())\n return false;\n if (nvec > static_cast(ABSL_ARRAYSIZE(vec)))\n return false;\n if (!re.Match(*str, 0, str->size(), UNANCHORED, vec, nvec))\n return false;\n\n std::string s;\n if (!re.Rewrite(&s, rewrite, vec, nvec))\n return false;\n\n ABSL_DCHECK_GE(vec[0].data(), str->data());\n ABSL_DCHECK_LE(vec[0].data() + vec[0].size(), str->data() + str->size());\n str->replace(vec[0].data() - str->data(), vec[0].size(), s);\n return true;\n}\n\nint RE2::GlobalReplace(std::string* str,\n const RE2& re,\n absl::string_view rewrite) {\n absl::string_view vec[kVecSize];\n int nvec = 1 + MaxSubmatch(rewrite);\n if (nvec > 1 + re.NumberOfCapturingGroups())\n return false;\n if (nvec > static_cast(ABSL_ARRAYSIZE(vec)))\n return false;\n\n const char* p = str->data();\n const char* ep = p + str->size();\n const char* lastend = NULL;\n std::string out;\n int count = 0;\n while (p <= ep) {\n if (maximum_global_replace_count != -1 &&\n count >= maximum_global_replace_count)\n break;\n if (!re.Match(*str, static_cast(p - str->data()),\n str->size(), UNANCHORED, vec, nvec))\n break;\n if (p < vec[0].data())\n out.append(p, vec[0].data() - p);\n if (vec[0].data() == lastend && vec[0].empty()) {\n // Disallow empty match at end of last match: skip ahead.\n //\n // fullrune() takes int, not ptrdiff_t. However, it just looks\n // at the leading byte and treats any length >= 4 the same.\n if (re.options().encoding() == RE2::Options::EncodingUTF8 &&\n fullrune(p, static_cast(std::min(ptrdiff_t{4}, ep - p)))) {\n // re is in UTF-8 mode and there is enough left of str\n // to allow us to advance by up to UTFmax bytes.\n Rune r;\n int n = chartorune(&r, p);\n // Some copies of chartorune have a bug that accepts\n // encodings of values in (10FFFF, 1FFFFF] as valid.\n if (r > Runemax) {\n n = 1;\n r = Runeerror;\n }\n if (!(n == 1 && r == Runeerror)) { // no decoding error\n out.append(p, n);\n p += n;\n continue;\n }\n }\n // Most likely, re is in Latin-1 mode. If it is in UTF-8 mode,\n // we fell through from above and the GIGO principle applies.\n if (p < ep)\n out.append(p, 1);\n p++;\n continue;\n }\n re.Rewrite(&out, rewrite, vec, nvec);\n p = vec[0].data() + vec[0].size();\n lastend = p;\n count++;\n }\n\n if (count == 0)\n return 0;\n\n if (p < ep)\n out.append(p, ep - p);\n using std::swap;\n swap(out, *str);\n return count;\n}\n\nbool RE2::Extract(absl::string_view text,\n const RE2& re,\n absl::string_view rewrite,\n std::string* out) {\n absl::string_view vec[kVecSize];\n int nvec = 1 + MaxSubmatch(rewrite);\n if (nvec > 1 + re.NumberOfCapturingGroups())\n return false;\n if (nvec > static_cast(ABSL_ARRAYSIZE(vec)))\n return false;\n if (!re.Match(text, 0, text.size(), UNANCHORED, vec, nvec))\n return false;\n\n out->clear();\n return re.Rewrite(out, rewrite, vec, nvec);\n}\n\nstd::string RE2::QuoteMeta(absl::string_view unquoted) {\n std::string result;\n result.reserve(unquoted.size() << 1);\n\n // Escape any ascii character not in [A-Za-z_0-9].\n //\n // Note that it's legal to escape a character even if it has no\n // special meaning in a regular expression -- so this function does\n // that. (This also makes it identical to the perl function of the\n // same name except for the null-character special case;\n // see `perldoc -f quotemeta`.)\n for (size_t ii = 0; ii < unquoted.size(); ++ii) {\n // Note that using 'isalnum' here raises the benchmark time from\n // 32ns to 58ns:\n if ((unquoted[ii] < 'a' || unquoted[ii] > 'z') &&\n (unquoted[ii] < 'A' || unquoted[ii] > 'Z') &&\n (unquoted[ii] < '0' || unquoted[ii] > '9') &&\n unquoted[ii] != '_' &&\n // If this is the part of a UTF8 or Latin1 character, we need\n // to copy this byte without escaping. Experimentally this is\n // what works correctly with the regexp library.\n !(unquoted[ii] & 128)) {\n if (unquoted[ii] == '\\0') { // Special handling for null chars.\n // Note that this special handling is not strictly required for RE2,\n // but this quoting is required for other regexp libraries such as\n // PCRE.\n // Can't use \"\\\\0\" since the next character might be a digit.\n result += \"\\\\x00\";\n continue;\n }\n result += '\\\\';\n }\n result += unquoted[ii];\n }\n\n return result;\n}\n\nbool RE2::PossibleMatchRange(std::string* min, std::string* max,\n int maxlen) const {\n if (prog_ == NULL)\n return false;\n\n int n = static_cast(prefix_.size());\n if (n > maxlen)\n n = maxlen;\n\n // Determine initial min max from prefix_ literal.\n *min = prefix_.substr(0, n);\n *max = prefix_.substr(0, n);\n if (prefix_foldcase_) {\n // prefix is ASCII lowercase; change *min to uppercase.\n for (int i = 0; i < n; i++) {\n char& c = (*min)[i];\n if ('a' <= c && c <= 'z')\n c += 'A' - 'a';\n }\n }\n\n // Add to prefix min max using PossibleMatchRange on regexp.\n std::string dmin, dmax;\n maxlen -= n;\n if (maxlen > 0 && prog_->PossibleMatchRange(&dmin, &dmax, maxlen)) {\n min->append(dmin);\n max->append(dmax);\n } else if (!max->empty()) {\n // prog_->PossibleMatchRange has failed us,\n // but we still have useful information from prefix_.\n // Round up *max to allow any possible suffix.\n PrefixSuccessor(max);\n } else {\n // Nothing useful.\n *min = \"\";\n *max = \"\";\n return false;\n }\n\n return true;\n}\n\n// Avoid possible locale nonsense in standard strcasecmp.\n// The string a is known to be all lowercase.\nstatic int ascii_strcasecmp(const char* a, const char* b, size_t len) {\n const char* ae = a + len;\n\n for (; a < ae; a++, b++) {\n uint8_t x = *a;\n uint8_t y = *b;\n if ('A' <= y && y <= 'Z')\n y += 'a' - 'A';\n if (x != y)\n return x - y;\n }\n return 0;\n}\n\n\n/***** Actual matching and rewriting code *****/\n\nbool RE2::Match(absl::string_view text,\n size_t startpos,\n size_t endpos,\n Anchor re_anchor,\n absl::string_view* submatch,\n int nsubmatch) const {\n if (!ok()) {\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"Invalid RE2: \" << *error_;\n return false;\n }\n\n if (startpos > endpos || endpos > text.size()) {\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"RE2: invalid startpos, endpos pair. [\"\n << \"startpos: \" << startpos << \", \"\n << \"endpos: \" << endpos << \", \"\n << \"text size: \" << text.size() << \"]\";\n return false;\n }\n\n absl::string_view subtext = text;\n subtext.remove_prefix(startpos);\n subtext.remove_suffix(text.size() - endpos);\n\n // Use DFAs to find exact location of match, filter out non-matches.\n\n // Don't ask for the location if we won't use it.\n // SearchDFA can do extra optimizations in that case.\n absl::string_view match;\n absl::string_view* matchp = &match;\n if (nsubmatch == 0)\n matchp = NULL;\n\n int ncap = 1 + NumberOfCapturingGroups();\n if (ncap > nsubmatch)\n ncap = nsubmatch;\n\n // If the regexp is anchored explicitly, must not be in middle of text.\n if (prog_->anchor_start() && startpos != 0)\n return false;\n if (prog_->anchor_end() && endpos != text.size())\n return false;\n\n // If the regexp is anchored explicitly, update re_anchor\n // so that we can potentially fall into a faster case below.\n if (prog_->anchor_start() && prog_->anchor_end())\n re_anchor = ANCHOR_BOTH;\n else if (prog_->anchor_start() && re_anchor != ANCHOR_BOTH)\n re_anchor = ANCHOR_START;\n\n // Check for the required prefix, if any.\n size_t prefixlen = 0;\n if (!prefix_.empty()) {\n if (startpos != 0)\n return false;\n prefixlen = prefix_.size();\n if (prefixlen > subtext.size())\n return false;\n if (prefix_foldcase_) {\n if (ascii_strcasecmp(&prefix_[0], subtext.data(), prefixlen) != 0)\n return false;\n } else {\n if (memcmp(&prefix_[0], subtext.data(), prefixlen) != 0)\n return false;\n }\n subtext.remove_prefix(prefixlen);\n // If there is a required prefix, the anchor must be at least ANCHOR_START.\n if (re_anchor != ANCHOR_BOTH)\n re_anchor = ANCHOR_START;\n }\n\n Prog::Anchor anchor = Prog::kUnanchored;\n Prog::MatchKind kind =\n longest_match_ ? Prog::kLongestMatch : Prog::kFirstMatch;\n\n bool can_one_pass = is_one_pass_ && ncap <= Prog::kMaxOnePassCapture;\n bool can_bit_state = prog_->CanBitState();\n size_t bit_state_text_max_size = prog_->bit_state_text_max_size();\n\n#ifdef RE2_HAVE_THREAD_LOCAL\n hooks::context = this;\n#endif\n bool dfa_failed = false;\n bool skipped_test = false;\n switch (re_anchor) {\n default:\n ABSL_LOG(DFATAL) << \"Unexpected re_anchor value: \" << re_anchor;\n return false;\n\n case UNANCHORED: {\n if (prog_->anchor_end()) {\n // This is a very special case: we don't need the forward DFA because\n // we already know where the match must end! Instead, the reverse DFA\n // can say whether there is a match and (optionally) where it starts.\n Prog* prog = ReverseProg();\n if (prog == NULL) {\n // Fall back to NFA below.\n skipped_test = true;\n break;\n }\n if (!prog->SearchDFA(subtext, text, Prog::kAnchored,\n Prog::kLongestMatch, matchp, &dfa_failed, NULL)) {\n if (dfa_failed) {\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"DFA out of memory: \"\n << \"pattern length \" << pattern_->size() << \", \"\n << \"program size \" << prog->size() << \", \"\n << \"list count \" << prog->list_count() << \", \"\n << \"bytemap range \" << prog->bytemap_range();\n // Fall back to NFA below.\n skipped_test = true;\n break;\n }\n return false;\n }\n if (matchp == NULL) // Matched. Don't care where.\n return true;\n break;\n }\n\n if (!prog_->SearchDFA(subtext, text, anchor, kind,\n matchp, &dfa_failed, NULL)) {\n if (dfa_failed) {\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"DFA out of memory: \"\n << \"pattern length \" << pattern_->size() << \", \"\n << \"program size \" << prog_->size() << \", \"\n << \"list count \" << prog_->list_count() << \", \"\n << \"bytemap range \" << prog_->bytemap_range();\n // Fall back to NFA below.\n skipped_test = true;\n break;\n }\n return false;\n }\n if (matchp == NULL) // Matched. Don't care where.\n return true;\n // SearchDFA set match.end() but didn't know where the\n // match started. Run the regexp backward from match.end()\n // to find the longest possible match -- that's where it started.\n Prog* prog = ReverseProg();\n if (prog == NULL) {\n // Fall back to NFA below.\n skipped_test = true;\n break;\n }\n if (!prog->SearchDFA(match, text, Prog::kAnchored,\n Prog::kLongestMatch, &match, &dfa_failed, NULL)) {\n if (dfa_failed) {\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"DFA out of memory: \"\n << \"pattern length \" << pattern_->size() << \", \"\n << \"program size \" << prog->size() << \", \"\n << \"list count \" << prog->list_count() << \", \"\n << \"bytemap range \" << prog->bytemap_range();\n // Fall back to NFA below.\n skipped_test = true;\n break;\n }\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"SearchDFA inconsistency\";\n return false;\n }\n break;\n }\n\n case ANCHOR_BOTH:\n case ANCHOR_START:\n if (re_anchor == ANCHOR_BOTH)\n kind = Prog::kFullMatch;\n anchor = Prog::kAnchored;\n\n // If only a small amount of text and need submatch\n // information anyway and we're going to use OnePass or BitState\n // to get it, we might as well not even bother with the DFA:\n // OnePass or BitState will be fast enough.\n // On tiny texts, OnePass outruns even the DFA, and\n // it doesn't have the shared state and occasional mutex that\n // the DFA does.\n if (can_one_pass && text.size() <= 4096 &&\n (ncap > 1 || text.size() <= 16)) {\n skipped_test = true;\n break;\n }\n if (can_bit_state && text.size() <= bit_state_text_max_size &&\n ncap > 1) {\n skipped_test = true;\n break;\n }\n if (!prog_->SearchDFA(subtext, text, anchor, kind,\n &match, &dfa_failed, NULL)) {\n if (dfa_failed) {\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"DFA out of memory: \"\n << \"pattern length \" << pattern_->size() << \", \"\n << \"program size \" << prog_->size() << \", \"\n << \"list count \" << prog_->list_count() << \", \"\n << \"bytemap range \" << prog_->bytemap_range();\n // Fall back to NFA below.\n skipped_test = true;\n break;\n }\n return false;\n }\n break;\n }\n\n if (!skipped_test && ncap <= 1) {\n // We know exactly where it matches. That's enough.\n if (ncap == 1)\n submatch[0] = match;\n } else {\n absl::string_view subtext1;\n if (skipped_test) {\n // DFA ran out of memory or was skipped:\n // need to search in entire original text.\n subtext1 = subtext;\n } else {\n // DFA found the exact match location:\n // let NFA run an anchored, full match search\n // to find submatch locations.\n subtext1 = match;\n anchor = Prog::kAnchored;\n kind = Prog::kFullMatch;\n }\n\n if (can_one_pass && anchor != Prog::kUnanchored) {\n if (!prog_->SearchOnePass(subtext1, text, anchor, kind, submatch, ncap)) {\n if (!skipped_test && options_.log_errors())\n ABSL_LOG(ERROR) << \"SearchOnePass inconsistency\";\n return false;\n }\n } else if (can_bit_state && subtext1.size() <= bit_state_text_max_size) {\n if (!prog_->SearchBitState(subtext1, text, anchor,\n kind, submatch, ncap)) {\n if (!skipped_test && options_.log_errors())\n ABSL_LOG(ERROR) << \"SearchBitState inconsistency\";\n return false;\n }\n } else {\n if (!prog_->SearchNFA(subtext1, text, anchor, kind, submatch, ncap)) {\n if (!skipped_test && options_.log_errors())\n ABSL_LOG(ERROR) << \"SearchNFA inconsistency\";\n return false;\n }\n }\n }\n\n // Adjust overall match for required prefix that we stripped off.\n if (prefixlen > 0 && nsubmatch > 0)\n submatch[0] = absl::string_view(submatch[0].data() - prefixlen,\n submatch[0].size() + prefixlen);\n\n // Zero submatches that don't exist in the regexp.\n for (int i = ncap; i < nsubmatch; i++)\n submatch[i] = absl::string_view();\n return true;\n}\n\n// Internal matcher - like Match() but takes Args not string_views.\nbool RE2::DoMatch(absl::string_view text,\n Anchor re_anchor,\n size_t* consumed,\n const Arg* const* args,\n int n) const {\n if (!ok()) {\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"Invalid RE2: \" << *error_;\n return false;\n }\n\n if (NumberOfCapturingGroups() < n) {\n // RE has fewer capturing groups than number of Arg pointers passed in.\n return false;\n }\n\n // Count number of capture groups needed.\n int nvec;\n if (n == 0 && consumed == NULL)\n nvec = 0;\n else\n nvec = n+1;\n\n absl::FixedArray vec_storage(nvec);\n absl::string_view* vec = vec_storage.data();\n\n if (!Match(text, 0, text.size(), re_anchor, vec, nvec)) {\n return false;\n }\n\n if (consumed != NULL)\n *consumed = static_cast(EndPtr(vec[0]) - BeginPtr(text));\n\n if (n == 0 || args == NULL) {\n // We are not interested in results\n return true;\n }\n\n // If we got here, we must have matched the whole pattern.\n for (int i = 0; i < n; i++) {\n absl::string_view s = vec[i+1];\n if (!args[i]->Parse(s.data(), s.size())) {\n // TODO: Should we indicate what the error was?\n return false;\n }\n }\n\n return true;\n}\n\n// Checks that the rewrite string is well-formed with respect to this\n// regular expression.\nbool RE2::CheckRewriteString(absl::string_view rewrite,\n std::string* error) const {\n int max_token = -1;\n for (const char *s = rewrite.data(), *end = s + rewrite.size();\n s < end; s++) {\n int c = *s;\n if (c != '\\\\') {\n continue;\n }\n if (++s == end) {\n *error = \"Rewrite schema error: '\\\\' not allowed at end.\";\n return false;\n }\n c = *s;\n if (c == '\\\\') {\n continue;\n }\n if (!absl::ascii_isdigit(c)) {\n *error = \"Rewrite schema error: \"\n \"'\\\\' must be followed by a digit or '\\\\'.\";\n return false;\n }\n int n = (c - '0');\n if (max_token < n) {\n max_token = n;\n }\n }\n\n if (max_token > NumberOfCapturingGroups()) {\n *error = absl::StrFormat(\n \"Rewrite schema requests %d matches, but the regexp only has %d \"\n \"parenthesized subexpressions.\",\n max_token, NumberOfCapturingGroups());\n return false;\n }\n return true;\n}\n\n// Returns the maximum submatch needed for the rewrite to be done by Replace().\n// E.g. if rewrite == \"foo \\\\2,\\\\1\", returns 2.\nint RE2::MaxSubmatch(absl::string_view rewrite) {\n int max = 0;\n for (const char *s = rewrite.data(), *end = s + rewrite.size();\n s < end; s++) {\n if (*s == '\\\\') {\n s++;\n int c = (s < end) ? *s : -1;\n if (absl::ascii_isdigit(c)) {\n int n = (c - '0');\n if (n > max)\n max = n;\n }\n }\n }\n return max;\n}\n\n// Append the \"rewrite\" string, with backslash substitutions from \"vec\",\n// to string \"out\".\nbool RE2::Rewrite(std::string* out,\n absl::string_view rewrite,\n const absl::string_view* vec,\n int veclen) const {\n for (const char *s = rewrite.data(), *end = s + rewrite.size();\n s < end; s++) {\n if (*s != '\\\\') {\n out->push_back(*s);\n continue;\n }\n s++;\n int c = (s < end) ? *s : -1;\n if (absl::ascii_isdigit(c)) {\n int n = (c - '0');\n if (n >= veclen) {\n if (options_.log_errors()) {\n ABSL_LOG(ERROR) << \"invalid substitution \\\\\" << n\n << \" from \" << veclen << \" groups\";\n }\n return false;\n }\n absl::string_view snip = vec[n];\n if (!snip.empty())\n out->append(snip.data(), snip.size());\n } else if (c == '\\\\') {\n out->push_back('\\\\');\n } else {\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"invalid rewrite pattern: \" << rewrite;\n return false;\n }\n }\n return true;\n}\n\n/***** Parsers for various types *****/\n\nnamespace re2_internal {\n\ntemplate <>\nbool Parse(const char* str, size_t n, void* dest) {\n // We fail if somebody asked us to store into a non-NULL void* pointer\n return (dest == NULL);\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, std::string* dest) {\n if (dest == NULL) return true;\n dest->assign(str, n);\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, absl::string_view* dest) {\n if (dest == NULL) return true;\n *dest = absl::string_view(str, n);\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, char* dest) {\n if (n != 1) return false;\n if (dest == NULL) return true;\n *dest = str[0];\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, signed char* dest) {\n if (n != 1) return false;\n if (dest == NULL) return true;\n *dest = str[0];\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, unsigned char* dest) {\n if (n != 1) return false;\n if (dest == NULL) return true;\n *dest = str[0];\n return true;\n}\n\n// Largest number spec that we are willing to parse\nstatic const int kMaxNumberLength = 32;\n\n// REQUIRES \"buf\" must have length at least nbuf.\n// Copies \"str\" into \"buf\" and null-terminates.\n// Overwrites *np with the new length.\nstatic const char* TerminateNumber(char* buf, size_t nbuf, const char* str,\n size_t* np, bool accept_spaces) {\n size_t n = *np;\n if (n == 0) return \"\";\n if (n > 0 && absl::ascii_isspace(*str)) {\n // We are less forgiving than the strtoxxx() routines and do not\n // allow leading spaces. We do allow leading spaces for floats.\n if (!accept_spaces) {\n return \"\";\n }\n while (n > 0 && absl::ascii_isspace(*str)) {\n n--;\n str++;\n }\n }\n\n // Although buf has a fixed maximum size, we can still handle\n // arbitrarily large integers correctly by omitting leading zeros.\n // (Numbers that are still too long will be out of range.)\n // Before deciding whether str is too long,\n // remove leading zeros with s/000+/00/.\n // Leaving the leading two zeros in place means that\n // we don't change 0000x123 (invalid) into 0x123 (valid).\n // Skip over leading - before replacing.\n bool neg = false;\n if (n >= 1 && str[0] == '-') {\n neg = true;\n n--;\n str++;\n }\n\n if (n >= 3 && str[0] == '0' && str[1] == '0') {\n while (n >= 3 && str[2] == '0') {\n n--;\n str++;\n }\n }\n\n if (neg) { // make room in buf for -\n n++;\n str--;\n }\n\n if (n > nbuf-1) return \"\";\n\n memmove(buf, str, n);\n if (neg) {\n buf[0] = '-';\n }\n buf[n] = '\\0';\n *np = n;\n return buf;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, float* dest) {\n if (n == 0) return false;\n static const int kMaxLength = 200;\n char buf[kMaxLength+1];\n str = TerminateNumber(buf, sizeof buf, str, &n, true);\n char* end;\n errno = 0;\n float r = strtof(str, &end);\n if (end != str + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n *dest = r;\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, double* dest) {\n if (n == 0) return false;\n static const int kMaxLength = 200;\n char buf[kMaxLength+1];\n str = TerminateNumber(buf, sizeof buf, str, &n, true);\n char* end;\n errno = 0;\n double r = strtod(str, &end);\n if (end != str + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n *dest = r;\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, long* dest, int radix) {\n if (n == 0) return false;\n char buf[kMaxNumberLength+1];\n str = TerminateNumber(buf, sizeof buf, str, &n, false);\n char* end;\n errno = 0;\n long r = strtol(str, &end, radix);\n if (end != str + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n *dest = r;\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, unsigned long* dest, int radix) {\n if (n == 0) return false;\n char buf[kMaxNumberLength+1];\n str = TerminateNumber(buf, sizeof buf, str, &n, false);\n if (str[0] == '-') {\n // strtoul() will silently accept negative numbers and parse\n // them. This module is more strict and treats them as errors.\n return false;\n }\n\n char* end;\n errno = 0;\n unsigned long r = strtoul(str, &end, radix);\n if (end != str + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n *dest = r;\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, short* dest, int radix) {\n long r;\n if (!Parse(str, n, &r, radix)) return false; // Could not parse\n if ((short)r != r) return false; // Out of range\n if (dest == NULL) return true;\n *dest = (short)r;\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, unsigned short* dest, int radix) {\n unsigned long r;\n if (!Parse(str, n, &r, radix)) return false; // Could not parse\n if ((unsigned short)r != r) return false; // Out of range\n if (dest == NULL) return true;\n *dest = (unsigned short)r;\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, int* dest, int radix) {\n long r;\n if (!Parse(str, n, &r, radix)) return false; // Could not parse\n if ((int)r != r) return false; // Out of range\n if (dest == NULL) return true;\n *dest = (int)r;\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, unsigned int* dest, int radix) {\n unsigned long r;\n if (!Parse(str, n, &r, radix)) return false; // Could not parse\n if ((unsigned int)r != r) return false; // Out of range\n if (dest == NULL) return true;\n *dest = (unsigned int)r;\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, long long* dest, int radix) {\n if (n == 0) return false;\n char buf[kMaxNumberLength+1];\n str = TerminateNumber(buf, sizeof buf, str, &n, false);\n char* end;\n errno = 0;\n long long r = strtoll(str, &end, radix);\n if (end != str + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n *dest = r;\n return true;\n}\n\ntemplate <>\nbool Parse(const char* str, size_t n, unsigned long long* dest, int radix) {\n if (n == 0) return false;\n char buf[kMaxNumberLength+1];\n str = TerminateNumber(buf, sizeof buf, str, &n, false);\n if (str[0] == '-') {\n // strtoull() will silently accept negative numbers and parse\n // them. This module is more strict and treats them as errors.\n return false;\n }\n char* end;\n errno = 0;\n unsigned long long r = strtoull(str, &end, radix);\n if (end != str + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n *dest = r;\n return true;\n}\n\n} // namespace re2_internal\n\nnamespace hooks {\n\n#ifdef RE2_HAVE_THREAD_LOCAL\nthread_local const RE2* context = NULL;\n#endif\n\ntemplate \nunion Hook {\n void Store(T* cb) { cb_.store(cb, std::memory_order_release); }\n T* Load() const { return cb_.load(std::memory_order_acquire); }\n\n#if !defined(__clang__) && defined(_MSC_VER)\n // Citing https://github.com/protocolbuffers/protobuf/pull/4777 as precedent,\n // this is a gross hack to make std::atomic constant-initialized on MSVC.\n static_assert(ATOMIC_POINTER_LOCK_FREE == 2,\n \"std::atomic must be always lock-free\");\n T* cb_for_constinit_;\n#endif\n\n std::atomic cb_;\n};\n\ntemplate \nstatic void DoNothing(const T&) {}\n\n#define DEFINE_HOOK(type, name) \\\n static Hook name##_hook = {{&DoNothing}}; \\\n void Set##type##Hook(type##Callback* cb) { name##_hook.Store(cb); } \\\n type##Callback* Get##type##Hook() { return name##_hook.Load(); }\n\nDEFINE_HOOK(DFAStateCacheReset, dfa_state_cache_reset)\nDEFINE_HOOK(DFASearchFailure, dfa_search_failure)\n\n#undef DEFINE_HOOK\n\n} // namespace hooks\n\n} // namespace re2\n" }, { "path": "re2/re2.h", "content": "// Copyright 2003-2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_RE2_H_\n#define RE2_RE2_H_\n\n// C++ interface to the re2 regular-expression library.\n// RE2 supports Perl-style regular expressions (with extensions like\n// \\d, \\w, \\s, ...).\n//\n// -----------------------------------------------------------------------\n// REGEXP SYNTAX:\n//\n// This module uses the re2 library and hence supports\n// its syntax for regular expressions, which is similar to Perl's with\n// some of the more complicated things thrown away. In particular,\n// backreferences and generalized assertions are not available, nor is \\Z.\n//\n// See https://github.com/google/re2/wiki/Syntax for the syntax\n// supported by RE2, and a comparison with PCRE and PERL regexps.\n//\n// For those not familiar with Perl's regular expressions,\n// here are some examples of the most commonly used extensions:\n//\n// \"hello (\\\\w+) world\" -- \\w matches a \"word\" character\n// \"version (\\\\d+)\" -- \\d matches a digit\n// \"hello\\\\s+world\" -- \\s matches any whitespace character\n// \"\\\\b(\\\\w+)\\\\b\" -- \\b matches non-empty string at word boundary\n// \"(?i)hello\" -- (?i) turns on case-insensitive matching\n// \"/\\\\*(.*?)\\\\*/\" -- .*? matches . minimum no. of times possible\n//\n// The double backslashes are needed when writing C++ string literals.\n// However, they should NOT be used when writing C++11 raw string literals:\n//\n// R\"(hello (\\w+) world)\" -- \\w matches a \"word\" character\n// R\"(version (\\d+))\" -- \\d matches a digit\n// R\"(hello\\s+world)\" -- \\s matches any whitespace character\n// R\"(\\b(\\w+)\\b)\" -- \\b matches non-empty string at word boundary\n// R\"((?i)hello)\" -- (?i) turns on case-insensitive matching\n// R\"(/\\*(.*?)\\*/)\" -- .*? matches . minimum no. of times possible\n//\n// When using UTF-8 encoding, case-insensitive matching will perform\n// simple case folding, not full case folding.\n//\n// -----------------------------------------------------------------------\n// MATCHING INTERFACE:\n//\n// The \"FullMatch\" operation checks that supplied text matches a\n// supplied pattern exactly.\n//\n// Example: successful match\n// ABSL_CHECK(RE2::FullMatch(\"hello\", \"h.*o\"));\n//\n// Example: unsuccessful match (requires full match):\n// ABSL_CHECK(!RE2::FullMatch(\"hello\", \"e\"));\n//\n// -----------------------------------------------------------------------\n// UTF-8 AND THE MATCHING INTERFACE:\n//\n// By default, the pattern and input text are interpreted as UTF-8.\n// The RE2::Latin1 option causes them to be interpreted as Latin-1.\n//\n// Example:\n// ABSL_CHECK(RE2::FullMatch(utf8_string, RE2(utf8_pattern)));\n// ABSL_CHECK(RE2::FullMatch(latin1_string, RE2(latin1_pattern,\n// RE2::Latin1)));\n//\n// -----------------------------------------------------------------------\n// SUBMATCH EXTRACTION:\n//\n// You can supply extra pointer arguments to extract submatches.\n// On match failure, none of the pointees will have been modified.\n// On match success, the submatches will be converted (as necessary) and\n// their values will be assigned to their pointees until all conversions\n// have succeeded or one conversion has failed.\n// On conversion failure, the pointees will be in an indeterminate state\n// because the caller has no way of knowing which conversion failed.\n// However, conversion cannot fail for types like string and string_view\n// that do not inspect the submatch contents. Hence, in the common case\n// where all of the pointees are of such types, failure is always due to\n// match failure and thus none of the pointees will have been modified.\n//\n// Example: extracts \"ruby\" into \"s\" and 1234 into \"i\"\n// int i;\n// std::string s;\n// ABSL_CHECK(RE2::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", &s, &i));\n//\n// Example: extracts \"ruby\" into \"s\" and no value into \"i\"\n// std::optional i;\n// std::string s;\n// ABSL_CHECK(RE2::FullMatch(\"ruby\", \"(\\\\w+)(?::(\\\\d+))?\", &s, &i));\n//\n// Example: fails because string cannot be stored in integer\n// ABSL_CHECK(!RE2::FullMatch(\"ruby\", \"(.*)\", &i));\n//\n// Example: fails because there aren't enough sub-patterns\n// ABSL_CHECK(!RE2::FullMatch(\"ruby:1234\", \"\\\\w+:\\\\d+\", &s));\n//\n// Example: does not try to extract any extra sub-patterns\n// ABSL_CHECK(RE2::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", &s));\n//\n// Example: does not try to extract into NULL\n// ABSL_CHECK(RE2::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", NULL, &i));\n//\n// Example: integer overflow causes failure\n// ABSL_CHECK(!RE2::FullMatch(\"ruby:1234567891234\", \"\\\\w+:(\\\\d+)\", &i));\n//\n// NOTE(rsc): Asking for submatches slows successful matches quite a bit.\n// This may get a little faster in the future, but right now is slower\n// than PCRE. On the other hand, failed matches run *very* fast (faster\n// than PCRE), as do matches without submatch extraction.\n//\n// -----------------------------------------------------------------------\n// PARTIAL MATCHES\n//\n// You can use the \"PartialMatch\" operation when you want the pattern\n// to match any substring of the text.\n//\n// Example: simple search for a string:\n// ABSL_CHECK(RE2::PartialMatch(\"hello\", \"ell\"));\n//\n// Example: find first number in a string\n// int number;\n// ABSL_CHECK(RE2::PartialMatch(\"x*100 + 20\", \"(\\\\d+)\", &number));\n// ABSL_CHECK_EQ(number, 100);\n//\n// -----------------------------------------------------------------------\n// PRE-COMPILED REGULAR EXPRESSIONS\n//\n// RE2 makes it easy to use any string as a regular expression, without\n// requiring a separate compilation step.\n//\n// If speed is of the essence, you can create a pre-compiled \"RE2\"\n// object from the pattern and use it multiple times. If you do so,\n// you can typically parse text faster than with sscanf.\n//\n// Example: precompile pattern for faster matching:\n// RE2 pattern(\"h.*o\");\n// while (ReadLine(&str)) {\n// if (RE2::FullMatch(str, pattern)) ...;\n// }\n//\n// -----------------------------------------------------------------------\n// SCANNING TEXT INCREMENTALLY\n//\n// The \"Consume\" operation may be useful if you want to repeatedly\n// match regular expressions at the front of a string and skip over\n// them as they match. This requires use of the string_view type,\n// which represents a sub-range of a real string.\n//\n// Example: read lines of the form \"var = value\" from a string.\n// std::string contents = ...; // Fill string somehow\n// absl::string_view input(contents); // Wrap a string_view around it\n//\n// std::string var;\n// int value;\n// while (RE2::Consume(&input, \"(\\\\w+) = (\\\\d+)\\n\", &var, &value)) {\n// ...;\n// }\n//\n// Each successful call to \"Consume\" will set \"var/value\", and also\n// advance \"input\" so it points past the matched text. Note that if the\n// regular expression matches an empty string, input will advance\n// by 0 bytes. If the regular expression being used might match\n// an empty string, the loop body must check for this case and either\n// advance the string or break out of the loop.\n//\n// The \"FindAndConsume\" operation is similar to \"Consume\" but does not\n// anchor your match at the beginning of the string. For example, you\n// could extract all words from a string by repeatedly calling\n// RE2::FindAndConsume(&input, \"(\\\\w+)\", &word)\n//\n// -----------------------------------------------------------------------\n// USING VARIABLE NUMBER OF ARGUMENTS\n//\n// The above operations require you to know the number of arguments\n// when you write the code. This is not always possible or easy (for\n// example, the regular expression may be calculated at run time).\n// You can use the \"N\" version of the operations when the number of\n// match arguments are determined at run time.\n//\n// Example:\n// const RE2::Arg* args[10];\n// int n;\n// // ... populate args with pointers to RE2::Arg values ...\n// // ... set n to the number of RE2::Arg objects ...\n// bool match = RE2::FullMatchN(input, pattern, args, n);\n//\n// The last statement is equivalent to\n//\n// bool match = RE2::FullMatch(input, pattern,\n// *args[0], *args[1], ..., *args[n - 1]);\n//\n// -----------------------------------------------------------------------\n// PARSING HEX/OCTAL/C-RADIX NUMBERS\n//\n// By default, if you pass a pointer to a numeric value, the\n// corresponding text is interpreted as a base-10 number. You can\n// instead wrap the pointer with a call to one of the operators Hex(),\n// Octal(), or CRadix() to interpret the text in another base. The\n// CRadix operator interprets C-style \"0\" (base-8) and \"0x\" (base-16)\n// prefixes, but defaults to base-10.\n//\n// Example:\n// int a, b, c, d;\n// ABSL_CHECK(RE2::FullMatch(\"100 40 0100 0x40\", \"(.*) (.*) (.*) (.*)\",\n// RE2::Octal(&a), RE2::Hex(&b), RE2::CRadix(&c), RE2::CRadix(&d));\n// will leave 64 in a, b, c, and d.\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"absl/base/call_once.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/stringpiece.h\"\n\n#if defined(__APPLE__)\n#include \n#endif\n\nnamespace re2 {\nclass Prog;\nclass Regexp;\n} // namespace re2\n\nnamespace re2 {\n\n// Interface for regular expression matching. Also corresponds to a\n// pre-compiled regular expression. An \"RE2\" object is safe for\n// concurrent use by multiple threads.\nclass RE2 {\n public:\n // We convert user-passed pointers into special Arg objects\n class Arg;\n class Options;\n\n // Defined in set.h.\n class Set;\n\n enum ErrorCode {\n NoError = 0,\n\n // Unexpected error\n ErrorInternal,\n\n // Parse errors\n ErrorBadEscape, // bad escape sequence\n ErrorBadCharClass, // bad character class\n ErrorBadCharRange, // bad character class range\n ErrorMissingBracket, // missing closing ]\n ErrorMissingParen, // missing closing )\n ErrorUnexpectedParen, // unexpected closing )\n ErrorTrailingBackslash, // trailing \\ at end of regexp\n ErrorRepeatArgument, // repeat argument missing, e.g. \"*\"\n ErrorRepeatSize, // bad repetition argument\n ErrorRepeatOp, // bad repetition operator\n ErrorBadPerlOp, // bad perl operator\n ErrorBadUTF8, // invalid UTF-8 in regexp\n ErrorBadNamedCapture, // bad named capture group\n ErrorPatternTooLarge // pattern too large (compile failed)\n };\n\n // Predefined common options.\n // If you need more complicated things, instantiate\n // an Option class, possibly passing one of these to\n // the Option constructor, change the settings, and pass that\n // Option class to the RE2 constructor.\n enum CannedOptions {\n DefaultOptions = 0,\n Latin1, // treat input as Latin-1 (default UTF-8)\n POSIX, // POSIX syntax, leftmost-longest match\n Quiet // do not log about regexp parse errors\n };\n\n // Need to have the const char* and const std::string& forms for implicit\n // conversions when passing string literals to FullMatch and PartialMatch.\n // Otherwise the absl::string_view form would be sufficient.\n RE2(const char* pattern);\n RE2(const std::string& pattern);\n RE2(absl::string_view pattern);\n RE2(absl::string_view pattern, const Options& options);\n ~RE2();\n\n // Not copyable.\n // RE2 objects are expensive. You should probably use std::shared_ptr\n // instead. If you really must copy, RE2(first.pattern(), first.options())\n // effectively does so: it produces a second object that mimics the first.\n RE2(const RE2&) = delete;\n RE2& operator=(const RE2&) = delete;\n // Not movable.\n // RE2 objects are thread-safe and logically immutable. You should probably\n // use std::unique_ptr instead. Otherwise, consider std::deque if\n // direct emplacement into a container is desired. If you really must move,\n // be prepared to submit a design document along with your feature request.\n RE2(RE2&&) = delete;\n RE2& operator=(RE2&&) = delete;\n\n // Returns whether RE2 was created properly.\n bool ok() const { return error_code() == NoError; }\n\n // The string specification for this RE2. E.g.\n // RE2 re(\"ab*c?d+\");\n // re.pattern(); // \"ab*c?d+\"\n const std::string& pattern() const { return *pattern_; }\n\n // If RE2 could not be created properly, returns an error string.\n // Else returns the empty string.\n const std::string& error() const { return *error_; }\n\n // If RE2 could not be created properly, returns an error code.\n // Else returns RE2::NoError (== 0).\n ErrorCode error_code() const { return error_code_; }\n\n // If RE2 could not be created properly, returns the offending\n // portion of the regexp.\n const std::string& error_arg() const { return *error_arg_; }\n\n // Returns the program size, a very approximate measure of a regexp's \"cost\".\n // Larger numbers are more expensive than smaller numbers.\n int ProgramSize() const;\n int ReverseProgramSize() const;\n\n // If histogram is not null, outputs the program fanout\n // as a histogram bucketed by powers of 2.\n // Returns the number of the largest non-empty bucket.\n int ProgramFanout(std::vector* histogram) const;\n int ReverseProgramFanout(std::vector* histogram) const;\n\n // Returns the underlying Regexp; not for general use.\n // Returns entire_regexp_ so that callers don't need\n // to know about prefix_ and prefix_foldcase_.\n re2::Regexp* Regexp() const { return entire_regexp_; }\n\n /***** The array-based matching interface ******/\n\n // The functions here have names ending in 'N' and are used to implement\n // the functions whose names are the prefix before the 'N'. It is sometimes\n // useful to invoke them directly, but the syntax is awkward, so the 'N'-less\n // versions should be preferred.\n static bool FullMatchN(absl::string_view text, const RE2& re,\n const Arg* const args[], int n);\n static bool PartialMatchN(absl::string_view text, const RE2& re,\n const Arg* const args[], int n);\n static bool ConsumeN(absl::string_view* input, const RE2& re,\n const Arg* const args[], int n);\n static bool FindAndConsumeN(absl::string_view* input, const RE2& re,\n const Arg* const args[], int n);\n\n private:\n template \n static inline bool Apply(F f, SP sp, const RE2& re) {\n return f(sp, re, NULL, 0);\n }\n\n template \n static inline bool Apply(F f, SP sp, const RE2& re, const A&... a) {\n const Arg* const args[] = {&a...};\n const int n = sizeof...(a);\n return f(sp, re, args, n);\n }\n\n public:\n // In order to allow FullMatch() et al. to be called with a varying number\n // of arguments of varying types, we use two layers of variadic templates.\n // The first layer constructs the temporary Arg objects. The second layer\n // (above) constructs the array of pointers to the temporary Arg objects.\n\n /***** The useful part: the matching interface *****/\n\n // Matches \"text\" against \"re\". If pointer arguments are\n // supplied, copies matched sub-patterns into them.\n //\n // You can pass in a \"const char*\" or a \"std::string\" for \"text\".\n // You can pass in a \"const char*\" or a \"std::string\" or a \"RE2\" for \"re\".\n //\n // The provided pointer arguments can be pointers to any scalar numeric\n // type, or one of:\n // std::string (matched piece is copied to string)\n // absl::string_view (string_view is mutated to point to matched piece)\n // std::optional (T is a supported numeric or string type as above)\n // T (\"bool T::ParseFrom(const char*, size_t)\" must exist)\n // (void*)NULL (the corresponding matched sub-pattern is not copied)\n //\n // Returns true iff all of the following conditions are satisfied:\n // a. \"text\" matches \"re\" fully - from the beginning to the end of \"text\".\n // b. The number of matched sub-patterns is >= number of supplied pointers.\n // c. The \"i\"th argument has a suitable type for holding the\n // string captured as the \"i\"th sub-pattern. If you pass in\n // NULL for the \"i\"th argument, or pass fewer arguments than\n // number of sub-patterns, the \"i\"th captured sub-pattern is\n // ignored.\n //\n // CAVEAT: An optional sub-pattern that does not exist in the\n // matched string is assigned the null string. Therefore, the\n // following returns false because the null string - absence of\n // a string (not even the empty string) - is not a valid number:\n //\n // int number;\n // RE2::FullMatch(\"abc\", \"[a-z]+(\\\\d+)?\", &number);\n //\n // Use std::optional instead to handle this case correctly.\n template \n static bool FullMatch(absl::string_view text, const RE2& re, A&&... a) {\n return Apply(FullMatchN, text, re, Arg(std::forward(a))...);\n }\n\n // Like FullMatch(), except that \"re\" is allowed to match a substring\n // of \"text\".\n //\n // Returns true iff all of the following conditions are satisfied:\n // a. \"text\" matches \"re\" partially - for some substring of \"text\".\n // b. The number of matched sub-patterns is >= number of supplied pointers.\n // c. The \"i\"th argument has a suitable type for holding the\n // string captured as the \"i\"th sub-pattern. If you pass in\n // NULL for the \"i\"th argument, or pass fewer arguments than\n // number of sub-patterns, the \"i\"th captured sub-pattern is\n // ignored.\n template \n static bool PartialMatch(absl::string_view text, const RE2& re, A&&... a) {\n return Apply(PartialMatchN, text, re, Arg(std::forward(a))...);\n }\n\n // Like FullMatch() and PartialMatch(), except that \"re\" has to match\n // a prefix of the text, and \"input\" is advanced past the matched\n // text. Note: \"input\" is modified iff this routine returns true\n // and \"re\" matched a non-empty substring of \"input\".\n //\n // Returns true iff all of the following conditions are satisfied:\n // a. \"input\" matches \"re\" partially - for some prefix of \"input\".\n // b. The number of matched sub-patterns is >= number of supplied pointers.\n // c. The \"i\"th argument has a suitable type for holding the\n // string captured as the \"i\"th sub-pattern. If you pass in\n // NULL for the \"i\"th argument, or pass fewer arguments than\n // number of sub-patterns, the \"i\"th captured sub-pattern is\n // ignored.\n template \n static bool Consume(absl::string_view* input, const RE2& re, A&&... a) {\n return Apply(ConsumeN, input, re, Arg(std::forward(a))...);\n }\n\n // Like Consume(), but does not anchor the match at the beginning of\n // the text. That is, \"re\" need not start its match at the beginning\n // of \"input\". For example, \"FindAndConsume(s, \"(\\\\w+)\", &word)\" finds\n // the next word in \"s\" and stores it in \"word\".\n //\n // Returns true iff all of the following conditions are satisfied:\n // a. \"input\" matches \"re\" partially - for some substring of \"input\".\n // b. The number of matched sub-patterns is >= number of supplied pointers.\n // c. The \"i\"th argument has a suitable type for holding the\n // string captured as the \"i\"th sub-pattern. If you pass in\n // NULL for the \"i\"th argument, or pass fewer arguments than\n // number of sub-patterns, the \"i\"th captured sub-pattern is\n // ignored.\n template \n static bool FindAndConsume(absl::string_view* input, const RE2& re, A&&... a) {\n return Apply(FindAndConsumeN, input, re, Arg(std::forward(a))...);\n }\n\n // Replace the first match of \"re\" in \"str\" with \"rewrite\".\n // Within \"rewrite\", backslash-escaped digits (\\1 to \\9) can be\n // used to insert text matching corresponding parenthesized group\n // from the pattern. \\0 in \"rewrite\" refers to the entire matching\n // text. E.g.,\n //\n // std::string s = \"yabba dabba doo\";\n // ABSL_CHECK(RE2::Replace(&s, \"b+\", \"d\"));\n //\n // will leave \"s\" containing \"yada dabba doo\"\n //\n // Returns true if the pattern matches and a replacement occurs,\n // false otherwise.\n static bool Replace(std::string* str,\n const RE2& re,\n absl::string_view rewrite);\n\n // Like Replace(), except replaces successive non-overlapping occurrences\n // of the pattern in the string with the rewrite. E.g.\n //\n // std::string s = \"yabba dabba doo\";\n // ABSL_CHECK(RE2::GlobalReplace(&s, \"b+\", \"d\"));\n //\n // will leave \"s\" containing \"yada dada doo\"\n // Replacements are not subject to re-matching.\n //\n // Because GlobalReplace only replaces non-overlapping matches,\n // replacing \"ana\" within \"banana\" makes only one replacement, not two.\n //\n // Returns the number of replacements made.\n static int GlobalReplace(std::string* str,\n const RE2& re,\n absl::string_view rewrite);\n\n // Like Replace, except that if the pattern matches, \"rewrite\"\n // is copied into \"out\" with substitutions. The non-matching\n // portions of \"text\" are ignored.\n //\n // Returns true iff a match occurred and the extraction happened\n // successfully; if no match occurs, the string is left unaffected.\n //\n // REQUIRES: \"text\" must not alias any part of \"*out\".\n static bool Extract(absl::string_view text,\n const RE2& re,\n absl::string_view rewrite,\n std::string* out);\n\n // Escapes all potentially meaningful regexp characters in\n // 'unquoted'. The returned string, used as a regular expression,\n // will match exactly the original string. For example,\n // 1.5-2.0?\n // may become:\n // 1\\.5\\-2\\.0\\?\n static std::string QuoteMeta(absl::string_view unquoted);\n\n // Computes range for any strings matching regexp. The min and max can in\n // some cases be arbitrarily precise, so the caller gets to specify the\n // maximum desired length of string returned.\n //\n // Assuming PossibleMatchRange(&min, &max, N) returns successfully, any\n // string s that is an anchored match for this regexp satisfies\n // min <= s && s <= max.\n //\n // Note that PossibleMatchRange() will only consider the first copy of an\n // infinitely repeated element (i.e., any regexp element followed by a '*' or\n // '+' operator). Regexps with \"{N}\" constructions are not affected, as those\n // do not compile down to infinite repetitions.\n //\n // Returns true on success, false on error.\n bool PossibleMatchRange(std::string* min, std::string* max,\n int maxlen) const;\n\n // Generic matching interface\n\n // Type of match.\n enum Anchor {\n UNANCHORED, // No anchoring\n ANCHOR_START, // Anchor at start only\n ANCHOR_BOTH // Anchor at start and end\n };\n\n // Return the number of capturing sub-patterns, or -1 if the\n // regexp wasn't valid on construction. The overall match ($0)\n // does not count: if the regexp is \"(a)(b)\", returns 2.\n int NumberOfCapturingGroups() const { return num_captures_; }\n\n // Return a map from names to capturing indices.\n // The map records the index of the leftmost group\n // with the given name.\n // Only valid until the re is deleted.\n const std::map& NamedCapturingGroups() const;\n\n // Return a map from capturing indices to names.\n // The map has no entries for unnamed groups.\n // Only valid until the re is deleted.\n const std::map& CapturingGroupNames() const;\n\n // General matching routine.\n // Match against text starting at offset startpos\n // and stopping the search at offset endpos.\n // Returns true if match found, false if not.\n // On a successful match, fills in submatch[] (up to nsubmatch entries)\n // with information about submatches.\n // I.e. matching RE2(\"(foo)|(bar)baz\") on \"barbazbla\" will return true, with\n // submatch[0] = \"barbaz\", submatch[1].data() = NULL, submatch[2] = \"bar\",\n // submatch[3].data() = NULL, ..., up to submatch[nsubmatch-1].data() = NULL.\n // Caveat: submatch[] may be clobbered even on match failure.\n //\n // Don't ask for more match information than you will use:\n // runs much faster with nsubmatch == 1 than nsubmatch > 1, and\n // runs even faster if nsubmatch == 0.\n // Doesn't make sense to use nsubmatch > 1 + NumberOfCapturingGroups(),\n // but will be handled correctly.\n //\n // Passing text == absl::string_view() will be handled like any other\n // empty string, but note that on return, it will not be possible to tell\n // whether submatch i matched the empty string or did not match:\n // either way, submatch[i].data() == NULL.\n bool Match(absl::string_view text,\n size_t startpos,\n size_t endpos,\n Anchor re_anchor,\n absl::string_view* submatch,\n int nsubmatch) const;\n\n // Check that the given rewrite string is suitable for use with this\n // regular expression. It checks that:\n // * The regular expression has enough parenthesized subexpressions\n // to satisfy all of the \\N tokens in rewrite\n // * The rewrite string doesn't have any syntax errors. E.g.,\n // '\\' followed by anything other than a digit or '\\'.\n // A true return value guarantees that Replace() and Extract() won't\n // fail because of a bad rewrite string.\n bool CheckRewriteString(absl::string_view rewrite,\n std::string* error) const;\n\n // Returns the maximum submatch needed for the rewrite to be done by\n // Replace(). E.g. if rewrite == \"foo \\\\2,\\\\1\", returns 2.\n static int MaxSubmatch(absl::string_view rewrite);\n\n // Append the \"rewrite\" string, with backslash substitutions from \"vec\",\n // to string \"out\".\n // Returns true on success. This method can fail because of a malformed\n // rewrite string. CheckRewriteString guarantees that the rewrite will\n // be sucessful.\n bool Rewrite(std::string* out,\n absl::string_view rewrite,\n const absl::string_view* vec,\n int veclen) const;\n\n // Constructor options\n class Options {\n public:\n // The options are (defaults in parentheses):\n //\n // utf8 (true) text and pattern are UTF-8; otherwise Latin-1\n // posix_syntax (false) restrict regexps to POSIX egrep syntax\n // longest_match (false) search for longest match, not first match\n // log_errors (true) log syntax and execution errors to ERROR\n // max_mem (see below) approx. max memory footprint of RE2\n // literal (false) interpret string as literal, not regexp\n // never_nl (false) never match \\n, even if it is in regexp\n // dot_nl (false) dot matches everything including new line\n // never_capture (false) parse all parens as non-capturing\n // case_sensitive (true) match is case-sensitive (regexp can override\n // with (?i) unless in posix_syntax mode)\n //\n // The following options are only consulted when posix_syntax == true.\n // When posix_syntax == false, these features are always enabled and\n // cannot be turned off; to perform multi-line matching in that case,\n // begin the regexp with (?m).\n // perl_classes (false) allow Perl's \\d \\s \\w \\D \\S \\W\n // word_boundary (false) allow Perl's \\b \\B (word boundary and not)\n // one_line (false) ^ and $ only match beginning and end of text\n //\n // The max_mem option controls how much memory can be used\n // to hold the compiled form of the regexp (the Prog) and\n // its cached DFA graphs. Code Search placed limits on the number\n // of Prog instructions and DFA states: 10,000 for both.\n // In RE2, those limits would translate to about 240 KB per Prog\n // and perhaps 2.5 MB per DFA (DFA state sizes vary by regexp; RE2 does a\n // better job of keeping them small than Code Search did).\n // Each RE2 has two Progs (one forward, one reverse), and each Prog\n // can have two DFAs (one first match, one longest match).\n // That makes 4 DFAs:\n //\n // forward, first-match - used for UNANCHORED or ANCHOR_START searches\n // if opt.longest_match() == false\n // forward, longest-match - used for all ANCHOR_BOTH searches,\n // and the other two kinds if\n // opt.longest_match() == true\n // reverse, first-match - never used\n // reverse, longest-match - used as second phase for unanchored searches\n //\n // The RE2 memory budget is statically divided between the two\n // Progs and then the DFAs: two thirds to the forward Prog\n // and one third to the reverse Prog. The forward Prog gives half\n // of what it has left over to each of its DFAs. The reverse Prog\n // gives it all to its longest-match DFA.\n //\n // Once a DFA fills its budget, it flushes its cache and starts over.\n // If this happens too often, RE2 falls back on the NFA implementation.\n\n // For now, make the default budget something close to Code Search.\n static const int kDefaultMaxMem = 8<<20;\n\n enum Encoding {\n EncodingUTF8 = 1,\n EncodingLatin1\n };\n\n Options() :\n max_mem_(kDefaultMaxMem),\n encoding_(EncodingUTF8),\n posix_syntax_(false),\n longest_match_(false),\n log_errors_(true),\n literal_(false),\n never_nl_(false),\n dot_nl_(false),\n never_capture_(false),\n case_sensitive_(true),\n perl_classes_(false),\n word_boundary_(false),\n one_line_(false) {\n }\n\n /*implicit*/ Options(CannedOptions);\n\n int64_t max_mem() const { return max_mem_; }\n void set_max_mem(int64_t m) { max_mem_ = m; }\n\n Encoding encoding() const { return encoding_; }\n void set_encoding(Encoding encoding) { encoding_ = encoding; }\n\n bool posix_syntax() const { return posix_syntax_; }\n void set_posix_syntax(bool b) { posix_syntax_ = b; }\n\n bool longest_match() const { return longest_match_; }\n void set_longest_match(bool b) { longest_match_ = b; }\n\n bool log_errors() const { return log_errors_; }\n void set_log_errors(bool b) { log_errors_ = b; }\n\n bool literal() const { return literal_; }\n void set_literal(bool b) { literal_ = b; }\n\n bool never_nl() const { return never_nl_; }\n void set_never_nl(bool b) { never_nl_ = b; }\n\n bool dot_nl() const { return dot_nl_; }\n void set_dot_nl(bool b) { dot_nl_ = b; }\n\n bool never_capture() const { return never_capture_; }\n void set_never_capture(bool b) { never_capture_ = b; }\n\n bool case_sensitive() const { return case_sensitive_; }\n void set_case_sensitive(bool b) { case_sensitive_ = b; }\n\n bool perl_classes() const { return perl_classes_; }\n void set_perl_classes(bool b) { perl_classes_ = b; }\n\n bool word_boundary() const { return word_boundary_; }\n void set_word_boundary(bool b) { word_boundary_ = b; }\n\n bool one_line() const { return one_line_; }\n void set_one_line(bool b) { one_line_ = b; }\n\n void Copy(const Options& src) {\n *this = src;\n }\n\n int ParseFlags() const;\n\n private:\n int64_t max_mem_;\n Encoding encoding_;\n bool posix_syntax_;\n bool longest_match_;\n bool log_errors_;\n bool literal_;\n bool never_nl_;\n bool dot_nl_;\n bool never_capture_;\n bool case_sensitive_;\n bool perl_classes_;\n bool word_boundary_;\n bool one_line_;\n };\n\n // Returns the options set in the constructor.\n const Options& options() const { return options_; }\n\n // Argument converters; see below.\n template \n static Arg CRadix(T* ptr);\n template \n static Arg Hex(T* ptr);\n template \n static Arg Octal(T* ptr);\n\n // Controls the maximum count permitted by GlobalReplace(); -1 is unlimited.\n // FOR FUZZING ONLY.\n static void FUZZING_ONLY_set_maximum_global_replace_count(int i);\n\n private:\n void Init(absl::string_view pattern, const Options& options);\n\n bool DoMatch(absl::string_view text,\n Anchor re_anchor,\n size_t* consumed,\n const Arg* const args[],\n int n) const;\n\n re2::Prog* ReverseProg() const;\n\n // First cache line is relatively cold fields.\n const std::string* pattern_; // string regular expression\n Options options_; // option flags\n re2::Regexp* entire_regexp_; // parsed regular expression\n re2::Regexp* suffix_regexp_; // parsed regular expression, prefix_ removed\n const std::string* error_; // error indicator (or points to empty string)\n const std::string* error_arg_; // fragment of regexp showing error (or ditto)\n\n // Second cache line is relatively hot fields.\n // These are ordered oddly to pack everything.\n int num_captures_; // number of capturing groups\n ErrorCode error_code_ : 29; // error code (29 bits is more than enough)\n bool longest_match_ : 1; // cached copy of options_.longest_match()\n bool is_one_pass_ : 1; // can use prog_->SearchOnePass?\n bool prefix_foldcase_ : 1; // prefix_ is ASCII case-insensitive\n std::string prefix_; // required prefix (before suffix_regexp_)\n re2::Prog* prog_; // compiled program for regexp\n\n // Reverse Prog for DFA execution only\n mutable re2::Prog* rprog_;\n // Map from capture names to indices\n mutable const std::map* named_groups_;\n // Map from capture indices to names\n mutable const std::map* group_names_;\n\n mutable absl::once_flag rprog_once_;\n mutable absl::once_flag named_groups_once_;\n mutable absl::once_flag group_names_once_;\n};\n\n/***** Implementation details *****/\n\nnamespace re2_internal {\n\n// Types for which the 3-ary Parse() function template has specializations.\ntemplate struct Parse3ary : public std::false_type {};\ntemplate <> struct Parse3ary : public std::true_type {};\ntemplate <> struct Parse3ary : public std::true_type {};\ntemplate <> struct Parse3ary : public std::true_type {};\ntemplate <> struct Parse3ary : public std::true_type {};\ntemplate <> struct Parse3ary : public std::true_type {};\ntemplate <> struct Parse3ary : public std::true_type {};\ntemplate <> struct Parse3ary : public std::true_type {};\ntemplate <> struct Parse3ary : public std::true_type {};\n\ntemplate \nbool Parse(const char* str, size_t n, T* dest);\n\n// Types for which the 4-ary Parse() function template has specializations.\ntemplate struct Parse4ary : public std::false_type {};\ntemplate <> struct Parse4ary : public std::true_type {};\ntemplate <> struct Parse4ary : public std::true_type {};\ntemplate <> struct Parse4ary : public std::true_type {};\ntemplate <> struct Parse4ary : public std::true_type {};\ntemplate <> struct Parse4ary : public std::true_type {};\ntemplate <> struct Parse4ary : public std::true_type {};\ntemplate <> struct Parse4ary : public std::true_type {};\ntemplate <> struct Parse4ary : public std::true_type {};\n\ntemplate \nbool Parse(const char* str, size_t n, T* dest, int radix);\n\n// Support std::optional for all T with a stock parser.\ntemplate struct Parse3ary> : public Parse3ary {};\ntemplate struct Parse4ary> : public Parse4ary {};\n\ntemplate \nbool Parse(const char* str, size_t n, std::optional* dest) {\n if (str == NULL) {\n if (dest != NULL)\n dest->reset();\n return true;\n }\n T tmp;\n if (Parse(str, n, &tmp)) {\n if (dest != NULL)\n dest->emplace(std::move(tmp));\n return true;\n }\n return false;\n}\n\ntemplate \nbool Parse(const char* str, size_t n, std::optional* dest, int radix) {\n if (str == NULL) {\n if (dest != NULL)\n dest->reset();\n return true;\n }\n T tmp;\n if (Parse(str, n, &tmp, radix)) {\n if (dest != NULL)\n dest->emplace(std::move(tmp));\n return true;\n }\n return false;\n}\n\n} // namespace re2_internal\n\nclass RE2::Arg {\n private:\n template \n using CanParse3ary = typename std::enable_if<\n re2_internal::Parse3ary::value,\n int>::type;\n\n template \n using CanParse4ary = typename std::enable_if<\n re2_internal::Parse4ary::value,\n int>::type;\n\n template \n using CanParseFrom = typename std::enable_if<\n std::is_member_function_pointer<\n decltype(static_cast(\n &T::ParseFrom))>::value,\n int>::type;\n\n public:\n Arg() : Arg(nullptr) {}\n Arg(std::nullptr_t ptr) : arg_(ptr), parser_(DoNothing) {}\n\n template = 0>\n Arg(T* ptr) : arg_(ptr), parser_(DoParse3ary) {}\n\n template = 0>\n Arg(T* ptr) : arg_(ptr), parser_(DoParse4ary) {}\n\n template = 0>\n Arg(T* ptr) : arg_(ptr), parser_(DoParseFrom) {}\n\n typedef bool (*Parser)(const char* str, size_t n, void* dest);\n\n template \n Arg(T* ptr, Parser parser) : arg_(ptr), parser_(parser) {}\n\n bool Parse(const char* str, size_t n) const {\n return (*parser_)(str, n, arg_);\n }\n\n private:\n static bool DoNothing(const char* /*str*/, size_t /*n*/, void* /*dest*/) {\n return true;\n }\n\n template \n static bool DoParse3ary(const char* str, size_t n, void* dest) {\n return re2_internal::Parse(str, n, reinterpret_cast(dest));\n }\n\n template \n static bool DoParse4ary(const char* str, size_t n, void* dest) {\n return re2_internal::Parse(str, n, reinterpret_cast(dest), 10);\n }\n\n template \n static bool DoParseFrom(const char* str, size_t n, void* dest) {\n if (dest == NULL) return true;\n return reinterpret_cast(dest)->ParseFrom(str, n);\n }\n\n void* arg_;\n Parser parser_;\n};\n\ntemplate \ninline RE2::Arg RE2::CRadix(T* ptr) {\n return RE2::Arg(ptr, [](const char* str, size_t n, void* dest) -> bool {\n return re2_internal::Parse(str, n, reinterpret_cast(dest), 0);\n });\n}\n\ntemplate \ninline RE2::Arg RE2::Hex(T* ptr) {\n return RE2::Arg(ptr, [](const char* str, size_t n, void* dest) -> bool {\n return re2_internal::Parse(str, n, reinterpret_cast(dest), 16);\n });\n}\n\ntemplate \ninline RE2::Arg RE2::Octal(T* ptr) {\n return RE2::Arg(ptr, [](const char* str, size_t n, void* dest) -> bool {\n return re2_internal::Parse(str, n, reinterpret_cast(dest), 8);\n });\n}\n\n// Silence warnings about missing initializers for members of LazyRE2.\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Wmissing-field-initializers\"\n#endif\n\n// Helper for writing global or static RE2s safely.\n// Write\n// static LazyRE2 re = {\".*\"};\n// and then use *re instead of writing\n// static RE2 re(\".*\");\n// The former is more careful about multithreaded\n// situations than the latter.\n//\n// N.B. This class never deletes the RE2 object that\n// it constructs: that's a feature, so that it can be used\n// for global and function static variables.\nclass LazyRE2 {\n private:\n struct NoArg {};\n\n public:\n typedef RE2 element_type; // support std::pointer_traits\n\n // Constructor omitted to preserve braced initialization in C++98.\n\n // Pretend to be a pointer to Type (never NULL due to on-demand creation):\n RE2& operator*() const { return *get(); }\n RE2* operator->() const { return get(); }\n\n // Named accessor/initializer:\n RE2* get() const {\n absl::call_once(once_, &LazyRE2::Init, this);\n return ptr_;\n }\n\n // All data fields must be public to support {\"foo\"} initialization.\n const char* pattern_;\n RE2::CannedOptions options_;\n NoArg barrier_against_excess_initializers_;\n\n mutable RE2* ptr_;\n mutable absl::once_flag once_;\n\n private:\n static void Init(const LazyRE2* lazy_re2) {\n lazy_re2->ptr_ = new RE2(lazy_re2->pattern_, lazy_re2->options_);\n }\n\n void operator=(const LazyRE2&); // disallowed\n};\n\nnamespace hooks {\n\n// Most platforms support thread_local. Older versions of iOS don't support\n// thread_local, but for the sake of brevity, we lump together all versions\n// of Apple platforms that aren't macOS. If an iOS application really needs\n// the context pointee someday, we can get more specific then...\n//\n// As per https://github.com/google/re2/issues/325, thread_local support in\n// MinGW seems to be buggy. (FWIW, Abseil folks also avoid it.)\n#define RE2_HAVE_THREAD_LOCAL\n#if (defined(__APPLE__) && !(defined(TARGET_OS_OSX) && TARGET_OS_OSX)) || defined(__MINGW32__)\n#undef RE2_HAVE_THREAD_LOCAL\n#endif\n\n// A hook must not make any assumptions regarding the lifetime of the context\n// pointee beyond the current invocation of the hook. Pointers and references\n// obtained via the context pointee should be considered invalidated when the\n// hook returns. Hence, any data about the context pointee (e.g. its pattern)\n// would have to be copied in order for it to be kept for an indefinite time.\n//\n// A hook must not use RE2 for matching. Control flow reentering RE2::Match()\n// could result in infinite mutual recursion. To discourage that possibility,\n// RE2 will not maintain the context pointer correctly when used in that way.\n#ifdef RE2_HAVE_THREAD_LOCAL\nextern thread_local const RE2* context;\n#endif\n\nstruct DFAStateCacheReset {\n int64_t state_budget;\n size_t state_cache_size;\n};\n\nstruct DFASearchFailure {\n // Nothing yet...\n};\n\n#define DECLARE_HOOK(type) \\\n using type##Callback = void(const type&); \\\n void Set##type##Hook(type##Callback* cb); \\\n type##Callback* Get##type##Hook();\n\nDECLARE_HOOK(DFAStateCacheReset)\nDECLARE_HOOK(DFASearchFailure)\n\n#undef DECLARE_HOOK\n\n} // namespace hooks\n\n} // namespace re2\n\nusing re2::RE2;\nusing re2::LazyRE2;\n\n#endif // RE2_RE2_H_\n" }, { "path": "re2/regexp.cc", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Regular expression representation.\n// Tested by parse_test.cc\n\n#include \"re2/regexp.h\"\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\n#include \"absl/base/call_once.h\"\n#include \"absl/base/macros.h\"\n#include \"absl/container/flat_hash_map.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/synchronization/mutex.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/walker-inl.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\n// Constructor. Allocates vectors as appropriate for operator.\nRegexp::Regexp(RegexpOp op, ParseFlags parse_flags)\n : op_(static_cast(op)),\n simple_(false),\n parse_flags_(static_cast(parse_flags)),\n ref_(1),\n nsub_(0),\n down_(NULL) {\n subone_ = NULL;\n memset(the_union_, 0, sizeof the_union_);\n}\n\n// Destructor. Assumes already cleaned up children.\n// Private: use Decref() instead of delete to destroy Regexps.\n// Can't call Decref on the sub-Regexps here because\n// that could cause arbitrarily deep recursion, so\n// required Decref() to have handled them for us.\nRegexp::~Regexp() {\n if (nsub_ > 0)\n ABSL_LOG(DFATAL) << \"Regexp not destroyed.\";\n\n switch (op_) {\n default:\n break;\n case kRegexpCapture:\n delete name_;\n break;\n case kRegexpLiteralString:\n delete[] runes_;\n break;\n case kRegexpCharClass:\n if (cc_)\n cc_->Delete();\n delete ccb_;\n break;\n }\n}\n\n// If it's possible to destroy this regexp without recurring,\n// do so and return true. Else return false.\nbool Regexp::QuickDestroy() {\n if (nsub_ == 0) {\n delete this;\n return true;\n }\n return false;\n}\n\n// Similar to EmptyStorage in re2.cc.\nstruct RefStorage {\n absl::Mutex ref_mutex;\n absl::flat_hash_map ref_map;\n};\nalignas(RefStorage) static char ref_storage[sizeof(RefStorage)];\n\nstatic inline absl::Mutex* ref_mutex() {\n return &reinterpret_cast(ref_storage)->ref_mutex;\n}\n\nstatic inline absl::flat_hash_map* ref_map() {\n return &reinterpret_cast(ref_storage)->ref_map;\n}\n\nint Regexp::Ref() {\n if (ref_ < kMaxRef)\n return ref_;\n\n absl::MutexLock l(ref_mutex());\n return (*ref_map())[this];\n}\n\n// Increments reference count, returns object as convenience.\nRegexp* Regexp::Incref() {\n if (ref_ >= kMaxRef-1) {\n static absl::once_flag ref_once;\n absl::call_once(ref_once, []() {\n (void) new (ref_storage) RefStorage;\n });\n\n // Store ref count in overflow map.\n absl::MutexLock l(ref_mutex());\n if (ref_ == kMaxRef) {\n // already overflowed\n (*ref_map())[this]++;\n } else {\n // overflowing now\n (*ref_map())[this] = kMaxRef;\n ref_ = kMaxRef;\n }\n return this;\n }\n\n ref_++;\n return this;\n}\n\n// Decrements reference count and deletes this object if count reaches 0.\nvoid Regexp::Decref() {\n if (ref_ == kMaxRef) {\n // Ref count is stored in overflow map.\n absl::MutexLock l(ref_mutex());\n int r = (*ref_map())[this] - 1;\n if (r < kMaxRef) {\n ref_ = static_cast(r);\n ref_map()->erase(this);\n } else {\n (*ref_map())[this] = r;\n }\n return;\n }\n ref_--;\n if (ref_ == 0)\n Destroy();\n}\n\n// Deletes this object; ref count has count reached 0.\nvoid Regexp::Destroy() {\n if (QuickDestroy())\n return;\n\n // Handle recursive Destroy with explicit stack\n // to avoid arbitrarily deep recursion on process stack [sigh].\n down_ = NULL;\n Regexp* stack = this;\n while (stack != NULL) {\n Regexp* re = stack;\n stack = re->down_;\n if (re->ref_ != 0)\n ABSL_LOG(DFATAL) << \"Bad reference count \" << re->ref_;\n if (re->nsub_ > 0) {\n Regexp** subs = re->sub();\n for (int i = 0; i < re->nsub_; i++) {\n Regexp* sub = subs[i];\n if (sub == NULL)\n continue;\n if (sub->ref_ == kMaxRef)\n sub->Decref();\n else\n --sub->ref_;\n if (sub->ref_ == 0 && !sub->QuickDestroy()) {\n sub->down_ = stack;\n stack = sub;\n }\n }\n if (re->nsub_ > 1)\n delete[] subs;\n re->nsub_ = 0;\n }\n delete re;\n }\n}\n\nvoid Regexp::AddRuneToString(Rune r) {\n ABSL_DCHECK(op_ == kRegexpLiteralString);\n if (nrunes_ == 0) {\n // start with 8\n runes_ = new Rune[8];\n } else if (nrunes_ >= 8 && (nrunes_ & (nrunes_ - 1)) == 0) {\n // double on powers of two\n Rune *old = runes_;\n runes_ = new Rune[nrunes_ * 2];\n for (int i = 0; i < nrunes_; i++)\n runes_[i] = old[i];\n delete[] old;\n }\n\n runes_[nrunes_++] = r;\n}\n\nRegexp* Regexp::HaveMatch(int match_id, ParseFlags flags) {\n Regexp* re = new Regexp(kRegexpHaveMatch, flags);\n re->match_id_ = match_id;\n return re;\n}\n\nRegexp* Regexp::StarPlusOrQuest(RegexpOp op, Regexp* sub, ParseFlags flags) {\n // Squash **, ++ and ??.\n if (op == sub->op() && flags == sub->parse_flags())\n return sub;\n\n // Squash *+, *?, +*, +?, ?* and ?+. They all squash to *, so because\n // op is Star/Plus/Quest, we just have to check that sub->op() is too.\n if ((sub->op() == kRegexpStar ||\n sub->op() == kRegexpPlus ||\n sub->op() == kRegexpQuest) &&\n flags == sub->parse_flags()) {\n // If sub is Star, no need to rewrite it.\n if (sub->op() == kRegexpStar)\n return sub;\n\n // Rewrite sub to Star.\n Regexp* re = new Regexp(kRegexpStar, flags);\n re->AllocSub(1);\n re->sub()[0] = sub->sub()[0]->Incref();\n sub->Decref(); // We didn't consume the reference after all.\n return re;\n }\n\n Regexp* re = new Regexp(op, flags);\n re->AllocSub(1);\n re->sub()[0] = sub;\n return re;\n}\n\nRegexp* Regexp::Plus(Regexp* sub, ParseFlags flags) {\n return StarPlusOrQuest(kRegexpPlus, sub, flags);\n}\n\nRegexp* Regexp::Star(Regexp* sub, ParseFlags flags) {\n return StarPlusOrQuest(kRegexpStar, sub, flags);\n}\n\nRegexp* Regexp::Quest(Regexp* sub, ParseFlags flags) {\n return StarPlusOrQuest(kRegexpQuest, sub, flags);\n}\n\nRegexp* Regexp::ConcatOrAlternate(RegexpOp op, Regexp** sub, int nsub,\n ParseFlags flags, bool can_factor) {\n if (nsub == 1)\n return sub[0];\n\n if (nsub == 0) {\n if (op == kRegexpAlternate)\n return new Regexp(kRegexpNoMatch, flags);\n else\n return new Regexp(kRegexpEmptyMatch, flags);\n }\n\n PODArray subcopy;\n if (op == kRegexpAlternate && can_factor) {\n // Going to edit sub; make a copy so we don't step on caller.\n subcopy = PODArray(nsub);\n memmove(subcopy.data(), sub, nsub * sizeof sub[0]);\n sub = subcopy.data();\n nsub = FactorAlternation(sub, nsub, flags);\n if (nsub == 1) {\n Regexp* re = sub[0];\n return re;\n }\n }\n\n if (nsub > kMaxNsub) {\n // Too many subexpressions to fit in a single Regexp.\n // Make a two-level tree. Two levels gets us to 65535^2.\n int nbigsub = (nsub+kMaxNsub-1)/kMaxNsub;\n Regexp* re = new Regexp(op, flags);\n re->AllocSub(nbigsub);\n Regexp** subs = re->sub();\n for (int i = 0; i < nbigsub - 1; i++)\n subs[i] = ConcatOrAlternate(op, sub+i*kMaxNsub, kMaxNsub, flags, false);\n subs[nbigsub - 1] = ConcatOrAlternate(op, sub+(nbigsub-1)*kMaxNsub,\n nsub - (nbigsub-1)*kMaxNsub, flags,\n false);\n return re;\n }\n\n Regexp* re = new Regexp(op, flags);\n re->AllocSub(nsub);\n Regexp** subs = re->sub();\n for (int i = 0; i < nsub; i++)\n subs[i] = sub[i];\n return re;\n}\n\nRegexp* Regexp::Concat(Regexp** sub, int nsub, ParseFlags flags) {\n return ConcatOrAlternate(kRegexpConcat, sub, nsub, flags, false);\n}\n\nRegexp* Regexp::Alternate(Regexp** sub, int nsub, ParseFlags flags) {\n return ConcatOrAlternate(kRegexpAlternate, sub, nsub, flags, true);\n}\n\nRegexp* Regexp::AlternateNoFactor(Regexp** sub, int nsub, ParseFlags flags) {\n return ConcatOrAlternate(kRegexpAlternate, sub, nsub, flags, false);\n}\n\nRegexp* Regexp::Capture(Regexp* sub, ParseFlags flags, int cap) {\n Regexp* re = new Regexp(kRegexpCapture, flags);\n re->AllocSub(1);\n re->sub()[0] = sub;\n re->cap_ = cap;\n return re;\n}\n\nRegexp* Regexp::Repeat(Regexp* sub, ParseFlags flags, int min, int max) {\n Regexp* re = new Regexp(kRegexpRepeat, flags);\n re->AllocSub(1);\n re->sub()[0] = sub;\n re->min_ = min;\n re->max_ = max;\n return re;\n}\n\nRegexp* Regexp::NewLiteral(Rune rune, ParseFlags flags) {\n Regexp* re = new Regexp(kRegexpLiteral, flags);\n re->rune_ = rune;\n return re;\n}\n\nRegexp* Regexp::LiteralString(Rune* runes, int nrunes, ParseFlags flags) {\n if (nrunes <= 0)\n return new Regexp(kRegexpEmptyMatch, flags);\n if (nrunes == 1)\n return NewLiteral(runes[0], flags);\n Regexp* re = new Regexp(kRegexpLiteralString, flags);\n for (int i = 0; i < nrunes; i++)\n re->AddRuneToString(runes[i]);\n return re;\n}\n\nRegexp* Regexp::NewCharClass(CharClass* cc, ParseFlags flags) {\n Regexp* re = new Regexp(kRegexpCharClass, flags);\n re->cc_ = cc;\n return re;\n}\n\nvoid Regexp::Swap(Regexp* that) {\n // Regexp is not trivially copyable, so we cannot freely copy it with\n // memmove(3), but swapping objects like so is safe for our purposes.\n char tmp[sizeof *this];\n void* vthis = reinterpret_cast(this);\n void* vthat = reinterpret_cast(that);\n memmove(tmp, vthis, sizeof *this);\n memmove(vthis, vthat, sizeof *this);\n memmove(vthat, tmp, sizeof *this);\n}\n\n// Tests equality of all top-level structure but not subregexps.\nstatic bool TopEqual(Regexp* a, Regexp* b) {\n if (a->op() != b->op())\n return false;\n\n switch (a->op()) {\n case kRegexpNoMatch:\n case kRegexpEmptyMatch:\n case kRegexpAnyChar:\n case kRegexpAnyByte:\n case kRegexpBeginLine:\n case kRegexpEndLine:\n case kRegexpWordBoundary:\n case kRegexpNoWordBoundary:\n case kRegexpBeginText:\n return true;\n\n case kRegexpEndText:\n // The parse flags remember whether it's \\z or (?-m:$),\n // which matters when testing against PCRE.\n return ((a->parse_flags() ^ b->parse_flags()) & Regexp::WasDollar) == 0;\n\n case kRegexpLiteral:\n return a->rune() == b->rune() &&\n ((a->parse_flags() ^ b->parse_flags()) & Regexp::FoldCase) == 0;\n\n case kRegexpLiteralString:\n return a->nrunes() == b->nrunes() &&\n ((a->parse_flags() ^ b->parse_flags()) & Regexp::FoldCase) == 0 &&\n memcmp(a->runes(), b->runes(),\n a->nrunes() * sizeof a->runes()[0]) == 0;\n\n case kRegexpAlternate:\n case kRegexpConcat:\n return a->nsub() == b->nsub();\n\n case kRegexpStar:\n case kRegexpPlus:\n case kRegexpQuest:\n return ((a->parse_flags() ^ b->parse_flags()) & Regexp::NonGreedy) == 0;\n\n case kRegexpRepeat:\n return ((a->parse_flags() ^ b->parse_flags()) & Regexp::NonGreedy) == 0 &&\n a->min() == b->min() &&\n a->max() == b->max();\n\n case kRegexpCapture:\n if (a->name() == NULL || b->name() == NULL) {\n // One pointer is null, so the other pointer should also be null.\n return a->cap() == b->cap() && a->name() == b->name();\n } else {\n // Neither pointer is null, so compare the pointees for equality.\n return a->cap() == b->cap() && *a->name() == *b->name();\n }\n\n case kRegexpHaveMatch:\n return a->match_id() == b->match_id();\n\n case kRegexpCharClass: {\n CharClass* acc = a->cc();\n CharClass* bcc = b->cc();\n return acc->size() == bcc->size() &&\n acc->end() - acc->begin() == bcc->end() - bcc->begin() &&\n memcmp(acc->begin(), bcc->begin(),\n (acc->end() - acc->begin()) * sizeof acc->begin()[0]) == 0;\n }\n }\n\n ABSL_LOG(DFATAL) << \"Unexpected op in Regexp::Equal: \" << a->op();\n return 0;\n}\n\nbool Regexp::Equal(Regexp* a, Regexp* b) {\n if (a == NULL || b == NULL)\n return a == b;\n\n if (!TopEqual(a, b))\n return false;\n\n // Fast path:\n // return without allocating vector if there are no subregexps.\n switch (a->op()) {\n case kRegexpAlternate:\n case kRegexpConcat:\n case kRegexpStar:\n case kRegexpPlus:\n case kRegexpQuest:\n case kRegexpRepeat:\n case kRegexpCapture:\n break;\n\n default:\n return true;\n }\n\n // Committed to doing real work.\n // The stack (vector) has pairs of regexps waiting to\n // be compared. The regexps are only equal if\n // all the pairs end up being equal.\n std::vector stk;\n\n for (;;) {\n // Invariant: TopEqual(a, b) == true.\n Regexp* a2;\n Regexp* b2;\n switch (a->op()) {\n default:\n break;\n case kRegexpAlternate:\n case kRegexpConcat:\n for (int i = 0; i < a->nsub(); i++) {\n a2 = a->sub()[i];\n b2 = b->sub()[i];\n if (!TopEqual(a2, b2))\n return false;\n stk.push_back(a2);\n stk.push_back(b2);\n }\n break;\n\n case kRegexpStar:\n case kRegexpPlus:\n case kRegexpQuest:\n case kRegexpRepeat:\n case kRegexpCapture:\n a2 = a->sub()[0];\n b2 = b->sub()[0];\n if (!TopEqual(a2, b2))\n return false;\n // Really:\n // stk.push_back(a2);\n // stk.push_back(b2);\n // break;\n // but faster to assign directly and loop.\n a = a2;\n b = b2;\n continue;\n }\n\n size_t n = stk.size();\n if (n == 0)\n break;\n\n ABSL_DCHECK_GE(n, size_t{2});\n a = stk[n-2];\n b = stk[n-1];\n stk.resize(n-2);\n }\n\n return true;\n}\n\n// Keep in sync with enum RegexpStatusCode in regexp.h\nstatic const char *kErrorStrings[] = {\n \"no error\",\n \"unexpected error\",\n \"invalid escape sequence\",\n \"invalid character class\",\n \"invalid character class range\",\n \"missing ]\",\n \"missing )\",\n \"unexpected )\",\n \"trailing \\\\\",\n \"no argument for repetition operator\",\n \"invalid repetition size\",\n \"bad repetition operator\",\n \"invalid perl operator\",\n \"invalid UTF-8\",\n \"invalid named capture group\",\n};\n\nstd::string RegexpStatus::CodeText(enum RegexpStatusCode code) {\n if (code < 0 || code >= ABSL_ARRAYSIZE(kErrorStrings))\n code = kRegexpInternalError;\n return kErrorStrings[code];\n}\n\nstd::string RegexpStatus::Text() const {\n if (error_arg_.empty())\n return CodeText(code_);\n std::string s;\n s.append(CodeText(code_));\n s.append(\": \");\n s.append(error_arg_.data(), error_arg_.size());\n return s;\n}\n\nvoid RegexpStatus::Copy(const RegexpStatus& status) {\n code_ = status.code_;\n error_arg_ = status.error_arg_;\n}\n\ntypedef int Ignored; // Walker doesn't exist\n\n// Walker subclass to count capturing parens in regexp.\nclass NumCapturesWalker : public Regexp::Walker {\n public:\n NumCapturesWalker() : ncapture_(0) {}\n int ncapture() { return ncapture_; }\n\n virtual Ignored PreVisit(Regexp* re, Ignored ignored, bool* stop) {\n if (re->op() == kRegexpCapture)\n ncapture_++;\n return ignored;\n }\n\n virtual Ignored ShortVisit(Regexp* re, Ignored ignored) {\n // Should never be called: we use Walk(), not WalkExponential().\n#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION\n ABSL_LOG(DFATAL) << \"NumCapturesWalker::ShortVisit called\";\n#endif\n return ignored;\n }\n\n private:\n int ncapture_;\n\n NumCapturesWalker(const NumCapturesWalker&) = delete;\n NumCapturesWalker& operator=(const NumCapturesWalker&) = delete;\n};\n\nint Regexp::NumCaptures() {\n NumCapturesWalker w;\n w.Walk(this, 0);\n return w.ncapture();\n}\n\n// Walker class to build map of named capture groups and their indices.\nclass NamedCapturesWalker : public Regexp::Walker {\n public:\n NamedCapturesWalker() : map_(NULL) {}\n ~NamedCapturesWalker() { delete map_; }\n\n std::map* TakeMap() {\n std::map* m = map_;\n map_ = NULL;\n return m;\n }\n\n virtual Ignored PreVisit(Regexp* re, Ignored ignored, bool* stop) {\n if (re->op() == kRegexpCapture && re->name() != NULL) {\n // Allocate map once we find a name.\n if (map_ == NULL)\n map_ = new std::map;\n\n // Record first occurrence of each name.\n // (The rule is that if you have the same name\n // multiple times, only the leftmost one counts.)\n map_->insert({*re->name(), re->cap()});\n }\n return ignored;\n }\n\n virtual Ignored ShortVisit(Regexp* re, Ignored ignored) {\n // Should never be called: we use Walk(), not WalkExponential().\n#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION\n ABSL_LOG(DFATAL) << \"NamedCapturesWalker::ShortVisit called\";\n#endif\n return ignored;\n }\n\n private:\n std::map* map_;\n\n NamedCapturesWalker(const NamedCapturesWalker&) = delete;\n NamedCapturesWalker& operator=(const NamedCapturesWalker&) = delete;\n};\n\nstd::map* Regexp::NamedCaptures() {\n NamedCapturesWalker w;\n w.Walk(this, 0);\n return w.TakeMap();\n}\n\n// Walker class to build map from capture group indices to their names.\nclass CaptureNamesWalker : public Regexp::Walker {\n public:\n CaptureNamesWalker() : map_(NULL) {}\n ~CaptureNamesWalker() { delete map_; }\n\n std::map* TakeMap() {\n std::map* m = map_;\n map_ = NULL;\n return m;\n }\n\n virtual Ignored PreVisit(Regexp* re, Ignored ignored, bool* stop) {\n if (re->op() == kRegexpCapture && re->name() != NULL) {\n // Allocate map once we find a name.\n if (map_ == NULL)\n map_ = new std::map;\n\n (*map_)[re->cap()] = *re->name();\n }\n return ignored;\n }\n\n virtual Ignored ShortVisit(Regexp* re, Ignored ignored) {\n // Should never be called: we use Walk(), not WalkExponential().\n#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION\n ABSL_LOG(DFATAL) << \"CaptureNamesWalker::ShortVisit called\";\n#endif\n return ignored;\n }\n\n private:\n std::map* map_;\n\n CaptureNamesWalker(const CaptureNamesWalker&) = delete;\n CaptureNamesWalker& operator=(const CaptureNamesWalker&) = delete;\n};\n\nstd::map* Regexp::CaptureNames() {\n CaptureNamesWalker w;\n w.Walk(this, 0);\n return w.TakeMap();\n}\n\nvoid ConvertRunesToBytes(bool latin1, Rune* runes, int nrunes,\n std::string* bytes) {\n if (latin1) {\n bytes->resize(nrunes);\n for (int i = 0; i < nrunes; i++)\n (*bytes)[i] = static_cast(runes[i]);\n } else {\n bytes->resize(nrunes * UTFmax); // worst case\n char* p = &(*bytes)[0];\n for (int i = 0; i < nrunes; i++)\n p += runetochar(p, &runes[i]);\n bytes->resize(p - &(*bytes)[0]);\n bytes->shrink_to_fit();\n }\n}\n\n// Determines whether regexp matches must be anchored\n// with a fixed string prefix. If so, returns the prefix and\n// the regexp that remains after the prefix. The prefix might\n// be ASCII case-insensitive.\nbool Regexp::RequiredPrefix(std::string* prefix, bool* foldcase,\n Regexp** suffix) {\n prefix->clear();\n *foldcase = false;\n *suffix = NULL;\n\n // No need for a walker: the regexp must be of the form\n // 1. some number of ^ anchors\n // 2. a literal char or string\n // 3. the rest\n if (op_ != kRegexpConcat)\n return false;\n int i = 0;\n while (i < nsub_ && sub()[i]->op_ == kRegexpBeginText)\n i++;\n if (i == 0 || i >= nsub_)\n return false;\n Regexp* re = sub()[i];\n if (re->op_ != kRegexpLiteral &&\n re->op_ != kRegexpLiteralString)\n return false;\n i++;\n if (i < nsub_) {\n for (int j = i; j < nsub_; j++)\n sub()[j]->Incref();\n *suffix = Concat(sub() + i, nsub_ - i, parse_flags());\n } else {\n *suffix = new Regexp(kRegexpEmptyMatch, parse_flags());\n }\n\n bool latin1 = (re->parse_flags() & Latin1) != 0;\n Rune* runes = re->op_ == kRegexpLiteral ? &re->rune_ : re->runes_;\n int nrunes = re->op_ == kRegexpLiteral ? 1 : re->nrunes_;\n ConvertRunesToBytes(latin1, runes, nrunes, prefix);\n *foldcase = (re->parse_flags() & FoldCase) != 0;\n return true;\n}\n\n// Determines whether regexp matches must be unanchored\n// with a fixed string prefix. If so, returns the prefix.\n// The prefix might be ASCII case-insensitive.\nbool Regexp::RequiredPrefixForAccel(std::string* prefix, bool* foldcase) {\n prefix->clear();\n *foldcase = false;\n\n // No need for a walker: the regexp must either begin with or be\n // a literal char or string. We \"see through\" capturing groups,\n // but make no effort to glue multiple prefix fragments together.\n Regexp* re = op_ == kRegexpConcat && nsub_ > 0 ? sub()[0] : this;\n while (re->op_ == kRegexpCapture) {\n re = re->sub()[0];\n if (re->op_ == kRegexpConcat && re->nsub_ > 0)\n re = re->sub()[0];\n }\n if (re->op_ != kRegexpLiteral &&\n re->op_ != kRegexpLiteralString)\n return false;\n\n bool latin1 = (re->parse_flags() & Latin1) != 0;\n Rune* runes = re->op_ == kRegexpLiteral ? &re->rune_ : re->runes_;\n int nrunes = re->op_ == kRegexpLiteral ? 1 : re->nrunes_;\n ConvertRunesToBytes(latin1, runes, nrunes, prefix);\n *foldcase = (re->parse_flags() & FoldCase) != 0;\n return true;\n}\n\n// Character class builder is a balanced binary tree (STL set)\n// containing non-overlapping, non-abutting RuneRanges.\n// The less-than operator used in the tree treats two\n// ranges as equal if they overlap at all, so that\n// lookups for a particular Rune are possible.\n\nCharClassBuilder::CharClassBuilder() {\n nrunes_ = 0;\n upper_ = 0;\n lower_ = 0;\n}\n\n// Add lo-hi to the class; return whether class got bigger.\nbool CharClassBuilder::AddRange(Rune lo, Rune hi) {\n if (hi < lo)\n return false;\n\n if (lo <= 'z' && hi >= 'A') {\n // Overlaps some alpha, maybe not all.\n // Update bitmaps telling which ASCII letters are in the set.\n Rune lo1 = std::max(lo, 'A');\n Rune hi1 = std::min(hi, 'Z');\n if (lo1 <= hi1)\n upper_ |= ((1 << (hi1 - lo1 + 1)) - 1) << (lo1 - 'A');\n\n lo1 = std::max(lo, 'a');\n hi1 = std::min(hi, 'z');\n if (lo1 <= hi1)\n lower_ |= ((1 << (hi1 - lo1 + 1)) - 1) << (lo1 - 'a');\n }\n\n { // Check whether lo, hi is already in the class.\n iterator it = ranges_.find(RuneRange(lo, lo));\n if (it != end() && it->lo <= lo && hi <= it->hi)\n return false;\n }\n\n // Look for a range abutting lo on the left.\n // If it exists, take it out and increase our range.\n if (lo > 0) {\n iterator it = ranges_.find(RuneRange(lo-1, lo-1));\n if (it != end()) {\n lo = it->lo;\n if (it->hi > hi)\n hi = it->hi;\n nrunes_ -= it->hi - it->lo + 1;\n ranges_.erase(it);\n }\n }\n\n // Look for a range abutting hi on the right.\n // If it exists, take it out and increase our range.\n if (hi < Runemax) {\n iterator it = ranges_.find(RuneRange(hi+1, hi+1));\n if (it != end()) {\n hi = it->hi;\n nrunes_ -= it->hi - it->lo + 1;\n ranges_.erase(it);\n }\n }\n\n // Look for ranges between lo and hi. Take them out.\n // This is only safe because the set has no overlapping ranges.\n // We've already removed any ranges abutting lo and hi, so\n // any that overlap [lo, hi] must be contained within it.\n for (;;) {\n iterator it = ranges_.find(RuneRange(lo, hi));\n if (it == end())\n break;\n nrunes_ -= it->hi - it->lo + 1;\n ranges_.erase(it);\n }\n\n // Finally, add [lo, hi].\n nrunes_ += hi - lo + 1;\n ranges_.insert(RuneRange(lo, hi));\n return true;\n}\n\nvoid CharClassBuilder::AddCharClass(CharClassBuilder *cc) {\n for (iterator it = cc->begin(); it != cc->end(); ++it)\n AddRange(it->lo, it->hi);\n}\n\nbool CharClassBuilder::Contains(Rune r) {\n return ranges_.find(RuneRange(r, r)) != end();\n}\n\n// Does the character class behave the same on A-Z as on a-z?\nbool CharClassBuilder::FoldsASCII() {\n return ((upper_ ^ lower_) & AlphaMask) == 0;\n}\n\nCharClassBuilder* CharClassBuilder::Copy() {\n CharClassBuilder* cc = new CharClassBuilder;\n for (iterator it = begin(); it != end(); ++it)\n cc->ranges_.insert(RuneRange(it->lo, it->hi));\n cc->upper_ = upper_;\n cc->lower_ = lower_;\n cc->nrunes_ = nrunes_;\n return cc;\n}\n\n\n\nvoid CharClassBuilder::RemoveAbove(Rune r) {\n if (r >= Runemax)\n return;\n\n if (r < 'z') {\n if (r < 'a')\n lower_ = 0;\n else\n lower_ &= AlphaMask >> ('z' - r);\n }\n\n if (r < 'Z') {\n if (r < 'A')\n upper_ = 0;\n else\n upper_ &= AlphaMask >> ('Z' - r);\n }\n\n for (;;) {\n\n iterator it = ranges_.find(RuneRange(r + 1, Runemax));\n if (it == end())\n break;\n RuneRange rr = *it;\n ranges_.erase(it);\n nrunes_ -= rr.hi - rr.lo + 1;\n if (rr.lo <= r) {\n rr.hi = r;\n ranges_.insert(rr);\n nrunes_ += rr.hi - rr.lo + 1;\n }\n }\n}\n\nvoid CharClassBuilder::Negate() {\n // Build up negation and then copy in.\n // Could edit ranges in place, but C++ won't let me.\n std::vector v;\n v.reserve(ranges_.size() + 1);\n\n // In negation, first range begins at 0, unless\n // the current class begins at 0.\n iterator it = begin();\n if (it == end()) {\n v.push_back(RuneRange(0, Runemax));\n } else {\n int nextlo = 0;\n if (it->lo == 0) {\n nextlo = it->hi + 1;\n ++it;\n }\n for (; it != end(); ++it) {\n v.push_back(RuneRange(nextlo, it->lo - 1));\n nextlo = it->hi + 1;\n }\n if (nextlo <= Runemax)\n v.push_back(RuneRange(nextlo, Runemax));\n }\n\n ranges_.clear();\n for (size_t i = 0; i < v.size(); i++)\n ranges_.insert(v[i]);\n\n upper_ = AlphaMask & ~upper_;\n lower_ = AlphaMask & ~lower_;\n nrunes_ = Runemax+1 - nrunes_;\n}\n\n// Character class is a sorted list of ranges.\n// The ranges are allocated in the same block as the header,\n// necessitating a special allocator and Delete method.\n\nCharClass* CharClass::New(size_t maxranges) {\n CharClass* cc;\n uint8_t* data = new uint8_t[sizeof *cc + maxranges*sizeof cc->ranges_[0]];\n cc = reinterpret_cast(data);\n cc->ranges_ = reinterpret_cast(data + sizeof *cc);\n cc->nranges_ = 0;\n cc->folds_ascii_ = false;\n cc->nrunes_ = 0;\n return cc;\n}\n\nvoid CharClass::Delete() {\n uint8_t* data = reinterpret_cast(this);\n delete[] data;\n}\n\nCharClass* CharClass::Negate() {\n CharClass* cc = CharClass::New(static_cast(nranges_+1));\n cc->folds_ascii_ = folds_ascii_;\n cc->nrunes_ = Runemax + 1 - nrunes_;\n int n = 0;\n int nextlo = 0;\n for (CharClass::iterator it = begin(); it != end(); ++it) {\n if (it->lo == nextlo) {\n nextlo = it->hi + 1;\n } else {\n cc->ranges_[n++] = RuneRange(nextlo, it->lo - 1);\n nextlo = it->hi + 1;\n }\n }\n if (nextlo <= Runemax)\n cc->ranges_[n++] = RuneRange(nextlo, Runemax);\n cc->nranges_ = n;\n return cc;\n}\n\nbool CharClass::Contains(Rune r) const {\n RuneRange* rr = ranges_;\n int n = nranges_;\n while (n > 0) {\n int m = n/2;\n if (rr[m].hi < r) {\n rr += m+1;\n n -= m+1;\n } else if (r < rr[m].lo) {\n n = m;\n } else { // rr[m].lo <= r && r <= rr[m].hi\n return true;\n }\n }\n return false;\n}\n\nCharClass* CharClassBuilder::GetCharClass() {\n CharClass* cc = CharClass::New(ranges_.size());\n int n = 0;\n for (iterator it = begin(); it != end(); ++it)\n cc->ranges_[n++] = *it;\n cc->nranges_ = n;\n ABSL_DCHECK_LE(n, static_cast(ranges_.size()));\n cc->nrunes_ = nrunes_;\n cc->folds_ascii_ = FoldsASCII();\n return cc;\n}\n\n} // namespace re2\n" }, { "path": "re2/regexp.h", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_REGEXP_H_\n#define RE2_REGEXP_H_\n\n// --- SPONSORED LINK --------------------------------------------------\n// If you want to use this library for regular expression matching,\n// you should use re2/re2.h, which provides a class RE2 that\n// mimics the PCRE interface provided by PCRE's C++ wrappers.\n// This header describes the low-level interface used to implement RE2\n// and may change in backwards-incompatible ways from time to time.\n// In contrast, RE2's interface will not.\n// ---------------------------------------------------------------------\n\n// Regular expression library: parsing, execution, and manipulation\n// of regular expressions.\n//\n// Any operation that traverses the Regexp structures should be written\n// using Regexp::Walker (see walker-inl.h), not recursively, because deeply nested\n// regular expressions such as x++++++++++++++++++++... might cause recursive\n// traversals to overflow the stack.\n//\n// It is the caller's responsibility to provide appropriate mutual exclusion\n// around manipulation of the regexps. RE2 does this.\n//\n// PARSING\n//\n// Regexp::Parse parses regular expressions encoded in UTF-8.\n// The default syntax is POSIX extended regular expressions,\n// with the following changes:\n//\n// 1. Backreferences (optional in POSIX EREs) are not supported.\n// (Supporting them precludes the use of DFA-based\n// matching engines.)\n//\n// 2. Collating elements and collation classes are not supported.\n// (No one has needed or wanted them.)\n//\n// The exact syntax accepted can be modified by passing flags to\n// Regexp::Parse. In particular, many of the basic Perl additions\n// are available. The flags are documented below (search for LikePerl).\n//\n// If parsed with the flag Regexp::Latin1, both the regular expression\n// and the input to the matching routines are assumed to be encoded in\n// Latin-1, not UTF-8.\n//\n// EXECUTION\n//\n// Once Regexp has parsed a regular expression, it provides methods\n// to search text using that regular expression. These methods are\n// implemented via calling out to other regular expression libraries.\n// (Let's call them the sublibraries.)\n//\n// To call a sublibrary, Regexp does not simply prepare a\n// string version of the regular expression and hand it to the\n// sublibrary. Instead, Regexp prepares, from its own parsed form, the\n// corresponding internal representation used by the sublibrary.\n// This has the drawback of needing to know the internal representation\n// used by the sublibrary, but it has two important benefits:\n//\n// 1. The syntax and meaning of regular expressions is guaranteed\n// to be that used by Regexp's parser, not the syntax expected\n// by the sublibrary. Regexp might accept a restricted or\n// expanded syntax for regular expressions as compared with\n// the sublibrary. As long as Regexp can translate from its\n// internal form into the sublibrary's, clients need not know\n// exactly which sublibrary they are using.\n//\n// 2. The sublibrary parsers are bypassed. For whatever reason,\n// sublibrary regular expression parsers often have security\n// problems. For example, plan9grep's regular expression parser\n// has a buffer overflow in its handling of large character\n// classes, and PCRE's parser has had buffer overflow problems\n// in the past. Security-team requires sandboxing of sublibrary\n// regular expression parsers. Avoiding the sublibrary parsers\n// avoids the sandbox.\n//\n// The execution methods we use now are provided by the compiled form,\n// Prog, described in prog.h\n//\n// MANIPULATION\n//\n// Unlike other regular expression libraries, Regexp makes its parsed\n// form accessible to clients, so that client code can analyze the\n// parsed regular expressions.\n\n#include \n#include \n\n#include \n#include \n#include \n\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/string_view.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\n// Keep in sync with string list kOpcodeNames[] in testing/dump.cc\nenum RegexpOp {\n // Matches no strings.\n kRegexpNoMatch = 1,\n\n // Matches empty string.\n kRegexpEmptyMatch,\n\n // Matches rune_.\n kRegexpLiteral,\n\n // Matches runes_.\n kRegexpLiteralString,\n\n // Matches concatenation of sub_[0..nsub-1].\n kRegexpConcat,\n // Matches union of sub_[0..nsub-1].\n kRegexpAlternate,\n\n // Matches sub_[0] zero or more times.\n kRegexpStar,\n // Matches sub_[0] one or more times.\n kRegexpPlus,\n // Matches sub_[0] zero or one times.\n kRegexpQuest,\n\n // Matches sub_[0] at least min_ times, at most max_ times.\n // max_ == -1 means no upper limit.\n kRegexpRepeat,\n\n // Parenthesized (capturing) subexpression. Index is cap_.\n // Optionally, capturing name is name_.\n kRegexpCapture,\n\n // Matches any character.\n kRegexpAnyChar,\n\n // Matches any byte [sic].\n kRegexpAnyByte,\n\n // Matches empty string at beginning of line.\n kRegexpBeginLine,\n // Matches empty string at end of line.\n kRegexpEndLine,\n\n // Matches word boundary \"\\b\".\n kRegexpWordBoundary,\n // Matches not-a-word boundary \"\\B\".\n kRegexpNoWordBoundary,\n\n // Matches empty string at beginning of text.\n kRegexpBeginText,\n // Matches empty string at end of text.\n kRegexpEndText,\n\n // Matches character class given by cc_.\n kRegexpCharClass,\n\n // Forces match of entire expression right now,\n // with match ID match_id_ (used by RE2::Set).\n kRegexpHaveMatch,\n\n kMaxRegexpOp = kRegexpHaveMatch,\n};\n\n// Keep in sync with string list in regexp.cc\nenum RegexpStatusCode {\n // No error\n kRegexpSuccess = 0,\n\n // Unexpected error\n kRegexpInternalError,\n\n // Parse errors\n kRegexpBadEscape, // bad escape sequence\n kRegexpBadCharClass, // bad character class\n kRegexpBadCharRange, // bad character class range\n kRegexpMissingBracket, // missing closing ]\n kRegexpMissingParen, // missing closing )\n kRegexpUnexpectedParen, // unexpected closing )\n kRegexpTrailingBackslash, // at end of regexp\n kRegexpRepeatArgument, // repeat argument missing, e.g. \"*\"\n kRegexpRepeatSize, // bad repetition argument\n kRegexpRepeatOp, // bad repetition operator\n kRegexpBadPerlOp, // bad perl operator\n kRegexpBadUTF8, // invalid UTF-8 in regexp\n kRegexpBadNamedCapture, // bad named capture\n};\n\n// Error status for certain operations.\nclass RegexpStatus {\n public:\n RegexpStatus() : code_(kRegexpSuccess), tmp_(NULL) {}\n ~RegexpStatus() { delete tmp_; }\n\n void set_code(RegexpStatusCode code) { code_ = code; }\n void set_error_arg(absl::string_view error_arg) { error_arg_ = error_arg; }\n void set_tmp(std::string* tmp) { delete tmp_; tmp_ = tmp; }\n RegexpStatusCode code() const { return code_; }\n absl::string_view error_arg() const { return error_arg_; }\n bool ok() const { return code() == kRegexpSuccess; }\n\n // Copies state from status.\n void Copy(const RegexpStatus& status);\n\n // Returns text equivalent of code, e.g.:\n // \"Bad character class\"\n static std::string CodeText(RegexpStatusCode code);\n\n // Returns text describing error, e.g.:\n // \"Bad character class: [z-a]\"\n std::string Text() const;\n\n private:\n RegexpStatusCode code_; // Kind of error.\n absl::string_view error_arg_; // Piece of regexp containing syntax error.\n std::string* tmp_; // Temporary storage, possibly for error_arg_.\n\n RegexpStatus(const RegexpStatus&) = delete;\n RegexpStatus& operator=(const RegexpStatus&) = delete;\n};\n\n// Compiled form; see prog.h\nclass Prog;\n\nstruct RuneRange {\n RuneRange() : lo(0), hi(0) { }\n RuneRange(int l, int h) : lo(l), hi(h) { }\n Rune lo;\n Rune hi;\n};\n\n// Less-than on RuneRanges treats a == b if they overlap at all.\n// This lets us look in a set to find the range covering a particular Rune.\nstruct RuneRangeLess {\n bool operator()(const RuneRange& a, const RuneRange& b) const {\n return a.hi < b.lo;\n }\n};\n\nclass CharClassBuilder;\n\nclass CharClass {\n public:\n void Delete();\n\n typedef RuneRange* iterator;\n iterator begin() { return ranges_; }\n iterator end() { return ranges_ + nranges_; }\n\n int size() { return nrunes_; }\n bool empty() { return nrunes_ == 0; }\n bool full() { return nrunes_ == Runemax+1; }\n bool FoldsASCII() { return folds_ascii_; }\n\n bool Contains(Rune r) const;\n CharClass* Negate();\n\n private:\n CharClass(); // not implemented\n ~CharClass(); // not implemented\n static CharClass* New(size_t maxranges);\n\n friend class CharClassBuilder;\n\n bool folds_ascii_;\n int nrunes_;\n RuneRange *ranges_;\n int nranges_;\n\n CharClass(const CharClass&) = delete;\n CharClass& operator=(const CharClass&) = delete;\n};\n\nclass Regexp {\n public:\n\n // Flags for parsing. Can be ORed together.\n enum ParseFlags {\n NoParseFlags = 0,\n FoldCase = 1<<0, // Fold case during matching (case-insensitive).\n Literal = 1<<1, // Treat s as literal string instead of a regexp.\n ClassNL = 1<<2, // Allow char classes like [^a-z] and \\D and \\s\n // and [[:space:]] to match newline.\n DotNL = 1<<3, // Allow . to match newline.\n MatchNL = ClassNL | DotNL,\n OneLine = 1<<4, // Treat ^ and $ as only matching at beginning and\n // end of text, not around embedded newlines.\n // (Perl's default)\n Latin1 = 1<<5, // Regexp and text are in Latin1, not UTF-8.\n NonGreedy = 1<<6, // Repetition operators are non-greedy by default.\n PerlClasses = 1<<7, // Allow Perl character classes like \\d.\n PerlB = 1<<8, // Allow Perl's \\b and \\B.\n PerlX = 1<<9, // Perl extensions:\n // non-capturing parens - (?: )\n // non-greedy operators - *? +? ?? {}?\n // flag edits - (?i) (?-i) (?i: )\n // i - FoldCase\n // m - !OneLine\n // s - DotNL\n // U - NonGreedy\n // line ends: \\A \\z\n // \\Q and \\E to disable/enable metacharacters\n // (?Pexpr) for named captures\n // \\C to match any single byte\n UnicodeGroups = 1<<10, // Allow \\p{Han} for Unicode Han group\n // and \\P{Han} for its negation.\n NeverNL = 1<<11, // Never match NL, even if the regexp mentions\n // it explicitly.\n NeverCapture = 1<<12, // Parse all parens as non-capturing.\n\n // As close to Perl as we can get.\n LikePerl = ClassNL | OneLine | PerlClasses | PerlB | PerlX |\n UnicodeGroups,\n\n // Internal use only.\n WasDollar = 1<<13, // on kRegexpEndText: was $ in regexp text\n AllParseFlags = (1<<14)-1,\n };\n\n // Get. No set, Regexps are logically immutable once created.\n RegexpOp op() { return static_cast(op_); }\n int nsub() { return nsub_; }\n bool simple() { return simple_ != 0; }\n ParseFlags parse_flags() { return static_cast(parse_flags_); }\n int Ref(); // For testing.\n\n Regexp** sub() {\n if(nsub_ <= 1)\n return &subone_;\n else\n return submany_;\n }\n\n int min() {\n ABSL_DCHECK_EQ(op_, kRegexpRepeat);\n return min_;\n }\n int max() {\n ABSL_DCHECK_EQ(op_, kRegexpRepeat);\n return max_;\n }\n Rune rune() {\n ABSL_DCHECK_EQ(op_, kRegexpLiteral);\n return rune_;\n }\n CharClass* cc() {\n ABSL_DCHECK_EQ(op_, kRegexpCharClass);\n return cc_;\n }\n int cap() {\n ABSL_DCHECK_EQ(op_, kRegexpCapture);\n return cap_;\n }\n const std::string* name() {\n ABSL_DCHECK_EQ(op_, kRegexpCapture);\n return name_;\n }\n Rune* runes() {\n ABSL_DCHECK_EQ(op_, kRegexpLiteralString);\n return runes_;\n }\n int nrunes() {\n ABSL_DCHECK_EQ(op_, kRegexpLiteralString);\n return nrunes_;\n }\n int match_id() {\n ABSL_DCHECK_EQ(op_, kRegexpHaveMatch);\n return match_id_;\n }\n\n // Increments reference count, returns object as convenience.\n Regexp* Incref();\n\n // Decrements reference count and deletes this object if count reaches 0.\n void Decref();\n\n // Parses string s to produce regular expression, returned.\n // Caller must release return value with re->Decref().\n // On failure, sets *status (if status != NULL) and returns NULL.\n static Regexp* Parse(absl::string_view s, ParseFlags flags,\n RegexpStatus* status);\n\n // Returns a _new_ simplified version of the current regexp.\n // Does not edit the current regexp.\n // Caller must release return value with re->Decref().\n // Simplified means that counted repetition has been rewritten\n // into simpler terms and all Perl/POSIX features have been\n // removed. The result will capture exactly the same\n // subexpressions the original did, unless formatted with ToString.\n Regexp* Simplify();\n friend class CoalesceWalker;\n friend class SimplifyWalker;\n\n // Parses the regexp src and then simplifies it and sets *dst to the\n // string representation of the simplified form. Returns true on success.\n // Returns false and sets *status (if status != NULL) on parse error.\n static bool SimplifyRegexp(absl::string_view src, ParseFlags flags,\n std::string* dst, RegexpStatus* status);\n\n // Returns the number of capturing groups in the regexp.\n int NumCaptures();\n friend class NumCapturesWalker;\n\n // Returns a map from names to capturing group indices,\n // or NULL if the regexp contains no named capture groups.\n // The caller is responsible for deleting the map.\n std::map* NamedCaptures();\n\n // Returns a map from capturing group indices to capturing group\n // names or NULL if the regexp contains no named capture groups. The\n // caller is responsible for deleting the map.\n std::map* CaptureNames();\n\n // Returns a string representation of the current regexp,\n // using as few parentheses as possible.\n std::string ToString();\n\n // Convenience functions. They consume the passed reference,\n // so in many cases you should use, e.g., Plus(re->Incref(), flags).\n // They do not consume allocated arrays like subs or runes.\n static Regexp* Plus(Regexp* sub, ParseFlags flags);\n static Regexp* Star(Regexp* sub, ParseFlags flags);\n static Regexp* Quest(Regexp* sub, ParseFlags flags);\n static Regexp* Concat(Regexp** subs, int nsubs, ParseFlags flags);\n static Regexp* Alternate(Regexp** subs, int nsubs, ParseFlags flags);\n static Regexp* Capture(Regexp* sub, ParseFlags flags, int cap);\n static Regexp* Repeat(Regexp* sub, ParseFlags flags, int min, int max);\n static Regexp* NewLiteral(Rune rune, ParseFlags flags);\n static Regexp* NewCharClass(CharClass* cc, ParseFlags flags);\n static Regexp* LiteralString(Rune* runes, int nrunes, ParseFlags flags);\n static Regexp* HaveMatch(int match_id, ParseFlags flags);\n\n // Like Alternate but does not factor out common prefixes.\n static Regexp* AlternateNoFactor(Regexp** subs, int nsubs, ParseFlags flags);\n\n // Debugging function. Returns string format for regexp\n // that makes structure clear. Does NOT use regexp syntax.\n std::string Dump();\n\n // Helper traversal class, defined fully in walker-inl.h.\n template class Walker;\n\n // Compile to Prog. See prog.h\n // Reverse prog expects to be run over text backward.\n // Construction and execution of prog will\n // stay within approximately max_mem bytes of memory.\n // If max_mem <= 0, a reasonable default is used.\n Prog* CompileToProg(int64_t max_mem);\n Prog* CompileToReverseProg(int64_t max_mem);\n\n // Whether to expect this library to find exactly the same answer as PCRE\n // when running this regexp. Most regexps do mimic PCRE exactly, but a few\n // obscure cases behave differently. Technically this is more a property\n // of the Prog than the Regexp, but the computation is much easier to do\n // on the Regexp. See mimics_pcre.cc for the exact conditions.\n bool MimicsPCRE();\n\n // Benchmarking function.\n void NullWalk();\n\n // Whether every match of this regexp must be anchored and\n // begin with a non-empty fixed string (perhaps after ASCII\n // case-folding). If so, returns the prefix and the sub-regexp that\n // follows it.\n // Callers should expect *prefix, *foldcase and *suffix to be \"zeroed\"\n // regardless of the return value.\n bool RequiredPrefix(std::string* prefix, bool* foldcase,\n Regexp** suffix);\n\n // Whether every match of this regexp must be unanchored and\n // begin with a non-empty fixed string (perhaps after ASCII\n // case-folding). If so, returns the prefix.\n // Callers should expect *prefix and *foldcase to be \"zeroed\"\n // regardless of the return value.\n bool RequiredPrefixForAccel(std::string* prefix, bool* foldcase);\n\n // Controls the maximum repeat count permitted by the parser.\n // FOR FUZZING ONLY.\n static void FUZZING_ONLY_set_maximum_repeat_count(int i);\n\n private:\n // Constructor allocates vectors as appropriate for operator.\n explicit Regexp(RegexpOp op, ParseFlags parse_flags);\n\n // Use Decref() instead of delete to release Regexps.\n // This is private to catch deletes at compile time.\n ~Regexp();\n void Destroy();\n bool QuickDestroy();\n\n // Helpers for Parse. Listed here so they can edit Regexps.\n class ParseState;\n\n friend class ParseState;\n friend bool ParseCharClass(absl::string_view* s, Regexp** out_re,\n RegexpStatus* status);\n\n // Helper for testing [sic].\n friend bool RegexpEqualTestingOnly(Regexp*, Regexp*);\n\n // Computes whether Regexp is already simple.\n bool ComputeSimple();\n\n // Constructor that generates a Star, Plus or Quest,\n // squashing the pair if sub is also a Star, Plus or Quest.\n static Regexp* StarPlusOrQuest(RegexpOp op, Regexp* sub, ParseFlags flags);\n\n // Constructor that generates a concatenation or alternation,\n // enforcing the limit on the number of subexpressions for\n // a particular Regexp.\n static Regexp* ConcatOrAlternate(RegexpOp op, Regexp** subs, int nsubs,\n ParseFlags flags, bool can_factor);\n\n // Returns the leading string that re starts with.\n // The returned Rune* points into a piece of re,\n // so it must not be used after the caller calls re->Decref().\n static Rune* LeadingString(Regexp* re, int* nrune, ParseFlags* flags);\n\n // Removes the first n leading runes from the beginning of re.\n // Edits re in place.\n static void RemoveLeadingString(Regexp* re, int n);\n\n // Returns the leading regexp in re's top-level concatenation.\n // The returned Regexp* points at re or a sub-expression of re,\n // so it must not be used after the caller calls re->Decref().\n static Regexp* LeadingRegexp(Regexp* re);\n\n // Removes LeadingRegexp(re) from re and returns the remainder.\n // Might edit re in place.\n static Regexp* RemoveLeadingRegexp(Regexp* re);\n\n // Simplifies an alternation of literal strings by factoring out\n // common prefixes.\n static int FactorAlternation(Regexp** sub, int nsub, ParseFlags flags);\n friend class FactorAlternationImpl;\n\n // Is a == b? Only efficient on regexps that have not been through\n // Simplify yet - the expansion of a kRegexpRepeat will make this\n // take a long time. Do not call on such regexps, hence private.\n static bool Equal(Regexp* a, Regexp* b);\n\n // Allocate space for n sub-regexps.\n void AllocSub(int n) {\n ABSL_DCHECK(n >= 0 && static_cast(n) == n);\n if (n > 1)\n submany_ = new Regexp*[n];\n nsub_ = static_cast(n);\n }\n\n // Add Rune to LiteralString\n void AddRuneToString(Rune r);\n\n // Swaps this with that, in place.\n void Swap(Regexp *that);\n\n // Operator. See description of operators above.\n // uint8_t instead of RegexpOp to control space usage.\n uint8_t op_;\n\n // Is this regexp structure already simple\n // (has it been returned by Simplify)?\n // uint8_t instead of bool to control space usage.\n uint8_t simple_;\n\n // Flags saved from parsing and used during execution.\n // (Only FoldCase is used.)\n // uint16_t instead of ParseFlags to control space usage.\n uint16_t parse_flags_;\n\n // Reference count. Exists so that SimplifyRegexp can build\n // regexp structures that are dags rather than trees to avoid\n // exponential blowup in space requirements.\n // uint16_t to control space usage.\n // The standard regexp routines will never generate a\n // ref greater than the maximum repeat count (kMaxRepeat),\n // but even so, Incref and Decref consult an overflow map\n // when ref_ reaches kMaxRef.\n uint16_t ref_;\n static const uint16_t kMaxRef = 0xffff;\n\n // Subexpressions.\n // uint16_t to control space usage.\n // Concat and Alternate handle larger numbers of subexpressions\n // by building concatenation or alternation trees.\n // Other routines should call Concat or Alternate instead of\n // filling in sub() by hand.\n uint16_t nsub_;\n static const uint16_t kMaxNsub = 0xffff;\n union {\n Regexp** submany_; // if nsub_ > 1\n Regexp* subone_; // if nsub_ == 1\n };\n\n // Extra space for parse and teardown stacks.\n Regexp* down_;\n\n // Arguments to operator. See description of operators above.\n union {\n struct { // Repeat\n int max_;\n int min_;\n };\n struct { // Capture\n int cap_;\n std::string* name_;\n };\n struct { // LiteralString\n int nrunes_;\n Rune* runes_;\n };\n struct { // CharClass\n // These two could be in separate union members,\n // but it wouldn't save any space (there are other two-word structs)\n // and keeping them separate avoids confusion during parsing.\n CharClass* cc_;\n CharClassBuilder* ccb_;\n };\n Rune rune_; // Literal\n int match_id_; // HaveMatch\n void *the_union_[2]; // as big as any other element, for memset\n };\n\n Regexp(const Regexp&) = delete;\n Regexp& operator=(const Regexp&) = delete;\n};\n\n// Character class set: contains non-overlapping, non-abutting RuneRanges.\ntypedef std::set RuneRangeSet;\n\nclass CharClassBuilder {\n public:\n CharClassBuilder();\n\n typedef RuneRangeSet::iterator iterator;\n iterator begin() { return ranges_.begin(); }\n iterator end() { return ranges_.end(); }\n\n int size() { return nrunes_; }\n bool empty() { return nrunes_ == 0; }\n bool full() { return nrunes_ == Runemax+1; }\n\n bool Contains(Rune r);\n bool FoldsASCII();\n bool AddRange(Rune lo, Rune hi); // returns whether class changed\n CharClassBuilder* Copy();\n void AddCharClass(CharClassBuilder* cc);\n void Negate();\n void RemoveAbove(Rune r);\n CharClass* GetCharClass();\n void AddRangeFlags(Rune lo, Rune hi, Regexp::ParseFlags parse_flags);\n\n private:\n static const uint32_t AlphaMask = (1<<26) - 1;\n uint32_t upper_; // bitmap of A-Z\n uint32_t lower_; // bitmap of a-z\n int nrunes_;\n RuneRangeSet ranges_;\n\n CharClassBuilder(const CharClassBuilder&) = delete;\n CharClassBuilder& operator=(const CharClassBuilder&) = delete;\n};\n\n// Bitwise ops on ParseFlags produce ParseFlags.\ninline Regexp::ParseFlags operator|(Regexp::ParseFlags a,\n Regexp::ParseFlags b) {\n return static_cast(\n static_cast(a) | static_cast(b));\n}\n\ninline Regexp::ParseFlags operator^(Regexp::ParseFlags a,\n Regexp::ParseFlags b) {\n return static_cast(\n static_cast(a) ^ static_cast(b));\n}\n\ninline Regexp::ParseFlags operator&(Regexp::ParseFlags a,\n Regexp::ParseFlags b) {\n return static_cast(\n static_cast(a) & static_cast(b));\n}\n\ninline Regexp::ParseFlags operator~(Regexp::ParseFlags a) {\n // Attempting to produce a value out of enum's range has undefined behaviour.\n return static_cast(\n ~static_cast(a) & static_cast(Regexp::AllParseFlags));\n}\n\n} // namespace re2\n\n#endif // RE2_REGEXP_H_\n" }, { "path": "re2/set.cc", "content": "// Copyright 2010 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \"re2/set.h\"\n\n#include \n\n#include \n#include \n#include \n#include \n#include \n\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/prog.h\"\n#include \"re2/re2.h\"\n#include \"re2/regexp.h\"\n#include \"re2/sparse_set.h\"\n\nnamespace re2 {\n\nRE2::Set::Set(const RE2::Options& options, RE2::Anchor anchor)\n : options_(options),\n anchor_(anchor),\n compiled_(false),\n size_(0) {\n options_.set_never_capture(true); // might unblock some optimisations\n}\n\nRE2::Set::~Set() {\n for (size_t i = 0; i < elem_.size(); i++)\n elem_[i].second->Decref();\n}\n\nRE2::Set::Set(Set&& other)\n : options_(other.options_),\n anchor_(other.anchor_),\n elem_(std::move(other.elem_)),\n compiled_(other.compiled_),\n size_(other.size_),\n prog_(std::move(other.prog_)) {\n other.elem_.clear();\n other.elem_.shrink_to_fit();\n other.compiled_ = false;\n other.size_ = 0;\n other.prog_.reset();\n}\n\nRE2::Set& RE2::Set::operator=(Set&& other) {\n this->~Set();\n (void) new (this) Set(std::move(other));\n return *this;\n}\n\nint RE2::Set::Size() const {\n if (!compiled_)\n return static_cast(elem_.size());\n return size_;\n}\n\nint RE2::Set::Add(absl::string_view pattern, std::string* error) {\n if (compiled_) {\n ABSL_LOG(DFATAL) << \"RE2::Set::Add() called after compiling\";\n return -1;\n }\n\n Regexp::ParseFlags pf = static_cast(\n options_.ParseFlags());\n RegexpStatus status;\n re2::Regexp* re = Regexp::Parse(pattern, pf, &status);\n if (re == NULL) {\n if (error != NULL)\n *error = status.Text();\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"Error parsing '\" << pattern << \"': \" << status.Text();\n return -1;\n }\n\n // Concatenate with match index and push on vector.\n int n = static_cast(elem_.size());\n re2::Regexp* m = re2::Regexp::HaveMatch(n, pf);\n if (re->op() == kRegexpConcat) {\n int nsub = re->nsub();\n PODArray sub(nsub + 1);\n for (int i = 0; i < nsub; i++)\n sub[i] = re->sub()[i]->Incref();\n sub[nsub] = m;\n re->Decref();\n re = re2::Regexp::Concat(sub.data(), nsub + 1, pf);\n } else {\n re2::Regexp* sub[2];\n sub[0] = re;\n sub[1] = m;\n re = re2::Regexp::Concat(sub, 2, pf);\n }\n elem_.emplace_back(std::string(pattern), re);\n return n;\n}\n\nbool RE2::Set::Compile() {\n if (compiled_) {\n ABSL_LOG(DFATAL) << \"RE2::Set::Compile() called more than once\";\n return false;\n }\n compiled_ = true;\n size_ = static_cast(elem_.size());\n\n // Sort the elements by their patterns. This is good enough for now\n // until we have a Regexp comparison function. (Maybe someday...)\n std::sort(elem_.begin(), elem_.end(),\n [](const Elem& a, const Elem& b) -> bool {\n return a.first < b.first;\n });\n\n PODArray sub(size_);\n for (int i = 0; i < size_; i++)\n sub[i] = elem_[i].second;\n elem_.clear();\n elem_.shrink_to_fit();\n\n Regexp::ParseFlags pf = static_cast(\n options_.ParseFlags());\n re2::Regexp* re = re2::Regexp::Alternate(sub.data(), size_, pf);\n\n prog_.reset(Prog::CompileSet(re, anchor_, options_.max_mem()));\n re->Decref();\n return prog_ != nullptr;\n}\n\nbool RE2::Set::Match(absl::string_view text, std::vector* v) const {\n return Match(text, v, NULL);\n}\n\nbool RE2::Set::Match(absl::string_view text, std::vector* v,\n ErrorInfo* error_info) const {\n if (!compiled_) {\n if (error_info != NULL)\n error_info->kind = kNotCompiled;\n ABSL_LOG(DFATAL) << \"RE2::Set::Match() called before compiling\";\n return false;\n }\n#ifdef RE2_HAVE_THREAD_LOCAL\n hooks::context = NULL;\n#endif\n bool dfa_failed = false;\n std::unique_ptr matches;\n if (v != NULL) {\n matches.reset(new SparseSet(size_));\n v->clear();\n }\n bool ret = prog_->SearchDFA(text, text, Prog::kAnchored, Prog::kManyMatch,\n NULL, &dfa_failed, matches.get());\n if (dfa_failed) {\n if (options_.log_errors())\n ABSL_LOG(ERROR) << \"DFA out of memory: \"\n << \"program size \" << prog_->size() << \", \"\n << \"list count \" << prog_->list_count() << \", \"\n << \"bytemap range \" << prog_->bytemap_range();\n if (error_info != NULL)\n error_info->kind = kOutOfMemory;\n return false;\n }\n if (ret == false) {\n if (error_info != NULL)\n error_info->kind = kNoError;\n return false;\n }\n if (v != NULL) {\n if (matches->empty()) {\n if (error_info != NULL)\n error_info->kind = kInconsistent;\n ABSL_LOG(DFATAL) << \"RE2::Set::Match() matched, but no matches returned\";\n return false;\n }\n v->assign(matches->begin(), matches->end());\n }\n if (error_info != NULL)\n error_info->kind = kNoError;\n return true;\n}\n\n} // namespace re2\n" }, { "path": "re2/set.h", "content": "// Copyright 2010 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_SET_H_\n#define RE2_SET_H_\n\n#include \n#include \n#include \n#include \n\n#include \"absl/strings/string_view.h\"\n#include \"re2/re2.h\"\n\nnamespace re2 {\nclass Prog;\nclass Regexp;\n} // namespace re2\n\nnamespace re2 {\n\n// An RE2::Set represents a collection of regexps that can\n// be searched for simultaneously.\nclass RE2::Set {\n public:\n enum ErrorKind {\n kNoError = 0,\n kNotCompiled, // The set is not compiled.\n kOutOfMemory, // The DFA ran out of memory.\n kInconsistent, // The result is inconsistent. This should never happen.\n };\n\n struct ErrorInfo {\n ErrorKind kind;\n };\n\n Set(const RE2::Options& options, RE2::Anchor anchor);\n ~Set();\n\n // Not copyable.\n Set(const Set&) = delete;\n Set& operator=(const Set&) = delete;\n // Movable.\n Set(Set&& other);\n Set& operator=(Set&& other);\n\n // Adds pattern to the set using the options passed to the constructor.\n // Returns the index that will identify the regexp in the output of Match(),\n // or -1 if the regexp cannot be parsed.\n // Indices are assigned in sequential order starting from 0.\n // Errors do not increment the index; if error is not NULL, *error will hold\n // the error message from the parser.\n int Add(absl::string_view pattern, std::string* error);\n\n // Returns the number of patterns in the set.\n // Can be called before or after Compile().\n int Size() const;\n\n // Compiles the set in preparation for matching.\n // Returns false if the compiler runs out of memory.\n // Add() must not be called again after Compile().\n // Compile() must be called before Match().\n bool Compile();\n\n // Returns true if text matches at least one of the regexps in the set.\n // Fills v (if not NULL) with the indices of the matching regexps.\n // Callers must not expect v to be sorted.\n // The indices are in the half-open interval [0, Size()).\n bool Match(absl::string_view text, std::vector* v) const;\n\n // As above, but populates error_info (if not NULL) when none of the regexps\n // in the set matched. This can inform callers when DFA execution fails, for\n // example, because they might wish to handle that case differently.\n bool Match(absl::string_view text, std::vector* v,\n ErrorInfo* error_info) const;\n\n private:\n typedef std::pair Elem;\n\n RE2::Options options_;\n RE2::Anchor anchor_;\n std::vector elem_;\n bool compiled_;\n int size_;\n std::unique_ptr prog_;\n};\n\n} // namespace re2\n\n#endif // RE2_SET_H_\n" }, { "path": "re2/simplify.cc", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Rewrite POSIX and other features in re\n// to use simple extended regular expression features.\n// Also sort and simplify character classes.\n\n#include \n\n#include \n#include \n\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/regexp.h\"\n#include \"re2/walker-inl.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\n// Parses the regexp src and then simplifies it and sets *dst to the\n// string representation of the simplified form. Returns true on success.\n// Returns false and sets *error (if error != NULL) on error.\nbool Regexp::SimplifyRegexp(absl::string_view src, ParseFlags flags,\n std::string* dst, RegexpStatus* status) {\n Regexp* re = Parse(src, flags, status);\n if (re == NULL)\n return false;\n Regexp* sre = re->Simplify();\n re->Decref();\n if (sre == NULL) {\n if (status) {\n status->set_code(kRegexpInternalError);\n status->set_error_arg(src);\n }\n return false;\n }\n *dst = sre->ToString();\n sre->Decref();\n return true;\n}\n\n// Assuming the simple_ flags on the children are accurate,\n// is this Regexp* simple?\nbool Regexp::ComputeSimple() {\n Regexp** subs;\n switch (op_) {\n case kRegexpNoMatch:\n case kRegexpEmptyMatch:\n case kRegexpLiteral:\n case kRegexpLiteralString:\n case kRegexpBeginLine:\n case kRegexpEndLine:\n case kRegexpBeginText:\n case kRegexpWordBoundary:\n case kRegexpNoWordBoundary:\n case kRegexpEndText:\n case kRegexpAnyChar:\n case kRegexpAnyByte:\n case kRegexpHaveMatch:\n return true;\n case kRegexpConcat:\n case kRegexpAlternate:\n // These are simple as long as the subpieces are simple.\n subs = sub();\n for (int i = 0; i < nsub_; i++)\n if (!subs[i]->simple())\n return false;\n return true;\n case kRegexpCharClass:\n // Simple as long as the char class is not empty, not full.\n if (ccb_ != NULL)\n return !ccb_->empty() && !ccb_->full();\n return !cc_->empty() && !cc_->full();\n case kRegexpCapture:\n subs = sub();\n return subs[0]->simple();\n case kRegexpStar:\n case kRegexpPlus:\n case kRegexpQuest:\n subs = sub();\n if (!subs[0]->simple())\n return false;\n switch (subs[0]->op_) {\n case kRegexpStar:\n case kRegexpPlus:\n case kRegexpQuest:\n case kRegexpEmptyMatch:\n case kRegexpNoMatch:\n return false;\n default:\n break;\n }\n return true;\n case kRegexpRepeat:\n return false;\n }\n ABSL_LOG(DFATAL) << \"Case not handled in ComputeSimple: \" << op_;\n return false;\n}\n\n// Walker subclass used by Simplify.\n// Coalesces runs of star/plus/quest/repeat of the same literal along with any\n// occurrences of that literal into repeats of that literal. It also works for\n// char classes, any char and any byte.\n// PostVisit creates the coalesced result, which should then be simplified.\nclass CoalesceWalker : public Regexp::Walker {\n public:\n CoalesceWalker() {}\n virtual Regexp* PostVisit(Regexp* re, Regexp* parent_arg, Regexp* pre_arg,\n Regexp** child_args, int nchild_args);\n virtual Regexp* Copy(Regexp* re);\n virtual Regexp* ShortVisit(Regexp* re, Regexp* parent_arg);\n\n private:\n // These functions are declared inside CoalesceWalker so that\n // they can edit the private fields of the Regexps they construct.\n\n // Returns true if r1 and r2 can be coalesced. In particular, ensures that\n // the parse flags are consistent. (They will not be checked again later.)\n static bool CanCoalesce(Regexp* r1, Regexp* r2);\n\n // Coalesces *r1ptr and *r2ptr. In most cases, the array elements afterwards\n // will be empty match and the coalesced op. In other cases, where part of a\n // literal string was removed to be coalesced, the array elements afterwards\n // will be the coalesced op and the remainder of the literal string.\n static void DoCoalesce(Regexp** r1ptr, Regexp** r2ptr);\n\n CoalesceWalker(const CoalesceWalker&) = delete;\n CoalesceWalker& operator=(const CoalesceWalker&) = delete;\n};\n\n// Walker subclass used by Simplify.\n// The simplify walk is purely post-recursive: given the simplified children,\n// PostVisit creates the simplified result.\n// The child_args are simplified Regexp*s.\nclass SimplifyWalker : public Regexp::Walker {\n public:\n SimplifyWalker() {}\n virtual Regexp* PreVisit(Regexp* re, Regexp* parent_arg, bool* stop);\n virtual Regexp* PostVisit(Regexp* re, Regexp* parent_arg, Regexp* pre_arg,\n Regexp** child_args, int nchild_args);\n virtual Regexp* Copy(Regexp* re);\n virtual Regexp* ShortVisit(Regexp* re, Regexp* parent_arg);\n\n private:\n // These functions are declared inside SimplifyWalker so that\n // they can edit the private fields of the Regexps they construct.\n\n // Creates a concatenation of two Regexp, consuming refs to re1 and re2.\n // Caller must Decref return value when done with it.\n static Regexp* Concat2(Regexp* re1, Regexp* re2, Regexp::ParseFlags flags);\n\n // Simplifies the expression re{min,max} in terms of *, +, and ?.\n // Returns a new regexp. Does not edit re. Does not consume reference to re.\n // Caller must Decref return value when done with it.\n static Regexp* SimplifyRepeat(Regexp* re, int min, int max,\n Regexp::ParseFlags parse_flags);\n\n // Simplifies a character class by expanding any named classes\n // into rune ranges. Does not edit re. Does not consume ref to re.\n // Caller must Decref return value when done with it.\n static Regexp* SimplifyCharClass(Regexp* re);\n\n SimplifyWalker(const SimplifyWalker&) = delete;\n SimplifyWalker& operator=(const SimplifyWalker&) = delete;\n};\n\n// Simplifies a regular expression, returning a new regexp.\n// The new regexp uses traditional Unix egrep features only,\n// plus the Perl (?:) non-capturing parentheses.\n// Otherwise, no POSIX or Perl additions. The new regexp\n// captures exactly the same subexpressions (with the same indices)\n// as the original.\n// Does not edit current object.\n// Caller must Decref() return value when done with it.\n\nRegexp* Regexp::Simplify() {\n CoalesceWalker cw;\n Regexp* cre = cw.Walk(this, NULL);\n if (cre == NULL)\n return NULL;\n if (cw.stopped_early()) {\n cre->Decref();\n return NULL;\n }\n SimplifyWalker sw;\n Regexp* sre = sw.Walk(cre, NULL);\n cre->Decref();\n if (sre == NULL)\n return NULL;\n if (sw.stopped_early()) {\n sre->Decref();\n return NULL;\n }\n return sre;\n}\n\n#define Simplify DontCallSimplify // Avoid accidental recursion\n\n// Utility function for PostVisit implementations that compares re->sub() with\n// child_args to determine whether any child_args changed. In the common case,\n// where nothing changed, calls Decref() for all child_args and returns false,\n// so PostVisit must return re->Incref(). Otherwise, returns true.\nstatic bool ChildArgsChanged(Regexp* re, Regexp** child_args) {\n for (int i = 0; i < re->nsub(); i++) {\n Regexp* sub = re->sub()[i];\n Regexp* newsub = child_args[i];\n if (newsub != sub)\n return true;\n }\n for (int i = 0; i < re->nsub(); i++) {\n Regexp* newsub = child_args[i];\n newsub->Decref();\n }\n return false;\n}\n\nRegexp* CoalesceWalker::Copy(Regexp* re) {\n return re->Incref();\n}\n\nRegexp* CoalesceWalker::ShortVisit(Regexp* re, Regexp* parent_arg) {\n // Should never be called: we use Walk(), not WalkExponential().\n#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION\n ABSL_LOG(DFATAL) << \"CoalesceWalker::ShortVisit called\";\n#endif\n return re->Incref();\n}\n\nRegexp* CoalesceWalker::PostVisit(Regexp* re,\n Regexp* parent_arg,\n Regexp* pre_arg,\n Regexp** child_args,\n int nchild_args) {\n if (re->nsub() == 0)\n return re->Incref();\n\n if (re->op() != kRegexpConcat) {\n if (!ChildArgsChanged(re, child_args))\n return re->Incref();\n\n // Something changed. Build a new op.\n Regexp* nre = new Regexp(re->op(), re->parse_flags());\n nre->AllocSub(re->nsub());\n Regexp** nre_subs = nre->sub();\n for (int i = 0; i < re->nsub(); i++)\n nre_subs[i] = child_args[i];\n // Repeats and Captures have additional data that must be copied.\n if (re->op() == kRegexpRepeat) {\n nre->min_ = re->min();\n nre->max_ = re->max();\n } else if (re->op() == kRegexpCapture) {\n nre->cap_ = re->cap();\n }\n return nre;\n }\n\n bool can_coalesce = false;\n for (int i = 0; i < re->nsub(); i++) {\n if (i+1 < re->nsub() &&\n CanCoalesce(child_args[i], child_args[i+1])) {\n can_coalesce = true;\n break;\n }\n }\n if (!can_coalesce) {\n if (!ChildArgsChanged(re, child_args))\n return re->Incref();\n\n // Something changed. Build a new op.\n Regexp* nre = new Regexp(re->op(), re->parse_flags());\n nre->AllocSub(re->nsub());\n Regexp** nre_subs = nre->sub();\n for (int i = 0; i < re->nsub(); i++)\n nre_subs[i] = child_args[i];\n return nre;\n }\n\n for (int i = 0; i < re->nsub(); i++) {\n if (i+1 < re->nsub() &&\n CanCoalesce(child_args[i], child_args[i+1]))\n DoCoalesce(&child_args[i], &child_args[i+1]);\n }\n // Determine how many empty matches were left by DoCoalesce.\n int n = 0;\n for (int i = n; i < re->nsub(); i++) {\n if (child_args[i]->op() == kRegexpEmptyMatch)\n n++;\n }\n // Build a new op.\n Regexp* nre = new Regexp(re->op(), re->parse_flags());\n nre->AllocSub(re->nsub() - n);\n Regexp** nre_subs = nre->sub();\n for (int i = 0, j = 0; i < re->nsub(); i++) {\n if (child_args[i]->op() == kRegexpEmptyMatch) {\n child_args[i]->Decref();\n continue;\n }\n nre_subs[j] = child_args[i];\n j++;\n }\n return nre;\n}\n\nbool CoalesceWalker::CanCoalesce(Regexp* r1, Regexp* r2) {\n // r1 must be a star/plus/quest/repeat of a literal, char class, any char or\n // any byte.\n if ((r1->op() == kRegexpStar ||\n r1->op() == kRegexpPlus ||\n r1->op() == kRegexpQuest ||\n r1->op() == kRegexpRepeat) &&\n (r1->sub()[0]->op() == kRegexpLiteral ||\n r1->sub()[0]->op() == kRegexpCharClass ||\n r1->sub()[0]->op() == kRegexpAnyChar ||\n r1->sub()[0]->op() == kRegexpAnyByte)) {\n // r2 must be a star/plus/quest/repeat of the same literal, char class,\n // any char or any byte.\n if ((r2->op() == kRegexpStar ||\n r2->op() == kRegexpPlus ||\n r2->op() == kRegexpQuest ||\n r2->op() == kRegexpRepeat) &&\n Regexp::Equal(r1->sub()[0], r2->sub()[0]) &&\n // The parse flags must be consistent.\n ((r1->parse_flags() & Regexp::NonGreedy) ==\n (r2->parse_flags() & Regexp::NonGreedy))) {\n return true;\n }\n // ... OR an occurrence of that literal, char class, any char or any byte\n if (Regexp::Equal(r1->sub()[0], r2)) {\n return true;\n }\n // ... OR a literal string that begins with that literal.\n if (r1->sub()[0]->op() == kRegexpLiteral &&\n r2->op() == kRegexpLiteralString &&\n r2->runes()[0] == r1->sub()[0]->rune() &&\n // The parse flags must be consistent.\n ((r1->sub()[0]->parse_flags() & Regexp::FoldCase) ==\n (r2->parse_flags() & Regexp::FoldCase))) {\n return true;\n }\n }\n return false;\n}\n\nvoid CoalesceWalker::DoCoalesce(Regexp** r1ptr, Regexp** r2ptr) {\n Regexp* r1 = *r1ptr;\n Regexp* r2 = *r2ptr;\n\n Regexp* nre = Regexp::Repeat(\n r1->sub()[0]->Incref(), r1->parse_flags(), 0, 0);\n\n switch (r1->op()) {\n case kRegexpStar:\n nre->min_ = 0;\n nre->max_ = -1;\n break;\n\n case kRegexpPlus:\n nre->min_ = 1;\n nre->max_ = -1;\n break;\n\n case kRegexpQuest:\n nre->min_ = 0;\n nre->max_ = 1;\n break;\n\n case kRegexpRepeat:\n nre->min_ = r1->min();\n nre->max_ = r1->max();\n break;\n\n default:\n nre->Decref();\n ABSL_LOG(DFATAL) << \"DoCoalesce failed: r1->op() is \" << r1->op();\n return;\n }\n\n switch (r2->op()) {\n case kRegexpStar:\n nre->max_ = -1;\n goto LeaveEmpty;\n\n case kRegexpPlus:\n nre->min_++;\n nre->max_ = -1;\n goto LeaveEmpty;\n\n case kRegexpQuest:\n if (nre->max() != -1)\n nre->max_++;\n goto LeaveEmpty;\n\n case kRegexpRepeat:\n nre->min_ += r2->min();\n if (r2->max() == -1)\n nre->max_ = -1;\n else if (nre->max() != -1)\n nre->max_ += r2->max();\n goto LeaveEmpty;\n\n case kRegexpLiteral:\n case kRegexpCharClass:\n case kRegexpAnyChar:\n case kRegexpAnyByte:\n nre->min_++;\n if (nre->max() != -1)\n nre->max_++;\n goto LeaveEmpty;\n\n LeaveEmpty:\n *r1ptr = new Regexp(kRegexpEmptyMatch, Regexp::NoParseFlags);\n *r2ptr = nre;\n break;\n\n case kRegexpLiteralString: {\n Rune r = r1->sub()[0]->rune();\n // Determine how much of the literal string is removed.\n // We know that we have at least one rune. :)\n int n = 1;\n while (n < r2->nrunes() && r2->runes()[n] == r)\n n++;\n nre->min_ += n;\n if (nre->max() != -1)\n nre->max_ += n;\n if (n == r2->nrunes())\n goto LeaveEmpty;\n *r1ptr = nre;\n *r2ptr = Regexp::LiteralString(\n &r2->runes()[n], r2->nrunes() - n, r2->parse_flags());\n break;\n }\n\n default:\n nre->Decref();\n ABSL_LOG(DFATAL) << \"DoCoalesce failed: r2->op() is \" << r2->op();\n return;\n }\n\n r1->Decref();\n r2->Decref();\n}\n\nRegexp* SimplifyWalker::Copy(Regexp* re) {\n return re->Incref();\n}\n\nRegexp* SimplifyWalker::ShortVisit(Regexp* re, Regexp* parent_arg) {\n // Should never be called: we use Walk(), not WalkExponential().\n#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION\n ABSL_LOG(DFATAL) << \"SimplifyWalker::ShortVisit called\";\n#endif\n return re->Incref();\n}\n\nRegexp* SimplifyWalker::PreVisit(Regexp* re, Regexp* parent_arg, bool* stop) {\n if (re->simple()) {\n *stop = true;\n return re->Incref();\n }\n return NULL;\n}\n\nRegexp* SimplifyWalker::PostVisit(Regexp* re,\n Regexp* parent_arg,\n Regexp* pre_arg,\n Regexp** child_args,\n int nchild_args) {\n switch (re->op()) {\n case kRegexpNoMatch:\n case kRegexpEmptyMatch:\n case kRegexpLiteral:\n case kRegexpLiteralString:\n case kRegexpBeginLine:\n case kRegexpEndLine:\n case kRegexpBeginText:\n case kRegexpWordBoundary:\n case kRegexpNoWordBoundary:\n case kRegexpEndText:\n case kRegexpAnyChar:\n case kRegexpAnyByte:\n case kRegexpHaveMatch:\n // All these are always simple.\n re->simple_ = true;\n return re->Incref();\n\n case kRegexpConcat:\n case kRegexpAlternate: {\n // These are simple as long as the subpieces are simple.\n if (!ChildArgsChanged(re, child_args)) {\n re->simple_ = true;\n return re->Incref();\n }\n Regexp* nre = new Regexp(re->op(), re->parse_flags());\n nre->AllocSub(re->nsub());\n Regexp** nre_subs = nre->sub();\n for (int i = 0; i < re->nsub(); i++)\n nre_subs[i] = child_args[i];\n nre->simple_ = true;\n return nre;\n }\n\n case kRegexpCapture: {\n Regexp* newsub = child_args[0];\n if (newsub == re->sub()[0]) {\n newsub->Decref();\n re->simple_ = true;\n return re->Incref();\n }\n Regexp* nre = new Regexp(kRegexpCapture, re->parse_flags());\n nre->AllocSub(1);\n nre->sub()[0] = newsub;\n nre->cap_ = re->cap();\n nre->simple_ = true;\n return nre;\n }\n\n case kRegexpStar:\n case kRegexpPlus:\n case kRegexpQuest: {\n Regexp* newsub = child_args[0];\n // Special case: repeat the empty string as much as\n // you want, but it's still the empty string.\n if (newsub->op() == kRegexpEmptyMatch)\n return newsub;\n\n // These are simple as long as the subpiece is simple.\n if (newsub == re->sub()[0]) {\n newsub->Decref();\n re->simple_ = true;\n return re->Incref();\n }\n\n // These are also idempotent if flags are constant.\n if (re->op() == newsub->op() &&\n re->parse_flags() == newsub->parse_flags())\n return newsub;\n\n Regexp* nre = new Regexp(re->op(), re->parse_flags());\n nre->AllocSub(1);\n nre->sub()[0] = newsub;\n nre->simple_ = true;\n return nre;\n }\n\n case kRegexpRepeat: {\n Regexp* newsub = child_args[0];\n // Special case: repeat the empty string as much as\n // you want, but it's still the empty string.\n if (newsub->op() == kRegexpEmptyMatch)\n return newsub;\n\n Regexp* nre = SimplifyRepeat(newsub, re->min_, re->max_,\n re->parse_flags());\n newsub->Decref();\n nre->simple_ = true;\n return nre;\n }\n\n case kRegexpCharClass: {\n Regexp* nre = SimplifyCharClass(re);\n nre->simple_ = true;\n return nre;\n }\n }\n\n ABSL_LOG(ERROR) << \"Simplify case not handled: \" << re->op();\n return re->Incref();\n}\n\n// Creates a concatenation of two Regexp, consuming refs to re1 and re2.\n// Returns a new Regexp, handing the ref to the caller.\nRegexp* SimplifyWalker::Concat2(Regexp* re1, Regexp* re2,\n Regexp::ParseFlags parse_flags) {\n Regexp* re = new Regexp(kRegexpConcat, parse_flags);\n re->AllocSub(2);\n Regexp** subs = re->sub();\n subs[0] = re1;\n subs[1] = re2;\n return re;\n}\n\n// Returns true if re is an empty-width op.\nstatic bool IsEmptyOp(Regexp* re) {\n return (re->op() == kRegexpBeginLine ||\n re->op() == kRegexpEndLine ||\n re->op() == kRegexpWordBoundary ||\n re->op() == kRegexpNoWordBoundary ||\n re->op() == kRegexpBeginText ||\n re->op() == kRegexpEndText);\n}\n\n// Simplifies the expression re{min,max} in terms of *, +, and ?.\n// Returns a new regexp. Does not edit re. Does not consume reference to re.\n// Caller must Decref return value when done with it.\n// The result will *not* necessarily have the right capturing parens\n// if you call ToString() and re-parse it: (x){2} becomes (x)(x),\n// but in the Regexp* representation, both (x) are marked as $1.\nRegexp* SimplifyWalker::SimplifyRepeat(Regexp* re, int min, int max,\n Regexp::ParseFlags f) {\n // For an empty-width op OR a concatenation or alternation of empty-width\n // ops, cap the repetition count at 1.\n if (IsEmptyOp(re) ||\n ((re->op() == kRegexpConcat ||\n re->op() == kRegexpAlternate) &&\n std::all_of(re->sub(), re->sub() + re->nsub(), IsEmptyOp))) {\n min = std::min(min, 1);\n max = std::min(max, 1);\n }\n\n // x{n,} means at least n matches of x.\n if (max == -1) {\n // Special case: x{0,} is x*\n if (min == 0)\n return Regexp::Star(re->Incref(), f);\n\n // Special case: x{1,} is x+\n if (min == 1)\n return Regexp::Plus(re->Incref(), f);\n\n // General case: x{4,} is xxxx+\n PODArray nre_subs(min);\n for (int i = 0; i < min-1; i++)\n nre_subs[i] = re->Incref();\n nre_subs[min-1] = Regexp::Plus(re->Incref(), f);\n return Regexp::Concat(nre_subs.data(), min, f);\n }\n\n // Special case: (x){0} matches only empty string.\n if (min == 0 && max == 0)\n return new Regexp(kRegexpEmptyMatch, f);\n\n // Special case: x{1} is just x.\n if (min == 1 && max == 1)\n return re->Incref();\n\n // General case: x{n,m} means n copies of x and m copies of x?.\n // The machine will do less work if we nest the final m copies,\n // so that x{2,5} = xx(x(x(x)?)?)?\n\n // Build leading prefix: xx. Capturing only on the last one.\n Regexp* nre = NULL;\n if (min > 0) {\n PODArray nre_subs(min);\n for (int i = 0; i < min; i++)\n nre_subs[i] = re->Incref();\n nre = Regexp::Concat(nre_subs.data(), min, f);\n }\n\n // Build and attach suffix: (x(x(x)?)?)?\n if (max > min) {\n Regexp* suf = Regexp::Quest(re->Incref(), f);\n for (int i = min+1; i < max; i++)\n suf = Regexp::Quest(Concat2(re->Incref(), suf, f), f);\n if (nre == NULL)\n nre = suf;\n else\n nre = Concat2(nre, suf, f);\n }\n\n if (nre == NULL) {\n // Some degenerate case, like min > max, or min < max < 0.\n // This shouldn't happen, because the parser rejects such regexps.\n ABSL_LOG(DFATAL) << \"Malformed repeat of \" << re->ToString()\n << \" min \" << min << \" max \" << max;\n return new Regexp(kRegexpNoMatch, f);\n }\n\n return nre;\n}\n\n// Simplifies a character class.\n// Caller must Decref return value when done with it.\nRegexp* SimplifyWalker::SimplifyCharClass(Regexp* re) {\n CharClass* cc = re->cc();\n\n // Special cases\n if (cc->empty())\n return new Regexp(kRegexpNoMatch, re->parse_flags());\n if (cc->full())\n return new Regexp(kRegexpAnyChar, re->parse_flags());\n\n return re->Incref();\n}\n\n} // namespace re2\n" }, { "path": "re2/sparse_array.h", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_SPARSE_ARRAY_H_\n#define RE2_SPARSE_ARRAY_H_\n\n// DESCRIPTION\n//\n// SparseArray(m) is a map from integers in [0, m) to T values.\n// It requires (sizeof(T)+sizeof(int))*m memory, but it provides\n// fast iteration through the elements in the array and fast clearing\n// of the array. The array has a concept of certain elements being\n// uninitialized (having no value).\n//\n// Insertion and deletion are constant time operations.\n//\n// Allocating the array is a constant time operation\n// when memory allocation is a constant time operation.\n//\n// Clearing the array is a constant time operation (unusual!).\n//\n// Iterating through the array is an O(n) operation, where n\n// is the number of items in the array (not O(m)).\n//\n// The array iterator visits entries in the order they were first\n// inserted into the array. It is safe to add items to the array while\n// using an iterator: the iterator will visit indices added to the array\n// during the iteration, but will not re-visit indices whose values\n// change after visiting. Thus SparseArray can be a convenient\n// implementation of a work queue.\n//\n// The SparseArray implementation is NOT thread-safe. It is up to the\n// caller to make sure only one thread is accessing the array. (Typically\n// these arrays are temporary values and used in situations where speed is\n// important.)\n//\n// The SparseArray interface does not present all the usual STL bells and\n// whistles.\n//\n// Implemented with reference to Briggs & Torczon, An Efficient\n// Representation for Sparse Sets, ACM Letters on Programming Languages\n// and Systems, Volume 2, Issue 1-4 (March-Dec. 1993), pp. 59-69.\n//\n// Briggs & Torczon popularized this technique, but it had been known\n// long before their paper. They point out that Aho, Hopcroft, and\n// Ullman's 1974 Design and Analysis of Computer Algorithms and Bentley's\n// 1986 Programming Pearls both hint at the technique in exercises to the\n// reader (in Aho & Hopcroft, exercise 2.12; in Bentley, column 1\n// exercise 8).\n//\n// Briggs & Torczon describe a sparse set implementation. I have\n// trivially generalized it to create a sparse array (actually the original\n// target of the AHU and Bentley exercises).\n\n// IMPLEMENTATION\n//\n// SparseArray is an array dense_ and an array sparse_ of identical size.\n// At any point, the number of elements in the sparse array is size_.\n//\n// The array dense_ contains the size_ elements in the sparse array (with\n// their indices),\n// in the order that the elements were first inserted. This array is dense:\n// the size_ pairs are dense_[0] through dense_[size_-1].\n//\n// The array sparse_ maps from indices in [0,m) to indices in [0,size_).\n// For indices present in the array, dense_[sparse_[i]].index_ == i.\n// For indices not present in the array, sparse_ can contain any value at all,\n// perhaps outside the range [0, size_) but perhaps not.\n//\n// The lax requirement on sparse_ values makes clearing the array very easy:\n// set size_ to 0. Lookups are slightly more complicated.\n// An index i has a value in the array if and only if:\n// sparse_[i] is in [0, size_) AND\n// dense_[sparse_[i]].index_ == i.\n// If both these properties hold, only then it is safe to refer to\n// dense_[sparse_[i]].value_\n// as the value associated with index i.\n//\n// To insert a new entry, set sparse_[i] to size_,\n// initialize dense_[size_], and then increment size_.\n//\n// To make the sparse array as efficient as possible for non-primitive types,\n// elements may or may not be destroyed when they are deleted from the sparse\n// array through a call to resize(). They immediately become inaccessible, but\n// they are only guaranteed to be destroyed when the SparseArray destructor is\n// called.\n//\n// A moved-from SparseArray will be empty.\n\n#include \n#include \n\n#include \n#include \n#include \n\n#include \"re2/pod_array.h\"\n\n// Doing this simplifies the logic below.\n#ifndef __has_feature\n#define __has_feature(x) 0\n#endif\n\n#if __has_feature(memory_sanitizer)\n#include \n#endif\n\nnamespace re2 {\n\ntemplate\nclass SparseArray {\n public:\n SparseArray();\n explicit SparseArray(int max_size);\n ~SparseArray();\n\n // IndexValue pairs: exposed in SparseArray::iterator.\n class IndexValue;\n\n typedef IndexValue* iterator;\n typedef const IndexValue* const_iterator;\n\n SparseArray(const SparseArray& src);\n SparseArray(SparseArray&& src);\n\n SparseArray& operator=(const SparseArray& src);\n SparseArray& operator=(SparseArray&& src);\n\n // Return the number of entries in the array.\n int size() const {\n return size_;\n }\n\n // Indicate whether the array is empty.\n int empty() const {\n return size_ == 0;\n }\n\n // Iterate over the array.\n iterator begin() {\n return dense_.data();\n }\n iterator end() {\n return dense_.data() + size_;\n }\n\n const_iterator begin() const {\n return dense_.data();\n }\n const_iterator end() const {\n return dense_.data() + size_;\n }\n\n // Change the maximum size of the array.\n // Invalidates all iterators.\n void resize(int new_max_size);\n\n // Return the maximum size of the array.\n // Indices can be in the range [0, max_size).\n int max_size() const {\n if (dense_.data() != NULL)\n return dense_.size();\n else\n return 0;\n }\n\n // Clear the array.\n void clear() {\n size_ = 0;\n }\n\n // Check whether index i is in the array.\n bool has_index(int i) const;\n\n // Comparison function for sorting.\n // Can sort the sparse array so that future iterations\n // will visit indices in increasing order using\n // std::sort(arr.begin(), arr.end(), arr.less);\n static bool less(const IndexValue& a, const IndexValue& b);\n\n public:\n // Set the value at index i to v.\n iterator set(int i, const Value& v) {\n return SetInternal(true, i, v);\n }\n\n // Set the value at new index i to v.\n // Fast but unsafe: only use if has_index(i) is false.\n iterator set_new(int i, const Value& v) {\n return SetInternal(false, i, v);\n }\n\n // Set the value at index i to v.\n // Fast but unsafe: only use if has_index(i) is true.\n iterator set_existing(int i, const Value& v) {\n return SetExistingInternal(i, v);\n }\n\n // Get the value at index i.\n // Fast but unsafe: only use if has_index(i) is true.\n Value& get_existing(int i) {\n assert(has_index(i));\n return dense_[sparse_[i]].value_;\n }\n const Value& get_existing(int i) const {\n assert(has_index(i));\n return dense_[sparse_[i]].value_;\n }\n\n private:\n iterator SetInternal(bool allow_existing, int i, const Value& v) {\n DebugCheckInvariants();\n if (static_cast(i) >= static_cast(max_size())) {\n assert(false && \"illegal index\");\n // Semantically, end() would be better here, but we already know\n // the user did something stupid, so begin() insulates them from\n // dereferencing an invalid pointer.\n return begin();\n }\n if (!allow_existing) {\n assert(!has_index(i));\n create_index(i);\n } else {\n if (!has_index(i))\n create_index(i);\n }\n return SetExistingInternal(i, v);\n }\n\n iterator SetExistingInternal(int i, const Value& v) {\n DebugCheckInvariants();\n assert(has_index(i));\n dense_[sparse_[i]].value_ = v;\n DebugCheckInvariants();\n return dense_.data() + sparse_[i];\n }\n\n // Add the index i to the array.\n // Only use if has_index(i) is known to be false.\n // Since it doesn't set the value associated with i,\n // this function is private, only intended as a helper\n // for other methods.\n void create_index(int i);\n\n // In debug mode, verify that some invariant properties of the class\n // are being maintained. This is called at the end of the constructor\n // and at the beginning and end of all public non-const member functions.\n void DebugCheckInvariants() const;\n\n // Initializes memory for elements [min, max).\n void MaybeInitializeMemory(int min, int max) {\n#if __has_feature(memory_sanitizer)\n __msan_unpoison(sparse_.data() + min, (max - min) * sizeof sparse_[0]);\n#elif defined(RE2_ON_VALGRIND)\n for (int i = min; i < max; i++) {\n sparse_[i] = 0xababababU;\n }\n#endif\n }\n\n int size_ = 0;\n PODArray sparse_;\n PODArray dense_;\n};\n\ntemplate\nSparseArray::SparseArray() = default;\n\ntemplate\nSparseArray::SparseArray(const SparseArray& src)\n : size_(src.size_),\n sparse_(src.max_size()),\n dense_(src.max_size()) {\n std::copy_n(src.sparse_.data(), src.max_size(), sparse_.data());\n std::copy_n(src.dense_.data(), src.max_size(), dense_.data());\n}\n\ntemplate\nSparseArray::SparseArray(SparseArray&& src)\n : size_(src.size_),\n sparse_(std::move(src.sparse_)),\n dense_(std::move(src.dense_)) {\n src.size_ = 0;\n}\n\ntemplate\nSparseArray& SparseArray::operator=(const SparseArray& src) {\n // Construct these first for exception safety.\n PODArray a(src.max_size());\n PODArray b(src.max_size());\n\n size_ = src.size_;\n sparse_ = std::move(a);\n dense_ = std::move(b);\n std::copy_n(src.sparse_.data(), src.max_size(), sparse_.data());\n std::copy_n(src.dense_.data(), src.max_size(), dense_.data());\n return *this;\n}\n\ntemplate\nSparseArray& SparseArray::operator=(SparseArray&& src) {\n size_ = src.size_;\n sparse_ = std::move(src.sparse_);\n dense_ = std::move(src.dense_);\n src.size_ = 0;\n return *this;\n}\n\n// IndexValue pairs: exposed in SparseArray::iterator.\ntemplate\nclass SparseArray::IndexValue {\n public:\n int index() const { return index_; }\n Value& value() { return value_; }\n const Value& value() const { return value_; }\n\n private:\n friend class SparseArray;\n int index_;\n Value value_;\n};\n\n// Change the maximum size of the array.\n// Invalidates all iterators.\ntemplate\nvoid SparseArray::resize(int new_max_size) {\n DebugCheckInvariants();\n if (new_max_size > max_size()) {\n const int old_max_size = max_size();\n\n // Construct these first for exception safety.\n PODArray a(new_max_size);\n PODArray b(new_max_size);\n\n std::copy_n(sparse_.data(), old_max_size, a.data());\n std::copy_n(dense_.data(), old_max_size, b.data());\n\n sparse_ = std::move(a);\n dense_ = std::move(b);\n\n MaybeInitializeMemory(old_max_size, new_max_size);\n }\n if (size_ > new_max_size)\n size_ = new_max_size;\n DebugCheckInvariants();\n}\n\n// Check whether index i is in the array.\ntemplate\nbool SparseArray::has_index(int i) const {\n assert(i >= 0);\n assert(i < max_size());\n if (static_cast(i) >= static_cast(max_size())) {\n return false;\n }\n // Unsigned comparison avoids checking sparse_[i] < 0.\n return (uint32_t)sparse_[i] < (uint32_t)size_ &&\n dense_[sparse_[i]].index_ == i;\n}\n\ntemplate\nvoid SparseArray::create_index(int i) {\n assert(!has_index(i));\n assert(size_ < max_size());\n sparse_[i] = size_;\n dense_[size_].index_ = i;\n size_++;\n}\n\ntemplate SparseArray::SparseArray(int max_size) :\n sparse_(max_size), dense_(max_size) {\n MaybeInitializeMemory(size_, max_size);\n DebugCheckInvariants();\n}\n\ntemplate SparseArray::~SparseArray() {\n DebugCheckInvariants();\n}\n\ntemplate void SparseArray::DebugCheckInvariants() const {\n assert(0 <= size_);\n assert(size_ <= max_size());\n}\n\n// Comparison function for sorting.\ntemplate bool SparseArray::less(const IndexValue& a,\n const IndexValue& b) {\n return a.index_ < b.index_;\n}\n\n} // namespace re2\n\n#endif // RE2_SPARSE_ARRAY_H_\n" }, { "path": "re2/sparse_set.h", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_SPARSE_SET_H_\n#define RE2_SPARSE_SET_H_\n\n// DESCRIPTION\n//\n// SparseSet(m) is a set of integers in [0, m).\n// It requires sizeof(int)*m memory, but it provides\n// fast iteration through the elements in the set and fast clearing\n// of the set.\n//\n// Insertion and deletion are constant time operations.\n//\n// Allocating the set is a constant time operation\n// when memory allocation is a constant time operation.\n//\n// Clearing the set is a constant time operation (unusual!).\n//\n// Iterating through the set is an O(n) operation, where n\n// is the number of items in the set (not O(m)).\n//\n// The set iterator visits entries in the order they were first\n// inserted into the set. It is safe to add items to the set while\n// using an iterator: the iterator will visit indices added to the set\n// during the iteration, but will not re-visit indices whose values\n// change after visiting. Thus SparseSet can be a convenient\n// implementation of a work queue.\n//\n// The SparseSet implementation is NOT thread-safe. It is up to the\n// caller to make sure only one thread is accessing the set. (Typically\n// these sets are temporary values and used in situations where speed is\n// important.)\n//\n// The SparseSet interface does not present all the usual STL bells and\n// whistles.\n//\n// Implemented with reference to Briggs & Torczon, An Efficient\n// Representation for Sparse Sets, ACM Letters on Programming Languages\n// and Systems, Volume 2, Issue 1-4 (March-Dec. 1993), pp. 59-69.\n//\n// This is a specialization of sparse array; see sparse_array.h.\n\n// IMPLEMENTATION\n//\n// See sparse_array.h for implementation details.\n\n#include \n#include \n\n#include \n#include \n#include \n\n#include \"re2/pod_array.h\"\n\n// Doing this simplifies the logic below.\n#ifndef __has_feature\n#define __has_feature(x) 0\n#endif\n\n#if __has_feature(memory_sanitizer)\n#include \n#endif\n\nnamespace re2 {\n\ntemplate\nclass SparseSetT {\n public:\n SparseSetT();\n explicit SparseSetT(int max_size);\n ~SparseSetT();\n\n typedef int* iterator;\n typedef const int* const_iterator;\n\n // Return the number of entries in the set.\n int size() const {\n return size_;\n }\n\n // Indicate whether the set is empty.\n int empty() const {\n return size_ == 0;\n }\n\n // Iterate over the set.\n iterator begin() {\n return dense_.data();\n }\n iterator end() {\n return dense_.data() + size_;\n }\n\n const_iterator begin() const {\n return dense_.data();\n }\n const_iterator end() const {\n return dense_.data() + size_;\n }\n\n // Change the maximum size of the set.\n // Invalidates all iterators.\n void resize(int new_max_size);\n\n // Return the maximum size of the set.\n // Indices can be in the range [0, max_size).\n int max_size() const {\n if (dense_.data() != NULL)\n return dense_.size();\n else\n return 0;\n }\n\n // Clear the set.\n void clear() {\n size_ = 0;\n }\n\n // Check whether index i is in the set.\n bool contains(int i) const;\n\n // Comparison function for sorting.\n // Can sort the sparse set so that future iterations\n // will visit indices in increasing order using\n // std::sort(arr.begin(), arr.end(), arr.less);\n static bool less(int a, int b);\n\n public:\n // Insert index i into the set.\n iterator insert(int i) {\n return InsertInternal(true, i);\n }\n\n // Insert index i into the set.\n // Fast but unsafe: only use if contains(i) is false.\n iterator insert_new(int i) {\n return InsertInternal(false, i);\n }\n\n private:\n iterator InsertInternal(bool allow_existing, int i) {\n DebugCheckInvariants();\n if (static_cast(i) >= static_cast(max_size())) {\n assert(false && \"illegal index\");\n // Semantically, end() would be better here, but we already know\n // the user did something stupid, so begin() insulates them from\n // dereferencing an invalid pointer.\n return begin();\n }\n if (!allow_existing) {\n assert(!contains(i));\n create_index(i);\n } else {\n if (!contains(i))\n create_index(i);\n }\n DebugCheckInvariants();\n return dense_.data() + sparse_[i];\n }\n\n // Add the index i to the set.\n // Only use if contains(i) is known to be false.\n // This function is private, only intended as a helper\n // for other methods.\n void create_index(int i);\n\n // In debug mode, verify that some invariant properties of the class\n // are being maintained. This is called at the end of the constructor\n // and at the beginning and end of all public non-const member functions.\n void DebugCheckInvariants() const;\n\n // Initializes memory for elements [min, max).\n void MaybeInitializeMemory(int min, int max) {\n#if __has_feature(memory_sanitizer)\n __msan_unpoison(sparse_.data() + min, (max - min) * sizeof sparse_[0]);\n#elif defined(RE2_ON_VALGRIND)\n for (int i = min; i < max; i++) {\n sparse_[i] = 0xababababU;\n }\n#endif\n }\n\n int size_ = 0;\n PODArray sparse_;\n PODArray dense_;\n};\n\ntemplate\nSparseSetT::SparseSetT() = default;\n\n// Change the maximum size of the set.\n// Invalidates all iterators.\ntemplate\nvoid SparseSetT::resize(int new_max_size) {\n DebugCheckInvariants();\n if (new_max_size > max_size()) {\n const int old_max_size = max_size();\n\n // Construct these first for exception safety.\n PODArray a(new_max_size);\n PODArray b(new_max_size);\n\n std::copy_n(sparse_.data(), old_max_size, a.data());\n std::copy_n(dense_.data(), old_max_size, b.data());\n\n sparse_ = std::move(a);\n dense_ = std::move(b);\n\n MaybeInitializeMemory(old_max_size, new_max_size);\n }\n if (size_ > new_max_size)\n size_ = new_max_size;\n DebugCheckInvariants();\n}\n\n// Check whether index i is in the set.\ntemplate\nbool SparseSetT::contains(int i) const {\n assert(i >= 0);\n assert(i < max_size());\n if (static_cast(i) >= static_cast(max_size())) {\n return false;\n }\n // Unsigned comparison avoids checking sparse_[i] < 0.\n return (uint32_t)sparse_[i] < (uint32_t)size_ &&\n dense_[sparse_[i]] == i;\n}\n\ntemplate\nvoid SparseSetT::create_index(int i) {\n assert(!contains(i));\n assert(size_ < max_size());\n sparse_[i] = size_;\n dense_[size_] = i;\n size_++;\n}\n\ntemplate SparseSetT::SparseSetT(int max_size) :\n sparse_(max_size), dense_(max_size) {\n MaybeInitializeMemory(size_, max_size);\n DebugCheckInvariants();\n}\n\ntemplate SparseSetT::~SparseSetT() {\n DebugCheckInvariants();\n}\n\ntemplate void SparseSetT::DebugCheckInvariants() const {\n assert(0 <= size_);\n assert(size_ <= max_size());\n}\n\n// Comparison function for sorting.\ntemplate bool SparseSetT::less(int a, int b) {\n return a < b;\n}\n\ntypedef SparseSetT SparseSet;\n\n} // namespace re2\n\n#endif // RE2_SPARSE_SET_H_\n" }, { "path": "re2/stringpiece.h", "content": "// Copyright 2022 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_STRINGPIECE_H_\n#define RE2_STRINGPIECE_H_\n\n#include \"absl/strings/string_view.h\"\n\nnamespace re2 {\n\n// Until RE2 requires C++17 and uses std::string_view, allow users to\n// continue to #include \"re2/stringpiece.h\" and use re2::StringPiece.\nusing StringPiece = absl::string_view;\n\n} // namespace re2\n\n#endif // RE2_STRINGPIECE_H_\n" }, { "path": "re2/testing/backtrack.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Tested by search_test.cc, exhaustive_test.cc, tester.cc\n//\n// Prog::UnsafeSearchBacktrack is a backtracking regular expression search,\n// except that it remembers where it has been, trading a lot of\n// memory for a lot of time. It exists only for testing purposes.\n//\n// Let me repeat that.\n//\n// THIS CODE SHOULD NEVER BE USED IN PRODUCTION:\n// - It uses a ton of memory.\n// - It uses a ton of stack.\n// - It uses ABSL_CHECK() and ABSL_LOG(FATAL).\n// - It implements unanchored search by repeated anchored search.\n//\n// On the other hand, it is very simple and a good reference\n// implementation for the more complicated regexp packages.\n//\n// In BUILD, this file is linked into the \":testing\" library,\n// not the main library, in order to make it harder to pick up\n// accidentally.\n\n#include \n#include \n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/pod_array.h\"\n#include \"re2/prog.h\"\n#include \"re2/regexp.h\"\n\nnamespace re2 {\n\n// Backtracker holds the state for a backtracking search.\n//\n// Excluding the search parameters, the main search state\n// is just the \"capture registers\", which record, for the\n// current execution, the string position at which each\n// parenthesis was passed. cap_[0] and cap_[1] are the\n// left and right parenthesis in $0, cap_[2] and cap_[3] in $1, etc.\n//\n// To avoid infinite loops during backtracking on expressions\n// like (a*)*, the visited_[] bitmap marks the (state, string-position)\n// pairs that have already been explored and are thus not worth\n// re-exploring if we get there via another path. Modern backtracking\n// libraries engineer their program representation differently, to make\n// such infinite loops possible to avoid without keeping a giant visited_\n// bitmap, but visited_ works fine for a reference implementation\n// and it has the nice benefit of making the search run in linear time.\nclass Backtracker {\n public:\n explicit Backtracker(Prog* prog);\n\n bool Search(absl::string_view text, absl::string_view context, bool anchored,\n bool longest, absl::string_view* submatch, int nsubmatch);\n\n private:\n // Explores from instruction id at string position p looking for a match.\n // Returns true if found (so that caller can stop trying other possibilities).\n bool Visit(int id, const char* p);\n\n // Tries instruction id at string position p.\n // Returns true if a match is found.\n bool Try(int id, const char* p);\n\n // Search parameters\n Prog* prog_; // program being run\n absl::string_view text_; // text being searched\n absl::string_view context_; // greater context of text being searched\n bool anchored_; // whether search is anchored at text.begin()\n bool longest_; // whether search wants leftmost-longest match\n bool endmatch_; // whether search must end at text.end()\n absl::string_view* submatch_; // submatches to fill in\n int nsubmatch_; // # of submatches to fill in\n\n // Search state\n const char* cap_[64]; // capture registers\n PODArray visited_; // bitmap: (Inst*, char*) pairs visited\n\n Backtracker(const Backtracker&) = delete;\n Backtracker& operator=(const Backtracker&) = delete;\n};\n\nBacktracker::Backtracker(Prog* prog)\n : prog_(prog),\n anchored_(false),\n longest_(false),\n endmatch_(false),\n submatch_(NULL),\n nsubmatch_(0) {\n}\n\n// Runs a backtracking search.\nbool Backtracker::Search(absl::string_view text, absl::string_view context,\n bool anchored, bool longest,\n absl::string_view* submatch, int nsubmatch) {\n text_ = text;\n context_ = context;\n if (context_.data() == NULL)\n context_ = text;\n if (prog_->anchor_start() && BeginPtr(text) > BeginPtr(context_))\n return false;\n if (prog_->anchor_end() && EndPtr(text) < EndPtr(context_))\n return false;\n anchored_ = anchored | prog_->anchor_start();\n longest_ = longest | prog_->anchor_end();\n endmatch_ = prog_->anchor_end();\n submatch_ = submatch;\n nsubmatch_ = nsubmatch;\n ABSL_CHECK_LT(2*nsubmatch_, static_cast(ABSL_ARRAYSIZE(cap_)));\n memset(cap_, 0, sizeof cap_);\n\n // We use submatch_[0] for our own bookkeeping,\n // so it had better exist.\n absl::string_view sp0;\n if (nsubmatch < 1) {\n submatch_ = &sp0;\n nsubmatch_ = 1;\n }\n submatch_[0] = absl::string_view();\n\n // Allocate new visited_ bitmap -- size is proportional\n // to text, so have to reallocate on each call to Search.\n int nvisited = prog_->size() * static_cast(text.size()+1);\n nvisited = (nvisited + 31) / 32;\n visited_ = PODArray(nvisited);\n memset(visited_.data(), 0, nvisited*sizeof visited_[0]);\n\n // Anchored search must start at text.begin().\n if (anchored_) {\n cap_[0] = text.data();\n return Visit(prog_->start(), text.data());\n }\n\n // Unanchored search, starting from each possible text position.\n // Notice that we have to try the empty string at the end of\n // the text, so the loop condition is p <= text.end(), not p < text.end().\n for (const char* p = text.data(); p <= text.data() + text.size(); p++) {\n cap_[0] = p;\n if (Visit(prog_->start(), p)) // Match must be leftmost; done.\n return true;\n // Avoid invoking undefined behavior (arithmetic on a null pointer)\n // by simply not continuing the loop.\n if (p == NULL)\n break;\n }\n return false;\n}\n\n// Explores from instruction id at string position p looking for a match.\n// Return true if found (so that caller can stop trying other possibilities).\nbool Backtracker::Visit(int id, const char* p) {\n // Check bitmap. If we've already explored from here,\n // either it didn't match or it did but we're hoping for a better match.\n // Either way, don't go down that road again.\n ABSL_CHECK(p <= text_.data() + text_.size());\n int n = id * static_cast(text_.size()+1) +\n static_cast(p-text_.data());\n ABSL_CHECK_LT(n/32, visited_.size());\n if (visited_[n/32] & (1 << (n&31)))\n return false;\n visited_[n/32] |= 1 << (n&31);\n\n Prog::Inst* ip = prog_->inst(id);\n if (Try(id, p)) {\n if (longest_ && !ip->last())\n Visit(id+1, p);\n return true;\n }\n if (!ip->last())\n return Visit(id+1, p);\n return false;\n}\n\n// Tries instruction id at string position p.\n// Returns true if a match is found.\nbool Backtracker::Try(int id, const char* p) {\n // Pick out byte at current position. If at end of string,\n // have to explore in hope of finishing a match. Use impossible byte -1.\n int c = -1;\n if (p < text_.data() + text_.size())\n c = *p & 0xFF;\n\n Prog::Inst* ip = prog_->inst(id);\n switch (ip->opcode()) {\n default:\n ABSL_LOG(FATAL) << \"Unexpected opcode: \" << ip->opcode();\n return false; // not reached\n\n case kInstAltMatch:\n // Ignored.\n return false;\n\n case kInstByteRange:\n if (ip->Matches(c))\n return Visit(ip->out(), p+1);\n return false;\n\n case kInstCapture:\n if (0 <= ip->cap() &&\n ip->cap() < static_cast(ABSL_ARRAYSIZE(cap_))) {\n // Capture p to register, but save old value.\n const char* q = cap_[ip->cap()];\n cap_[ip->cap()] = p;\n bool ret = Visit(ip->out(), p);\n // Restore old value as we backtrack.\n cap_[ip->cap()] = q;\n return ret;\n }\n return Visit(ip->out(), p);\n\n case kInstEmptyWidth:\n if (ip->empty() & ~Prog::EmptyFlags(context_, p))\n return false;\n return Visit(ip->out(), p);\n\n case kInstNop:\n return Visit(ip->out(), p);\n\n case kInstMatch:\n // We found a match. If it's the best so far, record the\n // parameters in the caller's submatch_ array.\n if (endmatch_ && p != context_.data() + context_.size())\n return false;\n cap_[1] = p;\n if (submatch_[0].data() == NULL ||\n (longest_ && p > submatch_[0].data() + submatch_[0].size())) {\n // First match so far - or better match.\n for (int i = 0; i < nsubmatch_; i++)\n submatch_[i] = absl::string_view(\n cap_[2 * i], static_cast(cap_[2 * i + 1] - cap_[2 * i]));\n }\n return true;\n\n case kInstFail:\n return false;\n }\n}\n\n// Runs a backtracking search.\nbool Prog::UnsafeSearchBacktrack(absl::string_view text,\n absl::string_view context, Anchor anchor,\n MatchKind kind, absl::string_view* match,\n int nmatch) {\n // If full match, we ask for an anchored longest match\n // and then check that match[0] == text.\n // So make sure match[0] exists.\n absl::string_view sp0;\n if (kind == kFullMatch) {\n anchor = kAnchored;\n if (nmatch < 1) {\n match = &sp0;\n nmatch = 1;\n }\n }\n\n // Run the search.\n Backtracker b(this);\n bool anchored = anchor == kAnchored;\n bool longest = kind != kFirstMatch;\n if (!b.Search(text, context, anchored, longest, match, nmatch))\n return false;\n if (kind == kFullMatch && EndPtr(match[0]) != EndPtr(text))\n return false;\n return true;\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/charclass_test.cc", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Test character class manipulations.\n\n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/strings/str_format.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/regexp.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\nstruct CCTest {\n struct {\n Rune lo;\n Rune hi;\n } add[10];\n int remove;\n struct {\n Rune lo;\n Rune hi;\n } final[10];\n};\n\nstatic CCTest tests[] = {\n { { { 10, 20 }, {-1} }, -1,\n { { 10, 20 }, {-1} } },\n\n { { { 10, 20 }, { 20, 30 }, {-1} }, -1,\n { { 10, 30 }, {-1} } },\n\n { { { 10, 20 }, { 30, 40 }, { 20, 30 }, {-1} }, -1,\n { { 10, 40 }, {-1} } },\n\n { { { 0, 50 }, { 20, 30 }, {-1} }, -1,\n { { 0, 50 }, {-1} } },\n\n { { { 10, 11 }, { 13, 14 }, { 16, 17 }, { 19, 20 }, { 22, 23 }, {-1} }, -1,\n { { 10, 11 }, { 13, 14 }, { 16, 17 }, { 19, 20 }, { 22, 23 }, {-1} } },\n\n { { { 13, 14 }, { 10, 11 }, { 22, 23 }, { 19, 20 }, { 16, 17 }, {-1} }, -1,\n { { 10, 11 }, { 13, 14 }, { 16, 17 }, { 19, 20 }, { 22, 23 }, {-1} } },\n\n { { { 13, 14 }, { 10, 11 }, { 22, 23 }, { 19, 20 }, { 16, 17 }, {-1} }, -1,\n { { 10, 11 }, { 13, 14 }, { 16, 17 }, { 19, 20 }, { 22, 23 }, {-1} } },\n\n { { { 13, 14 }, { 10, 11 }, { 22, 23 }, { 19, 20 }, { 16, 17 }, { 5, 25 }, {-1} }, -1,\n { { 5, 25 }, {-1} } },\n\n { { { 13, 14 }, { 10, 11 }, { 22, 23 }, { 19, 20 }, { 16, 17 }, { 12, 21 }, {-1} }, -1,\n { { 10, 23 }, {-1} } },\n\n // These check boundary cases during negation.\n { { { 0, Runemax }, {-1} }, -1,\n { { 0, Runemax }, {-1} } },\n\n { { { 0, 50 }, {-1} }, -1,\n { { 0, 50 }, {-1} } },\n\n { { { 50, Runemax }, {-1} }, -1,\n { { 50, Runemax }, {-1} } },\n\n // Check RemoveAbove.\n { { { 50, Runemax }, {-1} }, 255,\n { { 50, 255 }, {-1} } },\n\n { { { 50, Runemax }, {-1} }, 65535,\n { { 50, 65535 }, {-1} } },\n\n { { { 50, Runemax }, {-1} }, Runemax,\n { { 50, Runemax }, {-1} } },\n\n { { { 50, 60 }, { 250, 260 }, { 350, 360 }, {-1} }, 255,\n { { 50, 60 }, { 250, 255 }, {-1} } },\n\n { { { 50, 60 }, {-1} }, 255,\n { { 50, 60 }, {-1} } },\n\n { { { 350, 360 }, {-1} }, 255,\n { {-1} } },\n\n { { {-1} }, 255,\n { {-1} } },\n};\n\ntemplate \nstatic void Broke(const char *desc, const CCTest* t, CharClass* cc) {\n if (t == NULL) {\n absl::PrintF(\"\\t%s:\", desc);\n } else {\n absl::PrintF(\"\\n\");\n absl::PrintF(\"CharClass added: [%s]\", desc);\n for (int k = 0; t->add[k].lo >= 0; k++)\n absl::PrintF(\" %d-%d\", t->add[k].lo, t->add[k].hi);\n absl::PrintF(\"\\n\");\n if (t->remove >= 0)\n absl::PrintF(\"Removed > %d\\n\", t->remove);\n absl::PrintF(\"\\twant:\");\n for (int k = 0; t->final[k].lo >= 0; k++)\n absl::PrintF(\" %d-%d\", t->final[k].lo, t->final[k].hi);\n absl::PrintF(\"\\n\");\n absl::PrintF(\"\\thave:\");\n }\n\n for (typename CharClass::iterator it = cc->begin(); it != cc->end(); ++it)\n absl::PrintF(\" %d-%d\", it->lo, it->hi);\n absl::PrintF(\"\\n\");\n}\n\nbool ShouldContain(CCTest *t, int x) {\n for (int j = 0; t->final[j].lo >= 0; j++)\n if (t->final[j].lo <= x && x <= t->final[j].hi)\n return true;\n return false;\n}\n\n// Helpers to make templated CorrectCC work with both CharClass and CharClassBuilder.\n\nCharClass* Negate(CharClass *cc) {\n return cc->Negate();\n}\n\nvoid Delete(CharClass* cc) {\n cc->Delete();\n}\n\nCharClassBuilder* Negate(CharClassBuilder* cc) {\n CharClassBuilder* ncc = cc->Copy();\n ncc->Negate();\n return ncc;\n}\n\nvoid Delete(CharClassBuilder* cc) {\n delete cc;\n}\n\ntemplate \nbool CorrectCC(CharClass *cc, CCTest *t, const char *desc) {\n typename CharClass::iterator it = cc->begin();\n int size = 0;\n for (int j = 0; t->final[j].lo >= 0; j++, ++it) {\n if (it == cc->end() ||\n it->lo != t->final[j].lo ||\n it->hi != t->final[j].hi) {\n Broke(desc, t, cc);\n return false;\n }\n size += it->hi - it->lo + 1;\n }\n if (it != cc->end()) {\n Broke(desc, t, cc);\n return false;\n }\n if (cc->size() != size) {\n Broke(desc, t, cc);\n absl::PrintF(\"wrong size: want %d have %d\\n\", size, cc->size());\n return false;\n }\n\n for (int j = 0; j < 101; j++) {\n if (j == 100)\n j = Runemax;\n if (ShouldContain(t, j) != cc->Contains(j)) {\n Broke(desc, t, cc);\n absl::PrintF(\"want contains(%d)=%d, got %d\\n\",\n j, ShouldContain(t, j), cc->Contains(j));\n return false;\n }\n }\n\n CharClass* ncc = Negate(cc);\n for (int j = 0; j < 101; j++) {\n if (j == 100)\n j = Runemax;\n if (ShouldContain(t, j) == ncc->Contains(j)) {\n Broke(desc, t, cc);\n Broke(\"ncc\", NULL, ncc);\n absl::PrintF(\"want ncc contains(%d)!=%d, got %d\\n\",\n j, ShouldContain(t, j), ncc->Contains(j));\n Delete(ncc);\n return false;\n }\n if (ncc->size() != Runemax+1 - cc->size()) {\n Broke(desc, t, cc);\n Broke(\"ncc\", NULL, ncc);\n absl::PrintF(\"ncc size should be %d is %d\\n\",\n Runemax+1 - cc->size(), ncc->size());\n Delete(ncc);\n return false;\n }\n }\n Delete(ncc);\n return true;\n}\n\nTEST(TestCharClassBuilder, Adds) {\n int nfail = 0;\n for (size_t i = 0; i < ABSL_ARRAYSIZE(tests); i++) {\n CharClassBuilder ccb;\n CCTest* t = &tests[i];\n for (int j = 0; t->add[j].lo >= 0; j++)\n ccb.AddRange(t->add[j].lo, t->add[j].hi);\n if (t->remove >= 0)\n ccb.RemoveAbove(t->remove);\n if (!CorrectCC(&ccb, t, \"before copy (CharClassBuilder)\"))\n nfail++;\n CharClass* cc = ccb.GetCharClass();\n if (!CorrectCC(cc, t, \"before copy (CharClass)\"))\n nfail++;\n cc->Delete();\n\n CharClassBuilder *ccb1 = ccb.Copy();\n if (!CorrectCC(ccb1, t, \"after copy (CharClassBuilder)\"))\n nfail++;\n cc = ccb.GetCharClass();\n if (!CorrectCC(cc, t, \"after copy (CharClass)\"))\n nfail++;\n cc->Delete();\n delete ccb1;\n }\n EXPECT_EQ(nfail, 0);\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/compile_test.cc", "content": "// Copyright 2007 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Test prog.cc, compile.cc\n\n#include \n\n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/string_view.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/prog.h\"\n#include \"re2/regexp.h\"\n\nnamespace re2 {\n\n// Simple input/output tests checking that\n// the regexp compiles to the expected code.\n// These are just to sanity check the basic implementation.\n// The real confidence tests happen by testing the NFA/DFA\n// that run the compiled code.\n\nstruct Test {\n const char* regexp;\n const char* code;\n};\n\nstatic Test tests[] = {\n { \"a\",\n \"3. byte [61-61] 0 -> 4\\n\"\n \"4. match! 0\\n\" },\n { \"ab\",\n \"3. byte [61-61] 0 -> 4\\n\"\n \"4. byte [62-62] 0 -> 5\\n\"\n \"5. match! 0\\n\" },\n { \"a|c\",\n \"3+ byte [61-61] 0 -> 5\\n\"\n \"4. byte [63-63] 0 -> 5\\n\"\n \"5. match! 0\\n\" },\n { \"a|b\",\n \"3. byte [61-62] 0 -> 4\\n\"\n \"4. match! 0\\n\" },\n { \"[ab]\",\n \"3. byte [61-62] 0 -> 4\\n\"\n \"4. match! 0\\n\" },\n { \"a+\",\n \"3. byte [61-61] 0 -> 4\\n\"\n \"4+ nop -> 3\\n\"\n \"5. match! 0\\n\" },\n { \"a+?\",\n \"3. byte [61-61] 0 -> 4\\n\"\n \"4+ match! 0\\n\"\n \"5. nop -> 3\\n\" },\n { \"a*\",\n \"3+ byte [61-61] 1 -> 3\\n\"\n \"4. match! 0\\n\" },\n { \"a*?\",\n \"3+ match! 0\\n\"\n \"4. byte [61-61] 0 -> 3\\n\" },\n { \"a?\",\n \"3+ byte [61-61] 1 -> 5\\n\"\n \"4. nop -> 5\\n\"\n \"5. match! 0\\n\" },\n { \"a??\",\n \"3+ nop -> 5\\n\"\n \"4. byte [61-61] 0 -> 5\\n\"\n \"5. match! 0\\n\" },\n { \"a{4}\",\n \"3. byte [61-61] 0 -> 4\\n\"\n \"4. byte [61-61] 0 -> 5\\n\"\n \"5. byte [61-61] 0 -> 6\\n\"\n \"6. byte [61-61] 0 -> 7\\n\"\n \"7. match! 0\\n\" },\n { \"(a)\",\n \"3. capture 2 -> 4\\n\"\n \"4. byte [61-61] 0 -> 5\\n\"\n \"5. capture 3 -> 6\\n\"\n \"6. match! 0\\n\" },\n { \"(?:a)\",\n \"3. byte [61-61] 0 -> 4\\n\"\n \"4. match! 0\\n\" },\n { \"\",\n \"3. match! 0\\n\" },\n { \".\",\n \"3+ byte [00-09] 0 -> 5\\n\"\n \"4. byte [0b-ff] 0 -> 5\\n\"\n \"5. match! 0\\n\" },\n { \"[^ab]\",\n \"3+ byte [00-09] 0 -> 6\\n\"\n \"4+ byte [0b-60] 0 -> 6\\n\"\n \"5. byte [63-ff] 0 -> 6\\n\"\n \"6. match! 0\\n\" },\n { \"[Aa]\",\n \"3. byte/i [61-61] 0 -> 4\\n\"\n \"4. match! 0\\n\" },\n { \"\\\\C+\",\n \"3. byte [00-ff] 0 -> 4\\n\"\n \"4+ altmatch -> 5 | 6\\n\"\n \"5+ nop -> 3\\n\"\n \"6. match! 0\\n\" },\n { \"\\\\C*\",\n \"3+ altmatch -> 4 | 5\\n\"\n \"4+ byte [00-ff] 1 -> 3\\n\"\n \"5. match! 0\\n\" },\n { \"\\\\C?\",\n \"3+ byte [00-ff] 1 -> 5\\n\"\n \"4. nop -> 5\\n\"\n \"5. match! 0\\n\" },\n // Issue 20992936\n { \"[[-`]\",\n \"3. byte [5b-60] 0 -> 4\\n\"\n \"4. match! 0\\n\" },\n // Issue 310\n { \"(?:|a)*\",\n \"3+ nop -> 7\\n\"\n \"4. nop -> 9\\n\"\n \"5+ nop -> 7\\n\"\n \"6. nop -> 9\\n\"\n \"7+ nop -> 5\\n\"\n \"8. byte [61-61] 0 -> 5\\n\"\n \"9. match! 0\\n\" },\n { \"(?:|a)+\",\n \"3+ nop -> 5\\n\"\n \"4. byte [61-61] 0 -> 5\\n\"\n \"5+ nop -> 3\\n\"\n \"6. match! 0\\n\" },\n};\n\nTEST(TestRegexpCompileToProg, Simple) {\n int failed = 0;\n for (size_t i = 0; i < ABSL_ARRAYSIZE(tests); i++) {\n const re2::Test& t = tests[i];\n Regexp* re = Regexp::Parse(t.regexp, Regexp::PerlX|Regexp::Latin1, NULL);\n if (re == NULL) {\n ABSL_LOG(ERROR) << \"Cannot parse: \" << t.regexp;\n failed++;\n continue;\n }\n Prog* prog = re->CompileToProg(0);\n if (prog == NULL) {\n ABSL_LOG(ERROR) << \"Cannot compile: \" << t.regexp;\n re->Decref();\n failed++;\n continue;\n }\n ASSERT_TRUE(re->CompileToProg(1) == NULL);\n std::string s = prog->Dump();\n if (s != t.code) {\n ABSL_LOG(ERROR) << \"Incorrect compiled code for: \" << t.regexp;\n ABSL_LOG(ERROR) << \"Want:\\n\" << t.code;\n ABSL_LOG(ERROR) << \"Got:\\n\" << s;\n failed++;\n }\n delete prog;\n re->Decref();\n }\n EXPECT_EQ(failed, 0);\n}\n\nstatic void DumpByteMap(absl::string_view pattern, Regexp::ParseFlags flags,\n std::string* bytemap) {\n Regexp* re = Regexp::Parse(pattern, flags, NULL);\n EXPECT_TRUE(re != NULL);\n\n {\n Prog* prog = re->CompileToProg(0);\n EXPECT_TRUE(prog != NULL);\n *bytemap = prog->DumpByteMap();\n delete prog;\n }\n\n {\n Prog* prog = re->CompileToReverseProg(0);\n EXPECT_TRUE(prog != NULL);\n EXPECT_EQ(*bytemap, prog->DumpByteMap());\n delete prog;\n }\n\n re->Decref();\n}\n\nTEST(TestCompile, Latin1Ranges) {\n // The distinct byte ranges involved in the Latin-1 dot ([^\\n]).\n\n std::string bytemap;\n\n DumpByteMap(\".\", Regexp::PerlX|Regexp::Latin1, &bytemap);\n EXPECT_EQ(\"[00-09] -> 0\\n\"\n \"[0a-0a] -> 1\\n\"\n \"[0b-ff] -> 0\\n\",\n bytemap);\n}\n\nTEST(TestCompile, OtherByteMapTests) {\n std::string bytemap;\n\n // Test that \"absent\" ranges are mapped to the same byte class.\n DumpByteMap(\"[0-9A-Fa-f]+\", Regexp::PerlX|Regexp::Latin1, &bytemap);\n EXPECT_EQ(\"[00-2f] -> 0\\n\"\n \"[30-39] -> 1\\n\"\n \"[3a-40] -> 0\\n\"\n \"[41-46] -> 1\\n\"\n \"[47-60] -> 0\\n\"\n \"[61-66] -> 1\\n\"\n \"[67-ff] -> 0\\n\",\n bytemap);\n\n // Test the byte classes for \\b.\n DumpByteMap(\"\\\\b\", Regexp::LikePerl|Regexp::Latin1, &bytemap);\n EXPECT_EQ(\"[00-2f] -> 0\\n\"\n \"[30-39] -> 1\\n\"\n \"[3a-40] -> 0\\n\"\n \"[41-5a] -> 1\\n\"\n \"[5b-5e] -> 0\\n\"\n \"[5f-5f] -> 1\\n\"\n \"[60-60] -> 0\\n\"\n \"[61-7a] -> 1\\n\"\n \"[7b-ff] -> 0\\n\",\n bytemap);\n\n // Bug in the ASCII case-folding optimization created too many byte classes.\n DumpByteMap(\"[^_]\", Regexp::LikePerl|Regexp::Latin1, &bytemap);\n EXPECT_EQ(\"[00-5e] -> 0\\n\"\n \"[5f-5f] -> 1\\n\"\n \"[60-ff] -> 0\\n\",\n bytemap);\n}\n\nTEST(TestCompile, UTF8Ranges) {\n // The distinct byte ranges involved in the UTF-8 dot ([^\\n]).\n // Once, erroneously split between 0x3f and 0x40 because it is\n // a 6-bit boundary.\n\n std::string bytemap;\n\n DumpByteMap(\".\", Regexp::PerlX, &bytemap);\n EXPECT_EQ(\"[00-09] -> 0\\n\"\n \"[0a-0a] -> 1\\n\"\n \"[0b-7f] -> 0\\n\"\n \"[80-bf] -> 2\\n\"\n \"[c0-c1] -> 1\\n\"\n \"[c2-df] -> 3\\n\"\n \"[e0-ef] -> 4\\n\"\n \"[f0-f4] -> 5\\n\"\n \"[f5-ff] -> 1\\n\",\n bytemap);\n}\n\nTEST(TestCompile, InsufficientMemory) {\n Regexp* re = Regexp::Parse(\n \"^(?P[^\\\\s]+)\\\\s+(?P[^\\\\s]+)\\\\s+(?P.+)$\",\n Regexp::LikePerl, NULL);\n EXPECT_TRUE(re != NULL);\n Prog* prog = re->CompileToProg(850);\n // If the memory budget has been exhausted, compilation should fail\n // and return NULL instead of trying to do anything with NoMatch().\n EXPECT_TRUE(prog == NULL);\n re->Decref();\n}\n\nstatic void Dump(absl::string_view pattern, Regexp::ParseFlags flags,\n std::string* forward, std::string* reverse) {\n Regexp* re = Regexp::Parse(pattern, flags, NULL);\n EXPECT_TRUE(re != NULL);\n\n if (forward != NULL) {\n Prog* prog = re->CompileToProg(0);\n EXPECT_TRUE(prog != NULL);\n *forward = prog->Dump();\n delete prog;\n }\n\n if (reverse != NULL) {\n Prog* prog = re->CompileToReverseProg(0);\n EXPECT_TRUE(prog != NULL);\n *reverse = prog->Dump();\n delete prog;\n }\n\n re->Decref();\n}\n\nTEST(TestCompile, Bug26705922) {\n // Bug in the compiler caused inefficient bytecode to be generated for Unicode\n // groups: common suffixes were cached, but common prefixes were not factored.\n\n std::string forward, reverse;\n\n Dump(\"[\\\\x{10000}\\\\x{10010}]\", Regexp::LikePerl, &forward, &reverse);\n EXPECT_EQ(\"3. byte [f0-f0] 0 -> 4\\n\"\n \"4. byte [90-90] 0 -> 5\\n\"\n \"5. byte [80-80] 0 -> 6\\n\"\n \"6+ byte [80-80] 0 -> 8\\n\"\n \"7. byte [90-90] 0 -> 8\\n\"\n \"8. match! 0\\n\",\n forward);\n EXPECT_EQ(\"3+ byte [80-80] 0 -> 5\\n\"\n \"4. byte [90-90] 0 -> 5\\n\"\n \"5. byte [80-80] 0 -> 6\\n\"\n \"6. byte [90-90] 0 -> 7\\n\"\n \"7. byte [f0-f0] 0 -> 8\\n\"\n \"8. match! 0\\n\",\n reverse);\n\n Dump(\"[\\\\x{8000}-\\\\x{10FFF}]\", Regexp::LikePerl, &forward, &reverse);\n EXPECT_EQ(\"3+ byte [e8-ef] 0 -> 5\\n\"\n \"4. byte [f0-f0] 0 -> 8\\n\"\n \"5. byte [80-bf] 0 -> 6\\n\"\n \"6. byte [80-bf] 0 -> 7\\n\"\n \"7. match! 0\\n\"\n \"8. byte [90-90] 0 -> 5\\n\",\n forward);\n EXPECT_EQ(\"3. byte [80-bf] 0 -> 4\\n\"\n \"4. byte [80-bf] 0 -> 5\\n\"\n \"5+ byte [e8-ef] 0 -> 7\\n\"\n \"6. byte [90-90] 0 -> 8\\n\"\n \"7. match! 0\\n\"\n \"8. byte [f0-f0] 0 -> 7\\n\",\n reverse);\n\n Dump(\"[\\\\x{80}-\\\\x{10FFFF}]\", Regexp::LikePerl, &forward, &reverse);\n EXPECT_EQ(\"3+ byte [c2-df] 0 -> 6\\n\"\n \"4+ byte [e0-ef] 0 -> 8\\n\"\n \"5. byte [f0-f4] 0 -> 9\\n\"\n \"6. byte [80-bf] 0 -> 7\\n\"\n \"7. match! 0\\n\"\n \"8. byte [80-bf] 0 -> 6\\n\"\n \"9. byte [80-bf] 0 -> 8\\n\",\n forward);\n EXPECT_EQ(\"3. byte [80-bf] 0 -> 4\\n\"\n \"4+ byte [c2-df] 0 -> 6\\n\"\n \"5. byte [80-bf] 0 -> 7\\n\"\n \"6. match! 0\\n\"\n \"7+ byte [e0-ef] 0 -> 6\\n\"\n \"8. byte [80-bf] 0 -> 9\\n\"\n \"9. byte [f0-f4] 0 -> 6\\n\",\n reverse);\n}\n\nTEST(TestCompile, Bug35237384) {\n // Bug in the compiler caused inefficient bytecode to be generated for\n // nested nullable subexpressions.\n\n std::string forward;\n\n Dump(\"a**{3,}\", Regexp::Latin1|Regexp::NeverCapture, &forward, NULL);\n EXPECT_EQ(\"3+ byte [61-61] 1 -> 3\\n\"\n \"4. nop -> 5\\n\"\n \"5+ byte [61-61] 1 -> 5\\n\"\n \"6. nop -> 7\\n\"\n \"7+ byte [61-61] 1 -> 7\\n\"\n \"8. match! 0\\n\",\n forward);\n\n Dump(\"(a*|b*)*{3,}\", Regexp::Latin1|Regexp::NeverCapture, &forward, NULL);\n EXPECT_EQ(\"3+ nop -> 28\\n\"\n \"4. nop -> 30\\n\"\n \"5+ byte [61-61] 1 -> 5\\n\"\n \"6. nop -> 32\\n\"\n \"7+ byte [61-61] 1 -> 7\\n\"\n \"8. nop -> 26\\n\"\n \"9+ byte [61-61] 1 -> 9\\n\"\n \"10. nop -> 20\\n\"\n \"11+ byte [62-62] 1 -> 11\\n\"\n \"12. nop -> 20\\n\"\n \"13+ byte [62-62] 1 -> 13\\n\"\n \"14. nop -> 26\\n\"\n \"15+ byte [62-62] 1 -> 15\\n\"\n \"16. nop -> 32\\n\"\n \"17+ nop -> 9\\n\"\n \"18. nop -> 11\\n\"\n \"19. match! 0\\n\"\n \"20+ nop -> 17\\n\"\n \"21. nop -> 19\\n\"\n \"22+ nop -> 7\\n\"\n \"23. nop -> 13\\n\"\n \"24+ nop -> 17\\n\"\n \"25. nop -> 19\\n\"\n \"26+ nop -> 22\\n\"\n \"27. nop -> 24\\n\"\n \"28+ nop -> 5\\n\"\n \"29. nop -> 15\\n\"\n \"30+ nop -> 22\\n\"\n \"31. nop -> 24\\n\"\n \"32+ nop -> 28\\n\"\n \"33. nop -> 30\\n\",\n forward);\n\n Dump(\"((|S.+)+|(|S.+)+|){2}\", Regexp::Latin1|Regexp::NeverCapture, &forward, NULL);\n EXPECT_EQ(\"3+ nop -> 36\\n\"\n \"4+ nop -> 31\\n\"\n \"5. nop -> 33\\n\"\n \"6+ byte [00-09] 0 -> 8\\n\"\n \"7. byte [0b-ff] 0 -> 8\\n\"\n \"8+ nop -> 6\\n\"\n \"9+ nop -> 29\\n\"\n \"10. nop -> 28\\n\"\n \"11+ byte [00-09] 0 -> 13\\n\"\n \"12. byte [0b-ff] 0 -> 13\\n\"\n \"13+ nop -> 11\\n\"\n \"14+ nop -> 26\\n\"\n \"15. nop -> 28\\n\"\n \"16+ byte [00-09] 0 -> 18\\n\"\n \"17. byte [0b-ff] 0 -> 18\\n\"\n \"18+ nop -> 16\\n\"\n \"19+ nop -> 36\\n\"\n \"20. nop -> 33\\n\"\n \"21+ byte [00-09] 0 -> 23\\n\"\n \"22. byte [0b-ff] 0 -> 23\\n\"\n \"23+ nop -> 21\\n\"\n \"24+ nop -> 31\\n\"\n \"25. nop -> 33\\n\"\n \"26+ nop -> 28\\n\"\n \"27. byte [53-53] 0 -> 11\\n\"\n \"28. match! 0\\n\"\n \"29+ nop -> 28\\n\"\n \"30. byte [53-53] 0 -> 6\\n\"\n \"31+ nop -> 33\\n\"\n \"32. byte [53-53] 0 -> 21\\n\"\n \"33+ nop -> 29\\n\"\n \"34+ nop -> 26\\n\"\n \"35. nop -> 28\\n\"\n \"36+ nop -> 33\\n\"\n \"37. byte [53-53] 0 -> 16\\n\",\n forward);\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/dfa_test.cc", "content": "// Copyright 2006-2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \n#include \n\n#include \n#include \n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/flags/flag.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/prog.h\"\n#include \"re2/re2.h\"\n#include \"re2/regexp.h\"\n#include \"re2/testing/string_generator.h\"\n#include \"util/malloc_counter.h\"\n\nstatic const bool UsingMallocCounter = false;\n\nABSL_FLAG(int, size, 8, \"log2(number of DFA nodes)\");\nABSL_FLAG(int, repeat, 2, \"Repetition count.\");\nABSL_FLAG(int, threads, 4, \"number of threads\");\n\nnamespace re2 {\n\nstatic int state_cache_resets = 0;\nstatic int search_failures = 0;\n\nstruct SetHooks {\n SetHooks() {\n hooks::SetDFAStateCacheResetHook([](const hooks::DFAStateCacheReset&) {\n ++state_cache_resets;\n });\n hooks::SetDFASearchFailureHook([](const hooks::DFASearchFailure&) {\n ++search_failures;\n });\n }\n} set_hooks;\n\n// Check that multithreaded access to DFA class works.\n\n// Helper function: builds entire DFA for prog.\nstatic void DoBuild(Prog* prog) {\n ASSERT_TRUE(prog->BuildEntireDFA(Prog::kFirstMatch, nullptr));\n}\n\nTEST(Multithreaded, BuildEntireDFA) {\n // Create regexp with 2^FLAGS_size states in DFA.\n std::string s = \"a\";\n for (int i = 0; i < absl::GetFlag(FLAGS_size); i++)\n s += \"[ab]\";\n s += \"b\";\n Regexp* re = Regexp::Parse(s, Regexp::LikePerl, NULL);\n ASSERT_TRUE(re != NULL);\n\n // Check that single-threaded code works.\n {\n Prog* prog = re->CompileToProg(0);\n ASSERT_TRUE(prog != NULL);\n\n std::thread t(DoBuild, prog);\n t.join();\n\n delete prog;\n }\n\n // Build the DFA simultaneously in a bunch of threads.\n for (int i = 0; i < absl::GetFlag(FLAGS_repeat); i++) {\n Prog* prog = re->CompileToProg(0);\n ASSERT_TRUE(prog != NULL);\n\n std::vector threads;\n for (int j = 0; j < absl::GetFlag(FLAGS_threads); j++)\n threads.emplace_back(DoBuild, prog);\n for (int j = 0; j < absl::GetFlag(FLAGS_threads); j++)\n threads[j].join();\n\n // One more compile, to make sure everything is okay.\n prog->BuildEntireDFA(Prog::kFirstMatch, nullptr);\n delete prog;\n }\n\n re->Decref();\n}\n\n// Check that DFA size requirements are followed.\n// BuildEntireDFA will, like SearchDFA, stop building out\n// the DFA once the memory limits are reached.\nTEST(SingleThreaded, BuildEntireDFA) {\n // Create regexp with 2^30 states in DFA.\n Regexp* re = Regexp::Parse(\"a[ab]{30}b\", Regexp::LikePerl, NULL);\n ASSERT_TRUE(re != NULL);\n\n for (int i = 17; i < 24; i++) {\n int64_t limit = int64_t{1}<CompileToProg(limit);\n ASSERT_TRUE(prog != NULL);\n //progusage = m.HeapGrowth();\n //dfamem = prog->dfa_mem();\n prog->BuildEntireDFA(Prog::kFirstMatch, nullptr);\n prog->BuildEntireDFA(Prog::kLongestMatch, nullptr);\n usage = m.HeapGrowth();\n delete prog;\n }\n if (UsingMallocCounter) {\n //ABSL_LOG(INFO) << \"limit \" << limit << \", \"\n // << \"prog usage \" << progusage << \", \"\n // << \"DFA budget \" << dfamem << \", \"\n // << \"total \" << usage;\n // Tolerate +/- 10%.\n ASSERT_GT(usage, limit*9/10);\n ASSERT_LT(usage, limit*11/10);\n }\n }\n re->Decref();\n}\n\n// Test that the DFA gets the right result even if it runs\n// out of memory during a search. The regular expression\n// 0[01]{n}$ matches a binary string of 0s and 1s only if\n// the (n+1)th-to-last character is a 0. Matching this in\n// a single forward pass (as done by the DFA) requires\n// keeping one bit for each of the last n+1 characters\n// (whether each was a 0), or 2^(n+1) possible states.\n// If we run this regexp to search in a string that contains\n// every possible n-character binary string as a substring,\n// then it will have to run through at least 2^n states.\n// States are big data structures -- certainly more than 1 byte --\n// so if the DFA can search correctly while staying within a\n// 2^n byte limit, it must be handling out-of-memory conditions\n// gracefully.\nTEST(SingleThreaded, SearchDFA) {\n // The De Bruijn string is the worst case input for this regexp.\n // By default, the DFA will notice that it is flushing its cache\n // too frequently and will bail out early, so that RE2 can use the\n // NFA implementation instead. (The DFA loses its speed advantage\n // if it can't get a good cache hit rate.)\n // Tell the DFA to trudge along instead.\n Prog::TESTING_ONLY_set_dfa_should_bail_when_slow(false);\n state_cache_resets = 0;\n search_failures = 0;\n\n // Choice of n is mostly arbitrary, except that:\n // * making n too big makes the test run for too long.\n // * making n too small makes the DFA refuse to run,\n // because it has so little memory compared to the program size.\n // Empirically, n = 18 is a good compromise between the two.\n const int n = 18;\n\n Regexp* re = Regexp::Parse(absl::StrFormat(\"0[01]{%d}$\", n),\n Regexp::LikePerl, NULL);\n ASSERT_TRUE(re != NULL);\n\n // The De Bruijn string for n ends with a 1 followed by n 0s in a row,\n // which is not a match for 0[01]{n}$. Adding one more 0 is a match.\n std::string no_match = DeBruijnString(n);\n std::string match = no_match + \"0\";\n\n int64_t usage;\n int64_t peak_usage;\n {\n testing::MallocCounter m(testing::MallocCounter::THIS_THREAD_ONLY);\n Prog* prog = re->CompileToProg(1<SearchDFA(match, absl::string_view(), Prog::kUnanchored,\n Prog::kFirstMatch, NULL, &failed, NULL);\n ASSERT_FALSE(failed);\n ASSERT_TRUE(matched);\n matched =\n prog->SearchDFA(no_match, absl::string_view(), Prog::kUnanchored,\n Prog::kFirstMatch, NULL, &failed, NULL);\n ASSERT_FALSE(failed);\n ASSERT_FALSE(matched);\n }\n usage = m.HeapGrowth();\n peak_usage = m.PeakHeapGrowth();\n delete prog;\n }\n if (UsingMallocCounter) {\n //ABSL_LOG(INFO) << \"usage \" << usage << \", \"\n // << \"peak usage \" << peak_usage;\n ASSERT_LT(usage, 1<Decref();\n\n // Reset to original behaviour.\n Prog::TESTING_ONLY_set_dfa_should_bail_when_slow(true);\n ASSERT_GT(state_cache_resets, 0);\n ASSERT_EQ(search_failures, 0);\n}\n\n// Helper function: searches for match, which should match,\n// and no_match, which should not.\nstatic void DoSearch(Prog* prog, absl::string_view match,\n absl::string_view no_match) {\n for (int i = 0; i < 2; i++) {\n bool matched = false;\n bool failed = false;\n matched =\n prog->SearchDFA(match, absl::string_view(), Prog::kUnanchored,\n Prog::kFirstMatch, NULL, &failed, NULL);\n ASSERT_FALSE(failed);\n ASSERT_TRUE(matched);\n matched =\n prog->SearchDFA(no_match, absl::string_view(), Prog::kUnanchored,\n Prog::kFirstMatch, NULL, &failed, NULL);\n ASSERT_FALSE(failed);\n ASSERT_FALSE(matched);\n }\n}\n\nTEST(Multithreaded, SearchDFA) {\n Prog::TESTING_ONLY_set_dfa_should_bail_when_slow(false);\n state_cache_resets = 0;\n search_failures = 0;\n\n // Same as single-threaded test above.\n const int n = 18;\n Regexp* re = Regexp::Parse(absl::StrFormat(\"0[01]{%d}$\", n),\n Regexp::LikePerl, NULL);\n ASSERT_TRUE(re != NULL);\n std::string no_match = DeBruijnString(n);\n std::string match = no_match + \"0\";\n\n // Check that single-threaded code works.\n {\n Prog* prog = re->CompileToProg(1<CompileToProg(1< threads;\n for (int j = 0; j < absl::GetFlag(FLAGS_threads); j++)\n threads.emplace_back(DoSearch, prog, match, no_match);\n for (int j = 0; j < absl::GetFlag(FLAGS_threads); j++)\n threads[j].join();\n\n delete prog;\n }\n\n re->Decref();\n\n // Reset to original behaviour.\n Prog::TESTING_ONLY_set_dfa_should_bail_when_slow(true);\n ASSERT_GT(state_cache_resets, 0);\n ASSERT_EQ(search_failures, 0);\n}\n\nstruct ReverseTest {\n const char* regexp;\n const char* text;\n bool match;\n};\n\n// Test that reverse DFA handles anchored/unanchored correctly.\n// It's in the DFA interface but not used by RE2.\nReverseTest reverse_tests[] = {\n { \"\\\\A(a|b)\", \"abc\", true },\n { \"(a|b)\\\\z\", \"cba\", true },\n { \"\\\\A(a|b)\", \"cba\", false },\n { \"(a|b)\\\\z\", \"abc\", false },\n};\n\nTEST(DFA, ReverseMatch) {\n int nfail = 0;\n for (size_t i = 0; i < ABSL_ARRAYSIZE(reverse_tests); i++) {\n const ReverseTest& t = reverse_tests[i];\n Regexp* re = Regexp::Parse(t.regexp, Regexp::LikePerl, NULL);\n ASSERT_TRUE(re != NULL);\n Prog* prog = re->CompileToReverseProg(0);\n ASSERT_TRUE(prog != NULL);\n bool failed = false;\n bool matched =\n prog->SearchDFA(t.text, absl::string_view(), Prog::kUnanchored,\n Prog::kFirstMatch, NULL, &failed, NULL);\n if (matched != t.match) {\n ABSL_LOG(ERROR) << t.regexp << \" on \" << t.text << \": want \" << t.match;\n nfail++;\n }\n delete prog;\n re->Decref();\n }\n EXPECT_EQ(nfail, 0);\n}\n\nstruct CallbackTest {\n const char* regexp;\n const char* dump;\n};\n\n// Test that DFA::BuildAllStates() builds the expected DFA states\n// and issues the expected callbacks. These test cases reflect the\n// very compact encoding of the callbacks, but that also makes them\n// very difficult to understand, so let's work through \"\\\\Aa\\\\z\".\n// There are three slots per DFA state because the bytemap has two\n// equivalence classes and there is a third slot for kByteEndText:\n// 0: all bytes that are not 'a'\n// 1: the byte 'a'\n// 2: kByteEndText\n// -1 means that there is no transition from that DFA state to any\n// other DFA state for that slot. The valid transitions are thus:\n// state 0 --slot 1--> state 1\n// state 1 --slot 2--> state 2\n// The double brackets indicate that state 2 is a matching state.\n// Putting it together, this means that the DFA must consume the\n// byte 'a' and then hit end of text. Q.E.D.\nCallbackTest callback_tests[] = {\n { \"\\\\Aa\\\\z\", \"[-1,1,-1] [-1,-1,2] [[-1,-1,-1]]\" },\n { \"\\\\Aab\\\\z\", \"[-1,1,-1,-1] [-1,-1,2,-1] [-1,-1,-1,3] [[-1,-1,-1,-1]]\" },\n { \"\\\\Aa*b\\\\z\", \"[-1,0,1,-1] [-1,-1,-1,2] [[-1,-1,-1,-1]]\" },\n { \"\\\\Aa+b\\\\z\", \"[-1,1,-1,-1] [-1,1,2,-1] [-1,-1,-1,3] [[-1,-1,-1,-1]]\" },\n { \"\\\\Aa?b\\\\z\", \"[-1,1,2,-1] [-1,-1,2,-1] [-1,-1,-1,3] [[-1,-1,-1,-1]]\" },\n { \"\\\\Aa\\\\C*\\\\z\", \"[-1,1,-1] [1,1,2] [[-1,-1,-1]]\" },\n { \"\\\\Aa\\\\C*\", \"[-1,1,-1] [2,2,3] [[2,2,2]] [[-1,-1,-1]]\" },\n { \"a\\\\C*\", \"[0,1,-1] [2,2,3] [[2,2,2]] [[-1,-1,-1]]\" },\n { \"\\\\C*\", \"[1,2] [[1,1]] [[-1,-1]]\" },\n { \"a\", \"[0,1,-1] [2,2,2] [[-1,-1,-1]]\"} ,\n};\n\nTEST(DFA, Callback) {\n int nfail = 0;\n for (size_t i = 0; i < ABSL_ARRAYSIZE(callback_tests); i++) {\n const CallbackTest& t = callback_tests[i];\n Regexp* re = Regexp::Parse(t.regexp, Regexp::LikePerl, NULL);\n ASSERT_TRUE(re != NULL);\n Prog* prog = re->CompileToProg(0);\n ASSERT_TRUE(prog != NULL);\n std::string dump;\n prog->BuildEntireDFA(Prog::kLongestMatch, [&](const int* next, bool match) {\n ASSERT_TRUE(next != NULL);\n if (!dump.empty())\n dump += \" \";\n dump += match ? \"[[\" : \"[\";\n for (int b = 0; b < prog->bytemap_range() + 1; b++)\n dump += absl::StrFormat(\"%d,\", next[b]);\n dump.pop_back();\n dump += match ? \"]]\" : \"]\";\n });\n if (dump != t.dump) {\n ABSL_LOG(ERROR) << t.regexp << \" bytemap:\\n\" << prog->DumpByteMap();\n ABSL_LOG(ERROR) << t.regexp << \" dump:\\n\" << \"got \" << dump << \"\\n\"\n << \"want \" << t.dump;\n nfail++;\n }\n delete prog;\n re->Decref();\n }\n EXPECT_EQ(nfail, 0);\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/dump.cc", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Dump the regexp into a string showing structure.\n// Tested by parse_unittest.cc\n\n// This function traverses the regexp recursively,\n// meaning that on inputs like Regexp::Simplify of\n// a{100}{100}{100}{100}{100}{100}{100}{100}{100}{100},\n// it takes time and space exponential in the size of the\n// original regular expression. It can also use stack space\n// linear in the size of the regular expression for inputs\n// like ((((((((((((((((a*)*)*)*)*)*)*)*)*)*)*)*)*)*)*)*)*.\n// IT IS NOT SAFE TO CALL FROM PRODUCTION CODE.\n// As a result, Dump is provided only in the testing\n// library (see BUILD).\n\n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/regexp.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\nstatic const char* kOpcodeNames[] = {\n \"bad\",\n \"no\",\n \"emp\",\n \"lit\",\n \"str\",\n \"cat\",\n \"alt\",\n \"star\",\n \"plus\",\n \"que\",\n \"rep\",\n \"cap\",\n \"dot\",\n \"byte\",\n \"bol\",\n \"eol\",\n \"wb\", // kRegexpWordBoundary\n \"nwb\", // kRegexpNoWordBoundary\n \"bot\",\n \"eot\",\n \"cc\",\n \"match\",\n};\n\n// Create string representation of regexp with explicit structure.\n// Nothing pretty, just for testing.\nstatic void DumpRegexpAppending(Regexp* re, std::string* s) {\n if (re->op() < 0 || re->op() >= ABSL_ARRAYSIZE(kOpcodeNames)) {\n *s += absl::StrFormat(\"op%d\", re->op());\n } else {\n switch (re->op()) {\n default:\n break;\n case kRegexpStar:\n case kRegexpPlus:\n case kRegexpQuest:\n case kRegexpRepeat:\n if (re->parse_flags() & Regexp::NonGreedy)\n s->append(\"n\");\n break;\n }\n s->append(kOpcodeNames[re->op()]);\n if (re->op() == kRegexpLiteral && (re->parse_flags() & Regexp::FoldCase)) {\n Rune r = re->rune();\n if ('a' <= r && r <= 'z')\n s->append(\"fold\");\n }\n if (re->op() == kRegexpLiteralString && (re->parse_flags() & Regexp::FoldCase)) {\n for (int i = 0; i < re->nrunes(); i++) {\n Rune r = re->runes()[i];\n if ('a' <= r && r <= 'z') {\n s->append(\"fold\");\n break;\n }\n }\n }\n }\n s->append(\"{\");\n switch (re->op()) {\n default:\n break;\n case kRegexpEndText:\n if (!(re->parse_flags() & Regexp::WasDollar)) {\n s->append(\"\\\\z\");\n }\n break;\n case kRegexpLiteral: {\n Rune r = re->rune();\n if (re->parse_flags() & Regexp::Latin1) {\n s->push_back(r);\n } else {\n char buf[UTFmax+1];\n buf[runetochar(buf, &r)] = 0;\n s->append(buf);\n }\n break;\n }\n case kRegexpLiteralString:\n for (int i = 0; i < re->nrunes(); i++) {\n Rune r = re->runes()[i];\n if (re->parse_flags() & Regexp::Latin1) {\n s->push_back(r);\n } else {\n char buf[UTFmax+1];\n buf[runetochar(buf, &r)] = 0;\n s->append(buf);\n }\n }\n break;\n case kRegexpConcat:\n case kRegexpAlternate:\n for (int i = 0; i < re->nsub(); i++)\n DumpRegexpAppending(re->sub()[i], s);\n break;\n case kRegexpStar:\n case kRegexpPlus:\n case kRegexpQuest:\n DumpRegexpAppending(re->sub()[0], s);\n break;\n case kRegexpCapture:\n if (re->cap() == 0)\n ABSL_LOG(DFATAL) << \"kRegexpCapture cap() == 0\";\n if (re->name()) {\n s->append(*re->name());\n s->append(\":\");\n }\n DumpRegexpAppending(re->sub()[0], s);\n break;\n case kRegexpRepeat:\n s->append(absl::StrFormat(\"%d,%d \", re->min(), re->max()));\n DumpRegexpAppending(re->sub()[0], s);\n break;\n case kRegexpCharClass: {\n std::string sep;\n for (CharClass::iterator it = re->cc()->begin();\n it != re->cc()->end(); ++it) {\n RuneRange rr = *it;\n s->append(sep);\n if (rr.lo == rr.hi)\n s->append(absl::StrFormat(\"%#x\", rr.lo));\n else\n s->append(absl::StrFormat(\"%#x-%#x\", rr.lo, rr.hi));\n sep = \" \";\n }\n break;\n }\n }\n s->append(\"}\");\n}\n\nstd::string Regexp::Dump() {\n // Make sure that we are being called from a unit test.\n // Should cause a link error if used outside of testing.\n ABSL_CHECK(!::testing::TempDir().empty());\n\n std::string s;\n DumpRegexpAppending(this, &s);\n return s;\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/exhaustive1_test.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Exhaustive testing of regular expression matching.\n\n#include \n#include \n\n#include \"gtest/gtest.h\"\n#include \"re2/testing/exhaustive_tester.h\"\n#include \"re2/testing/regexp_generator.h\"\n\nnamespace re2 {\n\n// Test simple repetition operators\nTEST(Repetition, Simple) {\n std::vector ops = Split(\" \",\n \"%s{0} %s{0,} %s{1} %s{1,} %s{0,1} %s{0,2} \"\n \"%s{1,2} %s{2} %s{2,} %s{3,4} %s{4,5} \"\n \"%s* %s+ %s? %s*? %s+? %s??\");\n ExhaustiveTest(3, 2, Explode(\"abc.\"), ops,\n 6, Explode(\"ab\"), \"(?:%s)\", \"\");\n ExhaustiveTest(3, 2, Explode(\"abc.\"), ops,\n 40, Explode(\"a\"), \"(?:%s)\", \"\");\n}\n\n// Test capturing parens -- (a) -- inside repetition operators\nTEST(Repetition, Capturing) {\n std::vector ops = Split(\" \",\n \"%s{0} %s{0,} %s{1} %s{1,} %s{0,1} %s{0,2} \"\n \"%s{1,2} %s{2} %s{2,} %s{3,4} %s{4,5} \"\n \"%s* %s+ %s? %s*? %s+? %s??\");\n ExhaustiveTest(3, 2, Split(\" \", \"a (a) b\"), ops,\n 7, Explode(\"ab\"), \"(?:%s)\", \"\");\n ExhaustiveTest(3, 2, Split(\" \", \"a (a)\"), ops,\n 50, Explode(\"a\"), \"(?:%s)\", \"\");\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/exhaustive2_test.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Exhaustive testing of regular expression matching.\n\n#include \n\n#include \n#include \n\n#include \"gtest/gtest.h\"\n#include \"re2/testing/exhaustive_tester.h\"\n#include \"re2/testing/regexp_generator.h\"\n\nnamespace re2 {\n\n// Test empty string matches (aka \"(?:)\")\nTEST(EmptyString, Exhaustive) {\n ExhaustiveTest(2, 2, Split(\" \", \"(?:) a\"),\n RegexpGenerator::EgrepOps(),\n 5, Split(\"\", \"ab\"), \"\", \"\");\n}\n\n// Test escaped versions of regexp syntax.\nTEST(Punctuation, Literals) {\n std::vector alphabet = Explode(\"()*+?{}[]\\\\^$.\");\n std::vector escaped = alphabet;\n for (size_t i = 0; i < escaped.size(); i++)\n escaped[i] = \"\\\\\" + escaped[i];\n ExhaustiveTest(1, 1, escaped, RegexpGenerator::EgrepOps(),\n 2, alphabet, \"\", \"\");\n}\n\n// Test ^ $ . \\A \\z in presence of line endings.\n// Have to wrap the empty-width ones in (?:) so that\n// they can be repeated -- PCRE rejects ^* but allows (?:^)*\nTEST(LineEnds, Exhaustive) {\n ExhaustiveTest(2, 2, Split(\" \", \"(?:^) (?:$) . a \\\\n (?:\\\\A) (?:\\\\z)\"),\n RegexpGenerator::EgrepOps(),\n 4, Explode(\"ab\\n\"), \"\", \"\");\n}\n\n// Test what does and does not match \\n.\n// This would be a good test, except that PCRE seems to have a bug:\n// in single-byte character set mode (the default),\n// [^a] matches \\n, but in UTF-8 mode it does not.\n// So when we run the test, the tester complains that\n// we don't agree with PCRE, but it's PCRE that is at fault.\n// For what it's worth, Perl gets this right (matches\n// regardless of whether UTF-8 input is selected):\n//\n// #!/usr/bin/perl\n// use POSIX qw(locale_h);\n// print \"matches in latin1\\n\" if \"\\n\" =~ /[^a]/;\n// setlocale(\"en_US.utf8\");\n// print \"matches in utf8\\n\" if \"\\n\" =~ /[^a]/;\n//\n// The rule chosen for RE2 is that by default, like Perl,\n// dot does not match \\n but negated character classes [^a] do.\n// (?s) will allow dot to match \\n; there is no way in RE2\n// to stop [^a] from matching \\n, though the underlying library\n// provides a mechanism, and RE2 could add new syntax if needed.\n//\n// TEST(Newlines, Exhaustive) {\n// std::vector empty_vector;\n// ExhaustiveTest(1, 1, Split(\" \", \"\\\\n . a [^a]\"),\n// RegexpGenerator::EgrepOps(),\n// 4, Explode(\"a\\n\"), \"\");\n// }\n\n} // namespace re2\n" }, { "path": "re2/testing/exhaustive3_test.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Exhaustive testing of regular expression matching.\n\n#include \n\n#include \n#include \n\n#include \"gtest/gtest.h\"\n#include \"re2/testing/exhaustive_tester.h\"\n#include \"re2/testing/regexp_generator.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\n// Test simple character classes by themselves.\nTEST(CharacterClasses, Exhaustive) {\n std::vector atoms = Split(\" \",\n \"[a] [b] [ab] [^bc] [b-d] [^b-d] []a] [-a] [a-] [^-a] [a-b-c] a b .\");\n ExhaustiveTest(2, 1, atoms, RegexpGenerator::EgrepOps(),\n 5, Explode(\"ab\"), \"\", \"\");\n}\n\n// Test simple character classes inside a___b (for example, a[a]b).\nTEST(CharacterClasses, ExhaustiveAB) {\n std::vector atoms = Split(\" \",\n \"[a] [b] [ab] [^bc] [b-d] [^b-d] []a] [-a] [a-] [^-a] [a-b-c] a b .\");\n ExhaustiveTest(2, 1, atoms, RegexpGenerator::EgrepOps(),\n 5, Explode(\"ab\"), \"a%sb\", \"\");\n}\n\n// Returns UTF8 for Rune r\nstatic std::string UTF8(Rune r) {\n char buf[UTFmax+1];\n buf[runetochar(buf, &r)] = 0;\n return std::string(buf);\n}\n\n// Returns a vector of \"interesting\" UTF8 characters.\n// Unicode is now too big to just return all of them,\n// so UTF8Characters return a set likely to be good test cases.\nstatic const std::vector& InterestingUTF8() {\n static bool init;\n static std::vector v;\n\n if (init)\n return v;\n\n init = true;\n // All the Latin1 equivalents are interesting.\n for (int i = 1; i < 256; i++)\n v.push_back(UTF8(i));\n\n // After that, the codes near bit boundaries are\n // interesting, because they span byte sequence lengths.\n for (int j = 0; j < 8; j++)\n v.push_back(UTF8(256 + j));\n for (int i = 512; i < Runemax; i <<= 1)\n for (int j = -8; j < 8; j++)\n v.push_back(UTF8(i + j));\n\n // The codes near Runemax, including Runemax itself, are interesting.\n for (int j = -8; j <= 0; j++)\n v.push_back(UTF8(Runemax + j));\n\n return v;\n}\n\n// Test interesting UTF-8 characters against character classes.\nTEST(InterestingUTF8, SingleOps) {\n std::vector atoms = Split(\" \",\n \". ^ $ \\\\a \\\\f \\\\n \\\\r \\\\t \\\\v \\\\d \\\\D \\\\s \\\\S \\\\w \\\\W \\\\b \\\\B \"\n \"[[:alnum:]] [[:alpha:]] [[:blank:]] [[:cntrl:]] [[:digit:]] \"\n \"[[:graph:]] [[:lower:]] [[:print:]] [[:punct:]] [[:space:]] \"\n \"[[:upper:]] [[:xdigit:]] [\\\\s\\\\S] [\\\\d\\\\D] [^\\\\w\\\\W] [^\\\\d\\\\D]\");\n std::vector ops; // no ops\n ExhaustiveTest(1, 0, atoms, ops,\n 1, InterestingUTF8(), \"\", \"\");\n}\n\n// Test interesting UTF-8 characters against character classes,\n// but wrap everything inside AB.\nTEST(InterestingUTF8, AB) {\n std::vector atoms = Split(\" \",\n \". ^ $ \\\\a \\\\f \\\\n \\\\r \\\\t \\\\v \\\\d \\\\D \\\\s \\\\S \\\\w \\\\W \\\\b \\\\B \"\n \"[[:alnum:]] [[:alpha:]] [[:blank:]] [[:cntrl:]] [[:digit:]] \"\n \"[[:graph:]] [[:lower:]] [[:print:]] [[:punct:]] [[:space:]] \"\n \"[[:upper:]] [[:xdigit:]] [\\\\s\\\\S] [\\\\d\\\\D] [^\\\\w\\\\W] [^\\\\d\\\\D]\");\n std::vector ops; // no ops\n std::vector alpha = InterestingUTF8();\n for (size_t i = 0; i < alpha.size(); i++)\n alpha[i] = \"a\" + alpha[i] + \"b\";\n ExhaustiveTest(1, 0, atoms, ops,\n 1, alpha, \"a%sb\", \"\");\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/exhaustive_test.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Exhaustive testing of regular expression matching.\n\n#include \"gtest/gtest.h\"\n#include \"re2/testing/exhaustive_tester.h\"\n\nnamespace re2 {\n\n// Test very simple expressions.\nTEST(EgrepLiterals, Lowercase) {\n EgrepTest(3, 2, \"abc.\", 3, \"abc\", \"\");\n}\n\n// Test mixed-case expressions.\nTEST(EgrepLiterals, MixedCase) {\n EgrepTest(3, 2, \"AaBb.\", 2, \"AaBb\", \"\");\n}\n\n// Test mixed-case in case-insensitive mode.\nTEST(EgrepLiterals, FoldCase) {\n // The punctuation characters surround A-Z and a-z\n // in the ASCII table. This looks for bugs in the\n // bytemap range code in the DFA.\n EgrepTest(3, 2, \"abAB.\", 2, \"aBc@_~\", \"(?i:%s)\");\n}\n\n// Test very simple expressions.\nTEST(EgrepLiterals, UTF8) {\n EgrepTest(3, 2, \"ab.\", 4, \"a\\xE2\\x98\\xBA\", \"\");\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/exhaustive_tester.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Exhaustive testing of regular expression matching.\n\n// Each test picks an alphabet (e.g., \"abc\"), a maximum string length,\n// a maximum regular expression length, and a maximum number of letters\n// that can appear in the regular expression. Given these parameters,\n// it tries every possible regular expression and string, verifying that\n// the NFA, DFA, and a trivial backtracking implementation agree about\n// the location of the match.\n\n#include \"re2/testing/exhaustive_tester.h\"\n\n#include \n\n#include \n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/flags/flag.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/prog.h\"\n#include \"re2/re2.h\"\n#include \"re2/testing/regexp_generator.h\"\n#include \"re2/testing/tester.h\"\n\n// For target `log' in the Makefile.\n#ifndef LOGGING\n#define LOGGING 0\n#endif\n\nABSL_FLAG(bool, show_regexps, false, \"show regexps during testing\");\n\nABSL_FLAG(int, max_bad_regexp_inputs, 1,\n \"Stop testing a regular expression after finding this many \"\n \"strings that break it.\");\n\nnamespace re2 {\n\nstatic char* escape(absl::string_view sp) {\n static char buf[512];\n char* p = buf;\n *p++ = '\\\"';\n for (size_t i = 0; i < sp.size(); i++) {\n if(p+5 >= buf+sizeof buf)\n ABSL_LOG(FATAL) << \"ExhaustiveTester escape: too long\";\n if(sp[i] == '\\\\' || sp[i] == '\\\"') {\n *p++ = '\\\\';\n *p++ = sp[i];\n } else if(sp[i] == '\\n') {\n *p++ = '\\\\';\n *p++ = 'n';\n } else {\n *p++ = sp[i];\n }\n }\n *p++ = '\\\"';\n *p = '\\0';\n return buf;\n}\n\nstatic void PrintResult(const RE2& re, absl::string_view input,\n RE2::Anchor anchor, absl::string_view* m, int n) {\n if (!re.Match(input, 0, input.size(), anchor, m, n)) {\n absl::PrintF(\"-\");\n return;\n }\n for (int i = 0; i < n; i++) {\n if (i > 0)\n absl::PrintF(\" \");\n if (m[i].data() == NULL)\n absl::PrintF(\"-\");\n else\n absl::PrintF(\"%d-%d\",\n BeginPtr(m[i]) - BeginPtr(input),\n EndPtr(m[i]) - BeginPtr(input));\n }\n}\n\n// Processes a single generated regexp.\n// Compiles it using Regexp interface and PCRE, and then\n// checks that NFA, DFA, and PCRE all return the same results.\nvoid ExhaustiveTester::HandleRegexp(const std::string& const_regexp) {\n regexps_++;\n std::string regexp = const_regexp;\n if (!topwrapper_.empty()) {\n auto fmt = absl::ParsedFormat<'s'>::New(topwrapper_);\n ABSL_CHECK(fmt != nullptr);\n regexp = absl::StrFormat(*fmt, regexp);\n }\n\n if (absl::GetFlag(FLAGS_show_regexps)) {\n absl::PrintF(\"\\r%s\", regexp);\n fflush(stdout);\n }\n\n if (LOGGING) {\n // Write out test cases and answers for use in testing\n // other implementations, such as Go's regexp package.\n if (randomstrings_)\n ABSL_LOG(ERROR) << \"Cannot log with random strings.\";\n if (regexps_ == 1) { // first\n absl::PrintF(\"strings\\n\");\n strgen_.Reset();\n while (strgen_.HasNext())\n absl::PrintF(\"%s\\n\", escape(strgen_.Next()));\n absl::PrintF(\"regexps\\n\");\n }\n absl::PrintF(\"%s\\n\", escape(regexp));\n\n RE2 re(regexp);\n RE2::Options longest;\n longest.set_longest_match(true);\n RE2 relongest(regexp, longest);\n int ngroup = re.NumberOfCapturingGroups()+1;\n absl::string_view* group = new absl::string_view[ngroup];\n\n strgen_.Reset();\n while (strgen_.HasNext()) {\n absl::string_view input = strgen_.Next();\n PrintResult(re, input, RE2::ANCHOR_BOTH, group, ngroup);\n absl::PrintF(\";\");\n PrintResult(re, input, RE2::UNANCHORED, group, ngroup);\n absl::PrintF(\";\");\n PrintResult(relongest, input, RE2::ANCHOR_BOTH, group, ngroup);\n absl::PrintF(\";\");\n PrintResult(relongest, input, RE2::UNANCHORED, group, ngroup);\n absl::PrintF(\"\\n\");\n }\n delete[] group;\n return;\n }\n\n Tester tester(regexp);\n if (tester.error())\n return;\n\n strgen_.Reset();\n strgen_.GenerateNULL();\n if (randomstrings_)\n strgen_.Random(stringseed_, stringcount_);\n int bad_inputs = 0;\n while (strgen_.HasNext()) {\n tests_++;\n if (!tester.TestInput(strgen_.Next())) {\n failures_++;\n if (++bad_inputs >= absl::GetFlag(FLAGS_max_bad_regexp_inputs))\n break;\n }\n }\n}\n\n// Runs an exhaustive test on the given parameters.\nvoid ExhaustiveTest(int maxatoms, int maxops,\n const std::vector& alphabet,\n const std::vector& ops,\n int maxstrlen,\n const std::vector& stralphabet,\n const std::string& wrapper,\n const std::string& topwrapper) {\n if (RE2_DEBUG_MODE) {\n if (maxatoms > 1)\n maxatoms--;\n if (maxops > 1)\n maxops--;\n if (maxstrlen > 1)\n maxstrlen--;\n }\n ExhaustiveTester t(maxatoms, maxops, alphabet, ops,\n maxstrlen, stralphabet, wrapper,\n topwrapper);\n t.Generate();\n if (!LOGGING) {\n absl::PrintF(\"%d regexps, %d tests, %d failures [%d/%d str]\\n\",\n t.regexps(), t.tests(), t.failures(), maxstrlen, stralphabet.size());\n }\n EXPECT_EQ(0, t.failures());\n}\n\n// Runs an exhaustive test using the given parameters and\n// the basic egrep operators.\nvoid EgrepTest(int maxatoms, int maxops, const std::string& alphabet,\n int maxstrlen, const std::string& stralphabet,\n const std::string& wrapper) {\n const char* tops[] = { \"\", \"^(?:%s)\", \"(?:%s)$\", \"^(?:%s)$\" };\n\n for (size_t i = 0; i < ABSL_ARRAYSIZE(tops); i++) {\n ExhaustiveTest(maxatoms, maxops,\n Split(\"\", alphabet),\n RegexpGenerator::EgrepOps(),\n maxstrlen,\n Split(\"\", stralphabet),\n wrapper,\n tops[i]);\n }\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/exhaustive_tester.h", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_TESTING_EXHAUSTIVE_TESTER_H_\n#define RE2_TESTING_EXHAUSTIVE_TESTER_H_\n\n#include \n\n#include \n#include \n\n#include \"re2/testing/regexp_generator.h\"\n#include \"re2/testing/string_generator.h\"\n\nnamespace re2 {\n\n// Doing this simplifies the logic below.\n#ifndef __has_feature\n#define __has_feature(x) 0\n#endif\n\n#if !defined(NDEBUG)\n// We are in a debug build.\nconst bool RE2_DEBUG_MODE = true;\n#elif __has_feature(address_sanitizer) || __has_feature(memory_sanitizer) || __has_feature(thread_sanitizer)\n// Not a debug build, but still under sanitizers.\nconst bool RE2_DEBUG_MODE = true;\n#else\nconst bool RE2_DEBUG_MODE = false;\n#endif\n\n// Exhaustive regular expression test: generate all regexps within parameters,\n// then generate all strings of a given length over a given alphabet,\n// then check that NFA, DFA, and PCRE agree about whether each regexp matches\n// each possible string, and if so, where the match is.\n//\n// Can also be used in a \"random\" mode that generates a given number\n// of random regexp and strings, allowing testing of larger expressions\n// and inputs.\nclass ExhaustiveTester : public RegexpGenerator {\n public:\n ExhaustiveTester(int maxatoms,\n int maxops,\n const std::vector& alphabet,\n const std::vector& ops,\n int maxstrlen,\n const std::vector& stralphabet,\n const std::string& wrapper,\n const std::string& topwrapper)\n : RegexpGenerator(maxatoms, maxops, alphabet, ops),\n strgen_(maxstrlen, stralphabet),\n wrapper_(wrapper),\n topwrapper_(topwrapper),\n regexps_(0), tests_(0), failures_(0),\n randomstrings_(0), stringseed_(0), stringcount_(0) { }\n\n int regexps() { return regexps_; }\n int tests() { return tests_; }\n int failures() { return failures_; }\n\n // Needed for RegexpGenerator interface.\n void HandleRegexp(const std::string& regexp);\n\n // Causes testing to generate random input strings.\n void RandomStrings(int32_t seed, int32_t count) {\n randomstrings_ = true;\n stringseed_ = seed;\n stringcount_ = count;\n }\n\n private:\n StringGenerator strgen_;\n std::string wrapper_; // Regexp wrapper - either empty or has one %s.\n std::string topwrapper_; // Regexp top-level wrapper.\n int regexps_; // Number of HandleRegexp calls\n int tests_; // Number of regexp tests.\n int failures_; // Number of tests failed.\n\n bool randomstrings_; // Whether to use random strings\n int32_t stringseed_; // If so, the seed.\n int stringcount_; // If so, how many to generate.\n\n ExhaustiveTester(const ExhaustiveTester&) = delete;\n ExhaustiveTester& operator=(const ExhaustiveTester&) = delete;\n};\n\n// Runs an exhaustive test on the given parameters.\nvoid ExhaustiveTest(int maxatoms, int maxops,\n const std::vector& alphabet,\n const std::vector& ops,\n int maxstrlen,\n const std::vector& stralphabet,\n const std::string& wrapper,\n const std::string& topwrapper);\n\n// Runs an exhaustive test using the given parameters and\n// the basic egrep operators.\nvoid EgrepTest(int maxatoms, int maxops, const std::string& alphabet,\n int maxstrlen, const std::string& stralphabet,\n const std::string& wrapper);\n\n} // namespace re2\n\n#endif // RE2_TESTING_EXHAUSTIVE_TESTER_H_\n" }, { "path": "re2/testing/filtered_re2_test.cc", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \"re2/filtered_re2.h\"\n\n#include \n\n#include \n#include \n#include \n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_log.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/re2.h\"\n\nnamespace re2 {\n\nstruct FilterTestVars {\n FilterTestVars() {}\n explicit FilterTestVars(int min_atom_len) : f(min_atom_len) {}\n\n std::vector atoms;\n std::vector atom_indices;\n std::vector matches;\n RE2::Options opts;\n FilteredRE2 f;\n};\n\nTEST(FilteredRE2Test, EmptyTest) {\n FilterTestVars v;\n\n v.f.Compile(&v.atoms);\n EXPECT_EQ(size_t{0}, v.atoms.size());\n\n // Compile has no effect at all when called before Add: it will not\n // record that it has been called and it will not clear the vector.\n // The second point does not matter here, but the first point means\n // that an error will be logged during the call to AllMatches.\n v.f.AllMatches(\"foo\", v.atom_indices, &v.matches);\n EXPECT_EQ(size_t{0}, v.matches.size());\n}\n\nTEST(FilteredRE2Test, SmallOrTest) {\n FilterTestVars v(4); // override the minimum atom length\n int id;\n v.f.Add(\"(foo|bar)\", v.opts, &id);\n\n v.f.Compile(&v.atoms);\n EXPECT_EQ(size_t{0}, v.atoms.size());\n\n v.f.AllMatches(\"lemurs bar\", v.atom_indices, &v.matches);\n EXPECT_EQ(size_t{1}, v.matches.size());\n EXPECT_EQ(id, v.matches[0]);\n}\n\nTEST(FilteredRE2Test, SmallLatinTest) {\n FilterTestVars v;\n int id;\n\n v.opts.set_encoding(RE2::Options::EncodingLatin1);\n v.f.Add(\"\\xde\\xadQ\\xbe\\xef\", v.opts, &id);\n v.f.Compile(&v.atoms);\n EXPECT_EQ(size_t{1}, v.atoms.size());\n EXPECT_EQ(v.atoms[0], \"\\xde\\xadq\\xbe\\xef\");\n\n v.atom_indices.push_back(0);\n v.f.AllMatches(\"foo\\xde\\xadQ\\xbe\\xeflemur\", v.atom_indices, &v.matches);\n EXPECT_EQ(size_t{1}, v.matches.size());\n EXPECT_EQ(id, v.matches[0]);\n}\n\nstruct AtomTest {\n const char* testname;\n // If any test needs more than this many regexps or atoms, increase\n // the size of the corresponding array.\n const char* regexps[20];\n const char* atoms[20];\n};\n\nAtomTest atom_tests[] = {\n {\n // This test checks to make sure empty patterns are allowed.\n \"CheckEmptyPattern\",\n {\"\"},\n {}\n }, {\n // This test checks that all atoms of length greater than min length\n // are found, and no atoms that are of smaller length are found.\n \"AllAtomsGtMinLengthFound\", {\n \"(abc123|def456|ghi789).*mnop[x-z]+\",\n \"abc..yyy..zz\",\n \"mnmnpp[a-z]+PPP\"\n }, {\n \"abc123\",\n \"def456\",\n \"ghi789\",\n \"mnop\",\n \"abc\",\n \"yyy\",\n \"mnmnpp\",\n \"ppp\"\n }\n }, {\n // Test to make sure that any atoms that have another atom as a\n // substring in an OR are removed; that is, only the shortest\n // substring is kept.\n \"SubstrAtomRemovesSuperStrInOr\", {\n \"(abc123|abc|defxyz|ghi789|abc1234|xyz).*[x-z]+\",\n \"abcd..yyy..yyyzzz\",\n \"mnmnpp[a-z]+PPP\"\n }, {\n \"abc\",\n \"ghi789\",\n \"xyz\",\n \"abcd\",\n \"yyy\",\n \"yyyzzz\",\n \"mnmnpp\",\n \"ppp\"\n }\n }, {\n // Test character class expansion.\n \"CharClassExpansion\", {\n \"m[a-c][d-f]n.*[x-z]+\",\n \"[x-y]bcde[ab]\"\n }, {\n \"madn\", \"maen\", \"mafn\",\n \"mbdn\", \"mben\", \"mbfn\",\n \"mcdn\", \"mcen\", \"mcfn\",\n \"xbcdea\", \"xbcdeb\",\n \"ybcdea\", \"ybcdeb\"\n }\n }, {\n // Test upper/lower of non-ASCII.\n \"UnicodeLower\", {\n \"(?i)ΔδΠϖπΣςσ\",\n \"ΛΜΝΟΠ\",\n \"ψρστυ\",\n }, {\n \"δδπππσσσ\",\n \"λμνοπ\",\n \"ψρστυ\",\n },\n },\n};\n\nvoid AddRegexpsAndCompile(const char* regexps[],\n size_t n,\n struct FilterTestVars* v) {\n for (size_t i = 0; i < n; i++) {\n int id;\n v->f.Add(regexps[i], v->opts, &id);\n }\n v->f.Compile(&v->atoms);\n}\n\nbool CheckExpectedAtoms(const char* atoms[],\n size_t n,\n const char* testname,\n struct FilterTestVars* v) {\n std::vector expected;\n for (size_t i = 0; i < n; i++)\n expected.push_back(atoms[i]);\n\n bool pass = expected.size() == v->atoms.size();\n\n std::sort(v->atoms.begin(), v->atoms.end());\n std::sort(expected.begin(), expected.end());\n for (size_t i = 0; pass && i < n; i++)\n pass = pass && expected[i] == v->atoms[i];\n\n if (!pass) {\n ABSL_LOG(ERROR) << \"Failed \" << testname;\n ABSL_LOG(ERROR) << \"Expected #atoms = \" << expected.size();\n for (size_t i = 0; i < expected.size(); i++)\n ABSL_LOG(ERROR) << expected[i];\n ABSL_LOG(ERROR) << \"Found #atoms = \" << v->atoms.size();\n for (size_t i = 0; i < v->atoms.size(); i++)\n ABSL_LOG(ERROR) << v->atoms[i];\n }\n\n return pass;\n}\n\nTEST(FilteredRE2Test, AtomTests) {\n int nfail = 0;\n for (size_t i = 0; i < ABSL_ARRAYSIZE(atom_tests); i++) {\n FilterTestVars v;\n AtomTest* t = &atom_tests[i];\n size_t nregexp, natom;\n for (nregexp = 0; nregexp < ABSL_ARRAYSIZE(t->regexps); nregexp++)\n if (t->regexps[nregexp] == NULL)\n break;\n for (natom = 0; natom < ABSL_ARRAYSIZE(t->atoms); natom++)\n if (t->atoms[natom] == NULL)\n break;\n AddRegexpsAndCompile(t->regexps, nregexp, &v);\n if (!CheckExpectedAtoms(t->atoms, natom, t->testname, &v))\n nfail++;\n }\n EXPECT_EQ(0, nfail);\n}\n\nvoid FindAtomIndices(const std::vector& atoms,\n const std::vector& matched_atoms,\n std::vector* atom_indices) {\n atom_indices->clear();\n for (size_t i = 0; i < matched_atoms.size(); i++) {\n for (size_t j = 0; j < atoms.size(); j++) {\n if (matched_atoms[i] == atoms[j]) {\n atom_indices->push_back(static_cast(j));\n break;\n }\n }\n }\n}\n\nTEST(FilteredRE2Test, MatchEmptyPattern) {\n FilterTestVars v;\n AtomTest* t = &atom_tests[0];\n // We are using the regexps used in one of the atom tests\n // for this test. Adding the EXPECT here to make sure\n // the index we use for the test is for the correct test.\n EXPECT_EQ(\"CheckEmptyPattern\", std::string(t->testname));\n size_t nregexp;\n for (nregexp = 0; nregexp < ABSL_ARRAYSIZE(t->regexps); nregexp++)\n if (t->regexps[nregexp] == NULL)\n break;\n AddRegexpsAndCompile(t->regexps, nregexp, &v);\n std::string text = \"0123\";\n std::vector atom_ids;\n std::vector matching_regexps;\n EXPECT_EQ(0, v.f.FirstMatch(text, atom_ids));\n}\n\nTEST(FilteredRE2Test, MatchTests) {\n FilterTestVars v;\n AtomTest* t = &atom_tests[2];\n // We are using the regexps used in one of the atom tests\n // for this test.\n EXPECT_EQ(\"SubstrAtomRemovesSuperStrInOr\", std::string(t->testname));\n size_t nregexp;\n for (nregexp = 0; nregexp < ABSL_ARRAYSIZE(t->regexps); nregexp++)\n if (t->regexps[nregexp] == NULL)\n break;\n AddRegexpsAndCompile(t->regexps, nregexp, &v);\n\n std::string text = \"abc121212xyz\";\n // atoms = abc\n std::vector atom_ids;\n std::vector atoms;\n atoms.push_back(\"abc\");\n FindAtomIndices(v.atoms, atoms, &atom_ids);\n std::vector matching_regexps;\n v.f.AllMatches(text, atom_ids, &matching_regexps);\n EXPECT_EQ(size_t{1}, matching_regexps.size());\n\n text = \"abc12312yyyzzz\";\n atoms.clear();\n atoms.push_back(\"abc\");\n atoms.push_back(\"yyy\");\n atoms.push_back(\"yyyzzz\");\n FindAtomIndices(v.atoms, atoms, &atom_ids);\n v.f.AllMatches(text, atom_ids, &matching_regexps);\n EXPECT_EQ(size_t{1}, matching_regexps.size());\n\n text = \"abcd12yyy32yyyzzz\";\n atoms.clear();\n atoms.push_back(\"abc\");\n atoms.push_back(\"abcd\");\n atoms.push_back(\"yyy\");\n atoms.push_back(\"yyyzzz\");\n FindAtomIndices(v.atoms, atoms, &atom_ids);\n ABSL_LOG(INFO) << \"S: \" << atom_ids.size();\n for (size_t i = 0; i < atom_ids.size(); i++)\n ABSL_LOG(INFO) << \"i: \" << i << \" : \" << atom_ids[i];\n v.f.AllMatches(text, atom_ids, &matching_regexps);\n EXPECT_EQ(size_t{2}, matching_regexps.size());\n}\n\nTEST(FilteredRE2Test, EmptyStringInStringSetBug) {\n // Bug due to find() finding \"\" at the start of everything in a string\n // set and thus SimplifyStringSet() would end up erasing everything.\n // In order to test this, we have to keep PrefilterTree from discarding\n // the OR entirely, so we have to make the minimum atom length zero.\n\n FilterTestVars v(0); // override the minimum atom length\n const char* regexps[] = {\"-R.+(|ADD=;AA){12}}\"};\n const char* atoms[] = {\"\", \"-r\", \"add=;aa\", \"}\"};\n AddRegexpsAndCompile(regexps, ABSL_ARRAYSIZE(regexps), &v);\n EXPECT_TRUE(CheckExpectedAtoms(atoms, ABSL_ARRAYSIZE(atoms),\n \"EmptyStringInStringSetBug\", &v));\n}\n\nTEST(FilteredRE2Test, MoveSemantics) {\n FilterTestVars v1;\n int id;\n v1.f.Add(\"foo\\\\d+\", v1.opts, &id);\n EXPECT_EQ(0, id);\n v1.f.Compile(&v1.atoms);\n EXPECT_EQ(size_t{1}, v1.atoms.size());\n EXPECT_EQ(\"foo\", v1.atoms[0]);\n v1.f.AllMatches(\"abc foo1 xyz\", {0}, &v1.matches);\n EXPECT_EQ(size_t{1}, v1.matches.size());\n EXPECT_EQ(0, v1.matches[0]);\n v1.f.AllMatches(\"abc bar2 xyz\", {0}, &v1.matches);\n EXPECT_EQ(size_t{0}, v1.matches.size());\n\n // The moved-to object should do what the moved-from object did.\n FilterTestVars v2;\n v2.f = std::move(v1.f);\n v2.f.AllMatches(\"abc foo1 xyz\", {0}, &v2.matches);\n EXPECT_EQ(size_t{1}, v2.matches.size());\n EXPECT_EQ(0, v2.matches[0]);\n v2.f.AllMatches(\"abc bar2 xyz\", {0}, &v2.matches);\n EXPECT_EQ(size_t{0}, v2.matches.size());\n\n // The moved-from object should have been reset and be reusable.\n v1.f.Add(\"bar\\\\d+\", v1.opts, &id);\n EXPECT_EQ(0, id);\n v1.f.Compile(&v1.atoms);\n EXPECT_EQ(size_t{1}, v1.atoms.size());\n EXPECT_EQ(\"bar\", v1.atoms[0]);\n v1.f.AllMatches(\"abc foo1 xyz\", {0}, &v1.matches);\n EXPECT_EQ(size_t{0}, v1.matches.size());\n v1.f.AllMatches(\"abc bar2 xyz\", {0}, &v1.matches);\n EXPECT_EQ(size_t{1}, v1.matches.size());\n EXPECT_EQ(0, v1.matches[0]);\n\n // Verify that \"overwriting\" works and also doesn't leak memory.\n // (The latter will need a leak detector such as LeakSanitizer.)\n v1.f = std::move(v2.f);\n v1.f.AllMatches(\"abc foo1 xyz\", {0}, &v1.matches);\n EXPECT_EQ(size_t{1}, v1.matches.size());\n EXPECT_EQ(0, v1.matches[0]);\n v1.f.AllMatches(\"abc bar2 xyz\", {0}, &v1.matches);\n EXPECT_EQ(size_t{0}, v1.matches.size());\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/mimics_pcre_test.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \n\n#include \"absl/base/macros.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/prog.h\"\n#include \"re2/regexp.h\"\n\nnamespace re2 {\n\nstruct PCRETest {\n const char* regexp;\n bool should_match;\n};\n\nstatic PCRETest tests[] = {\n // Most things should behave exactly.\n { \"abc\", true },\n { \"(a|b)c\", true },\n { \"(a*|b)c\", true },\n { \"(a|b*)c\", true },\n { \"a(b|c)d\", true },\n { \"a(()|())c\", true },\n { \"ab*c\", true },\n { \"ab+c\", true },\n { \"a(b*|c*)d\", true },\n { \"\\\\W\", true },\n { \"\\\\W{1,2}\", true },\n { \"\\\\d\", true },\n\n // Check that repeated empty strings do not.\n { \"(a*)*\", false },\n { \"x(a*)*y\", false },\n { \"(a*)+\", false },\n { \"(a+)*\", true },\n { \"(a+)+\", true },\n { \"(a+)+\", true },\n\n // \\v is the only character class that shouldn't.\n { \"\\\\b\", true },\n { \"\\\\v\", false },\n { \"\\\\d\", true },\n\n // The handling of ^ in multi-line mode is different, as is\n // the handling of $ in single-line mode. (Both involve\n // boundary cases if the string ends with \\n.)\n { \"\\\\A\", true },\n { \"\\\\z\", true },\n { \"(?m)^\", false },\n { \"(?m)$\", true },\n { \"(?-m)^\", true },\n { \"(?-m)$\", false }, // In PCRE, == \\Z\n { \"(?m)\\\\A\", true },\n { \"(?m)\\\\z\", true },\n { \"(?-m)\\\\A\", true },\n { \"(?-m)\\\\z\", true },\n};\n\nTEST(MimicsPCRE, SimpleTests) {\n for (size_t i = 0; i < ABSL_ARRAYSIZE(tests); i++) {\n const PCRETest& t = tests[i];\n for (size_t j = 0; j < 2; j++) {\n Regexp::ParseFlags flags = Regexp::LikePerl;\n if (j == 0)\n flags = flags | Regexp::Latin1;\n Regexp* re = Regexp::Parse(t.regexp, flags, NULL);\n ASSERT_TRUE(re != NULL) << \" \" << t.regexp;\n ASSERT_EQ(t.should_match, re->MimicsPCRE())\n << \" \" << t.regexp << \" \"\n << (j == 0 ? \"latin1\" : \"utf\");\n re->Decref();\n }\n }\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/null_walker.cc", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \"absl/log/absl_log.h\"\n#include \"re2/regexp.h\"\n#include \"re2/walker-inl.h\"\n\nnamespace re2 {\n\n// Null walker. For benchmarking the walker itself.\n\nclass NullWalker : public Regexp::Walker {\n public:\n NullWalker() {}\n\n virtual bool PostVisit(Regexp* re, bool parent_arg, bool pre_arg,\n bool* child_args, int nchild_args);\n\n virtual bool ShortVisit(Regexp* re, bool a) {\n // Should never be called: we use Walk(), not WalkExponential().\n#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION\n ABSL_LOG(DFATAL) << \"NullWalker::ShortVisit called\";\n#endif\n return a;\n }\n\n private:\n NullWalker(const NullWalker&) = delete;\n NullWalker& operator=(const NullWalker&) = delete;\n};\n\n// Called after visiting re's children. child_args contains the return\n// value from each of the children's PostVisits (i.e., whether each child\n// can match an empty string). Returns whether this clause can match an\n// empty string.\nbool NullWalker::PostVisit(Regexp* re, bool parent_arg, bool pre_arg,\n bool* child_args, int nchild_args) {\n return false;\n}\n\n// Returns whether re can match an empty string.\nvoid Regexp::NullWalk() {\n NullWalker w;\n w.Walk(this, false);\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/parse_test.cc", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Test parse.cc, dump.cc, and tostring.cc.\n\n#include \n\n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_log.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/regexp.h\"\n\nnamespace re2 {\n\n// In the past, we used 1<<30 here and zeroed the bit later, but that\n// has undefined behaviour, so now we use an internal-only flag because\n// otherwise we would have to introduce a new flag value just for this.\nstatic const Regexp::ParseFlags TestZeroFlags = Regexp::WasDollar;\n\nstruct Test {\n const char* regexp;\n const char* parse;\n Regexp::ParseFlags flags;\n};\n\nstatic Regexp::ParseFlags kTestFlags = Regexp::MatchNL |\n Regexp::PerlX |\n Regexp::PerlClasses |\n Regexp::UnicodeGroups;\n\nstatic Test tests[] = {\n // Base cases\n { \"a\", \"lit{a}\" },\n { \"a.\", \"cat{lit{a}dot{}}\" },\n { \"a.b\", \"cat{lit{a}dot{}lit{b}}\" },\n { \"ab\", \"str{ab}\" },\n { \"a.b.c\", \"cat{lit{a}dot{}lit{b}dot{}lit{c}}\" },\n { \"abc\", \"str{abc}\" },\n { \"a|^\", \"alt{lit{a}bol{}}\" },\n { \"a|b\", \"cc{0x61-0x62}\" },\n { \"(a)\", \"cap{lit{a}}\" },\n { \"(a)|b\", \"alt{cap{lit{a}}lit{b}}\" },\n { \"a*\", \"star{lit{a}}\" },\n { \"a+\", \"plus{lit{a}}\" },\n { \"a?\", \"que{lit{a}}\" },\n { \"a{2}\", \"rep{2,2 lit{a}}\" },\n { \"a{2,3}\", \"rep{2,3 lit{a}}\" },\n { \"a{2,}\", \"rep{2,-1 lit{a}}\" },\n { \"a*?\", \"nstar{lit{a}}\" },\n { \"a+?\", \"nplus{lit{a}}\" },\n { \"a??\", \"nque{lit{a}}\" },\n { \"a{2}?\", \"nrep{2,2 lit{a}}\" },\n { \"a{2,3}?\", \"nrep{2,3 lit{a}}\" },\n { \"a{2,}?\", \"nrep{2,-1 lit{a}}\" },\n { \"\", \"emp{}\" },\n { \"|\", \"alt{emp{}emp{}}\" },\n { \"|x|\", \"alt{emp{}lit{x}emp{}}\" },\n { \".\", \"dot{}\" },\n { \"^\", \"bol{}\" },\n { \"$\", \"eol{}\" },\n { \"\\\\|\", \"lit{|}\" },\n { \"\\\\(\", \"lit{(}\" },\n { \"\\\\)\", \"lit{)}\" },\n { \"\\\\*\", \"lit{*}\" },\n { \"\\\\+\", \"lit{+}\" },\n { \"\\\\?\", \"lit{?}\" },\n { \"{\", \"lit{{}\" },\n { \"}\", \"lit{}}\" },\n { \"\\\\.\", \"lit{.}\" },\n { \"\\\\^\", \"lit{^}\" },\n { \"\\\\$\", \"lit{$}\" },\n { \"\\\\\\\\\", \"lit{\\\\}\" },\n { \"[ace]\", \"cc{0x61 0x63 0x65}\" },\n { \"[abc]\", \"cc{0x61-0x63}\" },\n { \"[a-z]\", \"cc{0x61-0x7a}\" },\n { \"[a]\", \"lit{a}\" },\n { \"\\\\-\", \"lit{-}\" },\n { \"-\", \"lit{-}\" },\n { \"\\\\_\", \"lit{_}\" },\n\n // Posix and Perl extensions\n { \"[[:lower:]]\", \"cc{0x61-0x7a}\" },\n { \"[a-z]\", \"cc{0x61-0x7a}\" },\n { \"[^[:lower:]]\", \"cc{0-0x60 0x7b-0x10ffff}\" },\n { \"[[:^lower:]]\", \"cc{0-0x60 0x7b-0x10ffff}\" },\n { \"(?i)[[:lower:]]\", \"cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}\" },\n { \"(?i)[a-z]\", \"cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}\" },\n { \"(?i)[^[:lower:]]\", \"cc{0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}\" },\n { \"(?i)[[:^lower:]]\", \"cc{0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}\" },\n { \"\\\\d\", \"cc{0x30-0x39}\" },\n { \"\\\\D\", \"cc{0-0x2f 0x3a-0x10ffff}\" },\n { \"\\\\s\", \"cc{0x9-0xa 0xc-0xd 0x20}\" },\n { \"\\\\S\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\" },\n { \"\\\\w\", \"cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}\" },\n { \"\\\\W\", \"cc{0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}\" },\n { \"(?i)\\\\w\", \"cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}\" },\n { \"(?i)\\\\W\", \"cc{0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}\" },\n { \"[^\\\\\\\\]\", \"cc{0-0x5b 0x5d-0x10ffff}\" },\n { \"\\\\C\", \"byte{}\" },\n\n // Unicode, negatives, and a double negative.\n { \"\\\\p{Braille}\", \"cc{0x2800-0x28ff}\" },\n { \"\\\\P{Braille}\", \"cc{0-0x27ff 0x2900-0x10ffff}\" },\n { \"\\\\p{^Braille}\", \"cc{0-0x27ff 0x2900-0x10ffff}\" },\n { \"\\\\P{^Braille}\", \"cc{0x2800-0x28ff}\" },\n\n // More interesting regular expressions.\n { \"a{,2}\", \"str{a{,2}}\" },\n { \"\\\\.\\\\^\\\\$\\\\\\\\\", \"str{.^$\\\\}\" },\n { \"[a-zABC]\", \"cc{0x41-0x43 0x61-0x7a}\" },\n { \"[^a]\", \"cc{0-0x60 0x62-0x10ffff}\" },\n { \"[\\xce\\xb1-\\xce\\xb5\\xe2\\x98\\xba]\", \"cc{0x3b1-0x3b5 0x263a}\" }, // utf-8\n { \"a*{\", \"cat{star{lit{a}}lit{{}}\" },\n\n // Test precedences\n { \"(?:ab)*\", \"star{str{ab}}\" },\n { \"(ab)*\", \"star{cap{str{ab}}}\" },\n { \"ab|cd\", \"alt{str{ab}str{cd}}\" },\n { \"a(b|c)d\", \"cat{lit{a}cap{cc{0x62-0x63}}lit{d}}\" },\n\n // Test squashing of **, ++, ?? et cetera.\n { \"(?:(?:a)*)*\", \"star{lit{a}}\" },\n { \"(?:(?:a)+)+\", \"plus{lit{a}}\" },\n { \"(?:(?:a)?)?\", \"que{lit{a}}\" },\n { \"(?:(?:a)*)+\", \"star{lit{a}}\" },\n { \"(?:(?:a)*)?\", \"star{lit{a}}\" },\n { \"(?:(?:a)+)*\", \"star{lit{a}}\" },\n { \"(?:(?:a)+)?\", \"star{lit{a}}\" },\n { \"(?:(?:a)?)*\", \"star{lit{a}}\" },\n { \"(?:(?:a)?)+\", \"star{lit{a}}\" },\n\n // Test flattening.\n { \"(?:a)\", \"lit{a}\" },\n { \"(?:ab)(?:cd)\", \"str{abcd}\" },\n { \"(?:a|b)|(?:c|d)\", \"cc{0x61-0x64}\" },\n { \"a|c\", \"cc{0x61 0x63}\" },\n { \"a|[cd]\", \"cc{0x61 0x63-0x64}\" },\n { \"a|.\", \"dot{}\" },\n { \"[ab]|c\", \"cc{0x61-0x63}\" },\n { \"[ab]|[cd]\", \"cc{0x61-0x64}\" },\n { \"[ab]|.\", \"dot{}\" },\n { \".|c\", \"dot{}\" },\n { \".|[cd]\", \"dot{}\" },\n { \".|.\", \"dot{}\" },\n\n // Test Perl quoted literals\n { \"\\\\Q+|*?{[\\\\E\", \"str{+|*?{[}\" },\n { \"\\\\Q+\\\\E+\", \"plus{lit{+}}\" },\n { \"\\\\Q\\\\\\\\E\", \"lit{\\\\}\" },\n { \"\\\\Q\\\\\\\\\\\\E\", \"str{\\\\\\\\}\" },\n { \"\\\\Qa\\\\E*\", \"star{lit{a}}\" },\n { \"\\\\Qab\\\\E*\", \"cat{lit{a}star{lit{b}}}\" },\n { \"\\\\Qabc\\\\E*\", \"cat{str{ab}star{lit{c}}}\" },\n\n // Test Perl \\A and \\z\n { \"(?m)^\", \"bol{}\" },\n { \"(?m)$\", \"eol{}\" },\n { \"(?-m)^\", \"bot{}\" },\n { \"(?-m)$\", \"eot{}\" },\n { \"(?m)\\\\A\", \"bot{}\" },\n { \"(?m)\\\\z\", \"eot{\\\\z}\" },\n { \"(?-m)\\\\A\", \"bot{}\" },\n { \"(?-m)\\\\z\", \"eot{\\\\z}\" },\n\n // Test named captures\n { \"(?Pa)\", \"cap{name:lit{a}}\" },\n { \"(?P<中文>a)\", \"cap{中文:lit{a}}\" },\n { \"(?a)\", \"cap{name:lit{a}}\" },\n { \"(?<中文>a)\", \"cap{中文:lit{a}}\" },\n\n // Case-folded literals\n { \"[Aa]\", \"litfold{a}\" },\n\n // Strings\n { \"abcde\", \"str{abcde}\" },\n { \"[Aa][Bb]cd\", \"cat{strfold{ab}str{cd}}\" },\n\n // Reported bug involving \\n leaking in despite use of NeverNL.\n { \"[^ ]\", \"cc{0-0x9 0xb-0x1f 0x21-0x10ffff}\", TestZeroFlags },\n { \"[^ ]\", \"cc{0-0x9 0xb-0x1f 0x21-0x10ffff}\", Regexp::FoldCase },\n { \"[^ ]\", \"cc{0-0x9 0xb-0x1f 0x21-0x10ffff}\", Regexp::NeverNL },\n { \"[^ ]\", \"cc{0-0x9 0xb-0x1f 0x21-0x10ffff}\", Regexp::NeverNL | Regexp::FoldCase },\n { \"[^ \\f]\", \"cc{0-0x9 0xb 0xd-0x1f 0x21-0x10ffff}\", TestZeroFlags },\n { \"[^ \\f]\", \"cc{0-0x9 0xb 0xd-0x1f 0x21-0x10ffff}\", Regexp::FoldCase },\n { \"[^ \\f]\", \"cc{0-0x9 0xb 0xd-0x1f 0x21-0x10ffff}\", Regexp::NeverNL },\n { \"[^ \\f]\", \"cc{0-0x9 0xb 0xd-0x1f 0x21-0x10ffff}\", Regexp::NeverNL | Regexp::FoldCase },\n { \"[^ \\r]\", \"cc{0-0x9 0xb-0xc 0xe-0x1f 0x21-0x10ffff}\", TestZeroFlags },\n { \"[^ \\r]\", \"cc{0-0x9 0xb-0xc 0xe-0x1f 0x21-0x10ffff}\", Regexp::FoldCase },\n { \"[^ \\r]\", \"cc{0-0x9 0xb-0xc 0xe-0x1f 0x21-0x10ffff}\", Regexp::NeverNL },\n { \"[^ \\r]\", \"cc{0-0x9 0xb-0xc 0xe-0x1f 0x21-0x10ffff}\", Regexp::NeverNL | Regexp::FoldCase },\n { \"[^ \\v]\", \"cc{0-0x9 0xc-0x1f 0x21-0x10ffff}\", TestZeroFlags },\n { \"[^ \\v]\", \"cc{0-0x9 0xc-0x1f 0x21-0x10ffff}\", Regexp::FoldCase },\n { \"[^ \\v]\", \"cc{0-0x9 0xc-0x1f 0x21-0x10ffff}\", Regexp::NeverNL },\n { \"[^ \\v]\", \"cc{0-0x9 0xc-0x1f 0x21-0x10ffff}\", Regexp::NeverNL | Regexp::FoldCase },\n { \"[^ \\t]\", \"cc{0-0x8 0xb-0x1f 0x21-0x10ffff}\", TestZeroFlags },\n { \"[^ \\t]\", \"cc{0-0x8 0xb-0x1f 0x21-0x10ffff}\", Regexp::FoldCase },\n { \"[^ \\t]\", \"cc{0-0x8 0xb-0x1f 0x21-0x10ffff}\", Regexp::NeverNL },\n { \"[^ \\t]\", \"cc{0-0x8 0xb-0x1f 0x21-0x10ffff}\", Regexp::NeverNL | Regexp::FoldCase },\n { \"[^ \\r\\f\\v]\", \"cc{0-0x9 0xe-0x1f 0x21-0x10ffff}\", Regexp::NeverNL },\n { \"[^ \\r\\f\\v]\", \"cc{0-0x9 0xe-0x1f 0x21-0x10ffff}\", Regexp::NeverNL | Regexp::FoldCase },\n { \"[^ \\r\\f\\t\\v]\", \"cc{0-0x8 0xe-0x1f 0x21-0x10ffff}\", Regexp::NeverNL },\n { \"[^ \\r\\f\\t\\v]\", \"cc{0-0x8 0xe-0x1f 0x21-0x10ffff}\", Regexp::NeverNL | Regexp::FoldCase },\n { \"[^ \\r\\n\\f\\t\\v]\", \"cc{0-0x8 0xe-0x1f 0x21-0x10ffff}\", Regexp::NeverNL },\n { \"[^ \\r\\n\\f\\t\\v]\", \"cc{0-0x8 0xe-0x1f 0x21-0x10ffff}\", Regexp::NeverNL | Regexp::FoldCase },\n { \"[^ \\r\\n\\f\\t]\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\", Regexp::NeverNL },\n { \"[^ \\r\\n\\f\\t]\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\", Regexp::NeverNL | Regexp::FoldCase },\n { \"[^\\t-\\n\\f-\\r ]\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\",\n Regexp::PerlClasses },\n { \"[^\\t-\\n\\f-\\r ]\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\",\n Regexp::PerlClasses | Regexp::FoldCase },\n { \"[^\\t-\\n\\f-\\r ]\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\",\n Regexp::PerlClasses | Regexp::NeverNL },\n { \"[^\\t-\\n\\f-\\r ]\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\",\n Regexp::PerlClasses | Regexp::NeverNL | Regexp::FoldCase },\n { \"\\\\S\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\",\n Regexp::PerlClasses },\n { \"\\\\S\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\",\n Regexp::PerlClasses | Regexp::FoldCase },\n { \"\\\\S\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\",\n Regexp::PerlClasses | Regexp::NeverNL },\n { \"\\\\S\", \"cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}\",\n Regexp::PerlClasses | Regexp::NeverNL | Regexp::FoldCase },\n\n // Bug in Regexp::ToString() that emitted [^], which\n // would (obviously) fail to parse when fed back in.\n { \"[\\\\s\\\\S]\", \"cc{0-0x10ffff}\" },\n\n // As per https://github.com/google/re2/issues/477,\n // there were long-standing bugs involving Latin-1.\n // Here, we exercise it WITHOUT case folding...\n { \"\\xa5\\x64\\xd1\", \"str{\\xa5\"\"d\\xd1}\", Regexp::Latin1 },\n { \"\\xa5\\xd1\\x64\", \"str{\\xa5\\xd1\"\"d}\", Regexp::Latin1 },\n { \"\\xa5\\x64[\\xd1\\xd2]\", \"cat{str{\\xa5\"\"d}cc{0xd1-0xd2}}\", Regexp::Latin1 },\n { \"\\xa5[\\xd1\\xd2]\\x64\", \"cat{lit{\\xa5}cc{0xd1-0xd2}lit{d}}\", Regexp::Latin1 },\n { \"\\xa5\\x64|\\xa5\\xd1\", \"cat{lit{\\xa5}cc{0x64 0xd1}}\", Regexp::Latin1 },\n { \"\\xa5\\xd1|\\xa5\\x64\", \"cat{lit{\\xa5}cc{0x64 0xd1}}\", Regexp::Latin1 },\n { \"\\xa5\\x64|\\xa5[\\xd1\\xd2]\", \"cat{lit{\\xa5}cc{0x64 0xd1-0xd2}}\", Regexp::Latin1 },\n { \"\\xa5[\\xd1\\xd2]|\\xa5\\x64\", \"cat{lit{\\xa5}cc{0x64 0xd1-0xd2}}\", Regexp::Latin1 },\n // Here, we exercise it WITH case folding...\n // 0x64 should fold to 0x44, but neither 0xD1 nor 0xD2\n // should fold to 0xF1 and 0xF2, respectively.\n { \"\\xa5\\x64\\xd1\", \"strfold{\\xa5\"\"d\\xd1}\", Regexp::Latin1 | Regexp::FoldCase },\n { \"\\xa5\\xd1\\x64\", \"strfold{\\xa5\\xd1\"\"d}\", Regexp::Latin1 | Regexp::FoldCase },\n { \"\\xa5\\x64[\\xd1\\xd2]\", \"cat{strfold{\\xa5\"\"d}cc{0xd1-0xd2}}\", Regexp::Latin1 | Regexp::FoldCase },\n { \"\\xa5[\\xd1\\xd2]\\x64\", \"cat{lit{\\xa5}cc{0xd1-0xd2}litfold{d}}\", Regexp::Latin1 | Regexp::FoldCase },\n { \"\\xa5\\x64|\\xa5\\xd1\", \"cat{lit{\\xa5}cc{0x44 0x64 0xd1}}\", Regexp::Latin1 | Regexp::FoldCase },\n { \"\\xa5\\xd1|\\xa5\\x64\", \"cat{lit{\\xa5}cc{0x44 0x64 0xd1}}\", Regexp::Latin1 | Regexp::FoldCase },\n { \"\\xa5\\x64|\\xa5[\\xd1\\xd2]\", \"cat{lit{\\xa5}cc{0x44 0x64 0xd1-0xd2}}\", Regexp::Latin1 | Regexp::FoldCase },\n { \"\\xa5[\\xd1\\xd2]|\\xa5\\x64\", \"cat{lit{\\xa5}cc{0x44 0x64 0xd1-0xd2}}\", Regexp::Latin1 | Regexp::FoldCase },\n};\n\nbool RegexpEqualTestingOnly(Regexp* a, Regexp* b) {\n return Regexp::Equal(a, b);\n}\n\nvoid TestParse(const Test* tests, int ntests, Regexp::ParseFlags flags,\n const std::string& title) {\n Regexp** re = new Regexp*[ntests];\n for (int i = 0; i < ntests; i++) {\n RegexpStatus status;\n Regexp::ParseFlags f = flags;\n if (tests[i].flags != 0) {\n f = tests[i].flags & ~TestZeroFlags;\n }\n re[i] = Regexp::Parse(tests[i].regexp, f, &status);\n ASSERT_TRUE(re[i] != NULL)\n << \" \" << tests[i].regexp << \" \" << status.Text();\n std::string s = re[i]->Dump();\n EXPECT_EQ(std::string(tests[i].parse), s)\n << \"Regexp: \" << tests[i].regexp\n << \"\\nparse: \" << std::string(tests[i].parse)\n << \" s: \" << s << \" flag=\" << f;\n }\n\n for (int i = 0; i < ntests; i++) {\n for (int j = 0; j < ntests; j++) {\n EXPECT_EQ(std::string(tests[i].parse) == std::string(tests[j].parse),\n RegexpEqualTestingOnly(re[i], re[j]))\n << \"Regexp: \" << tests[i].regexp << \" \" << tests[j].regexp;\n }\n }\n\n for (int i = 0; i < ntests; i++)\n re[i]->Decref();\n delete[] re;\n}\n\n// Test that regexps parse to expected structures.\nTEST(TestParse, SimpleRegexps) {\n TestParse(tests, ABSL_ARRAYSIZE(tests), kTestFlags, \"simple\");\n}\n\nTest foldcase_tests[] = {\n { \"AbCdE\", \"strfold{abcde}\" },\n { \"[Aa]\", \"litfold{a}\" },\n { \"a\", \"litfold{a}\" },\n\n // 0x17F is an old English long s (looks like an f) and folds to s.\n // 0x212A is the Kelvin symbol and folds to k.\n { \"A[F-g]\", \"cat{litfold{a}cc{0x41-0x7a 0x17f 0x212a}}\" }, // [Aa][A-z...]\n { \"[[:upper:]]\", \"cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}\" },\n { \"[[:lower:]]\", \"cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}\" },\n};\n\n// Test that parsing with FoldCase works.\nTEST(TestParse, FoldCase) {\n TestParse(foldcase_tests, ABSL_ARRAYSIZE(foldcase_tests), Regexp::FoldCase, \"foldcase\");\n}\n\nTest literal_tests[] = {\n { \"(|)^$.[*+?]{5,10},\\\\\", \"str{(|)^$.[*+?]{5,10},\\\\}\" },\n};\n\n// Test that parsing with Literal works.\nTEST(TestParse, Literal) {\n TestParse(literal_tests, ABSL_ARRAYSIZE(literal_tests), Regexp::Literal, \"literal\");\n}\n\nTest matchnl_tests[] = {\n { \".\", \"dot{}\" },\n { \"\\n\", \"lit{\\n}\" },\n { \"[^a]\", \"cc{0-0x60 0x62-0x10ffff}\" },\n { \"[a\\\\n]\", \"cc{0xa 0x61}\" },\n};\n\n// Test that parsing with MatchNL works.\n// (Also tested above during simple cases.)\nTEST(TestParse, MatchNL) {\n TestParse(matchnl_tests, ABSL_ARRAYSIZE(matchnl_tests), Regexp::MatchNL, \"with MatchNL\");\n}\n\nTest nomatchnl_tests[] = {\n { \".\", \"cc{0-0x9 0xb-0x10ffff}\" },\n { \"\\n\", \"lit{\\n}\" },\n { \"[^a]\", \"cc{0-0x9 0xb-0x60 0x62-0x10ffff}\" },\n { \"[a\\\\n]\", \"cc{0xa 0x61}\" },\n};\n\n// Test that parsing without MatchNL works.\nTEST(TestParse, NoMatchNL) {\n TestParse(nomatchnl_tests, ABSL_ARRAYSIZE(nomatchnl_tests), Regexp::NoParseFlags, \"without MatchNL\");\n}\n\nTest prefix_tests[] = {\n { \"abc|abd\", \"cat{str{ab}cc{0x63-0x64}}\" },\n { \"a(?:b)c|abd\", \"cat{str{ab}cc{0x63-0x64}}\" },\n { \"abc|abd|aef|bcx|bcy\",\n \"alt{cat{lit{a}alt{cat{lit{b}cc{0x63-0x64}}str{ef}}}\"\n \"cat{str{bc}cc{0x78-0x79}}}\" },\n { \"abc|x|abd\", \"alt{str{abc}lit{x}str{abd}}\" },\n { \"(?i)abc|ABD\", \"cat{strfold{ab}cc{0x43-0x44 0x63-0x64}}\" },\n { \"[ab]c|[ab]d\", \"cat{cc{0x61-0x62}cc{0x63-0x64}}\" },\n { \".c|.d\", \"cat{cc{0-0x9 0xb-0x10ffff}cc{0x63-0x64}}\" },\n { \"\\\\Cc|\\\\Cd\", \"cat{byte{}cc{0x63-0x64}}\" },\n { \"x{2}|x{2}[0-9]\",\n \"cat{rep{2,2 lit{x}}alt{emp{}cc{0x30-0x39}}}\" },\n { \"x{2}y|x{2}[0-9]y\",\n \"cat{rep{2,2 lit{x}}alt{lit{y}cat{cc{0x30-0x39}lit{y}}}}\" },\n { \"n|r|rs\",\n \"alt{lit{n}cat{lit{r}alt{emp{}lit{s}}}}\" },\n { \"n|rs|r\",\n \"alt{lit{n}cat{lit{r}alt{lit{s}emp{}}}}\" },\n { \"r|rs|n\",\n \"alt{cat{lit{r}alt{emp{}lit{s}}}lit{n}}\" },\n { \"rs|r|n\",\n \"alt{cat{lit{r}alt{lit{s}emp{}}}lit{n}}\" },\n { \"a\\\\C*?c|a\\\\C*?b\",\n \"cat{lit{a}alt{cat{nstar{byte{}}lit{c}}cat{nstar{byte{}}lit{b}}}}\" },\n { \"^/a/bc|^/a/de\",\n \"cat{bol{}cat{str{/a/}alt{str{bc}str{de}}}}\" },\n // In the past, factoring was limited to kFactorAlternationMaxDepth (8).\n { \"a|aa|aaa|aaaa|aaaaa|aaaaaa|aaaaaaa|aaaaaaaa|aaaaaaaaa|aaaaaaaaaa\",\n \"cat{lit{a}alt{emp{}\" \"cat{lit{a}alt{emp{}\" \"cat{lit{a}alt{emp{}\"\n \"cat{lit{a}alt{emp{}\" \"cat{lit{a}alt{emp{}\" \"cat{lit{a}alt{emp{}\"\n \"cat{lit{a}alt{emp{}\" \"cat{lit{a}alt{emp{}\" \"cat{lit{a}alt{emp{}\"\n \"lit{a}}}}}}}}}}}}}}}}}}}\" },\n { \"a|aardvark|aardvarks|abaci|aback|abacus|abacuses|abaft|abalone|abalones\",\n \"cat{lit{a}alt{emp{}cat{str{ardvark}alt{emp{}lit{s}}}\"\n \"cat{str{ba}alt{cat{lit{c}alt{cc{0x69 0x6b}cat{str{us}alt{emp{}str{es}}}}}\"\n \"str{ft}cat{str{lone}alt{emp{}lit{s}}}}}}}\" },\n // As per https://github.com/google/re2/issues/467,\n // these should factor identically, but they didn't\n // because AddFoldedRange() terminated prematurely.\n { \"0A|0[aA]\", \"cat{lit{0}cc{0x41 0x61}}\" },\n { \"0a|0[aA]\", \"cat{lit{0}cc{0x41 0x61}}\" },\n { \"0[aA]|0A\", \"cat{lit{0}cc{0x41 0x61}}\" },\n { \"0[aA]|0a\", \"cat{lit{0}cc{0x41 0x61}}\" },\n};\n\n// Test that prefix factoring works.\nTEST(TestParse, Prefix) {\n TestParse(prefix_tests, ABSL_ARRAYSIZE(prefix_tests), Regexp::PerlX, \"prefix\");\n}\n\nTest nested_tests[] = {\n { \"((((((((((x{2}){2}){2}){2}){2}){2}){2}){2}){2}))\",\n \"cap{cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 lit{x}}}}}}}}}}}}}}}}}}}}\" },\n { \"((((((((((x{1}){2}){2}){2}){2}){2}){2}){2}){2}){2})\",\n \"cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{1,1 lit{x}}}}}}}}}}}}}}}}}}}}}\" },\n { \"((((((((((x{0}){2}){2}){2}){2}){2}){2}){2}){2}){2})\",\n \"cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{0,0 lit{x}}}}}}}}}}}}}}}}}}}}}\" },\n { \"((((((x{2}){2}){2}){5}){5}){5})\",\n \"cap{rep{5,5 cap{rep{5,5 cap{rep{5,5 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 lit{x}}}}}}}}}}}}}\" },\n};\n\n// Test that nested repetition works.\nTEST(TestParse, Nested) {\n TestParse(nested_tests, ABSL_ARRAYSIZE(nested_tests), Regexp::PerlX, \"nested\");\n}\n\n// Invalid regular expressions\nconst char* badtests[] = {\n \"(\",\n \")\",\n \"(a\",\n \"(a|b|\",\n \"(a|b\",\n \"[a-z\",\n \"([a-z)\",\n \"x{1001}\",\n \"\\xff\", // Invalid UTF-8\n \"[\\xff]\",\n \"[\\\\\\xff]\",\n \"\\\\\\xff\",\n \"(?Pa\",\n \"(?P\",\n \"(?Pa)\",\n \"(?P<>a)\",\n \"(?a\",\n \"(?\",\n \"(?a)\",\n \"(?<>a)\",\n \"[a-Z]\",\n \"(?i)[a-Z]\",\n \"a{100000}\",\n \"a{100000,}\",\n \"((((((((((x{2}){2}){2}){2}){2}){2}){2}){2}){2}){2})\",\n \"(((x{7}){11}){13})\",\n \"\\\\Q\\\\E*\",\n};\n\n// Valid in Perl, bad in POSIX\nconst char* only_perl[] = {\n \"[a-b-c]\",\n \"\\\\Qabc\\\\E\",\n \"\\\\Q*+?{[\\\\E\",\n \"\\\\Q\\\\\\\\E\",\n \"\\\\Q\\\\\\\\\\\\E\",\n \"\\\\Q\\\\\\\\\\\\\\\\E\",\n \"\\\\Q\\\\\\\\\\\\\\\\\\\\E\",\n \"(?:a)\",\n \"(?Pa)\",\n \"(?a)\",\n};\n\n// Valid in POSIX, bad in Perl.\nconst char* only_posix[] = {\n \"a++\",\n \"a**\",\n \"a?*\",\n \"a+*\",\n \"a{1}*\",\n};\n\n// Test that parser rejects bad regexps.\nTEST(TestParse, InvalidRegexps) {\n for (size_t i = 0; i < ABSL_ARRAYSIZE(badtests); i++) {\n ASSERT_TRUE(Regexp::Parse(badtests[i], Regexp::PerlX, NULL) == NULL)\n << \" \" << badtests[i];\n ASSERT_TRUE(Regexp::Parse(badtests[i], Regexp::NoParseFlags, NULL) == NULL)\n << \" \" << badtests[i];\n }\n for (size_t i = 0; i < ABSL_ARRAYSIZE(only_posix); i++) {\n ASSERT_TRUE(Regexp::Parse(only_posix[i], Regexp::PerlX, NULL) == NULL)\n << \" \" << only_posix[i];\n Regexp* re = Regexp::Parse(only_posix[i], Regexp::NoParseFlags, NULL);\n ASSERT_TRUE(re != NULL) << \" \" << only_posix[i];\n re->Decref();\n }\n for (size_t i = 0; i < ABSL_ARRAYSIZE(only_perl); i++) {\n ASSERT_TRUE(Regexp::Parse(only_perl[i], Regexp::NoParseFlags, NULL) == NULL)\n << \" \" << only_perl[i];\n Regexp* re = Regexp::Parse(only_perl[i], Regexp::PerlX, NULL);\n ASSERT_TRUE(re != NULL) << \" \" << only_perl[i];\n re->Decref();\n }\n}\n\n// Test that ToString produces original regexp or equivalent one.\nTEST(TestToString, EquivalentParse) {\n for (size_t i = 0; i < ABSL_ARRAYSIZE(tests); i++) {\n RegexpStatus status;\n Regexp::ParseFlags f = kTestFlags;\n if (tests[i].flags != 0) {\n f = tests[i].flags & ~TestZeroFlags;\n }\n Regexp* re = Regexp::Parse(tests[i].regexp, f, &status);\n ASSERT_TRUE(re != NULL) << \" \" << tests[i].regexp << \" \" << status.Text();\n std::string s = re->Dump();\n EXPECT_EQ(std::string(tests[i].parse), s)\n << \"Regexp: \" << tests[i].regexp\n << \"\\nparse: \" << std::string(tests[i].parse)\n << \" s: \" << s << \" flag=\" << f;\n std::string t = re->ToString();\n if (t != tests[i].regexp) {\n // If ToString didn't return the original regexp,\n // it must have found one with fewer parens.\n // Unfortunately we can't check the length here, because\n // ToString produces \"\\\\{\" for a literal brace,\n // but \"{\" is a shorter equivalent.\n // ASSERT_LT(t.size(), strlen(tests[i].regexp))\n // << \" t=\" << t << \" regexp=\" << tests[i].regexp;\n\n // Test that if we parse the new regexp we get the same structure.\n Regexp* nre = Regexp::Parse(t, f, &status);\n ASSERT_TRUE(nre != NULL) << \" reparse \" << t << \" \" << status.Text();\n std::string ss = nre->Dump();\n std::string tt = nre->ToString();\n if (s != ss || t != tt)\n ABSL_LOG(INFO) << \"ToString(\" << tests[i].regexp << \") = \" << t;\n EXPECT_EQ(s, ss);\n EXPECT_EQ(t, tt);\n nre->Decref();\n }\n re->Decref();\n }\n}\n\n// Test that capture error args are correct.\nTEST(NamedCaptures, ErrorArgs) {\n RegexpStatus status;\n Regexp* re;\n\n re = Regexp::Parse(\"test(?Pz)\", Regexp::LikePerl, &status);\n EXPECT_TRUE(re == NULL);\n EXPECT_EQ(status.code(), kRegexpBadNamedCapture);\n EXPECT_EQ(status.error_arg(), \"(?P\");\n\n re = Regexp::Parse(\"test(?z)\", Regexp::LikePerl, &status);\n EXPECT_TRUE(re == NULL);\n EXPECT_EQ(status.code(), kRegexpBadNamedCapture);\n EXPECT_EQ(status.error_arg(), \"(?\");\n}\n\n// Test that look-around error args are correct.\nTEST(LookAround, ErrorArgs) {\n RegexpStatus status;\n Regexp* re;\n\n re = Regexp::Parse(\"(?=foo).*\", Regexp::LikePerl, &status);\n EXPECT_TRUE(re == NULL);\n EXPECT_EQ(status.code(), kRegexpBadPerlOp);\n EXPECT_EQ(status.error_arg(), \"(?=\");\n\n re = Regexp::Parse(\"(?!foo).*\", Regexp::LikePerl, &status);\n EXPECT_TRUE(re == NULL);\n EXPECT_EQ(status.code(), kRegexpBadPerlOp);\n EXPECT_EQ(status.error_arg(), \"(?!\");\n\n re = Regexp::Parse(\"(?<=foo).*\", Regexp::LikePerl, &status);\n EXPECT_TRUE(re == NULL);\n EXPECT_EQ(status.code(), kRegexpBadPerlOp);\n EXPECT_EQ(status.error_arg(), \"(?<=\");\n\n re = Regexp::Parse(\"(?\n\n#include \n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/escaping.h\"\n#include \"absl/strings/string_view.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/prog.h\"\n#include \"re2/re2.h\"\n#include \"re2/regexp.h\"\n#include \"re2/testing/exhaustive_tester.h\"\n#include \"re2/testing/regexp_generator.h\"\n#include \"re2/testing/string_generator.h\"\n\nnamespace re2 {\n\n// Test that C++ strings are compared as uint8s, not int8s.\n// PossibleMatchRange doesn't depend on this, but callers probably will.\nTEST(CplusplusStrings, EightBit) {\n std::string s = \"\\x70\";\n std::string t = \"\\xA0\";\n EXPECT_LT(s, t);\n}\n\nstruct PrefixTest {\n const char* regexp;\n int maxlen;\n const char* min;\n const char* max;\n};\n\nstatic PrefixTest tests[] = {\n { \"\", 10, \"\", \"\", },\n { \"Abcdef\", 10, \"Abcdef\", \"Abcdef\" },\n { \"abc(def|ghi)\", 10, \"abcdef\", \"abcghi\" },\n { \"a+hello\", 10, \"aa\", \"ahello\" },\n { \"a*hello\", 10, \"a\", \"hello\" },\n { \"def|abc\", 10, \"abc\", \"def\" },\n { \"a(b)(c)[d]\", 10, \"abcd\", \"abcd\" },\n { \"ab(cab|cat)\", 10, \"abcab\", \"abcat\" },\n { \"ab(cab|ca)x\", 10, \"abcabx\", \"abcax\" },\n { \"(ab|x)(c|de)\", 10, \"abc\", \"xde\" },\n { \"(ab|x)?(c|z)?\", 10, \"\", \"z\" },\n { \"[^\\\\s\\\\S]\", 10, \"\", \"\" },\n { \"(abc)+\", 5, \"abc\", \"abcac\" },\n { \"(abc)+\", 2, \"ab\", \"ac\" },\n { \"(abc)+\", 1, \"a\", \"b\" },\n { \"[a\\xC3\\xA1]\", 4, \"a\", \"\\xC3\\xA1\" },\n { \"a*\", 10, \"\", \"ab\" },\n\n { \"(?i)Abcdef\", 10, \"ABCDEF\", \"abcdef\" },\n { \"(?i)abc(def|ghi)\", 10, \"ABCDEF\", \"abcghi\" },\n { \"(?i)a+hello\", 10, \"AA\", \"ahello\" },\n { \"(?i)a*hello\", 10, \"A\", \"hello\" },\n { \"(?i)def|abc\", 10, \"ABC\", \"def\" },\n { \"(?i)a(b)(c)[d]\", 10, \"ABCD\", \"abcd\" },\n { \"(?i)ab(cab|cat)\", 10, \"ABCAB\", \"abcat\" },\n { \"(?i)ab(cab|ca)x\", 10, \"ABCABX\", \"abcax\" },\n { \"(?i)(ab|x)(c|de)\", 10, \"ABC\", \"xde\" },\n { \"(?i)(ab|x)?(c|z)?\", 10, \"\", \"z\" },\n { \"(?i)[^\\\\s\\\\S]\", 10, \"\", \"\" },\n { \"(?i)(abc)+\", 5, \"ABC\", \"abcac\" },\n { \"(?i)(abc)+\", 2, \"AB\", \"ac\" },\n { \"(?i)(abc)+\", 1, \"A\", \"b\" },\n { \"(?i)[a\\xC3\\xA1]\", 4, \"A\", \"\\xC3\\xA1\" },\n { \"(?i)a*\", 10, \"\", \"ab\" },\n { \"(?i)A*\", 10, \"\", \"ab\" },\n\n { \"\\\\AAbcdef\", 10, \"Abcdef\", \"Abcdef\" },\n { \"\\\\Aabc(def|ghi)\", 10, \"abcdef\", \"abcghi\" },\n { \"\\\\Aa+hello\", 10, \"aa\", \"ahello\" },\n { \"\\\\Aa*hello\", 10, \"a\", \"hello\" },\n { \"\\\\Adef|abc\", 10, \"abc\", \"def\" },\n { \"\\\\Aa(b)(c)[d]\", 10, \"abcd\", \"abcd\" },\n { \"\\\\Aab(cab|cat)\", 10, \"abcab\", \"abcat\" },\n { \"\\\\Aab(cab|ca)x\", 10, \"abcabx\", \"abcax\" },\n { \"\\\\A(ab|x)(c|de)\", 10, \"abc\", \"xde\" },\n { \"\\\\A(ab|x)?(c|z)?\", 10, \"\", \"z\" },\n { \"\\\\A[^\\\\s\\\\S]\", 10, \"\", \"\" },\n { \"\\\\A(abc)+\", 5, \"abc\", \"abcac\" },\n { \"\\\\A(abc)+\", 2, \"ab\", \"ac\" },\n { \"\\\\A(abc)+\", 1, \"a\", \"b\" },\n { \"\\\\A[a\\xC3\\xA1]\", 4, \"a\", \"\\xC3\\xA1\" },\n { \"\\\\Aa*\", 10, \"\", \"ab\" },\n\n { \"(?i)\\\\AAbcdef\", 10, \"ABCDEF\", \"abcdef\" },\n { \"(?i)\\\\Aabc(def|ghi)\", 10, \"ABCDEF\", \"abcghi\" },\n { \"(?i)\\\\Aa+hello\", 10, \"AA\", \"ahello\" },\n { \"(?i)\\\\Aa*hello\", 10, \"A\", \"hello\" },\n { \"(?i)\\\\Adef|abc\", 10, \"ABC\", \"def\" },\n { \"(?i)\\\\Aa(b)(c)[d]\", 10, \"ABCD\", \"abcd\" },\n { \"(?i)\\\\Aab(cab|cat)\", 10, \"ABCAB\", \"abcat\" },\n { \"(?i)\\\\Aab(cab|ca)x\", 10, \"ABCABX\", \"abcax\" },\n { \"(?i)\\\\A(ab|x)(c|de)\", 10, \"ABC\", \"xde\" },\n { \"(?i)\\\\A(ab|x)?(c|z)?\", 10, \"\", \"z\" },\n { \"(?i)\\\\A[^\\\\s\\\\S]\", 10, \"\", \"\" },\n { \"(?i)\\\\A(abc)+\", 5, \"ABC\", \"abcac\" },\n { \"(?i)\\\\A(abc)+\", 2, \"AB\", \"ac\" },\n { \"(?i)\\\\A(abc)+\", 1, \"A\", \"b\" },\n { \"(?i)\\\\A[a\\xC3\\xA1]\", 4, \"A\", \"\\xC3\\xA1\" },\n { \"(?i)\\\\Aa*\", 10, \"\", \"ab\" },\n { \"(?i)\\\\AA*\", 10, \"\", \"ab\" },\n};\n\nTEST(PossibleMatchRange, HandWritten) {\n for (size_t i = 0; i < ABSL_ARRAYSIZE(tests); i++) {\n for (size_t j = 0; j < 2; j++) {\n const PrefixTest& t = tests[i];\n std::string min, max;\n if (j == 0) {\n ABSL_LOG(INFO) << \"Checking regexp=\" << absl::CEscape(t.regexp);\n Regexp* re = Regexp::Parse(t.regexp, Regexp::LikePerl, NULL);\n ASSERT_TRUE(re != NULL);\n Prog* prog = re->CompileToProg(0);\n ASSERT_TRUE(prog != NULL);\n ASSERT_TRUE(prog->PossibleMatchRange(&min, &max, t.maxlen))\n << \" \" << t.regexp;\n delete prog;\n re->Decref();\n } else {\n ASSERT_TRUE(RE2(t.regexp).PossibleMatchRange(&min, &max, t.maxlen));\n }\n EXPECT_EQ(t.min, min) << t.regexp;\n EXPECT_EQ(t.max, max) << t.regexp;\n }\n }\n}\n\n// Test cases where PossibleMatchRange should return false.\nTEST(PossibleMatchRange, Failures) {\n std::string min, max;\n\n // Fails because no room to write max.\n EXPECT_FALSE(RE2(\"abc\").PossibleMatchRange(&min, &max, 0));\n\n // Fails because there is no max -- any non-empty string matches\n // or begins a match. Have to use Latin-1 input, because there\n // are no valid UTF-8 strings beginning with byte 0xFF.\n EXPECT_FALSE(RE2(\"[\\\\s\\\\S]+\", RE2::Latin1).\n PossibleMatchRange(&min, &max, 10))\n << \"min=\" << absl::CEscape(min) << \", max=\" << absl::CEscape(max);\n EXPECT_FALSE(RE2(\"[\\\\0-\\xFF]+\", RE2::Latin1).\n PossibleMatchRange(&min, &max, 10))\n << \"min=\" << absl::CEscape(min) << \", max=\" << absl::CEscape(max);\n EXPECT_FALSE(RE2(\".+hello\", RE2::Latin1).\n PossibleMatchRange(&min, &max, 10))\n << \"min=\" << absl::CEscape(min) << \", max=\" << absl::CEscape(max);\n EXPECT_FALSE(RE2(\".*hello\", RE2::Latin1).\n PossibleMatchRange(&min, &max, 10))\n << \"min=\" << absl::CEscape(min) << \", max=\" << absl::CEscape(max);\n EXPECT_FALSE(RE2(\".*\", RE2::Latin1).\n PossibleMatchRange(&min, &max, 10))\n << \"min=\" << absl::CEscape(min) << \", max=\" << absl::CEscape(max);\n EXPECT_FALSE(RE2(\"\\\\C*\").\n PossibleMatchRange(&min, &max, 10))\n << \"min=\" << absl::CEscape(min) << \", max=\" << absl::CEscape(max);\n\n // Fails because it's a malformed regexp.\n EXPECT_FALSE(RE2(\"*hello\").PossibleMatchRange(&min, &max, 10))\n << \"min=\" << absl::CEscape(min) << \", max=\" << absl::CEscape(max);\n}\n\n// Exhaustive test: generate all regexps within parameters,\n// then generate all strings of a given length over a given alphabet,\n// then check that the prefix information agrees with whether\n// the regexp matches each of the strings.\nclass PossibleMatchTester : public RegexpGenerator {\n public:\n PossibleMatchTester(int maxatoms,\n int maxops,\n const std::vector& alphabet,\n const std::vector& ops,\n int maxstrlen,\n const std::vector& stralphabet)\n : RegexpGenerator(maxatoms, maxops, alphabet, ops),\n strgen_(maxstrlen, stralphabet),\n regexps_(0), tests_(0) { }\n\n int regexps() { return regexps_; }\n int tests() { return tests_; }\n\n // Needed for RegexpGenerator interface.\n void HandleRegexp(const std::string& regexp);\n\n private:\n StringGenerator strgen_;\n\n int regexps_; // Number of HandleRegexp calls\n int tests_; // Number of regexp tests.\n\n PossibleMatchTester(const PossibleMatchTester&) = delete;\n PossibleMatchTester& operator=(const PossibleMatchTester&) = delete;\n};\n\n// Processes a single generated regexp.\n// Checks that all accepted strings agree with the prefix range.\nvoid PossibleMatchTester::HandleRegexp(const std::string& regexp) {\n regexps_++;\n\n ABSL_VLOG(3) << absl::CEscape(regexp);\n\n RE2 re(regexp, RE2::Latin1);\n ASSERT_EQ(re.error(), \"\");\n\n std::string min, max;\n if(!re.PossibleMatchRange(&min, &max, 10)) {\n // There's no good max for \"\\\\C*\". Can't use strcmp\n // because sometimes it gets embedded in more\n // complicated expressions.\n if(strstr(regexp.c_str(), \"\\\\C*\"))\n return;\n ABSL_LOG(QFATAL) << \"PossibleMatchRange failed on: \"\n << absl::CEscape(regexp);\n }\n\n strgen_.Reset();\n while (strgen_.HasNext()) {\n absl::string_view s = strgen_.Next();\n tests_++;\n if (!RE2::FullMatch(s, re))\n continue;\n ASSERT_GE(s, min) << \" regexp: \" << regexp << \" max: \" << max;\n ASSERT_LE(s, max) << \" regexp: \" << regexp << \" min: \" << min;\n }\n}\n\nTEST(PossibleMatchRange, Exhaustive) {\n int natom = 3;\n int noperator = 3;\n int stringlen = 5;\n if (RE2_DEBUG_MODE) {\n natom = 2;\n noperator = 3;\n stringlen = 3;\n }\n PossibleMatchTester t(natom, noperator, Split(\" \", \"a b [0-9]\"),\n RegexpGenerator::EgrepOps(),\n stringlen, Explode(\"ab4\"));\n t.Generate();\n ABSL_LOG(INFO) << t.regexps() << \" regexps, \"\n << t.tests() << \" tests\";\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/random_test.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Random testing of regular expression matching.\n\n#include \n#include \n\n#include \"absl/flags/flag.h\"\n#include \"absl/strings/str_format.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/testing/exhaustive_tester.h\"\n#include \"re2/testing/regexp_generator.h\"\n\nABSL_FLAG(int, regexpseed, 404, \"Random regexp seed.\");\nABSL_FLAG(int, regexpcount, 100, \"How many random regexps to generate.\");\nABSL_FLAG(int, stringseed, 200, \"Random string seed.\");\nABSL_FLAG(int, stringcount, 100, \"How many random strings to generate.\");\n\nnamespace re2 {\n\n// Runs a random test on the given parameters.\n// (Always uses the same random seeds for reproducibility.\n// Can give different seeds on command line.)\nstatic void RandomTest(int maxatoms, int maxops,\n const std::vector& alphabet,\n const std::vector& ops,\n int maxstrlen,\n const std::vector& stralphabet,\n const std::string& wrapper) {\n // Limit to smaller test cases in debug mode,\n // because everything is so much slower.\n if (RE2_DEBUG_MODE) {\n maxatoms--;\n maxops--;\n maxstrlen /= 2;\n }\n\n ExhaustiveTester t(maxatoms, maxops, alphabet, ops,\n maxstrlen, stralphabet, wrapper, \"\");\n t.RandomStrings(absl::GetFlag(FLAGS_stringseed),\n absl::GetFlag(FLAGS_stringcount));\n t.GenerateRandom(absl::GetFlag(FLAGS_regexpseed),\n absl::GetFlag(FLAGS_regexpcount));\n absl::PrintF(\"%d regexps, %d tests, %d failures [%d/%d str]\\n\",\n t.regexps(), t.tests(), t.failures(), maxstrlen, stralphabet.size());\n EXPECT_EQ(0, t.failures());\n}\n\n// Tests random small regexps involving literals and egrep operators.\nTEST(Random, SmallEgrepLiterals) {\n RandomTest(5, 5, Explode(\"abc.\"), RegexpGenerator::EgrepOps(),\n 15, Explode(\"abc\"),\n \"\");\n}\n\n// Tests random bigger regexps involving literals and egrep operators.\nTEST(Random, BigEgrepLiterals) {\n RandomTest(10, 10, Explode(\"abc.\"), RegexpGenerator::EgrepOps(),\n 15, Explode(\"abc\"),\n \"\");\n}\n\n// Tests random small regexps involving literals, capturing parens,\n// and egrep operators.\nTEST(Random, SmallEgrepCaptures) {\n RandomTest(5, 5, Split(\" \", \"a (b) .\"), RegexpGenerator::EgrepOps(),\n 15, Explode(\"abc\"),\n \"\");\n}\n\n// Tests random bigger regexps involving literals, capturing parens,\n// and egrep operators.\nTEST(Random, BigEgrepCaptures) {\n RandomTest(10, 10, Split(\" \", \"a (b) .\"), RegexpGenerator::EgrepOps(),\n 15, Explode(\"abc\"),\n \"\");\n}\n\n// Tests random large complicated expressions, using all the possible\n// operators, some literals, some parenthesized literals, and predefined\n// character classes like \\d. (Adding larger character classes would\n// make for too many possibilities.)\nTEST(Random, Complicated) {\n std::vector ops = Split(\" \",\n \"%s%s %s|%s %s* %s*? %s+ %s+? %s? %s?? \"\n \"%s{0} %s{0,} %s{1} %s{1,} %s{0,1} %s{0,2} %s{1,2} \"\n \"%s{2} %s{2,} %s{3,4} %s{4,5}\");\n\n // Use (?:\\b) and (?:\\B) instead of \\b and \\B,\n // because PCRE rejects \\b* but accepts (?:\\b)*.\n // Ditto ^ and $.\n std::vector atoms = Split(\" \",\n \". (?:^) (?:$) \\\\a \\\\f \\\\n \\\\r \\\\t \\\\v \"\n \"\\\\d \\\\D \\\\s \\\\S \\\\w \\\\W (?:\\\\b) (?:\\\\B) \"\n \"a (a) b c - \\\\\\\\\");\n std::vector alphabet = Explode(\"abc123\\001\\002\\003\\t\\r\\n\\v\\f\\a\");\n RandomTest(10, 10, atoms, ops, 20, alphabet, \"\");\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/re2_arg_test.cc", "content": "// Copyright 2005 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// This tests to make sure numbers are parsed from strings\n// correctly.\n// Todo: Expand the test to validate strings parsed to the other types\n// supported by RE2::Arg class\n\n#include \n#include \n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_log.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/re2.h\"\n\nnamespace re2 {\n\nstruct SuccessTable {\n const char * value_string;\n int64_t value;\n bool success[6];\n};\n\n// Test boundary cases for different integral sizes.\n// Specifically I want to make sure that values outside the boundries\n// of an integral type will fail and that negative numbers will fail\n// for unsigned types. The following table contains the boundaries for\n// the various integral types and has entries for whether or not each\n// type can contain the given value.\nconst SuccessTable kSuccessTable[] = {\n// string integer value i16 u16 i32 u32 i64 u64\n// 0 to 2^7-1\n{ \"0\", 0, { true, true, true, true, true, true }},\n{ \"127\", 127, { true, true, true, true, true, true }},\n\n// -1 to -2^7\n{ \"-1\", -1, { true, false, true, false, true, false }},\n{ \"-128\", -128, { true, false, true, false, true, false }},\n\n// 2^7 to 2^8-1\n{ \"128\", 128, { true, true, true, true, true, true }},\n{ \"255\", 255, { true, true, true, true, true, true }},\n\n// 2^8 to 2^15-1\n{ \"256\", 256, { true, true, true, true, true, true }},\n{ \"32767\", 32767, { true, true, true, true, true, true }},\n\n// -2^7-1 to -2^15\n{ \"-129\", -129, { true, false, true, false, true, false }},\n{ \"-32768\", -32768, { true, false, true, false, true, false }},\n\n// 2^15 to 2^16-1\n{ \"32768\", 32768, { false, true, true, true, true, true }},\n{ \"65535\", 65535, { false, true, true, true, true, true }},\n\n// 2^16 to 2^31-1\n{ \"65536\", 65536, { false, false, true, true, true, true }},\n{ \"2147483647\", 2147483647, { false, false, true, true, true, true }},\n\n// -2^15-1 to -2^31\n{ \"-32769\", -32769, { false, false, true, false, true, false }},\n{ \"-2147483648\", static_cast(0xFFFFFFFF80000000LL),\n { false, false, true, false, true, false }},\n\n// 2^31 to 2^32-1\n{ \"2147483648\", 2147483648U, { false, false, false, true, true, true }},\n{ \"4294967295\", 4294967295U, { false, false, false, true, true, true }},\n\n// 2^32 to 2^63-1\n{ \"4294967296\", 4294967296LL, { false, false, false, false, true, true }},\n{ \"9223372036854775807\",\n 9223372036854775807LL, { false, false, false, false, true, true }},\n\n// -2^31-1 to -2^63\n{ \"-2147483649\", -2147483649LL, { false, false, false, false, true, false }},\n{ \"-9223372036854775808\", static_cast(0x8000000000000000LL),\n { false, false, false, false, true, false }},\n\n// 2^63 to 2^64-1\n{ \"9223372036854775808\", static_cast(9223372036854775808ULL),\n { false, false, false, false, false, true }},\n{ \"18446744073709551615\", static_cast(18446744073709551615ULL),\n { false, false, false, false, false, true }},\n\n// >= 2^64\n{ \"18446744073709551616\", 0, { false, false, false, false, false, false }},\n};\n\nconst int kNumStrings = ABSL_ARRAYSIZE(kSuccessTable);\n\n// It's ugly to use a macro, but we apparently can't use the EXPECT_EQ\n// macro outside of a TEST block and this seems to be the only way to\n// avoid code duplication. I can also pull off a couple nice tricks\n// using concatenation for the type I'm checking against.\n#define PARSE_FOR_TYPE(type, column) { \\\n type r; \\\n for (int i = 0; i < kNumStrings; ++i) { \\\n RE2::Arg arg(&r); \\\n const char* const p = kSuccessTable[i].value_string; \\\n bool retval = arg.Parse(p, strlen(p)); \\\n bool success = kSuccessTable[i].success[column]; \\\n EXPECT_EQ(retval, success) \\\n << \"Parsing '\" << p << \"' for type \" #type \" should return \" \\\n << success; \\\n if (success) { \\\n EXPECT_EQ(r, (type)kSuccessTable[i].value); \\\n } \\\n } \\\n}\n\nTEST(RE2ArgTest, Int16Test) {\n PARSE_FOR_TYPE(int16_t, 0);\n}\n\nTEST(RE2ArgTest, Uint16Test) {\n PARSE_FOR_TYPE(uint16_t, 1);\n}\n\nTEST(RE2ArgTest, Int32Test) {\n PARSE_FOR_TYPE(int32_t, 2);\n}\n\nTEST(RE2ArgTest, Uint32Test) {\n PARSE_FOR_TYPE(uint32_t, 3);\n}\n\nTEST(RE2ArgTest, Int64Test) {\n PARSE_FOR_TYPE(int64_t, 4);\n}\n\nTEST(RE2ArgTest, Uint64Test) {\n PARSE_FOR_TYPE(uint64_t, 5);\n}\n\nTEST(RE2ArgTest, ParseFromTest) {\n struct {\n bool ParseFrom(const char* str, size_t n) {\n ABSL_LOG(INFO) << \"str = \" << str << \", n = \" << n;\n return true;\n }\n } obj1;\n RE2::Arg arg1(&obj1);\n EXPECT_TRUE(arg1.Parse(\"one\", 3));\n\n struct {\n bool ParseFrom(const char* str, size_t n) {\n ABSL_LOG(INFO) << \"str = \" << str << \", n = \" << n;\n return false;\n }\n // Ensure that RE2::Arg works even with overloaded ParseFrom().\n void ParseFrom(const char* str) {}\n } obj2;\n RE2::Arg arg2(&obj2);\n EXPECT_FALSE(arg2.Parse(\"two\", 3));\n}\n\nTEST(RE2ArgTest, OptionalDoubleTest) {\n std::optional opt;\n RE2::Arg arg(&opt);\n EXPECT_TRUE(arg.Parse(NULL, 0));\n EXPECT_FALSE(opt.has_value());\n EXPECT_FALSE(arg.Parse(\"\", 0));\n EXPECT_TRUE(arg.Parse(\"28.30\", 5));\n EXPECT_TRUE(opt.has_value());\n EXPECT_EQ(*opt, 28.30);\n}\n\nTEST(RE2ArgTest, OptionalIntWithCRadixTest) {\n std::optional opt;\n RE2::Arg arg = RE2::CRadix(&opt);\n EXPECT_TRUE(arg.Parse(NULL, 0));\n EXPECT_FALSE(opt.has_value());\n EXPECT_FALSE(arg.Parse(\"\", 0));\n EXPECT_TRUE(arg.Parse(\"0xb0e\", 5));\n EXPECT_TRUE(opt.has_value());\n EXPECT_EQ(*opt, 2830);\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/re2_test.cc", "content": "// -*- coding: utf-8 -*-\n// Copyright 2002-2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// TODO: Test extractions for PartialMatch/Consume\n\n#include \"re2/re2.h\"\n\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/regexp.h\"\n\n#if !defined(_MSC_VER) && !defined(__CYGWIN__) && !defined(__MINGW32__)\n#include \n#include \n#endif\n\nnamespace re2 {\n\nTEST(RE2, HexTests) {\n#define ASSERT_HEX(type, value) \\\n do { \\\n type v; \\\n ASSERT_TRUE( \\\n RE2::FullMatch(#value, \"([0-9a-fA-F]+)[uUlL]*\", RE2::Hex(&v))); \\\n ASSERT_EQ(v, 0x##value); \\\n ASSERT_TRUE(RE2::FullMatch(\"0x\" #value, \"([0-9a-fA-FxX]+)[uUlL]*\", \\\n RE2::CRadix(&v))); \\\n ASSERT_EQ(v, 0x##value); \\\n } while (0)\n\n ASSERT_HEX(short, 2bad);\n ASSERT_HEX(unsigned short, 2badU);\n ASSERT_HEX(int, dead);\n ASSERT_HEX(unsigned int, deadU);\n ASSERT_HEX(long, 7eadbeefL);\n ASSERT_HEX(unsigned long, deadbeefUL);\n ASSERT_HEX(long long, 12345678deadbeefLL);\n ASSERT_HEX(unsigned long long, cafebabedeadbeefULL);\n\n#undef ASSERT_HEX\n}\n\nTEST(RE2, OctalTests) {\n#define ASSERT_OCTAL(type, value) \\\n do { \\\n type v; \\\n ASSERT_TRUE(RE2::FullMatch(#value, \"([0-7]+)[uUlL]*\", RE2::Octal(&v))); \\\n ASSERT_EQ(v, 0##value); \\\n ASSERT_TRUE(RE2::FullMatch(\"0\" #value, \"([0-9a-fA-FxX]+)[uUlL]*\", \\\n RE2::CRadix(&v))); \\\n ASSERT_EQ(v, 0##value); \\\n } while (0)\n\n ASSERT_OCTAL(short, 77777);\n ASSERT_OCTAL(unsigned short, 177777U);\n ASSERT_OCTAL(int, 17777777777);\n ASSERT_OCTAL(unsigned int, 37777777777U);\n ASSERT_OCTAL(long, 17777777777L);\n ASSERT_OCTAL(unsigned long, 37777777777UL);\n ASSERT_OCTAL(long long, 777777777777777777777LL);\n ASSERT_OCTAL(unsigned long long, 1777777777777777777777ULL);\n\n#undef ASSERT_OCTAL\n}\n\nTEST(RE2, DecimalTests) {\n#define ASSERT_DECIMAL(type, value) \\\n do { \\\n type v; \\\n ASSERT_TRUE(RE2::FullMatch(#value, \"(-?[0-9]+)[uUlL]*\", &v)); \\\n ASSERT_EQ(v, value); \\\n ASSERT_TRUE( \\\n RE2::FullMatch(#value, \"(-?[0-9a-fA-FxX]+)[uUlL]*\", RE2::CRadix(&v))); \\\n ASSERT_EQ(v, value); \\\n } while (0)\n\n ASSERT_DECIMAL(short, -1);\n ASSERT_DECIMAL(unsigned short, 9999);\n ASSERT_DECIMAL(int, -1000);\n ASSERT_DECIMAL(unsigned int, 12345U);\n ASSERT_DECIMAL(long, -10000000L);\n ASSERT_DECIMAL(unsigned long, 3083324652U);\n ASSERT_DECIMAL(long long, -100000000000000LL);\n ASSERT_DECIMAL(unsigned long long, 1234567890987654321ULL);\n\n#undef ASSERT_DECIMAL\n}\n\nTEST(RE2, Replace) {\n struct ReplaceTest {\n const char *regexp;\n const char *rewrite;\n const char *original;\n const char *single;\n const char *global;\n int greplace_count;\n };\n static const ReplaceTest tests[] = {\n { \"(qu|[b-df-hj-np-tv-z]*)([a-z]+)\",\n \"\\\\2\\\\1ay\",\n \"the quick brown fox jumps over the lazy dogs.\",\n \"ethay quick brown fox jumps over the lazy dogs.\",\n \"ethay ickquay ownbray oxfay umpsjay overay ethay azylay ogsday.\",\n 9 },\n { \"\\\\w+\",\n \"\\\\0-NOSPAM\",\n \"abcd.efghi@google.com\",\n \"abcd-NOSPAM.efghi@google.com\",\n \"abcd-NOSPAM.efghi-NOSPAM@google-NOSPAM.com-NOSPAM\",\n 4 },\n { \"^\",\n \"(START)\",\n \"foo\",\n \"(START)foo\",\n \"(START)foo\",\n 1 },\n { \"^\",\n \"(START)\",\n \"\",\n \"(START)\",\n \"(START)\",\n 1 },\n { \"$\",\n \"(END)\",\n \"\",\n \"(END)\",\n \"(END)\",\n 1 },\n { \"b\",\n \"bb\",\n \"ababababab\",\n \"abbabababab\",\n \"abbabbabbabbabb\",\n 5 },\n { \"b\",\n \"bb\",\n \"bbbbbb\",\n \"bbbbbbb\",\n \"bbbbbbbbbbbb\",\n 6 },\n { \"b+\",\n \"bb\",\n \"bbbbbb\",\n \"bb\",\n \"bb\",\n 1 },\n { \"b*\",\n \"bb\",\n \"bbbbbb\",\n \"bb\",\n \"bb\",\n 1 },\n { \"b*\",\n \"bb\",\n \"aaaaa\",\n \"bbaaaaa\",\n \"bbabbabbabbabbabb\",\n 6 },\n // Check newline handling\n { \"a.*a\",\n \"(\\\\0)\",\n \"aba\\naba\",\n \"(aba)\\naba\",\n \"(aba)\\n(aba)\",\n 2 },\n { \"\", NULL, NULL, NULL, NULL, 0 }\n };\n\n for (const ReplaceTest* t = tests; t->original != NULL; t++) {\n std::string one(t->original);\n ASSERT_TRUE(RE2::Replace(&one, t->regexp, t->rewrite));\n ASSERT_EQ(one, t->single);\n std::string all(t->original);\n ASSERT_EQ(RE2::GlobalReplace(&all, t->regexp, t->rewrite), t->greplace_count)\n << \"Got: \" << all;\n ASSERT_EQ(all, t->global);\n }\n}\n\nstatic void TestCheckRewriteString(const char* regexp, const char* rewrite,\n bool expect_ok) {\n std::string error;\n RE2 exp(regexp);\n bool actual_ok = exp.CheckRewriteString(rewrite, &error);\n EXPECT_EQ(expect_ok, actual_ok) << \" for \" << rewrite << \" error: \" << error;\n}\n\nTEST(CheckRewriteString, all) {\n TestCheckRewriteString(\"abc\", \"foo\", true);\n TestCheckRewriteString(\"abc\", \"foo\\\\\", false);\n TestCheckRewriteString(\"abc\", \"foo\\\\0bar\", true);\n\n TestCheckRewriteString(\"a(b)c\", \"foo\", true);\n TestCheckRewriteString(\"a(b)c\", \"foo\\\\0bar\", true);\n TestCheckRewriteString(\"a(b)c\", \"foo\\\\1bar\", true);\n TestCheckRewriteString(\"a(b)c\", \"foo\\\\2bar\", false);\n TestCheckRewriteString(\"a(b)c\", \"f\\\\\\\\2o\\\\1o\", true);\n\n TestCheckRewriteString(\"a(b)(c)\", \"foo\\\\12\", true);\n TestCheckRewriteString(\"a(b)(c)\", \"f\\\\2o\\\\1o\", true);\n TestCheckRewriteString(\"a(b)(c)\", \"f\\\\oo\\\\1\", false);\n}\n\nTEST(RE2, Extract) {\n std::string s;\n\n ASSERT_TRUE(RE2::Extract(\"boris@kremvax.ru\", \"(.*)@([^.]*)\", \"\\\\2!\\\\1\", &s));\n ASSERT_EQ(s, \"kremvax!boris\");\n\n ASSERT_TRUE(RE2::Extract(\"foo\", \".*\", \"'\\\\0'\", &s));\n ASSERT_EQ(s, \"'foo'\");\n // check that false match doesn't overwrite\n ASSERT_FALSE(RE2::Extract(\"baz\", \"bar\", \"'\\\\0'\", &s));\n ASSERT_EQ(s, \"'foo'\");\n}\n\nTEST(RE2, MaxSubmatchTooLarge) {\n std::string s;\n ASSERT_FALSE(RE2::Extract(\"foo\", \"f(o+)\", \"\\\\1\\\\2\", &s));\n s = \"foo\";\n ASSERT_FALSE(RE2::Replace(&s, \"f(o+)\", \"\\\\1\\\\2\"));\n s = \"foo\";\n ASSERT_FALSE(RE2::GlobalReplace(&s, \"f(o+)\", \"\\\\1\\\\2\"));\n}\n\nTEST(RE2, Consume) {\n RE2 r(\"\\\\s*(\\\\w+)\"); // matches a word, possibly proceeded by whitespace\n std::string word;\n\n std::string s(\" aaa b!@#$@#$cccc\");\n absl::string_view input(s);\n\n ASSERT_TRUE(RE2::Consume(&input, r, &word));\n ASSERT_EQ(word, \"aaa\") << \" input: \" << input;\n ASSERT_TRUE(RE2::Consume(&input, r, &word));\n ASSERT_EQ(word, \"b\") << \" input: \" << input;\n ASSERT_FALSE(RE2::Consume(&input, r, &word)) << \" input: \" << input;\n}\n\nTEST(RE2, ConsumeN) {\n const std::string s(\" one two three 4\");\n absl::string_view input(s);\n\n RE2::Arg argv[2];\n const RE2::Arg* const args[2] = { &argv[0], &argv[1] };\n\n // 0 arg\n EXPECT_TRUE(RE2::ConsumeN(&input, \"\\\\s*(\\\\w+)\", args, 0)); // Skips \"one\".\n\n // 1 arg\n std::string word;\n argv[0] = &word;\n EXPECT_TRUE(RE2::ConsumeN(&input, \"\\\\s*(\\\\w+)\", args, 1));\n EXPECT_EQ(\"two\", word);\n\n // Multi-args\n int n;\n argv[1] = &n;\n EXPECT_TRUE(RE2::ConsumeN(&input, \"\\\\s*(\\\\w+)\\\\s*(\\\\d+)\", args, 2));\n EXPECT_EQ(\"three\", word);\n EXPECT_EQ(4, n);\n}\n\nTEST(RE2, FindAndConsume) {\n RE2 r(\"(\\\\w+)\"); // matches a word\n std::string word;\n\n std::string s(\" aaa b!@#$@#$cccc\");\n absl::string_view input(s);\n\n ASSERT_TRUE(RE2::FindAndConsume(&input, r, &word));\n ASSERT_EQ(word, \"aaa\");\n ASSERT_TRUE(RE2::FindAndConsume(&input, r, &word));\n ASSERT_EQ(word, \"b\");\n ASSERT_TRUE(RE2::FindAndConsume(&input, r, &word));\n ASSERT_EQ(word, \"cccc\");\n ASSERT_FALSE(RE2::FindAndConsume(&input, r, &word));\n\n // Check that FindAndConsume works without any submatches.\n // Earlier version used uninitialized data for\n // length to consume.\n input = \"aaa\";\n ASSERT_TRUE(RE2::FindAndConsume(&input, \"aaa\"));\n ASSERT_EQ(input, \"\");\n}\n\nTEST(RE2, FindAndConsumeN) {\n const std::string s(\" one two three 4\");\n absl::string_view input(s);\n\n RE2::Arg argv[2];\n const RE2::Arg* const args[2] = { &argv[0], &argv[1] };\n\n // 0 arg\n EXPECT_TRUE(RE2::FindAndConsumeN(&input, \"(\\\\w+)\", args, 0)); // Skips \"one\".\n\n // 1 arg\n std::string word;\n argv[0] = &word;\n EXPECT_TRUE(RE2::FindAndConsumeN(&input, \"(\\\\w+)\", args, 1));\n EXPECT_EQ(\"two\", word);\n\n // Multi-args\n int n;\n argv[1] = &n;\n EXPECT_TRUE(RE2::FindAndConsumeN(&input, \"(\\\\w+)\\\\s*(\\\\d+)\", args, 2));\n EXPECT_EQ(\"three\", word);\n EXPECT_EQ(4, n);\n}\n\nTEST(RE2, MatchNumberPeculiarity) {\n RE2 r(\"(foo)|(bar)|(baz)\");\n std::string word1;\n std::string word2;\n std::string word3;\n\n ASSERT_TRUE(RE2::PartialMatch(\"foo\", r, &word1, &word2, &word3));\n ASSERT_EQ(word1, \"foo\");\n ASSERT_EQ(word2, \"\");\n ASSERT_EQ(word3, \"\");\n ASSERT_TRUE(RE2::PartialMatch(\"bar\", r, &word1, &word2, &word3));\n ASSERT_EQ(word1, \"\");\n ASSERT_EQ(word2, \"bar\");\n ASSERT_EQ(word3, \"\");\n ASSERT_TRUE(RE2::PartialMatch(\"baz\", r, &word1, &word2, &word3));\n ASSERT_EQ(word1, \"\");\n ASSERT_EQ(word2, \"\");\n ASSERT_EQ(word3, \"baz\");\n ASSERT_FALSE(RE2::PartialMatch(\"f\", r, &word1, &word2, &word3));\n\n std::string a;\n ASSERT_TRUE(RE2::FullMatch(\"hello\", \"(foo)|hello\", &a));\n ASSERT_EQ(a, \"\");\n}\n\nTEST(RE2, Match) {\n RE2 re(\"((\\\\w+):([0-9]+))\"); // extracts host and port\n absl::string_view group[4];\n\n // No match.\n absl::string_view s = \"zyzzyva\";\n ASSERT_FALSE(\n re.Match(s, 0, s.size(), RE2::UNANCHORED, group, ABSL_ARRAYSIZE(group)));\n\n // Matches and extracts.\n s = \"a chrisr:9000 here\";\n ASSERT_TRUE(\n re.Match(s, 0, s.size(), RE2::UNANCHORED, group, ABSL_ARRAYSIZE(group)));\n ASSERT_EQ(group[0], \"chrisr:9000\");\n ASSERT_EQ(group[1], \"chrisr:9000\");\n ASSERT_EQ(group[2], \"chrisr\");\n ASSERT_EQ(group[3], \"9000\");\n\n std::string all, host;\n int port;\n ASSERT_TRUE(RE2::PartialMatch(\"a chrisr:9000 here\", re, &all, &host, &port));\n ASSERT_EQ(all, \"chrisr:9000\");\n ASSERT_EQ(host, \"chrisr\");\n ASSERT_EQ(port, 9000);\n}\n\nstatic void TestRecursion(int size, const char* pattern) {\n // Fill up a string repeating the pattern given\n std::string domain;\n domain.resize(size);\n size_t patlen = strlen(pattern);\n for (int i = 0; i < size; i++) {\n domain[i] = pattern[i % patlen];\n }\n // Just make sure it doesn't crash due to too much recursion.\n RE2 re(\"([a-zA-Z0-9]|-)+(\\\\.([a-zA-Z0-9]|-)+)*(\\\\.)?\", RE2::Quiet);\n RE2::FullMatch(domain, re);\n}\n\n// A meta-quoted string, interpreted as a pattern, should always match\n// the original unquoted string.\nstatic void TestQuoteMeta(const std::string& unquoted,\n const RE2::Options& options = RE2::DefaultOptions) {\n std::string quoted = RE2::QuoteMeta(unquoted);\n RE2 re(quoted, options);\n EXPECT_TRUE(RE2::FullMatch(unquoted, re))\n << \"Unquoted='\" << unquoted << \"', quoted='\" << quoted << \"'.\";\n}\n\n// A meta-quoted string, interpreted as a pattern, should always match\n// the original unquoted string.\nstatic void NegativeTestQuoteMeta(\n const std::string& unquoted, const std::string& should_not_match,\n const RE2::Options& options = RE2::DefaultOptions) {\n std::string quoted = RE2::QuoteMeta(unquoted);\n RE2 re(quoted, options);\n EXPECT_FALSE(RE2::FullMatch(should_not_match, re))\n << \"Unquoted='\" << unquoted << \"', quoted='\" << quoted << \"'.\";\n}\n\n// Tests that quoted meta characters match their original strings,\n// and that a few things that shouldn't match indeed do not.\nTEST(QuoteMeta, Simple) {\n TestQuoteMeta(\"foo\");\n TestQuoteMeta(\"foo.bar\");\n TestQuoteMeta(\"foo\\\\.bar\");\n TestQuoteMeta(\"[1-9]\");\n TestQuoteMeta(\"1.5-2.0?\");\n TestQuoteMeta(\"\\\\d\");\n TestQuoteMeta(\"Who doesn't like ice cream?\");\n TestQuoteMeta(\"((a|b)c?d*e+[f-h]i)\");\n TestQuoteMeta(\"((?!)xxx).*yyy\");\n TestQuoteMeta(\"([\");\n}\nTEST(QuoteMeta, SimpleNegative) {\n NegativeTestQuoteMeta(\"foo\", \"bar\");\n NegativeTestQuoteMeta(\"...\", \"bar\");\n NegativeTestQuoteMeta(\"\\\\.\", \".\");\n NegativeTestQuoteMeta(\"\\\\.\", \"..\");\n NegativeTestQuoteMeta(\"(a)\", \"a\");\n NegativeTestQuoteMeta(\"(a|b)\", \"a\");\n NegativeTestQuoteMeta(\"(a|b)\", \"(a)\");\n NegativeTestQuoteMeta(\"(a|b)\", \"a|b\");\n NegativeTestQuoteMeta(\"[0-9]\", \"0\");\n NegativeTestQuoteMeta(\"[0-9]\", \"0-9\");\n NegativeTestQuoteMeta(\"[0-9]\", \"[9]\");\n NegativeTestQuoteMeta(\"((?!)xxx)\", \"xxx\");\n}\n\nTEST(QuoteMeta, Latin1) {\n TestQuoteMeta(\"3\\xb2 = 9\", RE2::Latin1);\n}\n\nTEST(QuoteMeta, UTF8) {\n TestQuoteMeta(\"Plácido Domingo\");\n TestQuoteMeta(\"xyz\"); // No fancy utf8.\n TestQuoteMeta(\"\\xc2\\xb0\"); // 2-byte utf8 -- a degree symbol.\n TestQuoteMeta(\"27\\xc2\\xb0 degrees\"); // As a middle character.\n TestQuoteMeta(\"\\xe2\\x80\\xb3\"); // 3-byte utf8 -- a double prime.\n TestQuoteMeta(\"\\xf0\\x9d\\x85\\x9f\"); // 4-byte utf8 -- a music note.\n TestQuoteMeta(\"27\\xc2\\xb0\"); // Interpreted as Latin-1, this should\n // still work.\n NegativeTestQuoteMeta(\"27\\xc2\\xb0\",\n \"27\\\\\\xc2\\\\\\xb0\"); // 2-byte utf8 -- a degree symbol.\n}\n\nTEST(QuoteMeta, HasNull) {\n std::string has_null;\n\n // string with one null character\n has_null += '\\0';\n TestQuoteMeta(has_null);\n NegativeTestQuoteMeta(has_null, \"\");\n\n // Don't want null-followed-by-'1' to be interpreted as '\\01'.\n has_null += '1';\n TestQuoteMeta(has_null);\n NegativeTestQuoteMeta(has_null, \"\\1\");\n}\n\nTEST(ProgramSize, BigProgram) {\n RE2 re_simple(\"simple regexp\");\n RE2 re_medium(\"medium.*regexp\");\n RE2 re_complex(\"complex.{1,128}regexp\");\n\n ASSERT_GT(re_simple.ProgramSize(), 0);\n ASSERT_GT(re_medium.ProgramSize(), re_simple.ProgramSize());\n ASSERT_GT(re_complex.ProgramSize(), re_medium.ProgramSize());\n\n ASSERT_GT(re_simple.ReverseProgramSize(), 0);\n ASSERT_GT(re_medium.ReverseProgramSize(), re_simple.ReverseProgramSize());\n ASSERT_GT(re_complex.ReverseProgramSize(), re_medium.ReverseProgramSize());\n}\n\nTEST(ProgramFanout, BigProgram) {\n RE2 re1(\"(?:(?:(?:(?:(?:.)?){1})*)+)\");\n RE2 re10(\"(?:(?:(?:(?:(?:.)?){10})*)+)\");\n RE2 re100(\"(?:(?:(?:(?:(?:.)?){100})*)+)\");\n RE2 re1000(\"(?:(?:(?:(?:(?:.)?){1000})*)+)\");\n\n std::vector histogram;\n\n // 3 is the largest non-empty bucket and has 2 element.\n ASSERT_EQ(3, re1.ProgramFanout(&histogram));\n ASSERT_EQ(2, histogram[3]);\n\n // 6 is the largest non-empty bucket and has 11 elements.\n ASSERT_EQ(6, re10.ProgramFanout(&histogram));\n ASSERT_EQ(11, histogram[6]);\n\n // 9 is the largest non-empty bucket and has 101 elements.\n ASSERT_EQ(9, re100.ProgramFanout(&histogram));\n ASSERT_EQ(101, histogram[9]);\n\n // 13 is the largest non-empty bucket and has 1001 elements.\n ASSERT_EQ(13, re1000.ProgramFanout(&histogram));\n ASSERT_EQ(1001, histogram[13]);\n\n // 2 is the largest non-empty bucket and has 2 element.\n ASSERT_EQ(2, re1.ReverseProgramFanout(&histogram));\n ASSERT_EQ(2, histogram[2]);\n\n // 5 is the largest non-empty bucket and has 11 elements.\n ASSERT_EQ(5, re10.ReverseProgramFanout(&histogram));\n ASSERT_EQ(11, histogram[5]);\n\n // 9 is the largest non-empty bucket and has 101 elements.\n ASSERT_EQ(9, re100.ReverseProgramFanout(&histogram));\n ASSERT_EQ(101, histogram[9]);\n\n // 12 is the largest non-empty bucket and has 1001 elements.\n ASSERT_EQ(12, re1000.ReverseProgramFanout(&histogram));\n ASSERT_EQ(1001, histogram[12]);\n}\n\n// Issue 956519: handling empty character sets was\n// causing NULL dereference. This tests a few empty character sets.\n// (The way to get an empty character set is to negate a full one.)\nTEST(EmptyCharset, Fuzz) {\n static const char *empties[] = {\n \"[^\\\\S\\\\s]\",\n \"[^\\\\S[:space:]]\",\n \"[^\\\\D\\\\d]\",\n \"[^\\\\D[:digit:]]\"\n };\n for (size_t i = 0; i < ABSL_ARRAYSIZE(empties); i++)\n ASSERT_FALSE(RE2(empties[i]).Match(\"abc\", 0, 3, RE2::UNANCHORED, NULL, 0));\n}\n\n// Bitstate assumes that kInstFail instructions in\n// alternations or capture groups have been \"compiled away\".\nTEST(EmptyCharset, BitstateAssumptions) {\n // Captures trigger use of Bitstate.\n static const char *nop_empties[] = {\n \"((((()))))\" \"[^\\\\S\\\\s]?\",\n \"((((()))))\" \"([^\\\\S\\\\s])?\",\n \"((((()))))\" \"([^\\\\S\\\\s]|[^\\\\S\\\\s])?\",\n \"((((()))))\" \"(([^\\\\S\\\\s]|[^\\\\S\\\\s])|)\"\n };\n absl::string_view group[6];\n for (size_t i = 0; i < ABSL_ARRAYSIZE(nop_empties); i++)\n ASSERT_TRUE(RE2(nop_empties[i]).Match(\"\", 0, 0, RE2::UNANCHORED, group, 6));\n}\n\n// Test that named groups work correctly.\nTEST(Capture, NamedGroups) {\n {\n RE2 re(\"(hello world)\");\n ASSERT_EQ(re.NumberOfCapturingGroups(), 1);\n const std::map& m = re.NamedCapturingGroups();\n ASSERT_EQ(m.size(), size_t{0});\n }\n\n {\n RE2 re(\"(?Pexpr(?Pexpr)(?Pexpr))((expr)(?Pexpr))\");\n ASSERT_EQ(re.NumberOfCapturingGroups(), 6);\n const std::map& m = re.NamedCapturingGroups();\n ASSERT_EQ(m.size(), size_t{4});\n ASSERT_EQ(m.find(\"A\")->second, 1);\n ASSERT_EQ(m.find(\"B\")->second, 2);\n ASSERT_EQ(m.find(\"C\")->second, 3);\n ASSERT_EQ(m.find(\"D\")->second, 6); // $4 and $5 are anonymous\n }\n}\n\nTEST(RE2, CapturedGroupTest) {\n RE2 re(\"directions from (?P.*) to (?P.*)\");\n int num_groups = re.NumberOfCapturingGroups();\n EXPECT_EQ(2, num_groups);\n std::string args[4];\n RE2::Arg arg0(&args[0]);\n RE2::Arg arg1(&args[1]);\n RE2::Arg arg2(&args[2]);\n RE2::Arg arg3(&args[3]);\n\n const RE2::Arg* const matches[4] = {&arg0, &arg1, &arg2, &arg3};\n EXPECT_TRUE(RE2::FullMatchN(\"directions from mountain view to san jose\",\n re, matches, num_groups));\n const std::map& named_groups = re.NamedCapturingGroups();\n EXPECT_TRUE(named_groups.find(\"S\") != named_groups.end());\n EXPECT_TRUE(named_groups.find(\"D\") != named_groups.end());\n\n // The named group index is 1-based.\n int source_group_index = named_groups.find(\"S\")->second;\n int destination_group_index = named_groups.find(\"D\")->second;\n EXPECT_EQ(1, source_group_index);\n EXPECT_EQ(2, destination_group_index);\n\n // The args is zero-based.\n EXPECT_EQ(\"mountain view\", args[source_group_index - 1]);\n EXPECT_EQ(\"san jose\", args[destination_group_index - 1]);\n}\n\nTEST(RE2, FullMatchWithNoArgs) {\n ASSERT_TRUE(RE2::FullMatch(\"h\", \"h\"));\n ASSERT_TRUE(RE2::FullMatch(\"hello\", \"hello\"));\n ASSERT_TRUE(RE2::FullMatch(\"hello\", \"h.*o\"));\n ASSERT_FALSE(RE2::FullMatch(\"othello\", \"h.*o\")); // Must be anchored at front\n ASSERT_FALSE(RE2::FullMatch(\"hello!\", \"h.*o\")); // Must be anchored at end\n}\n\nTEST(RE2, PartialMatch) {\n ASSERT_TRUE(RE2::PartialMatch(\"x\", \"x\"));\n ASSERT_TRUE(RE2::PartialMatch(\"hello\", \"h.*o\"));\n ASSERT_TRUE(RE2::PartialMatch(\"othello\", \"h.*o\"));\n ASSERT_TRUE(RE2::PartialMatch(\"hello!\", \"h.*o\"));\n ASSERT_TRUE(RE2::PartialMatch(\"x\", \"((((((((((((((((((((x))))))))))))))))))))\"));\n}\n\nTEST(RE2, PartialMatchN) {\n RE2::Arg argv[2];\n const RE2::Arg* const args[2] = { &argv[0], &argv[1] };\n\n // 0 arg\n EXPECT_TRUE(RE2::PartialMatchN(\"hello\", \"e.*o\", args, 0));\n EXPECT_FALSE(RE2::PartialMatchN(\"othello\", \"a.*o\", args, 0));\n\n // 1 arg\n int i;\n argv[0] = &i;\n EXPECT_TRUE(RE2::PartialMatchN(\"1001 nights\", \"(\\\\d+)\", args, 1));\n EXPECT_EQ(1001, i);\n EXPECT_FALSE(RE2::PartialMatchN(\"three\", \"(\\\\d+)\", args, 1));\n\n // Multi-arg\n std::string s;\n argv[1] = &s;\n EXPECT_TRUE(RE2::PartialMatchN(\"answer: 42:life\", \"(\\\\d+):(\\\\w+)\", args, 2));\n EXPECT_EQ(42, i);\n EXPECT_EQ(\"life\", s);\n EXPECT_FALSE(RE2::PartialMatchN(\"hi1\", \"(\\\\w+)(1)\", args, 2));\n}\n\nTEST(RE2, FullMatchZeroArg) {\n // Zero-arg\n ASSERT_TRUE(RE2::FullMatch(\"1001\", \"\\\\d+\"));\n}\n\nTEST(RE2, FullMatchOneArg) {\n int i;\n\n // Single-arg\n ASSERT_TRUE(RE2::FullMatch(\"1001\", \"(\\\\d+)\", &i));\n ASSERT_EQ(i, 1001);\n ASSERT_TRUE(RE2::FullMatch(\"-123\", \"(-?\\\\d+)\", &i));\n ASSERT_EQ(i, -123);\n ASSERT_FALSE(RE2::FullMatch(\"10\", \"()\\\\d+\", &i));\n ASSERT_FALSE(\n RE2::FullMatch(\"1234567890123456789012345678901234567890\", \"(\\\\d+)\", &i));\n}\n\nTEST(RE2, FullMatchIntegerArg) {\n int i;\n\n // Digits surrounding integer-arg\n ASSERT_TRUE(RE2::FullMatch(\"1234\", \"1(\\\\d*)4\", &i));\n ASSERT_EQ(i, 23);\n ASSERT_TRUE(RE2::FullMatch(\"1234\", \"(\\\\d)\\\\d+\", &i));\n ASSERT_EQ(i, 1);\n ASSERT_TRUE(RE2::FullMatch(\"-1234\", \"(-\\\\d)\\\\d+\", &i));\n ASSERT_EQ(i, -1);\n ASSERT_TRUE(RE2::PartialMatch(\"1234\", \"(\\\\d)\", &i));\n ASSERT_EQ(i, 1);\n ASSERT_TRUE(RE2::PartialMatch(\"-1234\", \"(-\\\\d)\", &i));\n ASSERT_EQ(i, -1);\n}\n\nTEST(RE2, FullMatchStringArg) {\n std::string s;\n // string-arg\n ASSERT_TRUE(RE2::FullMatch(\"hello\", \"h(.*)o\", &s));\n ASSERT_EQ(s, std::string(\"ell\"));\n}\n\nTEST(RE2, FullMatchStringViewArg) {\n int i;\n absl::string_view sp;\n // string_view-arg\n ASSERT_TRUE(RE2::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", &sp, &i));\n ASSERT_EQ(sp.size(), size_t{4});\n ASSERT_TRUE(memcmp(sp.data(), \"ruby\", 4) == 0);\n ASSERT_EQ(i, 1234);\n}\n\nTEST(RE2, FullMatchMultiArg) {\n int i;\n std::string s;\n // Multi-arg\n ASSERT_TRUE(RE2::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", &s, &i));\n ASSERT_EQ(s, std::string(\"ruby\"));\n ASSERT_EQ(i, 1234);\n}\n\nTEST(RE2, FullMatchN) {\n RE2::Arg argv[2];\n const RE2::Arg* const args[2] = { &argv[0], &argv[1] };\n\n // 0 arg\n EXPECT_TRUE(RE2::FullMatchN(\"hello\", \"h.*o\", args, 0));\n EXPECT_FALSE(RE2::FullMatchN(\"othello\", \"h.*o\", args, 0));\n\n // 1 arg\n int i;\n argv[0] = &i;\n EXPECT_TRUE(RE2::FullMatchN(\"1001\", \"(\\\\d+)\", args, 1));\n EXPECT_EQ(1001, i);\n EXPECT_FALSE(RE2::FullMatchN(\"three\", \"(\\\\d+)\", args, 1));\n\n // Multi-arg\n std::string s;\n argv[1] = &s;\n EXPECT_TRUE(RE2::FullMatchN(\"42:life\", \"(\\\\d+):(\\\\w+)\", args, 2));\n EXPECT_EQ(42, i);\n EXPECT_EQ(\"life\", s);\n EXPECT_FALSE(RE2::FullMatchN(\"hi1\", \"(\\\\w+)(1)\", args, 2));\n}\n\nTEST(RE2, FullMatchIgnoredArg) {\n int i;\n std::string s;\n\n // Old-school NULL should be ignored.\n ASSERT_TRUE(\n RE2::FullMatch(\"ruby:1234\", \"(\\\\w+)(:)(\\\\d+)\", &s, (void*)NULL, &i));\n ASSERT_EQ(s, std::string(\"ruby\"));\n ASSERT_EQ(i, 1234);\n\n // C++11 nullptr should also be ignored.\n ASSERT_TRUE(RE2::FullMatch(\"rubz:1235\", \"(\\\\w+)(:)(\\\\d+)\", &s, nullptr, &i));\n ASSERT_EQ(s, std::string(\"rubz\"));\n ASSERT_EQ(i, 1235);\n}\n\nTEST(RE2, FullMatchTypedNullArg) {\n std::string s;\n\n // Ignore non-void* NULL arg\n ASSERT_TRUE(RE2::FullMatch(\"hello\", \"he(.*)lo\", (char*)NULL));\n ASSERT_TRUE(RE2::FullMatch(\"hello\", \"h(.*)o\", (std::string*)NULL));\n ASSERT_TRUE(RE2::FullMatch(\"hello\", \"h(.*)o\", (absl::string_view*)NULL));\n ASSERT_TRUE(RE2::FullMatch(\"1234\", \"(.*)\", (int*)NULL));\n ASSERT_TRUE(RE2::FullMatch(\"1234567890123456\", \"(.*)\", (long long*)NULL));\n ASSERT_TRUE(RE2::FullMatch(\"123.4567890123456\", \"(.*)\", (double*)NULL));\n ASSERT_TRUE(RE2::FullMatch(\"123.4567890123456\", \"(.*)\", (float*)NULL));\n\n // Fail on non-void* NULL arg if the match doesn't parse for the given type.\n ASSERT_FALSE(RE2::FullMatch(\"hello\", \"h(.*)lo\", &s, (char*)NULL));\n ASSERT_FALSE(RE2::FullMatch(\"hello\", \"(.*)\", (int*)NULL));\n ASSERT_FALSE(RE2::FullMatch(\"1234567890123456\", \"(.*)\", (int*)NULL));\n ASSERT_FALSE(RE2::FullMatch(\"hello\", \"(.*)\", (double*)NULL));\n ASSERT_FALSE(RE2::FullMatch(\"hello\", \"(.*)\", (float*)NULL));\n}\n\n// Check that numeric parsing code does not read past the end of\n// the number being parsed.\n// This implementation requires mmap(2) et al. and thus cannot\n// be used unless they are available.\nTEST(RE2, NULTerminated) {\n#if defined(_POSIX_MAPPED_FILES) && _POSIX_MAPPED_FILES > 0\n char *v;\n int x;\n long pagesize = sysconf(_SC_PAGE_SIZE);\n\n#ifndef MAP_ANONYMOUS\n#define MAP_ANONYMOUS MAP_ANON\n#endif\n v = static_cast(mmap(NULL, 2*pagesize, PROT_READ|PROT_WRITE,\n MAP_ANONYMOUS|MAP_PRIVATE, -1, 0));\n ASSERT_TRUE(v != reinterpret_cast(-1));\n ABSL_LOG(INFO) << \"Memory at \" << reinterpret_cast(v);\n ASSERT_EQ(munmap(v + pagesize, pagesize), 0) << \" error \" << errno;\n v[pagesize - 1] = '1';\n\n x = 0;\n ASSERT_TRUE(\n RE2::FullMatch(absl::string_view(v + pagesize - 1, 1), \"(.*)\", &x));\n ASSERT_EQ(x, 1);\n#endif\n}\n\nTEST(RE2, FullMatchTypeTests) {\n // Type tests\n std::string zeros(1000, '0');\n {\n char c;\n ASSERT_TRUE(RE2::FullMatch(\"Hello\", \"(H)ello\", &c));\n ASSERT_EQ(c, 'H');\n }\n {\n signed char c;\n ASSERT_TRUE(RE2::FullMatch(\"Hello\", \"(H)ello\", &c));\n ASSERT_EQ(c, static_cast('H'));\n }\n {\n unsigned char c;\n ASSERT_TRUE(RE2::FullMatch(\"Hello\", \"(H)ello\", &c));\n ASSERT_EQ(c, static_cast('H'));\n }\n {\n int16_t v;\n ASSERT_TRUE(RE2::FullMatch(\"100\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, 100);\n ASSERT_TRUE(RE2::FullMatch(\"-100\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, -100);\n ASSERT_TRUE(RE2::FullMatch(\"32767\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, 32767);\n ASSERT_TRUE(RE2::FullMatch(\"-32768\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, -32768);\n ASSERT_FALSE(RE2::FullMatch(\"-32769\", \"(-?\\\\d+)\", &v));\n ASSERT_FALSE(RE2::FullMatch(\"32768\", \"(-?\\\\d+)\", &v));\n }\n {\n uint16_t v;\n ASSERT_TRUE(RE2::FullMatch(\"100\", \"(\\\\d+)\", &v)); ASSERT_EQ(v, 100);\n ASSERT_TRUE(RE2::FullMatch(\"32767\", \"(\\\\d+)\", &v)); ASSERT_EQ(v, 32767);\n ASSERT_TRUE(RE2::FullMatch(\"65535\", \"(\\\\d+)\", &v)); ASSERT_EQ(v, 65535);\n ASSERT_FALSE(RE2::FullMatch(\"65536\", \"(\\\\d+)\", &v));\n }\n {\n int32_t v;\n static const int32_t max = INT32_C(0x7fffffff);\n static const int32_t min = -max - 1;\n ASSERT_TRUE(RE2::FullMatch(\"100\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, 100);\n ASSERT_TRUE(RE2::FullMatch(\"-100\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, -100);\n ASSERT_TRUE(RE2::FullMatch(\"2147483647\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, max);\n ASSERT_TRUE(RE2::FullMatch(\"-2147483648\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, min);\n ASSERT_FALSE(RE2::FullMatch(\"-2147483649\", \"(-?\\\\d+)\", &v));\n ASSERT_FALSE(RE2::FullMatch(\"2147483648\", \"(-?\\\\d+)\", &v));\n\n ASSERT_TRUE(RE2::FullMatch(zeros + \"2147483647\", \"(-?\\\\d+)\", &v));\n ASSERT_EQ(v, max);\n ASSERT_TRUE(RE2::FullMatch(\"-\" + zeros + \"2147483648\", \"(-?\\\\d+)\", &v));\n ASSERT_EQ(v, min);\n\n ASSERT_FALSE(RE2::FullMatch(\"-\" + zeros + \"2147483649\", \"(-?\\\\d+)\", &v));\n ASSERT_TRUE(RE2::FullMatch(\"0x7fffffff\", \"(.*)\", RE2::CRadix(&v)));\n ASSERT_EQ(v, max);\n ASSERT_FALSE(RE2::FullMatch(\"000x7fffffff\", \"(.*)\", RE2::CRadix(&v)));\n }\n {\n uint32_t v;\n static const uint32_t max = UINT32_C(0xffffffff);\n ASSERT_TRUE(RE2::FullMatch(\"100\", \"(\\\\d+)\", &v)); ASSERT_EQ(v, uint32_t{100});\n ASSERT_TRUE(RE2::FullMatch(\"4294967295\", \"(\\\\d+)\", &v)); ASSERT_EQ(v, max);\n ASSERT_FALSE(RE2::FullMatch(\"4294967296\", \"(\\\\d+)\", &v));\n ASSERT_FALSE(RE2::FullMatch(\"-1\", \"(\\\\d+)\", &v));\n\n ASSERT_TRUE(RE2::FullMatch(zeros + \"4294967295\", \"(\\\\d+)\", &v)); ASSERT_EQ(v, max);\n }\n {\n int64_t v;\n static const int64_t max = INT64_C(0x7fffffffffffffff);\n static const int64_t min = -max - 1;\n std::string str;\n\n ASSERT_TRUE(RE2::FullMatch(\"100\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, 100);\n ASSERT_TRUE(RE2::FullMatch(\"-100\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, -100);\n\n str = std::to_string(max);\n ASSERT_TRUE(RE2::FullMatch(str, \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, max);\n\n str = std::to_string(min);\n ASSERT_TRUE(RE2::FullMatch(str, \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, min);\n\n str = std::to_string(max);\n ASSERT_NE(str.back(), '9');\n str.back()++;\n ASSERT_FALSE(RE2::FullMatch(str, \"(-?\\\\d+)\", &v));\n\n str = std::to_string(min);\n ASSERT_NE(str.back(), '9');\n str.back()++;\n ASSERT_FALSE(RE2::FullMatch(str, \"(-?\\\\d+)\", &v));\n }\n {\n uint64_t v;\n int64_t v2;\n static const uint64_t max = UINT64_C(0xffffffffffffffff);\n std::string str;\n\n ASSERT_TRUE(RE2::FullMatch(\"100\", \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, uint64_t{100});\n ASSERT_TRUE(RE2::FullMatch(\"-100\", \"(-?\\\\d+)\", &v2)); ASSERT_EQ(v2, -100);\n\n str = std::to_string(max);\n ASSERT_TRUE(RE2::FullMatch(str, \"(-?\\\\d+)\", &v)); ASSERT_EQ(v, max);\n\n ASSERT_NE(str.back(), '9');\n str.back()++;\n ASSERT_FALSE(RE2::FullMatch(str, \"(-?\\\\d+)\", &v));\n }\n}\n\nTEST(RE2, FloatingPointFullMatchTypes) {\n std::string zeros(1000, '0');\n {\n float v;\n ASSERT_TRUE(RE2::FullMatch(\"100\", \"(.*)\", &v)); ASSERT_EQ(v, 100);\n ASSERT_TRUE(RE2::FullMatch(\"-100.\", \"(.*)\", &v)); ASSERT_EQ(v, -100);\n ASSERT_TRUE(RE2::FullMatch(\"1e23\", \"(.*)\", &v)); ASSERT_EQ(v, float{1e23});\n ASSERT_TRUE(RE2::FullMatch(\" 100\", \"(.*)\", &v)); ASSERT_EQ(v, 100);\n\n ASSERT_TRUE(RE2::FullMatch(zeros + \"1e23\", \"(.*)\", &v));\n ASSERT_EQ(v, float{1e23});\n\n // 6700000000081920.1 is an edge case.\n // 6700000000081920 is exactly halfway between\n // two float32s, so the .1 should make it round up.\n // However, the .1 is outside the precision possible with\n // a float64: the nearest float64 is 6700000000081920.\n // So if the code uses strtod and then converts to float32,\n // round-to-even will make it round down instead of up.\n // To pass the test, the parser must call strtof directly.\n // This test case is carefully chosen to use only a 17-digit\n // number, since C does not guarantee to get the correctly\n // rounded answer for strtod and strtof unless the input is\n // short.\n //\n // This is known to fail on Cygwin and MinGW due to a broken\n // implementation of strtof(3). And apparently MSVC too. Sigh.\n#if !defined(_MSC_VER) && !defined(__CYGWIN__) && !defined(__MINGW32__)\n ASSERT_TRUE(RE2::FullMatch(\"0.1\", \"(.*)\", &v));\n ASSERT_EQ(v, 0.1f) << absl::StrFormat(\"%.8g != %.8g\", v, 0.1f);\n ASSERT_TRUE(RE2::FullMatch(\"6700000000081920.1\", \"(.*)\", &v));\n ASSERT_EQ(v, 6700000000081920.1f)\n << absl::StrFormat(\"%.8g != %.8g\", v, 6700000000081920.1f);\n#endif\n }\n {\n double v;\n ASSERT_TRUE(RE2::FullMatch(\"100\", \"(.*)\", &v)); ASSERT_EQ(v, 100);\n ASSERT_TRUE(RE2::FullMatch(\"-100.\", \"(.*)\", &v)); ASSERT_EQ(v, -100);\n ASSERT_TRUE(RE2::FullMatch(\"1e23\", \"(.*)\", &v)); ASSERT_EQ(v, double{1e23});\n ASSERT_TRUE(RE2::FullMatch(\" 100\", \"(.*)\", &v)); ASSERT_EQ(v, 100);\n\n ASSERT_TRUE(RE2::FullMatch(zeros + \"1e23\", \"(.*)\", &v));\n ASSERT_EQ(v, double{1e23});\n\n ASSERT_TRUE(RE2::FullMatch(\"0.1\", \"(.*)\", &v));\n ASSERT_EQ(v, 0.1) << absl::StrFormat(\"%.17g != %.17g\", v, 0.1);\n ASSERT_TRUE(RE2::FullMatch(\"1.00000005960464485\", \"(.*)\", &v));\n ASSERT_EQ(v, 1.0000000596046448)\n << absl::StrFormat(\"%.17g != %.17g\", v, 1.0000000596046448);\n }\n}\n\nTEST(RE2, FullMatchAnchored) {\n int i;\n // Check that matching is fully anchored\n ASSERT_FALSE(RE2::FullMatch(\"x1001\", \"(\\\\d+)\", &i));\n ASSERT_FALSE(RE2::FullMatch(\"1001x\", \"(\\\\d+)\", &i));\n ASSERT_TRUE(RE2::FullMatch(\"x1001\", \"x(\\\\d+)\", &i)); ASSERT_EQ(i, 1001);\n ASSERT_TRUE(RE2::FullMatch(\"1001x\", \"(\\\\d+)x\", &i)); ASSERT_EQ(i, 1001);\n}\n\nTEST(RE2, FullMatchBraces) {\n // Braces\n ASSERT_TRUE(RE2::FullMatch(\"0abcd\", \"[0-9a-f+.-]{5,}\"));\n ASSERT_TRUE(RE2::FullMatch(\"0abcde\", \"[0-9a-f+.-]{5,}\"));\n ASSERT_FALSE(RE2::FullMatch(\"0abc\", \"[0-9a-f+.-]{5,}\"));\n}\n\nTEST(RE2, Complicated) {\n // Complicated RE2\n ASSERT_TRUE(RE2::FullMatch(\"foo\", \"foo|bar|[A-Z]\"));\n ASSERT_TRUE(RE2::FullMatch(\"bar\", \"foo|bar|[A-Z]\"));\n ASSERT_TRUE(RE2::FullMatch(\"X\", \"foo|bar|[A-Z]\"));\n ASSERT_FALSE(RE2::FullMatch(\"XY\", \"foo|bar|[A-Z]\"));\n}\n\nTEST(RE2, FullMatchEnd) {\n // Check full-match handling (needs '$' tacked on internally)\n ASSERT_TRUE(RE2::FullMatch(\"fo\", \"fo|foo\"));\n ASSERT_TRUE(RE2::FullMatch(\"foo\", \"fo|foo\"));\n ASSERT_TRUE(RE2::FullMatch(\"fo\", \"fo|foo$\"));\n ASSERT_TRUE(RE2::FullMatch(\"foo\", \"fo|foo$\"));\n ASSERT_TRUE(RE2::FullMatch(\"foo\", \"foo$\"));\n ASSERT_FALSE(RE2::FullMatch(\"foo$bar\", \"foo\\\\$\"));\n ASSERT_FALSE(RE2::FullMatch(\"fox\", \"fo|bar\"));\n\n // Uncomment the following if we change the handling of '$' to\n // prevent it from matching a trailing newline\n if (false) {\n // Check that we don't get bitten by pcre's special handling of a\n // '\\n' at the end of the string matching '$'\n ASSERT_FALSE(RE2::PartialMatch(\"foo\\n\", \"foo$\"));\n }\n}\n\nTEST(RE2, FullMatchArgCount) {\n // Number of args\n int a[16];\n ASSERT_TRUE(RE2::FullMatch(\"\", \"\"));\n\n memset(a, 0, sizeof(0));\n ASSERT_TRUE(RE2::FullMatch(\"1\", \"(\\\\d){1}\", &a[0]));\n ASSERT_EQ(a[0], 1);\n\n memset(a, 0, sizeof(0));\n ASSERT_TRUE(RE2::FullMatch(\"12\", \"(\\\\d)(\\\\d)\", &a[0], &a[1]));\n ASSERT_EQ(a[0], 1);\n ASSERT_EQ(a[1], 2);\n\n memset(a, 0, sizeof(0));\n ASSERT_TRUE(RE2::FullMatch(\"123\", \"(\\\\d)(\\\\d)(\\\\d)\", &a[0], &a[1], &a[2]));\n ASSERT_EQ(a[0], 1);\n ASSERT_EQ(a[1], 2);\n ASSERT_EQ(a[2], 3);\n\n memset(a, 0, sizeof(0));\n ASSERT_TRUE(RE2::FullMatch(\"1234\", \"(\\\\d)(\\\\d)(\\\\d)(\\\\d)\", &a[0], &a[1],\n &a[2], &a[3]));\n ASSERT_EQ(a[0], 1);\n ASSERT_EQ(a[1], 2);\n ASSERT_EQ(a[2], 3);\n ASSERT_EQ(a[3], 4);\n\n memset(a, 0, sizeof(0));\n ASSERT_TRUE(RE2::FullMatch(\"12345\", \"(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)\", &a[0], &a[1],\n &a[2], &a[3], &a[4]));\n ASSERT_EQ(a[0], 1);\n ASSERT_EQ(a[1], 2);\n ASSERT_EQ(a[2], 3);\n ASSERT_EQ(a[3], 4);\n ASSERT_EQ(a[4], 5);\n\n memset(a, 0, sizeof(0));\n ASSERT_TRUE(RE2::FullMatch(\"123456\", \"(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)\", &a[0],\n &a[1], &a[2], &a[3], &a[4], &a[5]));\n ASSERT_EQ(a[0], 1);\n ASSERT_EQ(a[1], 2);\n ASSERT_EQ(a[2], 3);\n ASSERT_EQ(a[3], 4);\n ASSERT_EQ(a[4], 5);\n ASSERT_EQ(a[5], 6);\n\n memset(a, 0, sizeof(0));\n ASSERT_TRUE(RE2::FullMatch(\"1234567\", \"(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)\",\n &a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6]));\n ASSERT_EQ(a[0], 1);\n ASSERT_EQ(a[1], 2);\n ASSERT_EQ(a[2], 3);\n ASSERT_EQ(a[3], 4);\n ASSERT_EQ(a[4], 5);\n ASSERT_EQ(a[5], 6);\n ASSERT_EQ(a[6], 7);\n\n memset(a, 0, sizeof(0));\n ASSERT_TRUE(RE2::FullMatch(\"1234567890123456\",\n \"(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)\"\n \"(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)(\\\\d)\",\n &a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6],\n &a[7], &a[8], &a[9], &a[10], &a[11], &a[12],\n &a[13], &a[14], &a[15]));\n ASSERT_EQ(a[0], 1);\n ASSERT_EQ(a[1], 2);\n ASSERT_EQ(a[2], 3);\n ASSERT_EQ(a[3], 4);\n ASSERT_EQ(a[4], 5);\n ASSERT_EQ(a[5], 6);\n ASSERT_EQ(a[6], 7);\n ASSERT_EQ(a[7], 8);\n ASSERT_EQ(a[8], 9);\n ASSERT_EQ(a[9], 0);\n ASSERT_EQ(a[10], 1);\n ASSERT_EQ(a[11], 2);\n ASSERT_EQ(a[12], 3);\n ASSERT_EQ(a[13], 4);\n ASSERT_EQ(a[14], 5);\n ASSERT_EQ(a[15], 6);\n}\n\nTEST(RE2, Accessors) {\n // Check the pattern() accessor\n {\n const std::string kPattern = \"http://([^/]+)/.*\";\n const RE2 re(kPattern);\n ASSERT_EQ(kPattern, re.pattern());\n }\n\n // Check RE2 error field.\n {\n RE2 re(\"foo\");\n ASSERT_TRUE(re.error().empty()); // Must have no error\n ASSERT_TRUE(re.ok());\n ASSERT_EQ(re.error_code(), RE2::NoError);\n }\n}\n\nTEST(RE2, UTF8) {\n // Check UTF-8 handling\n // Three Japanese characters (nihongo)\n const char utf8_string[] = {\n (char)0xe6, (char)0x97, (char)0xa5, // 65e5\n (char)0xe6, (char)0x9c, (char)0xac, // 627c\n (char)0xe8, (char)0xaa, (char)0x9e, // 8a9e\n 0\n };\n const char utf8_pattern[] = {\n '.',\n (char)0xe6, (char)0x9c, (char)0xac, // 627c\n '.',\n 0\n };\n\n // Both should match in either mode, bytes or UTF-8\n RE2 re_test1(\".........\", RE2::Latin1);\n ASSERT_TRUE(RE2::FullMatch(utf8_string, re_test1));\n RE2 re_test2(\"...\");\n ASSERT_TRUE(RE2::FullMatch(utf8_string, re_test2));\n\n // Check that '.' matches one byte or UTF-8 character\n // according to the mode.\n std::string s;\n RE2 re_test3(\"(.)\", RE2::Latin1);\n ASSERT_TRUE(RE2::PartialMatch(utf8_string, re_test3, &s));\n ASSERT_EQ(s, std::string(\"\\xe6\"));\n RE2 re_test4(\"(.)\");\n ASSERT_TRUE(RE2::PartialMatch(utf8_string, re_test4, &s));\n ASSERT_EQ(s, std::string(\"\\xe6\\x97\\xa5\"));\n\n // Check that string matches itself in either mode\n RE2 re_test5(utf8_string, RE2::Latin1);\n ASSERT_TRUE(RE2::FullMatch(utf8_string, re_test5));\n RE2 re_test6(utf8_string);\n ASSERT_TRUE(RE2::FullMatch(utf8_string, re_test6));\n\n // Check that pattern matches string only in UTF8 mode\n RE2 re_test7(utf8_pattern, RE2::Latin1);\n ASSERT_FALSE(RE2::FullMatch(utf8_string, re_test7));\n RE2 re_test8(utf8_pattern);\n ASSERT_TRUE(RE2::FullMatch(utf8_string, re_test8));\n}\n\nTEST(RE2, UngreedyUTF8) {\n // Check that ungreedy, UTF8 regular expressions don't match when they\n // oughtn't -- see bug 82246.\n {\n // This code always worked.\n const char* pattern = \"\\\\w+X\";\n const std::string target = \"a aX\";\n RE2 match_sentence(pattern, RE2::Latin1);\n RE2 match_sentence_re(pattern);\n\n ASSERT_FALSE(RE2::FullMatch(target, match_sentence));\n ASSERT_FALSE(RE2::FullMatch(target, match_sentence_re));\n }\n {\n const char* pattern = \"(?U)\\\\w+X\";\n const std::string target = \"a aX\";\n RE2 match_sentence(pattern, RE2::Latin1);\n ASSERT_EQ(match_sentence.error(), \"\");\n RE2 match_sentence_re(pattern);\n\n ASSERT_FALSE(RE2::FullMatch(target, match_sentence));\n ASSERT_FALSE(RE2::FullMatch(target, match_sentence_re));\n }\n}\n\nTEST(RE2, Rejects) {\n {\n RE2 re(\"a\\\\1\", RE2::Quiet);\n ASSERT_FALSE(re.ok()); }\n {\n RE2 re(\"a[x\", RE2::Quiet);\n ASSERT_FALSE(re.ok());\n }\n {\n RE2 re(\"a[z-a]\", RE2::Quiet);\n ASSERT_FALSE(re.ok());\n }\n {\n RE2 re(\"a[[:foobar:]]\", RE2::Quiet);\n ASSERT_FALSE(re.ok());\n }\n {\n RE2 re(\"a(b\", RE2::Quiet);\n ASSERT_FALSE(re.ok());\n }\n {\n RE2 re(\"a\\\\\", RE2::Quiet);\n ASSERT_FALSE(re.ok());\n }\n}\n\nTEST(RE2, NoCrash) {\n // Test that using a bad regexp doesn't crash.\n {\n RE2 re(\"a\\\\\", RE2::Quiet);\n ASSERT_FALSE(re.ok());\n ASSERT_FALSE(RE2::PartialMatch(\"a\\\\b\", re));\n }\n\n // Test that using an enormous regexp doesn't crash\n {\n RE2 re(\"(((.{100}){100}){100}){100}\", RE2::Quiet);\n ASSERT_FALSE(re.ok());\n ASSERT_FALSE(RE2::PartialMatch(\"aaa\", re));\n }\n\n // Test that a crazy regexp still compiles and runs.\n {\n RE2 re(\".{512}x\", RE2::Quiet);\n ASSERT_TRUE(re.ok());\n std::string s;\n s.append(515, 'c');\n s.append(\"x\");\n ASSERT_TRUE(RE2::PartialMatch(s, re));\n }\n}\n\nTEST(RE2, Recursion) {\n // Test that recursion is stopped.\n // This test is PCRE-legacy -- there's no recursion in RE2.\n int bytes = 15 * 1024; // enough to crash PCRE\n TestRecursion(bytes, \".\");\n TestRecursion(bytes, \"a\");\n TestRecursion(bytes, \"a.\");\n TestRecursion(bytes, \"ab.\");\n TestRecursion(bytes, \"abc.\");\n}\n\nTEST(RE2, BigCountedRepetition) {\n // Test that counted repetition works, given tons of memory.\n RE2::Options opt;\n opt.set_max_mem(256<<20);\n\n RE2 re(\".{512}x\", opt);\n ASSERT_TRUE(re.ok());\n std::string s;\n s.append(515, 'c');\n s.append(\"x\");\n ASSERT_TRUE(RE2::PartialMatch(s, re));\n}\n\nTEST(RE2, DeepRecursion) {\n // Test for deep stack recursion. This would fail with a\n // segmentation violation due to stack overflow before pcre was\n // patched.\n // Again, a PCRE legacy test. RE2 doesn't recurse.\n std::string comment(\"x*\");\n std::string a(131072, 'a');\n comment += a;\n comment += \"*x\";\n RE2 re(\"((?:\\\\s|xx.*\\n|x[*](?:\\n|.)*?[*]x)*)\");\n ASSERT_TRUE(RE2::FullMatch(comment, re));\n}\n\n// Suggested by Josh Hyman. Failed when SearchOnePass was\n// not implementing case-folding.\nTEST(CaseInsensitive, MatchAndConsume) {\n std::string text = \"A fish named *Wanda*\";\n absl::string_view sp(text);\n absl::string_view result;\n EXPECT_TRUE(RE2::PartialMatch(text, \"(?i)([wand]{5})\", &result));\n EXPECT_TRUE(RE2::FindAndConsume(&sp, \"(?i)([wand]{5})\", &result));\n}\n\n// RE2 should permit implicit conversions from string, string_view, const char*,\n// and C string literals.\nTEST(RE2, ImplicitConversions) {\n std::string re_string(\".\");\n absl::string_view re_string_view(\".\");\n const char* re_c_string = \".\";\n EXPECT_TRUE(RE2::PartialMatch(\"e\", re_string));\n EXPECT_TRUE(RE2::PartialMatch(\"e\", re_string_view));\n EXPECT_TRUE(RE2::PartialMatch(\"e\", re_c_string));\n EXPECT_TRUE(RE2::PartialMatch(\"e\", \".\"));\n}\n\n// Bugs introduced by 8622304\nTEST(RE2, CL8622304) {\n // reported by ingow\n std::string dir;\n EXPECT_TRUE(RE2::FullMatch(\"D\", \"([^\\\\\\\\])\")); // ok\n EXPECT_TRUE(RE2::FullMatch(\"D\", \"([^\\\\\\\\])\", &dir)); // fails\n\n // reported by jacobsa\n std::string key, val;\n EXPECT_TRUE(RE2::PartialMatch(\"bar:1,0x2F,030,4,5;baz:true;fooby:false,true\",\n \"(\\\\w+)(?::((?:[^;\\\\\\\\]|\\\\\\\\.)*))?;?\",\n &key,\n &val));\n EXPECT_EQ(key, \"bar\");\n EXPECT_EQ(val, \"1,0x2F,030,4,5\");\n}\n\n// Check that RE2 returns correct regexp pieces on error.\n// In particular, make sure it returns whole runes\n// and that it always reports invalid UTF-8.\n// Also check that Perl error flag piece is big enough.\nstatic struct ErrorTest {\n const char *regexp;\n RE2::ErrorCode error_code;\n const char *error_arg;\n} error_tests[] = {\n { \"ab\\\\αcd\", RE2::ErrorBadEscape, \"\\\\α\" },\n { \"ef\\\\x☺01\", RE2::ErrorBadEscape, \"\\\\x☺0\" },\n { \"gh\\\\x1☺01\", RE2::ErrorBadEscape, \"\\\\x1☺\" },\n { \"ij\\\\x1\", RE2::ErrorBadEscape, \"\\\\x1\" },\n { \"kl\\\\x\", RE2::ErrorBadEscape, \"\\\\x\" },\n { \"uv\\\\x{0000☺}\", RE2::ErrorBadEscape, \"\\\\x{0000☺\" },\n { \"wx\\\\p{ABC\", RE2::ErrorBadCharRange, \"\\\\p{ABC\" },\n // used to return (?s but the error is X\n { \"yz(?smiUX:abc)\", RE2::ErrorBadPerlOp, \"(?smiUX\" },\n { \"aa(?sm☺i\", RE2::ErrorBadPerlOp, \"(?sm☺\" },\n { \"bb[abc\", RE2::ErrorMissingBracket, \"[abc\" },\n { \"abc(def\", RE2::ErrorMissingParen, \"abc(def\" },\n { \"abc)def\", RE2::ErrorUnexpectedParen, \"abc)def\" },\n\n // no argument string returned for invalid UTF-8\n { \"mn\\\\x1\\377\", RE2::ErrorBadUTF8, \"\" },\n { \"op\\377qr\", RE2::ErrorBadUTF8, \"\" },\n { \"st\\\\x{00000\\377\", RE2::ErrorBadUTF8, \"\" },\n { \"zz\\\\p{\\377}\", RE2::ErrorBadUTF8, \"\" },\n { \"zz\\\\x{00\\377}\", RE2::ErrorBadUTF8, \"\" },\n { \"zz(?Pabc)\", RE2::ErrorBadUTF8, \"\" },\n};\nTEST(RE2, ErrorCodeAndArg) {\n for (size_t i = 0; i < ABSL_ARRAYSIZE(error_tests); i++) {\n RE2 re(error_tests[i].regexp, RE2::Quiet);\n EXPECT_FALSE(re.ok());\n EXPECT_EQ(re.error_code(), error_tests[i].error_code) << re.error();\n EXPECT_EQ(re.error_arg(), error_tests[i].error_arg) << re.error();\n }\n}\n\n// Check that \"never match \\n\" mode never matches \\n.\nstatic struct NeverTest {\n const char* regexp;\n const char* text;\n const char* match;\n} never_tests[] = {\n { \"(.*)\", \"abc\\ndef\\nghi\\n\", \"abc\" },\n { \"(?s)(abc.*def)\", \"abc\\ndef\\n\", NULL },\n { \"(abc(.|\\n)*def)\", \"abc\\ndef\\n\", NULL },\n { \"(abc[^x]*def)\", \"abc\\ndef\\n\", NULL },\n { \"(abc[^x]*def)\", \"abczzzdef\\ndef\\n\", \"abczzzdef\" },\n};\nTEST(RE2, NeverNewline) {\n RE2::Options opt;\n opt.set_never_nl(true);\n for (size_t i = 0; i < ABSL_ARRAYSIZE(never_tests); i++) {\n const NeverTest& t = never_tests[i];\n RE2 re(t.regexp, opt);\n if (t.match == NULL) {\n EXPECT_FALSE(re.PartialMatch(t.text, re));\n } else {\n absl::string_view m;\n EXPECT_TRUE(re.PartialMatch(t.text, re, &m));\n EXPECT_EQ(m, t.match);\n }\n }\n}\n\n// Check that dot_nl option works.\nTEST(RE2, DotNL) {\n RE2::Options opt;\n opt.set_dot_nl(true);\n EXPECT_TRUE(RE2::PartialMatch(\"\\n\", RE2(\".\", opt)));\n EXPECT_FALSE(RE2::PartialMatch(\"\\n\", RE2(\"(?-s).\", opt)));\n opt.set_never_nl(true);\n EXPECT_FALSE(RE2::PartialMatch(\"\\n\", RE2(\".\", opt)));\n}\n\n// Check that there are no capturing groups in \"never capture\" mode.\nTEST(RE2, NeverCapture) {\n RE2::Options opt;\n opt.set_never_capture(true);\n RE2 re(\"(r)(e)\", opt);\n EXPECT_EQ(0, re.NumberOfCapturingGroups());\n}\n\n// Bitstate bug was looking at submatch[0] even if nsubmatch == 0.\n// Triggered by a failed DFA search falling back to Bitstate when\n// using Match with a NULL submatch set. Bitstate tried to read\n// the submatch[0] entry even if nsubmatch was 0.\nTEST(RE2, BitstateCaptureBug) {\n RE2::Options opt;\n opt.set_max_mem(20000);\n RE2 re(\"(_________$)\", opt);\n absl::string_view s = \"xxxxxxxxxxxxxxxxxxxxxxxxxx_________x\";\n EXPECT_FALSE(re.Match(s, 0, s.size(), RE2::UNANCHORED, NULL, 0));\n}\n\n// C++ version of bug 609710.\nTEST(RE2, UnicodeClasses) {\n const std::string str = \"ABCDEFGHI譚永鋒\";\n std::string a, b, c;\n\n EXPECT_TRUE(RE2::FullMatch(\"A\", \"\\\\p{L}\"));\n EXPECT_TRUE(RE2::FullMatch(\"A\", \"\\\\p{Lu}\"));\n EXPECT_FALSE(RE2::FullMatch(\"A\", \"\\\\p{Ll}\"));\n EXPECT_FALSE(RE2::FullMatch(\"A\", \"\\\\P{L}\"));\n EXPECT_FALSE(RE2::FullMatch(\"A\", \"\\\\P{Lu}\"));\n EXPECT_TRUE(RE2::FullMatch(\"A\", \"\\\\P{Ll}\"));\n\n EXPECT_TRUE(RE2::FullMatch(\"譚\", \"\\\\p{L}\"));\n EXPECT_FALSE(RE2::FullMatch(\"譚\", \"\\\\p{Lu}\"));\n EXPECT_FALSE(RE2::FullMatch(\"譚\", \"\\\\p{Ll}\"));\n EXPECT_FALSE(RE2::FullMatch(\"譚\", \"\\\\P{L}\"));\n EXPECT_TRUE(RE2::FullMatch(\"譚\", \"\\\\P{Lu}\"));\n EXPECT_TRUE(RE2::FullMatch(\"譚\", \"\\\\P{Ll}\"));\n\n EXPECT_TRUE(RE2::FullMatch(\"永\", \"\\\\p{L}\"));\n EXPECT_FALSE(RE2::FullMatch(\"永\", \"\\\\p{Lu}\"));\n EXPECT_FALSE(RE2::FullMatch(\"永\", \"\\\\p{Ll}\"));\n EXPECT_FALSE(RE2::FullMatch(\"永\", \"\\\\P{L}\"));\n EXPECT_TRUE(RE2::FullMatch(\"永\", \"\\\\P{Lu}\"));\n EXPECT_TRUE(RE2::FullMatch(\"永\", \"\\\\P{Ll}\"));\n\n EXPECT_TRUE(RE2::FullMatch(\"鋒\", \"\\\\p{L}\"));\n EXPECT_FALSE(RE2::FullMatch(\"鋒\", \"\\\\p{Lu}\"));\n EXPECT_FALSE(RE2::FullMatch(\"鋒\", \"\\\\p{Ll}\"));\n EXPECT_FALSE(RE2::FullMatch(\"鋒\", \"\\\\P{L}\"));\n EXPECT_TRUE(RE2::FullMatch(\"鋒\", \"\\\\P{Lu}\"));\n EXPECT_TRUE(RE2::FullMatch(\"鋒\", \"\\\\P{Ll}\"));\n\n EXPECT_TRUE(RE2::PartialMatch(str, \"(.).*?(.).*?(.)\", &a, &b, &c));\n EXPECT_EQ(\"A\", a);\n EXPECT_EQ(\"B\", b);\n EXPECT_EQ(\"C\", c);\n\n EXPECT_TRUE(RE2::PartialMatch(str, \"(.).*?([\\\\p{L}]).*?(.)\", &a, &b, &c));\n EXPECT_EQ(\"A\", a);\n EXPECT_EQ(\"B\", b);\n EXPECT_EQ(\"C\", c);\n\n EXPECT_FALSE(RE2::PartialMatch(str, \"\\\\P{L}\"));\n\n EXPECT_TRUE(RE2::PartialMatch(str, \"(.).*?([\\\\p{Lu}]).*?(.)\", &a, &b, &c));\n EXPECT_EQ(\"A\", a);\n EXPECT_EQ(\"B\", b);\n EXPECT_EQ(\"C\", c);\n\n EXPECT_FALSE(RE2::PartialMatch(str, \"[^\\\\p{Lu}\\\\p{Lo}]\"));\n\n EXPECT_TRUE(RE2::PartialMatch(str, \".*(.).*?([\\\\p{Lu}\\\\p{Lo}]).*?(.)\", &a, &b, &c));\n EXPECT_EQ(\"譚\", a);\n EXPECT_EQ(\"永\", b);\n EXPECT_EQ(\"鋒\", c);\n}\n\nTEST(RE2, LazyRE2) {\n // Test with and without options.\n static LazyRE2 a = {\"a\"};\n static LazyRE2 b = {\"b\", RE2::Latin1};\n\n EXPECT_EQ(\"a\", a->pattern());\n EXPECT_EQ(RE2::Options::EncodingUTF8, a->options().encoding());\n\n EXPECT_EQ(\"b\", b->pattern());\n EXPECT_EQ(RE2::Options::EncodingLatin1, b->options().encoding());\n}\n\n// Bug reported by saito. 2009/02/17\nTEST(RE2, NullVsEmptyString) {\n RE2 re(\".*\");\n EXPECT_TRUE(re.ok());\n\n absl::string_view null;\n EXPECT_TRUE(RE2::FullMatch(null, re));\n\n absl::string_view empty(\"\");\n EXPECT_TRUE(RE2::FullMatch(empty, re));\n}\n\n// Similar to the previous test, check that the null string and the empty\n// string both match, but also that the null string can only provide null\n// submatches whereas the empty string can also provide empty submatches.\nTEST(RE2, NullVsEmptyStringSubmatches) {\n RE2 re(\"()|(foo)\");\n EXPECT_TRUE(re.ok());\n\n // matches[0] is overall match, [1] is (), [2] is (foo), [3] is nonexistent.\n absl::string_view matches[4];\n\n for (size_t i = 0; i < ABSL_ARRAYSIZE(matches); i++)\n matches[i] = \"bar\";\n\n absl::string_view null;\n EXPECT_TRUE(re.Match(null, 0, null.size(), RE2::UNANCHORED,\n matches, ABSL_ARRAYSIZE(matches)));\n for (size_t i = 0; i < ABSL_ARRAYSIZE(matches); i++) {\n EXPECT_TRUE(matches[i].data() == NULL); // always null\n EXPECT_TRUE(matches[i].empty());\n }\n\n for (size_t i = 0; i < ABSL_ARRAYSIZE(matches); i++)\n matches[i] = \"bar\";\n\n absl::string_view empty(\"\");\n EXPECT_TRUE(re.Match(empty, 0, empty.size(), RE2::UNANCHORED,\n matches, ABSL_ARRAYSIZE(matches)));\n EXPECT_TRUE(matches[0].data() != NULL); // empty, not null\n EXPECT_TRUE(matches[0].empty());\n EXPECT_TRUE(matches[1].data() != NULL); // empty, not null\n EXPECT_TRUE(matches[1].empty());\n EXPECT_TRUE(matches[2].data() == NULL);\n EXPECT_TRUE(matches[2].empty());\n EXPECT_TRUE(matches[3].data() == NULL);\n EXPECT_TRUE(matches[3].empty());\n}\n\n// Issue 1816809\nTEST(RE2, Bug1816809) {\n RE2 re(\"(((((llx((-3)|(4)))(;(llx((-3)|(4))))*))))\");\n absl::string_view piece(\"llx-3;llx4\");\n std::string x;\n EXPECT_TRUE(RE2::Consume(&piece, re, &x));\n}\n\n// Issue 3061120\nTEST(RE2, Bug3061120) {\n RE2 re(\"(?i)\\\\W\");\n EXPECT_FALSE(RE2::PartialMatch(\"x\", re)); // always worked\n EXPECT_FALSE(RE2::PartialMatch(\"k\", re)); // broke because of kelvin\n EXPECT_FALSE(RE2::PartialMatch(\"s\", re)); // broke because of latin long s\n}\n\nTEST(RE2, CapturingGroupNames) {\n // Opening parentheses annotated with group IDs:\n // 12 3 45 6 7\n RE2 re(\"((abc)(?P)|((e+)(?P.*)(?Pu+)))\");\n EXPECT_TRUE(re.ok());\n const std::map& have = re.CapturingGroupNames();\n std::map want;\n want[3] = \"G2\";\n want[6] = \"G2\";\n want[7] = \"G1\";\n EXPECT_EQ(want, have);\n}\n\nTEST(RE2, RegexpToStringLossOfAnchor) {\n EXPECT_EQ(RE2(\"^[a-c]at\", RE2::POSIX).Regexp()->ToString(), \"^[a-c]at\");\n EXPECT_EQ(RE2(\"^[a-c]at\").Regexp()->ToString(), \"(?-m:^)[a-c]at\");\n EXPECT_EQ(RE2(\"ca[t-z]$\", RE2::POSIX).Regexp()->ToString(), \"ca[t-z]$\");\n EXPECT_EQ(RE2(\"ca[t-z]$\").Regexp()->ToString(), \"ca[t-z](?-m:$)\");\n}\n\n// Issue 10131674\nTEST(RE2, Bug10131674) {\n // Some of these escapes describe values that do not fit in a byte.\n RE2 re(\"\\\\140\\\\440\\\\174\\\\271\\\\150\\\\656\\\\106\\\\201\\\\004\\\\332\", RE2::Latin1);\n EXPECT_FALSE(re.ok());\n EXPECT_FALSE(RE2::FullMatch(\"hello world\", re));\n}\n\nTEST(RE2, Bug18391750) {\n // Stray write past end of match_ in nfa.cc, caught by fuzzing + address sanitizer.\n const char t[] = {\n (char)0x28, (char)0x28, (char)0xfc, (char)0xfc, (char)0x08, (char)0x08,\n (char)0x26, (char)0x26, (char)0x28, (char)0xc2, (char)0x9b, (char)0xc5,\n (char)0xc5, (char)0xd4, (char)0x8f, (char)0x8f, (char)0x69, (char)0x69,\n (char)0xe7, (char)0x29, (char)0x7b, (char)0x37, (char)0x31, (char)0x31,\n (char)0x7d, (char)0xae, (char)0x7c, (char)0x7c, (char)0xf3, (char)0x29,\n (char)0xae, (char)0xae, (char)0x2e, (char)0x2a, (char)0x29, (char)0x00,\n };\n RE2::Options opt;\n opt.set_encoding(RE2::Options::EncodingLatin1);\n opt.set_longest_match(true);\n opt.set_dot_nl(true);\n opt.set_case_sensitive(false);\n RE2 re(t, opt);\n ASSERT_TRUE(re.ok());\n RE2::PartialMatch(t, re);\n}\n\nTEST(RE2, Bug18458852) {\n // Bug in parser accepting invalid (too large) rune,\n // causing compiler to fail in ABSL_DCHECK() in UTF-8\n // character class code.\n const char b[] = {\n (char)0x28, (char)0x05, (char)0x05, (char)0x41, (char)0x41, (char)0x28,\n (char)0x24, (char)0x5b, (char)0x5e, (char)0xf5, (char)0x87, (char)0x87,\n (char)0x90, (char)0x29, (char)0x5d, (char)0x29, (char)0x29, (char)0x00,\n };\n RE2 re(b);\n ASSERT_FALSE(re.ok());\n}\n\nTEST(RE2, Bug18523943) {\n // Bug in BitState: case kFailInst failed the match entirely.\n\n RE2::Options opt;\n const char a[] = {\n (char)0x29, (char)0x29, (char)0x24, (char)0x00,\n };\n const char b[] = {\n (char)0x28, (char)0x0a, (char)0x2a, (char)0x2a, (char)0x29, (char)0x00,\n };\n opt.set_log_errors(false);\n opt.set_encoding(RE2::Options::EncodingLatin1);\n opt.set_posix_syntax(true);\n opt.set_longest_match(true);\n opt.set_literal(false);\n opt.set_never_nl(true);\n\n RE2 re((const char*)b, opt);\n ASSERT_TRUE(re.ok());\n std::string s1;\n ASSERT_TRUE(RE2::PartialMatch((const char*)a, re, &s1));\n}\n\nTEST(RE2, Bug21371806) {\n // Bug in parser accepting Unicode groups in Latin-1 mode,\n // causing compiler to fail in ABSL_DCHECK() in prog.cc.\n\n RE2::Options opt;\n opt.set_encoding(RE2::Options::EncodingLatin1);\n\n RE2 re(\"g\\\\p{Zl}]\", opt);\n ASSERT_TRUE(re.ok());\n}\n\nTEST(RE2, Bug26356109) {\n // Bug in parser caused by factoring of common prefixes in alternations.\n\n // In the past, this was factored to \"a\\\\C*?[bc]\". Thus, the automaton would\n // consume \"ab\" and then stop (when unanchored) whereas it should consume all\n // of \"abc\" as per first-match semantics.\n RE2 re(\"a\\\\C*?c|a\\\\C*?b\");\n ASSERT_TRUE(re.ok());\n\n std::string s = \"abc\";\n absl::string_view m;\n\n ASSERT_TRUE(re.Match(s, 0, s.size(), RE2::UNANCHORED, &m, 1));\n ASSERT_EQ(m, s) << \" (UNANCHORED) got m='\" << m << \"', want '\" << s << \"'\";\n\n ASSERT_TRUE(re.Match(s, 0, s.size(), RE2::ANCHOR_BOTH, &m, 1));\n ASSERT_EQ(m, s) << \" (ANCHOR_BOTH) got m='\" << m << \"', want '\" << s << \"'\";\n}\n\nTEST(RE2, Issue104) {\n // RE2::GlobalReplace always advanced by one byte when the empty string was\n // matched, which would clobber any rune that is longer than one byte.\n\n std::string s = \"bc\";\n ASSERT_EQ(3, RE2::GlobalReplace(&s, \"a*\", \"d\"));\n ASSERT_EQ(\"dbdcd\", s);\n\n s = \"ąć\";\n ASSERT_EQ(3, RE2::GlobalReplace(&s, \"Ć*\", \"Ĉ\"));\n ASSERT_EQ(\"ĈąĈćĈ\", s);\n\n s = \"人类\";\n ASSERT_EQ(3, RE2::GlobalReplace(&s, \"大*\", \"小\"));\n ASSERT_EQ(\"小人小类小\", s);\n}\n\nTEST(RE2, Issue310) {\n // (?:|a)* matched more text than (?:|a)+ did.\n\n std::string s = \"aaa\";\n absl::string_view m;\n\n RE2 star(\"(?:|a)*\");\n ASSERT_TRUE(star.Match(s, 0, s.size(), RE2::UNANCHORED, &m, 1));\n ASSERT_EQ(m, \"\") << \" got m='\" << m << \"', want ''\";\n\n RE2 plus(\"(?:|a)+\");\n ASSERT_TRUE(plus.Match(s, 0, s.size(), RE2::UNANCHORED, &m, 1));\n ASSERT_EQ(m, \"\") << \" got m='\" << m << \"', want ''\";\n}\n\nTEST(RE2, Issue477) {\n // Regexp::LeadingString didn't output Latin1 into flags.\n // In the given pattern, 0xA5 should be factored out, but\n // shouldn't lose its Latin1-ness in the process. Because\n // that was happening, the prefix for accel was 0xC2 0xA5\n // instead of 0xA5. Note that the former doesn't occur in\n // the given input and so replacements weren't occurring.\n\n const char bytes[] = {\n (char)0xa5, (char)0xd1, (char)0xa5, (char)0xd1,\n (char)0x61, (char)0x63, (char)0xa5, (char)0x64,\n };\n std::string s(bytes, ABSL_ARRAYSIZE(bytes));\n RE2 re(\"\\xa5\\xd1|\\xa5\\x64\", RE2::Latin1);\n int n = RE2::GlobalReplace(&s, re, \"\");\n ASSERT_EQ(n, 3);\n ASSERT_EQ(s, \"\\x61\\x63\");\n}\n\nTEST(RE2, InitNULL) {\n // RE2::RE2 accepts NULL. Make sure it keeps doing that.\n RE2 re(NULL);\n ASSERT_TRUE(re.ok());\n ASSERT_TRUE(RE2::FullMatch(\"\", re));\n ASSERT_TRUE(RE2::FullMatch(\"\", NULL));\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/regexp_benchmark.cc", "content": "// Copyright 2006-2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Benchmarks for regular expression implementations.\n\n#include \n#include \n#include \n\n#include \n#include \n\n#include \"absl/container/flat_hash_map.h\"\n#include \"absl/flags/flag.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"absl/synchronization/mutex.h\"\n#include \"benchmark/benchmark.h\"\n#include \"re2/prog.h\"\n#include \"re2/re2.h\"\n#include \"re2/regexp.h\"\n#include \"util/malloc_counter.h\"\n#include \"util/pcre.h\"\n\nnamespace re2 {\nvoid Test();\nvoid MemoryUsage();\n} // namespace re2\n\ntypedef testing::MallocCounter MallocCounter;\n\nnamespace re2 {\n\nvoid Test() {\n Regexp* re = Regexp::Parse(\"(\\\\d+)-(\\\\d+)-(\\\\d+)\", Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n ABSL_CHECK(prog->IsOnePass());\n ABSL_CHECK(prog->CanBitState());\n const char* text = \"650-253-0001\";\n absl::string_view sp[4];\n ABSL_CHECK(prog->SearchOnePass(text, text, Prog::kAnchored, Prog::kFullMatch,\n sp, 4));\n ABSL_CHECK_EQ(sp[0], \"650-253-0001\");\n ABSL_CHECK_EQ(sp[1], \"650\");\n ABSL_CHECK_EQ(sp[2], \"253\");\n ABSL_CHECK_EQ(sp[3], \"0001\");\n delete prog;\n re->Decref();\n ABSL_LOG(INFO) << \"test passed\\n\";\n}\n\nvoid MemoryUsage() {\n const char* regexp = \"(\\\\d+)-(\\\\d+)-(\\\\d+)\";\n const char* text = \"650-253-0001\";\n {\n MallocCounter mc(MallocCounter::THIS_THREAD_ONLY);\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n // Can't pass mc.HeapGrowth() and mc.PeakHeapGrowth() to ABSL_LOG(INFO)\n // directly because ABSL_LOG(INFO) might do a big allocation before they\n // get evaluated.\n absl::FPrintF(stderr, \"Regexp: %7d bytes (peak=%d)\\n\",\n mc.HeapGrowth(), mc.PeakHeapGrowth());\n mc.Reset();\n\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n ABSL_CHECK(prog->IsOnePass());\n ABSL_CHECK(prog->CanBitState());\n absl::FPrintF(stderr, \"Prog: %7d bytes (peak=%d)\\n\",\n mc.HeapGrowth(), mc.PeakHeapGrowth());\n mc.Reset();\n\n absl::string_view sp[4];\n ABSL_CHECK(prog->SearchOnePass(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 4));\n absl::FPrintF(stderr, \"Search: %7d bytes (peak=%d)\\n\",\n mc.HeapGrowth(), mc.PeakHeapGrowth());\n delete prog;\n re->Decref();\n }\n\n {\n MallocCounter mc(MallocCounter::THIS_THREAD_ONLY);\n\n PCRE re(regexp, PCRE::UTF8);\n absl::FPrintF(stderr, \"RE: %7d bytes (peak=%d)\\n\",\n mc.HeapGrowth(), mc.PeakHeapGrowth());\n PCRE::FullMatch(text, re);\n absl::FPrintF(stderr, \"RE: %7d bytes (peak=%d)\\n\",\n mc.HeapGrowth(), mc.PeakHeapGrowth());\n }\n\n {\n MallocCounter mc(MallocCounter::THIS_THREAD_ONLY);\n\n PCRE* re = new PCRE(regexp, PCRE::UTF8);\n absl::FPrintF(stderr, \"PCRE*: %7d bytes (peak=%d)\\n\",\n mc.HeapGrowth(), mc.PeakHeapGrowth());\n PCRE::FullMatch(text, *re);\n absl::FPrintF(stderr, \"PCRE*: %7d bytes (peak=%d)\\n\",\n mc.HeapGrowth(), mc.PeakHeapGrowth());\n delete re;\n }\n\n {\n MallocCounter mc(MallocCounter::THIS_THREAD_ONLY);\n\n RE2 re(regexp);\n absl::FPrintF(stderr, \"RE2: %7d bytes (peak=%d)\\n\",\n mc.HeapGrowth(), mc.PeakHeapGrowth());\n RE2::FullMatch(text, re);\n absl::FPrintF(stderr, \"RE2: %7d bytes (peak=%d)\\n\",\n mc.HeapGrowth(), mc.PeakHeapGrowth());\n }\n\n absl::FPrintF(stderr, \"sizeof: PCRE=%d RE2=%d Prog=%d Inst=%d\\n\",\n sizeof(PCRE), sizeof(RE2), sizeof(Prog), sizeof(Prog::Inst));\n}\n\nint NumCPUs() {\n return static_cast(std::thread::hardware_concurrency());\n}\n\n// Regular expression implementation wrappers.\n// Defined at bottom of file, but they are repetitive\n// and not interesting.\n\ntypedef void SearchImpl(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match);\n\nSearchImpl SearchDFA, SearchNFA, SearchOnePass, SearchBitState, SearchPCRE,\n SearchRE2, SearchCachedDFA, SearchCachedNFA, SearchCachedOnePass,\n SearchCachedBitState, SearchCachedPCRE, SearchCachedRE2;\n\ntypedef void ParseImpl(benchmark::State& state, const char* regexp,\n absl::string_view text);\n\nParseImpl Parse1NFA, Parse1OnePass, Parse1BitState, Parse1PCRE, Parse1RE2,\n Parse1Backtrack, Parse1CachedNFA, Parse1CachedOnePass, Parse1CachedBitState,\n Parse1CachedPCRE, Parse1CachedRE2, Parse1CachedBacktrack;\n\nParseImpl Parse3NFA, Parse3OnePass, Parse3BitState, Parse3PCRE, Parse3RE2,\n Parse3Backtrack, Parse3CachedNFA, Parse3CachedOnePass, Parse3CachedBitState,\n Parse3CachedPCRE, Parse3CachedRE2, Parse3CachedBacktrack;\n\nParseImpl SearchParse2CachedPCRE, SearchParse2CachedRE2;\n\nParseImpl SearchParse1CachedPCRE, SearchParse1CachedRE2;\n\n// Benchmark: failed search for regexp in random text.\n\n// Generate random text that won't contain the search string,\n// to test worst-case search behavior.\nstd::string RandomText(int64_t nbytes) {\n static const std::string* const text = []() {\n std::string* text = new std::string;\n srand(1);\n text->resize(16<<20);\n for (int64_t i = 0; i < 16<<20; i++) {\n // Generate a one-byte rune that isn't a control character (e.g. '\\n').\n // Clipping to 0x20 introduces some bias, but we don't need uniformity.\n int byte = rand() & 0x7F;\n if (byte < 0x20)\n byte = 0x20;\n (*text)[i] = byte;\n }\n return text;\n }();\n ABSL_CHECK_LE(nbytes, 16<<20);\n return text->substr(0, nbytes);\n}\n\n// Makes text of size nbytes, then calls run to search\n// the text for regexp iters times.\nvoid Search(benchmark::State& state, const char* regexp, SearchImpl* search) {\n std::string s = RandomText(state.range(0));\n search(state, regexp, s, Prog::kUnanchored, false);\n state.SetBytesProcessed(state.iterations() * state.range(0));\n}\n\n// These three are easy because they have prefixes,\n// giving the search loop something to prefix accel.\n#define EASY0 \"ABCDEFGHIJKLMNOPQRSTUVWXYZ$\"\n#define EASY1 \"A[AB]B[BC]C[CD]D[DE]E[EF]F[FG]G[GH]H[HI]I[IJ]J$\"\n#define EASY2 \"(?i)\" EASY0\n\n// This is a little harder, since it starts with a character class\n// and thus can't be memchr'ed. Could look for ABC and work backward,\n// but no one does that.\n#define MEDIUM \"[XYZ]ABCDEFGHIJKLMNOPQRSTUVWXYZ$\"\n\n// This is a fair amount harder, because of the leading [ -~]*.\n// A bad backtracking implementation will take O(text^2) time to\n// figure out there's no match.\n#define HARD \"[ -~]*ABCDEFGHIJKLMNOPQRSTUVWXYZ$\"\n\n// This has quite a high degree of fanout.\n// NFA execution will be particularly slow.\n#define FANOUT \"(?:[\\\\x{80}-\\\\x{10FFFF}]?){100}[\\\\x{80}-\\\\x{10FFFF}]\"\n\n// This stresses engines that are trying to track parentheses.\n#define PARENS \"([ -~])*(A)(B)(C)(D)(E)(F)(G)(H)(I)(J)(K)(L)(M)\" \\\n \"(N)(O)(P)(Q)(R)(S)(T)(U)(V)(W)(X)(Y)(Z)$\"\n\nvoid Search_Easy0_CachedDFA(benchmark::State& state) { Search(state, EASY0, SearchCachedDFA); }\nvoid Search_Easy0_CachedNFA(benchmark::State& state) { Search(state, EASY0, SearchCachedNFA); }\nvoid Search_Easy0_CachedPCRE(benchmark::State& state) { Search(state, EASY0, SearchCachedPCRE); }\nvoid Search_Easy0_CachedRE2(benchmark::State& state) { Search(state, EASY0, SearchCachedRE2); }\n\nBENCHMARK_RANGE(Search_Easy0_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Easy0_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Easy0_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Easy0_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\nvoid Search_Easy1_CachedDFA(benchmark::State& state) { Search(state, EASY1, SearchCachedDFA); }\nvoid Search_Easy1_CachedNFA(benchmark::State& state) { Search(state, EASY1, SearchCachedNFA); }\nvoid Search_Easy1_CachedPCRE(benchmark::State& state) { Search(state, EASY1, SearchCachedPCRE); }\nvoid Search_Easy1_CachedRE2(benchmark::State& state) { Search(state, EASY1, SearchCachedRE2); }\n\nBENCHMARK_RANGE(Search_Easy1_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Easy1_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Easy1_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Easy1_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\nvoid Search_Easy2_CachedDFA(benchmark::State& state) { Search(state, EASY2, SearchCachedDFA); }\nvoid Search_Easy2_CachedNFA(benchmark::State& state) { Search(state, EASY2, SearchCachedNFA); }\nvoid Search_Easy2_CachedPCRE(benchmark::State& state) { Search(state, EASY2, SearchCachedPCRE); }\nvoid Search_Easy2_CachedRE2(benchmark::State& state) { Search(state, EASY2, SearchCachedRE2); }\n\nBENCHMARK_RANGE(Search_Easy2_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Easy2_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Easy2_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Easy2_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\nvoid Search_Medium_CachedDFA(benchmark::State& state) { Search(state, MEDIUM, SearchCachedDFA); }\nvoid Search_Medium_CachedNFA(benchmark::State& state) { Search(state, MEDIUM, SearchCachedNFA); }\nvoid Search_Medium_CachedPCRE(benchmark::State& state) { Search(state, MEDIUM, SearchCachedPCRE); }\nvoid Search_Medium_CachedRE2(benchmark::State& state) { Search(state, MEDIUM, SearchCachedRE2); }\n\nBENCHMARK_RANGE(Search_Medium_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Medium_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Medium_CachedPCRE, 8, 256<<10)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Medium_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\nvoid Search_Hard_CachedDFA(benchmark::State& state) { Search(state, HARD, SearchCachedDFA); }\nvoid Search_Hard_CachedNFA(benchmark::State& state) { Search(state, HARD, SearchCachedNFA); }\nvoid Search_Hard_CachedPCRE(benchmark::State& state) { Search(state, HARD, SearchCachedPCRE); }\nvoid Search_Hard_CachedRE2(benchmark::State& state) { Search(state, HARD, SearchCachedRE2); }\n\nBENCHMARK_RANGE(Search_Hard_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Hard_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Hard_CachedPCRE, 8, 4<<10)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Hard_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\nvoid Search_Fanout_CachedDFA(benchmark::State& state) { Search(state, FANOUT, SearchCachedDFA); }\nvoid Search_Fanout_CachedNFA(benchmark::State& state) { Search(state, FANOUT, SearchCachedNFA); }\nvoid Search_Fanout_CachedPCRE(benchmark::State& state) { Search(state, FANOUT, SearchCachedPCRE); }\nvoid Search_Fanout_CachedRE2(benchmark::State& state) { Search(state, FANOUT, SearchCachedRE2); }\n\nBENCHMARK_RANGE(Search_Fanout_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Fanout_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Fanout_CachedPCRE, 8, 4<<10)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Fanout_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\nvoid Search_Parens_CachedDFA(benchmark::State& state) { Search(state, PARENS, SearchCachedDFA); }\nvoid Search_Parens_CachedNFA(benchmark::State& state) { Search(state, PARENS, SearchCachedNFA); }\nvoid Search_Parens_CachedPCRE(benchmark::State& state) { Search(state, PARENS, SearchCachedPCRE); }\nvoid Search_Parens_CachedRE2(benchmark::State& state) { Search(state, PARENS, SearchCachedRE2); }\n\nBENCHMARK_RANGE(Search_Parens_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Parens_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Parens_CachedPCRE, 8, 8)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Parens_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\nvoid SearchBigFixed(benchmark::State& state, SearchImpl* search) {\n std::string s;\n s.append(state.range(0)/2, 'x');\n std::string regexp = \"^\" + s + \".*$\";\n std::string t = RandomText(state.range(0)/2);\n s += t;\n search(state, regexp.c_str(), s, Prog::kUnanchored, true);\n state.SetBytesProcessed(state.iterations() * state.range(0));\n}\n\nvoid Search_BigFixed_CachedDFA(benchmark::State& state) { SearchBigFixed(state, SearchCachedDFA); }\nvoid Search_BigFixed_CachedNFA(benchmark::State& state) { SearchBigFixed(state, SearchCachedNFA); }\nvoid Search_BigFixed_CachedPCRE(benchmark::State& state) { SearchBigFixed(state, SearchCachedPCRE); }\nvoid Search_BigFixed_CachedRE2(benchmark::State& state) { SearchBigFixed(state, SearchCachedRE2); }\n\nBENCHMARK_RANGE(Search_BigFixed_CachedDFA, 8, 1<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_BigFixed_CachedNFA, 8, 32<<10)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_BigFixed_CachedPCRE, 8, 32<<10)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_BigFixed_CachedRE2, 8, 1<<20)->ThreadRange(1, NumCPUs());\n\n// Benchmark: FindAndConsume\n\nvoid FindAndConsume(benchmark::State& state) {\n std::string s = RandomText(state.range(0));\n s.append(\"Hello World\");\n RE2 re(\"((Hello World))\");\n for (auto _ : state) {\n absl::string_view t = s;\n absl::string_view u;\n ABSL_CHECK(RE2::FindAndConsume(&t, re, &u));\n ABSL_CHECK_EQ(u, \"Hello World\");\n }\n state.SetBytesProcessed(state.iterations() * state.range(0));\n}\n\nBENCHMARK_RANGE(FindAndConsume, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\n// Benchmark: successful anchored search.\n\nvoid SearchSuccess(benchmark::State& state, const char* regexp,\n SearchImpl* search) {\n std::string s = RandomText(state.range(0));\n search(state, regexp, s, Prog::kAnchored, true);\n state.SetBytesProcessed(state.iterations() * state.range(0));\n}\n\n// Unambiguous search (RE2 can use OnePass).\n\nvoid Search_Success_DFA(benchmark::State& state) { SearchSuccess(state, \".*$\", SearchDFA); }\nvoid Search_Success_NFA(benchmark::State& state) { SearchSuccess(state, \".*$\", SearchNFA); }\nvoid Search_Success_PCRE(benchmark::State& state) { SearchSuccess(state, \".*$\", SearchPCRE); }\nvoid Search_Success_RE2(benchmark::State& state) { SearchSuccess(state, \".*$\", SearchRE2); }\nvoid Search_Success_OnePass(benchmark::State& state) { SearchSuccess(state, \".*$\", SearchOnePass); }\n\nBENCHMARK_RANGE(Search_Success_DFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Success_NFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Success_PCRE, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Success_RE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Success_OnePass, 8, 2<<20)->ThreadRange(1, NumCPUs());\n\nvoid Search_Success_CachedDFA(benchmark::State& state) { SearchSuccess(state, \".*$\", SearchCachedDFA); }\nvoid Search_Success_CachedNFA(benchmark::State& state) { SearchSuccess(state, \".*$\", SearchCachedNFA); }\nvoid Search_Success_CachedPCRE(benchmark::State& state) { SearchSuccess(state, \".*$\", SearchCachedPCRE); }\nvoid Search_Success_CachedRE2(benchmark::State& state) { SearchSuccess(state, \".*$\", SearchCachedRE2); }\nvoid Search_Success_CachedOnePass(benchmark::State& state) { SearchSuccess(state, \".*$\", SearchCachedOnePass); }\n\nBENCHMARK_RANGE(Search_Success_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Success_CachedNFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Success_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Success_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Success_CachedOnePass, 8, 2<<20)->ThreadRange(1, NumCPUs());\n\n// Ambiguous search (RE2 cannot use OnePass).\n// Used to be \".*.$\", but that is coalesced to \".+$\" these days.\n\nvoid Search_Success1_DFA(benchmark::State& state) { SearchSuccess(state, \".*\\\\C$\", SearchDFA); }\nvoid Search_Success1_NFA(benchmark::State& state) { SearchSuccess(state, \".*\\\\C$\", SearchNFA); }\nvoid Search_Success1_PCRE(benchmark::State& state) { SearchSuccess(state, \".*\\\\C$\", SearchPCRE); }\nvoid Search_Success1_RE2(benchmark::State& state) { SearchSuccess(state, \".*\\\\C$\", SearchRE2); }\nvoid Search_Success1_BitState(benchmark::State& state) { SearchSuccess(state, \".*\\\\C$\", SearchBitState); }\n\nBENCHMARK_RANGE(Search_Success1_DFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Success1_NFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Success1_PCRE, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Success1_RE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Success1_BitState, 8, 2<<20)->ThreadRange(1, NumCPUs());\n\nvoid Search_Success1_CachedDFA(benchmark::State& state) { SearchSuccess(state, \".*\\\\C$\", SearchCachedDFA); }\nvoid Search_Success1_CachedNFA(benchmark::State& state) { SearchSuccess(state, \".*\\\\C$\", SearchCachedNFA); }\nvoid Search_Success1_CachedPCRE(benchmark::State& state) { SearchSuccess(state, \".*\\\\C$\", SearchCachedPCRE); }\nvoid Search_Success1_CachedRE2(benchmark::State& state) { SearchSuccess(state, \".*\\\\C$\", SearchCachedRE2); }\nvoid Search_Success1_CachedBitState(benchmark::State& state) { SearchSuccess(state, \".*\\\\C$\", SearchCachedBitState); }\n\nBENCHMARK_RANGE(Search_Success1_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Success1_CachedNFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_Success1_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_Success1_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_Success1_CachedBitState, 8, 2<<20)->ThreadRange(1, NumCPUs());\n\n// Benchmark: AltMatch optimisation (just to verify that it works)\n// Note that OnePass doesn't implement it!\n\nvoid SearchAltMatch(benchmark::State& state, SearchImpl* search) {\n std::string s = RandomText(state.range(0));\n search(state, \"\\\\C*\", s, Prog::kAnchored, true);\n state.SetBytesProcessed(state.iterations() * state.range(0));\n}\n\nvoid Search_AltMatch_DFA(benchmark::State& state) { SearchAltMatch(state, SearchDFA); }\nvoid Search_AltMatch_NFA(benchmark::State& state) { SearchAltMatch(state, SearchNFA); }\nvoid Search_AltMatch_OnePass(benchmark::State& state) { SearchAltMatch(state, SearchOnePass); }\nvoid Search_AltMatch_BitState(benchmark::State& state) { SearchAltMatch(state, SearchBitState); }\nvoid Search_AltMatch_PCRE(benchmark::State& state) { SearchAltMatch(state, SearchPCRE); }\nvoid Search_AltMatch_RE2(benchmark::State& state) { SearchAltMatch(state, SearchRE2); }\n\nBENCHMARK_RANGE(Search_AltMatch_DFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_AltMatch_NFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_AltMatch_OnePass, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_AltMatch_BitState, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_AltMatch_PCRE, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_AltMatch_RE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\nvoid Search_AltMatch_CachedDFA(benchmark::State& state) { SearchAltMatch(state, SearchCachedDFA); }\nvoid Search_AltMatch_CachedNFA(benchmark::State& state) { SearchAltMatch(state, SearchCachedNFA); }\nvoid Search_AltMatch_CachedOnePass(benchmark::State& state) { SearchAltMatch(state, SearchCachedOnePass); }\nvoid Search_AltMatch_CachedBitState(benchmark::State& state) { SearchAltMatch(state, SearchCachedBitState); }\nvoid Search_AltMatch_CachedPCRE(benchmark::State& state) { SearchAltMatch(state, SearchCachedPCRE); }\nvoid Search_AltMatch_CachedRE2(benchmark::State& state) { SearchAltMatch(state, SearchCachedRE2); }\n\nBENCHMARK_RANGE(Search_AltMatch_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_AltMatch_CachedNFA, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_AltMatch_CachedOnePass, 8, 16<<20)->ThreadRange(1, NumCPUs());\nBENCHMARK_RANGE(Search_AltMatch_CachedBitState, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK_RANGE(Search_AltMatch_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(Search_AltMatch_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\n// Benchmark: use regexp to find phone number.\n\nvoid SearchDigits(benchmark::State& state, SearchImpl* search) {\n absl::string_view s(\"650-253-0001\");\n search(state, \"([0-9]+)-([0-9]+)-([0-9]+)\", s, Prog::kAnchored, true);\n state.SetItemsProcessed(state.iterations());\n}\n\nvoid Search_Digits_DFA(benchmark::State& state) { SearchDigits(state, SearchDFA); }\nvoid Search_Digits_NFA(benchmark::State& state) { SearchDigits(state, SearchNFA); }\nvoid Search_Digits_OnePass(benchmark::State& state) { SearchDigits(state, SearchOnePass); }\nvoid Search_Digits_PCRE(benchmark::State& state) { SearchDigits(state, SearchPCRE); }\nvoid Search_Digits_RE2(benchmark::State& state) { SearchDigits(state, SearchRE2); }\nvoid Search_Digits_BitState(benchmark::State& state) { SearchDigits(state, SearchBitState); }\n\nBENCHMARK(Search_Digits_DFA)->ThreadRange(1, NumCPUs());\nBENCHMARK(Search_Digits_NFA)->ThreadRange(1, NumCPUs());\nBENCHMARK(Search_Digits_OnePass)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK(Search_Digits_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Search_Digits_RE2)->ThreadRange(1, NumCPUs());\nBENCHMARK(Search_Digits_BitState)->ThreadRange(1, NumCPUs());\n\n// Benchmark: use regexp to parse digit fields in phone number.\n\nvoid Parse3Digits(benchmark::State& state,\n void (*parse3)(benchmark::State&, const char*,\n absl::string_view)) {\n parse3(state, \"([0-9]+)-([0-9]+)-([0-9]+)\", \"650-253-0001\");\n state.SetItemsProcessed(state.iterations());\n}\n\nvoid Parse_Digits_NFA(benchmark::State& state) { Parse3Digits(state, Parse3NFA); }\nvoid Parse_Digits_OnePass(benchmark::State& state) { Parse3Digits(state, Parse3OnePass); }\nvoid Parse_Digits_PCRE(benchmark::State& state) { Parse3Digits(state, Parse3PCRE); }\nvoid Parse_Digits_RE2(benchmark::State& state) { Parse3Digits(state, Parse3RE2); }\nvoid Parse_Digits_Backtrack(benchmark::State& state) { Parse3Digits(state, Parse3Backtrack); }\nvoid Parse_Digits_BitState(benchmark::State& state) { Parse3Digits(state, Parse3BitState); }\n\nBENCHMARK(Parse_Digits_NFA)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_Digits_OnePass)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK(Parse_Digits_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Parse_Digits_RE2)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_Digits_Backtrack)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_Digits_BitState)->ThreadRange(1, NumCPUs());\n\nvoid Parse_CachedDigits_NFA(benchmark::State& state) { Parse3Digits(state, Parse3CachedNFA); }\nvoid Parse_CachedDigits_OnePass(benchmark::State& state) { Parse3Digits(state, Parse3CachedOnePass); }\nvoid Parse_CachedDigits_PCRE(benchmark::State& state) { Parse3Digits(state, Parse3CachedPCRE); }\nvoid Parse_CachedDigits_RE2(benchmark::State& state) { Parse3Digits(state, Parse3CachedRE2); }\nvoid Parse_CachedDigits_Backtrack(benchmark::State& state) { Parse3Digits(state, Parse3CachedBacktrack); }\nvoid Parse_CachedDigits_BitState(benchmark::State& state) { Parse3Digits(state, Parse3CachedBitState); }\n\nBENCHMARK(Parse_CachedDigits_NFA)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedDigits_OnePass)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK(Parse_CachedDigits_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Parse_CachedDigits_Backtrack)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedDigits_RE2)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedDigits_BitState)->ThreadRange(1, NumCPUs());\n\nvoid Parse3DigitDs(benchmark::State& state,\n void (*parse3)(benchmark::State&, const char*,\n absl::string_view)) {\n parse3(state, \"(\\\\d+)-(\\\\d+)-(\\\\d+)\", \"650-253-0001\");\n state.SetItemsProcessed(state.iterations());\n}\n\nvoid Parse_DigitDs_NFA(benchmark::State& state) { Parse3DigitDs(state, Parse3NFA); }\nvoid Parse_DigitDs_OnePass(benchmark::State& state) { Parse3DigitDs(state, Parse3OnePass); }\nvoid Parse_DigitDs_PCRE(benchmark::State& state) { Parse3DigitDs(state, Parse3PCRE); }\nvoid Parse_DigitDs_RE2(benchmark::State& state) { Parse3DigitDs(state, Parse3RE2); }\nvoid Parse_DigitDs_Backtrack(benchmark::State& state) { Parse3DigitDs(state, Parse3CachedBacktrack); }\nvoid Parse_DigitDs_BitState(benchmark::State& state) { Parse3DigitDs(state, Parse3CachedBitState); }\n\nBENCHMARK(Parse_DigitDs_NFA)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_DigitDs_OnePass)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK(Parse_DigitDs_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Parse_DigitDs_RE2)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_DigitDs_Backtrack)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_DigitDs_BitState)->ThreadRange(1, NumCPUs());\n\nvoid Parse_CachedDigitDs_NFA(benchmark::State& state) { Parse3DigitDs(state, Parse3CachedNFA); }\nvoid Parse_CachedDigitDs_OnePass(benchmark::State& state) { Parse3DigitDs(state, Parse3CachedOnePass); }\nvoid Parse_CachedDigitDs_PCRE(benchmark::State& state) { Parse3DigitDs(state, Parse3CachedPCRE); }\nvoid Parse_CachedDigitDs_RE2(benchmark::State& state) { Parse3DigitDs(state, Parse3CachedRE2); }\nvoid Parse_CachedDigitDs_Backtrack(benchmark::State& state) { Parse3DigitDs(state, Parse3CachedBacktrack); }\nvoid Parse_CachedDigitDs_BitState(benchmark::State& state) { Parse3DigitDs(state, Parse3CachedBitState); }\n\nBENCHMARK(Parse_CachedDigitDs_NFA)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedDigitDs_OnePass)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK(Parse_CachedDigitDs_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Parse_CachedDigitDs_Backtrack)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedDigitDs_RE2)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedDigitDs_BitState)->ThreadRange(1, NumCPUs());\n\n// Benchmark: splitting off leading number field.\n\nvoid Parse1Split(benchmark::State& state,\n void (*parse1)(benchmark::State&, const char*,\n absl::string_view)) {\n parse1(state, \"[0-9]+-(.*)\", \"650-253-0001\");\n state.SetItemsProcessed(state.iterations());\n}\n\nvoid Parse_Split_NFA(benchmark::State& state) { Parse1Split(state, Parse1NFA); }\nvoid Parse_Split_OnePass(benchmark::State& state) { Parse1Split(state, Parse1OnePass); }\nvoid Parse_Split_PCRE(benchmark::State& state) { Parse1Split(state, Parse1PCRE); }\nvoid Parse_Split_RE2(benchmark::State& state) { Parse1Split(state, Parse1RE2); }\nvoid Parse_Split_BitState(benchmark::State& state) { Parse1Split(state, Parse1BitState); }\n\nBENCHMARK(Parse_Split_NFA)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_Split_OnePass)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK(Parse_Split_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Parse_Split_RE2)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_Split_BitState)->ThreadRange(1, NumCPUs());\n\nvoid Parse_CachedSplit_NFA(benchmark::State& state) { Parse1Split(state, Parse1CachedNFA); }\nvoid Parse_CachedSplit_OnePass(benchmark::State& state) { Parse1Split(state, Parse1CachedOnePass); }\nvoid Parse_CachedSplit_PCRE(benchmark::State& state) { Parse1Split(state, Parse1CachedPCRE); }\nvoid Parse_CachedSplit_RE2(benchmark::State& state) { Parse1Split(state, Parse1CachedRE2); }\nvoid Parse_CachedSplit_BitState(benchmark::State& state) { Parse1Split(state, Parse1CachedBitState); }\n\nBENCHMARK(Parse_CachedSplit_NFA)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedSplit_OnePass)->ThreadRange(1, NumCPUs());\n#ifdef USEPCRE\nBENCHMARK(Parse_CachedSplit_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Parse_CachedSplit_RE2)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedSplit_BitState)->ThreadRange(1, NumCPUs());\n\n// Benchmark: splitting off leading number field but harder (ambiguous regexp).\n\nvoid Parse1SplitHard(benchmark::State& state,\n void (*run)(benchmark::State&, const char*,\n absl::string_view)) {\n run(state, \"[0-9]+.(.*)\", \"650-253-0001\");\n state.SetItemsProcessed(state.iterations());\n}\n\nvoid Parse_SplitHard_NFA(benchmark::State& state) { Parse1SplitHard(state, Parse1NFA); }\nvoid Parse_SplitHard_PCRE(benchmark::State& state) { Parse1SplitHard(state, Parse1PCRE); }\nvoid Parse_SplitHard_RE2(benchmark::State& state) { Parse1SplitHard(state, Parse1RE2); }\nvoid Parse_SplitHard_BitState(benchmark::State& state) { Parse1SplitHard(state, Parse1BitState); }\n\n#ifdef USEPCRE\nBENCHMARK(Parse_SplitHard_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Parse_SplitHard_RE2)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_SplitHard_BitState)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_SplitHard_NFA)->ThreadRange(1, NumCPUs());\n\nvoid Parse_CachedSplitHard_NFA(benchmark::State& state) { Parse1SplitHard(state, Parse1CachedNFA); }\nvoid Parse_CachedSplitHard_PCRE(benchmark::State& state) { Parse1SplitHard(state, Parse1CachedPCRE); }\nvoid Parse_CachedSplitHard_RE2(benchmark::State& state) { Parse1SplitHard(state, Parse1CachedRE2); }\nvoid Parse_CachedSplitHard_BitState(benchmark::State& state) { Parse1SplitHard(state, Parse1CachedBitState); }\nvoid Parse_CachedSplitHard_Backtrack(benchmark::State& state) { Parse1SplitHard(state, Parse1CachedBacktrack); }\n\n#ifdef USEPCRE\nBENCHMARK(Parse_CachedSplitHard_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Parse_CachedSplitHard_RE2)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedSplitHard_BitState)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedSplitHard_NFA)->ThreadRange(1, NumCPUs());\nBENCHMARK(Parse_CachedSplitHard_Backtrack)->ThreadRange(1, NumCPUs());\n\n// Benchmark: Parse1SplitHard, big text, small match.\n\nvoid Parse1SplitBig1(benchmark::State& state,\n void (*run)(benchmark::State&, const char*,\n absl::string_view)) {\n std::string s;\n s.append(100000, 'x');\n s.append(\"650-253-0001\");\n run(state, \"[0-9]+.(.*)\", s);\n state.SetItemsProcessed(state.iterations());\n}\n\nvoid Parse_CachedSplitBig1_PCRE(benchmark::State& state) { Parse1SplitBig1(state, SearchParse1CachedPCRE); }\nvoid Parse_CachedSplitBig1_RE2(benchmark::State& state) { Parse1SplitBig1(state, SearchParse1CachedRE2); }\n\n#ifdef USEPCRE\nBENCHMARK(Parse_CachedSplitBig1_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Parse_CachedSplitBig1_RE2)->ThreadRange(1, NumCPUs());\n\n// Benchmark: Parse1SplitHard, big text, big match.\n\nvoid Parse1SplitBig2(benchmark::State& state,\n void (*run)(benchmark::State&, const char*,\n absl::string_view)) {\n std::string s;\n s.append(\"650-253-\");\n s.append(100000, '0');\n run(state, \"[0-9]+.(.*)\", s);\n state.SetItemsProcessed(state.iterations());\n}\n\nvoid Parse_CachedSplitBig2_PCRE(benchmark::State& state) { Parse1SplitBig2(state, SearchParse1CachedPCRE); }\nvoid Parse_CachedSplitBig2_RE2(benchmark::State& state) { Parse1SplitBig2(state, SearchParse1CachedRE2); }\n\n#ifdef USEPCRE\nBENCHMARK(Parse_CachedSplitBig2_PCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(Parse_CachedSplitBig2_RE2)->ThreadRange(1, NumCPUs());\n\n// Benchmark: measure time required to parse (but not execute)\n// a simple regular expression.\n\nvoid ParseRegexp(benchmark::State& state, const std::string& regexp) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n re->Decref();\n }\n}\n\nvoid SimplifyRegexp(benchmark::State& state, const std::string& regexp) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Regexp* sre = re->Simplify();\n ABSL_CHECK(sre);\n sre->Decref();\n re->Decref();\n }\n}\n\nvoid NullWalkRegexp(benchmark::State& state, const std::string& regexp) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n for (auto _ : state) {\n re->NullWalk();\n }\n re->Decref();\n}\n\nvoid SimplifyCompileRegexp(benchmark::State& state, const std::string& regexp) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Regexp* sre = re->Simplify();\n ABSL_CHECK(sre);\n Prog* prog = sre->CompileToProg(0);\n ABSL_CHECK(prog);\n delete prog;\n sre->Decref();\n re->Decref();\n }\n}\n\nvoid CompileRegexp(benchmark::State& state, const std::string& regexp) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n delete prog;\n re->Decref();\n }\n}\n\nvoid CompileToProg(benchmark::State& state, const std::string& regexp) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n for (auto _ : state) {\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n delete prog;\n }\n re->Decref();\n}\n\nvoid CompileByteMap(benchmark::State& state, const std::string& regexp) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n for (auto _ : state) {\n prog->ComputeByteMap();\n }\n delete prog;\n re->Decref();\n}\n\nvoid CompilePCRE(benchmark::State& state, const std::string& regexp) {\n for (auto _ : state) {\n PCRE re(regexp, PCRE::UTF8);\n ABSL_CHECK_EQ(re.error(), \"\");\n }\n}\n\nvoid CompileRE2(benchmark::State& state, const std::string& regexp) {\n for (auto _ : state) {\n RE2 re(regexp);\n ABSL_CHECK_EQ(re.error(), \"\");\n }\n}\n\nvoid RunBuild(benchmark::State& state, const std::string& regexp,\n void (*run)(benchmark::State&, const std::string&)) {\n run(state, regexp);\n state.SetItemsProcessed(state.iterations());\n}\n\n} // namespace re2\n\nABSL_FLAG(std::string, compile_regexp, \"(.*)-(\\\\d+)-of-(\\\\d+)\",\n \"regexp for compile benchmarks\");\n\nnamespace re2 {\n\nvoid BM_PCRE_Compile(benchmark::State& state) { RunBuild(state, absl::GetFlag(FLAGS_compile_regexp), CompilePCRE); }\nvoid BM_Regexp_Parse(benchmark::State& state) { RunBuild(state, absl::GetFlag(FLAGS_compile_regexp), ParseRegexp); }\nvoid BM_Regexp_Simplify(benchmark::State& state) { RunBuild(state, absl::GetFlag(FLAGS_compile_regexp), SimplifyRegexp); }\nvoid BM_CompileToProg(benchmark::State& state) { RunBuild(state, absl::GetFlag(FLAGS_compile_regexp), CompileToProg); }\nvoid BM_CompileByteMap(benchmark::State& state) { RunBuild(state, absl::GetFlag(FLAGS_compile_regexp), CompileByteMap); }\nvoid BM_Regexp_Compile(benchmark::State& state) { RunBuild(state, absl::GetFlag(FLAGS_compile_regexp), CompileRegexp); }\nvoid BM_Regexp_SimplifyCompile(benchmark::State& state) { RunBuild(state, absl::GetFlag(FLAGS_compile_regexp), SimplifyCompileRegexp); }\nvoid BM_Regexp_NullWalk(benchmark::State& state) { RunBuild(state, absl::GetFlag(FLAGS_compile_regexp), NullWalkRegexp); }\nvoid BM_RE2_Compile(benchmark::State& state) { RunBuild(state, absl::GetFlag(FLAGS_compile_regexp), CompileRE2); }\n\n#ifdef USEPCRE\nBENCHMARK(BM_PCRE_Compile)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(BM_Regexp_Parse)->ThreadRange(1, NumCPUs());\nBENCHMARK(BM_Regexp_Simplify)->ThreadRange(1, NumCPUs());\nBENCHMARK(BM_CompileToProg)->ThreadRange(1, NumCPUs());\nBENCHMARK(BM_CompileByteMap)->ThreadRange(1, NumCPUs());\nBENCHMARK(BM_Regexp_Compile)->ThreadRange(1, NumCPUs());\nBENCHMARK(BM_Regexp_SimplifyCompile)->ThreadRange(1, NumCPUs());\nBENCHMARK(BM_Regexp_NullWalk)->ThreadRange(1, NumCPUs());\nBENCHMARK(BM_RE2_Compile)->ThreadRange(1, NumCPUs());\n\n// Makes text of size nbytes, then calls run to search\n// the text for regexp iters times.\nvoid SearchPhone(benchmark::State& state, ParseImpl* search) {\n std::string s = RandomText(state.range(0));\n s.append(\"(650) 253-0001\");\n search(state, \"(\\\\d{3}-|\\\\(\\\\d{3}\\\\)\\\\s+)(\\\\d{3}-\\\\d{4})\", s);\n state.SetBytesProcessed(state.iterations() * state.range(0));\n}\n\nvoid SearchPhone_CachedPCRE(benchmark::State& state) {\n SearchPhone(state, SearchParse2CachedPCRE);\n}\n\nvoid SearchPhone_CachedRE2(benchmark::State& state) {\n SearchPhone(state, SearchParse2CachedRE2);\n}\n\n#ifdef USEPCRE\nBENCHMARK_RANGE(SearchPhone_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK_RANGE(SearchPhone_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs());\n\n/*\nTODO(rsc): Make this work again.\nvoid CacheFill(int iters, int n, SearchImpl *srch) {\n std::string s = DeBruijnString(n+1);\n std::string t;\n for (int i = n+1; i < 20; i++) {\n t = s + s;\n using std::swap;\n swap(s, t);\n }\n srch(iters, StringPrintf(\"0[01]{%d}$\", n).c_str(), s,\n Prog::kUnanchored, true);\n SetBenchmarkBytesProcessed(static_cast(iters)*s.size());\n}\n\nvoid CacheFillPCRE(int i, int n) { CacheFill(i, n, SearchCachedPCRE); }\nvoid CacheFillRE2(int i, int n) { CacheFill(i, n, SearchCachedRE2); }\nvoid CacheFillNFA(int i, int n) { CacheFill(i, n, SearchCachedNFA); }\nvoid CacheFillDFA(int i, int n) { CacheFill(i, n, SearchCachedDFA); }\n\n// BENCHMARK_WITH_ARG uses __LINE__ to generate distinct identifiers\n// for the static BenchmarkRegisterer, which makes it unusable inside\n// a macro like DO24 below. MY_BENCHMARK_WITH_ARG uses the argument a\n// to make the identifiers distinct (only possible when 'a' is a simple\n// expression like 2, not like 1+1).\n#define MY_BENCHMARK_WITH_ARG(n, a) \\\n bool __benchmark_ ## n ## a = \\\n (new ::testing::Benchmark(#n, NewPermanentCallback(&n)))->ThreadRange(1, NumCPUs());\n\n#define DO24(A, B) \\\n A(B, 1); A(B, 2); A(B, 3); A(B, 4); A(B, 5); A(B, 6); \\\n A(B, 7); A(B, 8); A(B, 9); A(B, 10); A(B, 11); A(B, 12); \\\n A(B, 13); A(B, 14); A(B, 15); A(B, 16); A(B, 17); A(B, 18); \\\n A(B, 19); A(B, 20); A(B, 21); A(B, 22); A(B, 23); A(B, 24);\n\nDO24(MY_BENCHMARK_WITH_ARG, CacheFillPCRE)\nDO24(MY_BENCHMARK_WITH_ARG, CacheFillNFA)\nDO24(MY_BENCHMARK_WITH_ARG, CacheFillRE2)\nDO24(MY_BENCHMARK_WITH_ARG, CacheFillDFA)\n\n#undef DO24\n#undef MY_BENCHMARK_WITH_ARG\n*/\n\n////////////////////////////////////////////////////////////////////////\n//\n// Implementation routines. Sad that there are so many,\n// but all the interfaces are slightly different.\n\n// Runs implementation to search for regexp in text, iters times.\n// Expect_match says whether the regexp should be found.\n// Anchored says whether to run an anchored search.\n\nvoid SearchDFA(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n bool failed = false;\n ABSL_CHECK_EQ(prog->SearchDFA(text, absl::string_view(), anchor,\n Prog::kFirstMatch, NULL, &failed, NULL),\n expect_match);\n ABSL_CHECK(!failed);\n delete prog;\n re->Decref();\n }\n}\n\nvoid SearchNFA(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n ABSL_CHECK_EQ(prog->SearchNFA(text, absl::string_view(), anchor,\n Prog::kFirstMatch, NULL, 0),\n expect_match);\n delete prog;\n re->Decref();\n }\n}\n\nvoid SearchOnePass(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n ABSL_CHECK(prog->IsOnePass());\n ABSL_CHECK_EQ(\n prog->SearchOnePass(text, text, anchor, Prog::kFirstMatch, NULL, 0),\n expect_match);\n delete prog;\n re->Decref();\n }\n}\n\nvoid SearchBitState(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n ABSL_CHECK(prog->CanBitState());\n ABSL_CHECK_EQ(\n prog->SearchBitState(text, text, anchor, Prog::kFirstMatch, NULL, 0),\n expect_match);\n delete prog;\n re->Decref();\n }\n}\n\nvoid SearchPCRE(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n for (auto _ : state) {\n PCRE re(regexp, PCRE::UTF8);\n ABSL_CHECK_EQ(re.error(), \"\");\n if (anchor == Prog::kAnchored) {\n ABSL_CHECK_EQ(PCRE::FullMatch(text, re), expect_match);\n } else {\n ABSL_CHECK_EQ(PCRE::PartialMatch(text, re), expect_match);\n }\n }\n}\n\nvoid SearchRE2(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n for (auto _ : state) {\n RE2 re(regexp);\n ABSL_CHECK_EQ(re.error(), \"\");\n if (anchor == Prog::kAnchored) {\n ABSL_CHECK_EQ(RE2::FullMatch(text, re), expect_match);\n } else {\n ABSL_CHECK_EQ(RE2::PartialMatch(text, re), expect_match);\n }\n }\n}\n\n// SearchCachedXXX is like SearchXXX but only does the\n// regexp parsing and compiling once. This lets us measure\n// search time without the per-regexp overhead.\n\nProg* GetCachedProg(const char* regexp) {\n static auto& mutex = *new absl::Mutex;\n absl::MutexLock lock(mutex);\n static auto& cache = *new absl::flat_hash_map;\n Prog* prog = cache[regexp];\n if (prog == NULL) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n prog = re->CompileToProg(int64_t{1}<<31); // mostly for the DFA\n ABSL_CHECK(prog);\n cache[regexp] = prog;\n re->Decref();\n // We must call this here - while we have exclusive access.\n prog->IsOnePass();\n }\n return prog;\n}\n\nPCRE* GetCachedPCRE(const char* regexp) {\n static auto& mutex = *new absl::Mutex;\n absl::MutexLock lock(mutex);\n static auto& cache = *new absl::flat_hash_map;\n PCRE* re = cache[regexp];\n if (re == NULL) {\n re = new PCRE(regexp, PCRE::UTF8);\n ABSL_CHECK_EQ(re->error(), \"\");\n cache[regexp] = re;\n }\n return re;\n}\n\nRE2* GetCachedRE2(const char* regexp) {\n static auto& mutex = *new absl::Mutex;\n absl::MutexLock lock(mutex);\n static auto& cache = *new absl::flat_hash_map;\n RE2* re = cache[regexp];\n if (re == NULL) {\n re = new RE2(regexp);\n ABSL_CHECK_EQ(re->error(), \"\");\n cache[regexp] = re;\n }\n return re;\n}\n\nvoid SearchCachedDFA(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n Prog* prog = GetCachedProg(regexp);\n for (auto _ : state) {\n bool failed = false;\n ABSL_CHECK_EQ(prog->SearchDFA(text, absl::string_view(), anchor,\n Prog::kFirstMatch, NULL, &failed, NULL),\n expect_match);\n ABSL_CHECK(!failed);\n }\n}\n\nvoid SearchCachedNFA(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n Prog* prog = GetCachedProg(regexp);\n for (auto _ : state) {\n ABSL_CHECK_EQ(prog->SearchNFA(text, absl::string_view(), anchor,\n Prog::kFirstMatch, NULL, 0),\n expect_match);\n }\n}\n\nvoid SearchCachedOnePass(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n Prog* prog = GetCachedProg(regexp);\n ABSL_CHECK(prog->IsOnePass());\n for (auto _ : state) {\n ABSL_CHECK_EQ(\n prog->SearchOnePass(text, text, anchor, Prog::kFirstMatch, NULL, 0),\n expect_match);\n }\n}\n\nvoid SearchCachedBitState(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n Prog* prog = GetCachedProg(regexp);\n ABSL_CHECK(prog->CanBitState());\n for (auto _ : state) {\n ABSL_CHECK_EQ(\n prog->SearchBitState(text, text, anchor, Prog::kFirstMatch, NULL, 0),\n expect_match);\n }\n}\n\nvoid SearchCachedPCRE(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n PCRE& re = *GetCachedPCRE(regexp);\n for (auto _ : state) {\n if (anchor == Prog::kAnchored) {\n ABSL_CHECK_EQ(PCRE::FullMatch(text, re), expect_match);\n } else {\n ABSL_CHECK_EQ(PCRE::PartialMatch(text, re), expect_match);\n }\n }\n}\n\nvoid SearchCachedRE2(benchmark::State& state, const char* regexp,\n absl::string_view text, Prog::Anchor anchor,\n bool expect_match) {\n RE2& re = *GetCachedRE2(regexp);\n for (auto _ : state) {\n if (anchor == Prog::kAnchored) {\n ABSL_CHECK_EQ(RE2::FullMatch(text, re), expect_match);\n } else {\n ABSL_CHECK_EQ(RE2::PartialMatch(text, re), expect_match);\n }\n }\n}\n\n// Runs implementation to full match regexp against text,\n// extracting three submatches. Expects match always.\n\nvoid Parse3NFA(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n absl::string_view sp[4]; // 4 because sp[0] is whole match.\n ABSL_CHECK(prog->SearchNFA(text, absl::string_view(), Prog::kAnchored,\n Prog::kFullMatch, sp, 4));\n delete prog;\n re->Decref();\n }\n}\n\nvoid Parse3OnePass(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n ABSL_CHECK(prog->IsOnePass());\n absl::string_view sp[4]; // 4 because sp[0] is whole match.\n ABSL_CHECK(prog->SearchOnePass(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 4));\n delete prog;\n re->Decref();\n }\n}\n\nvoid Parse3BitState(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n ABSL_CHECK(prog->CanBitState());\n absl::string_view sp[4]; // 4 because sp[0] is whole match.\n ABSL_CHECK(prog->SearchBitState(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 4));\n delete prog;\n re->Decref();\n }\n}\n\nvoid Parse3Backtrack(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n absl::string_view sp[4]; // 4 because sp[0] is whole match.\n ABSL_CHECK(prog->UnsafeSearchBacktrack(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 4));\n delete prog;\n re->Decref();\n }\n}\n\nvoid Parse3PCRE(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n PCRE re(regexp, PCRE::UTF8);\n ABSL_CHECK_EQ(re.error(), \"\");\n absl::string_view sp1, sp2, sp3;\n ABSL_CHECK(PCRE::FullMatch(text, re, &sp1, &sp2, &sp3));\n }\n}\n\nvoid Parse3RE2(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n RE2 re(regexp);\n ABSL_CHECK_EQ(re.error(), \"\");\n absl::string_view sp1, sp2, sp3;\n ABSL_CHECK(RE2::FullMatch(text, re, &sp1, &sp2, &sp3));\n }\n}\n\nvoid Parse3CachedNFA(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n Prog* prog = GetCachedProg(regexp);\n absl::string_view sp[4]; // 4 because sp[0] is whole match.\n for (auto _ : state) {\n ABSL_CHECK(prog->SearchNFA(text, absl::string_view(), Prog::kAnchored,\n Prog::kFullMatch, sp, 4));\n }\n}\n\nvoid Parse3CachedOnePass(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n Prog* prog = GetCachedProg(regexp);\n ABSL_CHECK(prog->IsOnePass());\n absl::string_view sp[4]; // 4 because sp[0] is whole match.\n for (auto _ : state) {\n ABSL_CHECK(prog->SearchOnePass(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 4));\n }\n}\n\nvoid Parse3CachedBitState(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n Prog* prog = GetCachedProg(regexp);\n ABSL_CHECK(prog->CanBitState());\n absl::string_view sp[4]; // 4 because sp[0] is whole match.\n for (auto _ : state) {\n ABSL_CHECK(prog->SearchBitState(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 4));\n }\n}\n\nvoid Parse3CachedBacktrack(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n Prog* prog = GetCachedProg(regexp);\n absl::string_view sp[4]; // 4 because sp[0] is whole match.\n for (auto _ : state) {\n ABSL_CHECK(prog->UnsafeSearchBacktrack(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 4));\n }\n}\n\nvoid Parse3CachedPCRE(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n PCRE& re = *GetCachedPCRE(regexp);\n absl::string_view sp1, sp2, sp3;\n for (auto _ : state) {\n ABSL_CHECK(PCRE::FullMatch(text, re, &sp1, &sp2, &sp3));\n }\n}\n\nvoid Parse3CachedRE2(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n RE2& re = *GetCachedRE2(regexp);\n absl::string_view sp1, sp2, sp3;\n for (auto _ : state) {\n ABSL_CHECK(RE2::FullMatch(text, re, &sp1, &sp2, &sp3));\n }\n}\n\n// Runs implementation to full match regexp against text,\n// extracting three submatches. Expects match always.\n\nvoid Parse1NFA(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n absl::string_view sp[2]; // 2 because sp[0] is whole match.\n ABSL_CHECK(prog->SearchNFA(text, absl::string_view(), Prog::kAnchored,\n Prog::kFullMatch, sp, 2));\n delete prog;\n re->Decref();\n }\n}\n\nvoid Parse1OnePass(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n ABSL_CHECK(prog->IsOnePass());\n absl::string_view sp[2]; // 2 because sp[0] is whole match.\n ABSL_CHECK(prog->SearchOnePass(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 2));\n delete prog;\n re->Decref();\n }\n}\n\nvoid Parse1BitState(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL);\n ABSL_CHECK(re);\n Prog* prog = re->CompileToProg(0);\n ABSL_CHECK(prog);\n ABSL_CHECK(prog->CanBitState());\n absl::string_view sp[2]; // 2 because sp[0] is whole match.\n ABSL_CHECK(prog->SearchBitState(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 2));\n delete prog;\n re->Decref();\n }\n}\n\nvoid Parse1PCRE(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n PCRE re(regexp, PCRE::UTF8);\n ABSL_CHECK_EQ(re.error(), \"\");\n absl::string_view sp1;\n ABSL_CHECK(PCRE::FullMatch(text, re, &sp1));\n }\n}\n\nvoid Parse1RE2(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n for (auto _ : state) {\n RE2 re(regexp);\n ABSL_CHECK_EQ(re.error(), \"\");\n absl::string_view sp1;\n ABSL_CHECK(RE2::FullMatch(text, re, &sp1));\n }\n}\n\nvoid Parse1CachedNFA(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n Prog* prog = GetCachedProg(regexp);\n absl::string_view sp[2]; // 2 because sp[0] is whole match.\n for (auto _ : state) {\n ABSL_CHECK(prog->SearchNFA(text, absl::string_view(), Prog::kAnchored,\n Prog::kFullMatch, sp, 2));\n }\n}\n\nvoid Parse1CachedOnePass(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n Prog* prog = GetCachedProg(regexp);\n ABSL_CHECK(prog->IsOnePass());\n absl::string_view sp[2]; // 2 because sp[0] is whole match.\n for (auto _ : state) {\n ABSL_CHECK(prog->SearchOnePass(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 2));\n }\n}\n\nvoid Parse1CachedBitState(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n Prog* prog = GetCachedProg(regexp);\n ABSL_CHECK(prog->CanBitState());\n absl::string_view sp[2]; // 2 because sp[0] is whole match.\n for (auto _ : state) {\n ABSL_CHECK(prog->SearchBitState(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 2));\n }\n}\n\nvoid Parse1CachedBacktrack(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n Prog* prog = GetCachedProg(regexp);\n absl::string_view sp[2]; // 2 because sp[0] is whole match.\n for (auto _ : state) {\n ABSL_CHECK(prog->UnsafeSearchBacktrack(text, text, Prog::kAnchored,\n Prog::kFullMatch, sp, 2));\n }\n}\n\nvoid Parse1CachedPCRE(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n PCRE& re = *GetCachedPCRE(regexp);\n absl::string_view sp1;\n for (auto _ : state) {\n ABSL_CHECK(PCRE::FullMatch(text, re, &sp1));\n }\n}\n\nvoid Parse1CachedRE2(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n RE2& re = *GetCachedRE2(regexp);\n absl::string_view sp1;\n for (auto _ : state) {\n ABSL_CHECK(RE2::FullMatch(text, re, &sp1));\n }\n}\n\nvoid SearchParse2CachedPCRE(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n PCRE& re = *GetCachedPCRE(regexp);\n for (auto _ : state) {\n absl::string_view sp1, sp2;\n ABSL_CHECK(PCRE::PartialMatch(text, re, &sp1, &sp2));\n }\n}\n\nvoid SearchParse2CachedRE2(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n RE2& re = *GetCachedRE2(regexp);\n for (auto _ : state) {\n absl::string_view sp1, sp2;\n ABSL_CHECK(RE2::PartialMatch(text, re, &sp1, &sp2));\n }\n}\n\nvoid SearchParse1CachedPCRE(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n PCRE& re = *GetCachedPCRE(regexp);\n for (auto _ : state) {\n absl::string_view sp1;\n ABSL_CHECK(PCRE::PartialMatch(text, re, &sp1));\n }\n}\n\nvoid SearchParse1CachedRE2(benchmark::State& state, const char* regexp,\n absl::string_view text) {\n RE2& re = *GetCachedRE2(regexp);\n for (auto _ : state) {\n absl::string_view sp1;\n ABSL_CHECK(RE2::PartialMatch(text, re, &sp1));\n }\n}\n\nvoid EmptyPartialMatchPCRE(benchmark::State& state) {\n PCRE re(\"\");\n for (auto _ : state) {\n PCRE::PartialMatch(\"\", re);\n }\n}\n\nvoid EmptyPartialMatchRE2(benchmark::State& state) {\n RE2 re(\"\");\n for (auto _ : state) {\n RE2::PartialMatch(\"\", re);\n }\n}\n#ifdef USEPCRE\nBENCHMARK(EmptyPartialMatchPCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(EmptyPartialMatchRE2)->ThreadRange(1, NumCPUs());\n\nvoid SimplePartialMatchPCRE(benchmark::State& state) {\n PCRE re(\"abcdefg\");\n for (auto _ : state) {\n PCRE::PartialMatch(\"abcdefg\", re);\n }\n}\n\nvoid SimplePartialMatchRE2(benchmark::State& state) {\n RE2 re(\"abcdefg\");\n for (auto _ : state) {\n RE2::PartialMatch(\"abcdefg\", re);\n }\n}\n#ifdef USEPCRE\nBENCHMARK(SimplePartialMatchPCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(SimplePartialMatchRE2)->ThreadRange(1, NumCPUs());\n\nstatic std::string http_text =\n \"GET /asdfhjasdhfasdlfhasdflkjasdfkljasdhflaskdjhf\"\n \"alksdjfhasdlkfhasdlkjfhasdljkfhadsjklf HTTP/1.1\";\n\nvoid HTTPPartialMatchPCRE(benchmark::State& state) {\n absl::string_view a;\n PCRE re(\"(?-s)^(?:GET|POST) +([^ ]+) HTTP\");\n for (auto _ : state) {\n PCRE::PartialMatch(http_text, re, &a);\n }\n}\n\nvoid HTTPPartialMatchRE2(benchmark::State& state) {\n absl::string_view a;\n RE2 re(\"(?-s)^(?:GET|POST) +([^ ]+) HTTP\");\n for (auto _ : state) {\n RE2::PartialMatch(http_text, re, &a);\n }\n}\n\n#ifdef USEPCRE\nBENCHMARK(HTTPPartialMatchPCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(HTTPPartialMatchRE2)->ThreadRange(1, NumCPUs());\n\nstatic std::string smallhttp_text =\n \"GET /abc HTTP/1.1\";\n\nvoid SmallHTTPPartialMatchPCRE(benchmark::State& state) {\n absl::string_view a;\n PCRE re(\"(?-s)^(?:GET|POST) +([^ ]+) HTTP\");\n for (auto _ : state) {\n PCRE::PartialMatch(smallhttp_text, re, &a);\n }\n}\n\nvoid SmallHTTPPartialMatchRE2(benchmark::State& state) {\n absl::string_view a;\n RE2 re(\"(?-s)^(?:GET|POST) +([^ ]+) HTTP\");\n for (auto _ : state) {\n RE2::PartialMatch(smallhttp_text, re, &a);\n }\n}\n\n#ifdef USEPCRE\nBENCHMARK(SmallHTTPPartialMatchPCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(SmallHTTPPartialMatchRE2)->ThreadRange(1, NumCPUs());\n\nvoid DotMatchPCRE(benchmark::State& state) {\n absl::string_view a;\n PCRE re(\"(?-s)^(.+)\");\n for (auto _ : state) {\n PCRE::PartialMatch(http_text, re, &a);\n }\n}\n\nvoid DotMatchRE2(benchmark::State& state) {\n absl::string_view a;\n RE2 re(\"(?-s)^(.+)\");\n for (auto _ : state) {\n RE2::PartialMatch(http_text, re, &a);\n }\n}\n\n#ifdef USEPCRE\nBENCHMARK(DotMatchPCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(DotMatchRE2)->ThreadRange(1, NumCPUs());\n\nvoid ASCIIMatchPCRE(benchmark::State& state) {\n absl::string_view a;\n PCRE re(\"(?-s)^([ -~]+)\");\n for (auto _ : state) {\n PCRE::PartialMatch(http_text, re, &a);\n }\n}\n\nvoid ASCIIMatchRE2(benchmark::State& state) {\n absl::string_view a;\n RE2 re(\"(?-s)^([ -~]+)\");\n for (auto _ : state) {\n RE2::PartialMatch(http_text, re, &a);\n }\n}\n\n#ifdef USEPCRE\nBENCHMARK(ASCIIMatchPCRE)->ThreadRange(1, NumCPUs());\n#endif\nBENCHMARK(ASCIIMatchRE2)->ThreadRange(1, NumCPUs());\n\nvoid FullMatchPCRE(benchmark::State& state, const char *regexp) {\n std::string s = RandomText(state.range(0));\n s += \"ABCDEFGHIJ\";\n PCRE re(regexp);\n for (auto _ : state) {\n ABSL_CHECK(PCRE::FullMatch(s, re));\n }\n state.SetBytesProcessed(state.iterations() * state.range(0));\n}\n\nvoid FullMatchRE2(benchmark::State& state, const char *regexp) {\n std::string s = RandomText(state.range(0));\n s += \"ABCDEFGHIJ\";\n RE2 re(regexp, RE2::Latin1);\n for (auto _ : state) {\n ABSL_CHECK(RE2::FullMatch(s, re));\n }\n state.SetBytesProcessed(state.iterations() * state.range(0));\n}\n\nvoid FullMatch_DotStar_CachedPCRE(benchmark::State& state) {\n FullMatchPCRE(state, \"(?s).*\");\n}\nvoid FullMatch_DotStar_CachedRE2(benchmark::State& state) {\n FullMatchRE2(state, \"(?s).*\");\n}\n\nvoid FullMatch_DotStarDollar_CachedPCRE(benchmark::State& state) {\n FullMatchPCRE(state, \"(?s).*$\");\n}\nvoid FullMatch_DotStarDollar_CachedRE2(benchmark::State& state) {\n FullMatchRE2(state, \"(?s).*$\");\n}\n\nvoid FullMatch_DotStarCapture_CachedPCRE(benchmark::State& state) {\n FullMatchPCRE(state, \"(?s)((.*)()()($))\");\n}\nvoid FullMatch_DotStarCapture_CachedRE2(benchmark::State& state) {\n FullMatchRE2(state, \"(?s)((.*)()()($))\");\n}\n\n#ifdef USEPCRE\nBENCHMARK_RANGE(FullMatch_DotStar_CachedPCRE, 8, 2<<20);\n#endif\nBENCHMARK_RANGE(FullMatch_DotStar_CachedRE2, 8, 2<<20);\n\n#ifdef USEPCRE\nBENCHMARK_RANGE(FullMatch_DotStarDollar_CachedPCRE, 8, 2<<20);\n#endif\nBENCHMARK_RANGE(FullMatch_DotStarDollar_CachedRE2, 8, 2<<20);\n\n#ifdef USEPCRE\nBENCHMARK_RANGE(FullMatch_DotStarCapture_CachedPCRE, 8, 2<<20);\n#endif\nBENCHMARK_RANGE(FullMatch_DotStarCapture_CachedRE2, 8, 2<<20);\n\nvoid PossibleMatchRangeCommon(benchmark::State& state, const char* regexp) {\n RE2 re(regexp);\n std::string min;\n std::string max;\n const int kMaxLen = 16;\n for (auto _ : state) {\n ABSL_CHECK(re.PossibleMatchRange(&min, &max, kMaxLen));\n }\n}\n\nvoid PossibleMatchRange_Trivial(benchmark::State& state) {\n PossibleMatchRangeCommon(state, \".*\");\n}\nvoid PossibleMatchRange_Complex(benchmark::State& state) {\n PossibleMatchRangeCommon(state, \"^abc[def]?[gh]{1,2}.*\");\n}\nvoid PossibleMatchRange_Prefix(benchmark::State& state) {\n PossibleMatchRangeCommon(state, \"^some_random_prefix.*\");\n}\nvoid PossibleMatchRange_NoProg(benchmark::State& state) {\n PossibleMatchRangeCommon(state, \"^some_random_string$\");\n}\n\nBENCHMARK(PossibleMatchRange_Trivial);\nBENCHMARK(PossibleMatchRange_Complex);\nBENCHMARK(PossibleMatchRange_Prefix);\nBENCHMARK(PossibleMatchRange_NoProg);\n\n} // namespace re2\n" }, { "path": "re2/testing/regexp_generator.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Regular expression generator: generates all possible\n// regular expressions within parameters (see regexp_generator.h for details).\n\n// The regexp generator first generates a sequence of commands in a simple\n// postfix language. Each command in the language is a string,\n// like \"a\" or \"%s*\" or \"%s|%s\".\n//\n// To evaluate a command, enough arguments are popped from the value stack to\n// plug into the %s slots. Then the result is pushed onto the stack.\n// For example, the command sequence\n// a b %s%s c\n// results in the stack\n// ab c\n//\n// GeneratePostfix generates all possible command sequences.\n// Then RunPostfix turns each sequence into a regular expression\n// and passes the regexp to HandleRegexp.\n\n#include \"re2/testing/regexp_generator.h\"\n\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/escaping.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\n// Returns a vector of the egrep regexp operators.\nconst std::vector& RegexpGenerator::EgrepOps() {\n static const char *ops[] = {\n \"%s%s\",\n \"%s|%s\",\n \"%s*\",\n \"%s+\",\n \"%s?\",\n \"%s\\\\C*\",\n };\n static std::vector v(ops, ops + ABSL_ARRAYSIZE(ops));\n return v;\n}\n\nRegexpGenerator::RegexpGenerator(int maxatoms, int maxops,\n const std::vector& atoms,\n const std::vector& ops)\n : maxatoms_(maxatoms), maxops_(maxops), atoms_(atoms), ops_(ops) {\n // Degenerate case.\n if (atoms_.empty())\n maxatoms_ = 0;\n if (ops_.empty())\n maxops_ = 0;\n}\n\n// Generates all possible regular expressions (within the parameters),\n// calling HandleRegexp for each one.\nvoid RegexpGenerator::Generate() {\n std::vector postfix;\n GeneratePostfix(&postfix, 0, 0, 0);\n}\n\n// Generates random regular expressions, calling HandleRegexp for each one.\nvoid RegexpGenerator::GenerateRandom(int32_t seed, int n) {\n rng_.seed(seed);\n\n for (int i = 0; i < n; i++) {\n std::vector postfix;\n GenerateRandomPostfix(&postfix, 0, 0, 0);\n }\n}\n\n// Counts and returns the number of occurrences of \"%s\" in s.\nstatic int CountArgs(const std::string& s) {\n const char *p = s.c_str();\n int n = 0;\n while ((p = strstr(p, \"%s\")) != NULL) {\n p += 2;\n n++;\n }\n return n;\n}\n\n// Generates all possible postfix command sequences.\n// Each sequence is handed off to RunPostfix to generate a regular expression.\n// The arguments are:\n// post: the current postfix sequence\n// nstk: the number of elements that would be on the stack after executing\n// the sequence\n// ops: the number of operators used in the sequence\n// atoms: the number of atoms used in the sequence\n// For example, if post were [\"a\", \"b\", \"%s%s\", \"c\"],\n// then nstk = 2, ops = 1, atoms = 3.\n//\n// The initial call should be GeneratePostfix([empty vector], 0, 0, 0).\n//\nvoid RegexpGenerator::GeneratePostfix(std::vector* post,\n int nstk, int ops, int atoms) {\n if (nstk == 1)\n RunPostfix(*post);\n\n // Early out: if used too many operators or can't\n // get back down to a single expression on the stack\n // using binary operators, give up.\n if (ops + nstk - 1 > maxops_)\n return;\n\n // Add atoms if there is room.\n if (atoms < maxatoms_) {\n for (size_t i = 0; i < atoms_.size(); i++) {\n post->push_back(atoms_[i]);\n GeneratePostfix(post, nstk + 1, ops, atoms + 1);\n post->pop_back();\n }\n }\n\n // Add operators if there are enough arguments.\n if (ops < maxops_) {\n for (size_t i = 0; i < ops_.size(); i++) {\n const std::string& fmt = ops_[i];\n int nargs = CountArgs(fmt);\n if (nargs <= nstk) {\n post->push_back(fmt);\n GeneratePostfix(post, nstk - nargs + 1, ops + 1, atoms);\n post->pop_back();\n }\n }\n }\n}\n\n// Generates a random postfix command sequence.\n// Stops and returns true once a single sequence has been generated.\nbool RegexpGenerator::GenerateRandomPostfix(std::vector* post,\n int nstk, int ops, int atoms) {\n std::uniform_int_distribution random_stop(0, maxatoms_ - atoms);\n std::uniform_int_distribution random_bit(0, 1);\n std::uniform_int_distribution random_ops_index(\n 0, static_cast(ops_.size()) - 1);\n std::uniform_int_distribution random_atoms_index(\n 0, static_cast(atoms_.size()) - 1);\n\n for (;;) {\n // Stop if we get to a single element, but only sometimes.\n if (nstk == 1 && random_stop(rng_) == 0) {\n RunPostfix(*post);\n return true;\n }\n\n // Early out: if used too many operators or can't\n // get back down to a single expression on the stack\n // using binary operators, give up.\n if (ops + nstk - 1 > maxops_)\n return false;\n\n // Add operators if there are enough arguments.\n if (ops < maxops_ && random_bit(rng_) == 0) {\n const std::string& fmt = ops_[random_ops_index(rng_)];\n int nargs = CountArgs(fmt);\n if (nargs <= nstk) {\n post->push_back(fmt);\n bool ret = GenerateRandomPostfix(post, nstk - nargs + 1,\n ops + 1, atoms);\n post->pop_back();\n if (ret)\n return true;\n }\n }\n\n // Add atoms if there is room.\n if (atoms < maxatoms_ && random_bit(rng_) == 0) {\n post->push_back(atoms_[random_atoms_index(rng_)]);\n bool ret = GenerateRandomPostfix(post, nstk + 1, ops, atoms + 1);\n post->pop_back();\n if (ret)\n return true;\n }\n }\n}\n\n// Interprets the postfix command sequence to create a regular expression\n// passed to HandleRegexp. The results of operators like %s|%s are wrapped\n// in (?: ) to avoid needing to maintain a precedence table.\nvoid RegexpGenerator::RunPostfix(const std::vector& post) {\n std::stack regexps;\n for (size_t i = 0; i < post.size(); i++) {\n switch (CountArgs(post[i])) {\n default:\n ABSL_LOG(FATAL) << \"Bad operator: \" << post[i];\n case 0:\n regexps.push(post[i]);\n break;\n case 1: {\n auto fmt = absl::ParsedFormat<'s'>::New(post[i]);\n ABSL_CHECK(fmt != nullptr);\n std::string a = regexps.top();\n regexps.pop();\n regexps.push(\"(?:\" + absl::StrFormat(*fmt, a) + \")\");\n break;\n }\n case 2: {\n auto fmt = absl::ParsedFormat<'s', 's'>::New(post[i]);\n ABSL_CHECK(fmt != nullptr);\n std::string b = regexps.top();\n regexps.pop();\n std::string a = regexps.top();\n regexps.pop();\n regexps.push(\"(?:\" + absl::StrFormat(*fmt, a, b) + \")\");\n break;\n }\n }\n }\n\n if (regexps.size() != 1) {\n // Internal error - should never happen.\n absl::PrintF(\"Bad regexp program:\\n\");\n for (size_t i = 0; i < post.size(); i++) {\n absl::PrintF(\" %s\\n\", absl::CEscape(post[i]));\n }\n absl::PrintF(\"Stack after running program:\\n\");\n while (!regexps.empty()) {\n absl::PrintF(\" %s\\n\", absl::CEscape(regexps.top()));\n regexps.pop();\n }\n ABSL_LOG(FATAL) << \"Bad regexp program.\";\n }\n\n HandleRegexp(regexps.top());\n HandleRegexp(\"^(?:\" + regexps.top() + \")$\");\n HandleRegexp(\"^(?:\" + regexps.top() + \")\");\n HandleRegexp(\"(?:\" + regexps.top() + \")$\");\n}\n\n// Split s into an vector of strings, one for each UTF-8 character.\nstd::vector Explode(absl::string_view s) {\n std::vector v;\n\n for (const char *q = s.data(); q < s.data() + s.size(); ) {\n const char* p = q;\n Rune r;\n q += chartorune(&r, q);\n v.push_back(std::string(p, q - p));\n }\n\n return v;\n}\n\n// Split string everywhere a substring is found, returning\n// vector of pieces.\nstd::vector Split(absl::string_view sep, absl::string_view s) {\n std::vector v;\n\n if (sep.empty())\n return Explode(s);\n\n const char *p = s.data();\n for (const char *q = s.data(); q + sep.size() <= s.data() + s.size(); q++) {\n if (absl::string_view(q, sep.size()) == sep) {\n v.push_back(std::string(p, q - p));\n p = q + sep.size();\n q = p - 1; // -1 for ++ in loop\n continue;\n }\n }\n if (p < s.data() + s.size())\n v.push_back(std::string(p, s.data() + s.size() - p));\n return v;\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/regexp_generator.h", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_TESTING_REGEXP_GENERATOR_H_\n#define RE2_TESTING_REGEXP_GENERATOR_H_\n\n// Regular expression generator: generates all possible\n// regular expressions within given parameters (see below for details).\n\n#include \n\n#include \n#include \n#include \n\n#include \"absl/strings/string_view.h\"\n\nnamespace re2 {\n\n// Regular expression generator.\n//\n// Given a set of atom expressions like \"a\", \"b\", or \".\"\n// and operators like \"%s*\", generates all possible regular expressions\n// using at most maxbases base expressions and maxops operators.\n// For each such expression re, calls HandleRegexp(re).\n//\n// Callers are expected to subclass RegexpGenerator and provide HandleRegexp.\n//\nclass RegexpGenerator {\n public:\n RegexpGenerator(int maxatoms, int maxops,\n const std::vector& atoms,\n const std::vector& ops);\n virtual ~RegexpGenerator() {}\n\n // Generates all the regular expressions, calling HandleRegexp(re) for each.\n void Generate();\n\n // Generates n random regular expressions, calling HandleRegexp(re) for each.\n void GenerateRandom(int32_t seed, int n);\n\n // Handles a regular expression. Must be provided by subclass.\n virtual void HandleRegexp(const std::string& regexp) = 0;\n\n // The egrep regexp operators: * + ? | and concatenation.\n static const std::vector& EgrepOps();\n\n private:\n void RunPostfix(const std::vector& post);\n void GeneratePostfix(std::vector* post,\n int nstk, int ops, int lits);\n bool GenerateRandomPostfix(std::vector* post,\n int nstk, int ops, int lits);\n\n int maxatoms_; // Maximum number of atoms allowed in expr.\n int maxops_; // Maximum number of ops allowed in expr.\n std::vector atoms_; // Possible atoms.\n std::vector ops_; // Possible ops.\n std::minstd_rand0 rng_; // Random number generator.\n\n RegexpGenerator(const RegexpGenerator&) = delete;\n RegexpGenerator& operator=(const RegexpGenerator&) = delete;\n};\n\n// Helpers for preparing arguments to RegexpGenerator constructor.\n\n// Returns one string for each character in s.\nstd::vector Explode(absl::string_view s);\n\n// Splits string everywhere sep is found, returning\n// vector of pieces.\nstd::vector Split(absl::string_view sep, absl::string_view s);\n\n} // namespace re2\n\n#endif // RE2_TESTING_REGEXP_GENERATOR_H_\n" }, { "path": "re2/testing/regexp_test.cc", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Test parse.cc, dump.cc, and tostring.cc.\n\n#include \"re2/regexp.h\"\n\n#include \n\n#include \n#include \n#include \n\n#include \"gtest/gtest.h\"\n\nnamespace re2 {\n\n// Test that overflowed ref counts work.\nTEST(Regexp, BigRef) {\n Regexp* re;\n re = Regexp::Parse(\"x\", Regexp::NoParseFlags, NULL);\n for (int i = 0; i < 100000; i++)\n re->Incref();\n for (int i = 0; i < 100000; i++)\n re->Decref();\n ASSERT_EQ(re->Ref(), 1);\n re->Decref();\n}\n\n// Test that very large Concats work.\n// Depends on overflowed ref counts working.\nTEST(Regexp, BigConcat) {\n Regexp* x;\n x = Regexp::Parse(\"x\", Regexp::NoParseFlags, NULL);\n std::vector v(90000, x); // ToString bails out at 100000\n for (size_t i = 0; i < v.size(); i++)\n x->Incref();\n ASSERT_EQ(x->Ref(), 1 + static_cast(v.size())) << x->Ref();\n Regexp* re = Regexp::Concat(v.data(), static_cast(v.size()),\n Regexp::NoParseFlags);\n ASSERT_EQ(re->ToString(), std::string(v.size(), 'x'));\n re->Decref();\n ASSERT_EQ(x->Ref(), 1) << x->Ref();\n x->Decref();\n}\n\nTEST(Regexp, NamedCaptures) {\n Regexp* x;\n RegexpStatus status;\n x = Regexp::Parse(\n \"(?Pa+)|(e)(?Pw*)+(?Pb+)\", Regexp::PerlX, &status);\n EXPECT_TRUE(status.ok());\n EXPECT_EQ(4, x->NumCaptures());\n const std::map* have = x->NamedCaptures();\n EXPECT_TRUE(have != NULL);\n // there are only two named groups in the regexp: 'g1' and 'g2'.\n EXPECT_EQ(size_t{2}, have->size());\n std::map want;\n want[\"g1\"] = 1;\n want[\"g2\"] = 3;\n EXPECT_EQ(want, *have);\n x->Decref();\n delete have;\n}\n\nTEST(Regexp, CaptureNames) {\n Regexp* x;\n RegexpStatus status;\n x = Regexp::Parse(\n \"(?Pa+)|(e)(?Pw*)+(?Pb+)\", Regexp::PerlX, &status);\n EXPECT_TRUE(status.ok());\n EXPECT_EQ(4, x->NumCaptures());\n const std::map* have = x->CaptureNames();\n EXPECT_TRUE(have != NULL);\n EXPECT_EQ(size_t{3}, have->size());\n std::map want;\n want[1] = \"g1\";\n want[3] = \"g2\";\n want[4] = \"g1\";\n\n EXPECT_EQ(want, *have);\n x->Decref();\n delete have;\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/required_prefix_test.cc", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \n\n#include \n\n#include \"absl/base/macros.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/prog.h\"\n#include \"re2/regexp.h\"\n\nnamespace re2 {\n\nstruct PrefixTest {\n const char* regexp;\n bool return_value;\n const char* prefix;\n bool foldcase;\n const char* suffix;\n};\n\nstatic PrefixTest tests[] = {\n // Empty cases.\n { \"\", false },\n { \"(?m)^\", false },\n { \"(?-m)^\", false },\n\n // If the regexp has no ^, there's no required prefix.\n { \"abc\", false },\n\n // If the regexp immediately goes into\n // something not a literal match, there's no required prefix.\n { \"^a*\", false },\n { \"^(abc)\", false },\n\n // Otherwise, it should work.\n { \"^abc$\", true, \"abc\", false, \"(?-m:$)\" },\n { \"^abc\", true, \"abc\", false, \"\" },\n { \"^(?i)abc\", true, \"abc\", true, \"\" },\n { \"^abcd*\", true, \"abc\", false, \"d*\" },\n { \"^[Aa][Bb]cd*\", true, \"ab\", true, \"cd*\" },\n { \"^ab[Cc]d*\", true, \"ab\", false, \"[Cc]d*\" },\n { \"^☺abc\", true, \"☺abc\", false, \"\" },\n};\n\nTEST(RequiredPrefix, SimpleTests) {\n for (size_t i = 0; i < ABSL_ARRAYSIZE(tests); i++) {\n const PrefixTest& t = tests[i];\n for (size_t j = 0; j < 2; j++) {\n Regexp::ParseFlags flags = Regexp::LikePerl;\n if (j == 0)\n flags = flags | Regexp::Latin1;\n Regexp* re = Regexp::Parse(t.regexp, flags, NULL);\n ASSERT_TRUE(re != NULL) << \" \" << t.regexp;\n\n std::string p;\n bool f;\n Regexp* s;\n ASSERT_EQ(t.return_value, re->RequiredPrefix(&p, &f, &s))\n << \" \" << t.regexp << \" \" << (j == 0 ? \"latin1\" : \"utf8\")\n << \" \" << re->Dump();\n if (t.return_value) {\n ASSERT_EQ(p, std::string(t.prefix))\n << \" \" << t.regexp << \" \" << (j == 0 ? \"latin1\" : \"utf8\");\n ASSERT_EQ(f, t.foldcase)\n << \" \" << t.regexp << \" \" << (j == 0 ? \"latin1\" : \"utf8\");\n ASSERT_EQ(s->ToString(), std::string(t.suffix))\n << \" \" << t.regexp << \" \" << (j == 0 ? \"latin1\" : \"utf8\");\n s->Decref();\n }\n re->Decref();\n }\n }\n}\n\nstatic PrefixTest for_accel_tests[] = {\n // Empty cases.\n { \"\", false },\n { \"(?m)^\", false },\n { \"(?-m)^\", false },\n\n // If the regexp has a ^, there's no required prefix.\n { \"^abc\", false },\n\n // If the regexp immediately goes into\n // something not a literal match, there's no required prefix.\n { \"a*\", false },\n\n // Unlike RequiredPrefix(), RequiredPrefixForAccel() can \"see through\"\n // capturing groups, but doesn't try to glue prefix fragments together.\n { \"(a?)def\", false },\n { \"(ab?)def\", true, \"a\", false },\n { \"(abc?)def\", true, \"ab\", false },\n { \"(()a)def\", false },\n { \"((a)b)def\", true, \"a\", false },\n { \"((ab)c)def\", true, \"ab\", false },\n\n // Otherwise, it should work.\n { \"abc$\", true, \"abc\", false },\n { \"abc\", true, \"abc\", false },\n { \"(?i)abc\", true, \"abc\", true },\n { \"abcd*\", true, \"abc\", false },\n { \"[Aa][Bb]cd*\", true, \"ab\", true },\n { \"ab[Cc]d*\", true, \"ab\", false },\n { \"☺abc\", true, \"☺abc\", false },\n};\n\nTEST(RequiredPrefixForAccel, SimpleTests) {\n for (size_t i = 0; i < ABSL_ARRAYSIZE(for_accel_tests); i++) {\n const PrefixTest& t = for_accel_tests[i];\n for (size_t j = 0; j < 2; j++) {\n Regexp::ParseFlags flags = Regexp::LikePerl;\n if (j == 0)\n flags = flags | Regexp::Latin1;\n Regexp* re = Regexp::Parse(t.regexp, flags, NULL);\n ASSERT_TRUE(re != NULL) << \" \" << t.regexp;\n\n std::string p;\n bool f;\n ASSERT_EQ(t.return_value, re->RequiredPrefixForAccel(&p, &f))\n << \" \" << t.regexp << \" \" << (j == 0 ? \"latin1\" : \"utf8\")\n << \" \" << re->Dump();\n if (t.return_value) {\n ASSERT_EQ(p, std::string(t.prefix))\n << \" \" << t.regexp << \" \" << (j == 0 ? \"latin1\" : \"utf8\");\n ASSERT_EQ(f, t.foldcase)\n << \" \" << t.regexp << \" \" << (j == 0 ? \"latin1\" : \"utf8\");\n }\n re->Decref();\n }\n }\n}\n\nTEST(RequiredPrefixForAccel, CaseFoldingForKAndS) {\n Regexp* re;\n std::string p;\n bool f;\n\n // With Latin-1 encoding, `(?i)` prefixes can include 'k' and 's'.\n re = Regexp::Parse(\"(?i)KLM\", Regexp::LikePerl|Regexp::Latin1, NULL);\n ASSERT_TRUE(re != NULL);\n ASSERT_TRUE(re->RequiredPrefixForAccel(&p, &f));\n ASSERT_EQ(p, \"klm\");\n ASSERT_EQ(f, true);\n re->Decref();\n\n re = Regexp::Parse(\"(?i)STU\", Regexp::LikePerl|Regexp::Latin1, NULL);\n ASSERT_TRUE(re != NULL);\n ASSERT_TRUE(re->RequiredPrefixForAccel(&p, &f));\n ASSERT_EQ(p, \"stu\");\n ASSERT_EQ(f, true);\n re->Decref();\n\n // With UTF-8 encoding, `(?i)` prefixes can't include 'k' and 's'.\n // This is because they match U+212A and U+017F, respectively, and\n // so the parser ends up emitting character classes, not literals.\n re = Regexp::Parse(\"(?i)KLM\", Regexp::LikePerl, NULL);\n ASSERT_TRUE(re != NULL);\n ASSERT_FALSE(re->RequiredPrefixForAccel(&p, &f));\n re->Decref();\n\n re = Regexp::Parse(\"(?i)STU\", Regexp::LikePerl, NULL);\n ASSERT_TRUE(re != NULL);\n ASSERT_FALSE(re->RequiredPrefixForAccel(&p, &f));\n re->Decref();\n}\n\nstatic const char* prefix_accel_tests[] = {\n \"aababc\\\\d+\",\n \"(?i)AABABC\\\\d+\",\n};\n\nTEST(PrefixAccel, SimpleTests) {\n for (size_t i = 0; i < ABSL_ARRAYSIZE(prefix_accel_tests); i++) {\n const char* pattern = prefix_accel_tests[i];\n Regexp* re = Regexp::Parse(pattern, Regexp::LikePerl, NULL);\n ASSERT_TRUE(re != NULL);\n Prog* prog = re->CompileToProg(0);\n ASSERT_TRUE(prog != NULL);\n ASSERT_TRUE(prog->can_prefix_accel());\n for (int j = 0; j < 100; j++) {\n std::string text(j, 'a');\n const char* p = reinterpret_cast(\n prog->PrefixAccel(text.data(), text.size()));\n EXPECT_TRUE(p == NULL);\n text.append(\"aababc\");\n for (int k = 0; k < 100; k++) {\n text.append(k, 'a');\n p = reinterpret_cast(\n prog->PrefixAccel(text.data(), text.size()));\n EXPECT_EQ(j, p - text.data());\n }\n }\n delete prog;\n re->Decref();\n }\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/search_test.cc", "content": "// Copyright 2006-2007 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \n\n#include \n#include \n\n#include \"absl/base/macros.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/testing/exhaustive_tester.h\"\n#include \"re2/testing/tester.h\"\n\n// For target `log' in the Makefile.\n#ifndef LOGGING\n#define LOGGING 0\n#endif\n\nnamespace re2 {\n\nstruct RegexpTest {\n const char* regexp;\n const char* text;\n};\n\nRegexpTest simple_tests[] = {\n { \"a\", \"a\" },\n { \"a\", \"zyzzyva\" },\n { \"a+\", \"aa\" },\n { \"(a+|b)+\", \"ab\" },\n { \"ab|cd\", \"xabcdx\" },\n { \"h.*od?\", \"hello\\ngoodbye\\n\" },\n { \"h.*o\", \"hello\\ngoodbye\\n\" },\n { \"h.*o\", \"goodbye\\nhello\\n\" },\n { \"h.*o\", \"hello world\" },\n { \"h.*o\", \"othello, world\" },\n { \"[^\\\\s\\\\S]\", \"aaaaaaa\" },\n { \"a\", \"aaaaaaa\" },\n { \"a*\", \"aaaaaaa\" },\n { \"a*\", \"\" },\n { \"ab|cd\", \"xabcdx\" },\n { \"a\", \"cab\" },\n { \"a*b\", \"cab\" },\n { \"((((((((((((((((((((x))))))))))))))))))))\", \"x\" },\n { \"[abcd]\", \"xxxabcdxxx\" },\n { \"[^x]\", \"xxxabcdxxx\" },\n { \"[abcd]+\", \"xxxabcdxxx\" },\n { \"[^x]+\", \"xxxabcdxxx\" },\n { \"(fo|foo)\", \"fo\" },\n { \"(foo|fo)\", \"foo\" },\n\n { \"aa\", \"aA\" },\n { \"a\", \"Aa\" },\n { \"a\", \"A\" },\n { \"ABC\", \"abc\" },\n { \"abc\", \"XABCY\" },\n { \"ABC\", \"xabcy\" },\n\n // Make sure ^ and $ work.\n // The pathological cases didn't work\n // in the original grep code.\n { \"foo|bar|[A-Z]\", \"foo\" },\n { \"^(foo|bar|[A-Z])\", \"foo\" },\n { \"(foo|bar|[A-Z])$\", \"foo\\n\" },\n { \"(foo|bar|[A-Z])$\", \"foo\" },\n { \"^(foo|bar|[A-Z])$\", \"foo\\n\" },\n { \"^(foo|bar|[A-Z])$\", \"foo\" },\n { \"^(foo|bar|[A-Z])$\", \"bar\" },\n { \"^(foo|bar|[A-Z])$\", \"X\" },\n { \"^(foo|bar|[A-Z])$\", \"XY\" },\n { \"^(fo|foo)$\", \"fo\" },\n { \"^(fo|foo)$\", \"foo\" },\n { \"^^(fo|foo)$\", \"fo\" },\n { \"^^(fo|foo)$\", \"foo\" },\n { \"^$\", \"\" },\n { \"^$\", \"x\" },\n { \"^^$\", \"\" },\n { \"^$$\", \"\" },\n { \"^^$\", \"x\" },\n { \"^$$\", \"x\" },\n { \"^^$$\", \"\" },\n { \"^^$$\", \"x\" },\n { \"^^^^^^^^$$$$$$$$\", \"\" },\n { \"^\", \"x\" },\n { \"$\", \"x\" },\n\n // Word boundaries.\n { \"\\\\bfoo\\\\b\", \"nofoo foo that\" },\n { \"a\\\\b\", \"faoa x\" },\n { \"\\\\bbar\", \"bar x\" },\n { \"\\\\bbar\", \"foo\\nbar x\" },\n { \"bar\\\\b\", \"foobar\" },\n { \"bar\\\\b\", \"foobar\\nxxx\" },\n { \"(foo|bar|[A-Z])\\\\b\", \"foo\" },\n { \"(foo|bar|[A-Z])\\\\b\", \"foo\\n\" },\n { \"\\\\b\", \"\" },\n { \"\\\\b\", \"x\" },\n { \"\\\\b(foo|bar|[A-Z])\", \"foo\" },\n { \"\\\\b(foo|bar|[A-Z])\\\\b\", \"X\" },\n { \"\\\\b(foo|bar|[A-Z])\\\\b\", \"XY\" },\n { \"\\\\b(foo|bar|[A-Z])\\\\b\", \"bar\" },\n { \"\\\\b(foo|bar|[A-Z])\\\\b\", \"foo\" },\n { \"\\\\b(foo|bar|[A-Z])\\\\b\", \"foo\\n\" },\n { \"\\\\b(foo|bar|[A-Z])\\\\b\", \"ffoo bbar N x\" },\n { \"\\\\b(fo|foo)\\\\b\", \"fo\" },\n { \"\\\\b(fo|foo)\\\\b\", \"foo\" },\n { \"\\\\b\\\\b\", \"\" },\n { \"\\\\b\\\\b\", \"x\" },\n { \"\\\\b$\", \"\" },\n { \"\\\\b$\", \"x\" },\n { \"\\\\b$\", \"y x\" },\n { \"\\\\b.$\", \"x\" },\n { \"^\\\\b(fo|foo)\\\\b\", \"fo\" },\n { \"^\\\\b(fo|foo)\\\\b\", \"foo\" },\n { \"^\\\\b\", \"\" },\n { \"^\\\\b\", \"x\" },\n { \"^\\\\b\\\\b\", \"\" },\n { \"^\\\\b\\\\b\", \"x\" },\n { \"^\\\\b$\", \"\" },\n { \"^\\\\b$\", \"x\" },\n { \"^\\\\b.$\", \"x\" },\n { \"^\\\\b.\\\\b$\", \"x\" },\n { \"^^^^^^^^\\\\b$$$$$$$\", \"\" },\n { \"^^^^^^^^\\\\b.$$$$$$\", \"x\" },\n { \"^^^^^^^^\\\\b$$$$$$$\", \"x\" },\n\n // Non-word boundaries.\n { \"\\\\Bfoo\\\\B\", \"n foo xfoox that\" },\n { \"a\\\\B\", \"faoa x\" },\n { \"\\\\Bbar\", \"bar x\" },\n { \"\\\\Bbar\", \"foo\\nbar x\" },\n { \"bar\\\\B\", \"foobar\" },\n { \"bar\\\\B\", \"foobar\\nxxx\" },\n { \"(foo|bar|[A-Z])\\\\B\", \"foox\" },\n { \"(foo|bar|[A-Z])\\\\B\", \"foo\\n\" },\n { \"\\\\B\", \"\" },\n { \"\\\\B\", \"x\" },\n { \"\\\\B(foo|bar|[A-Z])\", \"foo\" },\n { \"\\\\B(foo|bar|[A-Z])\\\\B\", \"xXy\" },\n { \"\\\\B(foo|bar|[A-Z])\\\\B\", \"XY\" },\n { \"\\\\B(foo|bar|[A-Z])\\\\B\", \"XYZ\" },\n { \"\\\\B(foo|bar|[A-Z])\\\\B\", \"abara\" },\n { \"\\\\B(foo|bar|[A-Z])\\\\B\", \"xfoo_\" },\n { \"\\\\B(foo|bar|[A-Z])\\\\B\", \"xfoo\\n\" },\n { \"\\\\B(foo|bar|[A-Z])\\\\B\", \"foo bar vNx\" },\n { \"\\\\B(fo|foo)\\\\B\", \"xfoo\" },\n { \"\\\\B(foo|fo)\\\\B\", \"xfooo\" },\n { \"\\\\B\\\\B\", \"\" },\n { \"\\\\B\\\\B\", \"x\" },\n { \"\\\\B$\", \"\" },\n { \"\\\\B$\", \"x\" },\n { \"\\\\B$\", \"y x\" },\n { \"\\\\B.$\", \"x\" },\n { \"^\\\\B(fo|foo)\\\\B\", \"fo\" },\n { \"^\\\\B(fo|foo)\\\\B\", \"foo\" },\n { \"^\\\\B\", \"\" },\n { \"^\\\\B\", \"x\" },\n { \"^\\\\B\\\\B\", \"\" },\n { \"^\\\\B\\\\B\", \"x\" },\n { \"^\\\\B$\", \"\" },\n { \"^\\\\B$\", \"x\" },\n { \"^\\\\B.$\", \"x\" },\n { \"^\\\\B.\\\\B$\", \"x\" },\n { \"^^^^^^^^\\\\B$$$$$$$\", \"\" },\n { \"^^^^^^^^\\\\B.$$$$$$\", \"x\" },\n { \"^^^^^^^^\\\\B$$$$$$$\", \"x\" },\n\n // PCRE uses only ASCII for \\b computation.\n // All non-ASCII are *not* word characters.\n { \"\\\\bx\\\\b\", \"x\" },\n { \"\\\\bx\\\\b\", \"x>\" },\n { \"\\\\bx\\\\b\", \"\" },\n { \"\\\\bx\\\\b\", \"ax\" },\n { \"\\\\bx\\\\b\", \"xb\" },\n { \"\\\\bx\\\\b\", \"axb\" },\n { \"\\\\bx\\\\b\", \"«x\" },\n { \"\\\\bx\\\\b\", \"x»\" },\n { \"\\\\bx\\\\b\", \"«x»\" },\n { \"\\\\bx\\\\b\", \"axb\" },\n { \"\\\\bx\\\\b\", \"áxβ\" },\n { \"\\\\Bx\\\\B\", \"axb\" },\n { \"\\\\Bx\\\\B\", \"áxβ\" },\n\n // Weird boundary cases.\n { \"^$^$\", \"\" },\n { \"^$^\", \"\" },\n { \"$^$\", \"\" },\n\n { \"^$^$\", \"x\" },\n { \"^$^\", \"x\" },\n { \"$^$\", \"x\" },\n\n { \"^$^$\", \"x\\ny\" },\n { \"^$^\", \"x\\ny\" },\n { \"$^$\", \"x\\ny\" },\n\n { \"^$^$\", \"x\\n\\ny\" },\n { \"^$^\", \"x\\n\\ny\" },\n { \"$^$\", \"x\\n\\ny\" },\n\n { \"^(foo\\\\$)$\", \"foo$bar\" },\n { \"(foo\\\\$)\", \"foo$bar\" },\n { \"^...$\", \"abc\" },\n\n // UTF-8\n { \"^\\xe6\\x9c\\xac$\", \"\\xe6\\x9c\\xac\" },\n { \"^...$\", \"\\xe6\\x97\\xa5\\xe6\\x9c\\xac\\xe8\\xaa\\x9e\" },\n { \"^...$\", \".\\xe6\\x9c\\xac.\" },\n\n { \"^\\\\C\\\\C\\\\C$\", \"\\xe6\\x9c\\xac\" },\n { \"^\\\\C$\", \"\\xe6\\x9c\\xac\" },\n { \"^\\\\C\\\\C\\\\C$\", \"\\xe6\\x97\\xa5\\xe6\\x9c\\xac\\xe8\\xaa\\x9e\" },\n\n // Latin1\n { \"^...$\", \"\\xe6\\x97\\xa5\\xe6\\x9c\\xac\\xe8\\xaa\\x9e\" },\n { \"^.........$\", \"\\xe6\\x97\\xa5\\xe6\\x9c\\xac\\xe8\\xaa\\x9e\" },\n { \"^...$\", \".\\xe6\\x9c\\xac.\" },\n { \"^.....$\", \".\\xe6\\x9c\\xac.\" },\n\n // Perl v Posix\n { \"\\\\B(fo|foo)\\\\B\", \"xfooo\" },\n { \"(fo|foo)\", \"foo\" },\n\n // Octal escapes.\n { \"\\\\141\", \"a\" },\n { \"\\\\060\", \"0\" },\n { \"\\\\0600\", \"00\" },\n { \"\\\\608\", \"08\" },\n { \"\\\\01\", \"\\01\" },\n { \"\\\\018\", \"\\01\" \"8\" },\n\n // Hexadecimal escapes\n { \"\\\\x{61}\", \"a\" },\n { \"\\\\x61\", \"a\" },\n { \"\\\\x{00000061}\", \"a\" },\n\n // Unicode scripts.\n { \"\\\\p{Greek}+\", \"aαβb\" },\n { \"\\\\P{Greek}+\", \"aαβb\" },\n { \"\\\\p{^Greek}+\", \"aαβb\" },\n { \"\\\\P{^Greek}+\", \"aαβb\" },\n\n // Unicode properties. Nd is decimal number. N is any number.\n { \"[^0-9]+\", \"abc123\" },\n { \"\\\\p{Nd}+\", \"abc123²³¼½¾₀₉\" },\n { \"\\\\p{^Nd}+\", \"abc123²³¼½¾₀₉\" },\n { \"\\\\P{Nd}+\", \"abc123²³¼½¾₀₉\" },\n { \"\\\\P{^Nd}+\", \"abc123²³¼½¾₀₉\" },\n { \"\\\\pN+\", \"abc123²³¼½¾₀₉\" },\n { \"\\\\p{N}+\", \"abc123²³¼½¾₀₉\" },\n { \"\\\\p{^N}+\", \"abc123²³¼½¾₀₉\" },\n\n { \"\\\\p{Any}+\", \"abc123\" },\n\n // Character classes & case folding.\n { \"(?i)[@-A]+\", \"@AaB\" }, // matches @Aa but not B\n { \"(?i)[A-Z]+\", \"aAzZ\" },\n { \"(?i)[^\\\\\\\\]+\", \"Aa\\\\\" }, // \\\\ is between A-Z and a-z -\n // splits the ranges in an interesting way.\n\n // would like to use, but PCRE mishandles in full-match, non-greedy mode\n // { \"(?i)[\\\\\\\\]+\", \"Aa\" },\n\n { \"(?i)[acegikmoqsuwy]+\", \"acegikmoqsuwyACEGIKMOQSUWY\" },\n\n // Character classes & case folding.\n { \"[@-A]+\", \"@AaB\" },\n { \"[A-Z]+\", \"aAzZ\" },\n { \"[^\\\\\\\\]+\", \"Aa\\\\\" },\n { \"[acegikmoqsuwy]+\", \"acegikmoqsuwyACEGIKMOQSUWY\" },\n\n // Anchoring. (^abc in aabcdef was a former bug)\n // The tester checks for a match in the text and\n // subpieces of the text with a byte removed on either side.\n { \"^abc\", \"abcdef\" },\n { \"^abc\", \"aabcdef\" },\n { \"^[ay]*[bx]+c\", \"abcdef\" },\n { \"^[ay]*[bx]+c\", \"aabcdef\" },\n { \"def$\", \"abcdef\" },\n { \"def$\", \"abcdeff\" },\n { \"d[ex][fy]$\", \"abcdef\" },\n { \"d[ex][fy]$\", \"abcdeff\" },\n { \"[dz][ex][fy]$\", \"abcdef\" },\n { \"[dz][ex][fy]$\", \"abcdeff\" },\n { \"(?m)^abc\", \"abcdef\" },\n { \"(?m)^abc\", \"aabcdef\" },\n { \"(?m)^[ay]*[bx]+c\", \"abcdef\" },\n { \"(?m)^[ay]*[bx]+c\", \"aabcdef\" },\n { \"(?m)def$\", \"abcdef\" },\n { \"(?m)def$\", \"abcdeff\" },\n { \"(?m)d[ex][fy]$\", \"abcdef\" },\n { \"(?m)d[ex][fy]$\", \"abcdeff\" },\n { \"(?m)[dz][ex][fy]$\", \"abcdef\" },\n { \"(?m)[dz][ex][fy]$\", \"abcdeff\" },\n { \"^\", \"a\" },\n { \"^^\", \"a\" },\n\n // Context.\n // The tester checks for a match in the text and\n // subpieces of the text with a byte removed on either side.\n { \"a\", \"a\" },\n { \"ab*\", \"a\" },\n { \"a\\\\C*\", \"a\" },\n { \"a\\\\C+\", \"a\" },\n { \"a\\\\C?\", \"a\" },\n { \"a\\\\C*?\", \"a\" },\n { \"a\\\\C+?\", \"a\" },\n { \"a\\\\C??\", \"a\" },\n\n // Former bugs.\n { \"a\\\\C*|ba\\\\C\", \"baba\" },\n { \"\\\\w*I\\\\w*\", \"Inc.\" },\n { \"(?:|a)*\", \"aaa\" },\n { \"(?:|a)+\", \"aaa\" },\n};\n\nTEST(Regexp, SearchTests) {\n int failures = 0;\n for (size_t i = 0; i < ABSL_ARRAYSIZE(simple_tests); i++) {\n const RegexpTest& t = simple_tests[i];\n if (!TestRegexpOnText(t.regexp, t.text))\n failures++;\n\n if (LOGGING) {\n // Build a dummy ExhaustiveTest call that will trigger just\n // this one test, so that we log the test case.\n std::vector atom, alpha, ops;\n atom.push_back(t.regexp);\n alpha.push_back(t.text);\n ExhaustiveTest(1, 0, atom, ops, 1, alpha, \"\", \"\");\n }\n }\n EXPECT_EQ(failures, 0);\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/set_test.cc", "content": "// Copyright 2010 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \"re2/set.h\"\n\n#include \n\n#include \n#include \n\n#include \"gtest/gtest.h\"\n#include \"re2/re2.h\"\n\nnamespace re2 {\n\nTEST(Set, Unanchored) {\n RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED);\n\n ASSERT_EQ(s.Size(), 0);\n ASSERT_EQ(s.Add(\"foo\", NULL), 0);\n ASSERT_EQ(s.Size(), 1);\n ASSERT_EQ(s.Add(\"(\", NULL), -1);\n ASSERT_EQ(s.Size(), 1);\n ASSERT_EQ(s.Add(\"bar\", NULL), 1);\n ASSERT_EQ(s.Size(), 2);\n ASSERT_EQ(s.Compile(), true);\n ASSERT_EQ(s.Size(), 2);\n\n ASSERT_EQ(s.Match(\"foobar\", NULL), true);\n ASSERT_EQ(s.Match(\"fooba\", NULL), true);\n ASSERT_EQ(s.Match(\"oobar\", NULL), true);\n\n std::vector v;\n ASSERT_EQ(s.Match(\"foobar\", &v), true);\n ASSERT_EQ(v.size(), size_t{2});\n ASSERT_EQ(v[0], 0);\n ASSERT_EQ(v[1], 1);\n\n ASSERT_EQ(s.Match(\"fooba\", &v), true);\n ASSERT_EQ(v.size(), size_t{1});\n ASSERT_EQ(v[0], 0);\n\n ASSERT_EQ(s.Match(\"oobar\", &v), true);\n ASSERT_EQ(v.size(), size_t{1});\n ASSERT_EQ(v[0], 1);\n}\n\nTEST(Set, UnanchoredFactored) {\n RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED);\n\n ASSERT_EQ(s.Add(\"foo\", NULL), 0);\n ASSERT_EQ(s.Add(\"(\", NULL), -1);\n ASSERT_EQ(s.Add(\"foobar\", NULL), 1);\n ASSERT_EQ(s.Compile(), true);\n\n ASSERT_EQ(s.Match(\"foobar\", NULL), true);\n ASSERT_EQ(s.Match(\"obarfoobaroo\", NULL), true);\n ASSERT_EQ(s.Match(\"fooba\", NULL), true);\n ASSERT_EQ(s.Match(\"oobar\", NULL), false);\n\n std::vector v;\n ASSERT_EQ(s.Match(\"foobar\", &v), true);\n ASSERT_EQ(v.size(), size_t{2});\n ASSERT_EQ(v[0], 0);\n ASSERT_EQ(v[1], 1);\n\n ASSERT_EQ(s.Match(\"obarfoobaroo\", &v), true);\n ASSERT_EQ(v.size(), size_t{2});\n ASSERT_EQ(v[0], 0);\n ASSERT_EQ(v[1], 1);\n\n ASSERT_EQ(s.Match(\"fooba\", &v), true);\n ASSERT_EQ(v.size(), size_t{1});\n ASSERT_EQ(v[0], 0);\n\n ASSERT_EQ(s.Match(\"oobar\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n}\n\nTEST(Set, UnanchoredDollar) {\n RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED);\n\n ASSERT_EQ(s.Add(\"foo$\", NULL), 0);\n ASSERT_EQ(s.Compile(), true);\n\n ASSERT_EQ(s.Match(\"foo\", NULL), true);\n ASSERT_EQ(s.Match(\"foobar\", NULL), false);\n\n std::vector v;\n ASSERT_EQ(s.Match(\"foo\", &v), true);\n ASSERT_EQ(v.size(), size_t{1});\n ASSERT_EQ(v[0], 0);\n\n ASSERT_EQ(s.Match(\"foobar\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n}\n\nTEST(Set, UnanchoredWordBoundary) {\n RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED);\n\n ASSERT_EQ(s.Add(\"foo\\\\b\", NULL), 0);\n ASSERT_EQ(s.Compile(), true);\n\n ASSERT_EQ(s.Match(\"foo\", NULL), true);\n ASSERT_EQ(s.Match(\"foobar\", NULL), false);\n ASSERT_EQ(s.Match(\"foo bar\", NULL), true);\n\n std::vector v;\n ASSERT_EQ(s.Match(\"foo\", &v), true);\n ASSERT_EQ(v.size(), size_t{1});\n ASSERT_EQ(v[0], 0);\n\n ASSERT_EQ(s.Match(\"foobar\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n\n ASSERT_EQ(s.Match(\"foo bar\", &v), true);\n ASSERT_EQ(v.size(), size_t{1});\n ASSERT_EQ(v[0], 0);\n}\n\nTEST(Set, Anchored) {\n RE2::Set s(RE2::DefaultOptions, RE2::ANCHOR_BOTH);\n\n ASSERT_EQ(s.Add(\"foo\", NULL), 0);\n ASSERT_EQ(s.Add(\"(\", NULL), -1);\n ASSERT_EQ(s.Add(\"bar\", NULL), 1);\n ASSERT_EQ(s.Compile(), true);\n\n ASSERT_EQ(s.Match(\"foobar\", NULL), false);\n ASSERT_EQ(s.Match(\"fooba\", NULL), false);\n ASSERT_EQ(s.Match(\"oobar\", NULL), false);\n ASSERT_EQ(s.Match(\"foo\", NULL), true);\n ASSERT_EQ(s.Match(\"bar\", NULL), true);\n\n std::vector v;\n ASSERT_EQ(s.Match(\"foobar\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n\n ASSERT_EQ(s.Match(\"fooba\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n\n ASSERT_EQ(s.Match(\"oobar\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n\n ASSERT_EQ(s.Match(\"foo\", &v), true);\n ASSERT_EQ(v.size(), size_t{1});\n ASSERT_EQ(v[0], 0);\n\n ASSERT_EQ(s.Match(\"bar\", &v), true);\n ASSERT_EQ(v.size(), size_t{1});\n ASSERT_EQ(v[0], 1);\n}\n\nTEST(Set, EmptyUnanchored) {\n RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED);\n\n ASSERT_EQ(s.Compile(), true);\n\n ASSERT_EQ(s.Match(\"\", NULL), false);\n ASSERT_EQ(s.Match(\"foobar\", NULL), false);\n\n std::vector v;\n ASSERT_EQ(s.Match(\"\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n\n ASSERT_EQ(s.Match(\"foobar\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n}\n\nTEST(Set, EmptyAnchored) {\n RE2::Set s(RE2::DefaultOptions, RE2::ANCHOR_BOTH);\n\n ASSERT_EQ(s.Compile(), true);\n\n ASSERT_EQ(s.Match(\"\", NULL), false);\n ASSERT_EQ(s.Match(\"foobar\", NULL), false);\n\n std::vector v;\n ASSERT_EQ(s.Match(\"\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n\n ASSERT_EQ(s.Match(\"foobar\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n}\n\nTEST(Set, Prefix) {\n RE2::Set s(RE2::DefaultOptions, RE2::ANCHOR_BOTH);\n\n ASSERT_EQ(s.Add(\"/prefix/\\\\d*\", NULL), 0);\n ASSERT_EQ(s.Compile(), true);\n\n ASSERT_EQ(s.Match(\"/prefix\", NULL), false);\n ASSERT_EQ(s.Match(\"/prefix/\", NULL), true);\n ASSERT_EQ(s.Match(\"/prefix/42\", NULL), true);\n\n std::vector v;\n ASSERT_EQ(s.Match(\"/prefix\", &v), false);\n ASSERT_EQ(v.size(), size_t{0});\n\n ASSERT_EQ(s.Match(\"/prefix/\", &v), true);\n ASSERT_EQ(v.size(), size_t{1});\n ASSERT_EQ(v[0], 0);\n\n ASSERT_EQ(s.Match(\"/prefix/42\", &v), true);\n ASSERT_EQ(v.size(), size_t{1});\n ASSERT_EQ(v[0], 0);\n}\n\nTEST(Set, MoveSemantics) {\n RE2::Set s1(RE2::DefaultOptions, RE2::UNANCHORED);\n ASSERT_EQ(s1.Add(\"foo\\\\d+\", NULL), 0);\n ASSERT_EQ(s1.Compile(), true);\n ASSERT_EQ(s1.Match(\"abc foo1 xyz\", NULL), true);\n ASSERT_EQ(s1.Match(\"abc bar2 xyz\", NULL), false);\n\n // The moved-to object should do what the moved-from object did.\n RE2::Set s2 = std::move(s1);\n ASSERT_EQ(s2.Match(\"abc foo1 xyz\", NULL), true);\n ASSERT_EQ(s2.Match(\"abc bar2 xyz\", NULL), false);\n\n // The moved-from object should have been reset and be reusable.\n ASSERT_EQ(s1.Add(\"bar\\\\d+\", NULL), 0);\n ASSERT_EQ(s1.Compile(), true);\n ASSERT_EQ(s1.Match(\"abc foo1 xyz\", NULL), false);\n ASSERT_EQ(s1.Match(\"abc bar2 xyz\", NULL), true);\n\n // Verify that \"overwriting\" works and also doesn't leak memory.\n // (The latter will need a leak detector such as LeakSanitizer.)\n s1 = std::move(s2);\n ASSERT_EQ(s1.Match(\"abc foo1 xyz\", NULL), true);\n ASSERT_EQ(s1.Match(\"abc bar2 xyz\", NULL), false);\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/simplify_test.cc", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Test simplify.cc.\n\n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_log.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/regexp.h\"\n\nnamespace re2 {\n\nstruct Test {\n const char* regexp;\n const char* simplified;\n};\n\nstatic Test tests[] = {\n // Already-simple constructs\n { \"a\", \"a\" },\n { \"ab\", \"ab\" },\n { \"a|b\", \"[a-b]\" },\n { \"ab|cd\", \"ab|cd\" },\n { \"(ab)*\", \"(ab)*\" },\n { \"(ab)+\", \"(ab)+\" },\n { \"(ab)?\", \"(ab)?\" },\n { \".\", \".\" },\n { \"^\", \"^\" },\n { \"$\", \"$\" },\n { \"[ac]\", \"[ac]\" },\n { \"[^ac]\", \"[^ac]\" },\n\n // Posix character classes\n { \"[[:alnum:]]\", \"[0-9A-Za-z]\" },\n { \"[[:alpha:]]\", \"[A-Za-z]\" },\n { \"[[:blank:]]\", \"[\\\\t ]\" },\n { \"[[:cntrl:]]\", \"[\\\\x00-\\\\x1f\\\\x7f]\" },\n { \"[[:digit:]]\", \"[0-9]\" },\n { \"[[:graph:]]\", \"[!-~]\" },\n { \"[[:lower:]]\", \"[a-z]\" },\n { \"[[:print:]]\", \"[ -~]\" },\n { \"[[:punct:]]\", \"[!-/:-@\\\\[-`{-~]\" },\n { \"[[:space:]]\" , \"[\\\\t-\\\\r ]\" },\n { \"[[:upper:]]\", \"[A-Z]\" },\n { \"[[:xdigit:]]\", \"[0-9A-Fa-f]\" },\n\n // Perl character classes\n { \"\\\\d\", \"[0-9]\" },\n { \"\\\\s\", \"[\\\\t-\\\\n\\\\f-\\\\r ]\" },\n { \"\\\\w\", \"[0-9A-Z_a-z]\" },\n { \"\\\\D\", \"[^0-9]\" },\n { \"\\\\S\", \"[^\\\\t-\\\\n\\\\f-\\\\r ]\" },\n { \"\\\\W\", \"[^0-9A-Z_a-z]\" },\n { \"[\\\\d]\", \"[0-9]\" },\n { \"[\\\\s]\", \"[\\\\t-\\\\n\\\\f-\\\\r ]\" },\n { \"[\\\\w]\", \"[0-9A-Z_a-z]\" },\n { \"[\\\\D]\", \"[^0-9]\" },\n { \"[\\\\S]\", \"[^\\\\t-\\\\n\\\\f-\\\\r ]\" },\n { \"[\\\\W]\", \"[^0-9A-Z_a-z]\" },\n\n // Posix repetitions\n { \"a{1}\", \"a\" },\n { \"a{2}\", \"aa\" },\n { \"a{5}\", \"aaaaa\" },\n { \"a{0,1}\", \"a?\" },\n // The next three are illegible because Simplify inserts (?:)\n // parens instead of () parens to avoid creating extra\n // captured subexpressions. The comments show a version fewer parens.\n { \"(a){0,2}\", \"(?:(a)(a)?)?\" }, // (aa?)?\n { \"(a){0,4}\", \"(?:(a)(?:(a)(?:(a)(a)?)?)?)?\" }, // (a(a(aa?)?)?)?\n { \"(a){2,6}\", \"(a)(a)(?:(a)(?:(a)(?:(a)(a)?)?)?)?\" }, // aa(a(a(aa?)?)?)?\n { \"a{0,2}\", \"(?:aa?)?\" }, // (aa?)?\n { \"a{0,4}\", \"(?:a(?:a(?:aa?)?)?)?\" }, // (a(a(aa?)?)?)?\n { \"a{2,6}\", \"aa(?:a(?:a(?:aa?)?)?)?\" }, // aa(a(a(aa?)?)?)?\n { \"a{0,}\", \"a*\" },\n { \"a{1,}\", \"a+\" },\n { \"a{2,}\", \"aa+\" },\n { \"a{5,}\", \"aaaaa+\" },\n\n // Test that operators simplify their arguments.\n // (Simplify used to not simplify arguments to a {} repeat.)\n { \"(?:a{1,}){1,}\", \"a+\" },\n { \"(a{1,}b{1,})\", \"(a+b+)\" },\n { \"a{1,}|b{1,}\", \"a+|b+\" },\n { \"(?:a{1,})*\", \"(?:a+)*\" },\n { \"(?:a{1,})+\", \"a+\" },\n { \"(?:a{1,})?\", \"(?:a+)?\" },\n { \"a{0}\", \"\" },\n\n // Character class simplification\n { \"[ab]\", \"[a-b]\" },\n { \"[a-za-za-z]\", \"[a-z]\" },\n { \"[A-Za-zA-Za-z]\", \"[A-Za-z]\" },\n { \"[ABCDEFGH]\", \"[A-H]\" },\n { \"[AB-CD-EF-GH]\", \"[A-H]\" },\n { \"[W-ZP-XE-R]\", \"[E-Z]\" },\n { \"[a-ee-gg-m]\", \"[a-m]\" },\n { \"[a-ea-ha-m]\", \"[a-m]\" },\n { \"[a-ma-ha-e]\", \"[a-m]\" },\n { \"[a-zA-Z0-9 -~]\", \"[ -~]\" },\n\n // Empty character classes\n { \"[^[:cntrl:][:^cntrl:]]\", \"[^\\\\x00-\\\\x{10ffff}]\" },\n\n // Full character classes\n { \"[[:cntrl:][:^cntrl:]]\", \".\" },\n\n // Unicode case folding.\n { \"(?i)A\", \"[Aa]\" },\n { \"(?i)a\", \"[Aa]\" },\n { \"(?i)K\", \"[Kk\\\\x{212a}]\" },\n { \"(?i)k\", \"[Kk\\\\x{212a}]\" },\n { \"(?i)\\\\x{212a}\", \"[Kk\\\\x{212a}]\" },\n { \"(?i)[a-z]\", \"[A-Za-z\\\\x{17f}\\\\x{212a}]\" },\n { \"(?i)[\\\\x00-\\\\x{FFFD}]\", \"[\\\\x00-\\\\x{fffd}]\" },\n { \"(?i)[\\\\x00-\\\\x{10ffff}]\", \".\" },\n\n // Empty string as a regular expression.\n // Empty string must be preserved inside parens in order\n // to make submatches work right, so these are less\n // interesting than they used to be. ToString inserts\n // explicit (?:) in place of non-parenthesized empty strings,\n // to make them easier to spot for other parsers.\n { \"(a|b|)\", \"([a-b]|(?:))\" },\n { \"(|)\", \"((?:)|(?:))\" },\n { \"a()\", \"a()\" },\n { \"(()|())\", \"(()|())\" },\n { \"(a|)\", \"(a|(?:))\" },\n { \"ab()cd()\", \"ab()cd()\" },\n { \"()\", \"()\" },\n { \"()*\", \"()*\" },\n { \"()+\", \"()+\" },\n { \"()?\" , \"()?\" },\n { \"(){0}\", \"\" },\n { \"(){1}\", \"()\" },\n { \"(){1,}\", \"()+\" },\n { \"(){0,2}\", \"(?:()()?)?\" },\n\n // For an empty-width op OR a concatenation or alternation of empty-width\n // ops, test that the repetition count is capped at 1.\n { \"(?:^){0,}\", \"^*\" }, // x{0,} -> x*\n { \"(?:$){28,}\", \"$+\" }, // x{N,} -> x{1,} -> x+\n { \"(?-m:^){0,30}\", \"(?-m:^)?\" }, // x{0,N} -> x{0,1} -> x?\n { \"(?-m:$){28,30}\", \"(?-m:$)\" }, // x{N,M} -> x{1,1} -> x\n { \"\\\\b(?:\\\\b\\\\B){999}\\\\B\", \"\\\\b\\\\b\\\\B\\\\B\" },\n { \"\\\\b(?:\\\\b|\\\\B){999}\\\\B\", \"\\\\b(?:\\\\b|\\\\B)\\\\B\" },\n // NonGreedy should also be handled.\n { \"(?:^){0,}?\", \"^*?\" },\n { \"(?:$){28,}?\", \"$+?\" },\n { \"(?-m:^){0,30}?\", \"(?-m:^)??\" },\n { \"(?-m:$){28,30}?\", \"(?-m:$)\" },\n { \"\\\\b(?:\\\\b\\\\B){999}?\\\\B\", \"\\\\b\\\\b\\\\B\\\\B\" },\n { \"\\\\b(?:\\\\b|\\\\B){999}?\\\\B\", \"\\\\b(?:\\\\b|\\\\B)\\\\B\" },\n\n // Test that coalescing occurs and that the resulting repeats are simplified.\n // Two-op combinations of *, +, ?, {n}, {n,} and {n,m} with a literal:\n { \"a*a*\", \"a*\" },\n { \"a*a+\", \"a+\" },\n { \"a*a?\", \"a*\" },\n { \"a*a{2}\", \"aa+\" },\n { \"a*a{2,}\", \"aa+\" },\n { \"a*a{2,3}\", \"aa+\" },\n { \"a+a*\", \"a+\" },\n { \"a+a+\", \"aa+\" },\n { \"a+a?\", \"a+\" },\n { \"a+a{2}\", \"aaa+\" },\n { \"a+a{2,}\", \"aaa+\" },\n { \"a+a{2,3}\", \"aaa+\" },\n { \"a?a*\", \"a*\" },\n { \"a?a+\", \"a+\" },\n { \"a?a?\", \"(?:aa?)?\" },\n { \"a?a{2}\", \"aaa?\" },\n { \"a?a{2,}\", \"aa+\" },\n { \"a?a{2,3}\", \"aa(?:aa?)?\" },\n { \"a{2}a*\", \"aa+\" },\n { \"a{2}a+\", \"aaa+\" },\n { \"a{2}a?\", \"aaa?\" },\n { \"a{2}a{2}\", \"aaaa\" },\n { \"a{2}a{2,}\", \"aaaa+\" },\n { \"a{2}a{2,3}\", \"aaaaa?\" },\n { \"a{2,}a*\", \"aa+\" },\n { \"a{2,}a+\", \"aaa+\" },\n { \"a{2,}a?\", \"aa+\" },\n { \"a{2,}a{2}\", \"aaaa+\" },\n { \"a{2,}a{2,}\", \"aaaa+\" },\n { \"a{2,}a{2,3}\", \"aaaa+\" },\n { \"a{2,3}a*\", \"aa+\" },\n { \"a{2,3}a+\", \"aaa+\" },\n { \"a{2,3}a?\", \"aa(?:aa?)?\" },\n { \"a{2,3}a{2}\", \"aaaaa?\" },\n { \"a{2,3}a{2,}\", \"aaaa+\" },\n { \"a{2,3}a{2,3}\", \"aaaa(?:aa?)?\" },\n // With a char class, any char and any byte:\n { \"\\\\d*\\\\d*\", \"[0-9]*\" },\n { \".*.*\", \".*\" },\n { \"\\\\C*\\\\C*\", \"\\\\C*\" },\n // FoldCase works, but must be consistent:\n { \"(?i)A*a*\", \"[Aa]*\" },\n { \"(?i)a+A+\", \"[Aa][Aa]+\" },\n { \"(?i)A*(?-i)a*\", \"[Aa]*a*\" },\n { \"(?i)a+(?-i)A+\", \"[Aa]+A+\" },\n // NonGreedy works, but must be consistent:\n { \"a*?a*?\", \"a*?\" },\n { \"a+?a+?\", \"aa+?\" },\n { \"a*?a*\", \"a*?a*\" },\n { \"a+a+?\", \"a+a+?\" },\n // The second element is the literal, char class, any char or any byte:\n { \"a*a\", \"a+\" },\n { \"\\\\d*\\\\d\", \"[0-9]+\" },\n { \".*.\", \".+\" },\n { \"\\\\C*\\\\C\", \"\\\\C+\" },\n // FoldCase works, but must be consistent:\n { \"(?i)A*a\", \"[Aa]+\" },\n { \"(?i)a+A\", \"[Aa][Aa]+\" },\n { \"(?i)A*(?-i)a\", \"[Aa]*a\" },\n { \"(?i)a+(?-i)A\", \"[Aa]+A\" },\n // The second element is a literal string that begins with the literal:\n { \"a*aa\", \"aa+\" },\n { \"a*aab\", \"aa+b\" },\n // FoldCase works, but must be consistent:\n { \"(?i)a*aa\", \"[Aa][Aa]+\" },\n { \"(?i)a*aab\", \"[Aa][Aa]+[Bb]\" },\n { \"(?i)a*(?-i)aa\", \"[Aa]*aa\" },\n { \"(?i)a*(?-i)aab\", \"[Aa]*aab\" },\n // Negative tests with mismatching ops:\n { \"a*b*\", \"a*b*\" },\n { \"\\\\d*\\\\D*\", \"[0-9]*[^0-9]*\" },\n { \"a+b\", \"a+b\" },\n { \"\\\\d+\\\\D\", \"[0-9]+[^0-9]\" },\n { \"a?bb\", \"a?bb\" },\n // Negative tests with capturing groups:\n { \"(a*)a*\", \"(a*)a*\" },\n { \"a+(a)\", \"a+(a)\" },\n { \"(a?)(aa)\", \"(a?)(aa)\" },\n // Just for fun:\n { \"aa*aa+aa?aa{2}aaa{2,}aaa{2,3}a\", \"aaaaaaaaaaaaaaaa+\" },\n\n // During coalescing, the child of the repeat changes, so we build a new\n // repeat. The new repeat must have the min and max of the old repeat.\n // Failure to copy them results in min=0 and max=0 -> empty match.\n { \"(?:a*aab){2}\", \"aa+baa+b\" },\n\n // During coalescing, the child of the capture changes, so we build a new\n // capture. The new capture must have the cap of the old capture.\n // Failure to copy it results in cap=0 -> ToString() logs a fatal error.\n { \"(a*aab)\", \"(aa+b)\" },\n\n // Test squashing of **, ++, ?? et cetera.\n { \"(?:(?:a){0,}){0,}\", \"a*\" },\n { \"(?:(?:a){1,}){1,}\", \"a+\" },\n { \"(?:(?:a){0,1}){0,1}\", \"a?\" },\n { \"(?:(?:a){0,}){1,}\", \"a*\" },\n { \"(?:(?:a){0,}){0,1}\", \"a*\" },\n { \"(?:(?:a){1,}){0,}\", \"a*\" },\n { \"(?:(?:a){1,}){0,1}\", \"a*\" },\n { \"(?:(?:a){0,1}){0,}\", \"a*\" },\n { \"(?:(?:a){0,1}){1,}\", \"a*\" },\n};\n\nTEST(TestSimplify, SimpleRegexps) {\n for (size_t i = 0; i < ABSL_ARRAYSIZE(tests); i++) {\n RegexpStatus status;\n ABSL_VLOG(1) << \"Testing \" << tests[i].regexp;\n Regexp* re = Regexp::Parse(tests[i].regexp,\n Regexp::MatchNL | (Regexp::LikePerl &\n ~Regexp::OneLine),\n &status);\n ASSERT_TRUE(re != NULL) << \" \" << tests[i].regexp << \" \" << status.Text();\n Regexp* sre = re->Simplify();\n ASSERT_TRUE(sre != NULL);\n\n // Check that already-simple regexps don't allocate new ones.\n if (strcmp(tests[i].regexp, tests[i].simplified) == 0) {\n ASSERT_TRUE(re == sre) << \" \" << tests[i].regexp\n << \" \" << re->ToString() << \" \" << sre->ToString();\n }\n\n EXPECT_EQ(tests[i].simplified, sre->ToString())\n << \" \" << tests[i].regexp << \" \" << sre->Dump();\n\n re->Decref();\n sre->Decref();\n }\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/string_generator.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// String generator: generates all possible strings of up to\n// maxlen letters using the set of letters in alpha.\n// Fetch strings using a Java-like Next()/HasNext() interface.\n\n#include \"re2/testing/string_generator.h\"\n\n#include \n#include \n\n#include \n#include \n#include \n\n#include \"absl/log/absl_check.h\"\n#include \"absl/strings/string_view.h\"\n\nnamespace re2 {\n\nStringGenerator::StringGenerator(int maxlen,\n const std::vector& alphabet)\n : maxlen_(maxlen), alphabet_(alphabet),\n generate_null_(false),\n random_(false), nrandom_(0) {\n\n // Degenerate case: no letters, no non-empty strings.\n if (alphabet_.empty())\n maxlen_ = 0;\n\n // Next() will return empty string (digits_ is empty).\n hasnext_ = true;\n}\n\n// Resets the string generator state to the beginning.\nvoid StringGenerator::Reset() {\n digits_.clear();\n hasnext_ = true;\n random_ = false;\n nrandom_ = 0;\n generate_null_ = false;\n}\n\n// Increments the big number in digits_, returning true if successful.\n// Returns false if all the numbers have been used.\nbool StringGenerator::IncrementDigits() {\n // First try to increment the current number.\n for (int i = static_cast(digits_.size()) - 1; i >= 0; i--) {\n if (++digits_[i] < static_cast(alphabet_.size()))\n return true;\n digits_[i] = 0;\n }\n\n // If that failed, make a longer number.\n if (static_cast(digits_.size()) < maxlen_) {\n digits_.push_back(0);\n return true;\n }\n\n return false;\n}\n\n// Generates random digits_, return true if successful.\n// Returns false if the random sequence is over.\nbool StringGenerator::RandomDigits() {\n if (--nrandom_ <= 0)\n return false;\n\n std::uniform_int_distribution random_len(0, maxlen_);\n std::uniform_int_distribution random_alphabet_index(\n 0, static_cast(alphabet_.size()) - 1);\n\n // Pick length.\n int len = random_len(rng_);\n digits_.resize(len);\n for (int i = 0; i < len; i++)\n digits_[i] = random_alphabet_index(rng_);\n return true;\n}\n\n// Returns the next string in the iteration, which is the one\n// currently described by digits_. Calls IncrementDigits\n// after computing the string, so that it knows the answer\n// for subsequent HasNext() calls.\nabsl::string_view StringGenerator::Next() {\n ABSL_CHECK(hasnext_);\n if (generate_null_) {\n generate_null_ = false;\n sp_ = absl::string_view();\n return sp_;\n }\n s_.clear();\n for (size_t i = 0; i < digits_.size(); i++) {\n s_ += alphabet_[digits_[i]];\n }\n hasnext_ = random_ ? RandomDigits() : IncrementDigits();\n sp_ = s_;\n return sp_;\n}\n\n// Sets generator up to return n random strings.\nvoid StringGenerator::Random(int32_t seed, int n) {\n rng_.seed(seed);\n\n random_ = true;\n nrandom_ = n;\n hasnext_ = nrandom_ > 0;\n}\n\nvoid StringGenerator::GenerateNULL() {\n generate_null_ = true;\n hasnext_ = true;\n}\n\nstd::string DeBruijnString(int n) {\n ABSL_CHECK_GE(n, 1);\n ABSL_CHECK_LE(n, 29);\n const size_t size = size_t{1} << static_cast(n);\n const size_t mask = size - 1;\n std::vector did(size, false);\n std::string s;\n s.reserve(static_cast(n) + size);\n for (size_t i = 0; i < static_cast(n - 1); i++)\n s += '0';\n size_t bits = 0;\n for (size_t i = 0; i < size; i++) {\n bits <<= 1;\n bits &= mask;\n if (!did[bits | 1]) {\n bits |= 1;\n s += '1';\n } else {\n s += '0';\n }\n ABSL_CHECK(!did[bits]);\n did[bits] = true;\n }\n ABSL_CHECK_EQ(s.size(), static_cast(n - 1) + size);\n return s;\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/string_generator.h", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_TESTING_STRING_GENERATOR_H_\n#define RE2_TESTING_STRING_GENERATOR_H_\n\n// String generator: generates all possible strings of up to\n// maxlen letters using the set of letters in alpha.\n// Fetch strings using a Java-like Next()/HasNext() interface.\n\n#include \n\n#include \n#include \n#include \n\n#include \"absl/strings/string_view.h\"\n\nnamespace re2 {\n\nclass StringGenerator {\n public:\n StringGenerator(int maxlen, const std::vector& alphabet);\n ~StringGenerator() {}\n\n absl::string_view Next();\n bool HasNext() { return hasnext_; }\n\n // Resets generator to start sequence over.\n void Reset();\n\n // Causes generator to emit random strings for next n calls to Next().\n void Random(int32_t seed, int n);\n\n // Causes generator to emit a NULL as the next call.\n void GenerateNULL();\n\n private:\n bool IncrementDigits();\n bool RandomDigits();\n\n // Global state.\n int maxlen_; // Maximum length string to generate.\n std::vector alphabet_; // Alphabet, one string per letter.\n\n // Iteration state.\n absl::string_view sp_; // Last string_view returned by Next().\n std::string s_; // String data in last string_view returned by Next().\n bool hasnext_; // Whether Next() can be called again.\n std::vector digits_; // Alphabet indices for next string.\n bool generate_null_; // Whether to generate a NULL string_view next.\n bool random_; // Whether generated strings are random.\n int nrandom_; // Number of random strings left to generate.\n std::minstd_rand0 rng_; // Random number generator.\n\n StringGenerator(const StringGenerator&) = delete;\n StringGenerator& operator=(const StringGenerator&) = delete;\n};\n\n// Generates and returns a string over binary alphabet {0,1} that contains\n// all possible binary sequences of length n as subsequences. The obvious\n// brute force method would generate a string of length n * 2^n, but this\n// generates a string of length n-1 + 2^n called a De Bruijn cycle.\n// See Knuth, The Art of Computer Programming, Vol 2, Exercise 3.2.2 #17.\n//\n// Such a string is useful for testing a DFA. If you have a DFA\n// where distinct last n bytes implies distinct states, then running on a\n// DeBruijn string causes the DFA to need to create a new state at every\n// position in the input, never reusing any states until it gets to the\n// end of the string. This is the worst possible case for DFA execution.\nstd::string DeBruijnString(int n);\n\n} // namespace re2\n\n#endif // RE2_TESTING_STRING_GENERATOR_H_\n" }, { "path": "re2/testing/string_generator_test.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Test StringGenerator.\n\n#include \"re2/testing/string_generator.h\"\n\n#include \n#include \n\n#include \n\n#include \"absl/strings/string_view.h\"\n#include \"gtest/gtest.h\"\n#include \"re2/testing/regexp_generator.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\n// Returns i to the e.\nstatic int64_t IntegerPower(int i, int e) {\n int64_t p = 1;\n while (e-- > 0)\n p *= i;\n return p;\n}\n\n// Checks that for given settings of the string generator:\n// * it generates strings that are non-decreasing in length.\n// * strings of the same length are sorted in alphabet order.\n// * it doesn't generate the same string twice.\n// * it generates the right number of strings.\n//\n// If all of these hold, the StringGenerator is behaving.\n// Assumes that the alphabet is sorted, so that the generated\n// strings can just be compared lexicographically.\nstatic void RunTest(int len, const std::string& alphabet, bool donull) {\n StringGenerator g(len, Explode(alphabet));\n\n int n = 0;\n int last_l = -1;\n std::string last_s;\n\n if (donull) {\n g.GenerateNULL();\n EXPECT_TRUE(g.HasNext());\n absl::string_view sp = g.Next();\n EXPECT_EQ(sp.data(), static_cast(NULL));\n EXPECT_EQ(sp.size(), size_t{0});\n }\n\n while (g.HasNext()) {\n std::string s = std::string(g.Next());\n n++;\n\n // Check that all characters in s appear in alphabet.\n for (const char *p = s.c_str(); *p != '\\0'; ) {\n Rune r;\n p += chartorune(&r, p);\n EXPECT_TRUE(utfrune(alphabet.c_str(), r) != NULL);\n }\n\n // Check that string is properly ordered w.r.t. previous string.\n int l = utflen(s.c_str());\n EXPECT_LE(l, len);\n if (last_l < l) {\n last_l = l;\n } else {\n EXPECT_EQ(last_l, l);\n EXPECT_LT(last_s, s);\n }\n last_s = s;\n }\n\n // Check total string count.\n int64_t m = 0;\n int alpha = utflen(alphabet.c_str());\n if (alpha == 0) // Degenerate case.\n len = 0;\n for (int i = 0; i <= len; i++)\n m += IntegerPower(alpha, i);\n EXPECT_EQ(n, m);\n}\n\nTEST(StringGenerator, NoLength) {\n RunTest(0, \"abc\", false);\n}\n\nTEST(StringGenerator, NoLengthNoAlphabet) {\n RunTest(0, \"\", false);\n}\n\nTEST(StringGenerator, NoAlphabet) {\n RunTest(5, \"\", false);\n}\n\nTEST(StringGenerator, Simple) {\n RunTest(3, \"abc\", false);\n}\n\nTEST(StringGenerator, UTF8) {\n RunTest(4, \"abc\\xE2\\x98\\xBA\", false);\n}\n\nTEST(StringGenerator, GenNULL) {\n RunTest(0, \"abc\", true);\n RunTest(0, \"\", true);\n RunTest(5, \"\", true);\n RunTest(3, \"abc\", true);\n RunTest(4, \"abc\\xE2\\x98\\xBA\", true);\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/tester.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Regular expression engine tester -- test all the implementations against each other.\n\n#include \"re2/testing/tester.h\"\n\n#include \n#include \n#include \n\n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/flags/flag.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/escaping.h\"\n#include \"absl/strings/str_format.h\"\n#include \"absl/strings/string_view.h\"\n#include \"re2/prog.h\"\n#include \"re2/re2.h\"\n#include \"re2/regexp.h\"\n#include \"util/pcre.h\"\n\nABSL_FLAG(bool, dump_prog, false, \"dump regexp program\");\nABSL_FLAG(bool, log_okay, false, \"log successful runs\");\nABSL_FLAG(bool, dump_rprog, false, \"dump reversed regexp program\");\n\nABSL_FLAG(int, max_regexp_failures, 100,\n \"maximum number of regexp test failures (-1 = unlimited)\");\n\nABSL_FLAG(std::string, regexp_engines, \"\",\n \"pattern to select regexp engines to test\");\n\nnamespace re2 {\n\nenum {\n kMaxSubmatch = 1+16, // $0...$16\n};\n\nconst char* engine_names[kEngineMax] = {\n \"Backtrack\",\n \"NFA\",\n \"DFA\",\n \"DFA1\",\n \"OnePass\",\n \"BitState\",\n \"RE2\",\n \"RE2a\",\n \"RE2b\",\n \"PCRE\",\n};\n\n// Returns the name of the engine.\nstatic const char* EngineName(Engine e) {\n ABSL_CHECK_GE(e, 0);\n ABSL_CHECK_LT(e, ABSL_ARRAYSIZE(engine_names));\n ABSL_CHECK(engine_names[e] != NULL);\n return engine_names[e];\n}\n\n// Returns bit mask of engines to use.\nstatic uint32_t Engines() {\n static bool did_parse = false;\n static uint32_t cached_engines = 0;\n\n if (did_parse)\n return cached_engines;\n\n if (absl::GetFlag(FLAGS_regexp_engines).empty()) {\n cached_engines = ~0;\n } else {\n for (Engine i = static_cast(0); i < kEngineMax; i++)\n if (absl::GetFlag(FLAGS_regexp_engines).find(EngineName(i)) != std::string::npos)\n cached_engines |= 1<(0); i < kEngineMax; i++) {\n if (cached_engines & (1<= 0x80) bytes.\nstatic bool NonASCII(absl::string_view text) {\n for (size_t i = 0; i < text.size(); i++)\n if ((uint8_t)text[i] >= 0x80)\n return true;\n return false;\n}\n\n// Returns string representation of match kind.\nstatic std::string FormatKind(Prog::MatchKind kind) {\n switch (kind) {\n case Prog::kFullMatch:\n return \"full match\";\n case Prog::kLongestMatch:\n return \"longest match\";\n case Prog::kFirstMatch:\n return \"first match\";\n case Prog::kManyMatch:\n return \"many match\";\n }\n return \"???\";\n}\n\n// Returns string representation of anchor kind.\nstatic std::string FormatAnchor(Prog::Anchor anchor) {\n switch (anchor) {\n case Prog::kAnchored:\n return \"anchored\";\n case Prog::kUnanchored:\n return \"unanchored\";\n }\n return \"???\";\n}\n\nstruct ParseMode {\n Regexp::ParseFlags parse_flags;\n std::string desc;\n};\n\nstatic const Regexp::ParseFlags single_line =\n Regexp::LikePerl;\nstatic const Regexp::ParseFlags multi_line =\n static_cast(Regexp::LikePerl & ~Regexp::OneLine);\n\nstatic ParseMode parse_modes[] = {\n { single_line, \"single-line\" },\n { single_line|Regexp::Latin1, \"single-line, latin1\" },\n { multi_line, \"multiline\" },\n { multi_line|Regexp::NonGreedy, \"multiline, nongreedy\" },\n { multi_line|Regexp::Latin1, \"multiline, latin1\" },\n};\n\nstatic std::string FormatMode(Regexp::ParseFlags flags) {\n for (size_t i = 0; i < ABSL_ARRAYSIZE(parse_modes); i++)\n if (parse_modes[i].parse_flags == flags)\n return parse_modes[i].desc;\n return absl::StrFormat(\"%#x\", static_cast(flags));\n}\n\n// Constructs and saves all the matching engines that\n// will be required for the given tests.\nTestInstance::TestInstance(absl::string_view regexp_str, Prog::MatchKind kind,\n Regexp::ParseFlags flags)\n : regexp_str_(regexp_str),\n kind_(kind),\n flags_(flags),\n error_(false),\n regexp_(NULL),\n num_captures_(0),\n prog_(NULL),\n rprog_(NULL),\n re_(NULL),\n re2_(NULL) {\n\n ABSL_VLOG(1) << absl::CEscape(regexp_str);\n\n // Compile regexp to prog.\n // Always required - needed for backtracking (reference implementation).\n RegexpStatus status;\n regexp_ = Regexp::Parse(regexp_str, flags, &status);\n if (regexp_ == NULL) {\n ABSL_LOG(INFO) << \"Cannot parse: \" << absl::CEscape(regexp_str_)\n << \" mode: \" << FormatMode(flags);\n error_ = true;\n return;\n }\n num_captures_ = regexp_->NumCaptures();\n prog_ = regexp_->CompileToProg(0);\n if (prog_ == NULL) {\n ABSL_LOG(INFO) << \"Cannot compile: \" << absl::CEscape(regexp_str_);\n error_ = true;\n return;\n }\n if (absl::GetFlag(FLAGS_dump_prog)) {\n ABSL_LOG(INFO) << \"Prog for \"\n << \" regexp \"\n << absl::CEscape(regexp_str_)\n << \" (\" << FormatKind(kind_)\n << \", \" << FormatMode(flags_)\n << \")\\n\"\n << prog_->Dump();\n }\n\n // Compile regexp to reversed prog. Only needed for DFA engines.\n if (Engines() & ((1<CompileToReverseProg(0);\n if (rprog_ == NULL) {\n ABSL_LOG(INFO) << \"Cannot reverse compile: \"\n << absl::CEscape(regexp_str_);\n error_ = true;\n return;\n }\n if (absl::GetFlag(FLAGS_dump_rprog))\n ABSL_LOG(INFO) << rprog_->Dump();\n }\n\n // Create re string that will be used for RE and RE2.\n std::string re = std::string(regexp_str);\n // Accomodate flags.\n // Regexp::Latin1 will be accomodated below.\n if (!(flags & Regexp::OneLine))\n re = \"(?m)\" + re;\n if (flags & Regexp::NonGreedy)\n re = \"(?U)\" + re;\n if (flags & Regexp::DotNL)\n re = \"(?s)\" + re;\n\n // Compile regexp to RE2.\n if (Engines() & ((1<error().empty()) {\n ABSL_LOG(INFO) << \"Cannot RE2: \" << absl::CEscape(re);\n error_ = true;\n return;\n }\n }\n\n // Compile regexp to RE.\n // PCRE as exposed by the RE interface isn't always usable.\n // 1. It disagrees about handling of empty-string reptitions\n // like matching (a*)* against \"b\". PCRE treats the (a*) as\n // occurring once, while we treat it as occurring not at all.\n // 2. It treats $ as this weird thing meaning end of string\n // or before the \\n at the end of the string.\n // 3. It doesn't implement POSIX leftmost-longest matching.\n // 4. It lets \\s match vertical tab.\n // MimicsPCRE() detects 1 and 2.\n if ((Engines() & (1<MimicsPCRE() &&\n kind_ != Prog::kLongestMatch) {\n PCRE_Options o;\n o.set_option(PCRE::UTF8);\n if (flags & Regexp::Latin1)\n o.set_option(PCRE::None);\n // PCRE has interface bug keeping us from finding $0, so\n // add one more layer of parens.\n re_ = new PCRE(\"(\"+re+\")\", o);\n if (!re_->error().empty()) {\n ABSL_LOG(INFO) << \"Cannot PCRE: \" << absl::CEscape(re);\n error_ = true;\n return;\n }\n }\n}\n\nTestInstance::~TestInstance() {\n if (regexp_)\n regexp_->Decref();\n delete prog_;\n delete rprog_;\n delete re_;\n delete re2_;\n}\n\n// Runs a single search using the named engine type.\n// This interface hides all the irregularities of the various\n// engine interfaces from the rest of this file.\nvoid TestInstance::RunSearch(Engine type, absl::string_view orig_text,\n absl::string_view orig_context,\n Prog::Anchor anchor, Result* result) {\n if (regexp_ == NULL) {\n result->skipped = true;\n return;\n }\n int nsubmatch = 1 + num_captures_; // NumCaptures doesn't count $0\n if (nsubmatch > kMaxSubmatch)\n nsubmatch = kMaxSubmatch;\n\n absl::string_view text = orig_text;\n absl::string_view context = orig_context;\n\n switch (type) {\n default:\n ABSL_LOG(FATAL) << \"Bad RunSearch type: \" << (int)type;\n\n case kEngineBacktrack:\n if (prog_ == NULL) {\n result->skipped = true;\n break;\n }\n result->matched =\n prog_->UnsafeSearchBacktrack(text, context, anchor, kind_,\n result->submatch, nsubmatch);\n result->have_submatch = true;\n break;\n\n case kEngineNFA:\n if (prog_ == NULL) {\n result->skipped = true;\n break;\n }\n result->matched =\n prog_->SearchNFA(text, context, anchor, kind_,\n result->submatch, nsubmatch);\n result->have_submatch = true;\n break;\n\n case kEngineDFA:\n if (prog_ == NULL) {\n result->skipped = true;\n break;\n }\n result->matched = prog_->SearchDFA(text, context, anchor, kind_, NULL,\n &result->skipped, NULL);\n break;\n\n case kEngineDFA1:\n if (prog_ == NULL || rprog_ == NULL) {\n result->skipped = true;\n break;\n }\n result->matched =\n prog_->SearchDFA(text, context, anchor, kind_, result->submatch,\n &result->skipped, NULL);\n // If anchored, no need for second run,\n // but do it anyway to find more bugs.\n if (result->matched) {\n if (!rprog_->SearchDFA(result->submatch[0], context,\n Prog::kAnchored, Prog::kLongestMatch,\n result->submatch,\n &result->skipped, NULL)) {\n ABSL_LOG(ERROR) << \"Reverse DFA inconsistency: \"\n << absl::CEscape(regexp_str_)\n << \" on \" << absl::CEscape(text);\n result->matched = false;\n }\n }\n result->have_submatch0 = true;\n break;\n\n case kEngineOnePass:\n if (prog_ == NULL ||\n !prog_->IsOnePass() ||\n anchor == Prog::kUnanchored ||\n nsubmatch > Prog::kMaxOnePassCapture) {\n result->skipped = true;\n break;\n }\n result->matched = prog_->SearchOnePass(text, context, anchor, kind_,\n result->submatch, nsubmatch);\n result->have_submatch = true;\n break;\n\n case kEngineBitState:\n if (prog_ == NULL ||\n !prog_->CanBitState()) {\n result->skipped = true;\n break;\n }\n result->matched = prog_->SearchBitState(text, context, anchor, kind_,\n result->submatch, nsubmatch);\n result->have_submatch = true;\n break;\n\n case kEngineRE2:\n case kEngineRE2a:\n case kEngineRE2b: {\n if (!re2_ || EndPtr(text) != EndPtr(context)) {\n result->skipped = true;\n break;\n }\n\n RE2::Anchor re_anchor;\n if (anchor == Prog::kAnchored)\n re_anchor = RE2::ANCHOR_START;\n else\n re_anchor = RE2::UNANCHORED;\n if (kind_ == Prog::kFullMatch)\n re_anchor = RE2::ANCHOR_BOTH;\n\n result->matched = re2_->Match(\n context,\n static_cast(BeginPtr(text) - BeginPtr(context)),\n static_cast(EndPtr(text) - BeginPtr(context)),\n re_anchor,\n result->submatch,\n nsubmatch);\n result->have_submatch = nsubmatch > 0;\n break;\n }\n\n case kEnginePCRE: {\n if (!re_ || BeginPtr(text) != BeginPtr(context) ||\n EndPtr(text) != EndPtr(context)) {\n result->skipped = true;\n break;\n }\n\n // In Perl/PCRE, \\v matches any character considered vertical\n // whitespace, not just vertical tab. Regexp::MimicsPCRE() is\n // unable to handle all cases of this, unfortunately, so just\n // catch them here. :(\n if (regexp_str_.find(\"\\\\v\") != absl::string_view::npos &&\n (text.find('\\n') != absl::string_view::npos ||\n text.find('\\f') != absl::string_view::npos ||\n text.find('\\r') != absl::string_view::npos)) {\n result->skipped = true;\n break;\n }\n\n // PCRE 8.34 or so started allowing vertical tab to match \\s,\n // following a change made in Perl 5.18. RE2 does not.\n if ((regexp_str_.find(\"\\\\s\") != absl::string_view::npos ||\n regexp_str_.find(\"\\\\S\") != absl::string_view::npos) &&\n text.find('\\v') != absl::string_view::npos) {\n result->skipped = true;\n break;\n }\n\n const PCRE::Arg **argptr = new const PCRE::Arg*[nsubmatch];\n PCRE::Arg *a = new PCRE::Arg[nsubmatch];\n for (int i = 0; i < nsubmatch; i++) {\n a[i] = PCRE::Arg(&result->submatch[i]);\n argptr[i] = &a[i];\n }\n size_t consumed;\n PCRE::Anchor pcre_anchor;\n if (anchor == Prog::kAnchored)\n pcre_anchor = PCRE::ANCHOR_START;\n else\n pcre_anchor = PCRE::UNANCHORED;\n if (kind_ == Prog::kFullMatch)\n pcre_anchor = PCRE::ANCHOR_BOTH;\n re_->ClearHitLimit();\n result->matched =\n re_->DoMatch(text,\n pcre_anchor,\n &consumed,\n argptr, nsubmatch);\n if (re_->HitLimit()) {\n result->untrusted = true;\n delete[] argptr;\n delete[] a;\n break;\n }\n result->have_submatch = true;\n delete[] argptr;\n delete[] a;\n break;\n }\n }\n\n if (!result->matched)\n result->ClearSubmatch();\n}\n\n// Checks whether r is okay given that correct is the right answer.\n// Specifically, r's answers have to match (but it doesn't have to\n// claim to have all the answers).\nstatic bool ResultOkay(const Result& r, const Result& correct) {\n if (r.skipped)\n return true;\n if (r.matched != correct.matched)\n return false;\n if (r.have_submatch || r.have_submatch0) {\n for (int i = 0; i < kMaxSubmatch; i++) {\n if (correct.submatch[i].data() != r.submatch[i].data() ||\n correct.submatch[i].size() != r.submatch[i].size())\n return false;\n if (!r.have_submatch)\n break;\n }\n }\n return true;\n}\n\n// Runs a single test.\nbool TestInstance::RunCase(absl::string_view text, absl::string_view context,\n Prog::Anchor anchor) {\n // Backtracking is the gold standard.\n Result correct;\n RunSearch(kEngineBacktrack, text, context, anchor, &correct);\n if (correct.skipped) {\n if (regexp_ == NULL)\n return true;\n ABSL_LOG(ERROR) << \"Skipped backtracking! \" << absl::CEscape(regexp_str_)\n << \" \" << FormatMode(flags_);\n return false;\n }\n ABSL_VLOG(1) << \"Try: regexp \" << absl::CEscape(regexp_str_)\n << \" text \" << absl::CEscape(text)\n << \" (\" << FormatKind(kind_)\n << \", \" << FormatAnchor(anchor)\n << \", \" << FormatMode(flags_)\n << \")\";\n\n // Compare the others.\n bool all_okay = true;\n for (Engine i = kEngineBacktrack+1; i < kEngineMax; i++) {\n if (!(Engines() & (1< 0 && --max_regexp_failures == 0)\n ABSL_LOG(QFATAL) << \"Too many regexp failures.\";\n }\n\n return all_okay;\n}\n\nvoid TestInstance::LogMatch(const char* prefix, Engine e,\n absl::string_view text, absl::string_view context,\n Prog::Anchor anchor) {\n ABSL_LOG(INFO) << prefix\n << EngineName(e)\n << \" regexp \"\n << absl::CEscape(regexp_str_)\n << \" \"\n << absl::CEscape(regexp_->ToString())\n << \" text \"\n << absl::CEscape(text)\n << \" (\"\n << BeginPtr(text) - BeginPtr(context)\n << \",\"\n << EndPtr(text) - BeginPtr(context)\n << \") of context \"\n << absl::CEscape(context)\n << \" (\" << FormatKind(kind_)\n << \", \" << FormatAnchor(anchor)\n << \", \" << FormatMode(flags_)\n << \")\";\n}\n\nstatic Prog::MatchKind kinds[] = {\n Prog::kFirstMatch,\n Prog::kLongestMatch,\n Prog::kFullMatch,\n};\n\n// Test all possible match kinds and parse modes.\nTester::Tester(absl::string_view regexp) {\n error_ = false;\n for (size_t i = 0; i < ABSL_ARRAYSIZE(kinds); i++) {\n for (size_t j = 0; j < ABSL_ARRAYSIZE(parse_modes); j++) {\n TestInstance* t = new TestInstance(regexp, kinds[i],\n parse_modes[j].parse_flags);\n error_ |= t->error();\n v_.push_back(t);\n }\n }\n}\n\nTester::~Tester() {\n for (size_t i = 0; i < v_.size(); i++)\n delete v_[i];\n}\n\nbool Tester::TestCase(absl::string_view text, absl::string_view context,\n Prog::Anchor anchor) {\n bool okay = true;\n for (size_t i = 0; i < v_.size(); i++)\n okay &= (!v_[i]->error() && v_[i]->RunCase(text, context, anchor));\n return okay;\n}\n\nstatic Prog::Anchor anchors[] = {\n Prog::kAnchored,\n Prog::kUnanchored\n};\n\nbool Tester::TestInput(absl::string_view text) {\n bool okay = TestInputInContext(text, text);\n if (!text.empty()) {\n absl::string_view sp;\n sp = text;\n sp.remove_prefix(1);\n okay &= TestInputInContext(sp, text);\n sp = text;\n sp.remove_suffix(1);\n okay &= TestInputInContext(sp, text);\n }\n return okay;\n}\n\nbool Tester::TestInputInContext(absl::string_view text,\n absl::string_view context) {\n bool okay = true;\n for (size_t i = 0; i < ABSL_ARRAYSIZE(anchors); i++)\n okay &= TestCase(text, context, anchors[i]);\n return okay;\n}\n\nbool TestRegexpOnText(absl::string_view regexp,\n absl::string_view text) {\n Tester t(regexp);\n return t.TestInput(text);\n}\n\n} // namespace re2\n" }, { "path": "re2/testing/tester.h", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_TESTING_TESTER_H_\n#define RE2_TESTING_TESTER_H_\n\n// Comparative tester for regular expression matching.\n// Checks all implementations against each other.\n\n#include \n\n#include \"absl/strings/string_view.h\"\n#include \"re2/prog.h\"\n#include \"re2/re2.h\"\n#include \"re2/regexp.h\"\n#include \"util/pcre.h\"\n\nnamespace re2 {\n\n// All the supported regexp engines.\nenum Engine {\n kEngineBacktrack = 0, // Prog::UnsafeSearchBacktrack\n kEngineNFA, // Prog::SearchNFA\n kEngineDFA, // Prog::SearchDFA, only ask whether it matched\n kEngineDFA1, // Prog::SearchDFA, ask for match[0]\n kEngineOnePass, // Prog::SearchOnePass, if applicable\n kEngineBitState, // Prog::SearchBitState\n kEngineRE2, // RE2, all submatches\n kEngineRE2a, // RE2, only ask for match[0]\n kEngineRE2b, // RE2, only ask whether it matched\n kEnginePCRE, // PCRE (util/pcre.h)\n\n kEngineMax,\n};\n\n// Make normal math on the enum preserve the type.\n// By default, C++ doesn't define ++ on enum, and e+1 has type int.\nstatic inline void operator++(Engine& e, int unused) {\n e = static_cast(e+1);\n}\n\nstatic inline Engine operator+(Engine e, int i) {\n return static_cast(static_cast(e)+i);\n}\n\n// A TestInstance caches per-regexp state for a given\n// regular expression in a given configuration\n// (UTF-8 vs Latin1, longest vs first match, etc.).\nclass TestInstance {\n public:\n struct Result;\n\n TestInstance(absl::string_view regexp, Prog::MatchKind kind,\n Regexp::ParseFlags flags);\n ~TestInstance();\n Regexp::ParseFlags flags() { return flags_; }\n bool error() { return error_; }\n\n // Runs a single test case: search in text, which is in context,\n // using the given anchoring.\n bool RunCase(absl::string_view text, absl::string_view context,\n Prog::Anchor anchor);\n\n private:\n // Runs a single search using the named engine type.\n void RunSearch(Engine type, absl::string_view text, absl::string_view context,\n Prog::Anchor anchor, Result* result);\n\n void LogMatch(const char* prefix, Engine e, absl::string_view text,\n absl::string_view context, Prog::Anchor anchor);\n\n absl::string_view regexp_str_; // regexp being tested\n Prog::MatchKind kind_; // kind of match\n Regexp::ParseFlags flags_; // flags for parsing regexp_str_\n bool error_; // error during constructor?\n\n Regexp* regexp_; // parsed regexp\n int num_captures_; // regexp_->NumCaptures() cached\n Prog* prog_; // compiled program\n Prog* rprog_; // compiled reverse program\n PCRE* re_; // PCRE implementation\n RE2* re2_; // RE2 implementation\n\n TestInstance(const TestInstance&) = delete;\n TestInstance& operator=(const TestInstance&) = delete;\n};\n\n// A group of TestInstances for all possible configurations.\nclass Tester {\n public:\n explicit Tester(absl::string_view regexp);\n ~Tester();\n\n bool error() { return error_; }\n\n // Runs a single test case: search in text, which is in context,\n // using the given anchoring.\n bool TestCase(absl::string_view text, absl::string_view context,\n Prog::Anchor anchor);\n\n // Run TestCase(text, text, anchor) for all anchoring modes.\n bool TestInput(absl::string_view text);\n\n // Run TestCase(text, context, anchor) for all anchoring modes.\n bool TestInputInContext(absl::string_view text, absl::string_view context);\n\n private:\n bool error_;\n std::vector v_;\n\n Tester(const Tester&) = delete;\n Tester& operator=(const Tester&) = delete;\n};\n\n// Run all possible tests using regexp and text.\nbool TestRegexpOnText(absl::string_view regexp, absl::string_view text);\n\n} // namespace re2\n\n#endif // RE2_TESTING_TESTER_H_\n" }, { "path": "re2/tostring.cc", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Format a regular expression structure as a string.\n// Tested by parse_test.cc\n\n#include \n\n#include \n\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"re2/regexp.h\"\n#include \"re2/walker-inl.h\"\n#include \"util/utf.h\"\n\nnamespace re2 {\n\nenum {\n PrecAtom,\n PrecUnary,\n PrecConcat,\n PrecAlternate,\n PrecEmpty,\n PrecParen,\n PrecToplevel,\n};\n\n// Helper function. See description below.\nstatic void AppendCCRange(std::string* t, Rune lo, Rune hi);\n\n// Walker to generate string in s_.\n// The arg pointers are actually integers giving the\n// context precedence.\n// The child_args are always NULL.\nclass ToStringWalker : public Regexp::Walker {\n public:\n explicit ToStringWalker(std::string* t) : t_(t) {}\n\n virtual int PreVisit(Regexp* re, int parent_arg, bool* stop);\n virtual int PostVisit(Regexp* re, int parent_arg, int pre_arg,\n int* child_args, int nchild_args);\n virtual int ShortVisit(Regexp* re, int parent_arg) {\n return 0;\n }\n\n private:\n std::string* t_; // The string the walker appends to.\n\n ToStringWalker(const ToStringWalker&) = delete;\n ToStringWalker& operator=(const ToStringWalker&) = delete;\n};\n\nstd::string Regexp::ToString() {\n std::string t;\n ToStringWalker w(&t);\n w.WalkExponential(this, PrecToplevel, 100000);\n if (w.stopped_early())\n t += \" [truncated]\";\n return t;\n}\n\n#define ToString DontCallToString // Avoid accidental recursion.\n\n// Visits re before children are processed.\n// Appends ( if needed and passes new precedence to children.\nint ToStringWalker::PreVisit(Regexp* re, int parent_arg, bool* stop) {\n int prec = parent_arg;\n int nprec = PrecAtom;\n\n switch (re->op()) {\n case kRegexpNoMatch:\n case kRegexpEmptyMatch:\n case kRegexpLiteral:\n case kRegexpAnyChar:\n case kRegexpAnyByte:\n case kRegexpBeginLine:\n case kRegexpEndLine:\n case kRegexpBeginText:\n case kRegexpEndText:\n case kRegexpWordBoundary:\n case kRegexpNoWordBoundary:\n case kRegexpCharClass:\n case kRegexpHaveMatch:\n nprec = PrecAtom;\n break;\n\n case kRegexpConcat:\n case kRegexpLiteralString:\n if (prec < PrecConcat)\n t_->append(\"(?:\");\n nprec = PrecConcat;\n break;\n\n case kRegexpAlternate:\n if (prec < PrecAlternate)\n t_->append(\"(?:\");\n nprec = PrecAlternate;\n break;\n\n case kRegexpCapture:\n t_->append(\"(\");\n if (re->cap() == 0)\n ABSL_LOG(DFATAL) << \"kRegexpCapture cap() == 0\";\n if (re->name()) {\n t_->append(\"?P<\");\n t_->append(*re->name());\n t_->append(\">\");\n }\n nprec = PrecParen;\n break;\n\n case kRegexpStar:\n case kRegexpPlus:\n case kRegexpQuest:\n case kRegexpRepeat:\n if (prec < PrecUnary)\n t_->append(\"(?:\");\n // The subprecedence here is PrecAtom instead of PrecUnary\n // because PCRE treats two unary ops in a row as a parse error.\n nprec = PrecAtom;\n break;\n }\n\n return nprec;\n}\n\nstatic void AppendLiteral(std::string *t, Rune r, bool foldcase) {\n if (r != 0 && r < 0x80 && strchr(\"(){}[]*+?|.^$\\\\\", r)) {\n t->append(1, '\\\\');\n t->append(1, static_cast(r));\n } else if (foldcase && 'a' <= r && r <= 'z') {\n r -= 'a' - 'A';\n t->append(1, '[');\n t->append(1, static_cast(r));\n t->append(1, static_cast(r) + 'a' - 'A');\n t->append(1, ']');\n } else {\n AppendCCRange(t, r, r);\n }\n}\n\n// Visits re after children are processed.\n// For childless regexps, all the work is done here.\n// For regexps with children, append any unary suffixes or ).\nint ToStringWalker::PostVisit(Regexp* re, int parent_arg, int pre_arg,\n int* child_args, int nchild_args) {\n int prec = parent_arg;\n switch (re->op()) {\n case kRegexpNoMatch:\n // There's no simple symbol for \"no match\", but\n // [^0-Runemax] excludes everything.\n t_->append(\"[^\\\\x00-\\\\x{10ffff}]\");\n break;\n\n case kRegexpEmptyMatch:\n // Append (?:) to make empty string visible,\n // unless this is already being parenthesized.\n if (prec < PrecEmpty)\n t_->append(\"(?:)\");\n break;\n\n case kRegexpLiteral:\n AppendLiteral(t_, re->rune(),\n (re->parse_flags() & Regexp::FoldCase) != 0);\n break;\n\n case kRegexpLiteralString:\n for (int i = 0; i < re->nrunes(); i++)\n AppendLiteral(t_, re->runes()[i],\n (re->parse_flags() & Regexp::FoldCase) != 0);\n if (prec < PrecConcat)\n t_->append(\")\");\n break;\n\n case kRegexpConcat:\n if (prec < PrecConcat)\n t_->append(\")\");\n break;\n\n case kRegexpAlternate:\n // Clumsy but workable: the children all appended |\n // at the end of their strings, so just remove the last one.\n if ((*t_)[t_->size()-1] == '|')\n t_->erase(t_->size()-1);\n else\n ABSL_LOG(DFATAL) << \"Bad final char: \" << t_;\n if (prec < PrecAlternate)\n t_->append(\")\");\n break;\n\n case kRegexpStar:\n t_->append(\"*\");\n if (re->parse_flags() & Regexp::NonGreedy)\n t_->append(\"?\");\n if (prec < PrecUnary)\n t_->append(\")\");\n break;\n\n case kRegexpPlus:\n t_->append(\"+\");\n if (re->parse_flags() & Regexp::NonGreedy)\n t_->append(\"?\");\n if (prec < PrecUnary)\n t_->append(\")\");\n break;\n\n case kRegexpQuest:\n t_->append(\"?\");\n if (re->parse_flags() & Regexp::NonGreedy)\n t_->append(\"?\");\n if (prec < PrecUnary)\n t_->append(\")\");\n break;\n\n case kRegexpRepeat:\n if (re->max() == -1)\n t_->append(absl::StrFormat(\"{%d,}\", re->min()));\n else if (re->min() == re->max())\n t_->append(absl::StrFormat(\"{%d}\", re->min()));\n else\n t_->append(absl::StrFormat(\"{%d,%d}\", re->min(), re->max()));\n if (re->parse_flags() & Regexp::NonGreedy)\n t_->append(\"?\");\n if (prec < PrecUnary)\n t_->append(\")\");\n break;\n\n case kRegexpAnyChar:\n t_->append(\".\");\n break;\n\n case kRegexpAnyByte:\n t_->append(\"\\\\C\");\n break;\n\n case kRegexpBeginLine:\n t_->append(\"^\");\n break;\n\n case kRegexpEndLine:\n t_->append(\"$\");\n break;\n\n case kRegexpBeginText:\n t_->append(\"(?-m:^)\");\n break;\n\n case kRegexpEndText:\n if (re->parse_flags() & Regexp::WasDollar)\n t_->append(\"(?-m:$)\");\n else\n t_->append(\"\\\\z\");\n break;\n\n case kRegexpWordBoundary:\n t_->append(\"\\\\b\");\n break;\n\n case kRegexpNoWordBoundary:\n t_->append(\"\\\\B\");\n break;\n\n case kRegexpCharClass: {\n if (re->cc()->size() == 0) {\n t_->append(\"[^\\\\x00-\\\\x{10ffff}]\");\n break;\n }\n t_->append(\"[\");\n // Heuristic: show class as negated if it contains the\n // non-character 0xFFFE and yet somehow isn't full.\n CharClass* cc = re->cc();\n if (cc->Contains(0xFFFE) && !cc->full()) {\n cc = cc->Negate();\n t_->append(\"^\");\n }\n for (CharClass::iterator i = cc->begin(); i != cc->end(); ++i)\n AppendCCRange(t_, i->lo, i->hi);\n if (cc != re->cc())\n cc->Delete();\n t_->append(\"]\");\n break;\n }\n\n case kRegexpCapture:\n t_->append(\")\");\n break;\n\n case kRegexpHaveMatch:\n // There's no syntax accepted by the parser to generate\n // this node (it is generated by RE2::Set) so make something\n // up that is readable but won't compile.\n t_->append(absl::StrFormat(\"(?HaveMatch:%d)\", re->match_id()));\n break;\n }\n\n // If the parent is an alternation, append the | for it.\n if (prec == PrecAlternate)\n t_->append(\"|\");\n\n return 0;\n}\n\n// Appends a rune for use in a character class to the string t.\nstatic void AppendCCChar(std::string* t, Rune r) {\n if (0x20 <= r && r <= 0x7E) {\n if (strchr(\"[]^-\\\\\", r))\n t->append(\"\\\\\");\n t->append(1, static_cast(r));\n return;\n }\n switch (r) {\n default:\n break;\n\n case '\\r':\n t->append(\"\\\\r\");\n return;\n\n case '\\t':\n t->append(\"\\\\t\");\n return;\n\n case '\\n':\n t->append(\"\\\\n\");\n return;\n\n case '\\f':\n t->append(\"\\\\f\");\n return;\n }\n\n if (r < 0x100) {\n *t += absl::StrFormat(\"\\\\x%02x\", static_cast(r));\n return;\n }\n *t += absl::StrFormat(\"\\\\x{%x}\", static_cast(r));\n}\n\nstatic void AppendCCRange(std::string* t, Rune lo, Rune hi) {\n if (lo > hi)\n return;\n AppendCCChar(t, lo);\n if (lo < hi) {\n t->append(\"-\");\n AppendCCChar(t, hi);\n }\n}\n\n} // namespace re2\n" }, { "path": "re2/unicode.py", "content": "# Copyright 2008 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n\"\"\"Parser for Unicode data files (as distributed by unicode.org).\"\"\"\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport os\nimport re\nimport urllib.request\n\n# Directory or URL where Unicode tables reside.\n_UNICODE_DIR = \"https://www.unicode.org/Public/15.1.0/ucd\"\n\n# Largest valid Unicode code value.\n_RUNE_MAX = 0x10FFFF\n\n\nclass Error(Exception):\n \"\"\"Unicode error base class.\"\"\"\n\n\nclass InputError(Error):\n \"\"\"Unicode input error class. Raised on invalid input.\"\"\"\n\n\ndef _UInt(s):\n \"\"\"Converts string to Unicode code point ('263A' => 0x263a).\n\n Args:\n s: string to convert\n\n Returns:\n Unicode code point\n\n Raises:\n InputError: the string is not a valid Unicode value.\n \"\"\"\n\n try:\n v = int(s, 16)\n except ValueError:\n v = -1\n if len(s) < 4 or len(s) > 6 or v < 0 or v > _RUNE_MAX:\n raise InputError(\"invalid Unicode value %s\" % (s,))\n return v\n\n\ndef _URange(s):\n \"\"\"Converts string to Unicode range.\n\n '0001..0003' => [1, 2, 3].\n '0001' => [1].\n\n Args:\n s: string to convert\n\n Returns:\n Unicode range\n\n Raises:\n InputError: the string is not a valid Unicode range.\n \"\"\"\n a = s.split(\"..\")\n if len(a) == 1:\n return [_UInt(a[0])]\n if len(a) == 2:\n lo = _UInt(a[0])\n hi = _UInt(a[1])\n if lo < hi:\n return range(lo, hi + 1)\n raise InputError(\"invalid Unicode range %s\" % (s,))\n\n\ndef _ParseContinue(s):\n \"\"\"Parses a Unicode continuation field.\n\n These are of the form '' or ''.\n Instead of giving an explicit range in a single table entry,\n some Unicode tables use two entries, one for the first\n code value in the range and one for the last.\n The first entry's description is '' instead of 'Name'\n and the second is ''.\n\n '' => ('Name', 'First')\n '' => ('Name', 'Last')\n 'Anything else' => ('Anything else', None)\n\n Args:\n s: continuation field string\n\n Returns:\n pair: name and ('First', 'Last', or None)\n \"\"\"\n\n match = re.match(\"<(.*), (First|Last)>\", s)\n if match is not None:\n return match.groups()\n return (s, None)\n\n\ndef ReadUnicodeTable(filename, nfields, doline):\n \"\"\"Generic Unicode table text file reader.\n\n The reader takes care of stripping out comments and also\n parsing the two different ways that the Unicode tables specify\n code ranges (using the .. notation and splitting the range across\n multiple lines).\n\n Each non-comment line in the table is expected to have the given\n number of fields. The first field is known to be the Unicode value\n and the second field its description.\n\n The reader calls doline(codes, fields) for each entry in the table.\n If fn raises an exception, the reader prints that exception,\n prefixed with the file name and line number, and continues\n processing the file. When done with the file, the reader re-raises\n the first exception encountered during the file.\n\n Arguments:\n filename: the Unicode data file to read, or a file-like object.\n nfields: the number of expected fields per line in that file.\n doline: the function to call for each table entry.\n\n Raises:\n InputError: nfields is invalid (must be >= 2).\n \"\"\"\n\n if nfields < 2:\n raise InputError(\"invalid number of fields %d\" % (nfields,))\n\n if type(filename) == str:\n if filename.startswith(\"https://\"):\n fil = urllib.request.urlopen(filename)\n else:\n fil = open(filename, \"rb\")\n else:\n fil = filename\n\n first = None # first code in multiline range\n expect_last = None # tag expected for \"Last\" line in multiline range\n lineno = 0 # current line number\n for line in fil:\n lineno += 1\n try:\n line = line.decode('latin1')\n\n # Chop # comments and white space; ignore empty lines.\n sharp = line.find(\"#\")\n if sharp >= 0:\n line = line[:sharp]\n line = line.strip()\n if not line:\n continue\n\n # Split fields on \";\", chop more white space.\n # Must have the expected number of fields.\n fields = [s.strip() for s in line.split(\";\")]\n if len(fields) != nfields:\n raise InputError(\"wrong number of fields %d %d - %s\" %\n (len(fields), nfields, line))\n\n # The Unicode text files have two different ways\n # to list a Unicode range. Either the first field is\n # itself a range (0000..FFFF), or the range is split\n # across two lines, with the second field noting\n # the continuation.\n codes = _URange(fields[0])\n (name, cont) = _ParseContinue(fields[1])\n\n if expect_last is not None:\n # If the last line gave the First code in a range,\n # this one had better give the Last one.\n if (len(codes) != 1 or codes[0] <= first or\n cont != \"Last\" or name != expect_last):\n raise InputError(\"expected Last line for %s\" %\n (expect_last,))\n codes = range(first, codes[0] + 1)\n first = None\n expect_last = None\n fields[0] = \"%04X..%04X\" % (codes[0], codes[-1])\n fields[1] = name\n elif cont == \"First\":\n # Otherwise, if this is the First code in a range,\n # remember it and go to the next line.\n if len(codes) != 1:\n raise InputError(\"bad First line: range given\")\n expect_last = name\n first = codes[0]\n continue\n\n doline(codes, fields)\n\n except Exception as e:\n print(\"%s:%d: %s\" % (filename, lineno, e))\n raise\n\n if expect_last is not None:\n raise InputError(\"expected Last line for %s; got EOF\" %\n (expect_last,))\n\n\ndef CaseGroups(unicode_dir=_UNICODE_DIR):\n \"\"\"Returns list of Unicode code groups equivalent under case folding.\n\n Each group is a sorted list of code points,\n and the list of groups is sorted by first code point\n in the group.\n\n Args:\n unicode_dir: Unicode data directory\n\n Returns:\n list of Unicode code groups\n \"\"\"\n\n # Dict mapping lowercase code point to fold-equivalent group.\n togroup = {}\n\n def DoLine(codes, fields):\n \"\"\"Process single CaseFolding.txt line, updating togroup.\"\"\"\n (_, foldtype, lower, _) = fields\n if foldtype not in (\"C\", \"S\"):\n return\n lower = _UInt(lower)\n togroup.setdefault(lower, [lower]).extend(codes)\n\n ReadUnicodeTable(unicode_dir+\"/CaseFolding.txt\", 4, DoLine)\n\n groups = list(togroup.values())\n for g in groups:\n g.sort()\n groups.sort()\n return togroup, groups\n\n\ndef Scripts(unicode_dir=_UNICODE_DIR):\n \"\"\"Returns dict mapping script names to code lists.\n\n Args:\n unicode_dir: Unicode data directory\n\n Returns:\n dict mapping script names to code lists\n \"\"\"\n\n scripts = {}\n\n def DoLine(codes, fields):\n \"\"\"Process single Scripts.txt line, updating scripts.\"\"\"\n (_, name) = fields\n scripts.setdefault(name, []).extend(codes)\n\n ReadUnicodeTable(unicode_dir+\"/Scripts.txt\", 2, DoLine)\n return scripts\n\n\ndef Categories(unicode_dir=_UNICODE_DIR):\n \"\"\"Returns dict mapping category names to code lists.\n\n Args:\n unicode_dir: Unicode data directory\n\n Returns:\n dict mapping category names to code lists\n \"\"\"\n\n categories = {}\n\n def DoLine(codes, fields):\n \"\"\"Process single UnicodeData.txt line, updating categories.\"\"\"\n category = fields[2]\n categories.setdefault(category, []).extend(codes)\n # Add codes from Lu into L, etc.\n if len(category) > 1:\n short = category[0]\n categories.setdefault(short, []).extend(codes)\n\n ReadUnicodeTable(unicode_dir+\"/UnicodeData.txt\", 15, DoLine)\n return categories\n\n" }, { "path": "re2/unicode_casefold.cc", "content": "\n// GENERATED BY make_unicode_casefold.py; DO NOT EDIT.\n// make_unicode_casefold.py >unicode_casefold.cc\n\n#include \"re2/unicode_casefold.h\"\n\nnamespace re2 {\n\n\n// 1427 groups, 2884 pairs, 372 ranges\nconst CaseFold unicode_casefold[] = {\n\t{ 65, 90, 32 },\n\t{ 97, 106, -32 },\n\t{ 107, 107, 8383 },\n\t{ 108, 114, -32 },\n\t{ 115, 115, 268 },\n\t{ 116, 122, -32 },\n\t{ 181, 181, 743 },\n\t{ 192, 214, 32 },\n\t{ 216, 222, 32 },\n\t{ 223, 223, 7615 },\n\t{ 224, 228, -32 },\n\t{ 229, 229, 8262 },\n\t{ 230, 246, -32 },\n\t{ 248, 254, -32 },\n\t{ 255, 255, 121 },\n\t{ 256, 303, EvenOdd },\n\t{ 306, 311, EvenOdd },\n\t{ 313, 328, OddEven },\n\t{ 330, 375, EvenOdd },\n\t{ 376, 376, -121 },\n\t{ 377, 382, OddEven },\n\t{ 383, 383, -300 },\n\t{ 384, 384, 195 },\n\t{ 385, 385, 210 },\n\t{ 386, 389, EvenOdd },\n\t{ 390, 390, 206 },\n\t{ 391, 392, OddEven },\n\t{ 393, 394, 205 },\n\t{ 395, 396, OddEven },\n\t{ 398, 398, 79 },\n\t{ 399, 399, 202 },\n\t{ 400, 400, 203 },\n\t{ 401, 402, OddEven },\n\t{ 403, 403, 205 },\n\t{ 404, 404, 207 },\n\t{ 405, 405, 97 },\n\t{ 406, 406, 211 },\n\t{ 407, 407, 209 },\n\t{ 408, 409, EvenOdd },\n\t{ 410, 410, 163 },\n\t{ 412, 412, 211 },\n\t{ 413, 413, 213 },\n\t{ 414, 414, 130 },\n\t{ 415, 415, 214 },\n\t{ 416, 421, EvenOdd },\n\t{ 422, 422, 218 },\n\t{ 423, 424, OddEven },\n\t{ 425, 425, 218 },\n\t{ 428, 429, EvenOdd },\n\t{ 430, 430, 218 },\n\t{ 431, 432, OddEven },\n\t{ 433, 434, 217 },\n\t{ 435, 438, OddEven },\n\t{ 439, 439, 219 },\n\t{ 440, 441, EvenOdd },\n\t{ 444, 445, EvenOdd },\n\t{ 447, 447, 56 },\n\t{ 452, 452, EvenOdd },\n\t{ 453, 453, OddEven },\n\t{ 454, 454, -2 },\n\t{ 455, 455, OddEven },\n\t{ 456, 456, EvenOdd },\n\t{ 457, 457, -2 },\n\t{ 458, 458, EvenOdd },\n\t{ 459, 459, OddEven },\n\t{ 460, 460, -2 },\n\t{ 461, 476, OddEven },\n\t{ 477, 477, -79 },\n\t{ 478, 495, EvenOdd },\n\t{ 497, 497, OddEven },\n\t{ 498, 498, EvenOdd },\n\t{ 499, 499, -2 },\n\t{ 500, 501, EvenOdd },\n\t{ 502, 502, -97 },\n\t{ 503, 503, -56 },\n\t{ 504, 543, EvenOdd },\n\t{ 544, 544, -130 },\n\t{ 546, 563, EvenOdd },\n\t{ 570, 570, 10795 },\n\t{ 571, 572, OddEven },\n\t{ 573, 573, -163 },\n\t{ 574, 574, 10792 },\n\t{ 575, 576, 10815 },\n\t{ 577, 578, OddEven },\n\t{ 579, 579, -195 },\n\t{ 580, 580, 69 },\n\t{ 581, 581, 71 },\n\t{ 582, 591, EvenOdd },\n\t{ 592, 592, 10783 },\n\t{ 593, 593, 10780 },\n\t{ 594, 594, 10782 },\n\t{ 595, 595, -210 },\n\t{ 596, 596, -206 },\n\t{ 598, 599, -205 },\n\t{ 601, 601, -202 },\n\t{ 603, 603, -203 },\n\t{ 604, 604, 42319 },\n\t{ 608, 608, -205 },\n\t{ 609, 609, 42315 },\n\t{ 611, 611, -207 },\n\t{ 613, 613, 42280 },\n\t{ 614, 614, 42308 },\n\t{ 616, 616, -209 },\n\t{ 617, 617, -211 },\n\t{ 618, 618, 42308 },\n\t{ 619, 619, 10743 },\n\t{ 620, 620, 42305 },\n\t{ 623, 623, -211 },\n\t{ 625, 625, 10749 },\n\t{ 626, 626, -213 },\n\t{ 629, 629, -214 },\n\t{ 637, 637, 10727 },\n\t{ 640, 640, -218 },\n\t{ 642, 642, 42307 },\n\t{ 643, 643, -218 },\n\t{ 647, 647, 42282 },\n\t{ 648, 648, -218 },\n\t{ 649, 649, -69 },\n\t{ 650, 651, -217 },\n\t{ 652, 652, -71 },\n\t{ 658, 658, -219 },\n\t{ 669, 669, 42261 },\n\t{ 670, 670, 42258 },\n\t{ 837, 837, 84 },\n\t{ 880, 883, EvenOdd },\n\t{ 886, 887, EvenOdd },\n\t{ 891, 893, 130 },\n\t{ 895, 895, 116 },\n\t{ 902, 902, 38 },\n\t{ 904, 906, 37 },\n\t{ 908, 908, 64 },\n\t{ 910, 911, 63 },\n\t{ 912, 912, 7235 },\n\t{ 913, 929, 32 },\n\t{ 931, 931, 31 },\n\t{ 932, 939, 32 },\n\t{ 940, 940, -38 },\n\t{ 941, 943, -37 },\n\t{ 944, 944, 7219 },\n\t{ 945, 945, -32 },\n\t{ 946, 946, 30 },\n\t{ 947, 948, -32 },\n\t{ 949, 949, 64 },\n\t{ 950, 951, -32 },\n\t{ 952, 952, 25 },\n\t{ 953, 953, 7173 },\n\t{ 954, 954, 54 },\n\t{ 955, 955, -32 },\n\t{ 956, 956, -775 },\n\t{ 957, 959, -32 },\n\t{ 960, 960, 22 },\n\t{ 961, 961, 48 },\n\t{ 962, 962, EvenOdd },\n\t{ 963, 965, -32 },\n\t{ 966, 966, 15 },\n\t{ 967, 968, -32 },\n\t{ 969, 969, 7517 },\n\t{ 970, 971, -32 },\n\t{ 972, 972, -64 },\n\t{ 973, 974, -63 },\n\t{ 975, 975, 8 },\n\t{ 976, 976, -62 },\n\t{ 977, 977, 35 },\n\t{ 981, 981, -47 },\n\t{ 982, 982, -54 },\n\t{ 983, 983, -8 },\n\t{ 984, 1007, EvenOdd },\n\t{ 1008, 1008, -86 },\n\t{ 1009, 1009, -80 },\n\t{ 1010, 1010, 7 },\n\t{ 1011, 1011, -116 },\n\t{ 1012, 1012, -92 },\n\t{ 1013, 1013, -96 },\n\t{ 1015, 1016, OddEven },\n\t{ 1017, 1017, -7 },\n\t{ 1018, 1019, EvenOdd },\n\t{ 1021, 1023, -130 },\n\t{ 1024, 1039, 80 },\n\t{ 1040, 1071, 32 },\n\t{ 1072, 1073, -32 },\n\t{ 1074, 1074, 6222 },\n\t{ 1075, 1075, -32 },\n\t{ 1076, 1076, 6221 },\n\t{ 1077, 1085, -32 },\n\t{ 1086, 1086, 6212 },\n\t{ 1087, 1088, -32 },\n\t{ 1089, 1090, 6210 },\n\t{ 1091, 1097, -32 },\n\t{ 1098, 1098, 6204 },\n\t{ 1099, 1103, -32 },\n\t{ 1104, 1119, -80 },\n\t{ 1120, 1122, EvenOdd },\n\t{ 1123, 1123, 6180 },\n\t{ 1124, 1153, EvenOdd },\n\t{ 1162, 1215, EvenOdd },\n\t{ 1216, 1216, 15 },\n\t{ 1217, 1230, OddEven },\n\t{ 1231, 1231, -15 },\n\t{ 1232, 1327, EvenOdd },\n\t{ 1329, 1366, 48 },\n\t{ 1377, 1414, -48 },\n\t{ 4256, 4293, 7264 },\n\t{ 4295, 4295, 7264 },\n\t{ 4301, 4301, 7264 },\n\t{ 4304, 4346, 3008 },\n\t{ 4349, 4351, 3008 },\n\t{ 5024, 5103, 38864 },\n\t{ 5104, 5109, 8 },\n\t{ 5112, 5117, -8 },\n\t{ 7296, 7296, -6254 },\n\t{ 7297, 7297, -6253 },\n\t{ 7298, 7298, -6244 },\n\t{ 7299, 7299, -6242 },\n\t{ 7300, 7300, EvenOdd },\n\t{ 7301, 7301, -6243 },\n\t{ 7302, 7302, -6236 },\n\t{ 7303, 7303, -6181 },\n\t{ 7304, 7304, 35266 },\n\t{ 7312, 7354, -3008 },\n\t{ 7357, 7359, -3008 },\n\t{ 7545, 7545, 35332 },\n\t{ 7549, 7549, 3814 },\n\t{ 7566, 7566, 35384 },\n\t{ 7680, 7776, EvenOdd },\n\t{ 7777, 7777, 58 },\n\t{ 7778, 7829, EvenOdd },\n\t{ 7835, 7835, -59 },\n\t{ 7838, 7838, -7615 },\n\t{ 7840, 7935, EvenOdd },\n\t{ 7936, 7943, 8 },\n\t{ 7944, 7951, -8 },\n\t{ 7952, 7957, 8 },\n\t{ 7960, 7965, -8 },\n\t{ 7968, 7975, 8 },\n\t{ 7976, 7983, -8 },\n\t{ 7984, 7991, 8 },\n\t{ 7992, 7999, -8 },\n\t{ 8000, 8005, 8 },\n\t{ 8008, 8013, -8 },\n\t{ 8017, 8017, 8 },\n\t{ 8019, 8019, 8 },\n\t{ 8021, 8021, 8 },\n\t{ 8023, 8023, 8 },\n\t{ 8025, 8025, -8 },\n\t{ 8027, 8027, -8 },\n\t{ 8029, 8029, -8 },\n\t{ 8031, 8031, -8 },\n\t{ 8032, 8039, 8 },\n\t{ 8040, 8047, -8 },\n\t{ 8048, 8049, 74 },\n\t{ 8050, 8053, 86 },\n\t{ 8054, 8055, 100 },\n\t{ 8056, 8057, 128 },\n\t{ 8058, 8059, 112 },\n\t{ 8060, 8061, 126 },\n\t{ 8064, 8071, 8 },\n\t{ 8072, 8079, -8 },\n\t{ 8080, 8087, 8 },\n\t{ 8088, 8095, -8 },\n\t{ 8096, 8103, 8 },\n\t{ 8104, 8111, -8 },\n\t{ 8112, 8113, 8 },\n\t{ 8115, 8115, 9 },\n\t{ 8120, 8121, -8 },\n\t{ 8122, 8123, -74 },\n\t{ 8124, 8124, -9 },\n\t{ 8126, 8126, -7289 },\n\t{ 8131, 8131, 9 },\n\t{ 8136, 8139, -86 },\n\t{ 8140, 8140, -9 },\n\t{ 8144, 8145, 8 },\n\t{ 8147, 8147, -7235 },\n\t{ 8152, 8153, -8 },\n\t{ 8154, 8155, -100 },\n\t{ 8160, 8161, 8 },\n\t{ 8163, 8163, -7219 },\n\t{ 8165, 8165, 7 },\n\t{ 8168, 8169, -8 },\n\t{ 8170, 8171, -112 },\n\t{ 8172, 8172, -7 },\n\t{ 8179, 8179, 9 },\n\t{ 8184, 8185, -128 },\n\t{ 8186, 8187, -126 },\n\t{ 8188, 8188, -9 },\n\t{ 8486, 8486, -7549 },\n\t{ 8490, 8490, -8415 },\n\t{ 8491, 8491, -8294 },\n\t{ 8498, 8498, 28 },\n\t{ 8526, 8526, -28 },\n\t{ 8544, 8559, 16 },\n\t{ 8560, 8575, -16 },\n\t{ 8579, 8580, OddEven },\n\t{ 9398, 9423, 26 },\n\t{ 9424, 9449, -26 },\n\t{ 11264, 11311, 48 },\n\t{ 11312, 11359, -48 },\n\t{ 11360, 11361, EvenOdd },\n\t{ 11362, 11362, -10743 },\n\t{ 11363, 11363, -3814 },\n\t{ 11364, 11364, -10727 },\n\t{ 11365, 11365, -10795 },\n\t{ 11366, 11366, -10792 },\n\t{ 11367, 11372, OddEven },\n\t{ 11373, 11373, -10780 },\n\t{ 11374, 11374, -10749 },\n\t{ 11375, 11375, -10783 },\n\t{ 11376, 11376, -10782 },\n\t{ 11378, 11379, EvenOdd },\n\t{ 11381, 11382, OddEven },\n\t{ 11390, 11391, -10815 },\n\t{ 11392, 11491, EvenOdd },\n\t{ 11499, 11502, OddEven },\n\t{ 11506, 11507, EvenOdd },\n\t{ 11520, 11557, -7264 },\n\t{ 11559, 11559, -7264 },\n\t{ 11565, 11565, -7264 },\n\t{ 42560, 42570, EvenOdd },\n\t{ 42571, 42571, -35267 },\n\t{ 42572, 42605, EvenOdd },\n\t{ 42624, 42651, EvenOdd },\n\t{ 42786, 42799, EvenOdd },\n\t{ 42802, 42863, EvenOdd },\n\t{ 42873, 42876, OddEven },\n\t{ 42877, 42877, -35332 },\n\t{ 42878, 42887, EvenOdd },\n\t{ 42891, 42892, OddEven },\n\t{ 42893, 42893, -42280 },\n\t{ 42896, 42899, EvenOdd },\n\t{ 42900, 42900, 48 },\n\t{ 42902, 42921, EvenOdd },\n\t{ 42922, 42922, -42308 },\n\t{ 42923, 42923, -42319 },\n\t{ 42924, 42924, -42315 },\n\t{ 42925, 42925, -42305 },\n\t{ 42926, 42926, -42308 },\n\t{ 42928, 42928, -42258 },\n\t{ 42929, 42929, -42282 },\n\t{ 42930, 42930, -42261 },\n\t{ 42931, 42931, 928 },\n\t{ 42932, 42947, EvenOdd },\n\t{ 42948, 42948, -48 },\n\t{ 42949, 42949, -42307 },\n\t{ 42950, 42950, -35384 },\n\t{ 42951, 42954, OddEven },\n\t{ 42960, 42961, EvenOdd },\n\t{ 42966, 42969, EvenOdd },\n\t{ 42997, 42998, OddEven },\n\t{ 43859, 43859, -928 },\n\t{ 43888, 43967, -38864 },\n\t{ 64261, 64262, OddEven },\n\t{ 65313, 65338, 32 },\n\t{ 65345, 65370, -32 },\n\t{ 66560, 66599, 40 },\n\t{ 66600, 66639, -40 },\n\t{ 66736, 66771, 40 },\n\t{ 66776, 66811, -40 },\n\t{ 66928, 66938, 39 },\n\t{ 66940, 66954, 39 },\n\t{ 66956, 66962, 39 },\n\t{ 66964, 66965, 39 },\n\t{ 66967, 66977, -39 },\n\t{ 66979, 66993, -39 },\n\t{ 66995, 67001, -39 },\n\t{ 67003, 67004, -39 },\n\t{ 68736, 68786, 64 },\n\t{ 68800, 68850, -64 },\n\t{ 71840, 71871, 32 },\n\t{ 71872, 71903, -32 },\n\t{ 93760, 93791, 32 },\n\t{ 93792, 93823, -32 },\n\t{ 125184, 125217, 34 },\n\t{ 125218, 125251, -34 },\n};\nconst int num_unicode_casefold = 372;\n\n// 1427 groups, 1457 pairs, 208 ranges\nconst CaseFold unicode_tolower[] = {\n\t{ 65, 90, 32 },\n\t{ 181, 181, 775 },\n\t{ 192, 214, 32 },\n\t{ 216, 222, 32 },\n\t{ 256, 302, EvenOddSkip },\n\t{ 306, 310, EvenOddSkip },\n\t{ 313, 327, OddEvenSkip },\n\t{ 330, 374, EvenOddSkip },\n\t{ 376, 376, -121 },\n\t{ 377, 381, OddEvenSkip },\n\t{ 383, 383, -268 },\n\t{ 385, 385, 210 },\n\t{ 386, 388, EvenOddSkip },\n\t{ 390, 390, 206 },\n\t{ 391, 391, OddEven },\n\t{ 393, 394, 205 },\n\t{ 395, 395, OddEven },\n\t{ 398, 398, 79 },\n\t{ 399, 399, 202 },\n\t{ 400, 400, 203 },\n\t{ 401, 401, OddEven },\n\t{ 403, 403, 205 },\n\t{ 404, 404, 207 },\n\t{ 406, 406, 211 },\n\t{ 407, 407, 209 },\n\t{ 408, 408, EvenOdd },\n\t{ 412, 412, 211 },\n\t{ 413, 413, 213 },\n\t{ 415, 415, 214 },\n\t{ 416, 420, EvenOddSkip },\n\t{ 422, 422, 218 },\n\t{ 423, 423, OddEven },\n\t{ 425, 425, 218 },\n\t{ 428, 428, EvenOdd },\n\t{ 430, 430, 218 },\n\t{ 431, 431, OddEven },\n\t{ 433, 434, 217 },\n\t{ 435, 437, OddEvenSkip },\n\t{ 439, 439, 219 },\n\t{ 440, 440, EvenOdd },\n\t{ 444, 444, EvenOdd },\n\t{ 452, 452, 2 },\n\t{ 453, 453, OddEven },\n\t{ 455, 455, 2 },\n\t{ 456, 456, EvenOdd },\n\t{ 458, 458, 2 },\n\t{ 459, 475, OddEvenSkip },\n\t{ 478, 494, EvenOddSkip },\n\t{ 497, 497, 2 },\n\t{ 498, 500, EvenOddSkip },\n\t{ 502, 502, -97 },\n\t{ 503, 503, -56 },\n\t{ 504, 542, EvenOddSkip },\n\t{ 544, 544, -130 },\n\t{ 546, 562, EvenOddSkip },\n\t{ 570, 570, 10795 },\n\t{ 571, 571, OddEven },\n\t{ 573, 573, -163 },\n\t{ 574, 574, 10792 },\n\t{ 577, 577, OddEven },\n\t{ 579, 579, -195 },\n\t{ 580, 580, 69 },\n\t{ 581, 581, 71 },\n\t{ 582, 590, EvenOddSkip },\n\t{ 837, 837, 116 },\n\t{ 880, 882, EvenOddSkip },\n\t{ 886, 886, EvenOdd },\n\t{ 895, 895, 116 },\n\t{ 902, 902, 38 },\n\t{ 904, 906, 37 },\n\t{ 908, 908, 64 },\n\t{ 910, 911, 63 },\n\t{ 913, 929, 32 },\n\t{ 931, 939, 32 },\n\t{ 962, 962, EvenOdd },\n\t{ 975, 975, 8 },\n\t{ 976, 976, -30 },\n\t{ 977, 977, -25 },\n\t{ 981, 981, -15 },\n\t{ 982, 982, -22 },\n\t{ 984, 1006, EvenOddSkip },\n\t{ 1008, 1008, -54 },\n\t{ 1009, 1009, -48 },\n\t{ 1012, 1012, -60 },\n\t{ 1013, 1013, -64 },\n\t{ 1015, 1015, OddEven },\n\t{ 1017, 1017, -7 },\n\t{ 1018, 1018, EvenOdd },\n\t{ 1021, 1023, -130 },\n\t{ 1024, 1039, 80 },\n\t{ 1040, 1071, 32 },\n\t{ 1120, 1152, EvenOddSkip },\n\t{ 1162, 1214, EvenOddSkip },\n\t{ 1216, 1216, 15 },\n\t{ 1217, 1229, OddEvenSkip },\n\t{ 1232, 1326, EvenOddSkip },\n\t{ 1329, 1366, 48 },\n\t{ 4256, 4293, 7264 },\n\t{ 4295, 4295, 7264 },\n\t{ 4301, 4301, 7264 },\n\t{ 5112, 5117, -8 },\n\t{ 7296, 7296, -6222 },\n\t{ 7297, 7297, -6221 },\n\t{ 7298, 7298, -6212 },\n\t{ 7299, 7300, -6210 },\n\t{ 7301, 7301, -6211 },\n\t{ 7302, 7302, -6204 },\n\t{ 7303, 7303, -6180 },\n\t{ 7304, 7304, 35267 },\n\t{ 7312, 7354, -3008 },\n\t{ 7357, 7359, -3008 },\n\t{ 7680, 7828, EvenOddSkip },\n\t{ 7835, 7835, -58 },\n\t{ 7838, 7838, -7615 },\n\t{ 7840, 7934, EvenOddSkip },\n\t{ 7944, 7951, -8 },\n\t{ 7960, 7965, -8 },\n\t{ 7976, 7983, -8 },\n\t{ 7992, 7999, -8 },\n\t{ 8008, 8013, -8 },\n\t{ 8025, 8025, -8 },\n\t{ 8027, 8027, -8 },\n\t{ 8029, 8029, -8 },\n\t{ 8031, 8031, -8 },\n\t{ 8040, 8047, -8 },\n\t{ 8072, 8079, -8 },\n\t{ 8088, 8095, -8 },\n\t{ 8104, 8111, -8 },\n\t{ 8120, 8121, -8 },\n\t{ 8122, 8123, -74 },\n\t{ 8124, 8124, -9 },\n\t{ 8126, 8126, -7173 },\n\t{ 8136, 8139, -86 },\n\t{ 8140, 8140, -9 },\n\t{ 8147, 8147, -7235 },\n\t{ 8152, 8153, -8 },\n\t{ 8154, 8155, -100 },\n\t{ 8163, 8163, -7219 },\n\t{ 8168, 8169, -8 },\n\t{ 8170, 8171, -112 },\n\t{ 8172, 8172, -7 },\n\t{ 8184, 8185, -128 },\n\t{ 8186, 8187, -126 },\n\t{ 8188, 8188, -9 },\n\t{ 8486, 8486, -7517 },\n\t{ 8490, 8490, -8383 },\n\t{ 8491, 8491, -8262 },\n\t{ 8498, 8498, 28 },\n\t{ 8544, 8559, 16 },\n\t{ 8579, 8579, OddEven },\n\t{ 9398, 9423, 26 },\n\t{ 11264, 11311, 48 },\n\t{ 11360, 11360, EvenOdd },\n\t{ 11362, 11362, -10743 },\n\t{ 11363, 11363, -3814 },\n\t{ 11364, 11364, -10727 },\n\t{ 11367, 11371, OddEvenSkip },\n\t{ 11373, 11373, -10780 },\n\t{ 11374, 11374, -10749 },\n\t{ 11375, 11375, -10783 },\n\t{ 11376, 11376, -10782 },\n\t{ 11378, 11378, EvenOdd },\n\t{ 11381, 11381, OddEven },\n\t{ 11390, 11391, -10815 },\n\t{ 11392, 11490, EvenOddSkip },\n\t{ 11499, 11501, OddEvenSkip },\n\t{ 11506, 11506, EvenOdd },\n\t{ 42560, 42604, EvenOddSkip },\n\t{ 42624, 42650, EvenOddSkip },\n\t{ 42786, 42798, EvenOddSkip },\n\t{ 42802, 42862, EvenOddSkip },\n\t{ 42873, 42875, OddEvenSkip },\n\t{ 42877, 42877, -35332 },\n\t{ 42878, 42886, EvenOddSkip },\n\t{ 42891, 42891, OddEven },\n\t{ 42893, 42893, -42280 },\n\t{ 42896, 42898, EvenOddSkip },\n\t{ 42902, 42920, EvenOddSkip },\n\t{ 42922, 42922, -42308 },\n\t{ 42923, 42923, -42319 },\n\t{ 42924, 42924, -42315 },\n\t{ 42925, 42925, -42305 },\n\t{ 42926, 42926, -42308 },\n\t{ 42928, 42928, -42258 },\n\t{ 42929, 42929, -42282 },\n\t{ 42930, 42930, -42261 },\n\t{ 42931, 42931, 928 },\n\t{ 42932, 42946, EvenOddSkip },\n\t{ 42948, 42948, -48 },\n\t{ 42949, 42949, -42307 },\n\t{ 42950, 42950, -35384 },\n\t{ 42951, 42953, OddEvenSkip },\n\t{ 42960, 42960, EvenOdd },\n\t{ 42966, 42968, EvenOddSkip },\n\t{ 42997, 42997, OddEven },\n\t{ 43888, 43967, -38864 },\n\t{ 64261, 64261, OddEven },\n\t{ 65313, 65338, 32 },\n\t{ 66560, 66599, 40 },\n\t{ 66736, 66771, 40 },\n\t{ 66928, 66938, 39 },\n\t{ 66940, 66954, 39 },\n\t{ 66956, 66962, 39 },\n\t{ 66964, 66965, 39 },\n\t{ 68736, 68786, 64 },\n\t{ 71840, 71871, 32 },\n\t{ 93760, 93791, 32 },\n\t{ 125184, 125217, 34 },\n};\nconst int num_unicode_tolower = 208;\n\n\n\n} // namespace re2\n\n\n" }, { "path": "re2/unicode_casefold.h", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_UNICODE_CASEFOLD_H_\n#define RE2_UNICODE_CASEFOLD_H_\n\n// Unicode case folding tables.\n\n// The Unicode case folding tables encode the mapping from one Unicode point\n// to the next largest Unicode point with equivalent folding. The largest\n// point wraps back to the first. For example, the tables map:\n//\n// 'A' -> 'a'\n// 'a' -> 'A'\n//\n// 'K' -> 'k'\n// 'k' -> 'K' (Kelvin symbol)\n// 'K' -> 'K'\n//\n// Like everything Unicode, these tables are big. If we represent the table\n// as a sorted list of uint32_t pairs, it has 2049 entries and is 16 kB.\n// Most table entries look like the ones around them:\n// 'A' maps to 'A'+32, 'B' maps to 'B'+32, etc.\n// Instead of listing all the pairs explicitly, we make a list of ranges\n// and deltas, so that the table entries for 'A' through 'Z' can be represented\n// as a single entry { 'A', 'Z', +32 }.\n//\n// In addition to blocks that map to each other (A-Z mapping to a-z)\n// there are blocks of pairs that individually map to each other\n// (for example, 0100<->0101, 0102<->0103, 0104<->0105, ...).\n// For those, the special delta value EvenOdd marks even/odd pairs\n// (if even, add 1; if odd, subtract 1), and OddEven marks odd/even pairs.\n//\n// In this form, the table has 274 entries, about 3kB. If we were to split\n// the table into one for 16-bit codes and an overflow table for larger ones,\n// we could get it down to about 1.5kB, but that's not worth the complexity.\n//\n// The grouped form also allows for efficient fold range calculations\n// rather than looping one character at a time.\n\n#include \n\n#include \"util/utf.h\"\n\nnamespace re2 {\n\nenum {\n EvenOdd = 1,\n OddEven = -1,\n EvenOddSkip = 1<<30,\n OddEvenSkip,\n};\n\nstruct CaseFold {\n Rune lo;\n Rune hi;\n int32_t delta;\n};\n\nextern const CaseFold unicode_casefold[];\nextern const int num_unicode_casefold;\n\nextern const CaseFold unicode_tolower[];\nextern const int num_unicode_tolower;\n\n// Returns the CaseFold* in the tables that contains rune.\n// If rune is not in the tables, returns the first CaseFold* after rune.\n// If rune is larger than any value in the tables, returns NULL.\nextern const CaseFold* LookupCaseFold(const CaseFold*, int, Rune rune);\n\n// Returns the result of applying the fold f to the rune r.\nextern Rune ApplyFold(const CaseFold *f, Rune r);\n\n} // namespace re2\n\n#endif // RE2_UNICODE_CASEFOLD_H_\n" }, { "path": "re2/unicode_groups.cc", "content": "\n// GENERATED BY make_unicode_groups.py; DO NOT EDIT.\n// make_unicode_groups.py >unicode_groups.cc\n\n#include \"re2/unicode_groups.h\"\n\nnamespace re2 {\n\n\nstatic const URange16 C_range16[] = {\n\t{ 0, 31 },\n\t{ 127, 159 },\n\t{ 173, 173 },\n\t{ 1536, 1541 },\n\t{ 1564, 1564 },\n\t{ 1757, 1757 },\n\t{ 1807, 1807 },\n\t{ 2192, 2193 },\n\t{ 2274, 2274 },\n\t{ 6158, 6158 },\n\t{ 8203, 8207 },\n\t{ 8234, 8238 },\n\t{ 8288, 8292 },\n\t{ 8294, 8303 },\n\t{ 55296, 63743 },\n\t{ 65279, 65279 },\n\t{ 65529, 65531 },\n};\nstatic const URange32 C_range32[] = {\n\t{ 69821, 69821 },\n\t{ 69837, 69837 },\n\t{ 78896, 78911 },\n\t{ 113824, 113827 },\n\t{ 119155, 119162 },\n\t{ 917505, 917505 },\n\t{ 917536, 917631 },\n\t{ 983040, 1048573 },\n\t{ 1048576, 1114109 },\n};\nstatic const URange16 Cc_range16[] = {\n\t{ 0, 31 },\n\t{ 127, 159 },\n};\nstatic const URange16 Cf_range16[] = {\n\t{ 173, 173 },\n\t{ 1536, 1541 },\n\t{ 1564, 1564 },\n\t{ 1757, 1757 },\n\t{ 1807, 1807 },\n\t{ 2192, 2193 },\n\t{ 2274, 2274 },\n\t{ 6158, 6158 },\n\t{ 8203, 8207 },\n\t{ 8234, 8238 },\n\t{ 8288, 8292 },\n\t{ 8294, 8303 },\n\t{ 65279, 65279 },\n\t{ 65529, 65531 },\n};\nstatic const URange32 Cf_range32[] = {\n\t{ 69821, 69821 },\n\t{ 69837, 69837 },\n\t{ 78896, 78911 },\n\t{ 113824, 113827 },\n\t{ 119155, 119162 },\n\t{ 917505, 917505 },\n\t{ 917536, 917631 },\n};\nstatic const URange16 Co_range16[] = {\n\t{ 57344, 63743 },\n};\nstatic const URange32 Co_range32[] = {\n\t{ 983040, 1048573 },\n\t{ 1048576, 1114109 },\n};\nstatic const URange16 Cs_range16[] = {\n\t{ 55296, 57343 },\n};\nstatic const URange16 L_range16[] = {\n\t{ 65, 90 },\n\t{ 97, 122 },\n\t{ 170, 170 },\n\t{ 181, 181 },\n\t{ 186, 186 },\n\t{ 192, 214 },\n\t{ 216, 246 },\n\t{ 248, 705 },\n\t{ 710, 721 },\n\t{ 736, 740 },\n\t{ 748, 748 },\n\t{ 750, 750 },\n\t{ 880, 884 },\n\t{ 886, 887 },\n\t{ 890, 893 },\n\t{ 895, 895 },\n\t{ 902, 902 },\n\t{ 904, 906 },\n\t{ 908, 908 },\n\t{ 910, 929 },\n\t{ 931, 1013 },\n\t{ 1015, 1153 },\n\t{ 1162, 1327 },\n\t{ 1329, 1366 },\n\t{ 1369, 1369 },\n\t{ 1376, 1416 },\n\t{ 1488, 1514 },\n\t{ 1519, 1522 },\n\t{ 1568, 1610 },\n\t{ 1646, 1647 },\n\t{ 1649, 1747 },\n\t{ 1749, 1749 },\n\t{ 1765, 1766 },\n\t{ 1774, 1775 },\n\t{ 1786, 1788 },\n\t{ 1791, 1791 },\n\t{ 1808, 1808 },\n\t{ 1810, 1839 },\n\t{ 1869, 1957 },\n\t{ 1969, 1969 },\n\t{ 1994, 2026 },\n\t{ 2036, 2037 },\n\t{ 2042, 2042 },\n\t{ 2048, 2069 },\n\t{ 2074, 2074 },\n\t{ 2084, 2084 },\n\t{ 2088, 2088 },\n\t{ 2112, 2136 },\n\t{ 2144, 2154 },\n\t{ 2160, 2183 },\n\t{ 2185, 2190 },\n\t{ 2208, 2249 },\n\t{ 2308, 2361 },\n\t{ 2365, 2365 },\n\t{ 2384, 2384 },\n\t{ 2392, 2401 },\n\t{ 2417, 2432 },\n\t{ 2437, 2444 },\n\t{ 2447, 2448 },\n\t{ 2451, 2472 },\n\t{ 2474, 2480 },\n\t{ 2482, 2482 },\n\t{ 2486, 2489 },\n\t{ 2493, 2493 },\n\t{ 2510, 2510 },\n\t{ 2524, 2525 },\n\t{ 2527, 2529 },\n\t{ 2544, 2545 },\n\t{ 2556, 2556 },\n\t{ 2565, 2570 },\n\t{ 2575, 2576 },\n\t{ 2579, 2600 },\n\t{ 2602, 2608 },\n\t{ 2610, 2611 },\n\t{ 2613, 2614 },\n\t{ 2616, 2617 },\n\t{ 2649, 2652 },\n\t{ 2654, 2654 },\n\t{ 2674, 2676 },\n\t{ 2693, 2701 },\n\t{ 2703, 2705 },\n\t{ 2707, 2728 },\n\t{ 2730, 2736 },\n\t{ 2738, 2739 },\n\t{ 2741, 2745 },\n\t{ 2749, 2749 },\n\t{ 2768, 2768 },\n\t{ 2784, 2785 },\n\t{ 2809, 2809 },\n\t{ 2821, 2828 },\n\t{ 2831, 2832 },\n\t{ 2835, 2856 },\n\t{ 2858, 2864 },\n\t{ 2866, 2867 },\n\t{ 2869, 2873 },\n\t{ 2877, 2877 },\n\t{ 2908, 2909 },\n\t{ 2911, 2913 },\n\t{ 2929, 2929 },\n\t{ 2947, 2947 },\n\t{ 2949, 2954 },\n\t{ 2958, 2960 },\n\t{ 2962, 2965 },\n\t{ 2969, 2970 },\n\t{ 2972, 2972 },\n\t{ 2974, 2975 },\n\t{ 2979, 2980 },\n\t{ 2984, 2986 },\n\t{ 2990, 3001 },\n\t{ 3024, 3024 },\n\t{ 3077, 3084 },\n\t{ 3086, 3088 },\n\t{ 3090, 3112 },\n\t{ 3114, 3129 },\n\t{ 3133, 3133 },\n\t{ 3160, 3162 },\n\t{ 3165, 3165 },\n\t{ 3168, 3169 },\n\t{ 3200, 3200 },\n\t{ 3205, 3212 },\n\t{ 3214, 3216 },\n\t{ 3218, 3240 },\n\t{ 3242, 3251 },\n\t{ 3253, 3257 },\n\t{ 3261, 3261 },\n\t{ 3293, 3294 },\n\t{ 3296, 3297 },\n\t{ 3313, 3314 },\n\t{ 3332, 3340 },\n\t{ 3342, 3344 },\n\t{ 3346, 3386 },\n\t{ 3389, 3389 },\n\t{ 3406, 3406 },\n\t{ 3412, 3414 },\n\t{ 3423, 3425 },\n\t{ 3450, 3455 },\n\t{ 3461, 3478 },\n\t{ 3482, 3505 },\n\t{ 3507, 3515 },\n\t{ 3517, 3517 },\n\t{ 3520, 3526 },\n\t{ 3585, 3632 },\n\t{ 3634, 3635 },\n\t{ 3648, 3654 },\n\t{ 3713, 3714 },\n\t{ 3716, 3716 },\n\t{ 3718, 3722 },\n\t{ 3724, 3747 },\n\t{ 3749, 3749 },\n\t{ 3751, 3760 },\n\t{ 3762, 3763 },\n\t{ 3773, 3773 },\n\t{ 3776, 3780 },\n\t{ 3782, 3782 },\n\t{ 3804, 3807 },\n\t{ 3840, 3840 },\n\t{ 3904, 3911 },\n\t{ 3913, 3948 },\n\t{ 3976, 3980 },\n\t{ 4096, 4138 },\n\t{ 4159, 4159 },\n\t{ 4176, 4181 },\n\t{ 4186, 4189 },\n\t{ 4193, 4193 },\n\t{ 4197, 4198 },\n\t{ 4206, 4208 },\n\t{ 4213, 4225 },\n\t{ 4238, 4238 },\n\t{ 4256, 4293 },\n\t{ 4295, 4295 },\n\t{ 4301, 4301 },\n\t{ 4304, 4346 },\n\t{ 4348, 4680 },\n\t{ 4682, 4685 },\n\t{ 4688, 4694 },\n\t{ 4696, 4696 },\n\t{ 4698, 4701 },\n\t{ 4704, 4744 },\n\t{ 4746, 4749 },\n\t{ 4752, 4784 },\n\t{ 4786, 4789 },\n\t{ 4792, 4798 },\n\t{ 4800, 4800 },\n\t{ 4802, 4805 },\n\t{ 4808, 4822 },\n\t{ 4824, 4880 },\n\t{ 4882, 4885 },\n\t{ 4888, 4954 },\n\t{ 4992, 5007 },\n\t{ 5024, 5109 },\n\t{ 5112, 5117 },\n\t{ 5121, 5740 },\n\t{ 5743, 5759 },\n\t{ 5761, 5786 },\n\t{ 5792, 5866 },\n\t{ 5873, 5880 },\n\t{ 5888, 5905 },\n\t{ 5919, 5937 },\n\t{ 5952, 5969 },\n\t{ 5984, 5996 },\n\t{ 5998, 6000 },\n\t{ 6016, 6067 },\n\t{ 6103, 6103 },\n\t{ 6108, 6108 },\n\t{ 6176, 6264 },\n\t{ 6272, 6276 },\n\t{ 6279, 6312 },\n\t{ 6314, 6314 },\n\t{ 6320, 6389 },\n\t{ 6400, 6430 },\n\t{ 6480, 6509 },\n\t{ 6512, 6516 },\n\t{ 6528, 6571 },\n\t{ 6576, 6601 },\n\t{ 6656, 6678 },\n\t{ 6688, 6740 },\n\t{ 6823, 6823 },\n\t{ 6917, 6963 },\n\t{ 6981, 6988 },\n\t{ 7043, 7072 },\n\t{ 7086, 7087 },\n\t{ 7098, 7141 },\n\t{ 7168, 7203 },\n\t{ 7245, 7247 },\n\t{ 7258, 7293 },\n\t{ 7296, 7304 },\n\t{ 7312, 7354 },\n\t{ 7357, 7359 },\n\t{ 7401, 7404 },\n\t{ 7406, 7411 },\n\t{ 7413, 7414 },\n\t{ 7418, 7418 },\n\t{ 7424, 7615 },\n\t{ 7680, 7957 },\n\t{ 7960, 7965 },\n\t{ 7968, 8005 },\n\t{ 8008, 8013 },\n\t{ 8016, 8023 },\n\t{ 8025, 8025 },\n\t{ 8027, 8027 },\n\t{ 8029, 8029 },\n\t{ 8031, 8061 },\n\t{ 8064, 8116 },\n\t{ 8118, 8124 },\n\t{ 8126, 8126 },\n\t{ 8130, 8132 },\n\t{ 8134, 8140 },\n\t{ 8144, 8147 },\n\t{ 8150, 8155 },\n\t{ 8160, 8172 },\n\t{ 8178, 8180 },\n\t{ 8182, 8188 },\n\t{ 8305, 8305 },\n\t{ 8319, 8319 },\n\t{ 8336, 8348 },\n\t{ 8450, 8450 },\n\t{ 8455, 8455 },\n\t{ 8458, 8467 },\n\t{ 8469, 8469 },\n\t{ 8473, 8477 },\n\t{ 8484, 8484 },\n\t{ 8486, 8486 },\n\t{ 8488, 8488 },\n\t{ 8490, 8493 },\n\t{ 8495, 8505 },\n\t{ 8508, 8511 },\n\t{ 8517, 8521 },\n\t{ 8526, 8526 },\n\t{ 8579, 8580 },\n\t{ 11264, 11492 },\n\t{ 11499, 11502 },\n\t{ 11506, 11507 },\n\t{ 11520, 11557 },\n\t{ 11559, 11559 },\n\t{ 11565, 11565 },\n\t{ 11568, 11623 },\n\t{ 11631, 11631 },\n\t{ 11648, 11670 },\n\t{ 11680, 11686 },\n\t{ 11688, 11694 },\n\t{ 11696, 11702 },\n\t{ 11704, 11710 },\n\t{ 11712, 11718 },\n\t{ 11720, 11726 },\n\t{ 11728, 11734 },\n\t{ 11736, 11742 },\n\t{ 11823, 11823 },\n\t{ 12293, 12294 },\n\t{ 12337, 12341 },\n\t{ 12347, 12348 },\n\t{ 12353, 12438 },\n\t{ 12445, 12447 },\n\t{ 12449, 12538 },\n\t{ 12540, 12543 },\n\t{ 12549, 12591 },\n\t{ 12593, 12686 },\n\t{ 12704, 12735 },\n\t{ 12784, 12799 },\n\t{ 13312, 19903 },\n\t{ 19968, 42124 },\n\t{ 42192, 42237 },\n\t{ 42240, 42508 },\n\t{ 42512, 42527 },\n\t{ 42538, 42539 },\n\t{ 42560, 42606 },\n\t{ 42623, 42653 },\n\t{ 42656, 42725 },\n\t{ 42775, 42783 },\n\t{ 42786, 42888 },\n\t{ 42891, 42954 },\n\t{ 42960, 42961 },\n\t{ 42963, 42963 },\n\t{ 42965, 42969 },\n\t{ 42994, 43009 },\n\t{ 43011, 43013 },\n\t{ 43015, 43018 },\n\t{ 43020, 43042 },\n\t{ 43072, 43123 },\n\t{ 43138, 43187 },\n\t{ 43250, 43255 },\n\t{ 43259, 43259 },\n\t{ 43261, 43262 },\n\t{ 43274, 43301 },\n\t{ 43312, 43334 },\n\t{ 43360, 43388 },\n\t{ 43396, 43442 },\n\t{ 43471, 43471 },\n\t{ 43488, 43492 },\n\t{ 43494, 43503 },\n\t{ 43514, 43518 },\n\t{ 43520, 43560 },\n\t{ 43584, 43586 },\n\t{ 43588, 43595 },\n\t{ 43616, 43638 },\n\t{ 43642, 43642 },\n\t{ 43646, 43695 },\n\t{ 43697, 43697 },\n\t{ 43701, 43702 },\n\t{ 43705, 43709 },\n\t{ 43712, 43712 },\n\t{ 43714, 43714 },\n\t{ 43739, 43741 },\n\t{ 43744, 43754 },\n\t{ 43762, 43764 },\n\t{ 43777, 43782 },\n\t{ 43785, 43790 },\n\t{ 43793, 43798 },\n\t{ 43808, 43814 },\n\t{ 43816, 43822 },\n\t{ 43824, 43866 },\n\t{ 43868, 43881 },\n\t{ 43888, 44002 },\n\t{ 44032, 55203 },\n\t{ 55216, 55238 },\n\t{ 55243, 55291 },\n\t{ 63744, 64109 },\n\t{ 64112, 64217 },\n\t{ 64256, 64262 },\n\t{ 64275, 64279 },\n\t{ 64285, 64285 },\n\t{ 64287, 64296 },\n\t{ 64298, 64310 },\n\t{ 64312, 64316 },\n\t{ 64318, 64318 },\n\t{ 64320, 64321 },\n\t{ 64323, 64324 },\n\t{ 64326, 64433 },\n\t{ 64467, 64829 },\n\t{ 64848, 64911 },\n\t{ 64914, 64967 },\n\t{ 65008, 65019 },\n\t{ 65136, 65140 },\n\t{ 65142, 65276 },\n\t{ 65313, 65338 },\n\t{ 65345, 65370 },\n\t{ 65382, 65470 },\n\t{ 65474, 65479 },\n\t{ 65482, 65487 },\n\t{ 65490, 65495 },\n\t{ 65498, 65500 },\n};\nstatic const URange32 L_range32[] = {\n\t{ 65536, 65547 },\n\t{ 65549, 65574 },\n\t{ 65576, 65594 },\n\t{ 65596, 65597 },\n\t{ 65599, 65613 },\n\t{ 65616, 65629 },\n\t{ 65664, 65786 },\n\t{ 66176, 66204 },\n\t{ 66208, 66256 },\n\t{ 66304, 66335 },\n\t{ 66349, 66368 },\n\t{ 66370, 66377 },\n\t{ 66384, 66421 },\n\t{ 66432, 66461 },\n\t{ 66464, 66499 },\n\t{ 66504, 66511 },\n\t{ 66560, 66717 },\n\t{ 66736, 66771 },\n\t{ 66776, 66811 },\n\t{ 66816, 66855 },\n\t{ 66864, 66915 },\n\t{ 66928, 66938 },\n\t{ 66940, 66954 },\n\t{ 66956, 66962 },\n\t{ 66964, 66965 },\n\t{ 66967, 66977 },\n\t{ 66979, 66993 },\n\t{ 66995, 67001 },\n\t{ 67003, 67004 },\n\t{ 67072, 67382 },\n\t{ 67392, 67413 },\n\t{ 67424, 67431 },\n\t{ 67456, 67461 },\n\t{ 67463, 67504 },\n\t{ 67506, 67514 },\n\t{ 67584, 67589 },\n\t{ 67592, 67592 },\n\t{ 67594, 67637 },\n\t{ 67639, 67640 },\n\t{ 67644, 67644 },\n\t{ 67647, 67669 },\n\t{ 67680, 67702 },\n\t{ 67712, 67742 },\n\t{ 67808, 67826 },\n\t{ 67828, 67829 },\n\t{ 67840, 67861 },\n\t{ 67872, 67897 },\n\t{ 67968, 68023 },\n\t{ 68030, 68031 },\n\t{ 68096, 68096 },\n\t{ 68112, 68115 },\n\t{ 68117, 68119 },\n\t{ 68121, 68149 },\n\t{ 68192, 68220 },\n\t{ 68224, 68252 },\n\t{ 68288, 68295 },\n\t{ 68297, 68324 },\n\t{ 68352, 68405 },\n\t{ 68416, 68437 },\n\t{ 68448, 68466 },\n\t{ 68480, 68497 },\n\t{ 68608, 68680 },\n\t{ 68736, 68786 },\n\t{ 68800, 68850 },\n\t{ 68864, 68899 },\n\t{ 69248, 69289 },\n\t{ 69296, 69297 },\n\t{ 69376, 69404 },\n\t{ 69415, 69415 },\n\t{ 69424, 69445 },\n\t{ 69488, 69505 },\n\t{ 69552, 69572 },\n\t{ 69600, 69622 },\n\t{ 69635, 69687 },\n\t{ 69745, 69746 },\n\t{ 69749, 69749 },\n\t{ 69763, 69807 },\n\t{ 69840, 69864 },\n\t{ 69891, 69926 },\n\t{ 69956, 69956 },\n\t{ 69959, 69959 },\n\t{ 69968, 70002 },\n\t{ 70006, 70006 },\n\t{ 70019, 70066 },\n\t{ 70081, 70084 },\n\t{ 70106, 70106 },\n\t{ 70108, 70108 },\n\t{ 70144, 70161 },\n\t{ 70163, 70187 },\n\t{ 70207, 70208 },\n\t{ 70272, 70278 },\n\t{ 70280, 70280 },\n\t{ 70282, 70285 },\n\t{ 70287, 70301 },\n\t{ 70303, 70312 },\n\t{ 70320, 70366 },\n\t{ 70405, 70412 },\n\t{ 70415, 70416 },\n\t{ 70419, 70440 },\n\t{ 70442, 70448 },\n\t{ 70450, 70451 },\n\t{ 70453, 70457 },\n\t{ 70461, 70461 },\n\t{ 70480, 70480 },\n\t{ 70493, 70497 },\n\t{ 70656, 70708 },\n\t{ 70727, 70730 },\n\t{ 70751, 70753 },\n\t{ 70784, 70831 },\n\t{ 70852, 70853 },\n\t{ 70855, 70855 },\n\t{ 71040, 71086 },\n\t{ 71128, 71131 },\n\t{ 71168, 71215 },\n\t{ 71236, 71236 },\n\t{ 71296, 71338 },\n\t{ 71352, 71352 },\n\t{ 71424, 71450 },\n\t{ 71488, 71494 },\n\t{ 71680, 71723 },\n\t{ 71840, 71903 },\n\t{ 71935, 71942 },\n\t{ 71945, 71945 },\n\t{ 71948, 71955 },\n\t{ 71957, 71958 },\n\t{ 71960, 71983 },\n\t{ 71999, 71999 },\n\t{ 72001, 72001 },\n\t{ 72096, 72103 },\n\t{ 72106, 72144 },\n\t{ 72161, 72161 },\n\t{ 72163, 72163 },\n\t{ 72192, 72192 },\n\t{ 72203, 72242 },\n\t{ 72250, 72250 },\n\t{ 72272, 72272 },\n\t{ 72284, 72329 },\n\t{ 72349, 72349 },\n\t{ 72368, 72440 },\n\t{ 72704, 72712 },\n\t{ 72714, 72750 },\n\t{ 72768, 72768 },\n\t{ 72818, 72847 },\n\t{ 72960, 72966 },\n\t{ 72968, 72969 },\n\t{ 72971, 73008 },\n\t{ 73030, 73030 },\n\t{ 73056, 73061 },\n\t{ 73063, 73064 },\n\t{ 73066, 73097 },\n\t{ 73112, 73112 },\n\t{ 73440, 73458 },\n\t{ 73474, 73474 },\n\t{ 73476, 73488 },\n\t{ 73490, 73523 },\n\t{ 73648, 73648 },\n\t{ 73728, 74649 },\n\t{ 74880, 75075 },\n\t{ 77712, 77808 },\n\t{ 77824, 78895 },\n\t{ 78913, 78918 },\n\t{ 82944, 83526 },\n\t{ 92160, 92728 },\n\t{ 92736, 92766 },\n\t{ 92784, 92862 },\n\t{ 92880, 92909 },\n\t{ 92928, 92975 },\n\t{ 92992, 92995 },\n\t{ 93027, 93047 },\n\t{ 93053, 93071 },\n\t{ 93760, 93823 },\n\t{ 93952, 94026 },\n\t{ 94032, 94032 },\n\t{ 94099, 94111 },\n\t{ 94176, 94177 },\n\t{ 94179, 94179 },\n\t{ 94208, 100343 },\n\t{ 100352, 101589 },\n\t{ 101632, 101640 },\n\t{ 110576, 110579 },\n\t{ 110581, 110587 },\n\t{ 110589, 110590 },\n\t{ 110592, 110882 },\n\t{ 110898, 110898 },\n\t{ 110928, 110930 },\n\t{ 110933, 110933 },\n\t{ 110948, 110951 },\n\t{ 110960, 111355 },\n\t{ 113664, 113770 },\n\t{ 113776, 113788 },\n\t{ 113792, 113800 },\n\t{ 113808, 113817 },\n\t{ 119808, 119892 },\n\t{ 119894, 119964 },\n\t{ 119966, 119967 },\n\t{ 119970, 119970 },\n\t{ 119973, 119974 },\n\t{ 119977, 119980 },\n\t{ 119982, 119993 },\n\t{ 119995, 119995 },\n\t{ 119997, 120003 },\n\t{ 120005, 120069 },\n\t{ 120071, 120074 },\n\t{ 120077, 120084 },\n\t{ 120086, 120092 },\n\t{ 120094, 120121 },\n\t{ 120123, 120126 },\n\t{ 120128, 120132 },\n\t{ 120134, 120134 },\n\t{ 120138, 120144 },\n\t{ 120146, 120485 },\n\t{ 120488, 120512 },\n\t{ 120514, 120538 },\n\t{ 120540, 120570 },\n\t{ 120572, 120596 },\n\t{ 120598, 120628 },\n\t{ 120630, 120654 },\n\t{ 120656, 120686 },\n\t{ 120688, 120712 },\n\t{ 120714, 120744 },\n\t{ 120746, 120770 },\n\t{ 120772, 120779 },\n\t{ 122624, 122654 },\n\t{ 122661, 122666 },\n\t{ 122928, 122989 },\n\t{ 123136, 123180 },\n\t{ 123191, 123197 },\n\t{ 123214, 123214 },\n\t{ 123536, 123565 },\n\t{ 123584, 123627 },\n\t{ 124112, 124139 },\n\t{ 124896, 124902 },\n\t{ 124904, 124907 },\n\t{ 124909, 124910 },\n\t{ 124912, 124926 },\n\t{ 124928, 125124 },\n\t{ 125184, 125251 },\n\t{ 125259, 125259 },\n\t{ 126464, 126467 },\n\t{ 126469, 126495 },\n\t{ 126497, 126498 },\n\t{ 126500, 126500 },\n\t{ 126503, 126503 },\n\t{ 126505, 126514 },\n\t{ 126516, 126519 },\n\t{ 126521, 126521 },\n\t{ 126523, 126523 },\n\t{ 126530, 126530 },\n\t{ 126535, 126535 },\n\t{ 126537, 126537 },\n\t{ 126539, 126539 },\n\t{ 126541, 126543 },\n\t{ 126545, 126546 },\n\t{ 126548, 126548 },\n\t{ 126551, 126551 },\n\t{ 126553, 126553 },\n\t{ 126555, 126555 },\n\t{ 126557, 126557 },\n\t{ 126559, 126559 },\n\t{ 126561, 126562 },\n\t{ 126564, 126564 },\n\t{ 126567, 126570 },\n\t{ 126572, 126578 },\n\t{ 126580, 126583 },\n\t{ 126585, 126588 },\n\t{ 126590, 126590 },\n\t{ 126592, 126601 },\n\t{ 126603, 126619 },\n\t{ 126625, 126627 },\n\t{ 126629, 126633 },\n\t{ 126635, 126651 },\n\t{ 131072, 173791 },\n\t{ 173824, 177977 },\n\t{ 177984, 178205 },\n\t{ 178208, 183969 },\n\t{ 183984, 191456 },\n\t{ 191472, 192093 },\n\t{ 194560, 195101 },\n\t{ 196608, 201546 },\n\t{ 201552, 205743 },\n};\nstatic const URange16 Ll_range16[] = {\n\t{ 97, 122 },\n\t{ 181, 181 },\n\t{ 223, 246 },\n\t{ 248, 255 },\n\t{ 257, 257 },\n\t{ 259, 259 },\n\t{ 261, 261 },\n\t{ 263, 263 },\n\t{ 265, 265 },\n\t{ 267, 267 },\n\t{ 269, 269 },\n\t{ 271, 271 },\n\t{ 273, 273 },\n\t{ 275, 275 },\n\t{ 277, 277 },\n\t{ 279, 279 },\n\t{ 281, 281 },\n\t{ 283, 283 },\n\t{ 285, 285 },\n\t{ 287, 287 },\n\t{ 289, 289 },\n\t{ 291, 291 },\n\t{ 293, 293 },\n\t{ 295, 295 },\n\t{ 297, 297 },\n\t{ 299, 299 },\n\t{ 301, 301 },\n\t{ 303, 303 },\n\t{ 305, 305 },\n\t{ 307, 307 },\n\t{ 309, 309 },\n\t{ 311, 312 },\n\t{ 314, 314 },\n\t{ 316, 316 },\n\t{ 318, 318 },\n\t{ 320, 320 },\n\t{ 322, 322 },\n\t{ 324, 324 },\n\t{ 326, 326 },\n\t{ 328, 329 },\n\t{ 331, 331 },\n\t{ 333, 333 },\n\t{ 335, 335 },\n\t{ 337, 337 },\n\t{ 339, 339 },\n\t{ 341, 341 },\n\t{ 343, 343 },\n\t{ 345, 345 },\n\t{ 347, 347 },\n\t{ 349, 349 },\n\t{ 351, 351 },\n\t{ 353, 353 },\n\t{ 355, 355 },\n\t{ 357, 357 },\n\t{ 359, 359 },\n\t{ 361, 361 },\n\t{ 363, 363 },\n\t{ 365, 365 },\n\t{ 367, 367 },\n\t{ 369, 369 },\n\t{ 371, 371 },\n\t{ 373, 373 },\n\t{ 375, 375 },\n\t{ 378, 378 },\n\t{ 380, 380 },\n\t{ 382, 384 },\n\t{ 387, 387 },\n\t{ 389, 389 },\n\t{ 392, 392 },\n\t{ 396, 397 },\n\t{ 402, 402 },\n\t{ 405, 405 },\n\t{ 409, 411 },\n\t{ 414, 414 },\n\t{ 417, 417 },\n\t{ 419, 419 },\n\t{ 421, 421 },\n\t{ 424, 424 },\n\t{ 426, 427 },\n\t{ 429, 429 },\n\t{ 432, 432 },\n\t{ 436, 436 },\n\t{ 438, 438 },\n\t{ 441, 442 },\n\t{ 445, 447 },\n\t{ 454, 454 },\n\t{ 457, 457 },\n\t{ 460, 460 },\n\t{ 462, 462 },\n\t{ 464, 464 },\n\t{ 466, 466 },\n\t{ 468, 468 },\n\t{ 470, 470 },\n\t{ 472, 472 },\n\t{ 474, 474 },\n\t{ 476, 477 },\n\t{ 479, 479 },\n\t{ 481, 481 },\n\t{ 483, 483 },\n\t{ 485, 485 },\n\t{ 487, 487 },\n\t{ 489, 489 },\n\t{ 491, 491 },\n\t{ 493, 493 },\n\t{ 495, 496 },\n\t{ 499, 499 },\n\t{ 501, 501 },\n\t{ 505, 505 },\n\t{ 507, 507 },\n\t{ 509, 509 },\n\t{ 511, 511 },\n\t{ 513, 513 },\n\t{ 515, 515 },\n\t{ 517, 517 },\n\t{ 519, 519 },\n\t{ 521, 521 },\n\t{ 523, 523 },\n\t{ 525, 525 },\n\t{ 527, 527 },\n\t{ 529, 529 },\n\t{ 531, 531 },\n\t{ 533, 533 },\n\t{ 535, 535 },\n\t{ 537, 537 },\n\t{ 539, 539 },\n\t{ 541, 541 },\n\t{ 543, 543 },\n\t{ 545, 545 },\n\t{ 547, 547 },\n\t{ 549, 549 },\n\t{ 551, 551 },\n\t{ 553, 553 },\n\t{ 555, 555 },\n\t{ 557, 557 },\n\t{ 559, 559 },\n\t{ 561, 561 },\n\t{ 563, 569 },\n\t{ 572, 572 },\n\t{ 575, 576 },\n\t{ 578, 578 },\n\t{ 583, 583 },\n\t{ 585, 585 },\n\t{ 587, 587 },\n\t{ 589, 589 },\n\t{ 591, 659 },\n\t{ 661, 687 },\n\t{ 881, 881 },\n\t{ 883, 883 },\n\t{ 887, 887 },\n\t{ 891, 893 },\n\t{ 912, 912 },\n\t{ 940, 974 },\n\t{ 976, 977 },\n\t{ 981, 983 },\n\t{ 985, 985 },\n\t{ 987, 987 },\n\t{ 989, 989 },\n\t{ 991, 991 },\n\t{ 993, 993 },\n\t{ 995, 995 },\n\t{ 997, 997 },\n\t{ 999, 999 },\n\t{ 1001, 1001 },\n\t{ 1003, 1003 },\n\t{ 1005, 1005 },\n\t{ 1007, 1011 },\n\t{ 1013, 1013 },\n\t{ 1016, 1016 },\n\t{ 1019, 1020 },\n\t{ 1072, 1119 },\n\t{ 1121, 1121 },\n\t{ 1123, 1123 },\n\t{ 1125, 1125 },\n\t{ 1127, 1127 },\n\t{ 1129, 1129 },\n\t{ 1131, 1131 },\n\t{ 1133, 1133 },\n\t{ 1135, 1135 },\n\t{ 1137, 1137 },\n\t{ 1139, 1139 },\n\t{ 1141, 1141 },\n\t{ 1143, 1143 },\n\t{ 1145, 1145 },\n\t{ 1147, 1147 },\n\t{ 1149, 1149 },\n\t{ 1151, 1151 },\n\t{ 1153, 1153 },\n\t{ 1163, 1163 },\n\t{ 1165, 1165 },\n\t{ 1167, 1167 },\n\t{ 1169, 1169 },\n\t{ 1171, 1171 },\n\t{ 1173, 1173 },\n\t{ 1175, 1175 },\n\t{ 1177, 1177 },\n\t{ 1179, 1179 },\n\t{ 1181, 1181 },\n\t{ 1183, 1183 },\n\t{ 1185, 1185 },\n\t{ 1187, 1187 },\n\t{ 1189, 1189 },\n\t{ 1191, 1191 },\n\t{ 1193, 1193 },\n\t{ 1195, 1195 },\n\t{ 1197, 1197 },\n\t{ 1199, 1199 },\n\t{ 1201, 1201 },\n\t{ 1203, 1203 },\n\t{ 1205, 1205 },\n\t{ 1207, 1207 },\n\t{ 1209, 1209 },\n\t{ 1211, 1211 },\n\t{ 1213, 1213 },\n\t{ 1215, 1215 },\n\t{ 1218, 1218 },\n\t{ 1220, 1220 },\n\t{ 1222, 1222 },\n\t{ 1224, 1224 },\n\t{ 1226, 1226 },\n\t{ 1228, 1228 },\n\t{ 1230, 1231 },\n\t{ 1233, 1233 },\n\t{ 1235, 1235 },\n\t{ 1237, 1237 },\n\t{ 1239, 1239 },\n\t{ 1241, 1241 },\n\t{ 1243, 1243 },\n\t{ 1245, 1245 },\n\t{ 1247, 1247 },\n\t{ 1249, 1249 },\n\t{ 1251, 1251 },\n\t{ 1253, 1253 },\n\t{ 1255, 1255 },\n\t{ 1257, 1257 },\n\t{ 1259, 1259 },\n\t{ 1261, 1261 },\n\t{ 1263, 1263 },\n\t{ 1265, 1265 },\n\t{ 1267, 1267 },\n\t{ 1269, 1269 },\n\t{ 1271, 1271 },\n\t{ 1273, 1273 },\n\t{ 1275, 1275 },\n\t{ 1277, 1277 },\n\t{ 1279, 1279 },\n\t{ 1281, 1281 },\n\t{ 1283, 1283 },\n\t{ 1285, 1285 },\n\t{ 1287, 1287 },\n\t{ 1289, 1289 },\n\t{ 1291, 1291 },\n\t{ 1293, 1293 },\n\t{ 1295, 1295 },\n\t{ 1297, 1297 },\n\t{ 1299, 1299 },\n\t{ 1301, 1301 },\n\t{ 1303, 1303 },\n\t{ 1305, 1305 },\n\t{ 1307, 1307 },\n\t{ 1309, 1309 },\n\t{ 1311, 1311 },\n\t{ 1313, 1313 },\n\t{ 1315, 1315 },\n\t{ 1317, 1317 },\n\t{ 1319, 1319 },\n\t{ 1321, 1321 },\n\t{ 1323, 1323 },\n\t{ 1325, 1325 },\n\t{ 1327, 1327 },\n\t{ 1376, 1416 },\n\t{ 4304, 4346 },\n\t{ 4349, 4351 },\n\t{ 5112, 5117 },\n\t{ 7296, 7304 },\n\t{ 7424, 7467 },\n\t{ 7531, 7543 },\n\t{ 7545, 7578 },\n\t{ 7681, 7681 },\n\t{ 7683, 7683 },\n\t{ 7685, 7685 },\n\t{ 7687, 7687 },\n\t{ 7689, 7689 },\n\t{ 7691, 7691 },\n\t{ 7693, 7693 },\n\t{ 7695, 7695 },\n\t{ 7697, 7697 },\n\t{ 7699, 7699 },\n\t{ 7701, 7701 },\n\t{ 7703, 7703 },\n\t{ 7705, 7705 },\n\t{ 7707, 7707 },\n\t{ 7709, 7709 },\n\t{ 7711, 7711 },\n\t{ 7713, 7713 },\n\t{ 7715, 7715 },\n\t{ 7717, 7717 },\n\t{ 7719, 7719 },\n\t{ 7721, 7721 },\n\t{ 7723, 7723 },\n\t{ 7725, 7725 },\n\t{ 7727, 7727 },\n\t{ 7729, 7729 },\n\t{ 7731, 7731 },\n\t{ 7733, 7733 },\n\t{ 7735, 7735 },\n\t{ 7737, 7737 },\n\t{ 7739, 7739 },\n\t{ 7741, 7741 },\n\t{ 7743, 7743 },\n\t{ 7745, 7745 },\n\t{ 7747, 7747 },\n\t{ 7749, 7749 },\n\t{ 7751, 7751 },\n\t{ 7753, 7753 },\n\t{ 7755, 7755 },\n\t{ 7757, 7757 },\n\t{ 7759, 7759 },\n\t{ 7761, 7761 },\n\t{ 7763, 7763 },\n\t{ 7765, 7765 },\n\t{ 7767, 7767 },\n\t{ 7769, 7769 },\n\t{ 7771, 7771 },\n\t{ 7773, 7773 },\n\t{ 7775, 7775 },\n\t{ 7777, 7777 },\n\t{ 7779, 7779 },\n\t{ 7781, 7781 },\n\t{ 7783, 7783 },\n\t{ 7785, 7785 },\n\t{ 7787, 7787 },\n\t{ 7789, 7789 },\n\t{ 7791, 7791 },\n\t{ 7793, 7793 },\n\t{ 7795, 7795 },\n\t{ 7797, 7797 },\n\t{ 7799, 7799 },\n\t{ 7801, 7801 },\n\t{ 7803, 7803 },\n\t{ 7805, 7805 },\n\t{ 7807, 7807 },\n\t{ 7809, 7809 },\n\t{ 7811, 7811 },\n\t{ 7813, 7813 },\n\t{ 7815, 7815 },\n\t{ 7817, 7817 },\n\t{ 7819, 7819 },\n\t{ 7821, 7821 },\n\t{ 7823, 7823 },\n\t{ 7825, 7825 },\n\t{ 7827, 7827 },\n\t{ 7829, 7837 },\n\t{ 7839, 7839 },\n\t{ 7841, 7841 },\n\t{ 7843, 7843 },\n\t{ 7845, 7845 },\n\t{ 7847, 7847 },\n\t{ 7849, 7849 },\n\t{ 7851, 7851 },\n\t{ 7853, 7853 },\n\t{ 7855, 7855 },\n\t{ 7857, 7857 },\n\t{ 7859, 7859 },\n\t{ 7861, 7861 },\n\t{ 7863, 7863 },\n\t{ 7865, 7865 },\n\t{ 7867, 7867 },\n\t{ 7869, 7869 },\n\t{ 7871, 7871 },\n\t{ 7873, 7873 },\n\t{ 7875, 7875 },\n\t{ 7877, 7877 },\n\t{ 7879, 7879 },\n\t{ 7881, 7881 },\n\t{ 7883, 7883 },\n\t{ 7885, 7885 },\n\t{ 7887, 7887 },\n\t{ 7889, 7889 },\n\t{ 7891, 7891 },\n\t{ 7893, 7893 },\n\t{ 7895, 7895 },\n\t{ 7897, 7897 },\n\t{ 7899, 7899 },\n\t{ 7901, 7901 },\n\t{ 7903, 7903 },\n\t{ 7905, 7905 },\n\t{ 7907, 7907 },\n\t{ 7909, 7909 },\n\t{ 7911, 7911 },\n\t{ 7913, 7913 },\n\t{ 7915, 7915 },\n\t{ 7917, 7917 },\n\t{ 7919, 7919 },\n\t{ 7921, 7921 },\n\t{ 7923, 7923 },\n\t{ 7925, 7925 },\n\t{ 7927, 7927 },\n\t{ 7929, 7929 },\n\t{ 7931, 7931 },\n\t{ 7933, 7933 },\n\t{ 7935, 7943 },\n\t{ 7952, 7957 },\n\t{ 7968, 7975 },\n\t{ 7984, 7991 },\n\t{ 8000, 8005 },\n\t{ 8016, 8023 },\n\t{ 8032, 8039 },\n\t{ 8048, 8061 },\n\t{ 8064, 8071 },\n\t{ 8080, 8087 },\n\t{ 8096, 8103 },\n\t{ 8112, 8116 },\n\t{ 8118, 8119 },\n\t{ 8126, 8126 },\n\t{ 8130, 8132 },\n\t{ 8134, 8135 },\n\t{ 8144, 8147 },\n\t{ 8150, 8151 },\n\t{ 8160, 8167 },\n\t{ 8178, 8180 },\n\t{ 8182, 8183 },\n\t{ 8458, 8458 },\n\t{ 8462, 8463 },\n\t{ 8467, 8467 },\n\t{ 8495, 8495 },\n\t{ 8500, 8500 },\n\t{ 8505, 8505 },\n\t{ 8508, 8509 },\n\t{ 8518, 8521 },\n\t{ 8526, 8526 },\n\t{ 8580, 8580 },\n\t{ 11312, 11359 },\n\t{ 11361, 11361 },\n\t{ 11365, 11366 },\n\t{ 11368, 11368 },\n\t{ 11370, 11370 },\n\t{ 11372, 11372 },\n\t{ 11377, 11377 },\n\t{ 11379, 11380 },\n\t{ 11382, 11387 },\n\t{ 11393, 11393 },\n\t{ 11395, 11395 },\n\t{ 11397, 11397 },\n\t{ 11399, 11399 },\n\t{ 11401, 11401 },\n\t{ 11403, 11403 },\n\t{ 11405, 11405 },\n\t{ 11407, 11407 },\n\t{ 11409, 11409 },\n\t{ 11411, 11411 },\n\t{ 11413, 11413 },\n\t{ 11415, 11415 },\n\t{ 11417, 11417 },\n\t{ 11419, 11419 },\n\t{ 11421, 11421 },\n\t{ 11423, 11423 },\n\t{ 11425, 11425 },\n\t{ 11427, 11427 },\n\t{ 11429, 11429 },\n\t{ 11431, 11431 },\n\t{ 11433, 11433 },\n\t{ 11435, 11435 },\n\t{ 11437, 11437 },\n\t{ 11439, 11439 },\n\t{ 11441, 11441 },\n\t{ 11443, 11443 },\n\t{ 11445, 11445 },\n\t{ 11447, 11447 },\n\t{ 11449, 11449 },\n\t{ 11451, 11451 },\n\t{ 11453, 11453 },\n\t{ 11455, 11455 },\n\t{ 11457, 11457 },\n\t{ 11459, 11459 },\n\t{ 11461, 11461 },\n\t{ 11463, 11463 },\n\t{ 11465, 11465 },\n\t{ 11467, 11467 },\n\t{ 11469, 11469 },\n\t{ 11471, 11471 },\n\t{ 11473, 11473 },\n\t{ 11475, 11475 },\n\t{ 11477, 11477 },\n\t{ 11479, 11479 },\n\t{ 11481, 11481 },\n\t{ 11483, 11483 },\n\t{ 11485, 11485 },\n\t{ 11487, 11487 },\n\t{ 11489, 11489 },\n\t{ 11491, 11492 },\n\t{ 11500, 11500 },\n\t{ 11502, 11502 },\n\t{ 11507, 11507 },\n\t{ 11520, 11557 },\n\t{ 11559, 11559 },\n\t{ 11565, 11565 },\n\t{ 42561, 42561 },\n\t{ 42563, 42563 },\n\t{ 42565, 42565 },\n\t{ 42567, 42567 },\n\t{ 42569, 42569 },\n\t{ 42571, 42571 },\n\t{ 42573, 42573 },\n\t{ 42575, 42575 },\n\t{ 42577, 42577 },\n\t{ 42579, 42579 },\n\t{ 42581, 42581 },\n\t{ 42583, 42583 },\n\t{ 42585, 42585 },\n\t{ 42587, 42587 },\n\t{ 42589, 42589 },\n\t{ 42591, 42591 },\n\t{ 42593, 42593 },\n\t{ 42595, 42595 },\n\t{ 42597, 42597 },\n\t{ 42599, 42599 },\n\t{ 42601, 42601 },\n\t{ 42603, 42603 },\n\t{ 42605, 42605 },\n\t{ 42625, 42625 },\n\t{ 42627, 42627 },\n\t{ 42629, 42629 },\n\t{ 42631, 42631 },\n\t{ 42633, 42633 },\n\t{ 42635, 42635 },\n\t{ 42637, 42637 },\n\t{ 42639, 42639 },\n\t{ 42641, 42641 },\n\t{ 42643, 42643 },\n\t{ 42645, 42645 },\n\t{ 42647, 42647 },\n\t{ 42649, 42649 },\n\t{ 42651, 42651 },\n\t{ 42787, 42787 },\n\t{ 42789, 42789 },\n\t{ 42791, 42791 },\n\t{ 42793, 42793 },\n\t{ 42795, 42795 },\n\t{ 42797, 42797 },\n\t{ 42799, 42801 },\n\t{ 42803, 42803 },\n\t{ 42805, 42805 },\n\t{ 42807, 42807 },\n\t{ 42809, 42809 },\n\t{ 42811, 42811 },\n\t{ 42813, 42813 },\n\t{ 42815, 42815 },\n\t{ 42817, 42817 },\n\t{ 42819, 42819 },\n\t{ 42821, 42821 },\n\t{ 42823, 42823 },\n\t{ 42825, 42825 },\n\t{ 42827, 42827 },\n\t{ 42829, 42829 },\n\t{ 42831, 42831 },\n\t{ 42833, 42833 },\n\t{ 42835, 42835 },\n\t{ 42837, 42837 },\n\t{ 42839, 42839 },\n\t{ 42841, 42841 },\n\t{ 42843, 42843 },\n\t{ 42845, 42845 },\n\t{ 42847, 42847 },\n\t{ 42849, 42849 },\n\t{ 42851, 42851 },\n\t{ 42853, 42853 },\n\t{ 42855, 42855 },\n\t{ 42857, 42857 },\n\t{ 42859, 42859 },\n\t{ 42861, 42861 },\n\t{ 42863, 42863 },\n\t{ 42865, 42872 },\n\t{ 42874, 42874 },\n\t{ 42876, 42876 },\n\t{ 42879, 42879 },\n\t{ 42881, 42881 },\n\t{ 42883, 42883 },\n\t{ 42885, 42885 },\n\t{ 42887, 42887 },\n\t{ 42892, 42892 },\n\t{ 42894, 42894 },\n\t{ 42897, 42897 },\n\t{ 42899, 42901 },\n\t{ 42903, 42903 },\n\t{ 42905, 42905 },\n\t{ 42907, 42907 },\n\t{ 42909, 42909 },\n\t{ 42911, 42911 },\n\t{ 42913, 42913 },\n\t{ 42915, 42915 },\n\t{ 42917, 42917 },\n\t{ 42919, 42919 },\n\t{ 42921, 42921 },\n\t{ 42927, 42927 },\n\t{ 42933, 42933 },\n\t{ 42935, 42935 },\n\t{ 42937, 42937 },\n\t{ 42939, 42939 },\n\t{ 42941, 42941 },\n\t{ 42943, 42943 },\n\t{ 42945, 42945 },\n\t{ 42947, 42947 },\n\t{ 42952, 42952 },\n\t{ 42954, 42954 },\n\t{ 42961, 42961 },\n\t{ 42963, 42963 },\n\t{ 42965, 42965 },\n\t{ 42967, 42967 },\n\t{ 42969, 42969 },\n\t{ 42998, 42998 },\n\t{ 43002, 43002 },\n\t{ 43824, 43866 },\n\t{ 43872, 43880 },\n\t{ 43888, 43967 },\n\t{ 64256, 64262 },\n\t{ 64275, 64279 },\n\t{ 65345, 65370 },\n};\nstatic const URange32 Ll_range32[] = {\n\t{ 66600, 66639 },\n\t{ 66776, 66811 },\n\t{ 66967, 66977 },\n\t{ 66979, 66993 },\n\t{ 66995, 67001 },\n\t{ 67003, 67004 },\n\t{ 68800, 68850 },\n\t{ 71872, 71903 },\n\t{ 93792, 93823 },\n\t{ 119834, 119859 },\n\t{ 119886, 119892 },\n\t{ 119894, 119911 },\n\t{ 119938, 119963 },\n\t{ 119990, 119993 },\n\t{ 119995, 119995 },\n\t{ 119997, 120003 },\n\t{ 120005, 120015 },\n\t{ 120042, 120067 },\n\t{ 120094, 120119 },\n\t{ 120146, 120171 },\n\t{ 120198, 120223 },\n\t{ 120250, 120275 },\n\t{ 120302, 120327 },\n\t{ 120354, 120379 },\n\t{ 120406, 120431 },\n\t{ 120458, 120485 },\n\t{ 120514, 120538 },\n\t{ 120540, 120545 },\n\t{ 120572, 120596 },\n\t{ 120598, 120603 },\n\t{ 120630, 120654 },\n\t{ 120656, 120661 },\n\t{ 120688, 120712 },\n\t{ 120714, 120719 },\n\t{ 120746, 120770 },\n\t{ 120772, 120777 },\n\t{ 120779, 120779 },\n\t{ 122624, 122633 },\n\t{ 122635, 122654 },\n\t{ 122661, 122666 },\n\t{ 125218, 125251 },\n};\nstatic const URange16 Lm_range16[] = {\n\t{ 688, 705 },\n\t{ 710, 721 },\n\t{ 736, 740 },\n\t{ 748, 748 },\n\t{ 750, 750 },\n\t{ 884, 884 },\n\t{ 890, 890 },\n\t{ 1369, 1369 },\n\t{ 1600, 1600 },\n\t{ 1765, 1766 },\n\t{ 2036, 2037 },\n\t{ 2042, 2042 },\n\t{ 2074, 2074 },\n\t{ 2084, 2084 },\n\t{ 2088, 2088 },\n\t{ 2249, 2249 },\n\t{ 2417, 2417 },\n\t{ 3654, 3654 },\n\t{ 3782, 3782 },\n\t{ 4348, 4348 },\n\t{ 6103, 6103 },\n\t{ 6211, 6211 },\n\t{ 6823, 6823 },\n\t{ 7288, 7293 },\n\t{ 7468, 7530 },\n\t{ 7544, 7544 },\n\t{ 7579, 7615 },\n\t{ 8305, 8305 },\n\t{ 8319, 8319 },\n\t{ 8336, 8348 },\n\t{ 11388, 11389 },\n\t{ 11631, 11631 },\n\t{ 11823, 11823 },\n\t{ 12293, 12293 },\n\t{ 12337, 12341 },\n\t{ 12347, 12347 },\n\t{ 12445, 12446 },\n\t{ 12540, 12542 },\n\t{ 40981, 40981 },\n\t{ 42232, 42237 },\n\t{ 42508, 42508 },\n\t{ 42623, 42623 },\n\t{ 42652, 42653 },\n\t{ 42775, 42783 },\n\t{ 42864, 42864 },\n\t{ 42888, 42888 },\n\t{ 42994, 42996 },\n\t{ 43000, 43001 },\n\t{ 43471, 43471 },\n\t{ 43494, 43494 },\n\t{ 43632, 43632 },\n\t{ 43741, 43741 },\n\t{ 43763, 43764 },\n\t{ 43868, 43871 },\n\t{ 43881, 43881 },\n\t{ 65392, 65392 },\n\t{ 65438, 65439 },\n};\nstatic const URange32 Lm_range32[] = {\n\t{ 67456, 67461 },\n\t{ 67463, 67504 },\n\t{ 67506, 67514 },\n\t{ 92992, 92995 },\n\t{ 94099, 94111 },\n\t{ 94176, 94177 },\n\t{ 94179, 94179 },\n\t{ 110576, 110579 },\n\t{ 110581, 110587 },\n\t{ 110589, 110590 },\n\t{ 122928, 122989 },\n\t{ 123191, 123197 },\n\t{ 124139, 124139 },\n\t{ 125259, 125259 },\n};\nstatic const URange16 Lo_range16[] = {\n\t{ 170, 170 },\n\t{ 186, 186 },\n\t{ 443, 443 },\n\t{ 448, 451 },\n\t{ 660, 660 },\n\t{ 1488, 1514 },\n\t{ 1519, 1522 },\n\t{ 1568, 1599 },\n\t{ 1601, 1610 },\n\t{ 1646, 1647 },\n\t{ 1649, 1747 },\n\t{ 1749, 1749 },\n\t{ 1774, 1775 },\n\t{ 1786, 1788 },\n\t{ 1791, 1791 },\n\t{ 1808, 1808 },\n\t{ 1810, 1839 },\n\t{ 1869, 1957 },\n\t{ 1969, 1969 },\n\t{ 1994, 2026 },\n\t{ 2048, 2069 },\n\t{ 2112, 2136 },\n\t{ 2144, 2154 },\n\t{ 2160, 2183 },\n\t{ 2185, 2190 },\n\t{ 2208, 2248 },\n\t{ 2308, 2361 },\n\t{ 2365, 2365 },\n\t{ 2384, 2384 },\n\t{ 2392, 2401 },\n\t{ 2418, 2432 },\n\t{ 2437, 2444 },\n\t{ 2447, 2448 },\n\t{ 2451, 2472 },\n\t{ 2474, 2480 },\n\t{ 2482, 2482 },\n\t{ 2486, 2489 },\n\t{ 2493, 2493 },\n\t{ 2510, 2510 },\n\t{ 2524, 2525 },\n\t{ 2527, 2529 },\n\t{ 2544, 2545 },\n\t{ 2556, 2556 },\n\t{ 2565, 2570 },\n\t{ 2575, 2576 },\n\t{ 2579, 2600 },\n\t{ 2602, 2608 },\n\t{ 2610, 2611 },\n\t{ 2613, 2614 },\n\t{ 2616, 2617 },\n\t{ 2649, 2652 },\n\t{ 2654, 2654 },\n\t{ 2674, 2676 },\n\t{ 2693, 2701 },\n\t{ 2703, 2705 },\n\t{ 2707, 2728 },\n\t{ 2730, 2736 },\n\t{ 2738, 2739 },\n\t{ 2741, 2745 },\n\t{ 2749, 2749 },\n\t{ 2768, 2768 },\n\t{ 2784, 2785 },\n\t{ 2809, 2809 },\n\t{ 2821, 2828 },\n\t{ 2831, 2832 },\n\t{ 2835, 2856 },\n\t{ 2858, 2864 },\n\t{ 2866, 2867 },\n\t{ 2869, 2873 },\n\t{ 2877, 2877 },\n\t{ 2908, 2909 },\n\t{ 2911, 2913 },\n\t{ 2929, 2929 },\n\t{ 2947, 2947 },\n\t{ 2949, 2954 },\n\t{ 2958, 2960 },\n\t{ 2962, 2965 },\n\t{ 2969, 2970 },\n\t{ 2972, 2972 },\n\t{ 2974, 2975 },\n\t{ 2979, 2980 },\n\t{ 2984, 2986 },\n\t{ 2990, 3001 },\n\t{ 3024, 3024 },\n\t{ 3077, 3084 },\n\t{ 3086, 3088 },\n\t{ 3090, 3112 },\n\t{ 3114, 3129 },\n\t{ 3133, 3133 },\n\t{ 3160, 3162 },\n\t{ 3165, 3165 },\n\t{ 3168, 3169 },\n\t{ 3200, 3200 },\n\t{ 3205, 3212 },\n\t{ 3214, 3216 },\n\t{ 3218, 3240 },\n\t{ 3242, 3251 },\n\t{ 3253, 3257 },\n\t{ 3261, 3261 },\n\t{ 3293, 3294 },\n\t{ 3296, 3297 },\n\t{ 3313, 3314 },\n\t{ 3332, 3340 },\n\t{ 3342, 3344 },\n\t{ 3346, 3386 },\n\t{ 3389, 3389 },\n\t{ 3406, 3406 },\n\t{ 3412, 3414 },\n\t{ 3423, 3425 },\n\t{ 3450, 3455 },\n\t{ 3461, 3478 },\n\t{ 3482, 3505 },\n\t{ 3507, 3515 },\n\t{ 3517, 3517 },\n\t{ 3520, 3526 },\n\t{ 3585, 3632 },\n\t{ 3634, 3635 },\n\t{ 3648, 3653 },\n\t{ 3713, 3714 },\n\t{ 3716, 3716 },\n\t{ 3718, 3722 },\n\t{ 3724, 3747 },\n\t{ 3749, 3749 },\n\t{ 3751, 3760 },\n\t{ 3762, 3763 },\n\t{ 3773, 3773 },\n\t{ 3776, 3780 },\n\t{ 3804, 3807 },\n\t{ 3840, 3840 },\n\t{ 3904, 3911 },\n\t{ 3913, 3948 },\n\t{ 3976, 3980 },\n\t{ 4096, 4138 },\n\t{ 4159, 4159 },\n\t{ 4176, 4181 },\n\t{ 4186, 4189 },\n\t{ 4193, 4193 },\n\t{ 4197, 4198 },\n\t{ 4206, 4208 },\n\t{ 4213, 4225 },\n\t{ 4238, 4238 },\n\t{ 4352, 4680 },\n\t{ 4682, 4685 },\n\t{ 4688, 4694 },\n\t{ 4696, 4696 },\n\t{ 4698, 4701 },\n\t{ 4704, 4744 },\n\t{ 4746, 4749 },\n\t{ 4752, 4784 },\n\t{ 4786, 4789 },\n\t{ 4792, 4798 },\n\t{ 4800, 4800 },\n\t{ 4802, 4805 },\n\t{ 4808, 4822 },\n\t{ 4824, 4880 },\n\t{ 4882, 4885 },\n\t{ 4888, 4954 },\n\t{ 4992, 5007 },\n\t{ 5121, 5740 },\n\t{ 5743, 5759 },\n\t{ 5761, 5786 },\n\t{ 5792, 5866 },\n\t{ 5873, 5880 },\n\t{ 5888, 5905 },\n\t{ 5919, 5937 },\n\t{ 5952, 5969 },\n\t{ 5984, 5996 },\n\t{ 5998, 6000 },\n\t{ 6016, 6067 },\n\t{ 6108, 6108 },\n\t{ 6176, 6210 },\n\t{ 6212, 6264 },\n\t{ 6272, 6276 },\n\t{ 6279, 6312 },\n\t{ 6314, 6314 },\n\t{ 6320, 6389 },\n\t{ 6400, 6430 },\n\t{ 6480, 6509 },\n\t{ 6512, 6516 },\n\t{ 6528, 6571 },\n\t{ 6576, 6601 },\n\t{ 6656, 6678 },\n\t{ 6688, 6740 },\n\t{ 6917, 6963 },\n\t{ 6981, 6988 },\n\t{ 7043, 7072 },\n\t{ 7086, 7087 },\n\t{ 7098, 7141 },\n\t{ 7168, 7203 },\n\t{ 7245, 7247 },\n\t{ 7258, 7287 },\n\t{ 7401, 7404 },\n\t{ 7406, 7411 },\n\t{ 7413, 7414 },\n\t{ 7418, 7418 },\n\t{ 8501, 8504 },\n\t{ 11568, 11623 },\n\t{ 11648, 11670 },\n\t{ 11680, 11686 },\n\t{ 11688, 11694 },\n\t{ 11696, 11702 },\n\t{ 11704, 11710 },\n\t{ 11712, 11718 },\n\t{ 11720, 11726 },\n\t{ 11728, 11734 },\n\t{ 11736, 11742 },\n\t{ 12294, 12294 },\n\t{ 12348, 12348 },\n\t{ 12353, 12438 },\n\t{ 12447, 12447 },\n\t{ 12449, 12538 },\n\t{ 12543, 12543 },\n\t{ 12549, 12591 },\n\t{ 12593, 12686 },\n\t{ 12704, 12735 },\n\t{ 12784, 12799 },\n\t{ 13312, 19903 },\n\t{ 19968, 40980 },\n\t{ 40982, 42124 },\n\t{ 42192, 42231 },\n\t{ 42240, 42507 },\n\t{ 42512, 42527 },\n\t{ 42538, 42539 },\n\t{ 42606, 42606 },\n\t{ 42656, 42725 },\n\t{ 42895, 42895 },\n\t{ 42999, 42999 },\n\t{ 43003, 43009 },\n\t{ 43011, 43013 },\n\t{ 43015, 43018 },\n\t{ 43020, 43042 },\n\t{ 43072, 43123 },\n\t{ 43138, 43187 },\n\t{ 43250, 43255 },\n\t{ 43259, 43259 },\n\t{ 43261, 43262 },\n\t{ 43274, 43301 },\n\t{ 43312, 43334 },\n\t{ 43360, 43388 },\n\t{ 43396, 43442 },\n\t{ 43488, 43492 },\n\t{ 43495, 43503 },\n\t{ 43514, 43518 },\n\t{ 43520, 43560 },\n\t{ 43584, 43586 },\n\t{ 43588, 43595 },\n\t{ 43616, 43631 },\n\t{ 43633, 43638 },\n\t{ 43642, 43642 },\n\t{ 43646, 43695 },\n\t{ 43697, 43697 },\n\t{ 43701, 43702 },\n\t{ 43705, 43709 },\n\t{ 43712, 43712 },\n\t{ 43714, 43714 },\n\t{ 43739, 43740 },\n\t{ 43744, 43754 },\n\t{ 43762, 43762 },\n\t{ 43777, 43782 },\n\t{ 43785, 43790 },\n\t{ 43793, 43798 },\n\t{ 43808, 43814 },\n\t{ 43816, 43822 },\n\t{ 43968, 44002 },\n\t{ 44032, 55203 },\n\t{ 55216, 55238 },\n\t{ 55243, 55291 },\n\t{ 63744, 64109 },\n\t{ 64112, 64217 },\n\t{ 64285, 64285 },\n\t{ 64287, 64296 },\n\t{ 64298, 64310 },\n\t{ 64312, 64316 },\n\t{ 64318, 64318 },\n\t{ 64320, 64321 },\n\t{ 64323, 64324 },\n\t{ 64326, 64433 },\n\t{ 64467, 64829 },\n\t{ 64848, 64911 },\n\t{ 64914, 64967 },\n\t{ 65008, 65019 },\n\t{ 65136, 65140 },\n\t{ 65142, 65276 },\n\t{ 65382, 65391 },\n\t{ 65393, 65437 },\n\t{ 65440, 65470 },\n\t{ 65474, 65479 },\n\t{ 65482, 65487 },\n\t{ 65490, 65495 },\n\t{ 65498, 65500 },\n};\nstatic const URange32 Lo_range32[] = {\n\t{ 65536, 65547 },\n\t{ 65549, 65574 },\n\t{ 65576, 65594 },\n\t{ 65596, 65597 },\n\t{ 65599, 65613 },\n\t{ 65616, 65629 },\n\t{ 65664, 65786 },\n\t{ 66176, 66204 },\n\t{ 66208, 66256 },\n\t{ 66304, 66335 },\n\t{ 66349, 66368 },\n\t{ 66370, 66377 },\n\t{ 66384, 66421 },\n\t{ 66432, 66461 },\n\t{ 66464, 66499 },\n\t{ 66504, 66511 },\n\t{ 66640, 66717 },\n\t{ 66816, 66855 },\n\t{ 66864, 66915 },\n\t{ 67072, 67382 },\n\t{ 67392, 67413 },\n\t{ 67424, 67431 },\n\t{ 67584, 67589 },\n\t{ 67592, 67592 },\n\t{ 67594, 67637 },\n\t{ 67639, 67640 },\n\t{ 67644, 67644 },\n\t{ 67647, 67669 },\n\t{ 67680, 67702 },\n\t{ 67712, 67742 },\n\t{ 67808, 67826 },\n\t{ 67828, 67829 },\n\t{ 67840, 67861 },\n\t{ 67872, 67897 },\n\t{ 67968, 68023 },\n\t{ 68030, 68031 },\n\t{ 68096, 68096 },\n\t{ 68112, 68115 },\n\t{ 68117, 68119 },\n\t{ 68121, 68149 },\n\t{ 68192, 68220 },\n\t{ 68224, 68252 },\n\t{ 68288, 68295 },\n\t{ 68297, 68324 },\n\t{ 68352, 68405 },\n\t{ 68416, 68437 },\n\t{ 68448, 68466 },\n\t{ 68480, 68497 },\n\t{ 68608, 68680 },\n\t{ 68864, 68899 },\n\t{ 69248, 69289 },\n\t{ 69296, 69297 },\n\t{ 69376, 69404 },\n\t{ 69415, 69415 },\n\t{ 69424, 69445 },\n\t{ 69488, 69505 },\n\t{ 69552, 69572 },\n\t{ 69600, 69622 },\n\t{ 69635, 69687 },\n\t{ 69745, 69746 },\n\t{ 69749, 69749 },\n\t{ 69763, 69807 },\n\t{ 69840, 69864 },\n\t{ 69891, 69926 },\n\t{ 69956, 69956 },\n\t{ 69959, 69959 },\n\t{ 69968, 70002 },\n\t{ 70006, 70006 },\n\t{ 70019, 70066 },\n\t{ 70081, 70084 },\n\t{ 70106, 70106 },\n\t{ 70108, 70108 },\n\t{ 70144, 70161 },\n\t{ 70163, 70187 },\n\t{ 70207, 70208 },\n\t{ 70272, 70278 },\n\t{ 70280, 70280 },\n\t{ 70282, 70285 },\n\t{ 70287, 70301 },\n\t{ 70303, 70312 },\n\t{ 70320, 70366 },\n\t{ 70405, 70412 },\n\t{ 70415, 70416 },\n\t{ 70419, 70440 },\n\t{ 70442, 70448 },\n\t{ 70450, 70451 },\n\t{ 70453, 70457 },\n\t{ 70461, 70461 },\n\t{ 70480, 70480 },\n\t{ 70493, 70497 },\n\t{ 70656, 70708 },\n\t{ 70727, 70730 },\n\t{ 70751, 70753 },\n\t{ 70784, 70831 },\n\t{ 70852, 70853 },\n\t{ 70855, 70855 },\n\t{ 71040, 71086 },\n\t{ 71128, 71131 },\n\t{ 71168, 71215 },\n\t{ 71236, 71236 },\n\t{ 71296, 71338 },\n\t{ 71352, 71352 },\n\t{ 71424, 71450 },\n\t{ 71488, 71494 },\n\t{ 71680, 71723 },\n\t{ 71935, 71942 },\n\t{ 71945, 71945 },\n\t{ 71948, 71955 },\n\t{ 71957, 71958 },\n\t{ 71960, 71983 },\n\t{ 71999, 71999 },\n\t{ 72001, 72001 },\n\t{ 72096, 72103 },\n\t{ 72106, 72144 },\n\t{ 72161, 72161 },\n\t{ 72163, 72163 },\n\t{ 72192, 72192 },\n\t{ 72203, 72242 },\n\t{ 72250, 72250 },\n\t{ 72272, 72272 },\n\t{ 72284, 72329 },\n\t{ 72349, 72349 },\n\t{ 72368, 72440 },\n\t{ 72704, 72712 },\n\t{ 72714, 72750 },\n\t{ 72768, 72768 },\n\t{ 72818, 72847 },\n\t{ 72960, 72966 },\n\t{ 72968, 72969 },\n\t{ 72971, 73008 },\n\t{ 73030, 73030 },\n\t{ 73056, 73061 },\n\t{ 73063, 73064 },\n\t{ 73066, 73097 },\n\t{ 73112, 73112 },\n\t{ 73440, 73458 },\n\t{ 73474, 73474 },\n\t{ 73476, 73488 },\n\t{ 73490, 73523 },\n\t{ 73648, 73648 },\n\t{ 73728, 74649 },\n\t{ 74880, 75075 },\n\t{ 77712, 77808 },\n\t{ 77824, 78895 },\n\t{ 78913, 78918 },\n\t{ 82944, 83526 },\n\t{ 92160, 92728 },\n\t{ 92736, 92766 },\n\t{ 92784, 92862 },\n\t{ 92880, 92909 },\n\t{ 92928, 92975 },\n\t{ 93027, 93047 },\n\t{ 93053, 93071 },\n\t{ 93952, 94026 },\n\t{ 94032, 94032 },\n\t{ 94208, 100343 },\n\t{ 100352, 101589 },\n\t{ 101632, 101640 },\n\t{ 110592, 110882 },\n\t{ 110898, 110898 },\n\t{ 110928, 110930 },\n\t{ 110933, 110933 },\n\t{ 110948, 110951 },\n\t{ 110960, 111355 },\n\t{ 113664, 113770 },\n\t{ 113776, 113788 },\n\t{ 113792, 113800 },\n\t{ 113808, 113817 },\n\t{ 122634, 122634 },\n\t{ 123136, 123180 },\n\t{ 123214, 123214 },\n\t{ 123536, 123565 },\n\t{ 123584, 123627 },\n\t{ 124112, 124138 },\n\t{ 124896, 124902 },\n\t{ 124904, 124907 },\n\t{ 124909, 124910 },\n\t{ 124912, 124926 },\n\t{ 124928, 125124 },\n\t{ 126464, 126467 },\n\t{ 126469, 126495 },\n\t{ 126497, 126498 },\n\t{ 126500, 126500 },\n\t{ 126503, 126503 },\n\t{ 126505, 126514 },\n\t{ 126516, 126519 },\n\t{ 126521, 126521 },\n\t{ 126523, 126523 },\n\t{ 126530, 126530 },\n\t{ 126535, 126535 },\n\t{ 126537, 126537 },\n\t{ 126539, 126539 },\n\t{ 126541, 126543 },\n\t{ 126545, 126546 },\n\t{ 126548, 126548 },\n\t{ 126551, 126551 },\n\t{ 126553, 126553 },\n\t{ 126555, 126555 },\n\t{ 126557, 126557 },\n\t{ 126559, 126559 },\n\t{ 126561, 126562 },\n\t{ 126564, 126564 },\n\t{ 126567, 126570 },\n\t{ 126572, 126578 },\n\t{ 126580, 126583 },\n\t{ 126585, 126588 },\n\t{ 126590, 126590 },\n\t{ 126592, 126601 },\n\t{ 126603, 126619 },\n\t{ 126625, 126627 },\n\t{ 126629, 126633 },\n\t{ 126635, 126651 },\n\t{ 131072, 173791 },\n\t{ 173824, 177977 },\n\t{ 177984, 178205 },\n\t{ 178208, 183969 },\n\t{ 183984, 191456 },\n\t{ 191472, 192093 },\n\t{ 194560, 195101 },\n\t{ 196608, 201546 },\n\t{ 201552, 205743 },\n};\nstatic const URange16 Lt_range16[] = {\n\t{ 453, 453 },\n\t{ 456, 456 },\n\t{ 459, 459 },\n\t{ 498, 498 },\n\t{ 8072, 8079 },\n\t{ 8088, 8095 },\n\t{ 8104, 8111 },\n\t{ 8124, 8124 },\n\t{ 8140, 8140 },\n\t{ 8188, 8188 },\n};\nstatic const URange16 Lu_range16[] = {\n\t{ 65, 90 },\n\t{ 192, 214 },\n\t{ 216, 222 },\n\t{ 256, 256 },\n\t{ 258, 258 },\n\t{ 260, 260 },\n\t{ 262, 262 },\n\t{ 264, 264 },\n\t{ 266, 266 },\n\t{ 268, 268 },\n\t{ 270, 270 },\n\t{ 272, 272 },\n\t{ 274, 274 },\n\t{ 276, 276 },\n\t{ 278, 278 },\n\t{ 280, 280 },\n\t{ 282, 282 },\n\t{ 284, 284 },\n\t{ 286, 286 },\n\t{ 288, 288 },\n\t{ 290, 290 },\n\t{ 292, 292 },\n\t{ 294, 294 },\n\t{ 296, 296 },\n\t{ 298, 298 },\n\t{ 300, 300 },\n\t{ 302, 302 },\n\t{ 304, 304 },\n\t{ 306, 306 },\n\t{ 308, 308 },\n\t{ 310, 310 },\n\t{ 313, 313 },\n\t{ 315, 315 },\n\t{ 317, 317 },\n\t{ 319, 319 },\n\t{ 321, 321 },\n\t{ 323, 323 },\n\t{ 325, 325 },\n\t{ 327, 327 },\n\t{ 330, 330 },\n\t{ 332, 332 },\n\t{ 334, 334 },\n\t{ 336, 336 },\n\t{ 338, 338 },\n\t{ 340, 340 },\n\t{ 342, 342 },\n\t{ 344, 344 },\n\t{ 346, 346 },\n\t{ 348, 348 },\n\t{ 350, 350 },\n\t{ 352, 352 },\n\t{ 354, 354 },\n\t{ 356, 356 },\n\t{ 358, 358 },\n\t{ 360, 360 },\n\t{ 362, 362 },\n\t{ 364, 364 },\n\t{ 366, 366 },\n\t{ 368, 368 },\n\t{ 370, 370 },\n\t{ 372, 372 },\n\t{ 374, 374 },\n\t{ 376, 377 },\n\t{ 379, 379 },\n\t{ 381, 381 },\n\t{ 385, 386 },\n\t{ 388, 388 },\n\t{ 390, 391 },\n\t{ 393, 395 },\n\t{ 398, 401 },\n\t{ 403, 404 },\n\t{ 406, 408 },\n\t{ 412, 413 },\n\t{ 415, 416 },\n\t{ 418, 418 },\n\t{ 420, 420 },\n\t{ 422, 423 },\n\t{ 425, 425 },\n\t{ 428, 428 },\n\t{ 430, 431 },\n\t{ 433, 435 },\n\t{ 437, 437 },\n\t{ 439, 440 },\n\t{ 444, 444 },\n\t{ 452, 452 },\n\t{ 455, 455 },\n\t{ 458, 458 },\n\t{ 461, 461 },\n\t{ 463, 463 },\n\t{ 465, 465 },\n\t{ 467, 467 },\n\t{ 469, 469 },\n\t{ 471, 471 },\n\t{ 473, 473 },\n\t{ 475, 475 },\n\t{ 478, 478 },\n\t{ 480, 480 },\n\t{ 482, 482 },\n\t{ 484, 484 },\n\t{ 486, 486 },\n\t{ 488, 488 },\n\t{ 490, 490 },\n\t{ 492, 492 },\n\t{ 494, 494 },\n\t{ 497, 497 },\n\t{ 500, 500 },\n\t{ 502, 504 },\n\t{ 506, 506 },\n\t{ 508, 508 },\n\t{ 510, 510 },\n\t{ 512, 512 },\n\t{ 514, 514 },\n\t{ 516, 516 },\n\t{ 518, 518 },\n\t{ 520, 520 },\n\t{ 522, 522 },\n\t{ 524, 524 },\n\t{ 526, 526 },\n\t{ 528, 528 },\n\t{ 530, 530 },\n\t{ 532, 532 },\n\t{ 534, 534 },\n\t{ 536, 536 },\n\t{ 538, 538 },\n\t{ 540, 540 },\n\t{ 542, 542 },\n\t{ 544, 544 },\n\t{ 546, 546 },\n\t{ 548, 548 },\n\t{ 550, 550 },\n\t{ 552, 552 },\n\t{ 554, 554 },\n\t{ 556, 556 },\n\t{ 558, 558 },\n\t{ 560, 560 },\n\t{ 562, 562 },\n\t{ 570, 571 },\n\t{ 573, 574 },\n\t{ 577, 577 },\n\t{ 579, 582 },\n\t{ 584, 584 },\n\t{ 586, 586 },\n\t{ 588, 588 },\n\t{ 590, 590 },\n\t{ 880, 880 },\n\t{ 882, 882 },\n\t{ 886, 886 },\n\t{ 895, 895 },\n\t{ 902, 902 },\n\t{ 904, 906 },\n\t{ 908, 908 },\n\t{ 910, 911 },\n\t{ 913, 929 },\n\t{ 931, 939 },\n\t{ 975, 975 },\n\t{ 978, 980 },\n\t{ 984, 984 },\n\t{ 986, 986 },\n\t{ 988, 988 },\n\t{ 990, 990 },\n\t{ 992, 992 },\n\t{ 994, 994 },\n\t{ 996, 996 },\n\t{ 998, 998 },\n\t{ 1000, 1000 },\n\t{ 1002, 1002 },\n\t{ 1004, 1004 },\n\t{ 1006, 1006 },\n\t{ 1012, 1012 },\n\t{ 1015, 1015 },\n\t{ 1017, 1018 },\n\t{ 1021, 1071 },\n\t{ 1120, 1120 },\n\t{ 1122, 1122 },\n\t{ 1124, 1124 },\n\t{ 1126, 1126 },\n\t{ 1128, 1128 },\n\t{ 1130, 1130 },\n\t{ 1132, 1132 },\n\t{ 1134, 1134 },\n\t{ 1136, 1136 },\n\t{ 1138, 1138 },\n\t{ 1140, 1140 },\n\t{ 1142, 1142 },\n\t{ 1144, 1144 },\n\t{ 1146, 1146 },\n\t{ 1148, 1148 },\n\t{ 1150, 1150 },\n\t{ 1152, 1152 },\n\t{ 1162, 1162 },\n\t{ 1164, 1164 },\n\t{ 1166, 1166 },\n\t{ 1168, 1168 },\n\t{ 1170, 1170 },\n\t{ 1172, 1172 },\n\t{ 1174, 1174 },\n\t{ 1176, 1176 },\n\t{ 1178, 1178 },\n\t{ 1180, 1180 },\n\t{ 1182, 1182 },\n\t{ 1184, 1184 },\n\t{ 1186, 1186 },\n\t{ 1188, 1188 },\n\t{ 1190, 1190 },\n\t{ 1192, 1192 },\n\t{ 1194, 1194 },\n\t{ 1196, 1196 },\n\t{ 1198, 1198 },\n\t{ 1200, 1200 },\n\t{ 1202, 1202 },\n\t{ 1204, 1204 },\n\t{ 1206, 1206 },\n\t{ 1208, 1208 },\n\t{ 1210, 1210 },\n\t{ 1212, 1212 },\n\t{ 1214, 1214 },\n\t{ 1216, 1217 },\n\t{ 1219, 1219 },\n\t{ 1221, 1221 },\n\t{ 1223, 1223 },\n\t{ 1225, 1225 },\n\t{ 1227, 1227 },\n\t{ 1229, 1229 },\n\t{ 1232, 1232 },\n\t{ 1234, 1234 },\n\t{ 1236, 1236 },\n\t{ 1238, 1238 },\n\t{ 1240, 1240 },\n\t{ 1242, 1242 },\n\t{ 1244, 1244 },\n\t{ 1246, 1246 },\n\t{ 1248, 1248 },\n\t{ 1250, 1250 },\n\t{ 1252, 1252 },\n\t{ 1254, 1254 },\n\t{ 1256, 1256 },\n\t{ 1258, 1258 },\n\t{ 1260, 1260 },\n\t{ 1262, 1262 },\n\t{ 1264, 1264 },\n\t{ 1266, 1266 },\n\t{ 1268, 1268 },\n\t{ 1270, 1270 },\n\t{ 1272, 1272 },\n\t{ 1274, 1274 },\n\t{ 1276, 1276 },\n\t{ 1278, 1278 },\n\t{ 1280, 1280 },\n\t{ 1282, 1282 },\n\t{ 1284, 1284 },\n\t{ 1286, 1286 },\n\t{ 1288, 1288 },\n\t{ 1290, 1290 },\n\t{ 1292, 1292 },\n\t{ 1294, 1294 },\n\t{ 1296, 1296 },\n\t{ 1298, 1298 },\n\t{ 1300, 1300 },\n\t{ 1302, 1302 },\n\t{ 1304, 1304 },\n\t{ 1306, 1306 },\n\t{ 1308, 1308 },\n\t{ 1310, 1310 },\n\t{ 1312, 1312 },\n\t{ 1314, 1314 },\n\t{ 1316, 1316 },\n\t{ 1318, 1318 },\n\t{ 1320, 1320 },\n\t{ 1322, 1322 },\n\t{ 1324, 1324 },\n\t{ 1326, 1326 },\n\t{ 1329, 1366 },\n\t{ 4256, 4293 },\n\t{ 4295, 4295 },\n\t{ 4301, 4301 },\n\t{ 5024, 5109 },\n\t{ 7312, 7354 },\n\t{ 7357, 7359 },\n\t{ 7680, 7680 },\n\t{ 7682, 7682 },\n\t{ 7684, 7684 },\n\t{ 7686, 7686 },\n\t{ 7688, 7688 },\n\t{ 7690, 7690 },\n\t{ 7692, 7692 },\n\t{ 7694, 7694 },\n\t{ 7696, 7696 },\n\t{ 7698, 7698 },\n\t{ 7700, 7700 },\n\t{ 7702, 7702 },\n\t{ 7704, 7704 },\n\t{ 7706, 7706 },\n\t{ 7708, 7708 },\n\t{ 7710, 7710 },\n\t{ 7712, 7712 },\n\t{ 7714, 7714 },\n\t{ 7716, 7716 },\n\t{ 7718, 7718 },\n\t{ 7720, 7720 },\n\t{ 7722, 7722 },\n\t{ 7724, 7724 },\n\t{ 7726, 7726 },\n\t{ 7728, 7728 },\n\t{ 7730, 7730 },\n\t{ 7732, 7732 },\n\t{ 7734, 7734 },\n\t{ 7736, 7736 },\n\t{ 7738, 7738 },\n\t{ 7740, 7740 },\n\t{ 7742, 7742 },\n\t{ 7744, 7744 },\n\t{ 7746, 7746 },\n\t{ 7748, 7748 },\n\t{ 7750, 7750 },\n\t{ 7752, 7752 },\n\t{ 7754, 7754 },\n\t{ 7756, 7756 },\n\t{ 7758, 7758 },\n\t{ 7760, 7760 },\n\t{ 7762, 7762 },\n\t{ 7764, 7764 },\n\t{ 7766, 7766 },\n\t{ 7768, 7768 },\n\t{ 7770, 7770 },\n\t{ 7772, 7772 },\n\t{ 7774, 7774 },\n\t{ 7776, 7776 },\n\t{ 7778, 7778 },\n\t{ 7780, 7780 },\n\t{ 7782, 7782 },\n\t{ 7784, 7784 },\n\t{ 7786, 7786 },\n\t{ 7788, 7788 },\n\t{ 7790, 7790 },\n\t{ 7792, 7792 },\n\t{ 7794, 7794 },\n\t{ 7796, 7796 },\n\t{ 7798, 7798 },\n\t{ 7800, 7800 },\n\t{ 7802, 7802 },\n\t{ 7804, 7804 },\n\t{ 7806, 7806 },\n\t{ 7808, 7808 },\n\t{ 7810, 7810 },\n\t{ 7812, 7812 },\n\t{ 7814, 7814 },\n\t{ 7816, 7816 },\n\t{ 7818, 7818 },\n\t{ 7820, 7820 },\n\t{ 7822, 7822 },\n\t{ 7824, 7824 },\n\t{ 7826, 7826 },\n\t{ 7828, 7828 },\n\t{ 7838, 7838 },\n\t{ 7840, 7840 },\n\t{ 7842, 7842 },\n\t{ 7844, 7844 },\n\t{ 7846, 7846 },\n\t{ 7848, 7848 },\n\t{ 7850, 7850 },\n\t{ 7852, 7852 },\n\t{ 7854, 7854 },\n\t{ 7856, 7856 },\n\t{ 7858, 7858 },\n\t{ 7860, 7860 },\n\t{ 7862, 7862 },\n\t{ 7864, 7864 },\n\t{ 7866, 7866 },\n\t{ 7868, 7868 },\n\t{ 7870, 7870 },\n\t{ 7872, 7872 },\n\t{ 7874, 7874 },\n\t{ 7876, 7876 },\n\t{ 7878, 7878 },\n\t{ 7880, 7880 },\n\t{ 7882, 7882 },\n\t{ 7884, 7884 },\n\t{ 7886, 7886 },\n\t{ 7888, 7888 },\n\t{ 7890, 7890 },\n\t{ 7892, 7892 },\n\t{ 7894, 7894 },\n\t{ 7896, 7896 },\n\t{ 7898, 7898 },\n\t{ 7900, 7900 },\n\t{ 7902, 7902 },\n\t{ 7904, 7904 },\n\t{ 7906, 7906 },\n\t{ 7908, 7908 },\n\t{ 7910, 7910 },\n\t{ 7912, 7912 },\n\t{ 7914, 7914 },\n\t{ 7916, 7916 },\n\t{ 7918, 7918 },\n\t{ 7920, 7920 },\n\t{ 7922, 7922 },\n\t{ 7924, 7924 },\n\t{ 7926, 7926 },\n\t{ 7928, 7928 },\n\t{ 7930, 7930 },\n\t{ 7932, 7932 },\n\t{ 7934, 7934 },\n\t{ 7944, 7951 },\n\t{ 7960, 7965 },\n\t{ 7976, 7983 },\n\t{ 7992, 7999 },\n\t{ 8008, 8013 },\n\t{ 8025, 8025 },\n\t{ 8027, 8027 },\n\t{ 8029, 8029 },\n\t{ 8031, 8031 },\n\t{ 8040, 8047 },\n\t{ 8120, 8123 },\n\t{ 8136, 8139 },\n\t{ 8152, 8155 },\n\t{ 8168, 8172 },\n\t{ 8184, 8187 },\n\t{ 8450, 8450 },\n\t{ 8455, 8455 },\n\t{ 8459, 8461 },\n\t{ 8464, 8466 },\n\t{ 8469, 8469 },\n\t{ 8473, 8477 },\n\t{ 8484, 8484 },\n\t{ 8486, 8486 },\n\t{ 8488, 8488 },\n\t{ 8490, 8493 },\n\t{ 8496, 8499 },\n\t{ 8510, 8511 },\n\t{ 8517, 8517 },\n\t{ 8579, 8579 },\n\t{ 11264, 11311 },\n\t{ 11360, 11360 },\n\t{ 11362, 11364 },\n\t{ 11367, 11367 },\n\t{ 11369, 11369 },\n\t{ 11371, 11371 },\n\t{ 11373, 11376 },\n\t{ 11378, 11378 },\n\t{ 11381, 11381 },\n\t{ 11390, 11392 },\n\t{ 11394, 11394 },\n\t{ 11396, 11396 },\n\t{ 11398, 11398 },\n\t{ 11400, 11400 },\n\t{ 11402, 11402 },\n\t{ 11404, 11404 },\n\t{ 11406, 11406 },\n\t{ 11408, 11408 },\n\t{ 11410, 11410 },\n\t{ 11412, 11412 },\n\t{ 11414, 11414 },\n\t{ 11416, 11416 },\n\t{ 11418, 11418 },\n\t{ 11420, 11420 },\n\t{ 11422, 11422 },\n\t{ 11424, 11424 },\n\t{ 11426, 11426 },\n\t{ 11428, 11428 },\n\t{ 11430, 11430 },\n\t{ 11432, 11432 },\n\t{ 11434, 11434 },\n\t{ 11436, 11436 },\n\t{ 11438, 11438 },\n\t{ 11440, 11440 },\n\t{ 11442, 11442 },\n\t{ 11444, 11444 },\n\t{ 11446, 11446 },\n\t{ 11448, 11448 },\n\t{ 11450, 11450 },\n\t{ 11452, 11452 },\n\t{ 11454, 11454 },\n\t{ 11456, 11456 },\n\t{ 11458, 11458 },\n\t{ 11460, 11460 },\n\t{ 11462, 11462 },\n\t{ 11464, 11464 },\n\t{ 11466, 11466 },\n\t{ 11468, 11468 },\n\t{ 11470, 11470 },\n\t{ 11472, 11472 },\n\t{ 11474, 11474 },\n\t{ 11476, 11476 },\n\t{ 11478, 11478 },\n\t{ 11480, 11480 },\n\t{ 11482, 11482 },\n\t{ 11484, 11484 },\n\t{ 11486, 11486 },\n\t{ 11488, 11488 },\n\t{ 11490, 11490 },\n\t{ 11499, 11499 },\n\t{ 11501, 11501 },\n\t{ 11506, 11506 },\n\t{ 42560, 42560 },\n\t{ 42562, 42562 },\n\t{ 42564, 42564 },\n\t{ 42566, 42566 },\n\t{ 42568, 42568 },\n\t{ 42570, 42570 },\n\t{ 42572, 42572 },\n\t{ 42574, 42574 },\n\t{ 42576, 42576 },\n\t{ 42578, 42578 },\n\t{ 42580, 42580 },\n\t{ 42582, 42582 },\n\t{ 42584, 42584 },\n\t{ 42586, 42586 },\n\t{ 42588, 42588 },\n\t{ 42590, 42590 },\n\t{ 42592, 42592 },\n\t{ 42594, 42594 },\n\t{ 42596, 42596 },\n\t{ 42598, 42598 },\n\t{ 42600, 42600 },\n\t{ 42602, 42602 },\n\t{ 42604, 42604 },\n\t{ 42624, 42624 },\n\t{ 42626, 42626 },\n\t{ 42628, 42628 },\n\t{ 42630, 42630 },\n\t{ 42632, 42632 },\n\t{ 42634, 42634 },\n\t{ 42636, 42636 },\n\t{ 42638, 42638 },\n\t{ 42640, 42640 },\n\t{ 42642, 42642 },\n\t{ 42644, 42644 },\n\t{ 42646, 42646 },\n\t{ 42648, 42648 },\n\t{ 42650, 42650 },\n\t{ 42786, 42786 },\n\t{ 42788, 42788 },\n\t{ 42790, 42790 },\n\t{ 42792, 42792 },\n\t{ 42794, 42794 },\n\t{ 42796, 42796 },\n\t{ 42798, 42798 },\n\t{ 42802, 42802 },\n\t{ 42804, 42804 },\n\t{ 42806, 42806 },\n\t{ 42808, 42808 },\n\t{ 42810, 42810 },\n\t{ 42812, 42812 },\n\t{ 42814, 42814 },\n\t{ 42816, 42816 },\n\t{ 42818, 42818 },\n\t{ 42820, 42820 },\n\t{ 42822, 42822 },\n\t{ 42824, 42824 },\n\t{ 42826, 42826 },\n\t{ 42828, 42828 },\n\t{ 42830, 42830 },\n\t{ 42832, 42832 },\n\t{ 42834, 42834 },\n\t{ 42836, 42836 },\n\t{ 42838, 42838 },\n\t{ 42840, 42840 },\n\t{ 42842, 42842 },\n\t{ 42844, 42844 },\n\t{ 42846, 42846 },\n\t{ 42848, 42848 },\n\t{ 42850, 42850 },\n\t{ 42852, 42852 },\n\t{ 42854, 42854 },\n\t{ 42856, 42856 },\n\t{ 42858, 42858 },\n\t{ 42860, 42860 },\n\t{ 42862, 42862 },\n\t{ 42873, 42873 },\n\t{ 42875, 42875 },\n\t{ 42877, 42878 },\n\t{ 42880, 42880 },\n\t{ 42882, 42882 },\n\t{ 42884, 42884 },\n\t{ 42886, 42886 },\n\t{ 42891, 42891 },\n\t{ 42893, 42893 },\n\t{ 42896, 42896 },\n\t{ 42898, 42898 },\n\t{ 42902, 42902 },\n\t{ 42904, 42904 },\n\t{ 42906, 42906 },\n\t{ 42908, 42908 },\n\t{ 42910, 42910 },\n\t{ 42912, 42912 },\n\t{ 42914, 42914 },\n\t{ 42916, 42916 },\n\t{ 42918, 42918 },\n\t{ 42920, 42920 },\n\t{ 42922, 42926 },\n\t{ 42928, 42932 },\n\t{ 42934, 42934 },\n\t{ 42936, 42936 },\n\t{ 42938, 42938 },\n\t{ 42940, 42940 },\n\t{ 42942, 42942 },\n\t{ 42944, 42944 },\n\t{ 42946, 42946 },\n\t{ 42948, 42951 },\n\t{ 42953, 42953 },\n\t{ 42960, 42960 },\n\t{ 42966, 42966 },\n\t{ 42968, 42968 },\n\t{ 42997, 42997 },\n\t{ 65313, 65338 },\n};\nstatic const URange32 Lu_range32[] = {\n\t{ 66560, 66599 },\n\t{ 66736, 66771 },\n\t{ 66928, 66938 },\n\t{ 66940, 66954 },\n\t{ 66956, 66962 },\n\t{ 66964, 66965 },\n\t{ 68736, 68786 },\n\t{ 71840, 71871 },\n\t{ 93760, 93791 },\n\t{ 119808, 119833 },\n\t{ 119860, 119885 },\n\t{ 119912, 119937 },\n\t{ 119964, 119964 },\n\t{ 119966, 119967 },\n\t{ 119970, 119970 },\n\t{ 119973, 119974 },\n\t{ 119977, 119980 },\n\t{ 119982, 119989 },\n\t{ 120016, 120041 },\n\t{ 120068, 120069 },\n\t{ 120071, 120074 },\n\t{ 120077, 120084 },\n\t{ 120086, 120092 },\n\t{ 120120, 120121 },\n\t{ 120123, 120126 },\n\t{ 120128, 120132 },\n\t{ 120134, 120134 },\n\t{ 120138, 120144 },\n\t{ 120172, 120197 },\n\t{ 120224, 120249 },\n\t{ 120276, 120301 },\n\t{ 120328, 120353 },\n\t{ 120380, 120405 },\n\t{ 120432, 120457 },\n\t{ 120488, 120512 },\n\t{ 120546, 120570 },\n\t{ 120604, 120628 },\n\t{ 120662, 120686 },\n\t{ 120720, 120744 },\n\t{ 120778, 120778 },\n\t{ 125184, 125217 },\n};\nstatic const URange16 M_range16[] = {\n\t{ 768, 879 },\n\t{ 1155, 1161 },\n\t{ 1425, 1469 },\n\t{ 1471, 1471 },\n\t{ 1473, 1474 },\n\t{ 1476, 1477 },\n\t{ 1479, 1479 },\n\t{ 1552, 1562 },\n\t{ 1611, 1631 },\n\t{ 1648, 1648 },\n\t{ 1750, 1756 },\n\t{ 1759, 1764 },\n\t{ 1767, 1768 },\n\t{ 1770, 1773 },\n\t{ 1809, 1809 },\n\t{ 1840, 1866 },\n\t{ 1958, 1968 },\n\t{ 2027, 2035 },\n\t{ 2045, 2045 },\n\t{ 2070, 2073 },\n\t{ 2075, 2083 },\n\t{ 2085, 2087 },\n\t{ 2089, 2093 },\n\t{ 2137, 2139 },\n\t{ 2200, 2207 },\n\t{ 2250, 2273 },\n\t{ 2275, 2307 },\n\t{ 2362, 2364 },\n\t{ 2366, 2383 },\n\t{ 2385, 2391 },\n\t{ 2402, 2403 },\n\t{ 2433, 2435 },\n\t{ 2492, 2492 },\n\t{ 2494, 2500 },\n\t{ 2503, 2504 },\n\t{ 2507, 2509 },\n\t{ 2519, 2519 },\n\t{ 2530, 2531 },\n\t{ 2558, 2558 },\n\t{ 2561, 2563 },\n\t{ 2620, 2620 },\n\t{ 2622, 2626 },\n\t{ 2631, 2632 },\n\t{ 2635, 2637 },\n\t{ 2641, 2641 },\n\t{ 2672, 2673 },\n\t{ 2677, 2677 },\n\t{ 2689, 2691 },\n\t{ 2748, 2748 },\n\t{ 2750, 2757 },\n\t{ 2759, 2761 },\n\t{ 2763, 2765 },\n\t{ 2786, 2787 },\n\t{ 2810, 2815 },\n\t{ 2817, 2819 },\n\t{ 2876, 2876 },\n\t{ 2878, 2884 },\n\t{ 2887, 2888 },\n\t{ 2891, 2893 },\n\t{ 2901, 2903 },\n\t{ 2914, 2915 },\n\t{ 2946, 2946 },\n\t{ 3006, 3010 },\n\t{ 3014, 3016 },\n\t{ 3018, 3021 },\n\t{ 3031, 3031 },\n\t{ 3072, 3076 },\n\t{ 3132, 3132 },\n\t{ 3134, 3140 },\n\t{ 3142, 3144 },\n\t{ 3146, 3149 },\n\t{ 3157, 3158 },\n\t{ 3170, 3171 },\n\t{ 3201, 3203 },\n\t{ 3260, 3260 },\n\t{ 3262, 3268 },\n\t{ 3270, 3272 },\n\t{ 3274, 3277 },\n\t{ 3285, 3286 },\n\t{ 3298, 3299 },\n\t{ 3315, 3315 },\n\t{ 3328, 3331 },\n\t{ 3387, 3388 },\n\t{ 3390, 3396 },\n\t{ 3398, 3400 },\n\t{ 3402, 3405 },\n\t{ 3415, 3415 },\n\t{ 3426, 3427 },\n\t{ 3457, 3459 },\n\t{ 3530, 3530 },\n\t{ 3535, 3540 },\n\t{ 3542, 3542 },\n\t{ 3544, 3551 },\n\t{ 3570, 3571 },\n\t{ 3633, 3633 },\n\t{ 3636, 3642 },\n\t{ 3655, 3662 },\n\t{ 3761, 3761 },\n\t{ 3764, 3772 },\n\t{ 3784, 3790 },\n\t{ 3864, 3865 },\n\t{ 3893, 3893 },\n\t{ 3895, 3895 },\n\t{ 3897, 3897 },\n\t{ 3902, 3903 },\n\t{ 3953, 3972 },\n\t{ 3974, 3975 },\n\t{ 3981, 3991 },\n\t{ 3993, 4028 },\n\t{ 4038, 4038 },\n\t{ 4139, 4158 },\n\t{ 4182, 4185 },\n\t{ 4190, 4192 },\n\t{ 4194, 4196 },\n\t{ 4199, 4205 },\n\t{ 4209, 4212 },\n\t{ 4226, 4237 },\n\t{ 4239, 4239 },\n\t{ 4250, 4253 },\n\t{ 4957, 4959 },\n\t{ 5906, 5909 },\n\t{ 5938, 5940 },\n\t{ 5970, 5971 },\n\t{ 6002, 6003 },\n\t{ 6068, 6099 },\n\t{ 6109, 6109 },\n\t{ 6155, 6157 },\n\t{ 6159, 6159 },\n\t{ 6277, 6278 },\n\t{ 6313, 6313 },\n\t{ 6432, 6443 },\n\t{ 6448, 6459 },\n\t{ 6679, 6683 },\n\t{ 6741, 6750 },\n\t{ 6752, 6780 },\n\t{ 6783, 6783 },\n\t{ 6832, 6862 },\n\t{ 6912, 6916 },\n\t{ 6964, 6980 },\n\t{ 7019, 7027 },\n\t{ 7040, 7042 },\n\t{ 7073, 7085 },\n\t{ 7142, 7155 },\n\t{ 7204, 7223 },\n\t{ 7376, 7378 },\n\t{ 7380, 7400 },\n\t{ 7405, 7405 },\n\t{ 7412, 7412 },\n\t{ 7415, 7417 },\n\t{ 7616, 7679 },\n\t{ 8400, 8432 },\n\t{ 11503, 11505 },\n\t{ 11647, 11647 },\n\t{ 11744, 11775 },\n\t{ 12330, 12335 },\n\t{ 12441, 12442 },\n\t{ 42607, 42610 },\n\t{ 42612, 42621 },\n\t{ 42654, 42655 },\n\t{ 42736, 42737 },\n\t{ 43010, 43010 },\n\t{ 43014, 43014 },\n\t{ 43019, 43019 },\n\t{ 43043, 43047 },\n\t{ 43052, 43052 },\n\t{ 43136, 43137 },\n\t{ 43188, 43205 },\n\t{ 43232, 43249 },\n\t{ 43263, 43263 },\n\t{ 43302, 43309 },\n\t{ 43335, 43347 },\n\t{ 43392, 43395 },\n\t{ 43443, 43456 },\n\t{ 43493, 43493 },\n\t{ 43561, 43574 },\n\t{ 43587, 43587 },\n\t{ 43596, 43597 },\n\t{ 43643, 43645 },\n\t{ 43696, 43696 },\n\t{ 43698, 43700 },\n\t{ 43703, 43704 },\n\t{ 43710, 43711 },\n\t{ 43713, 43713 },\n\t{ 43755, 43759 },\n\t{ 43765, 43766 },\n\t{ 44003, 44010 },\n\t{ 44012, 44013 },\n\t{ 64286, 64286 },\n\t{ 65024, 65039 },\n\t{ 65056, 65071 },\n};\nstatic const URange32 M_range32[] = {\n\t{ 66045, 66045 },\n\t{ 66272, 66272 },\n\t{ 66422, 66426 },\n\t{ 68097, 68099 },\n\t{ 68101, 68102 },\n\t{ 68108, 68111 },\n\t{ 68152, 68154 },\n\t{ 68159, 68159 },\n\t{ 68325, 68326 },\n\t{ 68900, 68903 },\n\t{ 69291, 69292 },\n\t{ 69373, 69375 },\n\t{ 69446, 69456 },\n\t{ 69506, 69509 },\n\t{ 69632, 69634 },\n\t{ 69688, 69702 },\n\t{ 69744, 69744 },\n\t{ 69747, 69748 },\n\t{ 69759, 69762 },\n\t{ 69808, 69818 },\n\t{ 69826, 69826 },\n\t{ 69888, 69890 },\n\t{ 69927, 69940 },\n\t{ 69957, 69958 },\n\t{ 70003, 70003 },\n\t{ 70016, 70018 },\n\t{ 70067, 70080 },\n\t{ 70089, 70092 },\n\t{ 70094, 70095 },\n\t{ 70188, 70199 },\n\t{ 70206, 70206 },\n\t{ 70209, 70209 },\n\t{ 70367, 70378 },\n\t{ 70400, 70403 },\n\t{ 70459, 70460 },\n\t{ 70462, 70468 },\n\t{ 70471, 70472 },\n\t{ 70475, 70477 },\n\t{ 70487, 70487 },\n\t{ 70498, 70499 },\n\t{ 70502, 70508 },\n\t{ 70512, 70516 },\n\t{ 70709, 70726 },\n\t{ 70750, 70750 },\n\t{ 70832, 70851 },\n\t{ 71087, 71093 },\n\t{ 71096, 71104 },\n\t{ 71132, 71133 },\n\t{ 71216, 71232 },\n\t{ 71339, 71351 },\n\t{ 71453, 71467 },\n\t{ 71724, 71738 },\n\t{ 71984, 71989 },\n\t{ 71991, 71992 },\n\t{ 71995, 71998 },\n\t{ 72000, 72000 },\n\t{ 72002, 72003 },\n\t{ 72145, 72151 },\n\t{ 72154, 72160 },\n\t{ 72164, 72164 },\n\t{ 72193, 72202 },\n\t{ 72243, 72249 },\n\t{ 72251, 72254 },\n\t{ 72263, 72263 },\n\t{ 72273, 72283 },\n\t{ 72330, 72345 },\n\t{ 72751, 72758 },\n\t{ 72760, 72767 },\n\t{ 72850, 72871 },\n\t{ 72873, 72886 },\n\t{ 73009, 73014 },\n\t{ 73018, 73018 },\n\t{ 73020, 73021 },\n\t{ 73023, 73029 },\n\t{ 73031, 73031 },\n\t{ 73098, 73102 },\n\t{ 73104, 73105 },\n\t{ 73107, 73111 },\n\t{ 73459, 73462 },\n\t{ 73472, 73473 },\n\t{ 73475, 73475 },\n\t{ 73524, 73530 },\n\t{ 73534, 73538 },\n\t{ 78912, 78912 },\n\t{ 78919, 78933 },\n\t{ 92912, 92916 },\n\t{ 92976, 92982 },\n\t{ 94031, 94031 },\n\t{ 94033, 94087 },\n\t{ 94095, 94098 },\n\t{ 94180, 94180 },\n\t{ 94192, 94193 },\n\t{ 113821, 113822 },\n\t{ 118528, 118573 },\n\t{ 118576, 118598 },\n\t{ 119141, 119145 },\n\t{ 119149, 119154 },\n\t{ 119163, 119170 },\n\t{ 119173, 119179 },\n\t{ 119210, 119213 },\n\t{ 119362, 119364 },\n\t{ 121344, 121398 },\n\t{ 121403, 121452 },\n\t{ 121461, 121461 },\n\t{ 121476, 121476 },\n\t{ 121499, 121503 },\n\t{ 121505, 121519 },\n\t{ 122880, 122886 },\n\t{ 122888, 122904 },\n\t{ 122907, 122913 },\n\t{ 122915, 122916 },\n\t{ 122918, 122922 },\n\t{ 123023, 123023 },\n\t{ 123184, 123190 },\n\t{ 123566, 123566 },\n\t{ 123628, 123631 },\n\t{ 124140, 124143 },\n\t{ 125136, 125142 },\n\t{ 125252, 125258 },\n\t{ 917760, 917999 },\n};\nstatic const URange16 Mc_range16[] = {\n\t{ 2307, 2307 },\n\t{ 2363, 2363 },\n\t{ 2366, 2368 },\n\t{ 2377, 2380 },\n\t{ 2382, 2383 },\n\t{ 2434, 2435 },\n\t{ 2494, 2496 },\n\t{ 2503, 2504 },\n\t{ 2507, 2508 },\n\t{ 2519, 2519 },\n\t{ 2563, 2563 },\n\t{ 2622, 2624 },\n\t{ 2691, 2691 },\n\t{ 2750, 2752 },\n\t{ 2761, 2761 },\n\t{ 2763, 2764 },\n\t{ 2818, 2819 },\n\t{ 2878, 2878 },\n\t{ 2880, 2880 },\n\t{ 2887, 2888 },\n\t{ 2891, 2892 },\n\t{ 2903, 2903 },\n\t{ 3006, 3007 },\n\t{ 3009, 3010 },\n\t{ 3014, 3016 },\n\t{ 3018, 3020 },\n\t{ 3031, 3031 },\n\t{ 3073, 3075 },\n\t{ 3137, 3140 },\n\t{ 3202, 3203 },\n\t{ 3262, 3262 },\n\t{ 3264, 3268 },\n\t{ 3271, 3272 },\n\t{ 3274, 3275 },\n\t{ 3285, 3286 },\n\t{ 3315, 3315 },\n\t{ 3330, 3331 },\n\t{ 3390, 3392 },\n\t{ 3398, 3400 },\n\t{ 3402, 3404 },\n\t{ 3415, 3415 },\n\t{ 3458, 3459 },\n\t{ 3535, 3537 },\n\t{ 3544, 3551 },\n\t{ 3570, 3571 },\n\t{ 3902, 3903 },\n\t{ 3967, 3967 },\n\t{ 4139, 4140 },\n\t{ 4145, 4145 },\n\t{ 4152, 4152 },\n\t{ 4155, 4156 },\n\t{ 4182, 4183 },\n\t{ 4194, 4196 },\n\t{ 4199, 4205 },\n\t{ 4227, 4228 },\n\t{ 4231, 4236 },\n\t{ 4239, 4239 },\n\t{ 4250, 4252 },\n\t{ 5909, 5909 },\n\t{ 5940, 5940 },\n\t{ 6070, 6070 },\n\t{ 6078, 6085 },\n\t{ 6087, 6088 },\n\t{ 6435, 6438 },\n\t{ 6441, 6443 },\n\t{ 6448, 6449 },\n\t{ 6451, 6456 },\n\t{ 6681, 6682 },\n\t{ 6741, 6741 },\n\t{ 6743, 6743 },\n\t{ 6753, 6753 },\n\t{ 6755, 6756 },\n\t{ 6765, 6770 },\n\t{ 6916, 6916 },\n\t{ 6965, 6965 },\n\t{ 6971, 6971 },\n\t{ 6973, 6977 },\n\t{ 6979, 6980 },\n\t{ 7042, 7042 },\n\t{ 7073, 7073 },\n\t{ 7078, 7079 },\n\t{ 7082, 7082 },\n\t{ 7143, 7143 },\n\t{ 7146, 7148 },\n\t{ 7150, 7150 },\n\t{ 7154, 7155 },\n\t{ 7204, 7211 },\n\t{ 7220, 7221 },\n\t{ 7393, 7393 },\n\t{ 7415, 7415 },\n\t{ 12334, 12335 },\n\t{ 43043, 43044 },\n\t{ 43047, 43047 },\n\t{ 43136, 43137 },\n\t{ 43188, 43203 },\n\t{ 43346, 43347 },\n\t{ 43395, 43395 },\n\t{ 43444, 43445 },\n\t{ 43450, 43451 },\n\t{ 43454, 43456 },\n\t{ 43567, 43568 },\n\t{ 43571, 43572 },\n\t{ 43597, 43597 },\n\t{ 43643, 43643 },\n\t{ 43645, 43645 },\n\t{ 43755, 43755 },\n\t{ 43758, 43759 },\n\t{ 43765, 43765 },\n\t{ 44003, 44004 },\n\t{ 44006, 44007 },\n\t{ 44009, 44010 },\n\t{ 44012, 44012 },\n};\nstatic const URange32 Mc_range32[] = {\n\t{ 69632, 69632 },\n\t{ 69634, 69634 },\n\t{ 69762, 69762 },\n\t{ 69808, 69810 },\n\t{ 69815, 69816 },\n\t{ 69932, 69932 },\n\t{ 69957, 69958 },\n\t{ 70018, 70018 },\n\t{ 70067, 70069 },\n\t{ 70079, 70080 },\n\t{ 70094, 70094 },\n\t{ 70188, 70190 },\n\t{ 70194, 70195 },\n\t{ 70197, 70197 },\n\t{ 70368, 70370 },\n\t{ 70402, 70403 },\n\t{ 70462, 70463 },\n\t{ 70465, 70468 },\n\t{ 70471, 70472 },\n\t{ 70475, 70477 },\n\t{ 70487, 70487 },\n\t{ 70498, 70499 },\n\t{ 70709, 70711 },\n\t{ 70720, 70721 },\n\t{ 70725, 70725 },\n\t{ 70832, 70834 },\n\t{ 70841, 70841 },\n\t{ 70843, 70846 },\n\t{ 70849, 70849 },\n\t{ 71087, 71089 },\n\t{ 71096, 71099 },\n\t{ 71102, 71102 },\n\t{ 71216, 71218 },\n\t{ 71227, 71228 },\n\t{ 71230, 71230 },\n\t{ 71340, 71340 },\n\t{ 71342, 71343 },\n\t{ 71350, 71350 },\n\t{ 71456, 71457 },\n\t{ 71462, 71462 },\n\t{ 71724, 71726 },\n\t{ 71736, 71736 },\n\t{ 71984, 71989 },\n\t{ 71991, 71992 },\n\t{ 71997, 71997 },\n\t{ 72000, 72000 },\n\t{ 72002, 72002 },\n\t{ 72145, 72147 },\n\t{ 72156, 72159 },\n\t{ 72164, 72164 },\n\t{ 72249, 72249 },\n\t{ 72279, 72280 },\n\t{ 72343, 72343 },\n\t{ 72751, 72751 },\n\t{ 72766, 72766 },\n\t{ 72873, 72873 },\n\t{ 72881, 72881 },\n\t{ 72884, 72884 },\n\t{ 73098, 73102 },\n\t{ 73107, 73108 },\n\t{ 73110, 73110 },\n\t{ 73461, 73462 },\n\t{ 73475, 73475 },\n\t{ 73524, 73525 },\n\t{ 73534, 73535 },\n\t{ 73537, 73537 },\n\t{ 94033, 94087 },\n\t{ 94192, 94193 },\n\t{ 119141, 119142 },\n\t{ 119149, 119154 },\n};\nstatic const URange16 Me_range16[] = {\n\t{ 1160, 1161 },\n\t{ 6846, 6846 },\n\t{ 8413, 8416 },\n\t{ 8418, 8420 },\n\t{ 42608, 42610 },\n};\nstatic const URange16 Mn_range16[] = {\n\t{ 768, 879 },\n\t{ 1155, 1159 },\n\t{ 1425, 1469 },\n\t{ 1471, 1471 },\n\t{ 1473, 1474 },\n\t{ 1476, 1477 },\n\t{ 1479, 1479 },\n\t{ 1552, 1562 },\n\t{ 1611, 1631 },\n\t{ 1648, 1648 },\n\t{ 1750, 1756 },\n\t{ 1759, 1764 },\n\t{ 1767, 1768 },\n\t{ 1770, 1773 },\n\t{ 1809, 1809 },\n\t{ 1840, 1866 },\n\t{ 1958, 1968 },\n\t{ 2027, 2035 },\n\t{ 2045, 2045 },\n\t{ 2070, 2073 },\n\t{ 2075, 2083 },\n\t{ 2085, 2087 },\n\t{ 2089, 2093 },\n\t{ 2137, 2139 },\n\t{ 2200, 2207 },\n\t{ 2250, 2273 },\n\t{ 2275, 2306 },\n\t{ 2362, 2362 },\n\t{ 2364, 2364 },\n\t{ 2369, 2376 },\n\t{ 2381, 2381 },\n\t{ 2385, 2391 },\n\t{ 2402, 2403 },\n\t{ 2433, 2433 },\n\t{ 2492, 2492 },\n\t{ 2497, 2500 },\n\t{ 2509, 2509 },\n\t{ 2530, 2531 },\n\t{ 2558, 2558 },\n\t{ 2561, 2562 },\n\t{ 2620, 2620 },\n\t{ 2625, 2626 },\n\t{ 2631, 2632 },\n\t{ 2635, 2637 },\n\t{ 2641, 2641 },\n\t{ 2672, 2673 },\n\t{ 2677, 2677 },\n\t{ 2689, 2690 },\n\t{ 2748, 2748 },\n\t{ 2753, 2757 },\n\t{ 2759, 2760 },\n\t{ 2765, 2765 },\n\t{ 2786, 2787 },\n\t{ 2810, 2815 },\n\t{ 2817, 2817 },\n\t{ 2876, 2876 },\n\t{ 2879, 2879 },\n\t{ 2881, 2884 },\n\t{ 2893, 2893 },\n\t{ 2901, 2902 },\n\t{ 2914, 2915 },\n\t{ 2946, 2946 },\n\t{ 3008, 3008 },\n\t{ 3021, 3021 },\n\t{ 3072, 3072 },\n\t{ 3076, 3076 },\n\t{ 3132, 3132 },\n\t{ 3134, 3136 },\n\t{ 3142, 3144 },\n\t{ 3146, 3149 },\n\t{ 3157, 3158 },\n\t{ 3170, 3171 },\n\t{ 3201, 3201 },\n\t{ 3260, 3260 },\n\t{ 3263, 3263 },\n\t{ 3270, 3270 },\n\t{ 3276, 3277 },\n\t{ 3298, 3299 },\n\t{ 3328, 3329 },\n\t{ 3387, 3388 },\n\t{ 3393, 3396 },\n\t{ 3405, 3405 },\n\t{ 3426, 3427 },\n\t{ 3457, 3457 },\n\t{ 3530, 3530 },\n\t{ 3538, 3540 },\n\t{ 3542, 3542 },\n\t{ 3633, 3633 },\n\t{ 3636, 3642 },\n\t{ 3655, 3662 },\n\t{ 3761, 3761 },\n\t{ 3764, 3772 },\n\t{ 3784, 3790 },\n\t{ 3864, 3865 },\n\t{ 3893, 3893 },\n\t{ 3895, 3895 },\n\t{ 3897, 3897 },\n\t{ 3953, 3966 },\n\t{ 3968, 3972 },\n\t{ 3974, 3975 },\n\t{ 3981, 3991 },\n\t{ 3993, 4028 },\n\t{ 4038, 4038 },\n\t{ 4141, 4144 },\n\t{ 4146, 4151 },\n\t{ 4153, 4154 },\n\t{ 4157, 4158 },\n\t{ 4184, 4185 },\n\t{ 4190, 4192 },\n\t{ 4209, 4212 },\n\t{ 4226, 4226 },\n\t{ 4229, 4230 },\n\t{ 4237, 4237 },\n\t{ 4253, 4253 },\n\t{ 4957, 4959 },\n\t{ 5906, 5908 },\n\t{ 5938, 5939 },\n\t{ 5970, 5971 },\n\t{ 6002, 6003 },\n\t{ 6068, 6069 },\n\t{ 6071, 6077 },\n\t{ 6086, 6086 },\n\t{ 6089, 6099 },\n\t{ 6109, 6109 },\n\t{ 6155, 6157 },\n\t{ 6159, 6159 },\n\t{ 6277, 6278 },\n\t{ 6313, 6313 },\n\t{ 6432, 6434 },\n\t{ 6439, 6440 },\n\t{ 6450, 6450 },\n\t{ 6457, 6459 },\n\t{ 6679, 6680 },\n\t{ 6683, 6683 },\n\t{ 6742, 6742 },\n\t{ 6744, 6750 },\n\t{ 6752, 6752 },\n\t{ 6754, 6754 },\n\t{ 6757, 6764 },\n\t{ 6771, 6780 },\n\t{ 6783, 6783 },\n\t{ 6832, 6845 },\n\t{ 6847, 6862 },\n\t{ 6912, 6915 },\n\t{ 6964, 6964 },\n\t{ 6966, 6970 },\n\t{ 6972, 6972 },\n\t{ 6978, 6978 },\n\t{ 7019, 7027 },\n\t{ 7040, 7041 },\n\t{ 7074, 7077 },\n\t{ 7080, 7081 },\n\t{ 7083, 7085 },\n\t{ 7142, 7142 },\n\t{ 7144, 7145 },\n\t{ 7149, 7149 },\n\t{ 7151, 7153 },\n\t{ 7212, 7219 },\n\t{ 7222, 7223 },\n\t{ 7376, 7378 },\n\t{ 7380, 7392 },\n\t{ 7394, 7400 },\n\t{ 7405, 7405 },\n\t{ 7412, 7412 },\n\t{ 7416, 7417 },\n\t{ 7616, 7679 },\n\t{ 8400, 8412 },\n\t{ 8417, 8417 },\n\t{ 8421, 8432 },\n\t{ 11503, 11505 },\n\t{ 11647, 11647 },\n\t{ 11744, 11775 },\n\t{ 12330, 12333 },\n\t{ 12441, 12442 },\n\t{ 42607, 42607 },\n\t{ 42612, 42621 },\n\t{ 42654, 42655 },\n\t{ 42736, 42737 },\n\t{ 43010, 43010 },\n\t{ 43014, 43014 },\n\t{ 43019, 43019 },\n\t{ 43045, 43046 },\n\t{ 43052, 43052 },\n\t{ 43204, 43205 },\n\t{ 43232, 43249 },\n\t{ 43263, 43263 },\n\t{ 43302, 43309 },\n\t{ 43335, 43345 },\n\t{ 43392, 43394 },\n\t{ 43443, 43443 },\n\t{ 43446, 43449 },\n\t{ 43452, 43453 },\n\t{ 43493, 43493 },\n\t{ 43561, 43566 },\n\t{ 43569, 43570 },\n\t{ 43573, 43574 },\n\t{ 43587, 43587 },\n\t{ 43596, 43596 },\n\t{ 43644, 43644 },\n\t{ 43696, 43696 },\n\t{ 43698, 43700 },\n\t{ 43703, 43704 },\n\t{ 43710, 43711 },\n\t{ 43713, 43713 },\n\t{ 43756, 43757 },\n\t{ 43766, 43766 },\n\t{ 44005, 44005 },\n\t{ 44008, 44008 },\n\t{ 44013, 44013 },\n\t{ 64286, 64286 },\n\t{ 65024, 65039 },\n\t{ 65056, 65071 },\n};\nstatic const URange32 Mn_range32[] = {\n\t{ 66045, 66045 },\n\t{ 66272, 66272 },\n\t{ 66422, 66426 },\n\t{ 68097, 68099 },\n\t{ 68101, 68102 },\n\t{ 68108, 68111 },\n\t{ 68152, 68154 },\n\t{ 68159, 68159 },\n\t{ 68325, 68326 },\n\t{ 68900, 68903 },\n\t{ 69291, 69292 },\n\t{ 69373, 69375 },\n\t{ 69446, 69456 },\n\t{ 69506, 69509 },\n\t{ 69633, 69633 },\n\t{ 69688, 69702 },\n\t{ 69744, 69744 },\n\t{ 69747, 69748 },\n\t{ 69759, 69761 },\n\t{ 69811, 69814 },\n\t{ 69817, 69818 },\n\t{ 69826, 69826 },\n\t{ 69888, 69890 },\n\t{ 69927, 69931 },\n\t{ 69933, 69940 },\n\t{ 70003, 70003 },\n\t{ 70016, 70017 },\n\t{ 70070, 70078 },\n\t{ 70089, 70092 },\n\t{ 70095, 70095 },\n\t{ 70191, 70193 },\n\t{ 70196, 70196 },\n\t{ 70198, 70199 },\n\t{ 70206, 70206 },\n\t{ 70209, 70209 },\n\t{ 70367, 70367 },\n\t{ 70371, 70378 },\n\t{ 70400, 70401 },\n\t{ 70459, 70460 },\n\t{ 70464, 70464 },\n\t{ 70502, 70508 },\n\t{ 70512, 70516 },\n\t{ 70712, 70719 },\n\t{ 70722, 70724 },\n\t{ 70726, 70726 },\n\t{ 70750, 70750 },\n\t{ 70835, 70840 },\n\t{ 70842, 70842 },\n\t{ 70847, 70848 },\n\t{ 70850, 70851 },\n\t{ 71090, 71093 },\n\t{ 71100, 71101 },\n\t{ 71103, 71104 },\n\t{ 71132, 71133 },\n\t{ 71219, 71226 },\n\t{ 71229, 71229 },\n\t{ 71231, 71232 },\n\t{ 71339, 71339 },\n\t{ 71341, 71341 },\n\t{ 71344, 71349 },\n\t{ 71351, 71351 },\n\t{ 71453, 71455 },\n\t{ 71458, 71461 },\n\t{ 71463, 71467 },\n\t{ 71727, 71735 },\n\t{ 71737, 71738 },\n\t{ 71995, 71996 },\n\t{ 71998, 71998 },\n\t{ 72003, 72003 },\n\t{ 72148, 72151 },\n\t{ 72154, 72155 },\n\t{ 72160, 72160 },\n\t{ 72193, 72202 },\n\t{ 72243, 72248 },\n\t{ 72251, 72254 },\n\t{ 72263, 72263 },\n\t{ 72273, 72278 },\n\t{ 72281, 72283 },\n\t{ 72330, 72342 },\n\t{ 72344, 72345 },\n\t{ 72752, 72758 },\n\t{ 72760, 72765 },\n\t{ 72767, 72767 },\n\t{ 72850, 72871 },\n\t{ 72874, 72880 },\n\t{ 72882, 72883 },\n\t{ 72885, 72886 },\n\t{ 73009, 73014 },\n\t{ 73018, 73018 },\n\t{ 73020, 73021 },\n\t{ 73023, 73029 },\n\t{ 73031, 73031 },\n\t{ 73104, 73105 },\n\t{ 73109, 73109 },\n\t{ 73111, 73111 },\n\t{ 73459, 73460 },\n\t{ 73472, 73473 },\n\t{ 73526, 73530 },\n\t{ 73536, 73536 },\n\t{ 73538, 73538 },\n\t{ 78912, 78912 },\n\t{ 78919, 78933 },\n\t{ 92912, 92916 },\n\t{ 92976, 92982 },\n\t{ 94031, 94031 },\n\t{ 94095, 94098 },\n\t{ 94180, 94180 },\n\t{ 113821, 113822 },\n\t{ 118528, 118573 },\n\t{ 118576, 118598 },\n\t{ 119143, 119145 },\n\t{ 119163, 119170 },\n\t{ 119173, 119179 },\n\t{ 119210, 119213 },\n\t{ 119362, 119364 },\n\t{ 121344, 121398 },\n\t{ 121403, 121452 },\n\t{ 121461, 121461 },\n\t{ 121476, 121476 },\n\t{ 121499, 121503 },\n\t{ 121505, 121519 },\n\t{ 122880, 122886 },\n\t{ 122888, 122904 },\n\t{ 122907, 122913 },\n\t{ 122915, 122916 },\n\t{ 122918, 122922 },\n\t{ 123023, 123023 },\n\t{ 123184, 123190 },\n\t{ 123566, 123566 },\n\t{ 123628, 123631 },\n\t{ 124140, 124143 },\n\t{ 125136, 125142 },\n\t{ 125252, 125258 },\n\t{ 917760, 917999 },\n};\nstatic const URange16 N_range16[] = {\n\t{ 48, 57 },\n\t{ 178, 179 },\n\t{ 185, 185 },\n\t{ 188, 190 },\n\t{ 1632, 1641 },\n\t{ 1776, 1785 },\n\t{ 1984, 1993 },\n\t{ 2406, 2415 },\n\t{ 2534, 2543 },\n\t{ 2548, 2553 },\n\t{ 2662, 2671 },\n\t{ 2790, 2799 },\n\t{ 2918, 2927 },\n\t{ 2930, 2935 },\n\t{ 3046, 3058 },\n\t{ 3174, 3183 },\n\t{ 3192, 3198 },\n\t{ 3302, 3311 },\n\t{ 3416, 3422 },\n\t{ 3430, 3448 },\n\t{ 3558, 3567 },\n\t{ 3664, 3673 },\n\t{ 3792, 3801 },\n\t{ 3872, 3891 },\n\t{ 4160, 4169 },\n\t{ 4240, 4249 },\n\t{ 4969, 4988 },\n\t{ 5870, 5872 },\n\t{ 6112, 6121 },\n\t{ 6128, 6137 },\n\t{ 6160, 6169 },\n\t{ 6470, 6479 },\n\t{ 6608, 6618 },\n\t{ 6784, 6793 },\n\t{ 6800, 6809 },\n\t{ 6992, 7001 },\n\t{ 7088, 7097 },\n\t{ 7232, 7241 },\n\t{ 7248, 7257 },\n\t{ 8304, 8304 },\n\t{ 8308, 8313 },\n\t{ 8320, 8329 },\n\t{ 8528, 8578 },\n\t{ 8581, 8585 },\n\t{ 9312, 9371 },\n\t{ 9450, 9471 },\n\t{ 10102, 10131 },\n\t{ 11517, 11517 },\n\t{ 12295, 12295 },\n\t{ 12321, 12329 },\n\t{ 12344, 12346 },\n\t{ 12690, 12693 },\n\t{ 12832, 12841 },\n\t{ 12872, 12879 },\n\t{ 12881, 12895 },\n\t{ 12928, 12937 },\n\t{ 12977, 12991 },\n\t{ 42528, 42537 },\n\t{ 42726, 42735 },\n\t{ 43056, 43061 },\n\t{ 43216, 43225 },\n\t{ 43264, 43273 },\n\t{ 43472, 43481 },\n\t{ 43504, 43513 },\n\t{ 43600, 43609 },\n\t{ 44016, 44025 },\n\t{ 65296, 65305 },\n};\nstatic const URange32 N_range32[] = {\n\t{ 65799, 65843 },\n\t{ 65856, 65912 },\n\t{ 65930, 65931 },\n\t{ 66273, 66299 },\n\t{ 66336, 66339 },\n\t{ 66369, 66369 },\n\t{ 66378, 66378 },\n\t{ 66513, 66517 },\n\t{ 66720, 66729 },\n\t{ 67672, 67679 },\n\t{ 67705, 67711 },\n\t{ 67751, 67759 },\n\t{ 67835, 67839 },\n\t{ 67862, 67867 },\n\t{ 68028, 68029 },\n\t{ 68032, 68047 },\n\t{ 68050, 68095 },\n\t{ 68160, 68168 },\n\t{ 68221, 68222 },\n\t{ 68253, 68255 },\n\t{ 68331, 68335 },\n\t{ 68440, 68447 },\n\t{ 68472, 68479 },\n\t{ 68521, 68527 },\n\t{ 68858, 68863 },\n\t{ 68912, 68921 },\n\t{ 69216, 69246 },\n\t{ 69405, 69414 },\n\t{ 69457, 69460 },\n\t{ 69573, 69579 },\n\t{ 69714, 69743 },\n\t{ 69872, 69881 },\n\t{ 69942, 69951 },\n\t{ 70096, 70105 },\n\t{ 70113, 70132 },\n\t{ 70384, 70393 },\n\t{ 70736, 70745 },\n\t{ 70864, 70873 },\n\t{ 71248, 71257 },\n\t{ 71360, 71369 },\n\t{ 71472, 71483 },\n\t{ 71904, 71922 },\n\t{ 72016, 72025 },\n\t{ 72784, 72812 },\n\t{ 73040, 73049 },\n\t{ 73120, 73129 },\n\t{ 73552, 73561 },\n\t{ 73664, 73684 },\n\t{ 74752, 74862 },\n\t{ 92768, 92777 },\n\t{ 92864, 92873 },\n\t{ 93008, 93017 },\n\t{ 93019, 93025 },\n\t{ 93824, 93846 },\n\t{ 119488, 119507 },\n\t{ 119520, 119539 },\n\t{ 119648, 119672 },\n\t{ 120782, 120831 },\n\t{ 123200, 123209 },\n\t{ 123632, 123641 },\n\t{ 124144, 124153 },\n\t{ 125127, 125135 },\n\t{ 125264, 125273 },\n\t{ 126065, 126123 },\n\t{ 126125, 126127 },\n\t{ 126129, 126132 },\n\t{ 126209, 126253 },\n\t{ 126255, 126269 },\n\t{ 127232, 127244 },\n\t{ 130032, 130041 },\n};\nstatic const URange16 Nd_range16[] = {\n\t{ 48, 57 },\n\t{ 1632, 1641 },\n\t{ 1776, 1785 },\n\t{ 1984, 1993 },\n\t{ 2406, 2415 },\n\t{ 2534, 2543 },\n\t{ 2662, 2671 },\n\t{ 2790, 2799 },\n\t{ 2918, 2927 },\n\t{ 3046, 3055 },\n\t{ 3174, 3183 },\n\t{ 3302, 3311 },\n\t{ 3430, 3439 },\n\t{ 3558, 3567 },\n\t{ 3664, 3673 },\n\t{ 3792, 3801 },\n\t{ 3872, 3881 },\n\t{ 4160, 4169 },\n\t{ 4240, 4249 },\n\t{ 6112, 6121 },\n\t{ 6160, 6169 },\n\t{ 6470, 6479 },\n\t{ 6608, 6617 },\n\t{ 6784, 6793 },\n\t{ 6800, 6809 },\n\t{ 6992, 7001 },\n\t{ 7088, 7097 },\n\t{ 7232, 7241 },\n\t{ 7248, 7257 },\n\t{ 42528, 42537 },\n\t{ 43216, 43225 },\n\t{ 43264, 43273 },\n\t{ 43472, 43481 },\n\t{ 43504, 43513 },\n\t{ 43600, 43609 },\n\t{ 44016, 44025 },\n\t{ 65296, 65305 },\n};\nstatic const URange32 Nd_range32[] = {\n\t{ 66720, 66729 },\n\t{ 68912, 68921 },\n\t{ 69734, 69743 },\n\t{ 69872, 69881 },\n\t{ 69942, 69951 },\n\t{ 70096, 70105 },\n\t{ 70384, 70393 },\n\t{ 70736, 70745 },\n\t{ 70864, 70873 },\n\t{ 71248, 71257 },\n\t{ 71360, 71369 },\n\t{ 71472, 71481 },\n\t{ 71904, 71913 },\n\t{ 72016, 72025 },\n\t{ 72784, 72793 },\n\t{ 73040, 73049 },\n\t{ 73120, 73129 },\n\t{ 73552, 73561 },\n\t{ 92768, 92777 },\n\t{ 92864, 92873 },\n\t{ 93008, 93017 },\n\t{ 120782, 120831 },\n\t{ 123200, 123209 },\n\t{ 123632, 123641 },\n\t{ 124144, 124153 },\n\t{ 125264, 125273 },\n\t{ 130032, 130041 },\n};\nstatic const URange16 Nl_range16[] = {\n\t{ 5870, 5872 },\n\t{ 8544, 8578 },\n\t{ 8581, 8584 },\n\t{ 12295, 12295 },\n\t{ 12321, 12329 },\n\t{ 12344, 12346 },\n\t{ 42726, 42735 },\n};\nstatic const URange32 Nl_range32[] = {\n\t{ 65856, 65908 },\n\t{ 66369, 66369 },\n\t{ 66378, 66378 },\n\t{ 66513, 66517 },\n\t{ 74752, 74862 },\n};\nstatic const URange16 No_range16[] = {\n\t{ 178, 179 },\n\t{ 185, 185 },\n\t{ 188, 190 },\n\t{ 2548, 2553 },\n\t{ 2930, 2935 },\n\t{ 3056, 3058 },\n\t{ 3192, 3198 },\n\t{ 3416, 3422 },\n\t{ 3440, 3448 },\n\t{ 3882, 3891 },\n\t{ 4969, 4988 },\n\t{ 6128, 6137 },\n\t{ 6618, 6618 },\n\t{ 8304, 8304 },\n\t{ 8308, 8313 },\n\t{ 8320, 8329 },\n\t{ 8528, 8543 },\n\t{ 8585, 8585 },\n\t{ 9312, 9371 },\n\t{ 9450, 9471 },\n\t{ 10102, 10131 },\n\t{ 11517, 11517 },\n\t{ 12690, 12693 },\n\t{ 12832, 12841 },\n\t{ 12872, 12879 },\n\t{ 12881, 12895 },\n\t{ 12928, 12937 },\n\t{ 12977, 12991 },\n\t{ 43056, 43061 },\n};\nstatic const URange32 No_range32[] = {\n\t{ 65799, 65843 },\n\t{ 65909, 65912 },\n\t{ 65930, 65931 },\n\t{ 66273, 66299 },\n\t{ 66336, 66339 },\n\t{ 67672, 67679 },\n\t{ 67705, 67711 },\n\t{ 67751, 67759 },\n\t{ 67835, 67839 },\n\t{ 67862, 67867 },\n\t{ 68028, 68029 },\n\t{ 68032, 68047 },\n\t{ 68050, 68095 },\n\t{ 68160, 68168 },\n\t{ 68221, 68222 },\n\t{ 68253, 68255 },\n\t{ 68331, 68335 },\n\t{ 68440, 68447 },\n\t{ 68472, 68479 },\n\t{ 68521, 68527 },\n\t{ 68858, 68863 },\n\t{ 69216, 69246 },\n\t{ 69405, 69414 },\n\t{ 69457, 69460 },\n\t{ 69573, 69579 },\n\t{ 69714, 69733 },\n\t{ 70113, 70132 },\n\t{ 71482, 71483 },\n\t{ 71914, 71922 },\n\t{ 72794, 72812 },\n\t{ 73664, 73684 },\n\t{ 93019, 93025 },\n\t{ 93824, 93846 },\n\t{ 119488, 119507 },\n\t{ 119520, 119539 },\n\t{ 119648, 119672 },\n\t{ 125127, 125135 },\n\t{ 126065, 126123 },\n\t{ 126125, 126127 },\n\t{ 126129, 126132 },\n\t{ 126209, 126253 },\n\t{ 126255, 126269 },\n\t{ 127232, 127244 },\n};\nstatic const URange16 P_range16[] = {\n\t{ 33, 35 },\n\t{ 37, 42 },\n\t{ 44, 47 },\n\t{ 58, 59 },\n\t{ 63, 64 },\n\t{ 91, 93 },\n\t{ 95, 95 },\n\t{ 123, 123 },\n\t{ 125, 125 },\n\t{ 161, 161 },\n\t{ 167, 167 },\n\t{ 171, 171 },\n\t{ 182, 183 },\n\t{ 187, 187 },\n\t{ 191, 191 },\n\t{ 894, 894 },\n\t{ 903, 903 },\n\t{ 1370, 1375 },\n\t{ 1417, 1418 },\n\t{ 1470, 1470 },\n\t{ 1472, 1472 },\n\t{ 1475, 1475 },\n\t{ 1478, 1478 },\n\t{ 1523, 1524 },\n\t{ 1545, 1546 },\n\t{ 1548, 1549 },\n\t{ 1563, 1563 },\n\t{ 1565, 1567 },\n\t{ 1642, 1645 },\n\t{ 1748, 1748 },\n\t{ 1792, 1805 },\n\t{ 2039, 2041 },\n\t{ 2096, 2110 },\n\t{ 2142, 2142 },\n\t{ 2404, 2405 },\n\t{ 2416, 2416 },\n\t{ 2557, 2557 },\n\t{ 2678, 2678 },\n\t{ 2800, 2800 },\n\t{ 3191, 3191 },\n\t{ 3204, 3204 },\n\t{ 3572, 3572 },\n\t{ 3663, 3663 },\n\t{ 3674, 3675 },\n\t{ 3844, 3858 },\n\t{ 3860, 3860 },\n\t{ 3898, 3901 },\n\t{ 3973, 3973 },\n\t{ 4048, 4052 },\n\t{ 4057, 4058 },\n\t{ 4170, 4175 },\n\t{ 4347, 4347 },\n\t{ 4960, 4968 },\n\t{ 5120, 5120 },\n\t{ 5742, 5742 },\n\t{ 5787, 5788 },\n\t{ 5867, 5869 },\n\t{ 5941, 5942 },\n\t{ 6100, 6102 },\n\t{ 6104, 6106 },\n\t{ 6144, 6154 },\n\t{ 6468, 6469 },\n\t{ 6686, 6687 },\n\t{ 6816, 6822 },\n\t{ 6824, 6829 },\n\t{ 7002, 7008 },\n\t{ 7037, 7038 },\n\t{ 7164, 7167 },\n\t{ 7227, 7231 },\n\t{ 7294, 7295 },\n\t{ 7360, 7367 },\n\t{ 7379, 7379 },\n\t{ 8208, 8231 },\n\t{ 8240, 8259 },\n\t{ 8261, 8273 },\n\t{ 8275, 8286 },\n\t{ 8317, 8318 },\n\t{ 8333, 8334 },\n\t{ 8968, 8971 },\n\t{ 9001, 9002 },\n\t{ 10088, 10101 },\n\t{ 10181, 10182 },\n\t{ 10214, 10223 },\n\t{ 10627, 10648 },\n\t{ 10712, 10715 },\n\t{ 10748, 10749 },\n\t{ 11513, 11516 },\n\t{ 11518, 11519 },\n\t{ 11632, 11632 },\n\t{ 11776, 11822 },\n\t{ 11824, 11855 },\n\t{ 11858, 11869 },\n\t{ 12289, 12291 },\n\t{ 12296, 12305 },\n\t{ 12308, 12319 },\n\t{ 12336, 12336 },\n\t{ 12349, 12349 },\n\t{ 12448, 12448 },\n\t{ 12539, 12539 },\n\t{ 42238, 42239 },\n\t{ 42509, 42511 },\n\t{ 42611, 42611 },\n\t{ 42622, 42622 },\n\t{ 42738, 42743 },\n\t{ 43124, 43127 },\n\t{ 43214, 43215 },\n\t{ 43256, 43258 },\n\t{ 43260, 43260 },\n\t{ 43310, 43311 },\n\t{ 43359, 43359 },\n\t{ 43457, 43469 },\n\t{ 43486, 43487 },\n\t{ 43612, 43615 },\n\t{ 43742, 43743 },\n\t{ 43760, 43761 },\n\t{ 44011, 44011 },\n\t{ 64830, 64831 },\n\t{ 65040, 65049 },\n\t{ 65072, 65106 },\n\t{ 65108, 65121 },\n\t{ 65123, 65123 },\n\t{ 65128, 65128 },\n\t{ 65130, 65131 },\n\t{ 65281, 65283 },\n\t{ 65285, 65290 },\n\t{ 65292, 65295 },\n\t{ 65306, 65307 },\n\t{ 65311, 65312 },\n\t{ 65339, 65341 },\n\t{ 65343, 65343 },\n\t{ 65371, 65371 },\n\t{ 65373, 65373 },\n\t{ 65375, 65381 },\n};\nstatic const URange32 P_range32[] = {\n\t{ 65792, 65794 },\n\t{ 66463, 66463 },\n\t{ 66512, 66512 },\n\t{ 66927, 66927 },\n\t{ 67671, 67671 },\n\t{ 67871, 67871 },\n\t{ 67903, 67903 },\n\t{ 68176, 68184 },\n\t{ 68223, 68223 },\n\t{ 68336, 68342 },\n\t{ 68409, 68415 },\n\t{ 68505, 68508 },\n\t{ 69293, 69293 },\n\t{ 69461, 69465 },\n\t{ 69510, 69513 },\n\t{ 69703, 69709 },\n\t{ 69819, 69820 },\n\t{ 69822, 69825 },\n\t{ 69952, 69955 },\n\t{ 70004, 70005 },\n\t{ 70085, 70088 },\n\t{ 70093, 70093 },\n\t{ 70107, 70107 },\n\t{ 70109, 70111 },\n\t{ 70200, 70205 },\n\t{ 70313, 70313 },\n\t{ 70731, 70735 },\n\t{ 70746, 70747 },\n\t{ 70749, 70749 },\n\t{ 70854, 70854 },\n\t{ 71105, 71127 },\n\t{ 71233, 71235 },\n\t{ 71264, 71276 },\n\t{ 71353, 71353 },\n\t{ 71484, 71486 },\n\t{ 71739, 71739 },\n\t{ 72004, 72006 },\n\t{ 72162, 72162 },\n\t{ 72255, 72262 },\n\t{ 72346, 72348 },\n\t{ 72350, 72354 },\n\t{ 72448, 72457 },\n\t{ 72769, 72773 },\n\t{ 72816, 72817 },\n\t{ 73463, 73464 },\n\t{ 73539, 73551 },\n\t{ 73727, 73727 },\n\t{ 74864, 74868 },\n\t{ 77809, 77810 },\n\t{ 92782, 92783 },\n\t{ 92917, 92917 },\n\t{ 92983, 92987 },\n\t{ 92996, 92996 },\n\t{ 93847, 93850 },\n\t{ 94178, 94178 },\n\t{ 113823, 113823 },\n\t{ 121479, 121483 },\n\t{ 125278, 125279 },\n};\nstatic const URange16 Pc_range16[] = {\n\t{ 95, 95 },\n\t{ 8255, 8256 },\n\t{ 8276, 8276 },\n\t{ 65075, 65076 },\n\t{ 65101, 65103 },\n\t{ 65343, 65343 },\n};\nstatic const URange16 Pd_range16[] = {\n\t{ 45, 45 },\n\t{ 1418, 1418 },\n\t{ 1470, 1470 },\n\t{ 5120, 5120 },\n\t{ 6150, 6150 },\n\t{ 8208, 8213 },\n\t{ 11799, 11799 },\n\t{ 11802, 11802 },\n\t{ 11834, 11835 },\n\t{ 11840, 11840 },\n\t{ 11869, 11869 },\n\t{ 12316, 12316 },\n\t{ 12336, 12336 },\n\t{ 12448, 12448 },\n\t{ 65073, 65074 },\n\t{ 65112, 65112 },\n\t{ 65123, 65123 },\n\t{ 65293, 65293 },\n};\nstatic const URange32 Pd_range32[] = {\n\t{ 69293, 69293 },\n};\nstatic const URange16 Pe_range16[] = {\n\t{ 41, 41 },\n\t{ 93, 93 },\n\t{ 125, 125 },\n\t{ 3899, 3899 },\n\t{ 3901, 3901 },\n\t{ 5788, 5788 },\n\t{ 8262, 8262 },\n\t{ 8318, 8318 },\n\t{ 8334, 8334 },\n\t{ 8969, 8969 },\n\t{ 8971, 8971 },\n\t{ 9002, 9002 },\n\t{ 10089, 10089 },\n\t{ 10091, 10091 },\n\t{ 10093, 10093 },\n\t{ 10095, 10095 },\n\t{ 10097, 10097 },\n\t{ 10099, 10099 },\n\t{ 10101, 10101 },\n\t{ 10182, 10182 },\n\t{ 10215, 10215 },\n\t{ 10217, 10217 },\n\t{ 10219, 10219 },\n\t{ 10221, 10221 },\n\t{ 10223, 10223 },\n\t{ 10628, 10628 },\n\t{ 10630, 10630 },\n\t{ 10632, 10632 },\n\t{ 10634, 10634 },\n\t{ 10636, 10636 },\n\t{ 10638, 10638 },\n\t{ 10640, 10640 },\n\t{ 10642, 10642 },\n\t{ 10644, 10644 },\n\t{ 10646, 10646 },\n\t{ 10648, 10648 },\n\t{ 10713, 10713 },\n\t{ 10715, 10715 },\n\t{ 10749, 10749 },\n\t{ 11811, 11811 },\n\t{ 11813, 11813 },\n\t{ 11815, 11815 },\n\t{ 11817, 11817 },\n\t{ 11862, 11862 },\n\t{ 11864, 11864 },\n\t{ 11866, 11866 },\n\t{ 11868, 11868 },\n\t{ 12297, 12297 },\n\t{ 12299, 12299 },\n\t{ 12301, 12301 },\n\t{ 12303, 12303 },\n\t{ 12305, 12305 },\n\t{ 12309, 12309 },\n\t{ 12311, 12311 },\n\t{ 12313, 12313 },\n\t{ 12315, 12315 },\n\t{ 12318, 12319 },\n\t{ 64830, 64830 },\n\t{ 65048, 65048 },\n\t{ 65078, 65078 },\n\t{ 65080, 65080 },\n\t{ 65082, 65082 },\n\t{ 65084, 65084 },\n\t{ 65086, 65086 },\n\t{ 65088, 65088 },\n\t{ 65090, 65090 },\n\t{ 65092, 65092 },\n\t{ 65096, 65096 },\n\t{ 65114, 65114 },\n\t{ 65116, 65116 },\n\t{ 65118, 65118 },\n\t{ 65289, 65289 },\n\t{ 65341, 65341 },\n\t{ 65373, 65373 },\n\t{ 65376, 65376 },\n\t{ 65379, 65379 },\n};\nstatic const URange16 Pf_range16[] = {\n\t{ 187, 187 },\n\t{ 8217, 8217 },\n\t{ 8221, 8221 },\n\t{ 8250, 8250 },\n\t{ 11779, 11779 },\n\t{ 11781, 11781 },\n\t{ 11786, 11786 },\n\t{ 11789, 11789 },\n\t{ 11805, 11805 },\n\t{ 11809, 11809 },\n};\nstatic const URange16 Pi_range16[] = {\n\t{ 171, 171 },\n\t{ 8216, 8216 },\n\t{ 8219, 8220 },\n\t{ 8223, 8223 },\n\t{ 8249, 8249 },\n\t{ 11778, 11778 },\n\t{ 11780, 11780 },\n\t{ 11785, 11785 },\n\t{ 11788, 11788 },\n\t{ 11804, 11804 },\n\t{ 11808, 11808 },\n};\nstatic const URange16 Po_range16[] = {\n\t{ 33, 35 },\n\t{ 37, 39 },\n\t{ 42, 42 },\n\t{ 44, 44 },\n\t{ 46, 47 },\n\t{ 58, 59 },\n\t{ 63, 64 },\n\t{ 92, 92 },\n\t{ 161, 161 },\n\t{ 167, 167 },\n\t{ 182, 183 },\n\t{ 191, 191 },\n\t{ 894, 894 },\n\t{ 903, 903 },\n\t{ 1370, 1375 },\n\t{ 1417, 1417 },\n\t{ 1472, 1472 },\n\t{ 1475, 1475 },\n\t{ 1478, 1478 },\n\t{ 1523, 1524 },\n\t{ 1545, 1546 },\n\t{ 1548, 1549 },\n\t{ 1563, 1563 },\n\t{ 1565, 1567 },\n\t{ 1642, 1645 },\n\t{ 1748, 1748 },\n\t{ 1792, 1805 },\n\t{ 2039, 2041 },\n\t{ 2096, 2110 },\n\t{ 2142, 2142 },\n\t{ 2404, 2405 },\n\t{ 2416, 2416 },\n\t{ 2557, 2557 },\n\t{ 2678, 2678 },\n\t{ 2800, 2800 },\n\t{ 3191, 3191 },\n\t{ 3204, 3204 },\n\t{ 3572, 3572 },\n\t{ 3663, 3663 },\n\t{ 3674, 3675 },\n\t{ 3844, 3858 },\n\t{ 3860, 3860 },\n\t{ 3973, 3973 },\n\t{ 4048, 4052 },\n\t{ 4057, 4058 },\n\t{ 4170, 4175 },\n\t{ 4347, 4347 },\n\t{ 4960, 4968 },\n\t{ 5742, 5742 },\n\t{ 5867, 5869 },\n\t{ 5941, 5942 },\n\t{ 6100, 6102 },\n\t{ 6104, 6106 },\n\t{ 6144, 6149 },\n\t{ 6151, 6154 },\n\t{ 6468, 6469 },\n\t{ 6686, 6687 },\n\t{ 6816, 6822 },\n\t{ 6824, 6829 },\n\t{ 7002, 7008 },\n\t{ 7037, 7038 },\n\t{ 7164, 7167 },\n\t{ 7227, 7231 },\n\t{ 7294, 7295 },\n\t{ 7360, 7367 },\n\t{ 7379, 7379 },\n\t{ 8214, 8215 },\n\t{ 8224, 8231 },\n\t{ 8240, 8248 },\n\t{ 8251, 8254 },\n\t{ 8257, 8259 },\n\t{ 8263, 8273 },\n\t{ 8275, 8275 },\n\t{ 8277, 8286 },\n\t{ 11513, 11516 },\n\t{ 11518, 11519 },\n\t{ 11632, 11632 },\n\t{ 11776, 11777 },\n\t{ 11782, 11784 },\n\t{ 11787, 11787 },\n\t{ 11790, 11798 },\n\t{ 11800, 11801 },\n\t{ 11803, 11803 },\n\t{ 11806, 11807 },\n\t{ 11818, 11822 },\n\t{ 11824, 11833 },\n\t{ 11836, 11839 },\n\t{ 11841, 11841 },\n\t{ 11843, 11855 },\n\t{ 11858, 11860 },\n\t{ 12289, 12291 },\n\t{ 12349, 12349 },\n\t{ 12539, 12539 },\n\t{ 42238, 42239 },\n\t{ 42509, 42511 },\n\t{ 42611, 42611 },\n\t{ 42622, 42622 },\n\t{ 42738, 42743 },\n\t{ 43124, 43127 },\n\t{ 43214, 43215 },\n\t{ 43256, 43258 },\n\t{ 43260, 43260 },\n\t{ 43310, 43311 },\n\t{ 43359, 43359 },\n\t{ 43457, 43469 },\n\t{ 43486, 43487 },\n\t{ 43612, 43615 },\n\t{ 43742, 43743 },\n\t{ 43760, 43761 },\n\t{ 44011, 44011 },\n\t{ 65040, 65046 },\n\t{ 65049, 65049 },\n\t{ 65072, 65072 },\n\t{ 65093, 65094 },\n\t{ 65097, 65100 },\n\t{ 65104, 65106 },\n\t{ 65108, 65111 },\n\t{ 65119, 65121 },\n\t{ 65128, 65128 },\n\t{ 65130, 65131 },\n\t{ 65281, 65283 },\n\t{ 65285, 65287 },\n\t{ 65290, 65290 },\n\t{ 65292, 65292 },\n\t{ 65294, 65295 },\n\t{ 65306, 65307 },\n\t{ 65311, 65312 },\n\t{ 65340, 65340 },\n\t{ 65377, 65377 },\n\t{ 65380, 65381 },\n};\nstatic const URange32 Po_range32[] = {\n\t{ 65792, 65794 },\n\t{ 66463, 66463 },\n\t{ 66512, 66512 },\n\t{ 66927, 66927 },\n\t{ 67671, 67671 },\n\t{ 67871, 67871 },\n\t{ 67903, 67903 },\n\t{ 68176, 68184 },\n\t{ 68223, 68223 },\n\t{ 68336, 68342 },\n\t{ 68409, 68415 },\n\t{ 68505, 68508 },\n\t{ 69461, 69465 },\n\t{ 69510, 69513 },\n\t{ 69703, 69709 },\n\t{ 69819, 69820 },\n\t{ 69822, 69825 },\n\t{ 69952, 69955 },\n\t{ 70004, 70005 },\n\t{ 70085, 70088 },\n\t{ 70093, 70093 },\n\t{ 70107, 70107 },\n\t{ 70109, 70111 },\n\t{ 70200, 70205 },\n\t{ 70313, 70313 },\n\t{ 70731, 70735 },\n\t{ 70746, 70747 },\n\t{ 70749, 70749 },\n\t{ 70854, 70854 },\n\t{ 71105, 71127 },\n\t{ 71233, 71235 },\n\t{ 71264, 71276 },\n\t{ 71353, 71353 },\n\t{ 71484, 71486 },\n\t{ 71739, 71739 },\n\t{ 72004, 72006 },\n\t{ 72162, 72162 },\n\t{ 72255, 72262 },\n\t{ 72346, 72348 },\n\t{ 72350, 72354 },\n\t{ 72448, 72457 },\n\t{ 72769, 72773 },\n\t{ 72816, 72817 },\n\t{ 73463, 73464 },\n\t{ 73539, 73551 },\n\t{ 73727, 73727 },\n\t{ 74864, 74868 },\n\t{ 77809, 77810 },\n\t{ 92782, 92783 },\n\t{ 92917, 92917 },\n\t{ 92983, 92987 },\n\t{ 92996, 92996 },\n\t{ 93847, 93850 },\n\t{ 94178, 94178 },\n\t{ 113823, 113823 },\n\t{ 121479, 121483 },\n\t{ 125278, 125279 },\n};\nstatic const URange16 Ps_range16[] = {\n\t{ 40, 40 },\n\t{ 91, 91 },\n\t{ 123, 123 },\n\t{ 3898, 3898 },\n\t{ 3900, 3900 },\n\t{ 5787, 5787 },\n\t{ 8218, 8218 },\n\t{ 8222, 8222 },\n\t{ 8261, 8261 },\n\t{ 8317, 8317 },\n\t{ 8333, 8333 },\n\t{ 8968, 8968 },\n\t{ 8970, 8970 },\n\t{ 9001, 9001 },\n\t{ 10088, 10088 },\n\t{ 10090, 10090 },\n\t{ 10092, 10092 },\n\t{ 10094, 10094 },\n\t{ 10096, 10096 },\n\t{ 10098, 10098 },\n\t{ 10100, 10100 },\n\t{ 10181, 10181 },\n\t{ 10214, 10214 },\n\t{ 10216, 10216 },\n\t{ 10218, 10218 },\n\t{ 10220, 10220 },\n\t{ 10222, 10222 },\n\t{ 10627, 10627 },\n\t{ 10629, 10629 },\n\t{ 10631, 10631 },\n\t{ 10633, 10633 },\n\t{ 10635, 10635 },\n\t{ 10637, 10637 },\n\t{ 10639, 10639 },\n\t{ 10641, 10641 },\n\t{ 10643, 10643 },\n\t{ 10645, 10645 },\n\t{ 10647, 10647 },\n\t{ 10712, 10712 },\n\t{ 10714, 10714 },\n\t{ 10748, 10748 },\n\t{ 11810, 11810 },\n\t{ 11812, 11812 },\n\t{ 11814, 11814 },\n\t{ 11816, 11816 },\n\t{ 11842, 11842 },\n\t{ 11861, 11861 },\n\t{ 11863, 11863 },\n\t{ 11865, 11865 },\n\t{ 11867, 11867 },\n\t{ 12296, 12296 },\n\t{ 12298, 12298 },\n\t{ 12300, 12300 },\n\t{ 12302, 12302 },\n\t{ 12304, 12304 },\n\t{ 12308, 12308 },\n\t{ 12310, 12310 },\n\t{ 12312, 12312 },\n\t{ 12314, 12314 },\n\t{ 12317, 12317 },\n\t{ 64831, 64831 },\n\t{ 65047, 65047 },\n\t{ 65077, 65077 },\n\t{ 65079, 65079 },\n\t{ 65081, 65081 },\n\t{ 65083, 65083 },\n\t{ 65085, 65085 },\n\t{ 65087, 65087 },\n\t{ 65089, 65089 },\n\t{ 65091, 65091 },\n\t{ 65095, 65095 },\n\t{ 65113, 65113 },\n\t{ 65115, 65115 },\n\t{ 65117, 65117 },\n\t{ 65288, 65288 },\n\t{ 65339, 65339 },\n\t{ 65371, 65371 },\n\t{ 65375, 65375 },\n\t{ 65378, 65378 },\n};\nstatic const URange16 S_range16[] = {\n\t{ 36, 36 },\n\t{ 43, 43 },\n\t{ 60, 62 },\n\t{ 94, 94 },\n\t{ 96, 96 },\n\t{ 124, 124 },\n\t{ 126, 126 },\n\t{ 162, 166 },\n\t{ 168, 169 },\n\t{ 172, 172 },\n\t{ 174, 177 },\n\t{ 180, 180 },\n\t{ 184, 184 },\n\t{ 215, 215 },\n\t{ 247, 247 },\n\t{ 706, 709 },\n\t{ 722, 735 },\n\t{ 741, 747 },\n\t{ 749, 749 },\n\t{ 751, 767 },\n\t{ 885, 885 },\n\t{ 900, 901 },\n\t{ 1014, 1014 },\n\t{ 1154, 1154 },\n\t{ 1421, 1423 },\n\t{ 1542, 1544 },\n\t{ 1547, 1547 },\n\t{ 1550, 1551 },\n\t{ 1758, 1758 },\n\t{ 1769, 1769 },\n\t{ 1789, 1790 },\n\t{ 2038, 2038 },\n\t{ 2046, 2047 },\n\t{ 2184, 2184 },\n\t{ 2546, 2547 },\n\t{ 2554, 2555 },\n\t{ 2801, 2801 },\n\t{ 2928, 2928 },\n\t{ 3059, 3066 },\n\t{ 3199, 3199 },\n\t{ 3407, 3407 },\n\t{ 3449, 3449 },\n\t{ 3647, 3647 },\n\t{ 3841, 3843 },\n\t{ 3859, 3859 },\n\t{ 3861, 3863 },\n\t{ 3866, 3871 },\n\t{ 3892, 3892 },\n\t{ 3894, 3894 },\n\t{ 3896, 3896 },\n\t{ 4030, 4037 },\n\t{ 4039, 4044 },\n\t{ 4046, 4047 },\n\t{ 4053, 4056 },\n\t{ 4254, 4255 },\n\t{ 5008, 5017 },\n\t{ 5741, 5741 },\n\t{ 6107, 6107 },\n\t{ 6464, 6464 },\n\t{ 6622, 6655 },\n\t{ 7009, 7018 },\n\t{ 7028, 7036 },\n\t{ 8125, 8125 },\n\t{ 8127, 8129 },\n\t{ 8141, 8143 },\n\t{ 8157, 8159 },\n\t{ 8173, 8175 },\n\t{ 8189, 8190 },\n\t{ 8260, 8260 },\n\t{ 8274, 8274 },\n\t{ 8314, 8316 },\n\t{ 8330, 8332 },\n\t{ 8352, 8384 },\n\t{ 8448, 8449 },\n\t{ 8451, 8454 },\n\t{ 8456, 8457 },\n\t{ 8468, 8468 },\n\t{ 8470, 8472 },\n\t{ 8478, 8483 },\n\t{ 8485, 8485 },\n\t{ 8487, 8487 },\n\t{ 8489, 8489 },\n\t{ 8494, 8494 },\n\t{ 8506, 8507 },\n\t{ 8512, 8516 },\n\t{ 8522, 8525 },\n\t{ 8527, 8527 },\n\t{ 8586, 8587 },\n\t{ 8592, 8967 },\n\t{ 8972, 9000 },\n\t{ 9003, 9254 },\n\t{ 9280, 9290 },\n\t{ 9372, 9449 },\n\t{ 9472, 10087 },\n\t{ 10132, 10180 },\n\t{ 10183, 10213 },\n\t{ 10224, 10626 },\n\t{ 10649, 10711 },\n\t{ 10716, 10747 },\n\t{ 10750, 11123 },\n\t{ 11126, 11157 },\n\t{ 11159, 11263 },\n\t{ 11493, 11498 },\n\t{ 11856, 11857 },\n\t{ 11904, 11929 },\n\t{ 11931, 12019 },\n\t{ 12032, 12245 },\n\t{ 12272, 12287 },\n\t{ 12292, 12292 },\n\t{ 12306, 12307 },\n\t{ 12320, 12320 },\n\t{ 12342, 12343 },\n\t{ 12350, 12351 },\n\t{ 12443, 12444 },\n\t{ 12688, 12689 },\n\t{ 12694, 12703 },\n\t{ 12736, 12771 },\n\t{ 12783, 12783 },\n\t{ 12800, 12830 },\n\t{ 12842, 12871 },\n\t{ 12880, 12880 },\n\t{ 12896, 12927 },\n\t{ 12938, 12976 },\n\t{ 12992, 13311 },\n\t{ 19904, 19967 },\n\t{ 42128, 42182 },\n\t{ 42752, 42774 },\n\t{ 42784, 42785 },\n\t{ 42889, 42890 },\n\t{ 43048, 43051 },\n\t{ 43062, 43065 },\n\t{ 43639, 43641 },\n\t{ 43867, 43867 },\n\t{ 43882, 43883 },\n\t{ 64297, 64297 },\n\t{ 64434, 64450 },\n\t{ 64832, 64847 },\n\t{ 64975, 64975 },\n\t{ 65020, 65023 },\n\t{ 65122, 65122 },\n\t{ 65124, 65126 },\n\t{ 65129, 65129 },\n\t{ 65284, 65284 },\n\t{ 65291, 65291 },\n\t{ 65308, 65310 },\n\t{ 65342, 65342 },\n\t{ 65344, 65344 },\n\t{ 65372, 65372 },\n\t{ 65374, 65374 },\n\t{ 65504, 65510 },\n\t{ 65512, 65518 },\n\t{ 65532, 65533 },\n};\nstatic const URange32 S_range32[] = {\n\t{ 65847, 65855 },\n\t{ 65913, 65929 },\n\t{ 65932, 65934 },\n\t{ 65936, 65948 },\n\t{ 65952, 65952 },\n\t{ 66000, 66044 },\n\t{ 67703, 67704 },\n\t{ 68296, 68296 },\n\t{ 71487, 71487 },\n\t{ 73685, 73713 },\n\t{ 92988, 92991 },\n\t{ 92997, 92997 },\n\t{ 113820, 113820 },\n\t{ 118608, 118723 },\n\t{ 118784, 119029 },\n\t{ 119040, 119078 },\n\t{ 119081, 119140 },\n\t{ 119146, 119148 },\n\t{ 119171, 119172 },\n\t{ 119180, 119209 },\n\t{ 119214, 119274 },\n\t{ 119296, 119361 },\n\t{ 119365, 119365 },\n\t{ 119552, 119638 },\n\t{ 120513, 120513 },\n\t{ 120539, 120539 },\n\t{ 120571, 120571 },\n\t{ 120597, 120597 },\n\t{ 120629, 120629 },\n\t{ 120655, 120655 },\n\t{ 120687, 120687 },\n\t{ 120713, 120713 },\n\t{ 120745, 120745 },\n\t{ 120771, 120771 },\n\t{ 120832, 121343 },\n\t{ 121399, 121402 },\n\t{ 121453, 121460 },\n\t{ 121462, 121475 },\n\t{ 121477, 121478 },\n\t{ 123215, 123215 },\n\t{ 123647, 123647 },\n\t{ 126124, 126124 },\n\t{ 126128, 126128 },\n\t{ 126254, 126254 },\n\t{ 126704, 126705 },\n\t{ 126976, 127019 },\n\t{ 127024, 127123 },\n\t{ 127136, 127150 },\n\t{ 127153, 127167 },\n\t{ 127169, 127183 },\n\t{ 127185, 127221 },\n\t{ 127245, 127405 },\n\t{ 127462, 127490 },\n\t{ 127504, 127547 },\n\t{ 127552, 127560 },\n\t{ 127568, 127569 },\n\t{ 127584, 127589 },\n\t{ 127744, 128727 },\n\t{ 128732, 128748 },\n\t{ 128752, 128764 },\n\t{ 128768, 128886 },\n\t{ 128891, 128985 },\n\t{ 128992, 129003 },\n\t{ 129008, 129008 },\n\t{ 129024, 129035 },\n\t{ 129040, 129095 },\n\t{ 129104, 129113 },\n\t{ 129120, 129159 },\n\t{ 129168, 129197 },\n\t{ 129200, 129201 },\n\t{ 129280, 129619 },\n\t{ 129632, 129645 },\n\t{ 129648, 129660 },\n\t{ 129664, 129672 },\n\t{ 129680, 129725 },\n\t{ 129727, 129733 },\n\t{ 129742, 129755 },\n\t{ 129760, 129768 },\n\t{ 129776, 129784 },\n\t{ 129792, 129938 },\n\t{ 129940, 129994 },\n};\nstatic const URange16 Sc_range16[] = {\n\t{ 36, 36 },\n\t{ 162, 165 },\n\t{ 1423, 1423 },\n\t{ 1547, 1547 },\n\t{ 2046, 2047 },\n\t{ 2546, 2547 },\n\t{ 2555, 2555 },\n\t{ 2801, 2801 },\n\t{ 3065, 3065 },\n\t{ 3647, 3647 },\n\t{ 6107, 6107 },\n\t{ 8352, 8384 },\n\t{ 43064, 43064 },\n\t{ 65020, 65020 },\n\t{ 65129, 65129 },\n\t{ 65284, 65284 },\n\t{ 65504, 65505 },\n\t{ 65509, 65510 },\n};\nstatic const URange32 Sc_range32[] = {\n\t{ 73693, 73696 },\n\t{ 123647, 123647 },\n\t{ 126128, 126128 },\n};\nstatic const URange16 Sk_range16[] = {\n\t{ 94, 94 },\n\t{ 96, 96 },\n\t{ 168, 168 },\n\t{ 175, 175 },\n\t{ 180, 180 },\n\t{ 184, 184 },\n\t{ 706, 709 },\n\t{ 722, 735 },\n\t{ 741, 747 },\n\t{ 749, 749 },\n\t{ 751, 767 },\n\t{ 885, 885 },\n\t{ 900, 901 },\n\t{ 2184, 2184 },\n\t{ 8125, 8125 },\n\t{ 8127, 8129 },\n\t{ 8141, 8143 },\n\t{ 8157, 8159 },\n\t{ 8173, 8175 },\n\t{ 8189, 8190 },\n\t{ 12443, 12444 },\n\t{ 42752, 42774 },\n\t{ 42784, 42785 },\n\t{ 42889, 42890 },\n\t{ 43867, 43867 },\n\t{ 43882, 43883 },\n\t{ 64434, 64450 },\n\t{ 65342, 65342 },\n\t{ 65344, 65344 },\n\t{ 65507, 65507 },\n};\nstatic const URange32 Sk_range32[] = {\n\t{ 127995, 127999 },\n};\nstatic const URange16 Sm_range16[] = {\n\t{ 43, 43 },\n\t{ 60, 62 },\n\t{ 124, 124 },\n\t{ 126, 126 },\n\t{ 172, 172 },\n\t{ 177, 177 },\n\t{ 215, 215 },\n\t{ 247, 247 },\n\t{ 1014, 1014 },\n\t{ 1542, 1544 },\n\t{ 8260, 8260 },\n\t{ 8274, 8274 },\n\t{ 8314, 8316 },\n\t{ 8330, 8332 },\n\t{ 8472, 8472 },\n\t{ 8512, 8516 },\n\t{ 8523, 8523 },\n\t{ 8592, 8596 },\n\t{ 8602, 8603 },\n\t{ 8608, 8608 },\n\t{ 8611, 8611 },\n\t{ 8614, 8614 },\n\t{ 8622, 8622 },\n\t{ 8654, 8655 },\n\t{ 8658, 8658 },\n\t{ 8660, 8660 },\n\t{ 8692, 8959 },\n\t{ 8992, 8993 },\n\t{ 9084, 9084 },\n\t{ 9115, 9139 },\n\t{ 9180, 9185 },\n\t{ 9655, 9655 },\n\t{ 9665, 9665 },\n\t{ 9720, 9727 },\n\t{ 9839, 9839 },\n\t{ 10176, 10180 },\n\t{ 10183, 10213 },\n\t{ 10224, 10239 },\n\t{ 10496, 10626 },\n\t{ 10649, 10711 },\n\t{ 10716, 10747 },\n\t{ 10750, 11007 },\n\t{ 11056, 11076 },\n\t{ 11079, 11084 },\n\t{ 64297, 64297 },\n\t{ 65122, 65122 },\n\t{ 65124, 65126 },\n\t{ 65291, 65291 },\n\t{ 65308, 65310 },\n\t{ 65372, 65372 },\n\t{ 65374, 65374 },\n\t{ 65506, 65506 },\n\t{ 65513, 65516 },\n};\nstatic const URange32 Sm_range32[] = {\n\t{ 120513, 120513 },\n\t{ 120539, 120539 },\n\t{ 120571, 120571 },\n\t{ 120597, 120597 },\n\t{ 120629, 120629 },\n\t{ 120655, 120655 },\n\t{ 120687, 120687 },\n\t{ 120713, 120713 },\n\t{ 120745, 120745 },\n\t{ 120771, 120771 },\n\t{ 126704, 126705 },\n};\nstatic const URange16 So_range16[] = {\n\t{ 166, 166 },\n\t{ 169, 169 },\n\t{ 174, 174 },\n\t{ 176, 176 },\n\t{ 1154, 1154 },\n\t{ 1421, 1422 },\n\t{ 1550, 1551 },\n\t{ 1758, 1758 },\n\t{ 1769, 1769 },\n\t{ 1789, 1790 },\n\t{ 2038, 2038 },\n\t{ 2554, 2554 },\n\t{ 2928, 2928 },\n\t{ 3059, 3064 },\n\t{ 3066, 3066 },\n\t{ 3199, 3199 },\n\t{ 3407, 3407 },\n\t{ 3449, 3449 },\n\t{ 3841, 3843 },\n\t{ 3859, 3859 },\n\t{ 3861, 3863 },\n\t{ 3866, 3871 },\n\t{ 3892, 3892 },\n\t{ 3894, 3894 },\n\t{ 3896, 3896 },\n\t{ 4030, 4037 },\n\t{ 4039, 4044 },\n\t{ 4046, 4047 },\n\t{ 4053, 4056 },\n\t{ 4254, 4255 },\n\t{ 5008, 5017 },\n\t{ 5741, 5741 },\n\t{ 6464, 6464 },\n\t{ 6622, 6655 },\n\t{ 7009, 7018 },\n\t{ 7028, 7036 },\n\t{ 8448, 8449 },\n\t{ 8451, 8454 },\n\t{ 8456, 8457 },\n\t{ 8468, 8468 },\n\t{ 8470, 8471 },\n\t{ 8478, 8483 },\n\t{ 8485, 8485 },\n\t{ 8487, 8487 },\n\t{ 8489, 8489 },\n\t{ 8494, 8494 },\n\t{ 8506, 8507 },\n\t{ 8522, 8522 },\n\t{ 8524, 8525 },\n\t{ 8527, 8527 },\n\t{ 8586, 8587 },\n\t{ 8597, 8601 },\n\t{ 8604, 8607 },\n\t{ 8609, 8610 },\n\t{ 8612, 8613 },\n\t{ 8615, 8621 },\n\t{ 8623, 8653 },\n\t{ 8656, 8657 },\n\t{ 8659, 8659 },\n\t{ 8661, 8691 },\n\t{ 8960, 8967 },\n\t{ 8972, 8991 },\n\t{ 8994, 9000 },\n\t{ 9003, 9083 },\n\t{ 9085, 9114 },\n\t{ 9140, 9179 },\n\t{ 9186, 9254 },\n\t{ 9280, 9290 },\n\t{ 9372, 9449 },\n\t{ 9472, 9654 },\n\t{ 9656, 9664 },\n\t{ 9666, 9719 },\n\t{ 9728, 9838 },\n\t{ 9840, 10087 },\n\t{ 10132, 10175 },\n\t{ 10240, 10495 },\n\t{ 11008, 11055 },\n\t{ 11077, 11078 },\n\t{ 11085, 11123 },\n\t{ 11126, 11157 },\n\t{ 11159, 11263 },\n\t{ 11493, 11498 },\n\t{ 11856, 11857 },\n\t{ 11904, 11929 },\n\t{ 11931, 12019 },\n\t{ 12032, 12245 },\n\t{ 12272, 12287 },\n\t{ 12292, 12292 },\n\t{ 12306, 12307 },\n\t{ 12320, 12320 },\n\t{ 12342, 12343 },\n\t{ 12350, 12351 },\n\t{ 12688, 12689 },\n\t{ 12694, 12703 },\n\t{ 12736, 12771 },\n\t{ 12783, 12783 },\n\t{ 12800, 12830 },\n\t{ 12842, 12871 },\n\t{ 12880, 12880 },\n\t{ 12896, 12927 },\n\t{ 12938, 12976 },\n\t{ 12992, 13311 },\n\t{ 19904, 19967 },\n\t{ 42128, 42182 },\n\t{ 43048, 43051 },\n\t{ 43062, 43063 },\n\t{ 43065, 43065 },\n\t{ 43639, 43641 },\n\t{ 64832, 64847 },\n\t{ 64975, 64975 },\n\t{ 65021, 65023 },\n\t{ 65508, 65508 },\n\t{ 65512, 65512 },\n\t{ 65517, 65518 },\n\t{ 65532, 65533 },\n};\nstatic const URange32 So_range32[] = {\n\t{ 65847, 65855 },\n\t{ 65913, 65929 },\n\t{ 65932, 65934 },\n\t{ 65936, 65948 },\n\t{ 65952, 65952 },\n\t{ 66000, 66044 },\n\t{ 67703, 67704 },\n\t{ 68296, 68296 },\n\t{ 71487, 71487 },\n\t{ 73685, 73692 },\n\t{ 73697, 73713 },\n\t{ 92988, 92991 },\n\t{ 92997, 92997 },\n\t{ 113820, 113820 },\n\t{ 118608, 118723 },\n\t{ 118784, 119029 },\n\t{ 119040, 119078 },\n\t{ 119081, 119140 },\n\t{ 119146, 119148 },\n\t{ 119171, 119172 },\n\t{ 119180, 119209 },\n\t{ 119214, 119274 },\n\t{ 119296, 119361 },\n\t{ 119365, 119365 },\n\t{ 119552, 119638 },\n\t{ 120832, 121343 },\n\t{ 121399, 121402 },\n\t{ 121453, 121460 },\n\t{ 121462, 121475 },\n\t{ 121477, 121478 },\n\t{ 123215, 123215 },\n\t{ 126124, 126124 },\n\t{ 126254, 126254 },\n\t{ 126976, 127019 },\n\t{ 127024, 127123 },\n\t{ 127136, 127150 },\n\t{ 127153, 127167 },\n\t{ 127169, 127183 },\n\t{ 127185, 127221 },\n\t{ 127245, 127405 },\n\t{ 127462, 127490 },\n\t{ 127504, 127547 },\n\t{ 127552, 127560 },\n\t{ 127568, 127569 },\n\t{ 127584, 127589 },\n\t{ 127744, 127994 },\n\t{ 128000, 128727 },\n\t{ 128732, 128748 },\n\t{ 128752, 128764 },\n\t{ 128768, 128886 },\n\t{ 128891, 128985 },\n\t{ 128992, 129003 },\n\t{ 129008, 129008 },\n\t{ 129024, 129035 },\n\t{ 129040, 129095 },\n\t{ 129104, 129113 },\n\t{ 129120, 129159 },\n\t{ 129168, 129197 },\n\t{ 129200, 129201 },\n\t{ 129280, 129619 },\n\t{ 129632, 129645 },\n\t{ 129648, 129660 },\n\t{ 129664, 129672 },\n\t{ 129680, 129725 },\n\t{ 129727, 129733 },\n\t{ 129742, 129755 },\n\t{ 129760, 129768 },\n\t{ 129776, 129784 },\n\t{ 129792, 129938 },\n\t{ 129940, 129994 },\n};\nstatic const URange16 Z_range16[] = {\n\t{ 32, 32 },\n\t{ 160, 160 },\n\t{ 5760, 5760 },\n\t{ 8192, 8202 },\n\t{ 8232, 8233 },\n\t{ 8239, 8239 },\n\t{ 8287, 8287 },\n\t{ 12288, 12288 },\n};\nstatic const URange16 Zl_range16[] = {\n\t{ 8232, 8232 },\n};\nstatic const URange16 Zp_range16[] = {\n\t{ 8233, 8233 },\n};\nstatic const URange16 Zs_range16[] = {\n\t{ 32, 32 },\n\t{ 160, 160 },\n\t{ 5760, 5760 },\n\t{ 8192, 8202 },\n\t{ 8239, 8239 },\n\t{ 8287, 8287 },\n\t{ 12288, 12288 },\n};\nstatic const URange32 Adlam_range32[] = {\n\t{ 125184, 125259 },\n\t{ 125264, 125273 },\n\t{ 125278, 125279 },\n};\nstatic const URange32 Ahom_range32[] = {\n\t{ 71424, 71450 },\n\t{ 71453, 71467 },\n\t{ 71472, 71494 },\n};\nstatic const URange32 Anatolian_Hieroglyphs_range32[] = {\n\t{ 82944, 83526 },\n};\nstatic const URange16 Arabic_range16[] = {\n\t{ 1536, 1540 },\n\t{ 1542, 1547 },\n\t{ 1549, 1562 },\n\t{ 1564, 1566 },\n\t{ 1568, 1599 },\n\t{ 1601, 1610 },\n\t{ 1622, 1647 },\n\t{ 1649, 1756 },\n\t{ 1758, 1791 },\n\t{ 1872, 1919 },\n\t{ 2160, 2190 },\n\t{ 2192, 2193 },\n\t{ 2200, 2273 },\n\t{ 2275, 2303 },\n\t{ 64336, 64450 },\n\t{ 64467, 64829 },\n\t{ 64832, 64911 },\n\t{ 64914, 64967 },\n\t{ 64975, 64975 },\n\t{ 65008, 65023 },\n\t{ 65136, 65140 },\n\t{ 65142, 65276 },\n};\nstatic const URange32 Arabic_range32[] = {\n\t{ 69216, 69246 },\n\t{ 69373, 69375 },\n\t{ 126464, 126467 },\n\t{ 126469, 126495 },\n\t{ 126497, 126498 },\n\t{ 126500, 126500 },\n\t{ 126503, 126503 },\n\t{ 126505, 126514 },\n\t{ 126516, 126519 },\n\t{ 126521, 126521 },\n\t{ 126523, 126523 },\n\t{ 126530, 126530 },\n\t{ 126535, 126535 },\n\t{ 126537, 126537 },\n\t{ 126539, 126539 },\n\t{ 126541, 126543 },\n\t{ 126545, 126546 },\n\t{ 126548, 126548 },\n\t{ 126551, 126551 },\n\t{ 126553, 126553 },\n\t{ 126555, 126555 },\n\t{ 126557, 126557 },\n\t{ 126559, 126559 },\n\t{ 126561, 126562 },\n\t{ 126564, 126564 },\n\t{ 126567, 126570 },\n\t{ 126572, 126578 },\n\t{ 126580, 126583 },\n\t{ 126585, 126588 },\n\t{ 126590, 126590 },\n\t{ 126592, 126601 },\n\t{ 126603, 126619 },\n\t{ 126625, 126627 },\n\t{ 126629, 126633 },\n\t{ 126635, 126651 },\n\t{ 126704, 126705 },\n};\nstatic const URange16 Armenian_range16[] = {\n\t{ 1329, 1366 },\n\t{ 1369, 1418 },\n\t{ 1421, 1423 },\n\t{ 64275, 64279 },\n};\nstatic const URange32 Avestan_range32[] = {\n\t{ 68352, 68405 },\n\t{ 68409, 68415 },\n};\nstatic const URange16 Balinese_range16[] = {\n\t{ 6912, 6988 },\n\t{ 6992, 7038 },\n};\nstatic const URange16 Bamum_range16[] = {\n\t{ 42656, 42743 },\n};\nstatic const URange32 Bamum_range32[] = {\n\t{ 92160, 92728 },\n};\nstatic const URange32 Bassa_Vah_range32[] = {\n\t{ 92880, 92909 },\n\t{ 92912, 92917 },\n};\nstatic const URange16 Batak_range16[] = {\n\t{ 7104, 7155 },\n\t{ 7164, 7167 },\n};\nstatic const URange16 Bengali_range16[] = {\n\t{ 2432, 2435 },\n\t{ 2437, 2444 },\n\t{ 2447, 2448 },\n\t{ 2451, 2472 },\n\t{ 2474, 2480 },\n\t{ 2482, 2482 },\n\t{ 2486, 2489 },\n\t{ 2492, 2500 },\n\t{ 2503, 2504 },\n\t{ 2507, 2510 },\n\t{ 2519, 2519 },\n\t{ 2524, 2525 },\n\t{ 2527, 2531 },\n\t{ 2534, 2558 },\n};\nstatic const URange32 Bhaiksuki_range32[] = {\n\t{ 72704, 72712 },\n\t{ 72714, 72758 },\n\t{ 72760, 72773 },\n\t{ 72784, 72812 },\n};\nstatic const URange16 Bopomofo_range16[] = {\n\t{ 746, 747 },\n\t{ 12549, 12591 },\n\t{ 12704, 12735 },\n};\nstatic const URange32 Brahmi_range32[] = {\n\t{ 69632, 69709 },\n\t{ 69714, 69749 },\n\t{ 69759, 69759 },\n};\nstatic const URange16 Braille_range16[] = {\n\t{ 10240, 10495 },\n};\nstatic const URange16 Buginese_range16[] = {\n\t{ 6656, 6683 },\n\t{ 6686, 6687 },\n};\nstatic const URange16 Buhid_range16[] = {\n\t{ 5952, 5971 },\n};\nstatic const URange16 Canadian_Aboriginal_range16[] = {\n\t{ 5120, 5759 },\n\t{ 6320, 6389 },\n};\nstatic const URange32 Canadian_Aboriginal_range32[] = {\n\t{ 72368, 72383 },\n};\nstatic const URange32 Carian_range32[] = {\n\t{ 66208, 66256 },\n};\nstatic const URange32 Caucasian_Albanian_range32[] = {\n\t{ 66864, 66915 },\n\t{ 66927, 66927 },\n};\nstatic const URange32 Chakma_range32[] = {\n\t{ 69888, 69940 },\n\t{ 69942, 69959 },\n};\nstatic const URange16 Cham_range16[] = {\n\t{ 43520, 43574 },\n\t{ 43584, 43597 },\n\t{ 43600, 43609 },\n\t{ 43612, 43615 },\n};\nstatic const URange16 Cherokee_range16[] = {\n\t{ 5024, 5109 },\n\t{ 5112, 5117 },\n\t{ 43888, 43967 },\n};\nstatic const URange32 Chorasmian_range32[] = {\n\t{ 69552, 69579 },\n};\nstatic const URange16 Common_range16[] = {\n\t{ 0, 64 },\n\t{ 91, 96 },\n\t{ 123, 169 },\n\t{ 171, 185 },\n\t{ 187, 191 },\n\t{ 215, 215 },\n\t{ 247, 247 },\n\t{ 697, 735 },\n\t{ 741, 745 },\n\t{ 748, 767 },\n\t{ 884, 884 },\n\t{ 894, 894 },\n\t{ 901, 901 },\n\t{ 903, 903 },\n\t{ 1541, 1541 },\n\t{ 1548, 1548 },\n\t{ 1563, 1563 },\n\t{ 1567, 1567 },\n\t{ 1600, 1600 },\n\t{ 1757, 1757 },\n\t{ 2274, 2274 },\n\t{ 2404, 2405 },\n\t{ 3647, 3647 },\n\t{ 4053, 4056 },\n\t{ 4347, 4347 },\n\t{ 5867, 5869 },\n\t{ 5941, 5942 },\n\t{ 6146, 6147 },\n\t{ 6149, 6149 },\n\t{ 7379, 7379 },\n\t{ 7393, 7393 },\n\t{ 7401, 7404 },\n\t{ 7406, 7411 },\n\t{ 7413, 7415 },\n\t{ 7418, 7418 },\n\t{ 8192, 8203 },\n\t{ 8206, 8292 },\n\t{ 8294, 8304 },\n\t{ 8308, 8318 },\n\t{ 8320, 8334 },\n\t{ 8352, 8384 },\n\t{ 8448, 8485 },\n\t{ 8487, 8489 },\n\t{ 8492, 8497 },\n\t{ 8499, 8525 },\n\t{ 8527, 8543 },\n\t{ 8585, 8587 },\n\t{ 8592, 9254 },\n\t{ 9280, 9290 },\n\t{ 9312, 10239 },\n\t{ 10496, 11123 },\n\t{ 11126, 11157 },\n\t{ 11159, 11263 },\n\t{ 11776, 11869 },\n\t{ 12272, 12292 },\n\t{ 12294, 12294 },\n\t{ 12296, 12320 },\n\t{ 12336, 12343 },\n\t{ 12348, 12351 },\n\t{ 12443, 12444 },\n\t{ 12448, 12448 },\n\t{ 12539, 12540 },\n\t{ 12688, 12703 },\n\t{ 12736, 12771 },\n\t{ 12783, 12783 },\n\t{ 12832, 12895 },\n\t{ 12927, 13007 },\n\t{ 13055, 13055 },\n\t{ 13144, 13311 },\n\t{ 19904, 19967 },\n\t{ 42752, 42785 },\n\t{ 42888, 42890 },\n\t{ 43056, 43065 },\n\t{ 43310, 43310 },\n\t{ 43471, 43471 },\n\t{ 43867, 43867 },\n\t{ 43882, 43883 },\n\t{ 64830, 64831 },\n\t{ 65040, 65049 },\n\t{ 65072, 65106 },\n\t{ 65108, 65126 },\n\t{ 65128, 65131 },\n\t{ 65279, 65279 },\n\t{ 65281, 65312 },\n\t{ 65339, 65344 },\n\t{ 65371, 65381 },\n\t{ 65392, 65392 },\n\t{ 65438, 65439 },\n\t{ 65504, 65510 },\n\t{ 65512, 65518 },\n\t{ 65529, 65533 },\n};\nstatic const URange32 Common_range32[] = {\n\t{ 65792, 65794 },\n\t{ 65799, 65843 },\n\t{ 65847, 65855 },\n\t{ 65936, 65948 },\n\t{ 66000, 66044 },\n\t{ 66273, 66299 },\n\t{ 113824, 113827 },\n\t{ 118608, 118723 },\n\t{ 118784, 119029 },\n\t{ 119040, 119078 },\n\t{ 119081, 119142 },\n\t{ 119146, 119162 },\n\t{ 119171, 119172 },\n\t{ 119180, 119209 },\n\t{ 119214, 119274 },\n\t{ 119488, 119507 },\n\t{ 119520, 119539 },\n\t{ 119552, 119638 },\n\t{ 119648, 119672 },\n\t{ 119808, 119892 },\n\t{ 119894, 119964 },\n\t{ 119966, 119967 },\n\t{ 119970, 119970 },\n\t{ 119973, 119974 },\n\t{ 119977, 119980 },\n\t{ 119982, 119993 },\n\t{ 119995, 119995 },\n\t{ 119997, 120003 },\n\t{ 120005, 120069 },\n\t{ 120071, 120074 },\n\t{ 120077, 120084 },\n\t{ 120086, 120092 },\n\t{ 120094, 120121 },\n\t{ 120123, 120126 },\n\t{ 120128, 120132 },\n\t{ 120134, 120134 },\n\t{ 120138, 120144 },\n\t{ 120146, 120485 },\n\t{ 120488, 120779 },\n\t{ 120782, 120831 },\n\t{ 126065, 126132 },\n\t{ 126209, 126269 },\n\t{ 126976, 127019 },\n\t{ 127024, 127123 },\n\t{ 127136, 127150 },\n\t{ 127153, 127167 },\n\t{ 127169, 127183 },\n\t{ 127185, 127221 },\n\t{ 127232, 127405 },\n\t{ 127462, 127487 },\n\t{ 127489, 127490 },\n\t{ 127504, 127547 },\n\t{ 127552, 127560 },\n\t{ 127568, 127569 },\n\t{ 127584, 127589 },\n\t{ 127744, 128727 },\n\t{ 128732, 128748 },\n\t{ 128752, 128764 },\n\t{ 128768, 128886 },\n\t{ 128891, 128985 },\n\t{ 128992, 129003 },\n\t{ 129008, 129008 },\n\t{ 129024, 129035 },\n\t{ 129040, 129095 },\n\t{ 129104, 129113 },\n\t{ 129120, 129159 },\n\t{ 129168, 129197 },\n\t{ 129200, 129201 },\n\t{ 129280, 129619 },\n\t{ 129632, 129645 },\n\t{ 129648, 129660 },\n\t{ 129664, 129672 },\n\t{ 129680, 129725 },\n\t{ 129727, 129733 },\n\t{ 129742, 129755 },\n\t{ 129760, 129768 },\n\t{ 129776, 129784 },\n\t{ 129792, 129938 },\n\t{ 129940, 129994 },\n\t{ 130032, 130041 },\n\t{ 917505, 917505 },\n\t{ 917536, 917631 },\n};\nstatic const URange16 Coptic_range16[] = {\n\t{ 994, 1007 },\n\t{ 11392, 11507 },\n\t{ 11513, 11519 },\n};\nstatic const URange32 Cuneiform_range32[] = {\n\t{ 73728, 74649 },\n\t{ 74752, 74862 },\n\t{ 74864, 74868 },\n\t{ 74880, 75075 },\n};\nstatic const URange32 Cypriot_range32[] = {\n\t{ 67584, 67589 },\n\t{ 67592, 67592 },\n\t{ 67594, 67637 },\n\t{ 67639, 67640 },\n\t{ 67644, 67644 },\n\t{ 67647, 67647 },\n};\nstatic const URange32 Cypro_Minoan_range32[] = {\n\t{ 77712, 77810 },\n};\nstatic const URange16 Cyrillic_range16[] = {\n\t{ 1024, 1156 },\n\t{ 1159, 1327 },\n\t{ 7296, 7304 },\n\t{ 7467, 7467 },\n\t{ 7544, 7544 },\n\t{ 11744, 11775 },\n\t{ 42560, 42655 },\n\t{ 65070, 65071 },\n};\nstatic const URange32 Cyrillic_range32[] = {\n\t{ 122928, 122989 },\n\t{ 123023, 123023 },\n};\nstatic const URange32 Deseret_range32[] = {\n\t{ 66560, 66639 },\n};\nstatic const URange16 Devanagari_range16[] = {\n\t{ 2304, 2384 },\n\t{ 2389, 2403 },\n\t{ 2406, 2431 },\n\t{ 43232, 43263 },\n};\nstatic const URange32 Devanagari_range32[] = {\n\t{ 72448, 72457 },\n};\nstatic const URange32 Dives_Akuru_range32[] = {\n\t{ 71936, 71942 },\n\t{ 71945, 71945 },\n\t{ 71948, 71955 },\n\t{ 71957, 71958 },\n\t{ 71960, 71989 },\n\t{ 71991, 71992 },\n\t{ 71995, 72006 },\n\t{ 72016, 72025 },\n};\nstatic const URange32 Dogra_range32[] = {\n\t{ 71680, 71739 },\n};\nstatic const URange32 Duployan_range32[] = {\n\t{ 113664, 113770 },\n\t{ 113776, 113788 },\n\t{ 113792, 113800 },\n\t{ 113808, 113817 },\n\t{ 113820, 113823 },\n};\nstatic const URange32 Egyptian_Hieroglyphs_range32[] = {\n\t{ 77824, 78933 },\n};\nstatic const URange32 Elbasan_range32[] = {\n\t{ 66816, 66855 },\n};\nstatic const URange32 Elymaic_range32[] = {\n\t{ 69600, 69622 },\n};\nstatic const URange16 Ethiopic_range16[] = {\n\t{ 4608, 4680 },\n\t{ 4682, 4685 },\n\t{ 4688, 4694 },\n\t{ 4696, 4696 },\n\t{ 4698, 4701 },\n\t{ 4704, 4744 },\n\t{ 4746, 4749 },\n\t{ 4752, 4784 },\n\t{ 4786, 4789 },\n\t{ 4792, 4798 },\n\t{ 4800, 4800 },\n\t{ 4802, 4805 },\n\t{ 4808, 4822 },\n\t{ 4824, 4880 },\n\t{ 4882, 4885 },\n\t{ 4888, 4954 },\n\t{ 4957, 4988 },\n\t{ 4992, 5017 },\n\t{ 11648, 11670 },\n\t{ 11680, 11686 },\n\t{ 11688, 11694 },\n\t{ 11696, 11702 },\n\t{ 11704, 11710 },\n\t{ 11712, 11718 },\n\t{ 11720, 11726 },\n\t{ 11728, 11734 },\n\t{ 11736, 11742 },\n\t{ 43777, 43782 },\n\t{ 43785, 43790 },\n\t{ 43793, 43798 },\n\t{ 43808, 43814 },\n\t{ 43816, 43822 },\n};\nstatic const URange32 Ethiopic_range32[] = {\n\t{ 124896, 124902 },\n\t{ 124904, 124907 },\n\t{ 124909, 124910 },\n\t{ 124912, 124926 },\n};\nstatic const URange16 Georgian_range16[] = {\n\t{ 4256, 4293 },\n\t{ 4295, 4295 },\n\t{ 4301, 4301 },\n\t{ 4304, 4346 },\n\t{ 4348, 4351 },\n\t{ 7312, 7354 },\n\t{ 7357, 7359 },\n\t{ 11520, 11557 },\n\t{ 11559, 11559 },\n\t{ 11565, 11565 },\n};\nstatic const URange16 Glagolitic_range16[] = {\n\t{ 11264, 11359 },\n};\nstatic const URange32 Glagolitic_range32[] = {\n\t{ 122880, 122886 },\n\t{ 122888, 122904 },\n\t{ 122907, 122913 },\n\t{ 122915, 122916 },\n\t{ 122918, 122922 },\n};\nstatic const URange32 Gothic_range32[] = {\n\t{ 66352, 66378 },\n};\nstatic const URange32 Grantha_range32[] = {\n\t{ 70400, 70403 },\n\t{ 70405, 70412 },\n\t{ 70415, 70416 },\n\t{ 70419, 70440 },\n\t{ 70442, 70448 },\n\t{ 70450, 70451 },\n\t{ 70453, 70457 },\n\t{ 70460, 70468 },\n\t{ 70471, 70472 },\n\t{ 70475, 70477 },\n\t{ 70480, 70480 },\n\t{ 70487, 70487 },\n\t{ 70493, 70499 },\n\t{ 70502, 70508 },\n\t{ 70512, 70516 },\n};\nstatic const URange16 Greek_range16[] = {\n\t{ 880, 883 },\n\t{ 885, 887 },\n\t{ 890, 893 },\n\t{ 895, 895 },\n\t{ 900, 900 },\n\t{ 902, 902 },\n\t{ 904, 906 },\n\t{ 908, 908 },\n\t{ 910, 929 },\n\t{ 931, 993 },\n\t{ 1008, 1023 },\n\t{ 7462, 7466 },\n\t{ 7517, 7521 },\n\t{ 7526, 7530 },\n\t{ 7615, 7615 },\n\t{ 7936, 7957 },\n\t{ 7960, 7965 },\n\t{ 7968, 8005 },\n\t{ 8008, 8013 },\n\t{ 8016, 8023 },\n\t{ 8025, 8025 },\n\t{ 8027, 8027 },\n\t{ 8029, 8029 },\n\t{ 8031, 8061 },\n\t{ 8064, 8116 },\n\t{ 8118, 8132 },\n\t{ 8134, 8147 },\n\t{ 8150, 8155 },\n\t{ 8157, 8175 },\n\t{ 8178, 8180 },\n\t{ 8182, 8190 },\n\t{ 8486, 8486 },\n\t{ 43877, 43877 },\n};\nstatic const URange32 Greek_range32[] = {\n\t{ 65856, 65934 },\n\t{ 65952, 65952 },\n\t{ 119296, 119365 },\n};\nstatic const URange16 Gujarati_range16[] = {\n\t{ 2689, 2691 },\n\t{ 2693, 2701 },\n\t{ 2703, 2705 },\n\t{ 2707, 2728 },\n\t{ 2730, 2736 },\n\t{ 2738, 2739 },\n\t{ 2741, 2745 },\n\t{ 2748, 2757 },\n\t{ 2759, 2761 },\n\t{ 2763, 2765 },\n\t{ 2768, 2768 },\n\t{ 2784, 2787 },\n\t{ 2790, 2801 },\n\t{ 2809, 2815 },\n};\nstatic const URange32 Gunjala_Gondi_range32[] = {\n\t{ 73056, 73061 },\n\t{ 73063, 73064 },\n\t{ 73066, 73102 },\n\t{ 73104, 73105 },\n\t{ 73107, 73112 },\n\t{ 73120, 73129 },\n};\nstatic const URange16 Gurmukhi_range16[] = {\n\t{ 2561, 2563 },\n\t{ 2565, 2570 },\n\t{ 2575, 2576 },\n\t{ 2579, 2600 },\n\t{ 2602, 2608 },\n\t{ 2610, 2611 },\n\t{ 2613, 2614 },\n\t{ 2616, 2617 },\n\t{ 2620, 2620 },\n\t{ 2622, 2626 },\n\t{ 2631, 2632 },\n\t{ 2635, 2637 },\n\t{ 2641, 2641 },\n\t{ 2649, 2652 },\n\t{ 2654, 2654 },\n\t{ 2662, 2678 },\n};\nstatic const URange16 Han_range16[] = {\n\t{ 11904, 11929 },\n\t{ 11931, 12019 },\n\t{ 12032, 12245 },\n\t{ 12293, 12293 },\n\t{ 12295, 12295 },\n\t{ 12321, 12329 },\n\t{ 12344, 12347 },\n\t{ 13312, 19903 },\n\t{ 19968, 40959 },\n\t{ 63744, 64109 },\n\t{ 64112, 64217 },\n};\nstatic const URange32 Han_range32[] = {\n\t{ 94178, 94179 },\n\t{ 94192, 94193 },\n\t{ 131072, 173791 },\n\t{ 173824, 177977 },\n\t{ 177984, 178205 },\n\t{ 178208, 183969 },\n\t{ 183984, 191456 },\n\t{ 191472, 192093 },\n\t{ 194560, 195101 },\n\t{ 196608, 201546 },\n\t{ 201552, 205743 },\n};\nstatic const URange16 Hangul_range16[] = {\n\t{ 4352, 4607 },\n\t{ 12334, 12335 },\n\t{ 12593, 12686 },\n\t{ 12800, 12830 },\n\t{ 12896, 12926 },\n\t{ 43360, 43388 },\n\t{ 44032, 55203 },\n\t{ 55216, 55238 },\n\t{ 55243, 55291 },\n\t{ 65440, 65470 },\n\t{ 65474, 65479 },\n\t{ 65482, 65487 },\n\t{ 65490, 65495 },\n\t{ 65498, 65500 },\n};\nstatic const URange32 Hanifi_Rohingya_range32[] = {\n\t{ 68864, 68903 },\n\t{ 68912, 68921 },\n};\nstatic const URange16 Hanunoo_range16[] = {\n\t{ 5920, 5940 },\n};\nstatic const URange32 Hatran_range32[] = {\n\t{ 67808, 67826 },\n\t{ 67828, 67829 },\n\t{ 67835, 67839 },\n};\nstatic const URange16 Hebrew_range16[] = {\n\t{ 1425, 1479 },\n\t{ 1488, 1514 },\n\t{ 1519, 1524 },\n\t{ 64285, 64310 },\n\t{ 64312, 64316 },\n\t{ 64318, 64318 },\n\t{ 64320, 64321 },\n\t{ 64323, 64324 },\n\t{ 64326, 64335 },\n};\nstatic const URange16 Hiragana_range16[] = {\n\t{ 12353, 12438 },\n\t{ 12445, 12447 },\n};\nstatic const URange32 Hiragana_range32[] = {\n\t{ 110593, 110879 },\n\t{ 110898, 110898 },\n\t{ 110928, 110930 },\n\t{ 127488, 127488 },\n};\nstatic const URange32 Imperial_Aramaic_range32[] = {\n\t{ 67648, 67669 },\n\t{ 67671, 67679 },\n};\nstatic const URange16 Inherited_range16[] = {\n\t{ 768, 879 },\n\t{ 1157, 1158 },\n\t{ 1611, 1621 },\n\t{ 1648, 1648 },\n\t{ 2385, 2388 },\n\t{ 6832, 6862 },\n\t{ 7376, 7378 },\n\t{ 7380, 7392 },\n\t{ 7394, 7400 },\n\t{ 7405, 7405 },\n\t{ 7412, 7412 },\n\t{ 7416, 7417 },\n\t{ 7616, 7679 },\n\t{ 8204, 8205 },\n\t{ 8400, 8432 },\n\t{ 12330, 12333 },\n\t{ 12441, 12442 },\n\t{ 65024, 65039 },\n\t{ 65056, 65069 },\n};\nstatic const URange32 Inherited_range32[] = {\n\t{ 66045, 66045 },\n\t{ 66272, 66272 },\n\t{ 70459, 70459 },\n\t{ 118528, 118573 },\n\t{ 118576, 118598 },\n\t{ 119143, 119145 },\n\t{ 119163, 119170 },\n\t{ 119173, 119179 },\n\t{ 119210, 119213 },\n\t{ 917760, 917999 },\n};\nstatic const URange32 Inscriptional_Pahlavi_range32[] = {\n\t{ 68448, 68466 },\n\t{ 68472, 68479 },\n};\nstatic const URange32 Inscriptional_Parthian_range32[] = {\n\t{ 68416, 68437 },\n\t{ 68440, 68447 },\n};\nstatic const URange16 Javanese_range16[] = {\n\t{ 43392, 43469 },\n\t{ 43472, 43481 },\n\t{ 43486, 43487 },\n};\nstatic const URange32 Kaithi_range32[] = {\n\t{ 69760, 69826 },\n\t{ 69837, 69837 },\n};\nstatic const URange16 Kannada_range16[] = {\n\t{ 3200, 3212 },\n\t{ 3214, 3216 },\n\t{ 3218, 3240 },\n\t{ 3242, 3251 },\n\t{ 3253, 3257 },\n\t{ 3260, 3268 },\n\t{ 3270, 3272 },\n\t{ 3274, 3277 },\n\t{ 3285, 3286 },\n\t{ 3293, 3294 },\n\t{ 3296, 3299 },\n\t{ 3302, 3311 },\n\t{ 3313, 3315 },\n};\nstatic const URange16 Katakana_range16[] = {\n\t{ 12449, 12538 },\n\t{ 12541, 12543 },\n\t{ 12784, 12799 },\n\t{ 13008, 13054 },\n\t{ 13056, 13143 },\n\t{ 65382, 65391 },\n\t{ 65393, 65437 },\n};\nstatic const URange32 Katakana_range32[] = {\n\t{ 110576, 110579 },\n\t{ 110581, 110587 },\n\t{ 110589, 110590 },\n\t{ 110592, 110592 },\n\t{ 110880, 110882 },\n\t{ 110933, 110933 },\n\t{ 110948, 110951 },\n};\nstatic const URange32 Kawi_range32[] = {\n\t{ 73472, 73488 },\n\t{ 73490, 73530 },\n\t{ 73534, 73561 },\n};\nstatic const URange16 Kayah_Li_range16[] = {\n\t{ 43264, 43309 },\n\t{ 43311, 43311 },\n};\nstatic const URange32 Kharoshthi_range32[] = {\n\t{ 68096, 68099 },\n\t{ 68101, 68102 },\n\t{ 68108, 68115 },\n\t{ 68117, 68119 },\n\t{ 68121, 68149 },\n\t{ 68152, 68154 },\n\t{ 68159, 68168 },\n\t{ 68176, 68184 },\n};\nstatic const URange32 Khitan_Small_Script_range32[] = {\n\t{ 94180, 94180 },\n\t{ 101120, 101589 },\n};\nstatic const URange16 Khmer_range16[] = {\n\t{ 6016, 6109 },\n\t{ 6112, 6121 },\n\t{ 6128, 6137 },\n\t{ 6624, 6655 },\n};\nstatic const URange32 Khojki_range32[] = {\n\t{ 70144, 70161 },\n\t{ 70163, 70209 },\n};\nstatic const URange32 Khudawadi_range32[] = {\n\t{ 70320, 70378 },\n\t{ 70384, 70393 },\n};\nstatic const URange16 Lao_range16[] = {\n\t{ 3713, 3714 },\n\t{ 3716, 3716 },\n\t{ 3718, 3722 },\n\t{ 3724, 3747 },\n\t{ 3749, 3749 },\n\t{ 3751, 3773 },\n\t{ 3776, 3780 },\n\t{ 3782, 3782 },\n\t{ 3784, 3790 },\n\t{ 3792, 3801 },\n\t{ 3804, 3807 },\n};\nstatic const URange16 Latin_range16[] = {\n\t{ 65, 90 },\n\t{ 97, 122 },\n\t{ 170, 170 },\n\t{ 186, 186 },\n\t{ 192, 214 },\n\t{ 216, 246 },\n\t{ 248, 696 },\n\t{ 736, 740 },\n\t{ 7424, 7461 },\n\t{ 7468, 7516 },\n\t{ 7522, 7525 },\n\t{ 7531, 7543 },\n\t{ 7545, 7614 },\n\t{ 7680, 7935 },\n\t{ 8305, 8305 },\n\t{ 8319, 8319 },\n\t{ 8336, 8348 },\n\t{ 8490, 8491 },\n\t{ 8498, 8498 },\n\t{ 8526, 8526 },\n\t{ 8544, 8584 },\n\t{ 11360, 11391 },\n\t{ 42786, 42887 },\n\t{ 42891, 42954 },\n\t{ 42960, 42961 },\n\t{ 42963, 42963 },\n\t{ 42965, 42969 },\n\t{ 42994, 43007 },\n\t{ 43824, 43866 },\n\t{ 43868, 43876 },\n\t{ 43878, 43881 },\n\t{ 64256, 64262 },\n\t{ 65313, 65338 },\n\t{ 65345, 65370 },\n};\nstatic const URange32 Latin_range32[] = {\n\t{ 67456, 67461 },\n\t{ 67463, 67504 },\n\t{ 67506, 67514 },\n\t{ 122624, 122654 },\n\t{ 122661, 122666 },\n};\nstatic const URange16 Lepcha_range16[] = {\n\t{ 7168, 7223 },\n\t{ 7227, 7241 },\n\t{ 7245, 7247 },\n};\nstatic const URange16 Limbu_range16[] = {\n\t{ 6400, 6430 },\n\t{ 6432, 6443 },\n\t{ 6448, 6459 },\n\t{ 6464, 6464 },\n\t{ 6468, 6479 },\n};\nstatic const URange32 Linear_A_range32[] = {\n\t{ 67072, 67382 },\n\t{ 67392, 67413 },\n\t{ 67424, 67431 },\n};\nstatic const URange32 Linear_B_range32[] = {\n\t{ 65536, 65547 },\n\t{ 65549, 65574 },\n\t{ 65576, 65594 },\n\t{ 65596, 65597 },\n\t{ 65599, 65613 },\n\t{ 65616, 65629 },\n\t{ 65664, 65786 },\n};\nstatic const URange16 Lisu_range16[] = {\n\t{ 42192, 42239 },\n};\nstatic const URange32 Lisu_range32[] = {\n\t{ 73648, 73648 },\n};\nstatic const URange32 Lycian_range32[] = {\n\t{ 66176, 66204 },\n};\nstatic const URange32 Lydian_range32[] = {\n\t{ 67872, 67897 },\n\t{ 67903, 67903 },\n};\nstatic const URange32 Mahajani_range32[] = {\n\t{ 69968, 70006 },\n};\nstatic const URange32 Makasar_range32[] = {\n\t{ 73440, 73464 },\n};\nstatic const URange16 Malayalam_range16[] = {\n\t{ 3328, 3340 },\n\t{ 3342, 3344 },\n\t{ 3346, 3396 },\n\t{ 3398, 3400 },\n\t{ 3402, 3407 },\n\t{ 3412, 3427 },\n\t{ 3430, 3455 },\n};\nstatic const URange16 Mandaic_range16[] = {\n\t{ 2112, 2139 },\n\t{ 2142, 2142 },\n};\nstatic const URange32 Manichaean_range32[] = {\n\t{ 68288, 68326 },\n\t{ 68331, 68342 },\n};\nstatic const URange32 Marchen_range32[] = {\n\t{ 72816, 72847 },\n\t{ 72850, 72871 },\n\t{ 72873, 72886 },\n};\nstatic const URange32 Masaram_Gondi_range32[] = {\n\t{ 72960, 72966 },\n\t{ 72968, 72969 },\n\t{ 72971, 73014 },\n\t{ 73018, 73018 },\n\t{ 73020, 73021 },\n\t{ 73023, 73031 },\n\t{ 73040, 73049 },\n};\nstatic const URange32 Medefaidrin_range32[] = {\n\t{ 93760, 93850 },\n};\nstatic const URange16 Meetei_Mayek_range16[] = {\n\t{ 43744, 43766 },\n\t{ 43968, 44013 },\n\t{ 44016, 44025 },\n};\nstatic const URange32 Mende_Kikakui_range32[] = {\n\t{ 124928, 125124 },\n\t{ 125127, 125142 },\n};\nstatic const URange32 Meroitic_Cursive_range32[] = {\n\t{ 68000, 68023 },\n\t{ 68028, 68047 },\n\t{ 68050, 68095 },\n};\nstatic const URange32 Meroitic_Hieroglyphs_range32[] = {\n\t{ 67968, 67999 },\n};\nstatic const URange32 Miao_range32[] = {\n\t{ 93952, 94026 },\n\t{ 94031, 94087 },\n\t{ 94095, 94111 },\n};\nstatic const URange32 Modi_range32[] = {\n\t{ 71168, 71236 },\n\t{ 71248, 71257 },\n};\nstatic const URange16 Mongolian_range16[] = {\n\t{ 6144, 6145 },\n\t{ 6148, 6148 },\n\t{ 6150, 6169 },\n\t{ 6176, 6264 },\n\t{ 6272, 6314 },\n};\nstatic const URange32 Mongolian_range32[] = {\n\t{ 71264, 71276 },\n};\nstatic const URange32 Mro_range32[] = {\n\t{ 92736, 92766 },\n\t{ 92768, 92777 },\n\t{ 92782, 92783 },\n};\nstatic const URange32 Multani_range32[] = {\n\t{ 70272, 70278 },\n\t{ 70280, 70280 },\n\t{ 70282, 70285 },\n\t{ 70287, 70301 },\n\t{ 70303, 70313 },\n};\nstatic const URange16 Myanmar_range16[] = {\n\t{ 4096, 4255 },\n\t{ 43488, 43518 },\n\t{ 43616, 43647 },\n};\nstatic const URange32 Nabataean_range32[] = {\n\t{ 67712, 67742 },\n\t{ 67751, 67759 },\n};\nstatic const URange32 Nag_Mundari_range32[] = {\n\t{ 124112, 124153 },\n};\nstatic const URange32 Nandinagari_range32[] = {\n\t{ 72096, 72103 },\n\t{ 72106, 72151 },\n\t{ 72154, 72164 },\n};\nstatic const URange16 New_Tai_Lue_range16[] = {\n\t{ 6528, 6571 },\n\t{ 6576, 6601 },\n\t{ 6608, 6618 },\n\t{ 6622, 6623 },\n};\nstatic const URange32 Newa_range32[] = {\n\t{ 70656, 70747 },\n\t{ 70749, 70753 },\n};\nstatic const URange16 Nko_range16[] = {\n\t{ 1984, 2042 },\n\t{ 2045, 2047 },\n};\nstatic const URange32 Nushu_range32[] = {\n\t{ 94177, 94177 },\n\t{ 110960, 111355 },\n};\nstatic const URange32 Nyiakeng_Puachue_Hmong_range32[] = {\n\t{ 123136, 123180 },\n\t{ 123184, 123197 },\n\t{ 123200, 123209 },\n\t{ 123214, 123215 },\n};\nstatic const URange16 Ogham_range16[] = {\n\t{ 5760, 5788 },\n};\nstatic const URange16 Ol_Chiki_range16[] = {\n\t{ 7248, 7295 },\n};\nstatic const URange32 Old_Hungarian_range32[] = {\n\t{ 68736, 68786 },\n\t{ 68800, 68850 },\n\t{ 68858, 68863 },\n};\nstatic const URange32 Old_Italic_range32[] = {\n\t{ 66304, 66339 },\n\t{ 66349, 66351 },\n};\nstatic const URange32 Old_North_Arabian_range32[] = {\n\t{ 68224, 68255 },\n};\nstatic const URange32 Old_Permic_range32[] = {\n\t{ 66384, 66426 },\n};\nstatic const URange32 Old_Persian_range32[] = {\n\t{ 66464, 66499 },\n\t{ 66504, 66517 },\n};\nstatic const URange32 Old_Sogdian_range32[] = {\n\t{ 69376, 69415 },\n};\nstatic const URange32 Old_South_Arabian_range32[] = {\n\t{ 68192, 68223 },\n};\nstatic const URange32 Old_Turkic_range32[] = {\n\t{ 68608, 68680 },\n};\nstatic const URange32 Old_Uyghur_range32[] = {\n\t{ 69488, 69513 },\n};\nstatic const URange16 Oriya_range16[] = {\n\t{ 2817, 2819 },\n\t{ 2821, 2828 },\n\t{ 2831, 2832 },\n\t{ 2835, 2856 },\n\t{ 2858, 2864 },\n\t{ 2866, 2867 },\n\t{ 2869, 2873 },\n\t{ 2876, 2884 },\n\t{ 2887, 2888 },\n\t{ 2891, 2893 },\n\t{ 2901, 2903 },\n\t{ 2908, 2909 },\n\t{ 2911, 2915 },\n\t{ 2918, 2935 },\n};\nstatic const URange32 Osage_range32[] = {\n\t{ 66736, 66771 },\n\t{ 66776, 66811 },\n};\nstatic const URange32 Osmanya_range32[] = {\n\t{ 66688, 66717 },\n\t{ 66720, 66729 },\n};\nstatic const URange32 Pahawh_Hmong_range32[] = {\n\t{ 92928, 92997 },\n\t{ 93008, 93017 },\n\t{ 93019, 93025 },\n\t{ 93027, 93047 },\n\t{ 93053, 93071 },\n};\nstatic const URange32 Palmyrene_range32[] = {\n\t{ 67680, 67711 },\n};\nstatic const URange32 Pau_Cin_Hau_range32[] = {\n\t{ 72384, 72440 },\n};\nstatic const URange16 Phags_Pa_range16[] = {\n\t{ 43072, 43127 },\n};\nstatic const URange32 Phoenician_range32[] = {\n\t{ 67840, 67867 },\n\t{ 67871, 67871 },\n};\nstatic const URange32 Psalter_Pahlavi_range32[] = {\n\t{ 68480, 68497 },\n\t{ 68505, 68508 },\n\t{ 68521, 68527 },\n};\nstatic const URange16 Rejang_range16[] = {\n\t{ 43312, 43347 },\n\t{ 43359, 43359 },\n};\nstatic const URange16 Runic_range16[] = {\n\t{ 5792, 5866 },\n\t{ 5870, 5880 },\n};\nstatic const URange16 Samaritan_range16[] = {\n\t{ 2048, 2093 },\n\t{ 2096, 2110 },\n};\nstatic const URange16 Saurashtra_range16[] = {\n\t{ 43136, 43205 },\n\t{ 43214, 43225 },\n};\nstatic const URange32 Sharada_range32[] = {\n\t{ 70016, 70111 },\n};\nstatic const URange32 Shavian_range32[] = {\n\t{ 66640, 66687 },\n};\nstatic const URange32 Siddham_range32[] = {\n\t{ 71040, 71093 },\n\t{ 71096, 71133 },\n};\nstatic const URange32 SignWriting_range32[] = {\n\t{ 120832, 121483 },\n\t{ 121499, 121503 },\n\t{ 121505, 121519 },\n};\nstatic const URange16 Sinhala_range16[] = {\n\t{ 3457, 3459 },\n\t{ 3461, 3478 },\n\t{ 3482, 3505 },\n\t{ 3507, 3515 },\n\t{ 3517, 3517 },\n\t{ 3520, 3526 },\n\t{ 3530, 3530 },\n\t{ 3535, 3540 },\n\t{ 3542, 3542 },\n\t{ 3544, 3551 },\n\t{ 3558, 3567 },\n\t{ 3570, 3572 },\n};\nstatic const URange32 Sinhala_range32[] = {\n\t{ 70113, 70132 },\n};\nstatic const URange32 Sogdian_range32[] = {\n\t{ 69424, 69465 },\n};\nstatic const URange32 Sora_Sompeng_range32[] = {\n\t{ 69840, 69864 },\n\t{ 69872, 69881 },\n};\nstatic const URange32 Soyombo_range32[] = {\n\t{ 72272, 72354 },\n};\nstatic const URange16 Sundanese_range16[] = {\n\t{ 7040, 7103 },\n\t{ 7360, 7367 },\n};\nstatic const URange16 Syloti_Nagri_range16[] = {\n\t{ 43008, 43052 },\n};\nstatic const URange16 Syriac_range16[] = {\n\t{ 1792, 1805 },\n\t{ 1807, 1866 },\n\t{ 1869, 1871 },\n\t{ 2144, 2154 },\n};\nstatic const URange16 Tagalog_range16[] = {\n\t{ 5888, 5909 },\n\t{ 5919, 5919 },\n};\nstatic const URange16 Tagbanwa_range16[] = {\n\t{ 5984, 5996 },\n\t{ 5998, 6000 },\n\t{ 6002, 6003 },\n};\nstatic const URange16 Tai_Le_range16[] = {\n\t{ 6480, 6509 },\n\t{ 6512, 6516 },\n};\nstatic const URange16 Tai_Tham_range16[] = {\n\t{ 6688, 6750 },\n\t{ 6752, 6780 },\n\t{ 6783, 6793 },\n\t{ 6800, 6809 },\n\t{ 6816, 6829 },\n};\nstatic const URange16 Tai_Viet_range16[] = {\n\t{ 43648, 43714 },\n\t{ 43739, 43743 },\n};\nstatic const URange32 Takri_range32[] = {\n\t{ 71296, 71353 },\n\t{ 71360, 71369 },\n};\nstatic const URange16 Tamil_range16[] = {\n\t{ 2946, 2947 },\n\t{ 2949, 2954 },\n\t{ 2958, 2960 },\n\t{ 2962, 2965 },\n\t{ 2969, 2970 },\n\t{ 2972, 2972 },\n\t{ 2974, 2975 },\n\t{ 2979, 2980 },\n\t{ 2984, 2986 },\n\t{ 2990, 3001 },\n\t{ 3006, 3010 },\n\t{ 3014, 3016 },\n\t{ 3018, 3021 },\n\t{ 3024, 3024 },\n\t{ 3031, 3031 },\n\t{ 3046, 3066 },\n};\nstatic const URange32 Tamil_range32[] = {\n\t{ 73664, 73713 },\n\t{ 73727, 73727 },\n};\nstatic const URange32 Tangsa_range32[] = {\n\t{ 92784, 92862 },\n\t{ 92864, 92873 },\n};\nstatic const URange32 Tangut_range32[] = {\n\t{ 94176, 94176 },\n\t{ 94208, 100343 },\n\t{ 100352, 101119 },\n\t{ 101632, 101640 },\n};\nstatic const URange16 Telugu_range16[] = {\n\t{ 3072, 3084 },\n\t{ 3086, 3088 },\n\t{ 3090, 3112 },\n\t{ 3114, 3129 },\n\t{ 3132, 3140 },\n\t{ 3142, 3144 },\n\t{ 3146, 3149 },\n\t{ 3157, 3158 },\n\t{ 3160, 3162 },\n\t{ 3165, 3165 },\n\t{ 3168, 3171 },\n\t{ 3174, 3183 },\n\t{ 3191, 3199 },\n};\nstatic const URange16 Thaana_range16[] = {\n\t{ 1920, 1969 },\n};\nstatic const URange16 Thai_range16[] = {\n\t{ 3585, 3642 },\n\t{ 3648, 3675 },\n};\nstatic const URange16 Tibetan_range16[] = {\n\t{ 3840, 3911 },\n\t{ 3913, 3948 },\n\t{ 3953, 3991 },\n\t{ 3993, 4028 },\n\t{ 4030, 4044 },\n\t{ 4046, 4052 },\n\t{ 4057, 4058 },\n};\nstatic const URange16 Tifinagh_range16[] = {\n\t{ 11568, 11623 },\n\t{ 11631, 11632 },\n\t{ 11647, 11647 },\n};\nstatic const URange32 Tirhuta_range32[] = {\n\t{ 70784, 70855 },\n\t{ 70864, 70873 },\n};\nstatic const URange32 Toto_range32[] = {\n\t{ 123536, 123566 },\n};\nstatic const URange32 Ugaritic_range32[] = {\n\t{ 66432, 66461 },\n\t{ 66463, 66463 },\n};\nstatic const URange16 Vai_range16[] = {\n\t{ 42240, 42539 },\n};\nstatic const URange32 Vithkuqi_range32[] = {\n\t{ 66928, 66938 },\n\t{ 66940, 66954 },\n\t{ 66956, 66962 },\n\t{ 66964, 66965 },\n\t{ 66967, 66977 },\n\t{ 66979, 66993 },\n\t{ 66995, 67001 },\n\t{ 67003, 67004 },\n};\nstatic const URange32 Wancho_range32[] = {\n\t{ 123584, 123641 },\n\t{ 123647, 123647 },\n};\nstatic const URange32 Warang_Citi_range32[] = {\n\t{ 71840, 71922 },\n\t{ 71935, 71935 },\n};\nstatic const URange32 Yezidi_range32[] = {\n\t{ 69248, 69289 },\n\t{ 69291, 69293 },\n\t{ 69296, 69297 },\n};\nstatic const URange16 Yi_range16[] = {\n\t{ 40960, 42124 },\n\t{ 42128, 42182 },\n};\nstatic const URange32 Zanabazar_Square_range32[] = {\n\t{ 72192, 72263 },\n};\n// 4042 16-bit ranges, 1778 32-bit ranges\nconst UGroup unicode_groups[] = {\n\t{ \"Adlam\", +1, 0, 0, Adlam_range32, 3 },\n\t{ \"Ahom\", +1, 0, 0, Ahom_range32, 3 },\n\t{ \"Anatolian_Hieroglyphs\", +1, 0, 0, Anatolian_Hieroglyphs_range32, 1 },\n\t{ \"Arabic\", +1, Arabic_range16, 22, Arabic_range32, 36 },\n\t{ \"Armenian\", +1, Armenian_range16, 4, 0, 0 },\n\t{ \"Avestan\", +1, 0, 0, Avestan_range32, 2 },\n\t{ \"Balinese\", +1, Balinese_range16, 2, 0, 0 },\n\t{ \"Bamum\", +1, Bamum_range16, 1, Bamum_range32, 1 },\n\t{ \"Bassa_Vah\", +1, 0, 0, Bassa_Vah_range32, 2 },\n\t{ \"Batak\", +1, Batak_range16, 2, 0, 0 },\n\t{ \"Bengali\", +1, Bengali_range16, 14, 0, 0 },\n\t{ \"Bhaiksuki\", +1, 0, 0, Bhaiksuki_range32, 4 },\n\t{ \"Bopomofo\", +1, Bopomofo_range16, 3, 0, 0 },\n\t{ \"Brahmi\", +1, 0, 0, Brahmi_range32, 3 },\n\t{ \"Braille\", +1, Braille_range16, 1, 0, 0 },\n\t{ \"Buginese\", +1, Buginese_range16, 2, 0, 0 },\n\t{ \"Buhid\", +1, Buhid_range16, 1, 0, 0 },\n\t{ \"C\", +1, C_range16, 17, C_range32, 9 },\n\t{ \"Canadian_Aboriginal\", +1, Canadian_Aboriginal_range16, 2, Canadian_Aboriginal_range32, 1 },\n\t{ \"Carian\", +1, 0, 0, Carian_range32, 1 },\n\t{ \"Caucasian_Albanian\", +1, 0, 0, Caucasian_Albanian_range32, 2 },\n\t{ \"Cc\", +1, Cc_range16, 2, 0, 0 },\n\t{ \"Cf\", +1, Cf_range16, 14, Cf_range32, 7 },\n\t{ \"Chakma\", +1, 0, 0, Chakma_range32, 2 },\n\t{ \"Cham\", +1, Cham_range16, 4, 0, 0 },\n\t{ \"Cherokee\", +1, Cherokee_range16, 3, 0, 0 },\n\t{ \"Chorasmian\", +1, 0, 0, Chorasmian_range32, 1 },\n\t{ \"Co\", +1, Co_range16, 1, Co_range32, 2 },\n\t{ \"Common\", +1, Common_range16, 91, Common_range32, 82 },\n\t{ \"Coptic\", +1, Coptic_range16, 3, 0, 0 },\n\t{ \"Cs\", +1, Cs_range16, 1, 0, 0 },\n\t{ \"Cuneiform\", +1, 0, 0, Cuneiform_range32, 4 },\n\t{ \"Cypriot\", +1, 0, 0, Cypriot_range32, 6 },\n\t{ \"Cypro_Minoan\", +1, 0, 0, Cypro_Minoan_range32, 1 },\n\t{ \"Cyrillic\", +1, Cyrillic_range16, 8, Cyrillic_range32, 2 },\n\t{ \"Deseret\", +1, 0, 0, Deseret_range32, 1 },\n\t{ \"Devanagari\", +1, Devanagari_range16, 4, Devanagari_range32, 1 },\n\t{ \"Dives_Akuru\", +1, 0, 0, Dives_Akuru_range32, 8 },\n\t{ \"Dogra\", +1, 0, 0, Dogra_range32, 1 },\n\t{ \"Duployan\", +1, 0, 0, Duployan_range32, 5 },\n\t{ \"Egyptian_Hieroglyphs\", +1, 0, 0, Egyptian_Hieroglyphs_range32, 1 },\n\t{ \"Elbasan\", +1, 0, 0, Elbasan_range32, 1 },\n\t{ \"Elymaic\", +1, 0, 0, Elymaic_range32, 1 },\n\t{ \"Ethiopic\", +1, Ethiopic_range16, 32, Ethiopic_range32, 4 },\n\t{ \"Georgian\", +1, Georgian_range16, 10, 0, 0 },\n\t{ \"Glagolitic\", +1, Glagolitic_range16, 1, Glagolitic_range32, 5 },\n\t{ \"Gothic\", +1, 0, 0, Gothic_range32, 1 },\n\t{ \"Grantha\", +1, 0, 0, Grantha_range32, 15 },\n\t{ \"Greek\", +1, Greek_range16, 33, Greek_range32, 3 },\n\t{ \"Gujarati\", +1, Gujarati_range16, 14, 0, 0 },\n\t{ \"Gunjala_Gondi\", +1, 0, 0, Gunjala_Gondi_range32, 6 },\n\t{ \"Gurmukhi\", +1, Gurmukhi_range16, 16, 0, 0 },\n\t{ \"Han\", +1, Han_range16, 11, Han_range32, 11 },\n\t{ \"Hangul\", +1, Hangul_range16, 14, 0, 0 },\n\t{ \"Hanifi_Rohingya\", +1, 0, 0, Hanifi_Rohingya_range32, 2 },\n\t{ \"Hanunoo\", +1, Hanunoo_range16, 1, 0, 0 },\n\t{ \"Hatran\", +1, 0, 0, Hatran_range32, 3 },\n\t{ \"Hebrew\", +1, Hebrew_range16, 9, 0, 0 },\n\t{ \"Hiragana\", +1, Hiragana_range16, 2, Hiragana_range32, 4 },\n\t{ \"Imperial_Aramaic\", +1, 0, 0, Imperial_Aramaic_range32, 2 },\n\t{ \"Inherited\", +1, Inherited_range16, 19, Inherited_range32, 10 },\n\t{ \"Inscriptional_Pahlavi\", +1, 0, 0, Inscriptional_Pahlavi_range32, 2 },\n\t{ \"Inscriptional_Parthian\", +1, 0, 0, Inscriptional_Parthian_range32, 2 },\n\t{ \"Javanese\", +1, Javanese_range16, 3, 0, 0 },\n\t{ \"Kaithi\", +1, 0, 0, Kaithi_range32, 2 },\n\t{ \"Kannada\", +1, Kannada_range16, 13, 0, 0 },\n\t{ \"Katakana\", +1, Katakana_range16, 7, Katakana_range32, 7 },\n\t{ \"Kawi\", +1, 0, 0, Kawi_range32, 3 },\n\t{ \"Kayah_Li\", +1, Kayah_Li_range16, 2, 0, 0 },\n\t{ \"Kharoshthi\", +1, 0, 0, Kharoshthi_range32, 8 },\n\t{ \"Khitan_Small_Script\", +1, 0, 0, Khitan_Small_Script_range32, 2 },\n\t{ \"Khmer\", +1, Khmer_range16, 4, 0, 0 },\n\t{ \"Khojki\", +1, 0, 0, Khojki_range32, 2 },\n\t{ \"Khudawadi\", +1, 0, 0, Khudawadi_range32, 2 },\n\t{ \"L\", +1, L_range16, 380, L_range32, 280 },\n\t{ \"Lao\", +1, Lao_range16, 11, 0, 0 },\n\t{ \"Latin\", +1, Latin_range16, 34, Latin_range32, 5 },\n\t{ \"Lepcha\", +1, Lepcha_range16, 3, 0, 0 },\n\t{ \"Limbu\", +1, Limbu_range16, 5, 0, 0 },\n\t{ \"Linear_A\", +1, 0, 0, Linear_A_range32, 3 },\n\t{ \"Linear_B\", +1, 0, 0, Linear_B_range32, 7 },\n\t{ \"Lisu\", +1, Lisu_range16, 1, Lisu_range32, 1 },\n\t{ \"Ll\", +1, Ll_range16, 617, Ll_range32, 41 },\n\t{ \"Lm\", +1, Lm_range16, 57, Lm_range32, 14 },\n\t{ \"Lo\", +1, Lo_range16, 290, Lo_range32, 221 },\n\t{ \"Lt\", +1, Lt_range16, 10, 0, 0 },\n\t{ \"Lu\", +1, Lu_range16, 605, Lu_range32, 41 },\n\t{ \"Lycian\", +1, 0, 0, Lycian_range32, 1 },\n\t{ \"Lydian\", +1, 0, 0, Lydian_range32, 2 },\n\t{ \"M\", +1, M_range16, 190, M_range32, 120 },\n\t{ \"Mahajani\", +1, 0, 0, Mahajani_range32, 1 },\n\t{ \"Makasar\", +1, 0, 0, Makasar_range32, 1 },\n\t{ \"Malayalam\", +1, Malayalam_range16, 7, 0, 0 },\n\t{ \"Mandaic\", +1, Mandaic_range16, 2, 0, 0 },\n\t{ \"Manichaean\", +1, 0, 0, Manichaean_range32, 2 },\n\t{ \"Marchen\", +1, 0, 0, Marchen_range32, 3 },\n\t{ \"Masaram_Gondi\", +1, 0, 0, Masaram_Gondi_range32, 7 },\n\t{ \"Mc\", +1, Mc_range16, 112, Mc_range32, 70 },\n\t{ \"Me\", +1, Me_range16, 5, 0, 0 },\n\t{ \"Medefaidrin\", +1, 0, 0, Medefaidrin_range32, 1 },\n\t{ \"Meetei_Mayek\", +1, Meetei_Mayek_range16, 3, 0, 0 },\n\t{ \"Mende_Kikakui\", +1, 0, 0, Mende_Kikakui_range32, 2 },\n\t{ \"Meroitic_Cursive\", +1, 0, 0, Meroitic_Cursive_range32, 3 },\n\t{ \"Meroitic_Hieroglyphs\", +1, 0, 0, Meroitic_Hieroglyphs_range32, 1 },\n\t{ \"Miao\", +1, 0, 0, Miao_range32, 3 },\n\t{ \"Mn\", +1, Mn_range16, 212, Mn_range32, 134 },\n\t{ \"Modi\", +1, 0, 0, Modi_range32, 2 },\n\t{ \"Mongolian\", +1, Mongolian_range16, 5, Mongolian_range32, 1 },\n\t{ \"Mro\", +1, 0, 0, Mro_range32, 3 },\n\t{ \"Multani\", +1, 0, 0, Multani_range32, 5 },\n\t{ \"Myanmar\", +1, Myanmar_range16, 3, 0, 0 },\n\t{ \"N\", +1, N_range16, 67, N_range32, 70 },\n\t{ \"Nabataean\", +1, 0, 0, Nabataean_range32, 2 },\n\t{ \"Nag_Mundari\", +1, 0, 0, Nag_Mundari_range32, 1 },\n\t{ \"Nandinagari\", +1, 0, 0, Nandinagari_range32, 3 },\n\t{ \"Nd\", +1, Nd_range16, 37, Nd_range32, 27 },\n\t{ \"New_Tai_Lue\", +1, New_Tai_Lue_range16, 4, 0, 0 },\n\t{ \"Newa\", +1, 0, 0, Newa_range32, 2 },\n\t{ \"Nko\", +1, Nko_range16, 2, 0, 0 },\n\t{ \"Nl\", +1, Nl_range16, 7, Nl_range32, 5 },\n\t{ \"No\", +1, No_range16, 29, No_range32, 43 },\n\t{ \"Nushu\", +1, 0, 0, Nushu_range32, 2 },\n\t{ \"Nyiakeng_Puachue_Hmong\", +1, 0, 0, Nyiakeng_Puachue_Hmong_range32, 4 },\n\t{ \"Ogham\", +1, Ogham_range16, 1, 0, 0 },\n\t{ \"Ol_Chiki\", +1, Ol_Chiki_range16, 1, 0, 0 },\n\t{ \"Old_Hungarian\", +1, 0, 0, Old_Hungarian_range32, 3 },\n\t{ \"Old_Italic\", +1, 0, 0, Old_Italic_range32, 2 },\n\t{ \"Old_North_Arabian\", +1, 0, 0, Old_North_Arabian_range32, 1 },\n\t{ \"Old_Permic\", +1, 0, 0, Old_Permic_range32, 1 },\n\t{ \"Old_Persian\", +1, 0, 0, Old_Persian_range32, 2 },\n\t{ \"Old_Sogdian\", +1, 0, 0, Old_Sogdian_range32, 1 },\n\t{ \"Old_South_Arabian\", +1, 0, 0, Old_South_Arabian_range32, 1 },\n\t{ \"Old_Turkic\", +1, 0, 0, Old_Turkic_range32, 1 },\n\t{ \"Old_Uyghur\", +1, 0, 0, Old_Uyghur_range32, 1 },\n\t{ \"Oriya\", +1, Oriya_range16, 14, 0, 0 },\n\t{ \"Osage\", +1, 0, 0, Osage_range32, 2 },\n\t{ \"Osmanya\", +1, 0, 0, Osmanya_range32, 2 },\n\t{ \"P\", +1, P_range16, 133, P_range32, 58 },\n\t{ \"Pahawh_Hmong\", +1, 0, 0, Pahawh_Hmong_range32, 5 },\n\t{ \"Palmyrene\", +1, 0, 0, Palmyrene_range32, 1 },\n\t{ \"Pau_Cin_Hau\", +1, 0, 0, Pau_Cin_Hau_range32, 1 },\n\t{ \"Pc\", +1, Pc_range16, 6, 0, 0 },\n\t{ \"Pd\", +1, Pd_range16, 18, Pd_range32, 1 },\n\t{ \"Pe\", +1, Pe_range16, 76, 0, 0 },\n\t{ \"Pf\", +1, Pf_range16, 10, 0, 0 },\n\t{ \"Phags_Pa\", +1, Phags_Pa_range16, 1, 0, 0 },\n\t{ \"Phoenician\", +1, 0, 0, Phoenician_range32, 2 },\n\t{ \"Pi\", +1, Pi_range16, 11, 0, 0 },\n\t{ \"Po\", +1, Po_range16, 130, Po_range32, 57 },\n\t{ \"Ps\", +1, Ps_range16, 79, 0, 0 },\n\t{ \"Psalter_Pahlavi\", +1, 0, 0, Psalter_Pahlavi_range32, 3 },\n\t{ \"Rejang\", +1, Rejang_range16, 2, 0, 0 },\n\t{ \"Runic\", +1, Runic_range16, 2, 0, 0 },\n\t{ \"S\", +1, S_range16, 152, S_range32, 81 },\n\t{ \"Samaritan\", +1, Samaritan_range16, 2, 0, 0 },\n\t{ \"Saurashtra\", +1, Saurashtra_range16, 2, 0, 0 },\n\t{ \"Sc\", +1, Sc_range16, 18, Sc_range32, 3 },\n\t{ \"Sharada\", +1, 0, 0, Sharada_range32, 1 },\n\t{ \"Shavian\", +1, 0, 0, Shavian_range32, 1 },\n\t{ \"Siddham\", +1, 0, 0, Siddham_range32, 2 },\n\t{ \"SignWriting\", +1, 0, 0, SignWriting_range32, 3 },\n\t{ \"Sinhala\", +1, Sinhala_range16, 12, Sinhala_range32, 1 },\n\t{ \"Sk\", +1, Sk_range16, 30, Sk_range32, 1 },\n\t{ \"Sm\", +1, Sm_range16, 53, Sm_range32, 11 },\n\t{ \"So\", +1, So_range16, 115, So_range32, 70 },\n\t{ \"Sogdian\", +1, 0, 0, Sogdian_range32, 1 },\n\t{ \"Sora_Sompeng\", +1, 0, 0, Sora_Sompeng_range32, 2 },\n\t{ \"Soyombo\", +1, 0, 0, Soyombo_range32, 1 },\n\t{ \"Sundanese\", +1, Sundanese_range16, 2, 0, 0 },\n\t{ \"Syloti_Nagri\", +1, Syloti_Nagri_range16, 1, 0, 0 },\n\t{ \"Syriac\", +1, Syriac_range16, 4, 0, 0 },\n\t{ \"Tagalog\", +1, Tagalog_range16, 2, 0, 0 },\n\t{ \"Tagbanwa\", +1, Tagbanwa_range16, 3, 0, 0 },\n\t{ \"Tai_Le\", +1, Tai_Le_range16, 2, 0, 0 },\n\t{ \"Tai_Tham\", +1, Tai_Tham_range16, 5, 0, 0 },\n\t{ \"Tai_Viet\", +1, Tai_Viet_range16, 2, 0, 0 },\n\t{ \"Takri\", +1, 0, 0, Takri_range32, 2 },\n\t{ \"Tamil\", +1, Tamil_range16, 16, Tamil_range32, 2 },\n\t{ \"Tangsa\", +1, 0, 0, Tangsa_range32, 2 },\n\t{ \"Tangut\", +1, 0, 0, Tangut_range32, 4 },\n\t{ \"Telugu\", +1, Telugu_range16, 13, 0, 0 },\n\t{ \"Thaana\", +1, Thaana_range16, 1, 0, 0 },\n\t{ \"Thai\", +1, Thai_range16, 2, 0, 0 },\n\t{ \"Tibetan\", +1, Tibetan_range16, 7, 0, 0 },\n\t{ \"Tifinagh\", +1, Tifinagh_range16, 3, 0, 0 },\n\t{ \"Tirhuta\", +1, 0, 0, Tirhuta_range32, 2 },\n\t{ \"Toto\", +1, 0, 0, Toto_range32, 1 },\n\t{ \"Ugaritic\", +1, 0, 0, Ugaritic_range32, 2 },\n\t{ \"Vai\", +1, Vai_range16, 1, 0, 0 },\n\t{ \"Vithkuqi\", +1, 0, 0, Vithkuqi_range32, 8 },\n\t{ \"Wancho\", +1, 0, 0, Wancho_range32, 2 },\n\t{ \"Warang_Citi\", +1, 0, 0, Warang_Citi_range32, 2 },\n\t{ \"Yezidi\", +1, 0, 0, Yezidi_range32, 3 },\n\t{ \"Yi\", +1, Yi_range16, 2, 0, 0 },\n\t{ \"Z\", +1, Z_range16, 8, 0, 0 },\n\t{ \"Zanabazar_Square\", +1, 0, 0, Zanabazar_Square_range32, 1 },\n\t{ \"Zl\", +1, Zl_range16, 1, 0, 0 },\n\t{ \"Zp\", +1, Zp_range16, 1, 0, 0 },\n\t{ \"Zs\", +1, Zs_range16, 7, 0, 0 },\n};\nconst int num_unicode_groups = 199;\n\n\n} // namespace re2\n\n\n" }, { "path": "re2/unicode_groups.h", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_UNICODE_GROUPS_H_\n#define RE2_UNICODE_GROUPS_H_\n\n// Unicode character groups.\n\n// The codes get split into ranges of 16-bit codes\n// and ranges of 32-bit codes. It would be simpler\n// to use only 32-bit ranges, but these tables are large\n// enough to warrant extra care.\n//\n// Using just 32-bit ranges gives 27 kB of data.\n// Adding 16-bit ranges gives 18 kB of data.\n// Adding an extra table of 16-bit singletons would reduce\n// to 16.5 kB of data but make the data harder to use;\n// we don't bother.\n\n#include \n\n#include \"util/utf.h\"\n\nnamespace re2 {\n\nstruct URange16\n{\n uint16_t lo;\n uint16_t hi;\n};\n\nstruct URange32\n{\n Rune lo;\n Rune hi;\n};\n\nstruct UGroup\n{\n const char *name;\n int sign; // +1 for [abc], -1 for [^abc]\n const URange16 *r16;\n int nr16;\n const URange32 *r32;\n int nr32;\n};\n\n// Named by property or script name (e.g., \"Nd\", \"N\", \"Han\").\n// Negated groups are not included.\nextern const UGroup unicode_groups[];\nextern const int num_unicode_groups;\n\n// Named by POSIX name (e.g., \"[:alpha:]\", \"[:^lower:]\").\n// Negated groups are included.\nextern const UGroup posix_groups[];\nextern const int num_posix_groups;\n\n// Named by Perl name (e.g., \"\\\\d\", \"\\\\D\").\n// Negated groups are included.\nextern const UGroup perl_groups[];\nextern const int num_perl_groups;\n\n} // namespace re2\n\n#endif // RE2_UNICODE_GROUPS_H_\n" }, { "path": "re2/walker-inl.h", "content": "// Copyright 2006 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef RE2_WALKER_INL_H_\n#define RE2_WALKER_INL_H_\n\n// Helper class for traversing Regexps without recursion.\n// Clients should declare their own subclasses that override\n// the PreVisit and PostVisit methods, which are called before\n// and after visiting the subexpressions.\n\n// Not quite the Visitor pattern, because (among other things)\n// the Visitor pattern is recursive.\n\n#include \n\n#include \"absl/base/macros.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"re2/regexp.h\"\n\nnamespace re2 {\n\ntemplate struct WalkState;\n\ntemplate class Regexp::Walker {\n public:\n Walker();\n virtual ~Walker();\n\n // Virtual method called before visiting re's children.\n // PreVisit passes ownership of its return value to its caller.\n // The Arg* that PreVisit returns will be passed to PostVisit as pre_arg\n // and passed to the child PreVisits and PostVisits as parent_arg.\n // At the top-most Regexp, parent_arg is arg passed to walk.\n // If PreVisit sets *stop to true, the walk does not recurse\n // into the children. Instead it behaves as though the return\n // value from PreVisit is the return value from PostVisit.\n // The default PreVisit returns parent_arg.\n virtual T PreVisit(Regexp* re, T parent_arg, bool* stop);\n\n // Virtual method called after visiting re's children.\n // The pre_arg is the T that PreVisit returned.\n // The child_args is a vector of the T that the child PostVisits returned.\n // PostVisit takes ownership of pre_arg.\n // PostVisit takes ownership of the Ts\n // in *child_args, but not the vector itself.\n // PostVisit passes ownership of its return value\n // to its caller.\n // The default PostVisit simply returns pre_arg.\n virtual T PostVisit(Regexp* re, T parent_arg, T pre_arg,\n T* child_args, int nchild_args);\n\n // Virtual method called to copy a T,\n // when Walk notices that more than one child is the same re.\n virtual T Copy(T arg);\n\n // Virtual method called to do a \"quick visit\" of the re,\n // but not its children. Only called once the visit budget\n // has been used up and we're trying to abort the walk\n // as quickly as possible. Should return a value that\n // makes sense for the parent PostVisits still to be run.\n // This function is (hopefully) only called by\n // WalkExponential, but must be implemented by all clients,\n // just in case.\n virtual T ShortVisit(Regexp* re, T parent_arg) = 0;\n\n // Walks over a regular expression.\n // Top_arg is passed as parent_arg to PreVisit and PostVisit of re.\n // Returns the T returned by PostVisit on re.\n T Walk(Regexp* re, T top_arg);\n\n // Like Walk, but doesn't use Copy. This can lead to\n // exponential runtimes on cross-linked Regexps like the\n // ones generated by Simplify. To help limit this,\n // at most max_visits nodes will be visited and then\n // the walk will be cut off early.\n // If the walk *is* cut off early, ShortVisit(re)\n // will be called on regexps that cannot be fully\n // visited rather than calling PreVisit/PostVisit.\n T WalkExponential(Regexp* re, T top_arg, int max_visits);\n\n // Clears the stack. Should never be necessary, since\n // Walk always enters and exits with an empty stack.\n // Logs DFATAL if stack is not already clear.\n void Reset();\n\n // Returns whether walk was cut off.\n bool stopped_early() { return stopped_early_; }\n\n private:\n // Walk state for the entire traversal.\n std::stack> stack_;\n bool stopped_early_;\n int max_visits_;\n\n T WalkInternal(Regexp* re, T top_arg, bool use_copy);\n\n Walker(const Walker&) = delete;\n Walker& operator=(const Walker&) = delete;\n};\n\ntemplate T Regexp::Walker::PreVisit(Regexp* re,\n T parent_arg,\n bool* stop) {\n return parent_arg;\n}\n\ntemplate T Regexp::Walker::PostVisit(Regexp* re,\n T parent_arg,\n T pre_arg,\n T* child_args,\n int nchild_args) {\n return pre_arg;\n}\n\ntemplate T Regexp::Walker::Copy(T arg) {\n return arg;\n}\n\n// State about a single level in the traversal.\ntemplate struct WalkState {\n WalkState(Regexp* re, T parent)\n : re(re),\n n(-1),\n parent_arg(parent),\n child_args(NULL) { }\n\n Regexp* re; // The regexp\n int n; // The index of the next child to process; -1 means need to PreVisit\n T parent_arg; // Accumulated arguments.\n T pre_arg;\n T child_arg; // One-element buffer for child_args.\n T* child_args;\n};\n\ntemplate Regexp::Walker::Walker() {\n stopped_early_ = false;\n}\n\ntemplate Regexp::Walker::~Walker() {\n Reset();\n}\n\n// Clears the stack. Should never be necessary, since\n// Walk always enters and exits with an empty stack.\n// Logs DFATAL if stack is not already clear.\ntemplate void Regexp::Walker::Reset() {\n if (!stack_.empty()) {\n ABSL_LOG(DFATAL) << \"Stack not empty.\";\n while (!stack_.empty()) {\n if (stack_.top().re->nsub_ > 1)\n delete[] stack_.top().child_args;\n stack_.pop();\n }\n }\n}\n\ntemplate T Regexp::Walker::WalkInternal(Regexp* re, T top_arg,\n bool use_copy) {\n Reset();\n\n if (re == NULL) {\n ABSL_LOG(DFATAL) << \"Walk NULL\";\n return top_arg;\n }\n\n stack_.push(WalkState(re, top_arg));\n\n WalkState* s;\n for (;;) {\n T t;\n s = &stack_.top();\n re = s->re;\n switch (s->n) {\n case -1: {\n if (--max_visits_ < 0) {\n stopped_early_ = true;\n t = ShortVisit(re, s->parent_arg);\n break;\n }\n bool stop = false;\n s->pre_arg = PreVisit(re, s->parent_arg, &stop);\n if (stop) {\n t = s->pre_arg;\n break;\n }\n s->n = 0;\n s->child_args = NULL;\n if (re->nsub_ == 1)\n s->child_args = &s->child_arg;\n else if (re->nsub_ > 1)\n s->child_args = new T[re->nsub_];\n [[fallthrough]];\n }\n default: {\n if (re->nsub_ > 0) {\n Regexp** sub = re->sub();\n if (s->n < re->nsub_) {\n if (use_copy && s->n > 0 && sub[s->n - 1] == sub[s->n]) {\n s->child_args[s->n] = Copy(s->child_args[s->n - 1]);\n s->n++;\n } else {\n stack_.push(WalkState(sub[s->n], s->pre_arg));\n }\n continue;\n }\n }\n\n t = PostVisit(re, s->parent_arg, s->pre_arg, s->child_args, s->n);\n if (re->nsub_ > 1)\n delete[] s->child_args;\n break;\n }\n }\n\n // We've finished stack_.top().\n // Update next guy down.\n stack_.pop();\n if (stack_.empty())\n return t;\n s = &stack_.top();\n if (s->child_args != NULL)\n s->child_args[s->n] = t;\n else\n s->child_arg = t;\n s->n++;\n }\n}\n\ntemplate T Regexp::Walker::Walk(Regexp* re, T top_arg) {\n // Without the exponential walking behavior,\n // this budget should be more than enough for any\n // regexp, and yet not enough to get us in trouble\n // as far as CPU time.\n max_visits_ = 1000000;\n return WalkInternal(re, top_arg, true);\n}\n\ntemplate T Regexp::Walker::WalkExponential(Regexp* re, T top_arg,\n int max_visits) {\n max_visits_ = max_visits;\n return WalkInternal(re, top_arg, false);\n}\n\n} // namespace re2\n\n#endif // RE2_WALKER_INL_H_\n" }, { "path": "re2.pc.in", "content": "includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@\nlibdir=@CMAKE_INSTALL_FULL_LIBDIR@\n\nName: re2\nDescription: RE2 is a fast, safe, thread-friendly regular expression engine.\nRequires: @REQUIRES@\nVersion: @SONAME@.0.0\nCflags: -pthread -I${includedir}\nLibs: -pthread -L${libdir} -lre2\n" }, { "path": "re2Config.cmake.in", "content": "# Copyright 2022 The RE2 Authors. All Rights Reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n@PACKAGE_INIT@\n\ninclude(CMakeFindDependencyMacro)\n\nif(UNIX)\n set(THREADS_PREFER_PTHREAD_FLAG ON)\n find_dependency(Threads REQUIRED)\nendif()\n\nfind_dependency(absl REQUIRED)\n\nif(@RE2_USE_ICU@)\n find_dependency(ICU REQUIRED COMPONENTS uc)\nendif()\n\ncheck_required_components(re2)\n\nif(TARGET re2::re2)\n return()\nendif()\n\ninclude(${CMAKE_CURRENT_LIST_DIR}/re2Targets.cmake)\n" }, { "path": "runtests", "content": "#!/usr/bin/env sh\n\n# System Integrity Protection on Darwin complicated these matters somewhat.\n# See https://github.com/google/re2/issues/175 for details.\nif [ \"x$1\" = \"x-shared-library-path\" ]; then\n\tif [ \"x$(uname)\" = \"xDarwin\" ]; then\n\t\tDYLD_LIBRARY_PATH=\"$2:$DYLD_LIBRARY_PATH\"\n\t\texport DYLD_LIBRARY_PATH\n\telse\n\t\tLD_LIBRARY_PATH=\"$2:$LD_LIBRARY_PATH\"\n\t\texport LD_LIBRARY_PATH\n\tfi\n\tshift 2\nfi\n\nsuccess=true\nfor i; do\n\tprintf \"%-40s\" $i\n\tif $($i >$i.log 2>&1) 2>/dev/null; then\n\t\techo PASS\n\telse\n\t\techo FAIL';' output in $i.log\n\t\tsuccess=false\n\tfi\ndone\n\nif $success; then\n\techo 'ALL TESTS PASSED.'\n\texit 0\nelse\n\techo 'TESTS FAILED.'\n\texit 1\nfi\n" }, { "path": "testinstall.cc", "content": "// Copyright 2008 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \n#include \n#include \n\nint main() {\n re2::FilteredRE2 f;\n int id;\n f.Add(\"a.*b.*c\", RE2::DefaultOptions, &id);\n std::vector v;\n f.Compile(&v);\n std::vector ids;\n f.FirstMatch(\"abbccc\", ids);\n\n int n;\n if (RE2::FullMatch(\"axbyc\", \"a.*b.*c\") &&\n RE2::PartialMatch(\"foo123bar\", \"(\\\\d+)\", &n) && n == 123) {\n printf(\"PASS\\n\");\n return 0;\n }\n\n printf(\"FAIL\\n\");\n return 2;\n}\n" }, { "path": "ucs2.diff", "content": "This is a dump from Google's source control system of the change\nthat removed UCS-2 support from RE2. As the explanation below\nsays, UCS-2 mode is fundamentally at odds with things like ^ and $,\nso it never really worked very well. But if you are interested in using\nit without those operators, it did work for that. It assumed that the\nUCS-2 data was in the native host byte order.\n\nIf you are interested in adding UCS-2 mode back, this patch might\nbe a good starting point.\n\n\nChange 12780686 by rsc@rsc-re2 on 2009/09/16 15:30:15\n\n\tRetire UCS-2 mode.\n\t\n\tI added it as an experiment for V8, but it\n\trequires 2-byte lookahead to do completely,\n\tand RE2 has 1-byte lookahead (enough for UTF-8)\n\tas a fairly deep fundamental assumption,\n\tso it did not support ^ or $.\n\n==== re2/bitstate.cc#2 - re2/bitstate.cc#3 ====\nre2/bitstate.cc#2:314,321 - re2/bitstate.cc#3:314,319\n cap_[0] = p;\n if (TrySearch(prog_->start(), p)) // Match must be leftmost; done.\n return true;\n- if (prog_->flags() & Regexp::UCS2)\n- p++;\n }\n return false;\n }\n==== re2/compile.cc#17 - re2/compile.cc#18 ====\nre2/compile.cc#17:95,101 - re2/compile.cc#18:95,100\n // Input encodings.\n enum Encoding {\n kEncodingUTF8 = 1, // UTF-8 (0-10FFFF)\n- kEncodingUCS2, // UCS-2 (0-FFFF), native byte order\n kEncodingLatin1, // Latin1 (0-FF)\n };\n \nre2/compile.cc#17:168,176 - re2/compile.cc#18:167,172\n void AddRuneRangeLatin1(Rune lo, Rune hi, bool foldcase);\n void AddRuneRangeUTF8(Rune lo, Rune hi, bool foldcase);\n void Add_80_10ffff();\n- void AddRuneRangeUCS2(Rune lo, Rune hi, bool foldcase);\n- void AddUCS2Pair(uint8 lo1, uint8 hi1, bool fold1,\n- uint8 lo2, uint8 hi2, bool fold2);\n \n // New suffix that matches the byte range lo-hi, then goes to next.\n Inst* RuneByteSuffix(uint8 lo, uint8 hi, bool foldcase, Inst* next);\nre2/compile.cc#17:475,481 - re2/compile.cc#18:471,477\n \n // Converts rune range lo-hi into a fragment that recognizes\n // the bytes that would make up those runes in the current\n- // encoding (Latin 1, UTF-8, or UCS-2).\n+ // encoding (Latin 1 or UTF-8).\n // This lets the machine work byte-by-byte even when\n // using multibyte encodings.\n \nre2/compile.cc#17:488,496 - re2/compile.cc#18:484,489\n case kEncodingLatin1:\n AddRuneRangeLatin1(lo, hi, foldcase);\n break;\n- case kEncodingUCS2:\n- AddRuneRangeUCS2(lo, hi, foldcase);\n- break;\n }\n }\n \nre2/compile.cc#17:503,581 - re2/compile.cc#18:496,501\n AddSuffix(RuneByteSuffix(lo, hi, foldcase, NULL));\n }\n \n- // Test whether 16-bit values are big or little endian.\n- static bool BigEndian() {\n- union {\n- char byte[2];\n- int16 endian;\n- } u;\n- \n- u.byte[0] = 1;\n- u.byte[1] = 2;\n- return u.endian == 0x0102;\n- }\n- \n- void Compiler::AddUCS2Pair(uint8 lo1, uint8 hi1, bool fold1,\n- uint8 lo2, uint8 hi2, bool fold2) {\n- Inst* ip;\n- if (reversed_) {\n- ip = RuneByteSuffix(lo1, hi1, fold1, NULL);\n- ip = RuneByteSuffix(lo2, hi2, fold2, ip);\n- } else {\n- ip = RuneByteSuffix(lo2, hi2, fold2, NULL);\n- ip = RuneByteSuffix(lo1, hi1, fold1, ip);\n- }\n- AddSuffix(ip);\n- }\n- \n- void Compiler::AddRuneRangeUCS2(Rune lo, Rune hi, bool foldcase) {\n- if (lo > hi || lo > 0xFFFF)\n- return;\n- if (hi > 0xFFFF)\n- hi = 0xFFFF;\n- \n- // We'll assemble a pattern assuming big endian.\n- // If the machine isn't, tell Cat to reverse its arguments.\n- bool oldreversed = reversed_;\n- if (!BigEndian()) {\n- reversed_ = !oldreversed;\n- }\n- \n- // Split into bytes.\n- int lo1 = lo >> 8;\n- int lo2 = lo & 0xFF;\n- int hi1 = hi >> 8;\n- int hi2 = hi & 0xFF;\n- \n- if (lo1 == hi1) {\n- // Easy case: high bits are same in both.\n- // Only do ASCII case folding on the second byte if the top byte is 00.\n- AddUCS2Pair(lo1, lo1, false, lo2, hi2, lo1==0 && foldcase);\n- } else {\n- // Harder case: different second byte ranges depending on first byte.\n- \n- // Initial fragment.\n- if (lo2 > 0) {\n- AddUCS2Pair(lo1, lo1, false, lo2, 0xFF, lo1==0 && foldcase);\n- lo1++;\n- }\n- \n- // Trailing fragment.\n- if (hi2 < 0xFF) {\n- AddUCS2Pair(hi1, hi1, false, 0, hi2, false);\n- hi1--;\n- }\n- \n- // Inner ranges.\n- if (lo1 <= hi1) {\n- AddUCS2Pair(lo1, hi1, false, 0, 0xFF, false);\n- }\n- }\n- \n- // Restore reverse setting.\n- reversed_ = oldreversed;\n- }\n- \n // Table describing how to make a UTF-8 matching machine\n // for the rune range 80-10FFFF (Runeself-Runemax).\n // This range happens frequently enough (for example /./ and /[^a-z]/)\nre2/compile.cc#17:707,716 - re2/compile.cc#18:627,634\n \n Frag Compiler::Literal(Rune r, bool foldcase) {\n switch (encoding_) {\n- default: // UCS-2 or something new\n- BeginRange();\n- AddRuneRange(r, r, foldcase);\n- return EndRange();\n+ default:\n+ return kNullFrag;\n \n case kEncodingLatin1:\n return ByteRange(r, r, foldcase);\nre2/compile.cc#17:927,934 - re2/compile.cc#18:845,850\n \n if (re->parse_flags() & Regexp::Latin1)\n c.encoding_ = kEncodingLatin1;\n- else if (re->parse_flags() & Regexp::UCS2)\n- c.encoding_ = kEncodingUCS2;\n c.reversed_ = reversed;\n if (max_mem <= 0) {\n c.max_inst_ = 100000; // more than enough\nre2/compile.cc#17:983,993 - re2/compile.cc#18:899,905\n c.prog_->set_start_unanchored(c.prog_->start());\n } else {\n Frag dot;\n- if (c.encoding_ == kEncodingUCS2) {\n- dot = c.Cat(c.ByteRange(0x00, 0xFF, false), c.ByteRange(0x00, 0xFF, false));\n- } else {\n- dot = c.ByteRange(0x00, 0xFF, false);\n- }\n+ dot = c.ByteRange(0x00, 0xFF, false);\n Frag dotloop = c.Star(dot, true);\n Frag unanchored = c.Cat(dotloop, all);\n c.prog_->set_start_unanchored(unanchored.begin);\n==== re2/nfa.cc#8 - re2/nfa.cc#9 ====\nre2/nfa.cc#8:426,432 - re2/nfa.cc#9:426,431\n const char* bp = context.begin();\n int c = -1;\n int wasword = 0;\n- bool ucs2 = prog_->flags() & Regexp::UCS2;\n \n if (text.begin() > context.begin()) {\n c = text.begin()[-1] & 0xFF;\nre2/nfa.cc#8:492,498 - re2/nfa.cc#9:491,497\n // If there's a required first byte for an unanchored search\n // and we're not in the middle of any possible matches,\n // use memchr to search for the byte quickly.\n- if (!ucs2 && !anchored && first_byte_ >= 0 && runq->size() == 0 &&\n+ if (!anchored && first_byte_ >= 0 && runq->size() == 0 &&\n p < text.end() && (p[0] & 0xFF) != first_byte_) {\n p = reinterpret_cast(memchr(p, first_byte_,\n text.end() - p));\nre2/nfa.cc#8:505,526 - re2/nfa.cc#9:504,514\n flag = Prog::EmptyFlags(context, p);\n }\n \n- // In UCS-2 mode, if we need to start a new thread,\n- // make sure to do it on an even boundary.\n- if(ucs2 && runq->size() == 0 &&\n- (p - context.begin()) % 2 && p < text.end()) {\n- p++;\n- flag = Prog::EmptyFlags(context, p);\n- }\n- \n // Steal match storage (cleared but unused as of yet)\n // temporarily to hold match boundaries for new thread.\n- // In UCS-2 mode, only start the thread on a 2-byte boundary.\n- if(!ucs2 || (p - context.begin()) % 2 == 0) {\n- match_[0] = p;\n- AddToThreadq(runq, start_, flag, p, match_);\n- match_[0] = NULL;\n- }\n+ match_[0] = p;\n+ AddToThreadq(runq, start_, flag, p, match_);\n+ match_[0] = NULL;\n }\n \n // If all the threads have died, stop early.\n==== re2/parse.cc#22 - re2/parse.cc#23 ====\nre2/parse.cc#22:160,167 - re2/parse.cc#23:160,165\n status_(status), stacktop_(NULL), ncap_(0) {\n if (flags_ & Latin1)\n rune_max_ = 0xFF;\n- else if (flags & UCS2)\n- rune_max_ = 0xFFFF;\n else\n rune_max_ = Runemax;\n }\nre2/parse.cc#22:365,387 - re2/parse.cc#23:363,374\n bool Regexp::ParseState::PushCarat() {\n if (flags_ & OneLine) {\n return PushSimpleOp(kRegexpBeginText);\n- } else {\n- if (flags_ & UCS2) {\n- status_->set_code(kRegexpUnsupported);\n- status_->set_error_arg(\"multiline ^ in UCS-2 mode\");\n- return false;\n- }\n- return PushSimpleOp(kRegexpBeginLine);\n }\n+ return PushSimpleOp(kRegexpBeginLine);\n }\n \n // Pushes a \\b or \\B onto the stack.\n bool Regexp::ParseState::PushWordBoundary(bool word) {\n- if (flags_ & UCS2) {\n- status_->set_code(kRegexpUnsupported);\n- status_->set_error_arg(\"\\\\b or \\\\B in UCS-2 mode\");\n- return false;\n- }\n if (word)\n return PushSimpleOp(kRegexpWordBoundary);\n return PushSimpleOp(kRegexpNoWordBoundary);\nre2/parse.cc#22:397,407 - re2/parse.cc#23:384,389\n bool ret = PushSimpleOp(kRegexpEndText);\n flags_ = oflags;\n return ret;\n- }\n- if (flags_ & UCS2) {\n- status_->set_code(kRegexpUnsupported);\n- status_->set_error_arg(\"multiline $ in UCS-2 mode\");\n- return false;\n }\n return PushSimpleOp(kRegexpEndLine);\n }\n==== re2/re2.cc#34 - re2/re2.cc#35 ====\nre2/re2.cc#34:79,86 - re2/re2.cc#35:79,84\n return RE2::ErrorBadUTF8;\n case re2::kRegexpBadNamedCapture:\n return RE2::ErrorBadNamedCapture;\n- case re2::kRegexpUnsupported:\n- return RE2::ErrorUnsupported;\n }\n return RE2::ErrorInternal;\n }\nre2/re2.cc#34:122,130 - re2/re2.cc#35:120,125\n break;\n case RE2::Options::EncodingLatin1:\n flags |= Regexp::Latin1;\n- break;\n- case RE2::Options::EncodingUCS2:\n- flags |= Regexp::UCS2;\n break;\n }\n \n==== re2/re2.h#36 - re2/re2.h#37 ====\nre2/re2.h#36:246,252 - re2/re2.h#37:246,251\n ErrorBadUTF8, // invalid UTF-8 in regexp\n ErrorBadNamedCapture, // bad named capture group\n ErrorPatternTooLarge, // pattern too large (compile failed)\n- ErrorUnsupported, // unsupported feature (in UCS-2 mode)\n };\n \n // Predefined common options.\nre2/re2.h#36:570,576 - re2/re2.h#37:569,574\n \n enum Encoding {\n EncodingUTF8 = 1,\n- EncodingUCS2, // 16-bit Unicode 0-FFFF only\n EncodingLatin1\n };\n \n==== re2/regexp.cc#15 - re2/regexp.cc#16 ====\nre2/regexp.cc#15:324,333 - re2/regexp.cc#16:324,329\n // the regexp that remains after the prefix. The prefix might\n // be ASCII case-insensitive.\n bool Regexp::RequiredPrefix(string *prefix, bool *foldcase, Regexp** suffix) {\n- // Don't even bother for UCS-2; it's time to throw that code away.\n- if (parse_flags_ & UCS2)\n- return false;\n- \n // No need for a walker: the regexp must be of the form\n // 1. some number of ^ anchors\n // 2. a literal char or string\n==== re2/regexp.h#20 - re2/regexp.h#21 ====\nre2/regexp.h#20:187,193 - re2/regexp.h#21:187,192\n kRegexpBadPerlOp, // bad perl operator\n kRegexpBadUTF8, // invalid UTF-8 in regexp\n kRegexpBadNamedCapture, // bad named capture\n- kRegexpUnsupported, // unsupported operator\n };\n \n // Error status for certain operations.\nre2/regexp.h#20:307,316 - re2/regexp.h#21:306,314\n // \\Q and \\E to disable/enable metacharacters\n // (?Pexpr) for named captures\n // \\C to match any single byte\n- UCS2 = 1<<10, // Text is in UCS-2, regexp is in UTF-8.\n- UnicodeGroups = 1<<11, // Allow \\p{Han} for Unicode Han group\n+ UnicodeGroups = 1<<10, // Allow \\p{Han} for Unicode Han group\n // and \\P{Han} for its negation.\n- NeverNL = 1<<12, // Never match NL, even if the regexp mentions\n+ NeverNL = 1<<11, // Never match NL, even if the regexp mentions\n // it explicitly.\n \n // As close to Perl as we can get.\n==== re2/testing/backtrack.cc#4 - re2/testing/backtrack.cc#5 ====\nre2/testing/backtrack.cc#4:134,141 - re2/testing/backtrack.cc#5:134,139\n cap_[0] = p;\n if (Visit(prog_->start(), p)) // Match must be leftmost; done.\n return true;\n- if (prog_->flags() & Regexp::UCS2)\n- p++;\n }\n return false;\n }\n==== re2/testing/tester.cc#12 - re2/testing/tester.cc#13 ====\nre2/testing/tester.cc#12:144,154 - re2/testing/tester.cc#13:144,152\n static ParseMode parse_modes[] = {\n { single_line, \"single-line\" },\n { single_line|Regexp::Latin1, \"single-line, latin1\" },\n- { single_line|Regexp::UCS2, \"single-line, ucs2\" },\n { multi_line, \"multiline\" },\n { multi_line|Regexp::NonGreedy, \"multiline, nongreedy\" },\n { multi_line|Regexp::Latin1, \"multiline, latin1\" },\n- { multi_line|Regexp::UCS2, \"multiline, ucs2\" },\n };\n \n static string FormatMode(Regexp::ParseFlags flags) {\nre2/testing/tester.cc#12:179,189 - re2/testing/tester.cc#13:177,185\n RegexpStatus status;\n regexp_ = Regexp::Parse(regexp_str, flags, &status);\n if (regexp_ == NULL) {\n- if (status.code() != kRegexpUnsupported) {\n- LOG(INFO) << \"Cannot parse: \" << CEscape(regexp_str_)\n- << \" mode: \" << FormatMode(flags);\n- error_ = true;\n- }\n+ LOG(INFO) << \"Cannot parse: \" << CEscape(regexp_str_)\n+ << \" mode: \" << FormatMode(flags);\n+ error_ = true;\n return;\n }\n prog_ = regexp_->CompileToProg(0);\nre2/testing/tester.cc#12:230,237 - re2/testing/tester.cc#13:226,231\n RE2::Options options;\n if (flags & Regexp::Latin1)\n options.set_encoding(RE2::Options::EncodingLatin1);\n- else if (flags & Regexp::UCS2)\n- options.set_encoding(RE2::Options::EncodingUCS2);\n if (kind_ == Prog::kLongestMatch)\n options.set_longest_match(true);\n re2_ = new RE2(re, options);\nre2/testing/tester.cc#12:281,379 - re2/testing/tester.cc#13:275,280\n delete re2_;\n }\n \n- // Converts UTF-8 string in text into UCS-2 string in new_text.\n- static bool ConvertUTF8ToUCS2(const StringPiece& text, StringPiece* new_text) {\n- const char* p = text.begin();\n- const char* ep = text.end();\n- uint16* q = new uint16[ep - p];\n- uint16* q0 = q;\n- \n- int n;\n- Rune r;\n- for (; p < ep; p += n) {\n- if (!fullrune(p, ep - p)) {\n- delete[] q0;\n- return false;\n- }\n- n = chartorune(&r, p);\n- if (r > 0xFFFF) {\n- delete[] q0;\n- return false;\n- }\n- *q++ = r;\n- }\n- *new_text = StringPiece(reinterpret_cast(q0), 2*(q - q0));\n- return true;\n- }\n- \n- // Rewrites *sp from being a pointer into text8 (UTF-8)\n- // to being a pointer into text16 (equivalent text but in UCS-2).\n- static void AdjustUTF8ToUCS2(const StringPiece& text8, const StringPiece& text16,\n- StringPiece *sp) {\n- if (sp->begin() == NULL && text8.begin() != NULL)\n- return;\n- \n- int nrune = 0;\n- int n;\n- Rune r;\n- const char* p = text8.begin();\n- const char* ep = text8.end();\n- const char* spbegin = NULL;\n- const char* spend = NULL;\n- for (;;) {\n- if (p == sp->begin())\n- spbegin = text16.begin() + sizeof(uint16)*nrune;\n- if (p == sp->end())\n- spend = text16.begin() + sizeof(uint16)*nrune;\n- if (p >= ep)\n- break;\n- n = chartorune(&r, p);\n- p += n;\n- nrune++;\n- }\n- if (spbegin == NULL || spend == NULL) {\n- LOG(FATAL) << \"Error in AdjustUTF8ToUCS2 \"\n- << CEscape(text8) << \" \"\n- << (int)(sp->begin() - text8.begin()) << \" \"\n- << (int)(sp->end() - text8.begin());\n- }\n- *sp = StringPiece(spbegin, spend - spbegin);\n- }\n- \n- // Rewrites *sp from begin a pointer into text16 (UCS-2)\n- // to being a pointer into text8 (equivalent text but in UTF-8).\n- static void AdjustUCS2ToUTF8(const StringPiece& text16, const StringPiece& text8,\n- StringPiece* sp) {\n- if (sp->begin() == NULL)\n- return;\n- \n- int nrune = 0;\n- int n;\n- Rune r;\n- const char* p = text8.begin();\n- const char* ep = text8.end();\n- const char* spbegin = NULL;\n- const char* spend = NULL;\n- for (;;) {\n- if (nrune == (sp->begin() - text16.begin())/2)\n- spbegin = p;\n- if (nrune == (sp->end() - text16.begin())/2)\n- spend = p;\n- if (p >= ep)\n- break;\n- n = chartorune(&r, p);\n- p += n;\n- nrune++;\n- }\n- if (text8.begin() != NULL && (spbegin == NULL || spend == NULL)) {\n- LOG(FATAL) << \"Error in AdjustUCS2ToUTF8 \"\n- << CEscape(text16) << \" \"\n- << (int)(sp->begin() - text16.begin()) << \" \"\n- << (int)(sp->end() - text16.begin());\n- }\n- *sp = StringPiece(spbegin, spend - spbegin);\n- }\n- \n // Runs a single search using the named engine type.\n // This interface hides all the irregularities of the various\n // engine interfaces from the rest of this file.\nre2/testing/tester.cc#12:393,411 - re2/testing/tester.cc#13:294,300\n \n StringPiece text = orig_text;\n StringPiece context = orig_context;\n- bool ucs2 = false;\n \n- if ((flags() & Regexp::UCS2) && type != kEnginePCRE) {\n- if (!ConvertUTF8ToUCS2(orig_context, &context)) {\n- result->skipped = true;\n- return;\n- }\n- \n- // Rewrite context to refer to new text.\n- AdjustUTF8ToUCS2(orig_context, context, &text);\n- ucs2 = true;\n- }\n- \n switch (type) {\n default:\n LOG(FATAL) << \"Bad RunSearch type: \" << (int)type;\nre2/testing/tester.cc#12:557,577 - re2/testing/tester.cc#13:446,451\n }\n }\n \n- // If we did UCS-2 matching, rewrite the matches to refer\n- // to the original UTF-8 text.\n- if (ucs2) {\n- if (result->matched) {\n- if (result->have_submatch0) {\n- AdjustUCS2ToUTF8(context, orig_context, &result->submatch[0]);\n- } else if (result->have_submatch) {\n- for (int i = 0; i < nsubmatch; i++) {\n- AdjustUCS2ToUTF8(context, orig_context, &result->submatch[i]);\n- }\n- }\n- }\n- delete[] context.begin();\n- }\n- \n if (!result->matched)\n memset(result->submatch, 0, sizeof result->submatch);\n }\nre2/testing/tester.cc#12:596,617 - re2/testing/tester.cc#13:470,475\n return true;\n }\n \n- // Check whether text uses only Unicode points <= 0xFFFF\n- // (in the BMP).\n- static bool IsBMP(const StringPiece& text) {\n- const char* p = text.begin();\n- const char* ep = text.end();\n- while (p < ep) {\n- if (!fullrune(p, ep - p))\n- return false;\n- Rune r;\n- p += chartorune(&r, p);\n- if (r > 0xFFFF)\n- return false;\n- }\n- return true;\n- }\n- \n // Runs a single test.\n bool TestInstance::RunCase(const StringPiece& text, const StringPiece& context,\n Prog::Anchor anchor) {\nre2/testing/tester.cc#12:619,625 - re2/testing/tester.cc#13:477,483\n Result correct;\n RunSearch(kEngineBacktrack, text, context, anchor, &correct);\n if (correct.skipped) {\n- if (regexp_ == NULL || !IsBMP(context)) // okay to skip in UCS-2 mode\n+ if (regexp_ == NULL)\n return true;\n LOG(ERROR) << \"Skipped backtracking! \" << CEscape(regexp_str_)\n << \" \" << FormatMode(flags_);\n" }, { "path": "util/malloc_counter.h", "content": "// Copyright 2009 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef UTIL_MALLOC_COUNTER_H_\n#define UTIL_MALLOC_COUNTER_H_\n\nnamespace testing {\nclass MallocCounter {\n public:\n MallocCounter(int x) {}\n static const int THIS_THREAD_ONLY = 0;\n long long HeapGrowth() { return 0; }\n long long PeakHeapGrowth() { return 0; }\n void Reset() {}\n};\n} // namespace testing\n\n#endif // UTIL_MALLOC_COUNTER_H_\n" }, { "path": "util/pcre.cc", "content": "// Copyright 2003-2009 Google Inc. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// This is a variant of PCRE's pcrecpp.cc, originally written at Google.\n// The main changes are the addition of the HitLimit method and\n// compilation as PCRE in namespace re2.\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"absl/flags/flag.h\"\n#include \"absl/log/absl_check.h\"\n#include \"absl/log/absl_log.h\"\n#include \"absl/strings/str_format.h\"\n#include \"util/pcre.h\"\n\n// Silence warnings about the wacky formatting in the operator() functions.\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Wmisleading-indentation\"\n#endif\n\n#define PCREPORT(level) ABSL_LOG(level)\n\n// Default PCRE limits.\n// Defaults chosen to allow a plausible amount of CPU and\n// not exceed main thread stacks. Note that other threads\n// often have smaller stacks, and therefore tightening\n// regexp_stack_limit may frequently be necessary.\nABSL_FLAG(int, regexp_stack_limit, 256 << 10,\n \"default PCRE stack limit (bytes)\");\nABSL_FLAG(int, regexp_match_limit, 1000000,\n \"default PCRE match limit (function calls)\");\n\n#ifndef USEPCRE\n\n// Fake just enough of the PCRE API to allow this file to build. :)\n\nstruct pcre_extra {\n int flags;\n int match_limit;\n int match_limit_recursion;\n};\n\n#define PCRE_EXTRA_MATCH_LIMIT 0\n#define PCRE_EXTRA_MATCH_LIMIT_RECURSION 0\n#define PCRE_ANCHORED 0\n#define PCRE_NOTEMPTY 0\n#define PCRE_ERROR_NOMATCH 1\n#define PCRE_ERROR_MATCHLIMIT 2\n#define PCRE_ERROR_RECURSIONLIMIT 3\n#define PCRE_INFO_CAPTURECOUNT 0\n\nvoid pcre_free(void*) {\n}\n\npcre* pcre_compile(const char*, int, const char**, int*, const unsigned char*) {\n return NULL;\n}\n\nint pcre_exec(const pcre*, const pcre_extra*, const char*, int, int, int, int*, int) {\n return 0;\n}\n\nint pcre_fullinfo(const pcre*, const pcre_extra*, int, void*) {\n return 0;\n}\n\n#endif\n\nnamespace re2 {\n\n// Maximum number of args we can set\nstatic const int kMaxArgs = 16;\nstatic const int kVecSize = (1 + kMaxArgs) * 3; // results + PCRE workspace\n\n// Approximate size of a recursive invocation of PCRE's\n// internal \"match()\" frame. This varies depending on the\n// compiler and architecture, of course, so the constant is\n// just a conservative estimate. To find the exact number,\n// run regexp_unittest with --regexp_stack_limit=0 under\n// a debugger and look at the frames when it crashes.\n// The exact frame size was 656 in production on 2008/02/03.\nstatic const int kPCREFrameSize = 700;\n\n// Special name for missing C++ arguments.\nPCRE::Arg PCRE::no_more_args((void*)NULL);\n\nconst PCRE::PartialMatchFunctor PCRE::PartialMatch = { };\nconst PCRE::FullMatchFunctor PCRE::FullMatch = { } ;\nconst PCRE::ConsumeFunctor PCRE::Consume = { };\nconst PCRE::FindAndConsumeFunctor PCRE::FindAndConsume = { };\n\n// If a regular expression has no error, its error_ field points here\nstatic const std::string empty_string;\n\nvoid PCRE::Init(const char* pattern, Option options, int match_limit,\n int stack_limit, bool report_errors) {\n pattern_ = pattern;\n options_ = options;\n match_limit_ = match_limit;\n stack_limit_ = stack_limit;\n hit_limit_ = false;\n error_ = &empty_string;\n report_errors_ = report_errors;\n re_full_ = NULL;\n re_partial_ = NULL;\n\n if (options & ~(EnabledCompileOptions | EnabledExecOptions)) {\n error_ = new std::string(\"illegal regexp option\");\n PCREPORT(ERROR)\n << \"Error compiling '\" << pattern << \"': illegal regexp option\";\n } else {\n re_partial_ = Compile(UNANCHORED);\n if (re_partial_ != NULL) {\n re_full_ = Compile(ANCHOR_BOTH);\n }\n }\n}\n\nPCRE::PCRE(const char* pattern) {\n Init(pattern, None, 0, 0, true);\n}\nPCRE::PCRE(const char* pattern, Option option) {\n Init(pattern, option, 0, 0, true);\n}\nPCRE::PCRE(const std::string& pattern) {\n Init(pattern.c_str(), None, 0, 0, true);\n}\nPCRE::PCRE(const std::string& pattern, Option option) {\n Init(pattern.c_str(), option, 0, 0, true);\n}\nPCRE::PCRE(const std::string& pattern, const PCRE_Options& re_option) {\n Init(pattern.c_str(), re_option.option(), re_option.match_limit(),\n re_option.stack_limit(), re_option.report_errors());\n}\n\nPCRE::PCRE(const char *pattern, const PCRE_Options& re_option) {\n Init(pattern, re_option.option(), re_option.match_limit(),\n re_option.stack_limit(), re_option.report_errors());\n}\n\nPCRE::~PCRE() {\n if (re_full_ != NULL) pcre_free(re_full_);\n if (re_partial_ != NULL) pcre_free(re_partial_);\n if (error_ != &empty_string) delete error_;\n}\n\npcre* PCRE::Compile(Anchor anchor) {\n // Special treatment for anchoring. This is needed because at\n // runtime pcre only provides an option for anchoring at the\n // beginning of a string.\n //\n // There are three types of anchoring we want:\n // UNANCHORED Compile the original pattern, and use\n // a pcre unanchored match.\n // ANCHOR_START Compile the original pattern, and use\n // a pcre anchored match.\n // ANCHOR_BOTH Tack a \"\\z\" to the end of the original pattern\n // and use a pcre anchored match.\n\n const char* error = \"\";\n int eoffset;\n pcre* re;\n if (anchor != ANCHOR_BOTH) {\n re = pcre_compile(pattern_.c_str(),\n (options_ & EnabledCompileOptions),\n &error, &eoffset, NULL);\n } else {\n // Tack a '\\z' at the end of PCRE. Parenthesize it first so that\n // the '\\z' applies to all top-level alternatives in the regexp.\n std::string wrapped = \"(?:\"; // A non-counting grouping operator\n wrapped += pattern_;\n wrapped += \")\\\\z\";\n re = pcre_compile(wrapped.c_str(),\n (options_ & EnabledCompileOptions),\n &error, &eoffset, NULL);\n }\n if (re == NULL) {\n if (error_ == &empty_string) error_ = new std::string(error);\n PCREPORT(ERROR) << \"Error compiling '\" << pattern_ << \"': \" << error;\n }\n return re;\n}\n\n/***** Convenience interfaces *****/\n\nbool PCRE::FullMatchFunctor::operator()(\n absl::string_view text, const PCRE& re, const Arg& a0, const Arg& a1,\n const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, const Arg& a6,\n const Arg& a7, const Arg& a8, const Arg& a9, const Arg& a10, const Arg& a11,\n const Arg& a12, const Arg& a13, const Arg& a14, const Arg& a15) const {\n const Arg* args[kMaxArgs];\n int n = 0;\n if (&a0 == &no_more_args) goto done; args[n++] = &a0;\n if (&a1 == &no_more_args) goto done; args[n++] = &a1;\n if (&a2 == &no_more_args) goto done; args[n++] = &a2;\n if (&a3 == &no_more_args) goto done; args[n++] = &a3;\n if (&a4 == &no_more_args) goto done; args[n++] = &a4;\n if (&a5 == &no_more_args) goto done; args[n++] = &a5;\n if (&a6 == &no_more_args) goto done; args[n++] = &a6;\n if (&a7 == &no_more_args) goto done; args[n++] = &a7;\n if (&a8 == &no_more_args) goto done; args[n++] = &a8;\n if (&a9 == &no_more_args) goto done; args[n++] = &a9;\n if (&a10 == &no_more_args) goto done; args[n++] = &a10;\n if (&a11 == &no_more_args) goto done; args[n++] = &a11;\n if (&a12 == &no_more_args) goto done; args[n++] = &a12;\n if (&a13 == &no_more_args) goto done; args[n++] = &a13;\n if (&a14 == &no_more_args) goto done; args[n++] = &a14;\n if (&a15 == &no_more_args) goto done; args[n++] = &a15;\ndone:\n\n size_t consumed;\n int vec[kVecSize] = {};\n return re.DoMatchImpl(text, ANCHOR_BOTH, &consumed, args, n, vec, kVecSize);\n}\n\nbool PCRE::PartialMatchFunctor::operator()(\n absl::string_view text, const PCRE& re, const Arg& a0, const Arg& a1,\n const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, const Arg& a6,\n const Arg& a7, const Arg& a8, const Arg& a9, const Arg& a10, const Arg& a11,\n const Arg& a12, const Arg& a13, const Arg& a14, const Arg& a15) const {\n const Arg* args[kMaxArgs];\n int n = 0;\n if (&a0 == &no_more_args) goto done; args[n++] = &a0;\n if (&a1 == &no_more_args) goto done; args[n++] = &a1;\n if (&a2 == &no_more_args) goto done; args[n++] = &a2;\n if (&a3 == &no_more_args) goto done; args[n++] = &a3;\n if (&a4 == &no_more_args) goto done; args[n++] = &a4;\n if (&a5 == &no_more_args) goto done; args[n++] = &a5;\n if (&a6 == &no_more_args) goto done; args[n++] = &a6;\n if (&a7 == &no_more_args) goto done; args[n++] = &a7;\n if (&a8 == &no_more_args) goto done; args[n++] = &a8;\n if (&a9 == &no_more_args) goto done; args[n++] = &a9;\n if (&a10 == &no_more_args) goto done; args[n++] = &a10;\n if (&a11 == &no_more_args) goto done; args[n++] = &a11;\n if (&a12 == &no_more_args) goto done; args[n++] = &a12;\n if (&a13 == &no_more_args) goto done; args[n++] = &a13;\n if (&a14 == &no_more_args) goto done; args[n++] = &a14;\n if (&a15 == &no_more_args) goto done; args[n++] = &a15;\ndone:\n\n size_t consumed;\n int vec[kVecSize] = {};\n return re.DoMatchImpl(text, UNANCHORED, &consumed, args, n, vec, kVecSize);\n}\n\nbool PCRE::ConsumeFunctor::operator()(\n absl::string_view* input, const PCRE& pattern, const Arg& a0, const Arg& a1,\n const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, const Arg& a6,\n const Arg& a7, const Arg& a8, const Arg& a9, const Arg& a10, const Arg& a11,\n const Arg& a12, const Arg& a13, const Arg& a14, const Arg& a15) const {\n const Arg* args[kMaxArgs];\n int n = 0;\n if (&a0 == &no_more_args) goto done; args[n++] = &a0;\n if (&a1 == &no_more_args) goto done; args[n++] = &a1;\n if (&a2 == &no_more_args) goto done; args[n++] = &a2;\n if (&a3 == &no_more_args) goto done; args[n++] = &a3;\n if (&a4 == &no_more_args) goto done; args[n++] = &a4;\n if (&a5 == &no_more_args) goto done; args[n++] = &a5;\n if (&a6 == &no_more_args) goto done; args[n++] = &a6;\n if (&a7 == &no_more_args) goto done; args[n++] = &a7;\n if (&a8 == &no_more_args) goto done; args[n++] = &a8;\n if (&a9 == &no_more_args) goto done; args[n++] = &a9;\n if (&a10 == &no_more_args) goto done; args[n++] = &a10;\n if (&a11 == &no_more_args) goto done; args[n++] = &a11;\n if (&a12 == &no_more_args) goto done; args[n++] = &a12;\n if (&a13 == &no_more_args) goto done; args[n++] = &a13;\n if (&a14 == &no_more_args) goto done; args[n++] = &a14;\n if (&a15 == &no_more_args) goto done; args[n++] = &a15;\ndone:\n\n size_t consumed;\n int vec[kVecSize] = {};\n if (pattern.DoMatchImpl(*input, ANCHOR_START, &consumed,\n args, n, vec, kVecSize)) {\n input->remove_prefix(consumed);\n return true;\n } else {\n return false;\n }\n}\n\nbool PCRE::FindAndConsumeFunctor::operator()(\n absl::string_view* input, const PCRE& pattern, const Arg& a0, const Arg& a1,\n const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, const Arg& a6,\n const Arg& a7, const Arg& a8, const Arg& a9, const Arg& a10, const Arg& a11,\n const Arg& a12, const Arg& a13, const Arg& a14, const Arg& a15) const {\n const Arg* args[kMaxArgs];\n int n = 0;\n if (&a0 == &no_more_args) goto done; args[n++] = &a0;\n if (&a1 == &no_more_args) goto done; args[n++] = &a1;\n if (&a2 == &no_more_args) goto done; args[n++] = &a2;\n if (&a3 == &no_more_args) goto done; args[n++] = &a3;\n if (&a4 == &no_more_args) goto done; args[n++] = &a4;\n if (&a5 == &no_more_args) goto done; args[n++] = &a5;\n if (&a6 == &no_more_args) goto done; args[n++] = &a6;\n if (&a7 == &no_more_args) goto done; args[n++] = &a7;\n if (&a8 == &no_more_args) goto done; args[n++] = &a8;\n if (&a9 == &no_more_args) goto done; args[n++] = &a9;\n if (&a10 == &no_more_args) goto done; args[n++] = &a10;\n if (&a11 == &no_more_args) goto done; args[n++] = &a11;\n if (&a12 == &no_more_args) goto done; args[n++] = &a12;\n if (&a13 == &no_more_args) goto done; args[n++] = &a13;\n if (&a14 == &no_more_args) goto done; args[n++] = &a14;\n if (&a15 == &no_more_args) goto done; args[n++] = &a15;\ndone:\n\n size_t consumed;\n int vec[kVecSize] = {};\n if (pattern.DoMatchImpl(*input, UNANCHORED, &consumed,\n args, n, vec, kVecSize)) {\n input->remove_prefix(consumed);\n return true;\n } else {\n return false;\n }\n}\n\nbool PCRE::Replace(std::string* str, const PCRE& pattern,\n absl::string_view rewrite) {\n int vec[kVecSize] = {};\n int matches = pattern.TryMatch(*str, 0, UNANCHORED, true, vec, kVecSize);\n if (matches == 0)\n return false;\n\n std::string s;\n if (!pattern.Rewrite(&s, rewrite, *str, vec, matches))\n return false;\n\n assert(vec[0] >= 0);\n assert(vec[1] >= 0);\n str->replace(vec[0], vec[1] - vec[0], s);\n return true;\n}\n\nint PCRE::GlobalReplace(std::string* str, const PCRE& pattern,\n absl::string_view rewrite) {\n int count = 0;\n int vec[kVecSize] = {};\n std::string out;\n size_t start = 0;\n bool last_match_was_empty_string = false;\n\n while (start <= str->size()) {\n // If the previous match was for the empty string, we shouldn't\n // just match again: we'll match in the same way and get an\n // infinite loop. Instead, we do the match in a special way:\n // anchored -- to force another try at the same position --\n // and with a flag saying that this time, ignore empty matches.\n // If this special match returns, that means there's a non-empty\n // match at this position as well, and we can continue. If not,\n // we do what perl does, and just advance by one.\n // Notice that perl prints '@@@' for this;\n // perl -le '$_ = \"aa\"; s/b*|aa/@/g; print'\n int matches;\n if (last_match_was_empty_string) {\n matches = pattern.TryMatch(*str, start, ANCHOR_START, false,\n vec, kVecSize);\n if (matches <= 0) {\n if (start < str->size())\n out.push_back((*str)[start]);\n start++;\n last_match_was_empty_string = false;\n continue;\n }\n } else {\n matches = pattern.TryMatch(*str, start, UNANCHORED, true,\n vec, kVecSize);\n if (matches <= 0)\n break;\n }\n size_t matchstart = vec[0], matchend = vec[1];\n assert(matchstart >= start);\n assert(matchend >= matchstart);\n\n out.append(*str, start, matchstart - start);\n pattern.Rewrite(&out, rewrite, *str, vec, matches);\n start = matchend;\n count++;\n last_match_was_empty_string = (matchstart == matchend);\n }\n\n if (count == 0)\n return 0;\n\n if (start < str->size())\n out.append(*str, start, str->size() - start);\n using std::swap;\n swap(out, *str);\n return count;\n}\n\nbool PCRE::Extract(absl::string_view text, const PCRE& pattern,\n absl::string_view rewrite, std::string* out) {\n int vec[kVecSize] = {};\n int matches = pattern.TryMatch(text, 0, UNANCHORED, true, vec, kVecSize);\n if (matches == 0)\n return false;\n out->clear();\n return pattern.Rewrite(out, rewrite, text, vec, matches);\n}\n\nstd::string PCRE::QuoteMeta(absl::string_view unquoted) {\n std::string result;\n result.reserve(unquoted.size() << 1);\n\n // Escape any ascii character not in [A-Za-z_0-9].\n //\n // Note that it's legal to escape a character even if it has no\n // special meaning in a regular expression -- so this function does\n // that. (This also makes it identical to the perl function of the\n // same name except for the null-character special case;\n // see `perldoc -f quotemeta`.)\n for (size_t ii = 0; ii < unquoted.size(); ++ii) {\n // Note that using 'isalnum' here raises the benchmark time from\n // 32ns to 58ns:\n if ((unquoted[ii] < 'a' || unquoted[ii] > 'z') &&\n (unquoted[ii] < 'A' || unquoted[ii] > 'Z') &&\n (unquoted[ii] < '0' || unquoted[ii] > '9') &&\n unquoted[ii] != '_' &&\n // If this is the part of a UTF8 or Latin1 character, we need\n // to copy this byte without escaping. Experimentally this is\n // what works correctly with the regexp library.\n !(unquoted[ii] & 128)) {\n if (unquoted[ii] == '\\0') { // Special handling for null chars.\n // Can't use \"\\\\0\" since the next character might be a digit.\n result += \"\\\\x00\";\n continue;\n }\n result += '\\\\';\n }\n result += unquoted[ii];\n }\n\n return result;\n}\n\n/***** Actual matching and rewriting code *****/\n\nbool PCRE::HitLimit() {\n return hit_limit_ != 0;\n}\n\nvoid PCRE::ClearHitLimit() {\n hit_limit_ = 0;\n}\n\nint PCRE::TryMatch(absl::string_view text, size_t startpos, Anchor anchor,\n bool empty_ok, int* vec, int vecsize) const {\n pcre* re = (anchor == ANCHOR_BOTH) ? re_full_ : re_partial_;\n if (re == NULL) {\n PCREPORT(ERROR) << \"Matching against invalid re: \" << *error_;\n return 0;\n }\n\n int match_limit = match_limit_;\n if (match_limit <= 0) {\n match_limit = absl::GetFlag(FLAGS_regexp_match_limit);\n }\n\n int stack_limit = stack_limit_;\n if (stack_limit <= 0) {\n stack_limit = absl::GetFlag(FLAGS_regexp_stack_limit);\n }\n\n pcre_extra extra = { 0 };\n if (match_limit > 0) {\n extra.flags |= PCRE_EXTRA_MATCH_LIMIT;\n extra.match_limit = match_limit;\n }\n if (stack_limit > 0) {\n extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION;\n extra.match_limit_recursion = stack_limit / kPCREFrameSize;\n }\n\n int options = 0;\n if (anchor != UNANCHORED)\n options |= PCRE_ANCHORED;\n if (!empty_ok)\n options |= PCRE_NOTEMPTY;\n\n int rc = pcre_exec(re, // The regular expression object\n &extra,\n (text.data() == NULL) ? \"\" : text.data(),\n static_cast(text.size()),\n static_cast(startpos),\n options,\n vec,\n vecsize);\n\n // Handle errors\n if (rc == 0) {\n // pcre_exec() returns 0 as a special case when the number of\n // capturing subpatterns exceeds the size of the vector.\n // When this happens, there is a match and the output vector\n // is filled, but we miss out on the positions of the extra subpatterns.\n rc = vecsize / 2;\n } else if (rc < 0) {\n switch (rc) {\n case PCRE_ERROR_NOMATCH:\n return 0;\n case PCRE_ERROR_MATCHLIMIT:\n // Writing to hit_limit is not safe if multiple threads\n // are using the PCRE, but the flag is only intended\n // for use by unit tests anyway, so we let it go.\n hit_limit_ = true;\n PCREPORT(WARNING) << \"Exceeded match limit of \" << match_limit\n << \" when matching '\" << pattern_ << \"'\"\n << \" against text that is \" << text.size() << \" bytes.\";\n return 0;\n case PCRE_ERROR_RECURSIONLIMIT:\n // See comment about hit_limit above.\n hit_limit_ = true;\n PCREPORT(WARNING) << \"Exceeded stack limit of \" << stack_limit\n << \" when matching '\" << pattern_ << \"'\"\n << \" against text that is \" << text.size() << \" bytes.\";\n return 0;\n default:\n // There are other return codes from pcre.h :\n // PCRE_ERROR_NULL (-2)\n // PCRE_ERROR_BADOPTION (-3)\n // PCRE_ERROR_BADMAGIC (-4)\n // PCRE_ERROR_UNKNOWN_NODE (-5)\n // PCRE_ERROR_NOMEMORY (-6)\n // PCRE_ERROR_NOSUBSTRING (-7)\n // ...\n PCREPORT(ERROR) << \"Unexpected return code: \" << rc\n << \" when matching '\" << pattern_ << \"'\"\n << \", re=\" << re\n << \", text=\" << text\n << \", vec=\" << vec\n << \", vecsize=\" << vecsize;\n return 0;\n }\n }\n\n return rc;\n}\n\nbool PCRE::DoMatchImpl(absl::string_view text, Anchor anchor, size_t* consumed,\n const Arg* const* args, int n, int* vec,\n int vecsize) const {\n assert((1 + n) * 3 <= vecsize); // results + PCRE workspace\n if (NumberOfCapturingGroups() < n) {\n // RE has fewer capturing groups than number of Arg pointers passed in.\n return false;\n }\n\n int matches = TryMatch(text, 0, anchor, true, vec, vecsize);\n assert(matches >= 0); // TryMatch never returns negatives\n if (matches == 0)\n return false;\n\n *consumed = vec[1];\n\n if (n == 0 || args == NULL) {\n // We are not interested in results\n return true;\n }\n\n // If we got here, we must have matched the whole pattern.\n // We do not need (can not do) any more checks on the value of 'matches' here\n // -- see the comment for TryMatch.\n for (int i = 0; i < n; i++) {\n const int start = vec[2*(i+1)];\n const int limit = vec[2*(i+1)+1];\n\n // Avoid invoking undefined behavior when text.data() happens\n // to be null and start happens to be -1, the latter being the\n // case for an unmatched subexpression. Even if text.data() is\n // not null, pointing one byte before was a longstanding bug.\n const char* addr = NULL;\n if (start != -1) {\n addr = text.data() + start;\n }\n\n if (!args[i]->Parse(addr, limit-start)) {\n // TODO: Should we indicate what the error was?\n return false;\n }\n }\n\n return true;\n}\n\nbool PCRE::DoMatch(absl::string_view text, Anchor anchor, size_t* consumed,\n const Arg* const args[], int n) const {\n assert(n >= 0);\n const int vecsize = (1 + n) * 3; // results + PCRE workspace\n // (as for kVecSize)\n int* vec = new int[vecsize];\n bool b = DoMatchImpl(text, anchor, consumed, args, n, vec, vecsize);\n delete[] vec;\n return b;\n}\n\nbool PCRE::Rewrite(std::string* out, absl::string_view rewrite,\n absl::string_view text, int* vec, int veclen) const {\n int number_of_capturing_groups = NumberOfCapturingGroups();\n for (const char *s = rewrite.data(), *end = s + rewrite.size();\n s < end; s++) {\n int c = *s;\n if (c == '\\\\') {\n c = *++s;\n if (isdigit(c)) {\n int n = (c - '0');\n if (n >= veclen) {\n if (n <= number_of_capturing_groups) {\n // unmatched optional capturing group. treat\n // its value as empty string; i.e., nothing to append.\n } else {\n PCREPORT(ERROR) << \"requested group \" << n\n << \" in regexp \" << rewrite.data();\n return false;\n }\n }\n int start = vec[2 * n];\n if (start >= 0)\n out->append(text.data() + start, vec[2 * n + 1] - start);\n } else if (c == '\\\\') {\n out->push_back('\\\\');\n } else {\n PCREPORT(ERROR) << \"invalid rewrite pattern: \" << rewrite.data();\n return false;\n }\n } else {\n out->push_back(c);\n }\n }\n return true;\n}\n\nbool PCRE::CheckRewriteString(absl::string_view rewrite,\n std::string* error) const {\n int max_token = -1;\n for (const char *s = rewrite.data(), *end = s + rewrite.size();\n s < end; s++) {\n int c = *s;\n if (c != '\\\\') {\n continue;\n }\n if (++s == end) {\n *error = \"Rewrite schema error: '\\\\' not allowed at end.\";\n return false;\n }\n c = *s;\n if (c == '\\\\') {\n continue;\n }\n if (!isdigit(c)) {\n *error = \"Rewrite schema error: \"\n \"'\\\\' must be followed by a digit or '\\\\'.\";\n return false;\n }\n int n = (c - '0');\n if (max_token < n) {\n max_token = n;\n }\n }\n\n if (max_token > NumberOfCapturingGroups()) {\n *error = absl::StrFormat(\n \"Rewrite schema requests %d matches, but the regexp only has %d \"\n \"parenthesized subexpressions.\",\n max_token, NumberOfCapturingGroups());\n return false;\n }\n return true;\n}\n\n// Return the number of capturing subpatterns, or -1 if the\n// regexp wasn't valid on construction.\nint PCRE::NumberOfCapturingGroups() const {\n if (re_partial_ == NULL) return -1;\n\n int result;\n int rc = pcre_fullinfo(re_partial_, // The regular expression object\n NULL, // We did not study the pattern\n PCRE_INFO_CAPTURECOUNT,\n &result);\n if (rc != 0) {\n PCREPORT(ERROR) << \"Unexpected return code: \" << rc;\n return -1;\n }\n return result;\n}\n\n\n/***** Parsers for various types *****/\n\nbool PCRE::Arg::parse_null(const char* str, size_t n, void* dest) {\n // We fail if somebody asked us to store into a non-NULL void* pointer\n return (dest == NULL);\n}\n\nbool PCRE::Arg::parse_string(const char* str, size_t n, void* dest) {\n if (dest == NULL) return true;\n reinterpret_cast(dest)->assign(str, n);\n return true;\n}\n\nbool PCRE::Arg::parse_string_view(const char* str, size_t n, void* dest) {\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = absl::string_view(str, n);\n return true;\n}\n\nbool PCRE::Arg::parse_char(const char* str, size_t n, void* dest) {\n if (n != 1) return false;\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = str[0];\n return true;\n}\n\nbool PCRE::Arg::parse_schar(const char* str, size_t n, void* dest) {\n if (n != 1) return false;\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = str[0];\n return true;\n}\n\nbool PCRE::Arg::parse_uchar(const char* str, size_t n, void* dest) {\n if (n != 1) return false;\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = str[0];\n return true;\n}\n\n// Largest number spec that we are willing to parse\nstatic const int kMaxNumberLength = 32;\n\n// PCREQUIPCRES \"buf\" must have length at least kMaxNumberLength+1\n// PCREQUIPCRES \"n > 0\"\n// Copies \"str\" into \"buf\" and null-terminates if necessary.\n// Returns one of:\n// a. \"str\" if no termination is needed\n// b. \"buf\" if the string was copied and null-terminated\n// c. \"\" if the input was invalid and has no hope of being parsed\nstatic const char* TerminateNumber(char* buf, const char* str, size_t n) {\n if ((n > 0) && isspace(*str)) {\n // We are less forgiving than the strtoxxx() routines and do not\n // allow leading spaces.\n return \"\";\n }\n\n // See if the character right after the input text may potentially\n // look like a digit.\n if (isdigit(str[n]) ||\n ((str[n] >= 'a') && (str[n] <= 'f')) ||\n ((str[n] >= 'A') && (str[n] <= 'F'))) {\n if (n > kMaxNumberLength) return \"\"; // Input too big to be a valid number\n memcpy(buf, str, n);\n buf[n] = '\\0';\n return buf;\n } else {\n // We can parse right out of the supplied string, so return it.\n return str;\n }\n}\n\nbool PCRE::Arg::parse_long_radix(const char* str,\n size_t n,\n void* dest,\n int radix) {\n if (n == 0) return false;\n char buf[kMaxNumberLength+1];\n str = TerminateNumber(buf, str, n);\n char* end;\n errno = 0;\n long r = strtol(str, &end, radix);\n if (end != str + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = r;\n return true;\n}\n\nbool PCRE::Arg::parse_ulong_radix(const char* str,\n size_t n,\n void* dest,\n int radix) {\n if (n == 0) return false;\n char buf[kMaxNumberLength+1];\n str = TerminateNumber(buf, str, n);\n if (str[0] == '-') {\n // strtoul() will silently accept negative numbers and parse\n // them. This module is more strict and treats them as errors.\n return false;\n }\n\n char* end;\n errno = 0;\n unsigned long r = strtoul(str, &end, radix);\n if (end != str + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = r;\n return true;\n}\n\nbool PCRE::Arg::parse_short_radix(const char* str,\n size_t n,\n void* dest,\n int radix) {\n long r;\n if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse\n if ((short)r != r) return false; // Out of range\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = (short)r;\n return true;\n}\n\nbool PCRE::Arg::parse_ushort_radix(const char* str,\n size_t n,\n void* dest,\n int radix) {\n unsigned long r;\n if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse\n if ((unsigned short)r != r) return false; // Out of range\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = (unsigned short)r;\n return true;\n}\n\nbool PCRE::Arg::parse_int_radix(const char* str,\n size_t n,\n void* dest,\n int radix) {\n long r;\n if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse\n if ((int)r != r) return false; // Out of range\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = (int)r;\n return true;\n}\n\nbool PCRE::Arg::parse_uint_radix(const char* str,\n size_t n,\n void* dest,\n int radix) {\n unsigned long r;\n if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse\n if ((unsigned int)r != r) return false; // Out of range\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = (unsigned int)r;\n return true;\n}\n\nbool PCRE::Arg::parse_longlong_radix(const char* str,\n size_t n,\n void* dest,\n int radix) {\n if (n == 0) return false;\n char buf[kMaxNumberLength+1];\n str = TerminateNumber(buf, str, n);\n char* end;\n errno = 0;\n long long r = strtoll(str, &end, radix);\n if (end != str + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = r;\n return true;\n}\n\nbool PCRE::Arg::parse_ulonglong_radix(const char* str,\n size_t n,\n void* dest,\n int radix) {\n if (n == 0) return false;\n char buf[kMaxNumberLength+1];\n str = TerminateNumber(buf, str, n);\n if (str[0] == '-') {\n // strtoull() will silently accept negative numbers and parse\n // them. This module is more strict and treats them as errors.\n return false;\n }\n char* end;\n errno = 0;\n unsigned long long r = strtoull(str, &end, radix);\n if (end != str + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n *(reinterpret_cast(dest)) = r;\n return true;\n}\n\nstatic bool parse_double_float(const char* str, size_t n, bool isfloat,\n void* dest) {\n if (n == 0) return false;\n static const int kMaxLength = 200;\n char buf[kMaxLength];\n if (n >= kMaxLength) return false;\n memcpy(buf, str, n);\n buf[n] = '\\0';\n char* end;\n errno = 0;\n double r;\n if (isfloat) {\n r = strtof(buf, &end);\n } else {\n r = strtod(buf, &end);\n }\n if (end != buf + n) return false; // Leftover junk\n if (errno) return false;\n if (dest == NULL) return true;\n if (isfloat) {\n *(reinterpret_cast(dest)) = (float)r;\n } else {\n *(reinterpret_cast(dest)) = r;\n }\n return true;\n}\n\nbool PCRE::Arg::parse_double(const char* str, size_t n, void* dest) {\n return parse_double_float(str, n, false, dest);\n}\n\nbool PCRE::Arg::parse_float(const char* str, size_t n, void* dest) {\n return parse_double_float(str, n, true, dest);\n}\n\n#define DEFINE_INTEGER_PARSER(name) \\\n bool PCRE::Arg::parse_##name(const char* str, size_t n, void* dest) { \\\n return parse_##name##_radix(str, n, dest, 10); \\\n } \\\n bool PCRE::Arg::parse_##name##_hex(const char* str, size_t n, void* dest) { \\\n return parse_##name##_radix(str, n, dest, 16); \\\n } \\\n bool PCRE::Arg::parse_##name##_octal(const char* str, size_t n, \\\n void* dest) { \\\n return parse_##name##_radix(str, n, dest, 8); \\\n } \\\n bool PCRE::Arg::parse_##name##_cradix(const char* str, size_t n, \\\n void* dest) { \\\n return parse_##name##_radix(str, n, dest, 0); \\\n }\n\nDEFINE_INTEGER_PARSER(short);\nDEFINE_INTEGER_PARSER(ushort);\nDEFINE_INTEGER_PARSER(int);\nDEFINE_INTEGER_PARSER(uint);\nDEFINE_INTEGER_PARSER(long);\nDEFINE_INTEGER_PARSER(ulong);\nDEFINE_INTEGER_PARSER(longlong);\nDEFINE_INTEGER_PARSER(ulonglong);\n\n#undef DEFINE_INTEGER_PARSER\n\n} // namespace re2\n" }, { "path": "util/pcre.h", "content": "// Copyright 2003-2010 Google Inc. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef UTIL_PCRE_H_\n#define UTIL_PCRE_H_\n\n// This is a variant of PCRE's pcrecpp.h, originally written at Google.\n// The main changes are the addition of the HitLimit method and\n// compilation as PCRE in namespace re2.\n\n// C++ interface to the pcre regular-expression library. PCRE supports\n// Perl-style regular expressions (with extensions like \\d, \\w, \\s,\n// ...).\n//\n// -----------------------------------------------------------------------\n// REGEXP SYNTAX:\n//\n// This module uses the pcre library and hence supports its syntax\n// for regular expressions:\n//\n// http://www.google.com/search?q=pcre\n//\n// The syntax is pretty similar to Perl's. For those not familiar\n// with Perl's regular expressions, here are some examples of the most\n// commonly used extensions:\n//\n// \"hello (\\\\w+) world\" -- \\w matches a \"word\" character\n// \"version (\\\\d+)\" -- \\d matches a digit\n// \"hello\\\\s+world\" -- \\s matches any whitespace character\n// \"\\\\b(\\\\w+)\\\\b\" -- \\b matches empty string at a word boundary\n// \"(?i)hello\" -- (?i) turns on case-insensitive matching\n// \"/\\\\*(.*?)\\\\*/\" -- .*? matches . minimum no. of times possible\n//\n// -----------------------------------------------------------------------\n// MATCHING INTERFACE:\n//\n// The \"FullMatch\" operation checks that supplied text matches a\n// supplied pattern exactly.\n//\n// Example: successful match\n// ABSL_CHECK(PCRE::FullMatch(\"hello\", \"h.*o\"));\n//\n// Example: unsuccessful match (requires full match):\n// ABSL_CHECK(!PCRE::FullMatch(\"hello\", \"e\"));\n//\n// -----------------------------------------------------------------------\n// UTF-8 AND THE MATCHING INTERFACE:\n//\n// By default, pattern and text are plain text, one byte per character.\n// The UTF8 flag, passed to the constructor, causes both pattern\n// and string to be treated as UTF-8 text, still a byte stream but\n// potentially multiple bytes per character. In practice, the text\n// is likelier to be UTF-8 than the pattern, but the match returned\n// may depend on the UTF8 flag, so always use it when matching\n// UTF8 text. E.g., \".\" will match one byte normally but with UTF8\n// set may match up to three bytes of a multi-byte character.\n//\n// Example:\n// PCRE re(utf8_pattern, PCRE::UTF8);\n// ABSL_CHECK(PCRE::FullMatch(utf8_string, re));\n//\n// -----------------------------------------------------------------------\n// MATCHING WITH SUBSTRING EXTRACTION:\n//\n// You can supply extra pointer arguments to extract matched substrings.\n//\n// Example: extracts \"ruby\" into \"s\" and 1234 into \"i\"\n// int i;\n// std::string s;\n// ABSL_CHECK(PCRE::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", &s, &i));\n//\n// Example: fails because string cannot be stored in integer\n// ABSL_CHECK(!PCRE::FullMatch(\"ruby\", \"(.*)\", &i));\n//\n// Example: fails because there aren't enough sub-patterns:\n// ABSL_CHECK(!PCRE::FullMatch(\"ruby:1234\", \"\\\\w+:\\\\d+\", &s));\n//\n// Example: does not try to extract any extra sub-patterns\n// ABSL_CHECK(PCRE::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", &s));\n//\n// Example: does not try to extract into NULL\n// ABSL_CHECK(PCRE::FullMatch(\"ruby:1234\", \"(\\\\w+):(\\\\d+)\", NULL, &i));\n//\n// Example: integer overflow causes failure\n// ABSL_CHECK(!PCRE::FullMatch(\"ruby:1234567891234\", \"\\\\w+:(\\\\d+)\", &i));\n//\n// -----------------------------------------------------------------------\n// PARTIAL MATCHES\n//\n// You can use the \"PartialMatch\" operation when you want the pattern\n// to match any substring of the text.\n//\n// Example: simple search for a string:\n// ABSL_CHECK(PCRE::PartialMatch(\"hello\", \"ell\"));\n//\n// Example: find first number in a string\n// int number;\n// ABSL_CHECK(PCRE::PartialMatch(\"x*100 + 20\", \"(\\\\d+)\", &number));\n// ABSL_CHECK_EQ(number, 100);\n//\n// -----------------------------------------------------------------------\n// PPCRE-COMPILED PCREGULAR EXPPCRESSIONS\n//\n// PCRE makes it easy to use any string as a regular expression, without\n// requiring a separate compilation step.\n//\n// If speed is of the essence, you can create a pre-compiled \"PCRE\"\n// object from the pattern and use it multiple times. If you do so,\n// you can typically parse text faster than with sscanf.\n//\n// Example: precompile pattern for faster matching:\n// PCRE pattern(\"h.*o\");\n// while (ReadLine(&str)) {\n// if (PCRE::FullMatch(str, pattern)) ...;\n// }\n//\n// -----------------------------------------------------------------------\n// SCANNING TEXT INCPCREMENTALLY\n//\n// The \"Consume\" operation may be useful if you want to repeatedly\n// match regular expressions at the front of a string and skip over\n// them as they match. This requires use of the string_view type,\n// which represents a sub-range of a real string.\n//\n// Example: read lines of the form \"var = value\" from a string.\n// std::string contents = ...; // Fill string somehow\n// absl::string_view input(contents); // Wrap a string_view around it\n//\n// std::string var;\n// int value;\n// while (PCRE::Consume(&input, \"(\\\\w+) = (\\\\d+)\\n\", &var, &value)) {\n// ...;\n// }\n//\n// Each successful call to \"Consume\" will set \"var/value\", and also\n// advance \"input\" so it points past the matched text. Note that if the\n// regular expression matches an empty string, input will advance\n// by 0 bytes. If the regular expression being used might match\n// an empty string, the loop body must check for this case and either\n// advance the string or break out of the loop.\n//\n// The \"FindAndConsume\" operation is similar to \"Consume\" but does not\n// anchor your match at the beginning of the string. For example, you\n// could extract all words from a string by repeatedly calling\n// PCRE::FindAndConsume(&input, \"(\\\\w+)\", &word)\n//\n// -----------------------------------------------------------------------\n// PARSING HEX/OCTAL/C-RADIX NUMBERS\n//\n// By default, if you pass a pointer to a numeric value, the\n// corresponding text is interpreted as a base-10 number. You can\n// instead wrap the pointer with a call to one of the operators Hex(),\n// Octal(), or CRadix() to interpret the text in another base. The\n// CRadix operator interprets C-style \"0\" (base-8) and \"0x\" (base-16)\n// prefixes, but defaults to base-10.\n//\n// Example:\n// int a, b, c, d;\n// ABSL_CHECK(PCRE::FullMatch(\"100 40 0100 0x40\", \"(.*) (.*) (.*) (.*)\",\n// Octal(&a), Hex(&b), CRadix(&c), CRadix(&d));\n// will leave 64 in a, b, c, and d.\n\n#include \"absl/strings/string_view.h\"\n\n#ifdef USEPCRE\n#include \nnamespace re2 {\nconst bool UsingPCRE = true;\n} // namespace re2\n#else\nstruct pcre; // opaque\nnamespace re2 {\nconst bool UsingPCRE = false;\n} // namespace re2\n#endif\n\n// To produce a DLL, CMake can automatically export code symbols,\n// but not data symbols, so we have to annotate those manually...\n#if defined(RE2_BUILD_TESTING_DLL)\n#define RE2_TESTING_DLL __declspec(dllexport)\n#elif defined(RE2_CONSUME_TESTING_DLL)\n#define RE2_TESTING_DLL __declspec(dllimport)\n#else\n#define RE2_TESTING_DLL\n#endif\n\nnamespace re2 {\n\nclass PCRE_Options;\n\n// Interface for regular expression matching. Also corresponds to a\n// pre-compiled regular expression. An \"PCRE\" object is safe for\n// concurrent use by multiple threads.\nclass PCRE {\n public:\n // We convert user-passed pointers into special Arg objects\n class Arg;\n\n // Marks end of arg list.\n // ONLY USE IN OPTIONAL ARG DEFAULTS.\n // DO NOT PASS EXPLICITLY.\n RE2_TESTING_DLL static Arg no_more_args;\n\n // Options are same value as those in pcre. We provide them here\n // to avoid users needing to include pcre.h and also to isolate\n // users from pcre should we change the underlying library.\n // Only those needed by Google programs are exposed here to\n // avoid collision with options employed internally by regexp.cc\n // Note that some options have equivalents that can be specified in\n // the regexp itself. For example, prefixing your regexp with\n // \"(?s)\" has the same effect as the PCRE_DOTALL option.\n enum Option {\n None = 0x0000,\n UTF8 = 0x0800, // == PCRE_UTF8\n EnabledCompileOptions = UTF8,\n EnabledExecOptions = 0x0000, // TODO: use to replace anchor flag\n };\n\n // We provide implicit conversions from strings so that users can\n // pass in a string or a \"const char*\" wherever an \"PCRE\" is expected.\n PCRE(const char* pattern);\n PCRE(const char* pattern, Option option);\n PCRE(const std::string& pattern);\n PCRE(const std::string& pattern, Option option);\n PCRE(const char *pattern, const PCRE_Options& re_option);\n PCRE(const std::string& pattern, const PCRE_Options& re_option);\n\n ~PCRE();\n\n // The string specification for this PCRE. E.g.\n // PCRE re(\"ab*c?d+\");\n // re.pattern(); // \"ab*c?d+\"\n const std::string& pattern() const { return pattern_; }\n\n // If PCRE could not be created properly, returns an error string.\n // Else returns the empty string.\n const std::string& error() const { return *error_; }\n\n // Whether the PCRE has hit a match limit during execution.\n // Not thread safe. Intended only for testing.\n // If hitting match limits is a problem,\n // you should be using PCRE2 (re2/re2.h)\n // instead of checking this flag.\n bool HitLimit();\n void ClearHitLimit();\n\n /***** The useful part: the matching interface *****/\n\n // Matches \"text\" against \"pattern\". If pointer arguments are\n // supplied, copies matched sub-patterns into them.\n //\n // You can pass in a \"const char*\" or a \"std::string\" for \"text\".\n // You can pass in a \"const char*\" or a \"std::string\" or a \"PCRE\" for \"pattern\".\n //\n // The provided pointer arguments can be pointers to any scalar numeric\n // type, or one of:\n // std::string (matched piece is copied to string)\n // absl::string_view (string_view is mutated to point to matched piece)\n // T (\"bool T::ParseFrom(const char*, size_t)\" must exist)\n // (void*)NULL (the corresponding matched sub-pattern is not copied)\n //\n // Returns true iff all of the following conditions are satisfied:\n // a. \"text\" matches \"pattern\" exactly\n // b. The number of matched sub-patterns is >= number of supplied pointers\n // c. The \"i\"th argument has a suitable type for holding the\n // string captured as the \"i\"th sub-pattern. If you pass in\n // NULL for the \"i\"th argument, or pass fewer arguments than\n // number of sub-patterns, \"i\"th captured sub-pattern is\n // ignored.\n //\n // CAVEAT: An optional sub-pattern that does not exist in the\n // matched string is assigned the empty string. Therefore, the\n // following will return false (because the empty string is not a\n // valid number):\n // int number;\n // PCRE::FullMatch(\"abc\", \"[a-z]+(\\\\d+)?\", &number);\n struct FullMatchFunctor {\n bool operator ()(absl::string_view text, const PCRE& re, // 3..16 args\n const Arg& ptr1 = no_more_args,\n const Arg& ptr2 = no_more_args,\n const Arg& ptr3 = no_more_args,\n const Arg& ptr4 = no_more_args,\n const Arg& ptr5 = no_more_args,\n const Arg& ptr6 = no_more_args,\n const Arg& ptr7 = no_more_args,\n const Arg& ptr8 = no_more_args,\n const Arg& ptr9 = no_more_args,\n const Arg& ptr10 = no_more_args,\n const Arg& ptr11 = no_more_args,\n const Arg& ptr12 = no_more_args,\n const Arg& ptr13 = no_more_args,\n const Arg& ptr14 = no_more_args,\n const Arg& ptr15 = no_more_args,\n const Arg& ptr16 = no_more_args) const;\n };\n\n RE2_TESTING_DLL static const FullMatchFunctor FullMatch;\n\n // Exactly like FullMatch(), except that \"pattern\" is allowed to match\n // a substring of \"text\".\n struct PartialMatchFunctor {\n bool operator ()(absl::string_view text, const PCRE& re, // 3..16 args\n const Arg& ptr1 = no_more_args,\n const Arg& ptr2 = no_more_args,\n const Arg& ptr3 = no_more_args,\n const Arg& ptr4 = no_more_args,\n const Arg& ptr5 = no_more_args,\n const Arg& ptr6 = no_more_args,\n const Arg& ptr7 = no_more_args,\n const Arg& ptr8 = no_more_args,\n const Arg& ptr9 = no_more_args,\n const Arg& ptr10 = no_more_args,\n const Arg& ptr11 = no_more_args,\n const Arg& ptr12 = no_more_args,\n const Arg& ptr13 = no_more_args,\n const Arg& ptr14 = no_more_args,\n const Arg& ptr15 = no_more_args,\n const Arg& ptr16 = no_more_args) const;\n };\n\n RE2_TESTING_DLL static const PartialMatchFunctor PartialMatch;\n\n // Like FullMatch() and PartialMatch(), except that pattern has to\n // match a prefix of \"text\", and \"input\" is advanced past the matched\n // text. Note: \"input\" is modified iff this routine returns true.\n struct ConsumeFunctor {\n bool operator ()(absl::string_view* input, const PCRE& pattern, // 3..16 args\n const Arg& ptr1 = no_more_args,\n const Arg& ptr2 = no_more_args,\n const Arg& ptr3 = no_more_args,\n const Arg& ptr4 = no_more_args,\n const Arg& ptr5 = no_more_args,\n const Arg& ptr6 = no_more_args,\n const Arg& ptr7 = no_more_args,\n const Arg& ptr8 = no_more_args,\n const Arg& ptr9 = no_more_args,\n const Arg& ptr10 = no_more_args,\n const Arg& ptr11 = no_more_args,\n const Arg& ptr12 = no_more_args,\n const Arg& ptr13 = no_more_args,\n const Arg& ptr14 = no_more_args,\n const Arg& ptr15 = no_more_args,\n const Arg& ptr16 = no_more_args) const;\n };\n\n RE2_TESTING_DLL static const ConsumeFunctor Consume;\n\n // Like Consume(..), but does not anchor the match at the beginning of the\n // string. That is, \"pattern\" need not start its match at the beginning of\n // \"input\". For example, \"FindAndConsume(s, \"(\\\\w+)\", &word)\" finds the next\n // word in \"s\" and stores it in \"word\".\n struct FindAndConsumeFunctor {\n bool operator ()(absl::string_view* input, const PCRE& pattern, // 3..16 args\n const Arg& ptr1 = no_more_args,\n const Arg& ptr2 = no_more_args,\n const Arg& ptr3 = no_more_args,\n const Arg& ptr4 = no_more_args,\n const Arg& ptr5 = no_more_args,\n const Arg& ptr6 = no_more_args,\n const Arg& ptr7 = no_more_args,\n const Arg& ptr8 = no_more_args,\n const Arg& ptr9 = no_more_args,\n const Arg& ptr10 = no_more_args,\n const Arg& ptr11 = no_more_args,\n const Arg& ptr12 = no_more_args,\n const Arg& ptr13 = no_more_args,\n const Arg& ptr14 = no_more_args,\n const Arg& ptr15 = no_more_args,\n const Arg& ptr16 = no_more_args) const;\n };\n\n RE2_TESTING_DLL static const FindAndConsumeFunctor FindAndConsume;\n\n // Replace the first match of \"pattern\" in \"str\" with \"rewrite\".\n // Within \"rewrite\", backslash-escaped digits (\\1 to \\9) can be\n // used to insert text matching corresponding parenthesized group\n // from the pattern. \\0 in \"rewrite\" refers to the entire matching\n // text. E.g.,\n //\n // std::string s = \"yabba dabba doo\";\n // ABSL_CHECK(PCRE::Replace(&s, \"b+\", \"d\"));\n //\n // will leave \"s\" containing \"yada dabba doo\"\n //\n // Returns true if the pattern matches and a replacement occurs,\n // false otherwise.\n static bool Replace(std::string* str, const PCRE& pattern,\n absl::string_view rewrite);\n\n // Like Replace(), except replaces all occurrences of the pattern in\n // the string with the rewrite. Replacements are not subject to\n // re-matching. E.g.,\n //\n // std::string s = \"yabba dabba doo\";\n // ABSL_CHECK(PCRE::GlobalReplace(&s, \"b+\", \"d\"));\n //\n // will leave \"s\" containing \"yada dada doo\"\n //\n // Returns the number of replacements made.\n static int GlobalReplace(std::string* str, const PCRE& pattern,\n absl::string_view rewrite);\n\n // Like Replace, except that if the pattern matches, \"rewrite\"\n // is copied into \"out\" with substitutions. The non-matching\n // portions of \"text\" are ignored.\n //\n // Returns true iff a match occurred and the extraction happened\n // successfully; if no match occurs, the string is left unaffected.\n static bool Extract(absl::string_view text, const PCRE& pattern,\n absl::string_view rewrite, std::string* out);\n\n // Check that the given @p rewrite string is suitable for use with\n // this PCRE. It checks that:\n // * The PCRE has enough parenthesized subexpressions to satisfy all\n // of the \\N tokens in @p rewrite, and\n // * The @p rewrite string doesn't have any syntax errors\n // ('\\' followed by anything besides [0-9] and '\\').\n // Making this test will guarantee that \"replace\" and \"extract\"\n // operations won't ABSL_LOG(ERROR) or fail because of a bad rewrite\n // string.\n // @param rewrite The proposed rewrite string.\n // @param error An error message is recorded here, iff we return false.\n // Otherwise, it is unchanged.\n // @return true, iff @p rewrite is suitable for use with the PCRE.\n bool CheckRewriteString(absl::string_view rewrite, std::string* error) const;\n\n // Returns a copy of 'unquoted' with all potentially meaningful\n // regexp characters backslash-escaped. The returned string, used\n // as a regular expression, will exactly match the original string.\n // For example,\n // 1.5-2.0?\n // becomes:\n // 1\\.5\\-2\\.0\\?\n static std::string QuoteMeta(absl::string_view unquoted);\n\n /***** Generic matching interface (not so nice to use) *****/\n\n // Type of match (TODO: Should be restructured as an Option)\n enum Anchor {\n UNANCHORED, // No anchoring\n ANCHOR_START, // Anchor at start only\n ANCHOR_BOTH, // Anchor at start and end\n };\n\n // General matching routine. Stores the length of the match in\n // \"*consumed\" if successful.\n bool DoMatch(absl::string_view text, Anchor anchor, size_t* consumed,\n const Arg* const* args, int n) const;\n\n // Return the number of capturing subpatterns, or -1 if the\n // regexp wasn't valid on construction.\n int NumberOfCapturingGroups() const;\n\n private:\n void Init(const char* pattern, Option option, int match_limit,\n int stack_limit, bool report_errors);\n\n // Match against \"text\", filling in \"vec\" (up to \"vecsize\" * 2/3) with\n // pairs of integers for the beginning and end positions of matched\n // text. The first pair corresponds to the entire matched text;\n // subsequent pairs correspond, in order, to parentheses-captured\n // matches. Returns the number of pairs (one more than the number of\n // the last subpattern with a match) if matching was successful\n // and zero if the match failed.\n // I.e. for PCRE(\"(foo)|(bar)|(baz)\") it will return 2, 3, and 4 when matching\n // against \"foo\", \"bar\", and \"baz\" respectively.\n // When matching PCRE(\"(foo)|hello\") against \"hello\", it will return 1.\n // But the values for all subpattern are filled in into \"vec\".\n int TryMatch(absl::string_view text, size_t startpos, Anchor anchor,\n bool empty_ok, int* vec, int vecsize) const;\n\n // Append the \"rewrite\" string, with backslash substitutions from \"text\"\n // and \"vec\", to string \"out\".\n bool Rewrite(std::string* out, absl::string_view rewrite,\n absl::string_view text, int* vec, int veclen) const;\n\n // internal implementation for DoMatch\n bool DoMatchImpl(absl::string_view text, Anchor anchor, size_t* consumed,\n const Arg* const args[], int n, int* vec, int vecsize) const;\n\n // Compile the regexp for the specified anchoring mode\n pcre* Compile(Anchor anchor);\n\n std::string pattern_;\n Option options_;\n pcre* re_full_; // For full matches\n pcre* re_partial_; // For partial matches\n const std::string* error_; // Error indicator (or empty string)\n bool report_errors_; // Silences error logging if false\n int match_limit_; // Limit on execution resources\n int stack_limit_; // Limit on stack resources (bytes)\n mutable int hit_limit_; // Hit limit during execution (bool)\n\n PCRE(const PCRE&) = delete;\n PCRE& operator=(const PCRE&) = delete;\n};\n\n// PCRE_Options allow you to set the PCRE::Options, plus any pcre\n// \"extra\" options. The only extras are match_limit, which limits\n// the CPU time of a match, and stack_limit, which limits the\n// stack usage. Setting a limit to <= 0 lets PCRE pick a sensible default\n// that should not cause too many problems in production code.\n// If PCRE hits a limit during a match, it may return a false negative,\n// but (hopefully) it won't crash.\n//\n// NOTE: If you are handling regular expressions specified by\n// (external or internal) users, rather than hard-coded ones,\n// you should be using PCRE2, which uses an alternate implementation\n// that avoids these issues. See http://go/re2quick.\nclass PCRE_Options {\n public:\n // constructor\n PCRE_Options() : option_(PCRE::None), match_limit_(0), stack_limit_(0), report_errors_(true) {}\n // accessors\n PCRE::Option option() const { return option_; }\n void set_option(PCRE::Option option) {\n option_ = option;\n }\n int match_limit() const { return match_limit_; }\n void set_match_limit(int match_limit) {\n match_limit_ = match_limit;\n }\n int stack_limit() const { return stack_limit_; }\n void set_stack_limit(int stack_limit) {\n stack_limit_ = stack_limit;\n }\n\n // If the regular expression is malformed, an error message will be printed\n // iff report_errors() is true. Default: true.\n bool report_errors() const { return report_errors_; }\n void set_report_errors(bool report_errors) {\n report_errors_ = report_errors;\n }\n private:\n PCRE::Option option_;\n int match_limit_;\n int stack_limit_;\n bool report_errors_;\n};\n\n\n/***** Implementation details *****/\n\n// Hex/Octal/Binary?\n\n// Special class for parsing into objects that define a ParseFrom() method\ntemplate \nclass _PCRE_MatchObject {\n public:\n static inline bool Parse(const char* str, size_t n, void* dest) {\n if (dest == NULL) return true;\n T* object = reinterpret_cast(dest);\n return object->ParseFrom(str, n);\n }\n};\n\nclass PCRE::Arg {\n public:\n // Empty constructor so we can declare arrays of PCRE::Arg\n Arg();\n\n // Constructor specially designed for NULL arguments\n Arg(void*);\n\n typedef bool (*Parser)(const char* str, size_t n, void* dest);\n\n// Type-specific parsers\n#define MAKE_PARSER(type, name) \\\n Arg(type* p) : arg_(p), parser_(name) {} \\\n Arg(type* p, Parser parser) : arg_(p), parser_(parser) {}\n\n MAKE_PARSER(char, parse_char);\n MAKE_PARSER(signed char, parse_schar);\n MAKE_PARSER(unsigned char, parse_uchar);\n MAKE_PARSER(float, parse_float);\n MAKE_PARSER(double, parse_double);\n MAKE_PARSER(std::string, parse_string);\n MAKE_PARSER(absl::string_view, parse_string_view);\n\n MAKE_PARSER(short, parse_short);\n MAKE_PARSER(unsigned short, parse_ushort);\n MAKE_PARSER(int, parse_int);\n MAKE_PARSER(unsigned int, parse_uint);\n MAKE_PARSER(long, parse_long);\n MAKE_PARSER(unsigned long, parse_ulong);\n MAKE_PARSER(long long, parse_longlong);\n MAKE_PARSER(unsigned long long, parse_ulonglong);\n\n#undef MAKE_PARSER\n\n // Generic constructor\n template Arg(T*, Parser parser);\n // Generic constructor template\n template Arg(T* p)\n : arg_(p), parser_(_PCRE_MatchObject::Parse) {\n }\n\n // Parse the data\n bool Parse(const char* str, size_t n) const;\n\n private:\n void* arg_;\n Parser parser_;\n\n static bool parse_null (const char* str, size_t n, void* dest);\n static bool parse_char (const char* str, size_t n, void* dest);\n static bool parse_schar (const char* str, size_t n, void* dest);\n static bool parse_uchar (const char* str, size_t n, void* dest);\n static bool parse_float (const char* str, size_t n, void* dest);\n static bool parse_double (const char* str, size_t n, void* dest);\n static bool parse_string (const char* str, size_t n, void* dest);\n static bool parse_string_view (const char* str, size_t n, void* dest);\n\n#define DECLARE_INTEGER_PARSER(name) \\\n private: \\\n static bool parse_##name(const char* str, size_t n, void* dest); \\\n static bool parse_##name##_radix(const char* str, size_t n, void* dest, \\\n int radix); \\\n \\\n public: \\\n static bool parse_##name##_hex(const char* str, size_t n, void* dest); \\\n static bool parse_##name##_octal(const char* str, size_t n, void* dest); \\\n static bool parse_##name##_cradix(const char* str, size_t n, void* dest)\n\n DECLARE_INTEGER_PARSER(short);\n DECLARE_INTEGER_PARSER(ushort);\n DECLARE_INTEGER_PARSER(int);\n DECLARE_INTEGER_PARSER(uint);\n DECLARE_INTEGER_PARSER(long);\n DECLARE_INTEGER_PARSER(ulong);\n DECLARE_INTEGER_PARSER(longlong);\n DECLARE_INTEGER_PARSER(ulonglong);\n\n#undef DECLARE_INTEGER_PARSER\n\n};\n\ninline PCRE::Arg::Arg() : arg_(NULL), parser_(parse_null) { }\ninline PCRE::Arg::Arg(void* p) : arg_(p), parser_(parse_null) { }\n\ninline bool PCRE::Arg::Parse(const char* str, size_t n) const {\n return (*parser_)(str, n, arg_);\n}\n\n// This part of the parser, appropriate only for ints, deals with bases\n#define MAKE_INTEGER_PARSER(type, name) \\\n inline PCRE::Arg Hex(type* ptr) { \\\n return PCRE::Arg(ptr, PCRE::Arg::parse_##name##_hex); \\\n } \\\n inline PCRE::Arg Octal(type* ptr) { \\\n return PCRE::Arg(ptr, PCRE::Arg::parse_##name##_octal); \\\n } \\\n inline PCRE::Arg CRadix(type* ptr) { \\\n return PCRE::Arg(ptr, PCRE::Arg::parse_##name##_cradix); \\\n }\n\nMAKE_INTEGER_PARSER(short, short);\nMAKE_INTEGER_PARSER(unsigned short, ushort);\nMAKE_INTEGER_PARSER(int, int);\nMAKE_INTEGER_PARSER(unsigned int, uint);\nMAKE_INTEGER_PARSER(long, long);\nMAKE_INTEGER_PARSER(unsigned long, ulong);\nMAKE_INTEGER_PARSER(long long, longlong);\nMAKE_INTEGER_PARSER(unsigned long long, ulonglong);\n\n#undef MAKE_INTEGER_PARSER\n\n} // namespace re2\n\n#endif // UTIL_PCRE_H_\n" }, { "path": "util/rune.cc", "content": "/*\n * The authors of this software are Rob Pike and Ken Thompson.\n * Copyright (c) 2002 by Lucent Technologies.\n * Permission to use, copy, modify, and distribute this software for any\n * purpose without fee is hereby granted, provided that this entire notice\n * is included in all copies of any software which is or includes a copy\n * or modification of this software and in all copies of the supporting\n * documentation for such software.\n * THIS SOFTWARE IS BEING PROVIDED \"AS IS\", WITHOUT ANY EXPRESS OR IMPLIED\n * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY\n * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY\n * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE.\n */\n\n#include \n#include \n\n#include \"util/utf.h\"\n\nnamespace re2 {\n\nenum\n{\n\tBit1\t= 7,\n\tBitx\t= 6,\n\tBit2\t= 5,\n\tBit3\t= 4,\n\tBit4\t= 3,\n\tBit5\t= 2, \n\n\tT1\t= ((1<<(Bit1+1))-1) ^ 0xFF,\t/* 0000 0000 */\n\tTx\t= ((1<<(Bitx+1))-1) ^ 0xFF,\t/* 1000 0000 */\n\tT2\t= ((1<<(Bit2+1))-1) ^ 0xFF,\t/* 1100 0000 */\n\tT3\t= ((1<<(Bit3+1))-1) ^ 0xFF,\t/* 1110 0000 */\n\tT4\t= ((1<<(Bit4+1))-1) ^ 0xFF,\t/* 1111 0000 */\n\tT5\t= ((1<<(Bit5+1))-1) ^ 0xFF,\t/* 1111 1000 */\n\n\tRune1\t= (1<<(Bit1+0*Bitx))-1,\t\t/* 0000 0000 0111 1111 */\n\tRune2\t= (1<<(Bit2+1*Bitx))-1,\t\t/* 0000 0111 1111 1111 */\n\tRune3\t= (1<<(Bit3+2*Bitx))-1,\t\t/* 1111 1111 1111 1111 */\n\tRune4\t= (1<<(Bit4+3*Bitx))-1,\n /* 0001 1111 1111 1111 1111 1111 */\n\n\tMaskx\t= (1< T1\n\t */\n\tc = *(unsigned char*)str;\n\tif(c < Tx) {\n\t\t*rune = c;\n\t\treturn 1;\n\t}\n\n\t/*\n\t * two character sequence\n\t *\t0080-07FF => T2 Tx\n\t */\n\tc1 = *(unsigned char*)(str+1) ^ Tx;\n\tif(c1 & Testx)\n\t\tgoto bad;\n\tif(c < T3) {\n\t\tif(c < T2)\n\t\t\tgoto bad;\n\t\tl = ((c << Bitx) | c1) & Rune2;\n\t\tif(l <= Rune1)\n\t\t\tgoto bad;\n\t\t*rune = l;\n\t\treturn 2;\n\t}\n\n\t/*\n\t * three character sequence\n\t *\t0800-FFFF => T3 Tx Tx\n\t */\n\tc2 = *(unsigned char*)(str+2) ^ Tx;\n\tif(c2 & Testx)\n\t\tgoto bad;\n\tif(c < T4) {\n\t\tl = ((((c << Bitx) | c1) << Bitx) | c2) & Rune3;\n\t\tif(l <= Rune2)\n\t\t\tgoto bad;\n\t\t*rune = l;\n\t\treturn 3;\n\t}\n\n\t/*\n\t * four character sequence (21-bit value)\n\t *\t10000-1FFFFF => T4 Tx Tx Tx\n\t */\n\tc3 = *(unsigned char*)(str+3) ^ Tx;\n\tif (c3 & Testx)\n\t\tgoto bad;\n\tif (c < T5) {\n\t\tl = ((((((c << Bitx) | c1) << Bitx) | c2) << Bitx) | c3) & Rune4;\n\t\tif (l <= Rune3)\n\t\t\tgoto bad;\n\t\t*rune = l;\n\t\treturn 4;\n\t}\n\n\t/*\n\t * Support for 5-byte or longer UTF-8 would go here, but\n\t * since we don't have that, we'll just fall through to bad.\n\t */\n\n\t/*\n\t * bad decoding\n\t */\nbad:\n\t*rune = Bad;\n\treturn 1;\n}\n\nint\nrunetochar(char *str, const Rune *rune)\n{\n\t/* Runes are signed, so convert to unsigned for range check. */\n\tunsigned int c;\n\n\t/*\n\t * one character sequence\n\t *\t00000-0007F => 00-7F\n\t */\n\tc = *rune;\n\tif(c <= Rune1) {\n\t\tstr[0] = static_cast(c);\n\t\treturn 1;\n\t}\n\n\t/*\n\t * two character sequence\n\t *\t0080-07FF => T2 Tx\n\t */\n\tif(c <= Rune2) {\n\t\tstr[0] = T2 | static_cast(c >> 1*Bitx);\n\t\tstr[1] = Tx | (c & Maskx);\n\t\treturn 2;\n\t}\n\n\t/*\n\t * If the Rune is out of range, convert it to the error rune.\n\t * Do this test here because the error rune encodes to three bytes.\n\t * Doing it earlier would duplicate work, since an out of range\n\t * Rune wouldn't have fit in one or two bytes.\n\t */\n\tif (c > Runemax)\n\t\tc = Runeerror;\n\n\t/*\n\t * three character sequence\n\t *\t0800-FFFF => T3 Tx Tx\n\t */\n\tif (c <= Rune3) {\n\t\tstr[0] = T3 | static_cast(c >> 2*Bitx);\n\t\tstr[1] = Tx | ((c >> 1*Bitx) & Maskx);\n\t\tstr[2] = Tx | (c & Maskx);\n\t\treturn 3;\n\t}\n\n\t/*\n\t * four character sequence (21-bit value)\n\t * 10000-1FFFFF => T4 Tx Tx Tx\n\t */\n\tstr[0] = T4 | static_cast(c >> 3*Bitx);\n\tstr[1] = Tx | ((c >> 2*Bitx) & Maskx);\n\tstr[2] = Tx | ((c >> 1*Bitx) & Maskx);\n\tstr[3] = Tx | (c & Maskx);\n\treturn 4;\n}\n\nint\nrunelen(Rune rune)\n{\n\tchar str[10];\n\n\treturn runetochar(str, &rune);\n}\n\nint\nfullrune(const char *str, int n)\n{\n\tif (n > 0) {\n\t\tint c = *(unsigned char*)str;\n\t\tif (c < Tx)\n\t\t\treturn 1;\n\t\tif (n > 1) {\n\t\t\tif (c < T3)\n\t\t\t\treturn 1;\n\t\t\tif (n > 2) {\n\t\t\t\tif (c < T4 || n > 3)\n\t\t\t\t\treturn 1;\n\t\t\t}\n\t\t}\n\t}\n\treturn 0;\n}\n\n\nint\nutflen(const char *s)\n{\n\tint c;\n\tint n;\n\tRune rune;\n\n\tn = 0;\n\tfor(;;) {\n\t\tc = *(unsigned char*)s;\n\t\tif(c < Runeself) {\n\t\t\tif(c == 0)\n\t\t\t\treturn n;\n\t\t\ts++;\n\t\t} else\n\t\t\ts += chartorune(&rune, s);\n\t\tn++;\n\t}\n\treturn 0;\n}\n\nchar*\nutfrune(const char *s, Rune c)\n{\n\tint c1;\n\tRune r;\n\tint n;\n\n\tif(c < Runesync)\t\t/* not part of utf sequence */\n\t\treturn strchr((char*)s, c);\n\n\tfor(;;) {\n\t\tc1 = *(unsigned char*)s;\n\t\tif(c1 < Runeself) {\t/* one byte rune */\n\t\t\tif(c1 == 0)\n\t\t\t\treturn 0;\n\t\t\tif(c1 == c)\n\t\t\t\treturn (char*)s;\n\t\t\ts++;\n\t\t\tcontinue;\n\t\t}\n\t\tn = chartorune(&r, s);\n\t\tif(r == c)\n\t\t\treturn (char*)s;\n\t\ts += n;\n\t}\n\treturn 0;\n}\n\n} // namespace re2\n" }, { "path": "util/strutil.cc", "content": "// Copyright 1999-2005 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#include \"util/strutil.h\"\n\nnamespace re2 {\n\nvoid PrefixSuccessor(std::string* prefix) {\n // We can increment the last character in the string and be done\n // unless that character is 255, in which case we have to erase the\n // last character and increment the previous character, unless that\n // is 255, etc. If the string is empty or consists entirely of\n // 255's, we just return the empty string.\n while (!prefix->empty()) {\n char& c = prefix->back();\n if (c == '\\xff') { // char literal avoids signed/unsigned.\n prefix->pop_back();\n } else {\n ++c;\n break;\n }\n }\n}\n\n} // namespace re2\n" }, { "path": "util/strutil.h", "content": "// Copyright 2016 The RE2 Authors. All Rights Reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n#ifndef UTIL_STRUTIL_H_\n#define UTIL_STRUTIL_H_\n\n#include \n\nnamespace re2 {\n\nvoid PrefixSuccessor(std::string* prefix);\n\n} // namespace re2\n\n#endif // UTIL_STRUTIL_H_\n" }, { "path": "util/utf.h", "content": "/*\n * The authors of this software are Rob Pike and Ken Thompson.\n * Copyright (c) 2002 by Lucent Technologies.\n * Permission to use, copy, modify, and distribute this software for any\n * purpose without fee is hereby granted, provided that this entire notice\n * is included in all copies of any software which is or includes a copy\n * or modification of this software and in all copies of the supporting\n * documentation for such software.\n * THIS SOFTWARE IS BEING PROVIDED \"AS IS\", WITHOUT ANY EXPRESS OR IMPLIED\n * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY\n * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY\n * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE.\n *\n * This file and rune.cc have been converted to compile as C++ code\n * in name space re2.\n */\n\n#ifndef UTIL_UTF_H_\n#define UTIL_UTF_H_\n\n#include \n\nnamespace re2 {\n\ntypedef signed int Rune;\t/* Code-point values in Unicode 4.0 are 21 bits wide.*/\n\nenum\n{\n UTFmax\t= 4,\t\t/* maximum bytes per rune */\n Runesync\t= 0x80,\t\t/* cannot represent part of a UTF sequence (<) */\n Runeself\t= 0x80,\t\t/* rune and UTF sequences are the same (<) */\n Runeerror\t= 0xFFFD,\t/* decoding error in UTF */\n Runemax\t= 0x10FFFF,\t/* maximum rune value */\n};\n\nint runetochar(char* s, const Rune* r);\nint chartorune(Rune* r, const char* s);\nint fullrune(const char* s, int n);\nint utflen(const char* s);\nchar* utfrune(const char*, Rune);\n\n} // namespace re2\n\n#endif // UTIL_UTF_H_\n" } ]