[ { "path": ".bazelversion", "content": "8.2.1\n" }, { "path": ".clang-format", "content": "---\nLanguage: Cpp\nBasedOnStyle: Google\nPointerAlignment: Left\n...\n" }, { "path": ".clang-tidy", "content": "---\nChecks: >\n abseil-*,\n bugprone-*,\n clang-analyzer-*,\n cppcoreguidelines-*,\n google-*,\n misc-*,\n performance-*,\n readability-*,\n -clang-analyzer-deadcode*,\n -clang-analyzer-optin*,\n -readability-identifier-length\nWarningsAsErrors: ''\nHeaderFilterRegex: ''\nFormatStyle: none\nCheckOptions:\n llvm-else-after-return.WarnOnConditionVariables: 'false'\n modernize-loop-convert.MinConfidence: reasonable\n modernize-replace-auto-ptr.IncludeStyle: llvm\n cert-str34-c.DiagnoseSignedUnsignedCharComparisons: 'false'\n google-readability-namespace-comments.ShortNamespaceLines: '10'\n cert-err33-c.CheckedFunctions: '::aligned_alloc;::asctime_s;::at_quick_exit;::atexit;::bsearch;::bsearch_s;::btowc;::c16rtomb;::c32rtomb;::calloc;::clock;::cnd_broadcast;::cnd_init;::cnd_signal;::cnd_timedwait;::cnd_wait;::ctime_s;::fclose;::fflush;::fgetc;::fgetpos;::fgets;::fgetwc;::fopen;::fopen_s;::fprintf;::fprintf_s;::fputc;::fputs;::fputwc;::fputws;::fread;::freopen;::freopen_s;::fscanf;::fscanf_s;::fseek;::fsetpos;::ftell;::fwprintf;::fwprintf_s;::fwrite;::fwscanf;::fwscanf_s;::getc;::getchar;::getenv;::getenv_s;::gets_s;::getwc;::getwchar;::gmtime;::gmtime_s;::localtime;::localtime_s;::malloc;::mbrtoc16;::mbrtoc32;::mbsrtowcs;::mbsrtowcs_s;::mbstowcs;::mbstowcs_s;::memchr;::mktime;::mtx_init;::mtx_lock;::mtx_timedlock;::mtx_trylock;::mtx_unlock;::printf_s;::putc;::putwc;::raise;::realloc;::remove;::rename;::scanf;::scanf_s;::setlocale;::setvbuf;::signal;::snprintf;::snprintf_s;::sprintf;::sprintf_s;::sscanf;::sscanf_s;::strchr;::strerror_s;::strftime;::strpbrk;::strrchr;::strstr;::strtod;::strtof;::strtoimax;::strtok;::strtok_s;::strtol;::strtold;::strtoll;::strtoul;::strtoull;::strtoumax;::strxfrm;::swprintf;::swprintf_s;::swscanf;::swscanf_s;::thrd_create;::thrd_detach;::thrd_join;::thrd_sleep;::time;::timespec_get;::tmpfile;::tmpfile_s;::tmpnam;::tmpnam_s;::tss_create;::tss_get;::tss_set;::ungetc;::ungetwc;::vfprintf;::vfprintf_s;::vfscanf;::vfscanf_s;::vfwprintf;::vfwprintf_s;::vfwscanf;::vfwscanf_s;::vprintf_s;::vscanf;::vscanf_s;::vsnprintf;::vsnprintf_s;::vsprintf;::vsprintf_s;::vsscanf;::vsscanf_s;::vswprintf;::vswprintf_s;::vswscanf;::vswscanf_s;::vwprintf_s;::vwscanf;::vwscanf_s;::wcrtomb;::wcschr;::wcsftime;::wcspbrk;::wcsrchr;::wcsrtombs;::wcsrtombs_s;::wcsstr;::wcstod;::wcstof;::wcstoimax;::wcstok;::wcstok_s;::wcstol;::wcstold;::wcstoll;::wcstombs;::wcstombs_s;::wcstoul;::wcstoull;::wcstoumax;::wcsxfrm;::wctob;::wctrans;::wctype;::wmemchr;::wprintf_s;::wscanf;::wscanf_s;'\n cert-oop54-cpp.WarnOnlyIfThisHasSuspiciousField: 'false'\n cert-dcl16-c.NewSuffixes: 'L;LL;LU;LLU'\n google-readability-braces-around-statements.ShortStatementLines: '1'\n cppcoreguidelines-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic: 'true'\n google-readability-namespace-comments.SpacesBeforeComments: '2'\n modernize-loop-convert.MaxCopySize: '16'\n modernize-pass-by-value.IncludeStyle: llvm\n modernize-use-nullptr.NullMacros: 'NULL'\n llvm-qualified-auto.AddConstToQualified: 'false'\n modernize-loop-convert.NamingStyle: CamelCase\n llvm-else-after-return.WarnOnUnfixable: 'false'\n google-readability-function-size.StatementThreshold: '800'\n...\n\n" }, { "path": ".clang-tidy.ignore", "content": ".*third_party/.*\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: \"[BUG]\"\nlabels: ''\nassignees: ''\n\n---\n\n**Describe the bug**\nA clear and concise description of what the bug is.\n\n**System**\nWhich OS, compiler, and compiler version are you using:\n - OS: \n - Compiler and version: \n\n**To reproduce**\nSteps to reproduce the behavior:\n1. sync to commit ...\n2. cmake/bazel...\n3. make ...\n4. See error\n\n**Expected behavior**\nA clear and concise description of what you expected to happen.\n\n**Screenshots**\nIf applicable, add screenshots to help explain your problem.\n\n**Additional context**\nAdd any other context about the problem here.\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: \"[FR]\"\nlabels: ''\nassignees: ''\n\n---\n\n**Is your feature request related to a problem? Please describe.**\nA clear and concise description of what the problem is. Ex. I'm always frustrated when [...]\n\n**Describe the solution you'd like**\nA clear and concise description of what you want to happen.\n\n**Describe alternatives you've considered**\nA clear and concise description of any alternative solutions or features you've considered.\n\n**Additional context**\nAdd any other context or screenshots about the feature request here.\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: github-actions\n directory: /\n schedule:\n interval: daily\n\n - package-ecosystem: pip\n directory: /tools\n schedule:\n interval: daily\n" }, { "path": ".github/install_bazel.sh", "content": "if ! bazel version; then\n arch=$(uname -m)\n if [ \"$arch\" == \"aarch64\" ]; then\n arch=\"arm64\"\n fi\n echo \"Downloading $arch Bazel binary from GitHub releases.\"\n curl -L -o $HOME/bin/bazel --create-dirs \"https://github.com/bazelbuild/bazel/releases/download/8.2.0/bazel-8.2.0-linux-$arch\"\n chmod +x $HOME/bin/bazel\nelse\n # Bazel is installed for the correct architecture\n exit 0\nfi\n" }, { "path": ".github/libcxx-setup.sh", "content": "#!/usr/bin/env bash\n\nset -e\n\n# Checkout LLVM sources\ngit clone --filter=blob:none --depth=1 --branch llvmorg-19.1.6 --no-checkout https://github.com/llvm/llvm-project.git llvm-project\ncd llvm-project\ngit sparse-checkout set --cone\ngit checkout llvmorg-19.1.6\ngit sparse-checkout set cmake llvm/cmake runtimes libcxx libcxxabi\ncd ..\n\n## Setup libc++ options\nif [ -z \"$BUILD_32_BITS\" ]; then\n export BUILD_32_BITS=OFF && echo disabling 32 bit build\nfi\n\n## Build and install libc++ (Use unstable ABI for better sanitizer coverage)\nmkdir llvm-build && cd llvm-build\ncmake -GNinja \\\n -DCMAKE_C_COMPILER=${CC} \\\n -DCMAKE_CXX_COMPILER=${CXX} \\\n -DCMAKE_BUILD_TYPE=RelWithDebInfo \\\n -DCMAKE_INSTALL_PREFIX=/usr \\\n -DLIBCXX_ABI_UNSTABLE=OFF \\\n -DLLVM_USE_SANITIZER=${LIBCXX_SANITIZER} \\\n -DLLVM_BUILD_32_BITS=${BUILD_32_BITS} \\\n -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \\\n -DLLVM_INCLUDE_TESTS=OFF \\\n -DLIBCXX_INCLUDE_TESTS=OFF \\\n -DLIBCXX_INCLUDE_BENCHMARKS=OFF \\\n -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \\\n ../llvm-project/runtimes/\ncmake --build . -- cxx cxxabi\ncd ..\n" }, { "path": ".github/workflows/bazel.yml", "content": "name: bazel\n\non:\n push: {}\n pull_request: {}\n\nenv:\n CMAKE_GENERATOR: Ninja\n\npermissions:\n contents: read\n\njobs:\n build_and_test_default:\n name: bazel.${{ matrix.os }}\n runs-on: ${{ matrix.os }}\n strategy:\n fail-fast: false\n matrix:\n os: [ubuntu-latest, macos-latest, windows-latest]\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: mount bazel cache\n uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4\n env:\n cache-name: bazel-cache\n with:\n path: \"~/.cache/bazel\"\n key: ${{ env.cache-name }}-${{ matrix.os }}-${{ github.ref }}\n restore-keys: |\n ${{ env.cache-name }}-${{ matrix.os }}-main\n\n - name: build\n run: |\n bazel build //:benchmark //:benchmark_main //test/...\n\n - name: test\n run: |\n bazel test --test_output=all //test/...\n" }, { "path": ".github/workflows/build-and-test-min-cmake.yml", "content": "name: build-and-test-min-cmake\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n\nenv:\n CMAKE_GENERATOR: Ninja\n\njobs:\n job:\n name: ${{ matrix.os }}.min-cmake\n runs-on: ${{ matrix.os }}\n strategy:\n fail-fast: false\n matrix:\n os: [ubuntu-latest, macos-latest]\n\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - uses: lukka/get-cmake@b78306120111dc2522750771cfd09ee7ca723687 # latest\n with:\n cmakeVersion: 3.13.0\n\n - name: create build environment\n run: cmake -E make_directory ${{ runner.workspace }}/_build\n\n - name: setup cmake initial cache\n run: touch compiler-cache.cmake\n\n - name: configure cmake\n env:\n CXX: ${{ matrix.compiler }}\n shell: bash\n working-directory: ${{ runner.workspace }}/_build\n run: >\n cmake -C ${{ github.workspace }}/compiler-cache.cmake\n $GITHUB_WORKSPACE\n -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON\n -DCMAKE_CXX_VISIBILITY_PRESET=hidden\n -DCMAKE_VISIBILITY_INLINES_HIDDEN=ON\n\n - name: build\n shell: bash\n working-directory: ${{ runner.workspace }}/_build\n run: cmake --build .\n" }, { "path": ".github/workflows/build-and-test-perfcounters.yml", "content": "name: build-and-test-perfcounters\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n\nenv:\n CMAKE_GENERATOR: Ninja\n\npermissions:\n contents: read\n\njobs:\n job:\n # TODO(dominic): Extend this to include compiler and set through env: CC/CXX.\n name: ${{ matrix.os }}.${{ matrix.build_type }}\n runs-on: ${{ matrix.os }}\n strategy:\n fail-fast: false\n matrix:\n os: [ubuntu-latest]\n build_type: ['Release', 'Debug']\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: install libpfm\n run: |\n sudo apt update\n sudo apt -y install libpfm4-dev\n\n - name: create build environment\n run: cmake -E make_directory ${{ runner.workspace }}/_build\n\n - name: configure cmake\n shell: bash\n working-directory: ${{ runner.workspace }}/_build\n run: >\n cmake $GITHUB_WORKSPACE\n -DBENCHMARK_ENABLE_LIBPFM=1\n -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON\n -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}\n\n - name: build\n shell: bash\n working-directory: ${{ runner.workspace }}/_build\n run: cmake --build . --config ${{ matrix.build_type }}\n\n # Skip testing, for now. It seems perf_event_open does not succeed on the\n # hosting machine, very likely a permissions issue.\n # TODO(mtrofin): Enable test.\n # - name: test\n # shell: bash\n # working-directory: ${{ runner.workspace }}/_build\n # run: ctest -C ${{ matrix.build_type }} --rerun-failed --output-on-failure\n\n" }, { "path": ".github/workflows/build-and-test.yml", "content": "name: build-and-test\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n\nenv:\n CMAKE_GENERATOR: Ninja\n\njobs:\n # TODO: add 32-bit builds (g++ and clang++) for ubuntu\n # (requires g++-multilib and libc6:i386)\n # TODO: add coverage build (requires lcov)\n # TODO: add clang + libc++ builds for ubuntu\n job:\n name: ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }}\n runs-on: ${{ matrix.os }}\n strategy:\n fail-fast: false\n matrix:\n os: [ubuntu-24.04, ubuntu-22.04, ubuntu-24.04-arm, macos-latest]\n build_type: ['Release', 'Debug']\n compiler: ['g++', 'clang++']\n lib: ['shared', 'static']\n\n steps:\n - name: Install dependencies (macos)\n if: runner.os == 'macOS'\n run: brew install ninja\n\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: build\n uses: threeal/cmake-action@725d1314ccf9ea922805d7e3f9d9bcbca892b406 # v2.1.0\n with:\n build-dir: ${{ runner.workspace }}/_build\n cxx-compiler: ${{ matrix.compiler }}\n options: |\n BENCHMARK_DOWNLOAD_DEPENDENCIES=ON\n BUILD_SHARED_LIBS=${{ matrix.lib == 'shared' }}\n CMAKE_BUILD_TYPE=${{ matrix.build_type }}\n CMAKE_CXX_COMPILER=${{ matrix.compiler }}\n CMAKE_CXX_VISIBILITY_PRESET=hidden\n CMAKE_VISIBILITY_INLINES_HIDDEN=ON\n\n - name: test\n shell: bash\n working-directory: ${{ runner.workspace }}/_build\n run: ctest -C ${{ matrix.build_type }} -VV\n\n msvc:\n name: ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msvc }}\n runs-on: ${{ matrix.os }}\n defaults:\n run:\n shell: powershell\n strategy:\n fail-fast: false\n matrix:\n msvc:\n - VS-17-2025\n - VS-17-2022\n build_type:\n - Debug\n - Release\n lib:\n - shared\n - static\n include:\n - msvc: VS-17-2025\n os: windows-2025\n generator: 'Visual Studio 17 2022'\n - msvc: VS-17-2022\n os: windows-2022\n generator: 'Visual Studio 17 2022'\n\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - uses: lukka/get-cmake@b78306120111dc2522750771cfd09ee7ca723687 # latest\n\n - name: configure cmake\n run: >\n cmake -S . -B ${{ runner.workspace }}/_build/\n -G \"${{ matrix.generator }}\"\n -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON\n -DBUILD_SHARED_LIBS=${{ matrix.lib == 'shared' }}\n\n - name: build\n run: cmake --build ${{ runner.workspace }}/_build/ --config ${{ matrix.build_type }}\n\n - name: test\n run: ctest --test-dir ${{ runner.workspace }}/_build/ -C ${{ matrix.build_type }} -VV\n\n msys2:\n name: ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }}\n runs-on: ${{ matrix.os }}\n defaults:\n run:\n shell: msys2 {0}\n strategy:\n fail-fast: false\n matrix:\n os: [ windows-latest ]\n msys2:\n - { msystem: MINGW64, arch: x86_64, family: GNU, compiler: g++ }\n - { msystem: CLANG64, arch: x86_64, family: LLVM, compiler: clang++ }\n - { msystem: UCRT64, arch: x86_64, family: GNU, compiler: g++ }\n build_type:\n - Debug\n - Release\n lib:\n - shared\n - static\n\n steps:\n - name: setup msys2\n uses: msys2/setup-msys2@4f806de0a5a7294ffabaff804b38a9b435a73bda # v2.30.0\n with:\n cache: false\n msystem: ${{ matrix.msys2.msystem }}\n update: true\n install: >-\n git\n base-devel\n pacboy: >-\n gcc:p\n clang:p\n cmake:p\n ninja:p\n\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n # NOTE: we can't use cmake actions here as we need to do everything in msys2 shell.\n - name: configure cmake\n env:\n CXX: ${{ matrix.msys2.compiler }}\n run: >\n cmake -S . -B _build/\n -GNinja\n -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON\n -DBUILD_SHARED_LIBS=${{ matrix.lib == 'shared' }}\n\n - name: build\n run: cmake --build _build/ --config ${{ matrix.build_type }}\n\n - name: test\n working-directory: _build\n run: ctest -C ${{ matrix.build_type }} -VV\n" }, { "path": ".github/workflows/clang-format-lint.yml", "content": "name: clang-format-lint\non:\n push: {}\n pull_request: {}\n\nenv:\n CMAKE_GENERATOR: Ninja\n\npermissions:\n contents: read\n\njobs:\n job:\n name: check-clang-format\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n - uses: DoozyX/clang-format-lint-action@bcb4eb2cb0d707ee4f3e5cc3b456eb075f12cf73 # v0.20\n with:\n source: './include/benchmark ./src ./test ./bindings'\n clangFormatVersion: 18\n" }, { "path": ".github/workflows/clang-tidy-lint.yml", "content": "name: clang-tidy\n\non:\n push: {}\n pull_request: {}\n\nenv:\n CMAKE_GENERATOR: Ninja\n\npermissions:\n contents: read\n\njobs:\n job:\n name: run-clang-tidy\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: install clang-tidy\n run: sudo apt update && sudo apt -y install clang-tidy\n\n - name: create build environment\n run: cmake -E make_directory ${{ github.workspace }}/_build\n\n - name: configure cmake\n shell: bash\n working-directory: ${{ github.workspace }}/_build\n run: >\n cmake $GITHUB_WORKSPACE\n -DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF\n -DBENCHMARK_ENABLE_LIBPFM=OFF\n -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON\n -DCMAKE_C_COMPILER=clang\n -DCMAKE_CXX_COMPILER=clang++\n -DCMAKE_EXPORT_COMPILE_COMMANDS=ON\n -DGTEST_COMPILE_COMMANDS=OFF\n\n - name: run\n shell: bash\n working-directory: ${{ github.workspace }}/_build\n run: run-clang-tidy -config-file=$GITHUB_WORKSPACE/.clang-tidy\n" }, { "path": ".github/workflows/doxygen.yml", "content": "name: doxygen\n\non:\n push:\n branches: [main]\n pull_request:\n branches: [main]\n\nenv:\n CMAKE_GENERATOR: Ninja\n\npermissions:\n contents: read\n\njobs:\n build-and-deploy:\n name: Build HTML documentation\n runs-on: ubuntu-latest\n steps:\n - name: Fetching sources\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Installing build dependencies\n run: |\n sudo apt update\n sudo apt install doxygen gcc git\n\n - name: Creating build directory\n run: mkdir build\n\n - name: Building HTML documentation with Doxygen\n run: |\n cmake -S . -B build -DBENCHMARK_ENABLE_TESTING:BOOL=OFF -DBENCHMARK_ENABLE_DOXYGEN:BOOL=ON -DBENCHMARK_INSTALL_DOCS:BOOL=ON\n cmake --build build --target benchmark_doxygen\n" }, { "path": ".github/workflows/ossf.yml", "content": "name: OSSF Scorecard Weekly\n\non:\n schedule:\n - cron: '0 0 * * 0' # Runs every Sunday at midnight UTC\n workflow_dispatch:\n\npermissions:\n contents: read\n\njobs:\n ossf-scorecard:\n # To write a badge\n permissions:\n id-token: write\n \n runs-on: ubuntu-latest\n steps:\n - name: Checkout repository\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Run analysis\n uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3\n with:\n publish_results: true\n results_file: ossf_scorecard.json\n results_format: json\n" }, { "path": ".github/workflows/pre-commit.yml", "content": "name: python + Bazel pre-commit checks\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n\njobs:\n pre-commit:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n persist-credentials: false\n - name: Install uv\n uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0\n with:\n python-version: 3.12\n - name: Run pre-commit checks\n run: uv run --only-group=dev pre-commit run --all-files --verbose --show-diff-on-failure\n" }, { "path": ".github/workflows/sanitizer.yml", "content": "name: sanitizer\n\non:\n push: {}\n pull_request: {}\n\nenv:\n CMAKE_GENERATOR: Ninja\n UBSAN_OPTIONS: \"print_stacktrace=1\"\n\njobs:\n job:\n name: ${{ matrix.sanitizer }}.${{ matrix.build_type }}\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n matrix:\n build_type: ['Debug', 'RelWithDebInfo']\n sanitizer: ['asan', 'ubsan', 'tsan', 'msan']\n\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: configure msan env\n if: matrix.sanitizer == 'msan'\n run: |\n echo \"EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins\" >> $GITHUB_ENV\n echo \"LIBCXX_SANITIZER=MemoryWithOrigins\" >> $GITHUB_ENV\n\n - name: configure ubsan env\n if: matrix.sanitizer == 'ubsan'\n run: |\n echo \"EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all\" >> $GITHUB_ENV\n echo \"LIBCXX_SANITIZER=Undefined\" >> $GITHUB_ENV\n\n - name: configure asan env\n if: matrix.sanitizer == 'asan'\n run: |\n echo \"EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=address -fno-sanitize-recover=all\" >> $GITHUB_ENV\n echo \"LIBCXX_SANITIZER=Address\" >> $GITHUB_ENV\n\n - name: configure tsan env\n if: matrix.sanitizer == 'tsan'\n run: |\n echo \"EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all\" >> $GITHUB_ENV\n echo \"LIBCXX_SANITIZER=Thread\" >> $GITHUB_ENV\n\n - name: fine-tune asan options\n # in asan we get an error from std::regex. ignore it.\n if: matrix.sanitizer == 'asan'\n run: |\n echo \"ASAN_OPTIONS=alloc_dealloc_mismatch=0\" >> $GITHUB_ENV\n\n - name: setup clang\n uses: egor-tensin/setup-clang@471a6f8ef1d449dba8e1a51780e7f943572a3f99 # v2.1\n with:\n version: latest\n platform: x64\n\n - name: configure clang\n run: |\n echo \"CC=cc\" >> $GITHUB_ENV\n echo \"CXX=c++\" >> $GITHUB_ENV\n\n - name: build libc++ (non-asan)\n if: matrix.sanitizer != 'asan'\n run: |\n \"${GITHUB_WORKSPACE}/.github/libcxx-setup.sh\"\n echo \"EXTRA_CXX_FLAGS=-stdlib=libc++ -L${GITHUB_WORKSPACE}/llvm-build/lib -lc++abi -I${GITHUB_WORKSPACE}/llvm-build/include/c++/v1 -Isystem${GITHUB_WORKSPACE}/llvm-build/include/c++/v1 -Wl,-rpath,${GITHUB_WORKSPACE}/llvm-build/lib\" >> $GITHUB_ENV\n\n - name: create build environment\n run: cmake -E make_directory ${{ runner.workspace }}/_build\n\n - name: configure cmake\n shell: bash\n working-directory: ${{ runner.workspace }}/_build\n run: >\n VERBOSE=1\n cmake -GNinja $GITHUB_WORKSPACE\n -DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF\n -DBENCHMARK_ENABLE_LIBPFM=OFF\n -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON\n -DCMAKE_C_COMPILER=${{ env.CC }}\n -DCMAKE_CXX_COMPILER=${{ env.CXX }}\n -DCMAKE_C_FLAGS=\"${{ env.EXTRA_FLAGS }}\"\n -DCMAKE_CXX_FLAGS=\"${{ env.EXTRA_FLAGS }} ${{ env.EXTRA_CXX_FLAGS }}\"\n -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}\n\n - name: build\n shell: bash\n working-directory: ${{ runner.workspace }}/_build\n run: cmake --build . --config ${{ matrix.build_type }}\n\n - name: test\n shell: bash\n working-directory: ${{ runner.workspace }}/_build\n run: ctest -C ${{ matrix.build_type }} -VV\n" }, { "path": ".github/workflows/test_bindings.yml", "content": "name: test-bindings\n\non:\n push:\n branches: [main]\n pull_request:\n branches: [main]\n\nenv:\n CMAKE_GENERATOR: Ninja\n\npermissions:\n contents: read\n\njobs:\n python_bindings:\n name: Test GBM Python ${{ matrix.python-version }} bindings on ${{ matrix.os }}\n runs-on: ${{ matrix.os }}\n strategy:\n fail-fast: false\n matrix:\n os: [ ubuntu-latest, macos-latest, windows-latest ]\n python-version: [ \"3.10\", \"3.11\", \"3.12\", \"3.13\" ]\n\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: ${{ matrix.python-version }}\n - name: Install GBM Python bindings on ${{ matrix.os }}\n run: python -m pip install .\n - name: Run example on ${{ matrix.os }} under Python ${{ matrix.python-version }}\n run: python bindings/python/google_benchmark/example.py\n" }, { "path": ".github/workflows/wheels.yml", "content": "name: Build and upload Python wheels\n\non:\n workflow_dispatch:\n release:\n types:\n - published\n\nenv:\n CMAKE_GENERATOR: Ninja\n\njobs:\n build_sdist:\n name: Build source distribution\n runs-on: ubuntu-latest\n steps:\n - name: Check out repo\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - name: Install Python 3.12\n uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: \"3.12\"\n - run: python -m pip install build\n - name: Build sdist\n run: python -m build --sdist\n - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0\n with:\n name: dist-sdist\n path: dist/*.tar.gz\n\n build_wheels:\n name: Build Google Benchmark wheels on ${{ matrix.os }}\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-latest, windows-latest]\n steps:\n - name: Check out Google Benchmark\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n name: Install Python 3.12\n with:\n python-version: \"3.12\"\n - name: Install the latest version of uv\n uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0\n\n - name: Build wheels on ${{ matrix.os }} using cibuildwheel\n uses: pypa/cibuildwheel@ee02a1537ce3071a004a6b08c41e72f0fdc42d9a # v3.4.0\n env:\n CIBW_BUILD: \"cp310-* cp311-* cp312-*\"\n CIBW_BUILD_FRONTEND: \"build[uv]\"\n CIBW_SKIP: \"*-musllinux_*\"\n CIBW_ARCHS: auto64\n CIBW_BEFORE_ALL_LINUX: bash .github/install_bazel.sh\n # Grab the rootless Bazel installation inside the container.\n CIBW_ENVIRONMENT_LINUX: PATH=$PATH:$HOME/bin\n CIBW_TEST_COMMAND: python {project}/bindings/python/google_benchmark/example.py\n # unused by Bazel, but needed explicitly by delocate on MacOS.\n MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-15-intel' && 10.14 || 11.0 }}\n\n - name: Upload Google Benchmark ${{ matrix.os }} wheels\n uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0\n with:\n name: dist-${{ matrix.os }}\n path: wheelhouse/*.whl\n\n pypi_upload:\n name: Publish google-benchmark wheels to PyPI\n needs: [build_sdist, build_wheels]\n runs-on: ubuntu-latest\n permissions:\n id-token: write\n steps:\n - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1\n with:\n path: dist\n pattern: dist-*\n merge-multiple: true\n - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1\n" }, { "path": ".gitignore", "content": "*.a\n*.so\n*.so.?*\n*.dll\n*.exe\n*.dylib\n*.cmake\n!/cmake/*.cmake\n!/test/AssemblyTests.cmake\n*~\n*.swp\n*.pyc\n__pycache__\n.DS_Store\n\n# lcov\n*.lcov\n/lcov\n\n# cmake files.\n/Testing\nCMakeCache.txt\nCMakeFiles/\ncmake_install.cmake\n\n# makefiles.\nMakefile\n\n# in-source build.\nbin/\nlib/\n/test/*_test\n\n# exuberant ctags.\ntags\n\n# YouCompleteMe configuration.\n.ycm_extra_conf.pyc\n\n# ninja generated files.\n.ninja_deps\n.ninja_log\nbuild.ninja\ninstall_manifest.txt\nrules.ninja\n\n# bazel output symlinks.\nbazel-*\nMODULE.bazel.lock\n\n# out-of-source build top-level folders.\nbuild/\n_build/\nbuild*/\n\n# in-source dependencies\n/googletest/\n\n# Visual Studio 2015/2017 cache/options directory\n.vs/\nCMakeSettings.json\n\n# Visual Studio Code cache/options directory\n.vscode/\n\n# Python build stuff\ndist/\n*.egg-info*\nuv.lock\n" }, { "path": ".pre-commit-config.yaml", "content": "repos:\n - repo: https://github.com/keith/pre-commit-buildifier\n rev: 8.2.1\n hooks:\n - id: buildifier\n - id: buildifier-lint\n - repo: https://github.com/pre-commit/mirrors-mypy\n rev: v1.18.2\n hooks:\n - id: mypy\n types_or: [ python, pyi ]\n args: [ \"--ignore-missing-imports\", \"--scripts-are-modules\" ]\n - repo: https://github.com/astral-sh/ruff-pre-commit\n rev: v0.14.0\n hooks:\n - id: ruff-check\n args: [ --fix, --exit-non-zero-on-fix ]\n - id: ruff-format\n" }, { "path": ".ycm_extra_conf.py", "content": "import os\n\nimport ycm_core\n\n# These are the compilation flags that will be used in case there's no\n# compilation database set (by default, one is not set).\n# CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR.\nflags = [\n \"-Wall\",\n \"-Werror\",\n \"-pedantic-errors\",\n \"-std=c++0x\",\n \"-fno-strict-aliasing\",\n \"-O3\",\n \"-DNDEBUG\",\n # ...and the same thing goes for the magic -x option which specifies the\n # language that the files to be compiled are written in. This is mostly\n # relevant for c++ headers.\n # For a C project, you would set this to 'c' instead of 'c++'.\n \"-x\",\n \"c++\",\n \"-I\",\n \"include\",\n \"-isystem\",\n \"/usr/include\",\n \"-isystem\",\n \"/usr/local/include\",\n]\n\n\n# Set this to the absolute path to the folder (NOT the file!) containing the\n# compile_commands.json file to use that instead of 'flags'. See here for\n# more details: http://clang.llvm.org/docs/JSONCompilationDatabase.html\n#\n# Most projects will NOT need to set this to anything; you can just change the\n# 'flags' list of compilation flags. Notice that YCM itself uses that approach.\ncompilation_database_folder = \"\"\n\nif os.path.exists(compilation_database_folder):\n database = ycm_core.CompilationDatabase(compilation_database_folder)\nelse:\n database = None\n\nSOURCE_EXTENSIONS = [\".cc\"]\n\n\ndef DirectoryOfThisScript():\n return os.path.dirname(os.path.abspath(__file__))\n\n\ndef MakeRelativePathsInFlagsAbsolute(flags, working_directory):\n if not working_directory:\n return list(flags)\n new_flags = []\n make_next_absolute = False\n path_flags = [\"-isystem\", \"-I\", \"-iquote\", \"--sysroot=\"]\n for flag in flags:\n new_flag = flag\n\n if make_next_absolute:\n make_next_absolute = False\n if not flag.startswith(\"/\"):\n new_flag = os.path.join(working_directory, flag)\n\n for path_flag in path_flags:\n if flag == path_flag:\n make_next_absolute = True\n break\n\n if flag.startswith(path_flag):\n path = flag[len(path_flag) :]\n new_flag = path_flag + os.path.join(working_directory, path)\n break\n\n if new_flag:\n new_flags.append(new_flag)\n return new_flags\n\n\ndef IsHeaderFile(filename):\n extension = os.path.splitext(filename)[1]\n return extension in [\".h\", \".hxx\", \".hpp\", \".hh\"]\n\n\ndef GetCompilationInfoForFile(filename):\n # The compilation_commands.json file generated by CMake does not have\n # entries for header files. So we do our best by asking the db for flags for\n # a corresponding source file, if any. If one exists, the flags for that\n # file should be good enough.\n if IsHeaderFile(filename):\n basename = os.path.splitext(filename)[0]\n for extension in SOURCE_EXTENSIONS:\n replacement_file = basename + extension\n if os.path.exists(replacement_file):\n compilation_info = database.GetCompilationInfoForFile(\n replacement_file\n )\n if compilation_info.compiler_flags_:\n return compilation_info\n return None\n return database.GetCompilationInfoForFile(filename)\n\n\ndef FlagsForFile(filename, **kwargs):\n if database:\n # Bear in mind that compilation_info.compiler_flags_ does NOT return a\n # python list, but a \"list-like\" StringVec object\n compilation_info = GetCompilationInfoForFile(filename)\n if not compilation_info:\n return None\n\n final_flags = MakeRelativePathsInFlagsAbsolute(\n compilation_info.compiler_flags_,\n compilation_info.compiler_working_dir_,\n )\n else:\n relative_to = DirectoryOfThisScript()\n final_flags = MakeRelativePathsInFlagsAbsolute(flags, relative_to)\n\n return {\"flags\": final_flags, \"do_cache\": True}\n" }, { "path": "AUTHORS", "content": "# This is the official list of benchmark authors for copyright purposes.\n# This file is distinct from the CONTRIBUTORS files.\n# See the latter for an explanation.\n#\n# Names should be added to this file as:\n#\tName or Organization \n# The email address is not required for organizations.\n#\n# Please keep the list sorted.\n\nAlbert Pretorius \nAlex Steele \nAndriy Berestovskyy \nArne Beer \nBenjamin King \nCarto\nCezary Skrzyński \nChristian Wassermann \nChristopher Seymour \nColin Braley \nDaniel Harvey \nDavid Coeurjolly \nDeniz Evrenci \nDirac Research \nDominik Czarnota \nDominik Korman \nDonald Aingworth \nEric Backus \nEric Fiselier \nEugene Zhuk \nEvgeny Safronov \nFabien Pichot \nFederico Ficarelli \nFelix Homann \nGergely Meszaros \nGergő Szitár \nGoogle Inc.\nHenrique Bucher \nInternational Business Machines Corporation\nIsmael Jimenez Martinez \nJern-Kuan Leong \nJianXiong Zhou \nJoao Paulo Magalhaes \nJordan Williams \nJussi Knuuttila \nKaito Udagawa \nKishan Kumar \nKostiantyn Lazukin \nLei Xu \nMarcel Jacobse \nMatt Clarkson \nMaxim Vafin \nMike Apodaca \nMin-Yih Hsu \nMongoDB Inc.\nNick Hutchinson \nNorman Heino \nOleksandr Sochka \nOlga Fadeeva \nOri Livneh \nPaul Redmond \nPrithvi Rao \nRadoslav Yovchev \nRaghu Raja \nRainer Orth \nRoman Lebedev \nSayan Bhattacharjee \nShapr3D \nShashank Thakur \nShuo Chen \nStaffan Tjernstrom \nSteinar H. Gunderson \nStripe, Inc.\nTobias Schmidt \nYixuan Qiu \nYusuke Suzuki \nZbigniew Skowron \n" }, { "path": "BUILD.bazel", "content": "load(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\n\nlicenses([\"notice\"])\n\nCOPTS = [\n \"-pedantic\",\n \"-pedantic-errors\",\n \"-std=c++17\",\n \"-Wall\",\n \"-Wconversion\",\n \"-Wextra\",\n \"-Wshadow\",\n # \"-Wshorten-64-to-32\",\n \"-Wfloat-equal\",\n \"-Wformat=2\",\n \"-fstrict-aliasing\",\n ## assert() are used a lot in tests upstream, which may be optimised out leading to\n ## unused-variable warning.\n \"-Wno-unused-variable\",\n \"-Werror=old-style-cast\",\n]\n\nMSVC_COPTS = [\n \"/std:c++17\",\n]\n\nconfig_setting(\n name = \"windows\",\n constraint_values = [\"@platforms//os:windows\"],\n visibility = [\":__subpackages__\"],\n)\n\nconfig_setting(\n name = \"perfcounters\",\n define_values = {\n \"pfm\": \"1\",\n },\n visibility = [\":__subpackages__\"],\n)\n\ncc_library(\n name = \"benchmark\",\n srcs = glob(\n [\n \"src/*.cc\",\n \"src/*.h\",\n ],\n exclude = [\"src/benchmark_main.cc\"],\n ),\n hdrs = [\n \"include/benchmark/benchmark.h\",\n \"include/benchmark/benchmark_api.h\",\n \"include/benchmark/counter.h\",\n \"include/benchmark/export.h\",\n \"include/benchmark/macros.h\",\n \"include/benchmark/managers.h\",\n \"include/benchmark/registration.h\",\n \"include/benchmark/reporter.h\",\n \"include/benchmark/state.h\",\n \"include/benchmark/statistics.h\",\n \"include/benchmark/sysinfo.h\",\n \"include/benchmark/types.h\",\n \"include/benchmark/utils.h\",\n ],\n copts = select({\n \":windows\": MSVC_COPTS,\n \"//conditions:default\": COPTS,\n }),\n defines = [\n \"BENCHMARK_STATIC_DEFINE\",\n \"BENCHMARK_VERSION=\\\\\\\"\" + (module_version() if module_version() != None else \"\") + \"\\\\\\\"\",\n ] + select({\n \":perfcounters\": [\"HAVE_LIBPFM\"],\n \"//conditions:default\": [],\n }),\n includes = [\"include\"],\n linkopts = select({\n \":windows\": [\"-DEFAULTLIB:shlwapi.lib\"],\n \"//conditions:default\": [\"-pthread\"],\n }),\n # Only static linking is allowed; no .so will be produced.\n # Using `defines` (i.e. not `local_defines`) means that no\n # dependent rules need to bother about defining the macro.\n linkstatic = True,\n local_defines = [\n # Turn on Large-file Support\n \"_FILE_OFFSET_BITS=64\",\n \"_LARGEFILE64_SOURCE\",\n \"_LARGEFILE_SOURCE\",\n ],\n visibility = [\"//visibility:public\"],\n deps = select({\n \":perfcounters\": [\"@libpfm\"],\n \"//conditions:default\": [],\n }),\n)\n\ncc_library(\n name = \"benchmark_main\",\n srcs = [\"src/benchmark_main.cc\"],\n hdrs = [\n \"include/benchmark/benchmark.h\",\n \"include/benchmark/benchmark_api.h\",\n \"include/benchmark/counter.h\",\n \"include/benchmark/export.h\",\n \"include/benchmark/macros.h\",\n \"include/benchmark/managers.h\",\n \"include/benchmark/registration.h\",\n \"include/benchmark/reporter.h\",\n \"include/benchmark/state.h\",\n \"include/benchmark/statistics.h\",\n \"include/benchmark/sysinfo.h\",\n \"include/benchmark/types.h\",\n \"include/benchmark/utils.h\",\n ],\n includes = [\"include\"],\n visibility = [\"//visibility:public\"],\n deps = [\":benchmark\"],\n)\n\ncc_library(\n name = \"benchmark_internal_headers\",\n hdrs = glob([\"src/*.h\"]),\n visibility = [\"//test:__pkg__\"],\n)\n" }, { "path": "CMakeLists.txt", "content": "# Require CMake 3.10. If available, use the policies up to CMake 3.22.\ncmake_minimum_required (VERSION 3.13...3.22)\n\nproject (benchmark VERSION 1.9.5 LANGUAGES CXX)\n\noption(BENCHMARK_ENABLE_TESTING \"Enable testing of the benchmark library.\" ON)\noption(BENCHMARK_ENABLE_EXCEPTIONS \"Enable the use of exceptions in the benchmark library.\" ON)\noption(BENCHMARK_ENABLE_LTO \"Enable link time optimisation of the benchmark library.\" OFF)\noption(BENCHMARK_USE_LIBCXX \"Build and test using libc++ as the standard library.\" OFF)\noption(BENCHMARK_ENABLE_WERROR \"Build Release candidates with -Werror.\" ON)\noption(BENCHMARK_FORCE_WERROR \"Build Release candidates with -Werror regardless of compiler issues.\" OFF)\n\nif(\"${CMAKE_CXX_COMPILER_ID}\" STREQUAL \"PGI\")\n # PGC++ maybe reporting false positives.\n set(BENCHMARK_ENABLE_WERROR OFF)\nendif()\nif(\"${CMAKE_CXX_COMPILER_ID}\" STREQUAL \"NVHPC\")\n set(BENCHMARK_ENABLE_WERROR OFF)\nendif()\nif(BENCHMARK_FORCE_WERROR)\n set(BENCHMARK_ENABLE_WERROR ON)\nendif(BENCHMARK_FORCE_WERROR)\n\nif(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL \"MSVC\"))\n option(BENCHMARK_BUILD_32_BITS \"Build a 32 bit version of the library.\" OFF)\nelse()\n set(BENCHMARK_BUILD_32_BITS OFF CACHE BOOL \"Build a 32 bit version of the library - unsupported when using MSVC)\" FORCE)\nendif()\noption(BENCHMARK_ENABLE_INSTALL \"Enable installation of benchmark. (Projects embedding benchmark may want to turn this OFF.)\" ON)\noption(BENCHMARK_ENABLE_DOXYGEN \"Build documentation with Doxygen.\" OFF)\noption(BENCHMARK_INSTALL_DOCS \"Enable installation of documentation.\" ON)\noption(BENCHMARK_INSTALL_TOOLS \"Enable installation of tools.\" ON)\n\n# Allow unmet dependencies to be met using CMake's ExternalProject mechanics, which\n# may require downloading the source code.\noption(BENCHMARK_DOWNLOAD_DEPENDENCIES \"Allow the downloading and in-tree building of unmet dependencies\" OFF)\n\n# This option can be used to disable building and running unit tests which depend on gtest\n# in cases where it is not possible to build or find a valid version of gtest.\noption(BENCHMARK_ENABLE_GTEST_TESTS \"Enable building the unit tests which depend on gtest\" ON)\noption(BENCHMARK_USE_BUNDLED_GTEST \"Use bundled GoogleTest. If disabled, the find_package(GTest) will be used.\" ON)\n\noption(BENCHMARK_ENABLE_LIBPFM \"Enable performance counters provided by libpfm\" OFF)\n\n# Export only public symbols\nset(CMAKE_CXX_VISIBILITY_PRESET hidden)\nset(CMAKE_VISIBILITY_INLINES_HIDDEN ON)\n\nif(CMAKE_CXX_COMPILER_ID STREQUAL \"MSVC\")\n # As of CMake 3.18, CMAKE_SYSTEM_PROCESSOR is not set properly for MSVC and\n # cross-compilation (e.g. Host=x86_64, target=aarch64) requires using the\n # undocumented, but working variable.\n # See https://gitlab.kitware.com/cmake/cmake/-/issues/15170\n set(CMAKE_SYSTEM_PROCESSOR ${MSVC_CXX_ARCHITECTURE_ID})\n if(${CMAKE_SYSTEM_PROCESSOR} MATCHES \"ARM\")\n set(CMAKE_CROSSCOMPILING TRUE)\n endif()\nendif()\n\nset(ENABLE_ASSEMBLY_TESTS_DEFAULT OFF)\nfunction(should_enable_assembly_tests)\n if(CMAKE_BUILD_TYPE)\n string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)\n if (${CMAKE_BUILD_TYPE_LOWER} MATCHES \"coverage\")\n # FIXME: The --coverage flag needs to be removed when building assembly\n # tests for this to work.\n return()\n endif()\n endif()\n if (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL \"MSVC\")\n return()\n elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES \"x86_64\")\n return()\n elseif(NOT CMAKE_SIZEOF_VOID_P EQUAL 8)\n # FIXME: Make these work on 32 bit builds\n return()\n elseif(BENCHMARK_BUILD_32_BITS)\n # FIXME: Make these work on 32 bit builds\n return()\n endif()\n find_program(LLVM_FILECHECK_EXE FileCheck)\n if (LLVM_FILECHECK_EXE)\n set(LLVM_FILECHECK_EXE \"${LLVM_FILECHECK_EXE}\" CACHE PATH \"llvm filecheck\" FORCE)\n message(STATUS \"LLVM FileCheck Found: ${LLVM_FILECHECK_EXE}\")\n else()\n message(STATUS \"Failed to find LLVM FileCheck\")\n return()\n endif()\n set(ENABLE_ASSEMBLY_TESTS_DEFAULT ON PARENT_SCOPE)\nendfunction()\nshould_enable_assembly_tests()\n\n# This option disables the building and running of the assembly verification tests\noption(BENCHMARK_ENABLE_ASSEMBLY_TESTS \"Enable building and running the assembly tests\"\n ${ENABLE_ASSEMBLY_TESTS_DEFAULT})\n\n# Make sure we can import out CMake functions\nlist(APPEND CMAKE_MODULE_PATH \"${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules\")\nlist(APPEND CMAKE_MODULE_PATH \"${CMAKE_CURRENT_SOURCE_DIR}/cmake\")\n\n\n# Read the git tags to determine the project version\ninclude(GetGitVersion)\nget_git_version(GIT_VERSION)\n\n# If no git version can be determined, use the version\n# from the project() command\nif (\"${GIT_VERSION}\" STREQUAL \"v0.0.0\")\n set(VERSION \"v${benchmark_VERSION}\")\nelse()\n set(VERSION \"${GIT_VERSION}\")\nendif()\n\n# Normalize version: drop \"v\" prefix, replace first \"-\" with \".\",\n# drop everything after second \"-\" (including said \"-\").\nstring(STRIP ${VERSION} VERSION)\nif(VERSION MATCHES v[^-]*-)\n string(REGEX REPLACE \"v([^-]*)-([0-9]+)-.*\" \"\\\\1.\\\\2\" NORMALIZED_VERSION ${VERSION})\nelse()\n string(REGEX REPLACE \"v(.*)\" \"\\\\1\" NORMALIZED_VERSION ${VERSION})\nendif()\n\n# Tell the user what versions we are using\nmessage(STATUS \"Google Benchmark version: ${VERSION}, normalized to ${NORMALIZED_VERSION}\")\n\n# The version of the libraries\nset(GENERIC_LIB_VERSION ${NORMALIZED_VERSION})\nstring(SUBSTRING ${NORMALIZED_VERSION} 0 1 GENERIC_LIB_SOVERSION)\n\n# Import our CMake modules\ninclude(AddCXXCompilerFlag)\ninclude(CheckCXXCompilerFlag)\ninclude(CheckLibraryExists)\ninclude(CXXFeatureCheck)\n\n# Check for rt library, but explicitly disable for QNX\nif(QNXNTO)\n set(HAVE_LIB_RT FALSE)\nelse()\n check_library_exists(rt shm_open \"\" HAVE_LIB_RT)\nendif()\n\nif (BENCHMARK_BUILD_32_BITS)\n add_required_cxx_compiler_flag(-m32)\nendif()\n\nset(BENCHMARK_CXX_STANDARD 17)\n\nset(CMAKE_CXX_STANDARD ${BENCHMARK_CXX_STANDARD})\nset(CMAKE_CXX_STANDARD_REQUIRED YES)\nset(CMAKE_CXX_EXTENSIONS OFF)\n\nif (MSVC)\n # Turn compiler warnings up to 11\n string(REGEX REPLACE \"[-/]W[1-4]\" \"\" CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS}\")\n set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} /W4\")\n\n # MP flag only applies to cl, not cl frontends to other compilers (e.g. clang-cl, icx-cl etc)\n if(CMAKE_CXX_COMPILER_ID MATCHES MSVC)\n set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} /MP\")\n endif()\n add_definitions(-D_CRT_SECURE_NO_WARNINGS)\n\n if(BENCHMARK_ENABLE_WERROR)\n add_cxx_compiler_flag(-WX)\n endif()\n\n if (NOT BENCHMARK_ENABLE_EXCEPTIONS)\n add_cxx_compiler_flag(-EHs-)\n add_cxx_compiler_flag(-EHa-)\n add_definitions(-D_HAS_EXCEPTIONS=0)\n endif()\n # Link time optimisation\n if (BENCHMARK_ENABLE_LTO)\n set(CMAKE_CXX_FLAGS_RELEASE \"${CMAKE_CXX_FLAGS_RELEASE} /GL\")\n set(CMAKE_STATIC_LINKER_FLAGS_RELEASE \"${CMAKE_STATIC_LINKER_FLAGS_RELEASE} /LTCG\")\n set(CMAKE_SHARED_LINKER_FLAGS_RELEASE \"${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG\")\n set(CMAKE_EXE_LINKER_FLAGS_RELEASE \"${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG\")\n\n set(CMAKE_CXX_FLAGS_RELWITHDEBINFO \"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /GL\")\n string(REGEX REPLACE \"[-/]INCREMENTAL\" \"/INCREMENTAL:NO\" CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO \"${CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO}\")\n set(CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO \"${CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO} /LTCG\")\n string(REGEX REPLACE \"[-/]INCREMENTAL\" \"/INCREMENTAL:NO\" CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO \"${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO}\")\n set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO \"${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO} /LTCG\")\n string(REGEX REPLACE \"[-/]INCREMENTAL\" \"/INCREMENTAL:NO\" CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO \"${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO}\")\n set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO \"${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} /LTCG\")\n\n set(CMAKE_CXX_FLAGS_MINSIZEREL \"${CMAKE_CXX_FLAGS_MINSIZEREL} /GL\")\n set(CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL \"${CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL} /LTCG\")\n set(CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL \"${CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL} /LTCG\")\n set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL \"${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL} /LTCG\")\n endif()\nelse()\n # Turn on Large-file Support\n add_definitions(-D_FILE_OFFSET_BITS=64)\n add_definitions(-D_LARGEFILE64_SOURCE)\n add_definitions(-D_LARGEFILE_SOURCE)\n # Turn compiler warnings up to 11\n add_cxx_compiler_flag(-Wall)\n add_cxx_compiler_flag(-Wextra)\n add_cxx_compiler_flag(-Wshadow)\n add_cxx_compiler_flag(-Wfloat-equal)\n add_cxx_compiler_flag(-Wold-style-cast)\n add_cxx_compiler_flag(-Wconversion)\n add_cxx_compiler_flag(-Wformat=2)\n if(BENCHMARK_ENABLE_WERROR)\n add_cxx_compiler_flag(-Werror)\n endif()\n if (NOT BENCHMARK_ENABLE_TESTING)\n # Disable warning when compiling tests as gtest does not use 'override'.\n add_cxx_compiler_flag(-Wsuggest-override)\n endif()\n add_cxx_compiler_flag(-pedantic)\n add_cxx_compiler_flag(-pedantic-errors)\n add_cxx_compiler_flag(-Wshorten-64-to-32)\n add_cxx_compiler_flag(-fstrict-aliasing)\n # Disable warnings regarding deprecated parts of the library while building\n # and testing those parts of the library.\n add_cxx_compiler_flag(-Wno-deprecated-declarations)\n if (CMAKE_CXX_COMPILER_ID STREQUAL \"Intel\" OR CMAKE_CXX_COMPILER_ID STREQUAL \"IntelLLVM\")\n # Intel silently ignores '-Wno-deprecated-declarations',\n # warning no. 1786 must be explicitly disabled.\n # See #631 for rationale.\n add_cxx_compiler_flag(-wd1786)\n add_cxx_compiler_flag(-fno-finite-math-only)\n # ICC17u2: overloaded virtual function \"benchmark::Fixture::SetUp\" is only partially\n # overridden (because of deprecated overload)\n add_cxx_compiler_flag(-wd654)\n endif()\n # Disable deprecation warnings for release builds (when -Werror is enabled).\n if(BENCHMARK_ENABLE_WERROR)\n add_cxx_compiler_flag(-Wno-deprecated)\n endif()\n if (NOT BENCHMARK_ENABLE_EXCEPTIONS)\n add_cxx_compiler_flag(-fno-exceptions)\n endif()\n\n if (HAVE_CXX_FLAG_FSTRICT_ALIASING)\n if (NOT CMAKE_CXX_COMPILER_ID STREQUAL \"Intel\" AND NOT CMAKE_CXX_COMPILER_ID STREQUAL \"IntelLLVM\") #ICC17u2: Many false positives for Wstrict-aliasing\n add_cxx_compiler_flag(-Wstrict-aliasing)\n endif()\n endif()\n\n add_cxx_compiler_flag(-Wthread-safety)\n if (HAVE_CXX_FLAG_WTHREAD_SAFETY)\n cxx_feature_check(THREAD_SAFETY_ATTRIBUTES \"-DINCLUDE_DIRECTORIES=${PROJECT_SOURCE_DIR}/include\")\n endif()\n\n # On most UNIX like platforms g++ and clang++ define _GNU_SOURCE as a\n # predefined macro, which turns on all of the wonderful libc extensions.\n # However g++ doesn't do this in Cygwin so we have to define it ourselves\n # since we depend on GNU/POSIX/BSD extensions.\n if (CYGWIN)\n add_definitions(-D_GNU_SOURCE=1)\n endif()\n\n if (QNXNTO)\n add_definitions(-D_QNX_SOURCE)\n endif()\n\n # Link time optimisation\n if (BENCHMARK_ENABLE_LTO)\n add_cxx_compiler_flag(-flto)\n add_cxx_compiler_flag(-Wno-lto-type-mismatch)\n if (\"${CMAKE_CXX_COMPILER_ID}\" STREQUAL \"GNU\")\n find_program(GCC_AR gcc-ar)\n if (GCC_AR)\n set(CMAKE_AR ${GCC_AR})\n endif()\n find_program(GCC_RANLIB gcc-ranlib)\n if (GCC_RANLIB)\n set(CMAKE_RANLIB ${GCC_RANLIB})\n endif()\n elseif(\"${CMAKE_CXX_COMPILER_ID}\" MATCHES \"Clang\")\n include(llvm-toolchain)\n endif()\n endif()\n\n # Coverage build type\n set(BENCHMARK_CXX_FLAGS_COVERAGE \"${CMAKE_CXX_FLAGS_DEBUG}\"\n CACHE STRING \"Flags used by the C++ compiler during coverage builds.\"\n FORCE)\n set(BENCHMARK_EXE_LINKER_FLAGS_COVERAGE \"${CMAKE_EXE_LINKER_FLAGS_DEBUG}\"\n CACHE STRING \"Flags used for linking binaries during coverage builds.\"\n FORCE)\n set(BENCHMARK_SHARED_LINKER_FLAGS_COVERAGE \"${CMAKE_SHARED_LINKER_FLAGS_DEBUG}\"\n CACHE STRING \"Flags used by the shared libraries linker during coverage builds.\"\n FORCE)\n mark_as_advanced(\n BENCHMARK_CXX_FLAGS_COVERAGE\n BENCHMARK_EXE_LINKER_FLAGS_COVERAGE\n BENCHMARK_SHARED_LINKER_FLAGS_COVERAGE)\n set(CMAKE_BUILD_TYPE \"${CMAKE_BUILD_TYPE}\" CACHE STRING\n \"Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel Coverage.\")\n add_cxx_compiler_flag(--coverage COVERAGE)\nendif()\n\nif (BENCHMARK_USE_LIBCXX)\n if (\"${CMAKE_CXX_COMPILER_ID}\" MATCHES \"Clang\")\n add_cxx_compiler_flag(-stdlib=libc++)\n elseif (\"${CMAKE_CXX_COMPILER_ID}\" STREQUAL \"GNU\" OR\n \"${CMAKE_CXX_COMPILER_ID}\" STREQUAL \"Intel\" OR\n \"${CMAKE_CXX_COMPILER_ID}\" STREQUAL \"IntelLLVM\")\n add_cxx_compiler_flag(-nostdinc++)\n message(WARNING \"libc++ header path must be manually specified using CMAKE_CXX_FLAGS\")\n # Adding -nodefaultlibs directly to CMAKE__LINKER_FLAGS will break\n # configuration checks such as 'find_package(Threads)'\n list(APPEND BENCHMARK_CXX_LINKER_FLAGS -nodefaultlibs)\n # -lc++ cannot be added directly to CMAKE__LINKER_FLAGS because\n # linker flags appear before all linker inputs and -lc++ must appear after.\n list(APPEND BENCHMARK_CXX_LIBRARIES c++)\n else()\n message(FATAL_ERROR \"-DBENCHMARK_USE_LIBCXX:BOOL=ON is not supported for compiler\")\n endif()\nendif(BENCHMARK_USE_LIBCXX)\n\n# C++ feature checks\n# Determine the correct regular expression engine to use\ncxx_feature_check(STD_REGEX)\ncxx_feature_check(GNU_POSIX_REGEX)\ncxx_feature_check(POSIX_REGEX)\nif(NOT HAVE_STD_REGEX AND NOT HAVE_GNU_POSIX_REGEX AND NOT HAVE_POSIX_REGEX)\n message(FATAL_ERROR \"Failed to determine the source files for the regular expression backend\")\nendif()\nif (NOT BENCHMARK_ENABLE_EXCEPTIONS AND HAVE_STD_REGEX\n AND NOT HAVE_GNU_POSIX_REGEX AND NOT HAVE_POSIX_REGEX)\n message(WARNING \"Using std::regex with exceptions disabled is not fully supported\")\nendif()\n\ncxx_feature_check(STEADY_CLOCK)\n# Ensure we have pthreads\nset(THREADS_PREFER_PTHREAD_FLAG ON)\nfind_package(Threads REQUIRED)\n\n# cxx_feature_check relies on try_run to probe compiler features. Because this\n# check does not produce a real target, target_link_libraries cannot be applied.\n# Therefore, link libraries are forwarded to try_run through\n# BENCHMARK_CXX_LIBRARIES.\nlist(APPEND BENCHMARK_CXX_LIBRARIES Threads::Threads)\ncxx_feature_check(PTHREAD_AFFINITY)\n\nif (BENCHMARK_ENABLE_LIBPFM)\n find_package(PFM REQUIRED)\nendif()\n\n# Set up directories\ninclude_directories(${PROJECT_SOURCE_DIR}/include)\n\n# Build the targets\nadd_subdirectory(src)\n\nif (BENCHMARK_ENABLE_TESTING)\n enable_testing()\n if (BENCHMARK_ENABLE_GTEST_TESTS AND\n NOT (TARGET gtest AND TARGET gtest_main AND\n TARGET gmock AND TARGET gmock_main))\n if (BENCHMARK_USE_BUNDLED_GTEST)\n include(GoogleTest)\n else()\n find_package(GTest CONFIG REQUIRED)\n add_library(gtest ALIAS GTest::gtest)\n add_library(gtest_main ALIAS GTest::gtest_main)\n add_library(gmock ALIAS GTest::gmock)\n add_library(gmock_main ALIAS GTest::gmock_main)\n endif()\n endif()\n add_subdirectory(test)\nendif()\n" }, { "path": "CONTRIBUTING.md", "content": "# How to contribute #\n\nWe'd love to accept your patches and contributions to this project. There are\na just a few small guidelines you need to follow.\n\n\n## Contributor License Agreement ##\n\nContributions to any Google project must be accompanied by a Contributor\nLicense Agreement. This is not a copyright **assignment**, it simply gives\nGoogle permission to use and redistribute your contributions as part of the\nproject.\n\n * If you are an individual writing original source code and you're sure you\n own the intellectual property, then you'll need to sign an [individual\n CLA][].\n\n * If you work for a company that wants to allow you to contribute your work,\n then you'll need to sign a [corporate CLA][].\n\nYou generally only need to submit a CLA once, so if you've already submitted\none (even if it was for a different project), you probably don't need to do it\nagain.\n\n[individual CLA]: https://developers.google.com/open-source/cla/individual\n[corporate CLA]: https://developers.google.com/open-source/cla/corporate\n\nOnce your CLA is submitted (or if you already submitted one for\nanother Google project), make a commit adding yourself to the\n[AUTHORS][] and [CONTRIBUTORS][] files. This commit can be part\nof your first [pull request][].\n\n[AUTHORS]: AUTHORS\n[CONTRIBUTORS]: CONTRIBUTORS\n\n\n## Submitting a patch ##\n\n 1. It's generally best to start by opening a new issue describing the bug or\n feature you're intending to fix. Even if you think it's relatively minor,\n it's helpful to know what people are working on. Mention in the initial\n issue that you are planning to work on that bug or feature so that it can\n be assigned to you.\n\n 1. Follow the normal process of [forking][] the project, and setup a new\n branch to work in. It's important that each group of changes be done in\n separate branches in order to ensure that a pull request only includes the\n commits related to that bug or feature.\n\n 1. Do your best to have [well-formed commit messages][] for each change.\n This provides consistency throughout the project, and ensures that commit\n messages are able to be formatted properly by various git tools.\n\n 1. Finally, push the commits to your fork and submit a [pull request][].\n\n[forking]: https://help.github.com/articles/fork-a-repo\n[well-formed commit messages]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html\n[pull request]: https://help.github.com/articles/creating-a-pull-request\n" }, { "path": "CONTRIBUTORS", "content": "# People who have agreed to one of the CLAs and can contribute patches.\n# The AUTHORS file lists the copyright holders; this file\n# lists people. For example, Google employees are listed here\n# but not in AUTHORS, because Google holds the copyright.\n#\n# Names should be added to this file only after verifying that\n# the individual or the individual's organization has agreed to\n# the appropriate Contributor License Agreement, found here:\n#\n# https://developers.google.com/open-source/cla/individual\n# https://developers.google.com/open-source/cla/corporate\n#\n# The agreement for individuals can be filled out on the web.\n#\n# When adding J Random Contributor's name to this file,\n# either J's name or J's organization's name should be\n# added to the AUTHORS file, depending on whether the\n# individual or corporate CLA was used.\n#\n# Names should be added to this file as:\n# Name \n#\n# Please keep the list sorted.\n\nAbhina Sreeskantharajan \nAlbert Pretorius \nAlex Steele \nAndriy Berestovskyy \nArne Beer \nBátor Tallér \nBenjamin King \nBilly Robert O'Neal III \nCezary Skrzyński \nChris Kennelly \nChristian Wassermann \nChristopher Seymour \nColin Braley \nCyrille Faucheux \nDaniel Harvey \nDavid Coeurjolly \nDeniz Evrenci \nDominic Hamon \nDominik Czarnota \nDominik Korman \nDonald Aingworth \nDoug Evans \nEric Backus \nEric Fiselier \nEugene Zhuk \nEvgeny Safronov \nFabien Pichot \nFanbo Meng \nFederico Ficarelli \nFelix Homann \nGeoffrey Martin-Noble \nGergely Meszaros \nGergő Szitár \nHannes Hauswedell \nHenrique Bucher \nIsmael Jimenez Martinez \nIakov Sergeev \nJern-Kuan Leong \nJianXiong Zhou \nJoao Paulo Magalhaes \nJohn Millikin \nJordan Williams \nJussi Knuuttila \nKaito Udagawa \nKai Wolf \nKishan Kumar \nKostiantyn Lazukin \nLei Xu \nMarcel Jacobse \nMatt Clarkson \nMaxim Vafin \nMike Apodaca \nMin-Yih Hsu \nNick Hutchinson \nNorman Heino \nOleksandr Sochka \nOlga Fadeeva \nOri Livneh \nPascal Leroy \nPaul Redmond \nPierre Phaneuf \nPrithvi Rao \nRadoslav Yovchev \nRaghu Raja \nRainer Orth \nRaul Marin \nRay Glover \nRobert Guo \nRoman Lebedev \nSayan Bhattacharjee \nShashank Thakur \nShengYi Hung \nShuo Chen \nSteven Wan \nTobias Schmidt \nTobias Ulvgård \nTom Madams \nYixuan Qiu \nYusuke Suzuki \nZbigniew Skowron \n" }, { "path": "LICENSE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "MODULE.bazel", "content": "module(\n name = \"google_benchmark\",\n version = \"1.9.5\",\n)\n\nbazel_dep(name = \"bazel_skylib\", version = \"1.7.1\")\nbazel_dep(name = \"platforms\", version = \"0.0.10\")\nbazel_dep(name = \"rules_cc\", version = \"0.0.9\")\n\nbazel_dep(name = \"rules_python\", version = \"1.0.0\", dev_dependency = True)\nbazel_dep(name = \"googletest\", version = \"1.14.0\", dev_dependency = True, repo_name = \"com_google_googletest\")\n\nbazel_dep(name = \"libpfm\", version = \"4.11.0.bcr.1\")\n\n# Register a toolchain for Python 3.9 to be able to build numpy. Python\n# versions >=3.10 are problematic.\n# A second reason for this is to be able to build Python hermetically instead\n# of relying on the changing default version from rules_python.\n\npython = use_extension(\"@rules_python//python/extensions:python.bzl\", \"python\", dev_dependency = True)\npython.toolchain(python_version = \"3.8\")\npython.toolchain(python_version = \"3.9\")\npython.toolchain(python_version = \"3.10\")\npython.toolchain(python_version = \"3.11\")\npython.toolchain(\n is_default = True,\n python_version = \"3.12\",\n)\npython.toolchain(python_version = \"3.13\")\n\npip = use_extension(\"@rules_python//python/extensions:pip.bzl\", \"pip\", dev_dependency = True)\npip.parse(\n hub_name = \"tools_pip_deps\",\n python_version = \"3.12\",\n requirements_lock = \"//tools:requirements.txt\",\n)\nuse_repo(pip, \"tools_pip_deps\")\n\n# -- bazel_dep definitions -- #\n\nbazel_dep(name = \"nanobind_bazel\", version = \"2.9.2\", dev_dependency = True)\n" }, { "path": "README.md", "content": "# Benchmark\n\n[![build-and-test](https://github.com/google/benchmark/workflows/build-and-test/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Abuild-and-test)\n[![bazel](https://github.com/google/benchmark/actions/workflows/bazel.yml/badge.svg)](https://github.com/google/benchmark/actions/workflows/bazel.yml)\n[![test-bindings](https://github.com/google/benchmark/workflows/test-bindings/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Atest-bindings)\n[![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark)\n[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/google/benchmark/badge)](https://securityscorecards.dev/viewer/?uri=github.com/google/benchmark)\n\n[![Discord](https://discordapp.com/api/guilds/1125694995928719494/widget.png?style=shield)](https://discord.gg/cz7UX7wKC2)\n\nA library to benchmark code snippets, similar to unit tests. Example:\n\n```c++\n#include \n#include \n\nstatic void BM_SomeFunction(benchmark::State& state) {\n // Perform setup here\n for (auto _ : state) {\n // This code gets timed\n SomeFunction();\n }\n}\n// Register the function as a benchmark\nBENCHMARK(BM_SomeFunction);\n// Run the benchmark\nBENCHMARK_MAIN();\n```\n\n## Getting Started\n\nTo get started, see [Requirements](#requirements) and\n[Installation](#installation). See [Usage](#usage) for a full example and the\n[User Guide](docs/user_guide.md) for a more comprehensive feature overview.\n\nIt may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/main/docs/primer.md)\nas some of the structural aspects of the APIs are similar.\n\n## Resources\n\n[Discussion group](https://groups.google.com/d/forum/benchmark-discuss)\n\nIRC channels:\n* [libera](https://libera.chat) #benchmark\n\n[Additional Tooling Documentation](docs/tools.md)\n\n[Assembly Testing Documentation](docs/AssemblyTests.md)\n\n[Building and installing Python bindings](docs/python_bindings.md)\n\n## Requirements\n\nThe library can be used with C++11. However, it requires C++17 to build,\nincluding compiler and standard library support.\n\n_See [dependencies.md](docs/dependencies.md) for more details regarding supported\ncompilers and standards._\n\nIf you have need for a particular compiler to be supported, patches are very welcome.\n\nSee [Platform-Specific Build Instructions](docs/platform_specific_build_instructions.md).\n\n## Installation\n\nThis describes the installation process using cmake. As pre-requisites, you'll\nneed git and cmake installed.\n\n_See [dependencies.md](docs/dependencies.md) for more details regarding supported\nversions of build tools._\n\n```bash\n# Check out the library.\n$ git clone https://github.com/google/benchmark.git\n# Go to the library root directory\n$ cd benchmark\n# Make a build directory to place the build output.\n$ cmake -E make_directory \"build\"\n# Generate build system files with cmake, and download any dependencies.\n$ cmake -E chdir \"build\" cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=on -DCMAKE_BUILD_TYPE=Release ../\n# or, starting with CMake 3.13, use a simpler form:\n# cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=on -DCMAKE_BUILD_TYPE=Release -S . -B \"build\"\n# Build the library.\n$ cmake --build \"build\" --config Release\n```\nThis builds the `benchmark` and `benchmark_main` libraries and tests.\nOn a unix system, the build directory should now look something like this:\n\n```\n/benchmark\n /build\n /src\n /libbenchmark.a\n /libbenchmark_main.a\n /test\n ...\n```\n\nNext, you can run the tests to check the build.\n\n```bash\n$ cmake -E chdir \"build\" ctest --build-config Release\n```\n\nIf you want to install the library globally, also run:\n\n```\nsudo cmake --build \"build\" --config Release --target install\n```\n\nNote that Google Benchmark requires Google Test to build and run the tests. This\ndependency can be provided two ways:\n\n* Checkout the Google Test sources into `benchmark/googletest`.\n* Otherwise, if `-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON` is specified during\n configuration as above, the library will automatically download and build\n any required dependencies.\n\nIf you do not wish to build and run the tests, add `-DBENCHMARK_ENABLE_GTEST_TESTS=OFF`\nto `CMAKE_ARGS`.\n\n### Debug vs Release\n\nBy default, benchmark builds as a debug library. You will see a warning in the\noutput when this is the case. To build it as a release library instead, add\n`-DCMAKE_BUILD_TYPE=Release` when generating the build system files, as shown\nabove. The use of `--config Release` in build commands is needed to properly\nsupport multi-configuration tools (like Visual Studio for example) and can be\nskipped for other build systems (like Makefile).\n\nTo enable link-time optimisation, also add `-DBENCHMARK_ENABLE_LTO=true` when\ngenerating the build system files.\n\nIf you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake\ncache variables, if autodetection fails.\n\nIf you are using clang, you may need to set `LLVMAR_EXECUTABLE`,\n`LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables.\n\nTo enable sanitizer checks (eg., `asan` and `tsan`), add:\n```\n -DCMAKE_C_FLAGS=\"-g -O2 -fno-omit-frame-pointer -fsanitize=address -fsanitize=thread -fno-sanitize-recover=all\"\n -DCMAKE_CXX_FLAGS=\"-g -O2 -fno-omit-frame-pointer -fsanitize=address -fsanitize=thread -fno-sanitize-recover=all \" \n```\n\n### Stable and Experimental Library Versions\n\nThe main branch contains the latest stable version of the benchmarking library;\nthe API of which can be considered largely stable, with source breaking changes\nbeing made only upon the release of a new major version.\n\nNewer, experimental, features are implemented and tested on the\n[`v2` branch](https://github.com/google/benchmark/tree/v2). Users who wish\nto use, test, and provide feedback on the new features are encouraged to try\nthis branch. However, this branch provides no stability guarantees and reserves\nthe right to change and break the API at any time.\n\n## Usage\n\n### Basic usage\n\nDefine a function that executes the code to measure, register it as a benchmark\nfunction using the `BENCHMARK` macro, and ensure an appropriate `main` function\nis available:\n\n```c++\n#include \n\nstatic void BM_StringCreation(benchmark::State& state) {\n for (auto _ : state)\n std::string empty_string;\n}\n// Register the function as a benchmark\nBENCHMARK(BM_StringCreation);\n\n// Define another benchmark\nstatic void BM_StringCopy(benchmark::State& state) {\n std::string x = \"hello\";\n for (auto _ : state)\n std::string copy(x);\n}\nBENCHMARK(BM_StringCopy);\n\nBENCHMARK_MAIN();\n```\n\nTo run the benchmark, compile and link against the `benchmark` library\n(libbenchmark.a/.so). If you followed the build steps above, this library will \nbe under the build directory you created.\n\n```bash\n# Example on linux after running the build steps above. Assumes the\n# `benchmark` and `build` directories are under the current directory.\n$ g++ mybenchmark.cc -std=c++11 -isystem benchmark/include \\\n -Lbenchmark/build/src -lbenchmark -lpthread -o mybenchmark\n```\n\nAlternatively, link against the `benchmark_main` library and remove\n`BENCHMARK_MAIN();` above to get the same behavior.\n\nThe compiled executable will run all benchmarks by default. Pass the `--help`\nflag for option information or see the [User Guide](docs/user_guide.md).\n\n### Usage with CMake\n\nIf using CMake, it is recommended to link against the project-provided\n`benchmark::benchmark` and `benchmark::benchmark_main` targets using\n`target_link_libraries`.\nIt is possible to use ```find_package``` to import an installed version of the\nlibrary.\n```cmake\nfind_package(benchmark REQUIRED)\n```\nAlternatively, ```add_subdirectory``` will incorporate the library directly in\nto one's CMake project.\n```cmake\nadd_subdirectory(benchmark)\n```\nEither way, link to the library as follows.\n```cmake\ntarget_link_libraries(MyTarget benchmark::benchmark)\n```\n" }, { "path": "WORKSPACE", "content": "workspace(name = \"com_github_google_benchmark\")\n\nload(\"//:bazel/benchmark_deps.bzl\", \"benchmark_deps\")\n\nbenchmark_deps()\n\nload(\"@rules_python//python:repositories.bzl\", \"py_repositories\")\n\npy_repositories()\n\nload(\"@rules_python//python:pip.bzl\", \"pip_parse\")\n\npip_parse(\n name = \"tools_pip_deps\",\n requirements_lock = \"//tools:requirements.txt\",\n)\n\nload(\"@tools_pip_deps//:requirements.bzl\", \"install_deps\")\n\ninstall_deps()\n" }, { "path": "WORKSPACE.bzlmod", "content": "# This file marks the root of the Bazel workspace.\n# See MODULE.bazel for dependencies and setup.\n" }, { "path": "_config.yml", "content": "theme: jekyll-theme-midnight\nmarkdown: GFM\n" }, { "path": "appveyor.yml", "content": "version: '{build}'\n\nimage: Visual Studio 2017\n\nconfiguration:\n - Debug\n - Release\n\nenvironment:\n matrix:\n - compiler: msvc-15-seh\n generator: \"Visual Studio 15 2017\"\n\n - compiler: msvc-15-seh\n generator: \"Visual Studio 15 2017 Win64\"\n\n - compiler: msvc-14-seh\n generator: \"Visual Studio 14 2015\"\n\n - compiler: msvc-14-seh\n generator: \"Visual Studio 14 2015 Win64\"\n\n - compiler: gcc-5.3.0-posix\n generator: \"MinGW Makefiles\"\n cxx_path: 'C:\\mingw-w64\\i686-5.3.0-posix-dwarf-rt_v4-rev0\\mingw32\\bin'\n APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015\n\nmatrix:\n fast_finish: true\n\ninstall:\n # git bash conflicts with MinGW makefiles\n - if \"%generator%\"==\"MinGW Makefiles\" (set \"PATH=%PATH:C:\\Program Files\\Git\\usr\\bin;=%\")\n - if not \"%cxx_path%\"==\"\" (set \"PATH=%PATH%;%cxx_path%\")\n\nbuild_script:\n - md _build -Force\n - cd _build\n - echo %configuration%\n - cmake -G \"%generator%\" \"-DCMAKE_BUILD_TYPE=%configuration%\" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON ..\n - cmake --build . --config %configuration%\n\ntest_script:\n - ctest --build-config %configuration% --timeout 300 --output-on-failure\n\nartifacts:\n - path: '_build/CMakeFiles/*.log'\n name: logs\n - path: '_build/Testing/**/*.xml'\n name: test_results\n" }, { "path": "bazel/benchmark_deps.bzl", "content": "\"\"\"\nThis file contains the Bazel build dependencies for Google Benchmark (both C++ source and Python bindings).\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\n\ndef benchmark_deps():\n \"\"\"Loads dependencies required to build Google Benchmark.\"\"\"\n\n if \"bazel_skylib\" not in native.existing_rules():\n http_archive(\n name = \"bazel_skylib\",\n sha256 = \"cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94\",\n urls = [\n \"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz\",\n \"https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz\",\n ],\n )\n\n if \"rules_python\" not in native.existing_rules():\n http_archive(\n name = \"rules_python\",\n sha256 = \"e85ae30de33625a63eca7fc40a94fea845e641888e52f32b6beea91e8b1b2793\",\n strip_prefix = \"rules_python-0.27.1\",\n url = \"https://github.com/bazelbuild/rules_python/releases/download/0.27.1/rules_python-0.27.1.tar.gz\",\n )\n\n if \"com_google_googletest\" not in native.existing_rules():\n new_git_repository(\n name = \"com_google_googletest\",\n remote = \"https://github.com/google/googletest.git\",\n tag = \"release-1.12.1\",\n )\n\n if \"nanobind\" not in native.existing_rules():\n new_git_repository(\n name = \"nanobind\",\n remote = \"https://github.com/wjakob/nanobind.git\",\n tag = \"v1.9.2\",\n build_file = \"@//bindings/python:nanobind.BUILD\",\n recursive_init_submodules = True,\n )\n\n if \"libpfm\" not in native.existing_rules():\n # Downloaded from v4.9.0 tag at https://sourceforge.net/p/perfmon2/libpfm4/ref/master/tags/\n http_archive(\n name = \"libpfm\",\n build_file = str(Label(\"//tools:libpfm.BUILD.bazel\")),\n sha256 = \"5da5f8872bde14b3634c9688d980f68bda28b510268723cc12973eedbab9fecc\",\n type = \"tar.gz\",\n strip_prefix = \"libpfm-4.11.0\",\n urls = [\"https://sourceforge.net/projects/perfmon2/files/libpfm4/libpfm-4.11.0.tar.gz/download\"],\n )\n" }, { "path": "bindings/python/google_benchmark/BUILD", "content": "load(\"@nanobind_bazel//:build_defs.bzl\", \"nanobind_extension\", \"nanobind_stubgen\")\nload(\"@rules_python//python:defs.bzl\", \"py_library\", \"py_test\")\n\npy_library(\n name = \"google_benchmark\",\n srcs = [\"__init__.py\"],\n visibility = [\"//visibility:public\"],\n deps = [\n \":_benchmark\",\n ],\n)\n\nnanobind_extension(\n name = \"_benchmark\",\n srcs = [\"benchmark.cc\"],\n deps = [\"//:benchmark\"],\n)\n\nnanobind_stubgen(\n name = \"benchmark_stubgen\",\n marker_file = \"bindings/python/google_benchmark/py.typed\",\n module = \":_benchmark\",\n)\n\npy_test(\n name = \"example\",\n srcs = [\"example.py\"],\n python_version = \"PY3\",\n srcs_version = \"PY3\",\n visibility = [\"//visibility:public\"],\n deps = [\n \":google_benchmark\",\n ],\n)\n" }, { "path": "bindings/python/google_benchmark/__init__.py", "content": "# Copyright 2020 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Python benchmarking utilities.\n\nExample usage:\n import google_benchmark as benchmark\n\n @benchmark.register\n def my_benchmark(state):\n ... # Code executed outside `while` loop is not timed.\n\n while state:\n ... # Code executed within `while` loop is timed.\n\n if __name__ == '__main__':\n benchmark.main()\n\"\"\"\n\nimport atexit\n\nfrom google_benchmark import _benchmark\nfrom google_benchmark._benchmark import (\n Counter as Counter,\n State as State,\n kMicrosecond as kMicrosecond,\n kMillisecond as kMillisecond,\n kNanosecond as kNanosecond,\n kSecond as kSecond,\n o1 as o1,\n oAuto as oAuto,\n oLambda as oLambda,\n oLogN as oLogN,\n oN as oN,\n oNCubed as oNCubed,\n oNLogN as oNLogN,\n oNone as oNone,\n oNSquared as oNSquared,\n)\n\n__version__ = \"1.9.5\"\n\n\nclass __OptionMaker:\n \"\"\"A stateless class to collect benchmark options.\n\n Collect all decorator calls like @option.range(start=0, limit=1<<5).\n \"\"\"\n\n class Options:\n \"\"\"Pure data class to store options calls, along with the benchmarked\n function.\"\"\"\n\n def __init__(self, func):\n self.func = func\n self.builder_calls = []\n\n @classmethod\n def make(cls, func_or_options):\n \"\"\"Make Options from Options or the benchmarked function.\"\"\"\n if isinstance(func_or_options, cls.Options):\n return func_or_options\n return cls.Options(func_or_options)\n\n def __getattr__(self, builder_name):\n \"\"\"Append option call in the Options.\"\"\"\n\n # The function that get returned on @option.range(start=0, limit=1<<5).\n def __builder_method(*args, **kwargs):\n # The decorator that get called, either with the benchmared function\n # or the previous Options\n def __decorator(func_or_options):\n options = self.make(func_or_options)\n options.builder_calls.append((builder_name, args, kwargs))\n # The decorator returns Options so it is not technically a\n # decorator and needs a final call to @register\n return options\n\n return __decorator\n\n return __builder_method\n\n\n# Alias for nicer API.\n# We have to instantiate an object, even if stateless, to be able to use\n# __getattr__ on option.range\noption = __OptionMaker()\n\n\ndef register(undefined=None, *, name=None):\n \"\"\"Register function for benchmarking.\"\"\"\n if undefined is None:\n # Decorator is called without parenthesis so we return a decorator\n return lambda f: register(f, name=name)\n\n # We have either the function to benchmark (simple case) or an instance of\n # Options (@option._ case).\n options = __OptionMaker.make(undefined)\n\n if name is None:\n name = options.func.__name__\n\n # We register the benchmark and reproduce all the @option._ calls onto the\n # benchmark builder pattern\n benchmark = _benchmark.RegisterBenchmark(name, options.func)\n for name, args, kwargs in options.builder_calls[::-1]:\n getattr(benchmark, name)(*args, **kwargs)\n\n # return the benchmarked function because the decorator does not modify it\n return options.func\n\n\ndef main(argv: list[str] | None = None) -> None:\n import sys\n\n _benchmark.Initialize(argv or sys.argv)\n return _benchmark.RunSpecifiedBenchmarks()\n\n\n# FIXME: can we rerun with disabled ASLR?\n\n# Methods for use with custom main function.\ninitialize = _benchmark.Initialize\nrun_benchmarks = _benchmark.RunSpecifiedBenchmarks\nadd_custom_context = _benchmark.AddCustomContext\natexit.register(_benchmark.ClearRegisteredBenchmarks)\n" }, { "path": "bindings/python/google_benchmark/benchmark.cc", "content": "// Benchmark for Python.\n\n#include \"benchmark/benchmark.h\"\n\n#include \"nanobind/nanobind.h\"\n#include \"nanobind/operators.h\"\n#include \"nanobind/stl/bind_map.h\"\n#include \"nanobind/stl/string.h\"\n#include \"nanobind/stl/vector.h\"\n\nNB_MAKE_OPAQUE(benchmark::UserCounters);\n\nnamespace {\nnamespace nb = nanobind;\n\nstd::vector Initialize(const std::vector& argv) {\n std::vector ptrs;\n ptrs.reserve(argv.size());\n for (auto& arg : argv) {\n ptrs.push_back(const_cast(arg.c_str()));\n }\n if (!ptrs.empty()) {\n // The `argv` pointers here become invalid when this function returns, but\n // benchmark holds the pointer to `argv[0]`. We create a static copy of it\n // so it persists, and replace the pointer below.\n static std::string executable_name(argv[0]);\n ptrs[0] = const_cast(executable_name.c_str());\n }\n int argc = static_cast(argv.size());\n benchmark::Initialize(&argc, ptrs.data());\n std::vector remaining_argv;\n remaining_argv.reserve(argc);\n for (int i = 0; i < argc; ++i) {\n remaining_argv.emplace_back(ptrs[i]);\n }\n return remaining_argv;\n}\n\nbenchmark::Benchmark* RegisterBenchmark(const std::string& name,\n nb::callable f) {\n return benchmark::RegisterBenchmark(\n name, [f](benchmark::State& state) { f(&state); });\n}\n\nNB_MODULE(_benchmark, m) {\n using benchmark::TimeUnit;\n nb::enum_(m, \"TimeUnit\")\n .value(\"kNanosecond\", TimeUnit::kNanosecond)\n .value(\"kMicrosecond\", TimeUnit::kMicrosecond)\n .value(\"kMillisecond\", TimeUnit::kMillisecond)\n .value(\"kSecond\", TimeUnit::kSecond)\n .export_values();\n\n using benchmark::BigO;\n nb::enum_(m, \"BigO\")\n .value(\"oNone\", BigO::oNone)\n .value(\"o1\", BigO::o1)\n .value(\"oN\", BigO::oN)\n .value(\"oNSquared\", BigO::oNSquared)\n .value(\"oNCubed\", BigO::oNCubed)\n .value(\"oLogN\", BigO::oLogN)\n .value(\"oNLogN\", BigO::oNLogN)\n .value(\"oAuto\", BigO::oAuto)\n .value(\"oLambda\", BigO::oLambda)\n .export_values();\n\n using benchmark::Benchmark;\n nb::class_(m, \"Benchmark\")\n // For methods returning a pointer to the current object, reference\n // return policy is used to ask nanobind not to take ownership of the\n // returned object and avoid calling delete on it.\n // https://pybind11.readthedocs.io/en/stable/advanced/functions.html#return-value-policies\n //\n // For methods taking a const std::vector<...>&, a copy is created\n // because a it is bound to a Python list.\n // https://pybind11.readthedocs.io/en/stable/advanced/cast/stl.html\n .def(\"unit\", &Benchmark::Unit, nb::rv_policy::reference)\n .def(\"arg\", &Benchmark::Arg, nb::rv_policy::reference)\n .def(\"args\", &Benchmark::Args, nb::rv_policy::reference)\n .def(\"range\", &Benchmark::Range, nb::rv_policy::reference,\n nb::arg(\"start\"), nb::arg(\"limit\"))\n .def(\"dense_range\", &Benchmark::DenseRange, nb::rv_policy::reference,\n nb::arg(\"start\"), nb::arg(\"limit\"), nb::arg(\"step\") = 1)\n .def(\"ranges\", &Benchmark::Ranges, nb::rv_policy::reference)\n .def(\"args_product\", &Benchmark::ArgsProduct, nb::rv_policy::reference)\n .def(\"arg_name\", &Benchmark::ArgName, nb::rv_policy::reference)\n .def(\"arg_names\", &Benchmark::ArgNames, nb::rv_policy::reference)\n .def(\"range_pair\", &Benchmark::RangePair, nb::rv_policy::reference,\n nb::arg(\"lo1\"), nb::arg(\"hi1\"), nb::arg(\"lo2\"), nb::arg(\"hi2\"))\n .def(\"range_multiplier\", &Benchmark::RangeMultiplier,\n nb::rv_policy::reference)\n .def(\"min_time\", &Benchmark::MinTime, nb::rv_policy::reference)\n .def(\"min_warmup_time\", &Benchmark::MinWarmUpTime,\n nb::rv_policy::reference)\n .def(\"iterations\", &Benchmark::Iterations, nb::rv_policy::reference)\n .def(\"repetitions\", &Benchmark::Repetitions, nb::rv_policy::reference)\n .def(\"report_aggregates_only\", &Benchmark::ReportAggregatesOnly,\n nb::rv_policy::reference, nb::arg(\"value\") = true)\n .def(\"display_aggregates_only\", &Benchmark::DisplayAggregatesOnly,\n nb::rv_policy::reference, nb::arg(\"value\") = true)\n .def(\"measure_process_cpu_time\", &Benchmark::MeasureProcessCPUTime,\n nb::rv_policy::reference)\n .def(\"use_real_time\", &Benchmark::UseRealTime, nb::rv_policy::reference)\n .def(\"use_manual_time\", &Benchmark::UseManualTime,\n nb::rv_policy::reference)\n .def(\n \"complexity\",\n (Benchmark * (Benchmark::*)(benchmark::BigO)) & Benchmark::Complexity,\n nb::rv_policy::reference, nb::arg(\"complexity\") = benchmark::oAuto);\n\n using benchmark::Counter;\n nb::class_ py_counter(m, \"Counter\");\n\n nb::enum_(py_counter, \"Flags\", nb::is_arithmetic(),\n nb::is_flag())\n .value(\"kDefaults\", Counter::Flags::kDefaults)\n .value(\"kIsRate\", Counter::Flags::kIsRate)\n .value(\"kAvgThreads\", Counter::Flags::kAvgThreads)\n .value(\"kAvgThreadsRate\", Counter::Flags::kAvgThreadsRate)\n .value(\"kIsIterationInvariant\", Counter::Flags::kIsIterationInvariant)\n .value(\"kIsIterationInvariantRate\",\n Counter::Flags::kIsIterationInvariantRate)\n .value(\"kAvgIterations\", Counter::Flags::kAvgIterations)\n .value(\"kAvgIterationsRate\", Counter::Flags::kAvgIterationsRate)\n .value(\"kInvert\", Counter::Flags::kInvert)\n .export_values();\n\n nb::enum_(py_counter, \"OneK\")\n .value(\"kIs1000\", Counter::OneK::kIs1000)\n .value(\"kIs1024\", Counter::OneK::kIs1024)\n .export_values();\n\n py_counter\n .def(nb::init(),\n nb::arg(\"value\") = 0., nb::arg(\"flags\") = Counter::kDefaults,\n nb::arg(\"k\") = Counter::kIs1000)\n .def(\"__init__\",\n ([](Counter* c, double value) { new (c) Counter(value); }))\n .def_rw(\"value\", &Counter::value)\n .def_rw(\"flags\", &Counter::flags)\n .def_rw(\"oneK\", &Counter::oneK)\n .def(nb::init_implicit());\n\n nb::implicitly_convertible();\n\n nb::bind_map(m, \"UserCounters\");\n\n using benchmark::State;\n nb::class_(m, \"State\")\n .def(\"__bool__\", &State::KeepRunning)\n .def_prop_ro(\"keep_running\", &State::KeepRunning)\n .def(\"pause_timing\", &State::PauseTiming)\n .def(\"resume_timing\", &State::ResumeTiming)\n .def(\"skip_with_error\", &State::SkipWithError)\n .def_prop_ro(\"error_occurred\", &State::error_occurred)\n .def(\"set_iteration_time\", &State::SetIterationTime)\n .def_prop_rw(\"bytes_processed\", &State::bytes_processed,\n &State::SetBytesProcessed)\n .def_prop_rw(\"complexity_n\", &State::complexity_length_n,\n &State::SetComplexityN)\n .def_prop_rw(\"items_processed\", &State::items_processed,\n &State::SetItemsProcessed)\n .def(\"set_label\", &State::SetLabel)\n .def(\n \"range\",\n [](const State& state, std::size_t pos = 0) -> int64_t {\n if (pos < state.range_size()) {\n return state.range(pos);\n }\n throw nb::index_error(\"pos is out of range\");\n },\n nb::arg(\"pos\") = 0)\n .def_prop_ro(\"iterations\", &State::iterations)\n .def_prop_ro(\"name\", &State::name)\n .def_rw(\"counters\", &State::counters)\n .def_prop_ro(\"thread_index\", &State::thread_index)\n .def_prop_ro(\"threads\", &State::threads);\n\n m.def(\"Initialize\", Initialize);\n m.def(\"RegisterBenchmark\", RegisterBenchmark, nb::rv_policy::reference);\n m.def(\"RunSpecifiedBenchmarks\",\n []() { benchmark::RunSpecifiedBenchmarks(); });\n m.def(\"ClearRegisteredBenchmarks\", benchmark::ClearRegisteredBenchmarks);\n m.def(\"AddCustomContext\", benchmark::AddCustomContext, nb::arg(\"key\"),\n nb::arg(\"value\"),\n \"Add a key-value pair to output as part of the context stanza in the \"\n \"report.\");\n};\n} // namespace\n" }, { "path": "bindings/python/google_benchmark/example.py", "content": "# Copyright 2020 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Example of Python using C++ benchmark framework.\n\nTo run this example, you must first install the `google_benchmark` Python\npackage.\n\nTo install using `setup.py`, download and extract the `google_benchmark` source.\nIn the extracted directory, execute:\n python setup.py install\n\"\"\"\n\nimport random\nimport sys\nimport time\n\nimport google_benchmark as benchmark\nfrom google_benchmark import Counter\n\n\n@benchmark.register\ndef empty(state):\n while state:\n pass\n\n\n@benchmark.register\ndef sum_million(state):\n while state:\n sum(range(1_000_000))\n\n\n@benchmark.register\ndef pause_timing(state):\n \"\"\"Pause timing every iteration.\"\"\"\n while state:\n # Construct a list of random ints every iteration without timing it\n state.pause_timing()\n random_list = [random.randint(0, 100) for _ in range(100)]\n state.resume_timing()\n # Time the in place sorting algorithm\n random_list.sort()\n\n\n@benchmark.register\ndef skipped(state):\n if True: # Test some predicate here.\n state.skip_with_error(\"some error\")\n return # NOTE: You must explicitly return, or benchmark will continue.\n\n # Benchmark code would be here.\n\n\n@benchmark.register\n@benchmark.option.use_manual_time()\ndef manual_timing(state):\n while state:\n # Manually count Python CPU time\n start = time.perf_counter() # perf_counter_ns() in Python 3.7+\n # Something to benchmark\n time.sleep(0.01)\n end = time.perf_counter()\n state.set_iteration_time(end - start)\n\n\n@benchmark.register\ndef custom_counters(state):\n \"\"\"Collect custom metric using benchmark.Counter.\"\"\"\n num_foo = 0.0\n while state:\n # Benchmark some code here\n # Collect some custom metric named foo\n num_foo += 0.13\n\n # Automatic Counter from numbers.\n state.counters[\"foo\"] = num_foo\n # Set a counter as a rate.\n state.counters[\"foo_rate\"] = Counter(num_foo, Counter.kIsRate)\n # Set a counter as an inverse of rate.\n state.counters[\"foo_inv_rate\"] = Counter(\n num_foo, Counter.kIsRate | Counter.kInvert\n )\n # Set a counter as a thread-average quantity.\n state.counters[\"foo_avg\"] = Counter(num_foo, Counter.kAvgThreads)\n # There's also a combined flag:\n state.counters[\"foo_avg_rate\"] = Counter(num_foo, Counter.kAvgThreadsRate)\n\n\n@benchmark.register\n@benchmark.option.measure_process_cpu_time()\n@benchmark.option.use_real_time()\ndef with_options(state):\n while state:\n sum(range(1_000_000))\n\n\n@benchmark.register(name=\"sum_million_microseconds\")\n@benchmark.option.unit(benchmark.kMicrosecond)\ndef with_options2(state):\n while state:\n sum(range(1_000_000))\n\n\n@benchmark.register\n@benchmark.option.arg(100)\n@benchmark.option.arg(1000)\ndef passing_argument(state):\n while state:\n sum(range(state.range(0)))\n\n\n@benchmark.register\n@benchmark.option.range(8, limit=8 << 10)\ndef using_range(state):\n while state:\n sum(range(state.range(0)))\n\n\n@benchmark.register\n@benchmark.option.range_multiplier(2)\n@benchmark.option.range(1 << 10, 1 << 18)\n@benchmark.option.complexity(benchmark.oN)\ndef computing_complexity(state):\n while state:\n sum(range(state.range(0)))\n state.complexity_n = state.range(0)\n\n\nif __name__ == \"__main__\":\n benchmark.add_custom_context(\"python\", sys.version)\n benchmark.main()\n" }, { "path": "cmake/AddCXXCompilerFlag.cmake", "content": "# - Adds a compiler flag if it is supported by the compiler\n#\n# This function checks that the supplied compiler flag is supported and then\n# adds it to the corresponding compiler flags\n#\n# add_cxx_compiler_flag( [])\n#\n# - Example\n#\n# include(AddCXXCompilerFlag)\n# add_cxx_compiler_flag(-Wall)\n# add_cxx_compiler_flag(-no-strict-aliasing RELEASE)\n# Requires CMake 2.6+\n\nif(__add_cxx_compiler_flag)\n return()\nendif()\nset(__add_cxx_compiler_flag INCLUDED)\n\ninclude(CheckCXXCompilerFlag)\n\nfunction(mangle_compiler_flag FLAG OUTPUT)\n string(TOUPPER \"HAVE_CXX_FLAG_${FLAG}\" SANITIZED_FLAG)\n string(REPLACE \"+\" \"X\" SANITIZED_FLAG ${SANITIZED_FLAG})\n string(REGEX REPLACE \"[^A-Za-z_0-9]\" \"_\" SANITIZED_FLAG ${SANITIZED_FLAG})\n string(REGEX REPLACE \"_+\" \"_\" SANITIZED_FLAG ${SANITIZED_FLAG})\n set(${OUTPUT} \"${SANITIZED_FLAG}\" PARENT_SCOPE)\nendfunction(mangle_compiler_flag)\n\nfunction(add_cxx_compiler_flag FLAG)\n mangle_compiler_flag(\"${FLAG}\" MANGLED_FLAG)\n set(OLD_CMAKE_REQUIRED_FLAGS \"${CMAKE_REQUIRED_FLAGS}\")\n set(CMAKE_REQUIRED_FLAGS \"${CMAKE_REQUIRED_FLAGS} ${FLAG}\")\n check_cxx_compiler_flag(\"${FLAG}\" ${MANGLED_FLAG})\n set(CMAKE_REQUIRED_FLAGS \"${OLD_CMAKE_REQUIRED_FLAGS}\")\n if(${MANGLED_FLAG})\n if(ARGC GREATER 1)\n set(VARIANT ${ARGV1})\n string(TOUPPER \"_${VARIANT}\" VARIANT)\n else()\n set(VARIANT \"\")\n endif()\n set(CMAKE_CXX_FLAGS${VARIANT} \"${CMAKE_CXX_FLAGS${VARIANT}} ${BENCHMARK_CXX_FLAGS${VARIANT}} ${FLAG}\" PARENT_SCOPE)\n endif()\nendfunction()\n\nfunction(add_required_cxx_compiler_flag FLAG)\n mangle_compiler_flag(\"${FLAG}\" MANGLED_FLAG)\n set(OLD_CMAKE_REQUIRED_FLAGS \"${CMAKE_REQUIRED_FLAGS}\")\n set(CMAKE_REQUIRED_FLAGS \"${CMAKE_REQUIRED_FLAGS} ${FLAG}\")\n check_cxx_compiler_flag(\"${FLAG}\" ${MANGLED_FLAG})\n set(CMAKE_REQUIRED_FLAGS \"${OLD_CMAKE_REQUIRED_FLAGS}\")\n if(${MANGLED_FLAG})\n if(ARGC GREATER 1)\n set(VARIANT ${ARGV1})\n string(TOUPPER \"_${VARIANT}\" VARIANT)\n else()\n set(VARIANT \"\")\n endif()\n set(CMAKE_CXX_FLAGS${VARIANT} \"${CMAKE_CXX_FLAGS${VARIANT}} ${FLAG}\" PARENT_SCOPE)\n set(CMAKE_EXE_LINKER_FLAGS \"${CMAKE_EXE_LINKER_FLAGS} ${FLAG}\" PARENT_SCOPE)\n set(CMAKE_SHARED_LINKER_FLAGS \"${CMAKE_SHARED_LINKER_FLAGS} ${FLAG}\" PARENT_SCOPE)\n set(CMAKE_MODULE_LINKER_FLAGS \"${CMAKE_MODULE_LINKER_FLAGS} ${FLAG}\" PARENT_SCOPE)\n set(CMAKE_REQUIRED_FLAGS \"${CMAKE_REQUIRED_FLAGS} ${FLAG}\" PARENT_SCOPE)\n else()\n message(FATAL_ERROR \"Required flag '${FLAG}' is not supported by the compiler\")\n endif()\nendfunction()\n\nfunction(check_cxx_warning_flag FLAG)\n mangle_compiler_flag(\"${FLAG}\" MANGLED_FLAG)\n set(OLD_CMAKE_REQUIRED_FLAGS \"${CMAKE_REQUIRED_FLAGS}\")\n # Add -Werror to ensure the compiler generates an error if the warning flag\n # doesn't exist.\n set(CMAKE_REQUIRED_FLAGS \"${CMAKE_REQUIRED_FLAGS} -Werror ${FLAG}\")\n check_cxx_compiler_flag(\"${FLAG}\" ${MANGLED_FLAG})\n set(CMAKE_REQUIRED_FLAGS \"${OLD_CMAKE_REQUIRED_FLAGS}\")\nendfunction()\n" }, { "path": "cmake/CXXFeatureCheck.cmake", "content": "# - Compile and run code to check for C++ features\n#\n# This functions compiles a source file under the `cmake` folder\n# and adds the corresponding `HAVE_[FILENAME]` flag to the CMake\n# environment\n#\n# cxx_feature_check( [])\n#\n# - Example\n#\n# include(CXXFeatureCheck)\n# cxx_feature_check(STD_REGEX)\n# Requires CMake 2.8.12+\n\nif(__cxx_feature_check)\n return()\nendif()\nset(__cxx_feature_check INCLUDED)\n\noption(CXXFEATURECHECK_DEBUG OFF)\n\nfunction(cxx_feature_check FILE)\n string(TOLOWER ${FILE} FILE)\n string(TOUPPER ${FILE} VAR)\n string(TOUPPER \"HAVE_${VAR}\" FEATURE)\n if (DEFINED HAVE_${VAR})\n if(HAVE_${VAR})\n add_definitions(-DHAVE_${VAR})\n endif()\n return()\n endif()\n\n set(FEATURE_CHECK_CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS})\n if (ARGC GREATER 1)\n message(STATUS \"Enabling additional flags: ${ARGV1}\")\n list(APPEND FEATURE_CHECK_CMAKE_FLAGS ${ARGV1})\n endif()\n\n if (NOT DEFINED COMPILE_${FEATURE})\n if(CMAKE_CROSSCOMPILING)\n message(STATUS \"Cross-compiling to test ${FEATURE}\")\n try_compile(COMPILE_${FEATURE}\n ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp\n CXX_STANDARD 17\n CXX_STANDARD_REQUIRED ON\n CMAKE_FLAGS ${FEATURE_CHECK_CMAKE_FLAGS}\n LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}\n OUTPUT_VARIABLE COMPILE_OUTPUT_VAR)\n if(COMPILE_${FEATURE})\n message(WARNING\n \"If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0\")\n set(RUN_${FEATURE} 0 CACHE INTERNAL \"\")\n else()\n set(RUN_${FEATURE} 1 CACHE INTERNAL \"\")\n endif()\n else()\n message(STATUS \"Compiling and running to test ${FEATURE}\")\n try_run(RUN_${FEATURE} COMPILE_${FEATURE}\n ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp\n CXX_STANDARD 17\n CXX_STANDARD_REQUIRED ON\n CMAKE_FLAGS ${FEATURE_CHECK_CMAKE_FLAGS}\n LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}\n COMPILE_OUTPUT_VARIABLE COMPILE_OUTPUT_VAR)\n endif()\n endif()\n\n if(COMPILE_${FEATURE})\n if(DEFINED RUN_${FEATURE} AND RUN_${FEATURE} EQUAL 0)\n message(STATUS \"Performing Test ${FEATURE} -- success\")\n set(HAVE_${VAR} 1 CACHE INTERNAL \"\")\n add_definitions(-DHAVE_${VAR})\n else()\n message(STATUS \"Performing Test ${FEATURE} -- compiled but failed to run\")\n set(HAVE_${VAR} 0 CACHE INTERNAL \"\")\n endif()\n else()\n if(CXXFEATURECHECK_DEBUG)\n message(STATUS \"Performing Test ${FEATURE} -- failed to compile: ${COMPILE_OUTPUT_VAR}\")\n else()\n message(STATUS \"Performing Test ${FEATURE} -- failed to compile\")\n endif()\n set(HAVE_${VAR} 0 CACHE INTERNAL \"\")\n endif()\nendfunction()\n" }, { "path": "cmake/Config.cmake.in", "content": "@PACKAGE_INIT@\n\ninclude (CMakeFindDependencyMacro)\n\nfind_dependency (Threads)\n\nif (@BENCHMARK_ENABLE_LIBPFM@)\n list(APPEND CMAKE_MODULE_PATH \"${CMAKE_CURRENT_LIST_DIR}\")\n find_dependency (PFM)\nendif()\n\ninclude(\"${CMAKE_CURRENT_LIST_DIR}/@targets_export_name@.cmake\")\n" }, { "path": "cmake/GetGitVersion.cmake", "content": "# - Returns a version string from Git tags\n#\n# This function inspects the annotated git tags for the project and returns a string\n# into a CMake variable\n#\n# get_git_version()\n#\n# - Example\n#\n# include(GetGitVersion)\n# get_git_version(GIT_VERSION)\n#\n# Requires CMake 2.8.11+\nfind_package(Git)\n\nif(__get_git_version)\n return()\nendif()\nset(__get_git_version INCLUDED)\n\nfunction(get_git_version var)\n if(GIT_EXECUTABLE)\n execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --match \"v[0-9]*.[0-9]*.[0-9]*\" --abbrev=8 --dirty\n WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}\n RESULT_VARIABLE status\n OUTPUT_VARIABLE GIT_VERSION\n ERROR_QUIET)\n if(status)\n set(GIT_VERSION \"v0.0.0\")\n endif()\n else()\n set(GIT_VERSION \"v0.0.0\")\n endif()\n\n set(${var} ${GIT_VERSION} PARENT_SCOPE)\nendfunction()\n" }, { "path": "cmake/GoogleTest.cmake", "content": "# Download and unpack googletest at configure time\nset(GOOGLETEST_PREFIX \"${benchmark_BINARY_DIR}/third_party/googletest\")\nconfigure_file(${benchmark_SOURCE_DIR}/cmake/GoogleTest.cmake.in ${GOOGLETEST_PREFIX}/CMakeLists.txt @ONLY)\n\nset(GOOGLETEST_PATH \"${CMAKE_CURRENT_SOURCE_DIR}/googletest\" CACHE PATH \"\") # Mind the quotes\nexecute_process(COMMAND ${CMAKE_COMMAND} -G \"${CMAKE_GENERATOR}\"\n -DALLOW_DOWNLOADING_GOOGLETEST=${BENCHMARK_DOWNLOAD_DEPENDENCIES} -DGOOGLETEST_PATH:PATH=${GOOGLETEST_PATH} .\n RESULT_VARIABLE result\n WORKING_DIRECTORY ${GOOGLETEST_PREFIX}\n)\n\nif(result)\n message(FATAL_ERROR \"CMake step for googletest failed: ${result}\")\nendif()\n\nexecute_process(\n COMMAND ${CMAKE_COMMAND} --build .\n RESULT_VARIABLE result\n WORKING_DIRECTORY ${GOOGLETEST_PREFIX}\n)\n\nif(result)\n message(FATAL_ERROR \"Build step for googletest failed: ${result}\")\nendif()\n\n# Prevent overriding the parent project's compiler/linker\n# settings on Windows\nset(gtest_force_shared_crt ON CACHE BOOL \"\" FORCE)\n\ninclude(${GOOGLETEST_PREFIX}/googletest-paths.cmake)\n\n# Add googletest directly to our build. This defines\n# the gtest and gtest_main targets.\nadd_subdirectory(${GOOGLETEST_SOURCE_DIR}\n ${GOOGLETEST_BINARY_DIR}\n EXCLUDE_FROM_ALL)\n\n# googletest doesn't seem to want to stay build warning clean so let's not hurt ourselves.\nif (MSVC)\n target_compile_options(gtest PRIVATE \"/wd4244\" \"/wd4722\")\n target_compile_options(gtest_main PRIVATE \"/wd4244\" \"/wd4722\")\n target_compile_options(gmock PRIVATE \"/wd4244\" \"/wd4722\")\n target_compile_options(gmock_main PRIVATE \"/wd4244\" \"/wd4722\")\nelse()\n target_compile_options(gtest PRIVATE \"-w\")\n target_compile_options(gtest_main PRIVATE \"-w\")\n target_compile_options(gmock PRIVATE \"-w\")\n target_compile_options(gmock_main PRIVATE \"-w\")\nendif()\n\nif(NOT DEFINED GTEST_COMPILE_COMMANDS)\n set(GTEST_COMPILE_COMMANDS ON)\nendif()\n\nset_target_properties(gtest PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $ EXPORT_COMPILE_COMMANDS ${GTEST_COMPILE_COMMANDS})\nset_target_properties(gtest_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $ EXPORT_COMPILE_COMMANDS ${GTEST_COMPILE_COMMANDS})\nset_target_properties(gmock PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $ EXPORT_COMPILE_COMMANDS ${GTEST_COMPILE_COMMANDS})\nset_target_properties(gmock_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $ EXPORT_COMPILE_COMMANDS ${GTEST_COMPILE_COMMANDS})\n" }, { "path": "cmake/GoogleTest.cmake.in", "content": "cmake_minimum_required (VERSION 3.13...3.22)\n\nproject(googletest-download NONE)\n\n# Enable ExternalProject CMake module\ninclude(ExternalProject)\n\noption(ALLOW_DOWNLOADING_GOOGLETEST \"If googletest src tree is not found in location specified by GOOGLETEST_PATH, do fetch the archive from internet\" OFF)\nset(GOOGLETEST_PATH \"/usr/src/googletest\" CACHE PATH\n \"Path to the googletest root tree. Should contain googletest and googlemock subdirs. And CMakeLists.txt in root, and in both of these subdirs\")\n\n# Download and install GoogleTest\n\nmessage(STATUS \"Looking for Google Test sources\")\nmessage(STATUS \"Looking for Google Test sources in ${GOOGLETEST_PATH}\")\nif(EXISTS \"${GOOGLETEST_PATH}\" AND IS_DIRECTORY \"${GOOGLETEST_PATH}\" AND EXISTS \"${GOOGLETEST_PATH}/CMakeLists.txt\" AND\n EXISTS \"${GOOGLETEST_PATH}/googletest\" AND IS_DIRECTORY \"${GOOGLETEST_PATH}/googletest\" AND EXISTS \"${GOOGLETEST_PATH}/googletest/CMakeLists.txt\" AND\n EXISTS \"${GOOGLETEST_PATH}/googlemock\" AND IS_DIRECTORY \"${GOOGLETEST_PATH}/googlemock\" AND EXISTS \"${GOOGLETEST_PATH}/googlemock/CMakeLists.txt\")\n message(STATUS \"Found Google Test in ${GOOGLETEST_PATH}\")\n\n ExternalProject_Add(\n googletest\n PREFIX \"${CMAKE_BINARY_DIR}\"\n DOWNLOAD_DIR \"${CMAKE_BINARY_DIR}/download\"\n SOURCE_DIR \"${GOOGLETEST_PATH}\" # use existing src dir.\n BINARY_DIR \"${CMAKE_BINARY_DIR}/build\"\n CONFIGURE_COMMAND \"\"\n BUILD_COMMAND \"\"\n INSTALL_COMMAND \"\"\n TEST_COMMAND \"\"\n )\nelse()\n if(NOT ALLOW_DOWNLOADING_GOOGLETEST)\n message(SEND_ERROR \"Did not find Google Test sources! Either pass correct path in GOOGLETEST_PATH, or enable BENCHMARK_DOWNLOAD_DEPENDENCIES, or disable BENCHMARK_USE_BUNDLED_GTEST, or disable BENCHMARK_ENABLE_GTEST_TESTS / BENCHMARK_ENABLE_TESTING.\")\n return()\n else()\n message(STATUS \"Did not find Google Test sources! Fetching from web...\")\n ExternalProject_Add(\n googletest\n GIT_REPOSITORY https://github.com/google/googletest.git\n GIT_TAG \"v1.15.2\"\n GIT_SHALLOW \"ON\"\n PREFIX \"${CMAKE_BINARY_DIR}\"\n STAMP_DIR \"${CMAKE_BINARY_DIR}/stamp\"\n DOWNLOAD_DIR \"${CMAKE_BINARY_DIR}/download\"\n SOURCE_DIR \"${CMAKE_BINARY_DIR}/src\"\n BINARY_DIR \"${CMAKE_BINARY_DIR}/build\"\n CONFIGURE_COMMAND \"\"\n BUILD_COMMAND \"\"\n INSTALL_COMMAND \"\"\n TEST_COMMAND \"\"\n )\n endif()\nendif()\n\nExternalProject_Get_Property(googletest SOURCE_DIR BINARY_DIR)\nfile(WRITE googletest-paths.cmake\n\"set(GOOGLETEST_SOURCE_DIR \\\"${SOURCE_DIR}\\\")\nset(GOOGLETEST_BINARY_DIR \\\"${BINARY_DIR}\\\")\n\")\n" }, { "path": "cmake/benchmark.pc.in", "content": "prefix=@CMAKE_INSTALL_PREFIX@\nexec_prefix=${prefix}\nlibdir=@CMAKE_INSTALL_FULL_LIBDIR@\nincludedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@\n\nName: @PROJECT_NAME@\nDescription: Google microbenchmark framework\nVersion: @NORMALIZED_VERSION@\n\nLibs: -L${libdir} -lbenchmark\nLibs.private: -lpthread @BENCHMARK_PRIVATE_LINK_LIBRARIES@\nCflags: -I${includedir}\n" }, { "path": "cmake/benchmark_main.pc.in", "content": "libdir=@CMAKE_INSTALL_FULL_LIBDIR@\n\nName: @PROJECT_NAME@\nDescription: Google microbenchmark framework (with main() function)\nVersion: @NORMALIZED_VERSION@\nRequires: benchmark\nLibs: -L${libdir} -lbenchmark_main\n" }, { "path": "cmake/gnu_posix_regex.cpp", "content": "#include \n#include \nint main() {\n std::string str = \"test0159\";\n regex_t re;\n int ec = regcomp(&re, \"^[a-z]+[0-9]+$\", REG_EXTENDED | REG_NOSUB);\n if (ec != 0) {\n return ec;\n }\n return regexec(&re, str.c_str(), 0, nullptr, 0) ? -1 : 0;\n}\n\n" }, { "path": "cmake/llvm-toolchain.cmake", "content": "find_package(LLVMAr REQUIRED)\nset(CMAKE_AR \"${LLVMAR_EXECUTABLE}\" CACHE FILEPATH \"\" FORCE)\n\nfind_package(LLVMNm REQUIRED)\nset(CMAKE_NM \"${LLVMNM_EXECUTABLE}\" CACHE FILEPATH \"\" FORCE)\n\nfind_package(LLVMRanLib REQUIRED)\nset(CMAKE_RANLIB \"${LLVMRANLIB_EXECUTABLE}\" CACHE FILEPATH \"\" FORCE)\n" }, { "path": "cmake/posix_regex.cpp", "content": "#include \n#include \nint main() {\n std::string str = \"test0159\";\n regex_t re;\n int ec = regcomp(&re, \"^[a-z]+[0-9]+$\", REG_EXTENDED | REG_NOSUB);\n if (ec != 0) {\n return ec;\n }\n int ret = regexec(&re, str.c_str(), 0, nullptr, 0) ? -1 : 0;\n regfree(&re);\n return ret;\n}\n\n" }, { "path": "cmake/pthread_affinity.cpp", "content": "#include \n#ifdef __FreeBSD__\n#include \n#endif\nint main() {\n cpu_set_t set;\n CPU_ZERO(&set);\n for (int i = 0; i < CPU_SETSIZE; ++i) {\n CPU_SET(i, &set);\n CPU_CLR(i, &set);\n }\n pthread_t self = pthread_self();\n int ret;\n ret = pthread_getaffinity_np(self, sizeof(set), &set);\n if (ret != 0) return ret;\n ret = pthread_setaffinity_np(self, sizeof(set), &set);\n if (ret != 0) return ret;\n return 0;\n}\n" }, { "path": "cmake/split_list.cmake", "content": "macro(split_list listname)\n string(REPLACE \";\" \" \" ${listname} \"${${listname}}\")\nendmacro()\n" }, { "path": "cmake/std_regex.cpp", "content": "#include \n#include \nint main() {\n const std::string str = \"test0159\";\n std::regex re;\n re = std::regex(\"^[a-z]+[0-9]+$\",\n std::regex_constants::extended | std::regex_constants::nosubs);\n return std::regex_search(str, re) ? 0 : -1;\n}\n\n" }, { "path": "cmake/steady_clock.cpp", "content": "#include \n\nint main() {\n typedef std::chrono::steady_clock Clock;\n Clock::time_point tp = Clock::now();\n ((void)tp);\n}\n" }, { "path": "cmake/thread_safety_attributes.cpp", "content": "#define HAVE_THREAD_SAFETY_ATTRIBUTES\n#include \"../src/mutex.h\"\n\nint main() {}\n" }, { "path": "docs/AssemblyTests.md", "content": "# Assembly Tests\n\nThe Benchmark library provides a number of functions whose primary\npurpose in to affect assembly generation, including `DoNotOptimize`\nand `ClobberMemory`. In addition there are other functions,\nsuch as `KeepRunning`, for which generating good assembly is paramount.\n\nFor these functions it's important to have tests that verify the\ncorrectness and quality of the implementation. This requires testing\nthe code generated by the compiler.\n\nThis document describes how the Benchmark library tests compiler output,\nas well as how to properly write new tests.\n\n\n## Anatomy of a Test\n\nWriting a test has two steps:\n\n* Write the code you want to generate assembly for.\n* Add `// CHECK` lines to match against the verified assembly.\n\nExample:\n```c++\n\n// CHECK-LABEL: test_add:\nextern \"C\" int test_add() {\n extern int ExternInt;\n return ExternInt + 1;\n\n // CHECK: movl ExternInt(%rip), %eax\n // CHECK: addl %eax\n // CHECK: ret\n}\n\n```\n\n#### LLVM Filecheck\n\n[LLVM's Filecheck](https://llvm.org/docs/CommandGuide/FileCheck.html)\nis used to test the generated assembly against the `// CHECK` lines\nspecified in the tests source file. Please see the documentation\nlinked above for information on how to write `CHECK` directives.\n\n#### Tips and Tricks:\n\n* Tests should match the minimal amount of output required to establish\ncorrectness. `CHECK` directives don't have to match on the exact next line\nafter the previous match, so tests should omit checks for unimportant\nbits of assembly. ([`CHECK-NEXT`](https://llvm.org/docs/CommandGuide/FileCheck.html#the-check-next-directive)\ncan be used to ensure a match occurs exactly after the previous match).\n\n* The tests are compiled with `-O3 -g0`. So we're only testing the\noptimized output.\n\n* The assembly output is further cleaned up using `tools/strip_asm.py`.\nThis removes comments, assembler directives, and unused labels before\nthe test is run.\n\n* The generated and stripped assembly file for a test is output under\n`/test/.s`\n\n* Filecheck supports using [`CHECK` prefixes](https://llvm.org/docs/CommandGuide/FileCheck.html#cmdoption-check-prefixes)\nto specify lines that should only match in certain situations.\nThe Benchmark tests use `CHECK-CLANG` and `CHECK-GNU` for lines that\nare only expected to match Clang or GCC's output respectively. Normal\n`CHECK` lines match against all compilers. (Note: `CHECK-NOT` and\n`CHECK-LABEL` are NOT prefixes. They are versions of non-prefixed\n`CHECK` lines)\n\n* Use `extern \"C\"` to disable name mangling for specific functions. This\nmakes them easier to name in the `CHECK` lines.\n\n\n## Problems Writing Portable Tests\n\nWriting tests which check the code generated by a compiler are\ninherently non-portable. Different compilers and even different compiler\nversions may generate entirely different code. The Benchmark tests\nmust tolerate this.\n\nLLVM Filecheck provides a number of mechanisms to help write\n\"more portable\" tests; including [matching using regular expressions](https://llvm.org/docs/CommandGuide/FileCheck.html#filecheck-pattern-matching-syntax),\nallowing the creation of [named variables](https://llvm.org/docs/CommandGuide/FileCheck.html#filecheck-variables)\nfor later matching, and [checking non-sequential matches](https://llvm.org/docs/CommandGuide/FileCheck.html#the-check-dag-directive).\n\n#### Capturing Variables\n\nFor example, say GCC stores a variable in a register but Clang stores\nit in memory. To write a test that tolerates both cases we \"capture\"\nthe destination of the store, and then use the captured expression\nto write the remainder of the test.\n\n```c++\n// CHECK-LABEL: test_div_no_op_into_shr:\nextern \"C\" void test_div_no_op_into_shr(int value) {\n int divisor = 2;\n benchmark::DoNotOptimize(divisor); // hide the value from the optimizer\n return value / divisor;\n\n // CHECK: movl $2, [[DEST:.*]]\n // CHECK: idivl [[DEST]]\n // CHECK: ret\n}\n```\n\n#### Using Regular Expressions to Match Differing Output\n\nOften tests require testing assembly lines which may subtly differ\nbetween compilers or compiler versions. A common example of this\nis matching stack frame addresses. In this case regular expressions\ncan be used to match the differing bits of output. For example:\n\n\n```c++\nint ExternInt;\nstruct Point { int x, y, z; };\n\n// CHECK-LABEL: test_store_point:\nextern \"C\" void test_store_point() {\n Point p{ExternInt, ExternInt, ExternInt};\n benchmark::DoNotOptimize(p);\n\n // CHECK: movl ExternInt(%rip), %eax\n // CHECK: movl %eax, -{{[0-9]+}}(%rsp)\n // CHECK: movl %eax, -{{[0-9]+}}(%rsp)\n // CHECK: movl %eax, -{{[0-9]+}}(%rsp)\n // CHECK: ret\n}\n```\n\n\n## Current Requirements and Limitations\n\nThe tests require Filecheck to be installed along the `PATH` of the\nbuild machine. Otherwise the tests will be disabled.\n\nAdditionally, as mentioned in the previous section, codegen tests are\ninherently non-portable. Currently the tests are limited to:\n\n* x86_64 targets.\n* Compiled with GCC or Clang\n\nFurther work could be done, at least on a limited basis, to extend the\ntests to other architectures and compilers (using `CHECK` prefixes).\n\nFurthermore, the tests fail for builds which specify additional flags\nthat modify code generation, including `--coverage` or `-fsanitize=`.\n\n" }, { "path": "docs/_config.yml", "content": "theme: jekyll-theme-minimal\nlogo: /assets/images/icon_black.png\nshow_downloads: true\n" }, { "path": "docs/dependencies.md", "content": "# Build tool dependency policy\n\nWe follow the [Foundational C++ support policy](https://opensource.google/documentation/policies/cplusplus-support) for our build tools. In\nparticular the [\"Build Systems\" section](https://opensource.google/documentation/policies/cplusplus-support#build-systems).\n\n## CMake\n\nThe current supported version is CMake 3.10 as of 2023-08-10. Most modern\ndistributions include newer versions, for example:\n\n* Ubuntu 20.04 provides CMake 3.16.3\n* Debian 11.4 provides CMake 3.18.4\n* Ubuntu 22.04 provides CMake 3.22.1\n\n## Python\n\nThe Python bindings require Python 3.10+ as of v1.9.0 (2024-08-16) for installation from PyPI.\nBuilding from source for older versions probably still works, though. See the [user guide](python_bindings.md) for details on how to build from source.\nThe minimum theoretically supported version is Python 3.8, since the used bindings generator (nanobind) only supports Python 3.8+.\n" }, { "path": "docs/index.md", "content": "# Benchmark\n\n* [Assembly Tests](AssemblyTests.md)\n* [Dependencies](dependencies.md)\n* [Perf Counters](perf_counters.md)\n* [Platform Specific Build Instructions](platform_specific_build_instructions.md)\n* [Python Bindings](python_bindings.md)\n* [Random Interleaving](random_interleaving.md)\n* [Reducing Variance](reducing_variance.md)\n* [Releasing](releasing.md)\n* [Tools](tools.md)\n* [User Guide](user_guide.md)\n" }, { "path": "docs/perf_counters.md", "content": "\n\n# User-Requested Performance Counters\n\nWhen running benchmarks, the user may choose to request collection of\nperformance counters. This may be useful in investigation scenarios - narrowing\ndown the cause of a regression; or verifying that the underlying cause of a\nperformance improvement matches expectations.\n\nThis feature is available if:\n\n* The benchmark is run on an architecture featuring a Performance Monitoring\n Unit (PMU),\n* The benchmark is compiled with support for collecting counters. Currently,\n this requires [libpfm](http://perfmon2.sourceforge.net/), which is built as a\n dependency via Bazel.\n\nThe feature does not require modifying benchmark code. Counter collection is\nhandled at the boundaries where timer collection is also handled. \n\nTo opt-in:\n* If using a Bazel build, add `--define pfm=1` to your build flags\n* If using CMake:\n * Install `libpfm4-dev`, e.g. `apt-get install libpfm4-dev`.\n * Enable the CMake flag `BENCHMARK_ENABLE_LIBPFM` in `CMakeLists.txt`.\n\nTo use, pass a comma-separated list of counter names through the\n`--benchmark_perf_counters` flag. The names are decoded through libpfm - meaning,\nthey are platform specific, but some (e.g. `CYCLES` or `INSTRUCTIONS`) are\nmapped by libpfm to platform-specifics - see libpfm\n[documentation](http://perfmon2.sourceforge.net/docs.html) for more details.\n\nThe counter values are reported back through the [User Counters](../README.md#custom-counters)\nmechanism, meaning, they are available in all the formats (e.g. JSON) supported\nby User Counters.\n" }, { "path": "docs/platform_specific_build_instructions.md", "content": "# Platform Specific Build Instructions\n\n## Building with GCC\n\nWhen the library is built using GCC it is necessary to link with the pthread\nlibrary due to how GCC implements `std::thread`. Failing to link to pthread will\nlead to runtime exceptions (unless you're using libc++), not linker errors. See\n[issue #67](https://github.com/google/benchmark/issues/67) for more details. You\ncan link to pthread by adding `-pthread` to your linker command. Note, you can\nalso use `-lpthread`, but there are potential issues with ordering of command\nline parameters if you use that.\n\nOn QNX, the pthread library is part of libc and usually included automatically\n(see\n[`pthread_create()`](https://www.qnx.com/developers/docs/7.1/index.html#com.qnx.doc.neutrino.lib_ref/topic/p/pthread_create.html)).\nThere's no separate pthread library to link.\n\n## Building with Visual Studio 2015, 2017 or 2022\n\nThe `shlwapi` library (`-lshlwapi`) is required to support a call to `CPUInfo` which reads the registry. Either add `shlwapi.lib` under `[ Configuration Properties > Linker > Input ]`, or use the following:\n\n```\n// Alternatively, can add libraries using linker options.\n\n// First, Add the path to the generated library files (directory containing the `benchmark.lib`) in `[Configuration Properties > Linker > General > Additional Library Directories]`. Then do the following:\n#ifdef _WIN32\n#pragma comment ( lib, \"Shlwapi.lib\" )\n#ifdef _DEBUG\n#pragma comment ( lib, \"benchmark.lib\" )\n#else\n#pragma comment ( lib, \"benchmark.lib\" )\n#endif\n#endif\n```\n\nWhen using the static library, make sure to add `BENCHMARK_STATIC_DEFINE` under `[Configuration Properties > C/C++ > Preprocessor > Preprocessor Definitions]`\n\nCan also use the graphical version of CMake:\n* Open `CMake GUI`.\n* Under `Where to build the binaries`, same path as source plus `build`.\n* Under `CMAKE_INSTALL_PREFIX`, same path as source plus `install`.\n* Click `Configure`, `Generate`, `Open Project`.\n* If build fails, try deleting entire directory and starting again, or unticking options to build less.\n\n## Building with Intel 2015 Update 1 or Intel System Studio Update 4\n\nSee instructions for building with Visual Studio. Once built, right click on the solution and change the build to Intel.\n\n## Building on Solaris\n\nIf you're running benchmarks on solaris, you'll want the kstat library linked in\ntoo (`-lkstat`)." }, { "path": "docs/python_bindings.md", "content": "# Building and installing Python bindings\n\nPython bindings are available as wheels on [PyPI](https://pypi.org/project/google-benchmark/) for importing and \nusing Google Benchmark directly in Python. \nCurrently, pre-built wheels exist for macOS (both ARM64 and Intel x86), Linux x86-64 and 64-bit Windows.\nSupported Python versions are Python 3.8 - 3.12.\n\nTo install Google Benchmark's Python bindings, run:\n\n```bash\npython -m pip install --upgrade pip # for manylinux2014 support\npython -m pip install google-benchmark\n```\n\nIn order to keep your system Python interpreter clean, it is advisable to run these commands in a virtual\nenvironment. See the [official Python documentation](https://docs.python.org/3/library/venv.html) \non how to create virtual environments.\n\nTo build a wheel directly from source, you can follow these steps:\n```bash\ngit clone https://github.com/google/benchmark.git\ncd benchmark\n# create a virtual environment and activate it\npython3 -m venv venv --system-site-packages\nsource venv/bin/activate # .\\venv\\Scripts\\Activate.ps1 on Windows\n\n# upgrade Python's system-wide packages\npython -m pip install --upgrade pip build\n# builds the wheel and stores it in the directory \"dist\".\npython -m build\n```\n\nNB: Building wheels from source requires Bazel. For platform-specific instructions on how to install Bazel,\nrefer to the [Bazel installation docs](https://bazel.build/install).\n" }, { "path": "docs/random_interleaving.md", "content": "\n\n# Random Interleaving\n\n[Random Interleaving](https://github.com/google/benchmark/issues/1051) is a\ntechnique to lower run-to-run variance. It randomly interleaves repetitions of a\nmicrobenchmark with repetitions from other microbenchmarks in the same benchmark\ntest. Data shows it is able to lower run-to-run variance by\n[40%](https://github.com/google/benchmark/issues/1051) on average.\n\nTo use, you mainly need to set `--benchmark_enable_random_interleaving=true`,\nand optionally specify non-zero repetition count `--benchmark_repetitions=9`\nand optionally decrease the per-repetition time `--benchmark_min_time=0.1`.\n" }, { "path": "docs/reducing_variance.md", "content": "# Reducing Variance\n\n\n\n## Disabling CPU Frequency Scaling\n\nIf you see this error:\n\n```\n***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead.\n```\n\nyou might want to disable the CPU frequency scaling while running the\nbenchmark, as well as consider other ways to stabilize the performance of\nyour system while benchmarking.\n\nExactly how to do this depends on the Linux distribution,\ndesktop environment, and installed programs. Specific details are a moving\ntarget, so we will not attempt to exhaustively document them here.\n\nOne simple option is to use the `cpupower` program to change the\nperformance governor to \"performance\". This tool is maintained along with\nthe Linux kernel and provided by your distribution.\n\nIt must be run as root, like this:\n\n```bash\nsudo cpupower frequency-set --governor performance\n```\n\nAfter this you can verify that all CPUs are using the performance governor\nby running this command:\n\n```bash\ncpupower frequency-info -o proc\n```\n\nThe benchmarks you subsequently run will have less variance.\n\n\n\n## Disabling ASLR\n\nIf you see this error:\n\n```\n***WARNING*** ASLR is enabled, the results may have unreproducible noise in them.\n```\n\nyou might want to disable the ASLR security hardening feature while running the\nbenchmark.\n\nThe simplest way is to add\n```\nbenchmark::MaybeReenterWithoutASLR(argc, argv);\n```\nas the first line of your `main()` function. It will try to disable ASLR\nfor the current processor, and, if successful, re-execute the binary.\nNote that `personality(2)` may be forbidden by e.g. seccomp (which happens\nby default if you are running in a Docker container).\n\nNote that if you link to `benchmark_main` already does that for you.\n\nTo globally disable ASLR on Linux, run\n```\necho 0 > /proc/sys/kernel/randomize_va_space\n```\n\nTo run a single benchmark with ASLR disabled on Linux, do:\n```\nsetarch `uname -m` -R ./a_benchmark\n```\n\nNote that for the information on how to disable ASLR on other operating systems,\nplease refer to their documentation.\n\n## Reducing Variance in Benchmarks\n\nThe Linux CPU frequency governor [discussed\nabove](user_guide#disabling-cpu-frequency-scaling) is not the only source\nof noise in benchmarks. Some, but not all, of the sources of variance\ninclude:\n\n1. On multi-core machines not all CPUs/CPU cores/CPU threads run the same\n speed, so running a benchmark one time and then again may give a\n different result depending on which CPU it ran on.\n2. CPU scaling features that run on the CPU, like Intel's Turbo Boost and\n AMD Turbo Core and Precision Boost, can temporarily change the CPU\n frequency even when the using the \"performance\" governor on Linux.\n3. Context switching between CPUs, or scheduling competition on the CPU the\n benchmark is running on.\n4. Intel Hyperthreading or AMD SMT causing the same issue as above.\n5. Cache effects caused by code running on other CPUs.\n6. Non-uniform memory architectures (NUMA).\n\nThese can cause variance in benchmarks results within a single run\n(`--benchmark_repetitions=N`) or across multiple runs of the benchmark\nprogram.\n\nReducing sources of variance is OS and architecture dependent, which is one\nreason some companies maintain machines dedicated to performance testing.\n\nSome of the easier and effective ways of reducing variance on a typical\nLinux workstation are:\n\n1. Use the performance governor as [discussed\nabove](user_guide#disabling-cpu-frequency-scaling).\n2. Disable processor boosting by:\n ```sh\n echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost\n ```\n See the Linux kernel's\n [boost.txt](https://www.kernel.org/doc/Documentation/cpu-freq/boost.txt)\n for more information.\n3. Set the benchmark program's task affinity to a fixed cpu. For example:\n ```sh\n taskset -c 0 ./mybenchmark\n ```\n4. Increase the program's scheduling priority to minimize context switches using `nice` or `chrt`:\n ```sh\n sudo nice -n -20 ./mybenchmark\n sudo chrt -f 80 ./mybenchmark\n ```\n5. Disabling Hyperthreading/SMT. This can be done in the Bios or using the\n `/sys` file system (see the LLVM project's [Benchmarking\n tips](https://llvm.org/docs/Benchmarking.html)).\n6. Close other programs that do non-trivial things based on timers, such as\n your web browser, desktop environment, etc.\n7. Reduce the working set of your benchmark to fit within the L1 cache, but\n do be aware that this may lead you to optimize for an unrealistic\n situation.\n\nFurther resources on this topic:\n\n1. The LLVM project's [Benchmarking\n tips](https://llvm.org/docs/Benchmarking.html).\n1. The Arch Wiki [Cpu frequency\nscaling](https://wiki.archlinux.org/title/CPU_frequency_scaling) page.\n" }, { "path": "docs/releasing.md", "content": "# How to release\n\n* Make sure you're on main and synced to HEAD\n* Ensure the project builds and tests run\n * `parallel -j0 exec ::: test/*_test` can help ensure everything at least\n passes\n* Prepare release notes\n * `git log $(git describe --abbrev=0 --tags)..HEAD` gives you the list of\n commits between the last annotated tag and HEAD\n * Pick the most interesting.\n* Create one last commit that updates the version saved in `CMakeLists.txt`, `MODULE.bazel`,\n and `bindings/python/google_benchmark/__init__.py` to the release version you're creating.\n (This version will be used if benchmark is installed from the archive you'll be creating\n in the next step.)\n\n```\n# CMakeLists.txt\nproject (benchmark VERSION 1.9.0 LANGUAGES CXX)\n```\n\n```\n# MODULE.bazel\nmodule(name = \"com_github_google_benchmark\", version=\"1.9.0\")\n```\n\n```\n# google_benchmark/__init__.py\n__version__ = \"1.9.0\"\n```\n\n* Create a release through github's interface\n * Note this will create a lightweight tag.\n * Update this to an annotated tag:\n * `git pull --tags`\n * `git tag -a -f `\n * `git push --force --tags origin`\n* Confirm that the \"Build and upload Python wheels\" action runs to completion\n * Run it manually if it hasn't run.\n" }, { "path": "docs/tools.md", "content": "# Benchmark Tools\n\n## compare.py\n\nThe `compare.py` can be used to compare the result of benchmarks.\n\n### Dependencies\nThe utility relies on the [scipy](https://www.scipy.org) package which can be installed using pip:\n```bash\npip3 install -r requirements.txt\n```\n\n### Displaying aggregates only\n\nThe switch `-a` / `--display_aggregates_only` can be used to control the\ndisplayment of the normal iterations vs the aggregates. When passed, it will\nbe passthrough to the benchmark binaries to be run, and will be accounted for\nin the tool itself; only the aggregates will be displayed, but not normal runs.\nIt only affects the display, the separate runs will still be used to calculate\nthe U test.\n\n### Modes of operation\n\nThere are three modes of operation:\n\n1. Just compare two benchmarks\nThe program is invoked like:\n\n``` bash\n$ compare.py benchmarks [benchmark options]...\n```\nWhere `` and `` either specify a benchmark executable file, or a JSON output file. The type of the input file is automatically detected. If a benchmark executable is specified then the benchmark is run to obtain the results. Otherwise the results are simply loaded from the output file.\n\n`[benchmark options]` will be passed to the benchmarks invocations. They can be anything that binary accepts, be it either normal `--benchmark_*` parameters, or some custom parameters your binary takes.\n\nExample output:\n```\n$ ./compare.py benchmarks ./a.out ./a.out\nRUNNING: ./a.out --benchmark_out=/tmp/tmprBT5nW\nRun on (8 X 4000 MHz CPU s)\n2017-11-07 21:16:44\n------------------------------------------------------\nBenchmark Time CPU Iterations\n------------------------------------------------------\nBM_memcpy/8 36 ns 36 ns 19101577 211.669MB/s\nBM_memcpy/64 76 ns 76 ns 9412571 800.199MB/s\nBM_memcpy/512 84 ns 84 ns 8249070 5.64771GB/s\nBM_memcpy/1024 116 ns 116 ns 6181763 8.19505GB/s\nBM_memcpy/8192 643 ns 643 ns 1062855 11.8636GB/s\nBM_copy/8 222 ns 222 ns 3137987 34.3772MB/s\nBM_copy/64 1608 ns 1608 ns 432758 37.9501MB/s\nBM_copy/512 12589 ns 12589 ns 54806 38.7867MB/s\nBM_copy/1024 25169 ns 25169 ns 27713 38.8003MB/s\nBM_copy/8192 201165 ns 201112 ns 3486 38.8466MB/s\nRUNNING: ./a.out --benchmark_out=/tmp/tmpt1wwG_\nRun on (8 X 4000 MHz CPU s)\n2017-11-07 21:16:53\n------------------------------------------------------\nBenchmark Time CPU Iterations\n------------------------------------------------------\nBM_memcpy/8 36 ns 36 ns 19397903 211.255MB/s\nBM_memcpy/64 73 ns 73 ns 9691174 839.635MB/s\nBM_memcpy/512 85 ns 85 ns 8312329 5.60101GB/s\nBM_memcpy/1024 118 ns 118 ns 6438774 8.11608GB/s\nBM_memcpy/8192 656 ns 656 ns 1068644 11.6277GB/s\nBM_copy/8 223 ns 223 ns 3146977 34.2338MB/s\nBM_copy/64 1611 ns 1611 ns 435340 37.8751MB/s\nBM_copy/512 12622 ns 12622 ns 54818 38.6844MB/s\nBM_copy/1024 25257 ns 25239 ns 27779 38.6927MB/s\nBM_copy/8192 205013 ns 205010 ns 3479 38.108MB/s\nComparing ./a.out to ./a.out\nBenchmark Time CPU Time Old Time New CPU Old CPU New\n------------------------------------------------------------------------------------------------------\nBM_memcpy/8 +0.0020 +0.0020 36 36 36 36\nBM_memcpy/64 -0.0468 -0.0470 76 73 76 73\nBM_memcpy/512 +0.0081 +0.0083 84 85 84 85\nBM_memcpy/1024 +0.0098 +0.0097 116 118 116 118\nBM_memcpy/8192 +0.0200 +0.0203 643 656 643 656\nBM_copy/8 +0.0046 +0.0042 222 223 222 223\nBM_copy/64 +0.0020 +0.0020 1608 1611 1608 1611\nBM_copy/512 +0.0027 +0.0026 12589 12622 12589 12622\nBM_copy/1024 +0.0035 +0.0028 25169 25257 25169 25239\nBM_copy/8192 +0.0191 +0.0194 201165 205013 201112 205010\n```\n\nWhat it does is for the every benchmark from the first run it looks for the benchmark with exactly the same name in the second run, and then compares the results. If the names differ, the benchmark is omitted from the diff.\nAs you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`.\n\n2. Compare two different filters of one benchmark\nThe program is invoked like:\n\n``` bash\n$ compare.py filters [benchmark options]...\n```\nWhere `` either specify a benchmark executable file, or a JSON output file. The type of the input file is automatically detected. If a benchmark executable is specified then the benchmark is run to obtain the results. Otherwise the results are simply loaded from the output file.\n\nWhere `` and `` are the same regex filters that you would pass to the `[--benchmark_filter=]` parameter of the benchmark binary.\n\n`[benchmark options]` will be passed to the benchmarks invocations. They can be anything that binary accepts, be it either normal `--benchmark_*` parameters, or some custom parameters your binary takes.\n\nExample output:\n```\n$ ./compare.py filters ./a.out BM_memcpy BM_copy\nRUNNING: ./a.out --benchmark_filter=BM_memcpy --benchmark_out=/tmp/tmpBWKk0k\nRun on (8 X 4000 MHz CPU s)\n2017-11-07 21:37:28\n------------------------------------------------------\nBenchmark Time CPU Iterations\n------------------------------------------------------\nBM_memcpy/8 36 ns 36 ns 17891491 211.215MB/s\nBM_memcpy/64 74 ns 74 ns 9400999 825.646MB/s\nBM_memcpy/512 87 ns 87 ns 8027453 5.46126GB/s\nBM_memcpy/1024 111 ns 111 ns 6116853 8.5648GB/s\nBM_memcpy/8192 657 ns 656 ns 1064679 11.6247GB/s\nRUNNING: ./a.out --benchmark_filter=BM_copy --benchmark_out=/tmp/tmpAvWcOM\nRun on (8 X 4000 MHz CPU s)\n2017-11-07 21:37:33\n----------------------------------------------------\nBenchmark Time CPU Iterations\n----------------------------------------------------\nBM_copy/8 227 ns 227 ns 3038700 33.6264MB/s\nBM_copy/64 1640 ns 1640 ns 426893 37.2154MB/s\nBM_copy/512 12804 ns 12801 ns 55417 38.1444MB/s\nBM_copy/1024 25409 ns 25407 ns 27516 38.4365MB/s\nBM_copy/8192 202986 ns 202990 ns 3454 38.4871MB/s\nComparing BM_memcpy to BM_copy (from ./a.out)\nBenchmark Time CPU Time Old Time New CPU Old CPU New\n--------------------------------------------------------------------------------------------------------------------\n[BM_memcpy vs. BM_copy]/8 +5.2829 +5.2812 36 227 36 227\n[BM_memcpy vs. BM_copy]/64 +21.1719 +21.1856 74 1640 74 1640\n[BM_memcpy vs. BM_copy]/512 +145.6487 +145.6097 87 12804 87 12801\n[BM_memcpy vs. BM_copy]/1024 +227.1860 +227.1776 111 25409 111 25407\n[BM_memcpy vs. BM_copy]/8192 +308.1664 +308.2898 657 202986 656 202990\n```\n\nAs you can see, it applies filter to the benchmarks, both when running the benchmark, and before doing the diff. And to make the diff work, the matches are replaced with some common string. Thus, you can compare two different benchmark families within one benchmark binary.\nAs you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`.\n\n3. Compare filter one from benchmark one to filter two from benchmark two:\nThe program is invoked like:\n\n``` bash\n$ compare.py filters [benchmark options]...\n```\n\nWhere `` and `` either specify a benchmark executable file, or a JSON output file. The type of the input file is automatically detected. If a benchmark executable is specified then the benchmark is run to obtain the results. Otherwise the results are simply loaded from the output file.\n\nWhere `` and `` are the same regex filters that you would pass to the `[--benchmark_filter=]` parameter of the benchmark binary.\n\n`[benchmark options]` will be passed to the benchmarks invocations. They can be anything that binary accepts, be it either normal `--benchmark_*` parameters, or some custom parameters your binary takes.\n\nExample output:\n```\n$ ./compare.py benchmarksfiltered ./a.out BM_memcpy ./a.out BM_copy\nRUNNING: ./a.out --benchmark_filter=BM_memcpy --benchmark_out=/tmp/tmp_FvbYg\nRun on (8 X 4000 MHz CPU s)\n2017-11-07 21:38:27\n------------------------------------------------------\nBenchmark Time CPU Iterations\n------------------------------------------------------\nBM_memcpy/8 37 ns 37 ns 18953482 204.118MB/s\nBM_memcpy/64 74 ns 74 ns 9206578 828.245MB/s\nBM_memcpy/512 91 ns 91 ns 8086195 5.25476GB/s\nBM_memcpy/1024 120 ns 120 ns 5804513 7.95662GB/s\nBM_memcpy/8192 664 ns 664 ns 1028363 11.4948GB/s\nRUNNING: ./a.out --benchmark_filter=BM_copy --benchmark_out=/tmp/tmpDfL5iE\nRun on (8 X 4000 MHz CPU s)\n2017-11-07 21:38:32\n----------------------------------------------------\nBenchmark Time CPU Iterations\n----------------------------------------------------\nBM_copy/8 230 ns 230 ns 2985909 33.1161MB/s\nBM_copy/64 1654 ns 1653 ns 419408 36.9137MB/s\nBM_copy/512 13122 ns 13120 ns 53403 37.2156MB/s\nBM_copy/1024 26679 ns 26666 ns 26575 36.6218MB/s\nBM_copy/8192 215068 ns 215053 ns 3221 36.3283MB/s\nComparing BM_memcpy (from ./a.out) to BM_copy (from ./a.out)\nBenchmark Time CPU Time Old Time New CPU Old CPU New\n--------------------------------------------------------------------------------------------------------------------\n[BM_memcpy vs. BM_copy]/8 +5.1649 +5.1637 37 230 37 230\n[BM_memcpy vs. BM_copy]/64 +21.4352 +21.4374 74 1654 74 1653\n[BM_memcpy vs. BM_copy]/512 +143.6022 +143.5865 91 13122 91 13120\n[BM_memcpy vs. BM_copy]/1024 +221.5903 +221.4790 120 26679 120 26666\n[BM_memcpy vs. BM_copy]/8192 +322.9059 +323.0096 664 215068 664 215053\n```\nThis is a mix of the previous two modes, two (potentially different) benchmark binaries are run, and a different filter is applied to each one.\nAs you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`.\n\n### Note: Interpreting the output\n\nPerformance measurements are an art, and performance comparisons are doubly so.\nResults are often noisy and don't necessarily have large absolute differences to\nthem, so just by visual inspection, it is not at all apparent if two\nmeasurements are actually showing a performance change or not. It is even more\nconfusing with multiple benchmark repetitions.\n\nThankfully, what we can do, is use statistical tests on the results to determine\nwhether the performance has statistically-significantly changed. `compare.py`\nuses [Mann–Whitney U\ntest](https://en.wikipedia.org/wiki/Mann%E2%80%93Whitney_U_test), with a null\nhypothesis being that there's no difference in performance.\n \n**The below output is a summary of a benchmark comparison with statistics\nprovided for a multi-threaded process.**\n```\nBenchmark Time CPU Time Old Time New CPU Old CPU New\n-----------------------------------------------------------------------------------------------------------------------------\nbenchmark/threads:1/process_time/real_time_pvalue 0.0000 0.0000 U Test, Repetitions: 27 vs 27\nbenchmark/threads:1/process_time/real_time_mean -0.1442 -0.1442 90 77 90 77\nbenchmark/threads:1/process_time/real_time_median -0.1444 -0.1444 90 77 90 77\nbenchmark/threads:1/process_time/real_time_stddev +0.3974 +0.3933 0 0 0 0\nbenchmark/threads:1/process_time/real_time_cv +0.6329 +0.6280 0 0 0 0\nOVERALL_GEOMEAN -0.1442 -0.1442 0 0 0 0\n```\n--------------------------------------------\nHere's a breakdown of each row:\n\n**benchmark/threads:1/process_time/real_time_pvalue**: This shows the _p-value_ for\nthe statistical test comparing the performance of the process running with one\nthread. A value of 0.0000 suggests a statistically significant difference in\nperformance. The comparison was conducted using the U Test (Mann-Whitney\nU Test) with 27 repetitions for each case.\n\n**benchmark/threads:1/process_time/real_time_mean**: This shows the relative\ndifference in mean execution time between two different cases. The negative\nvalue (-0.1442) implies that the new process is faster by about 14.42%. The old\ntime was 90 units, while the new time is 77 units.\n\n**benchmark/threads:1/process_time/real_time_median**: Similarly, this shows the\nrelative difference in the median execution time. Again, the new process is\nfaster by 14.44%.\n\n**benchmark/threads:1/process_time/real_time_stddev**: This is the relative\ndifference in the standard deviation of the execution time, which is a measure\nof how much variation or dispersion there is from the mean. A positive value\n(+0.3974) implies there is more variance in the execution time in the new\nprocess.\n\n**benchmark/threads:1/process_time/real_time_cv**: CV stands for Coefficient of\nVariation. It is the ratio of the standard deviation to the mean. It provides a\nstandardized measure of dispersion. An increase (+0.6329) indicates more\nrelative variability in the new process.\n\n**OVERALL_GEOMEAN**: Geomean stands for geometric mean, a type of average that is\nless influenced by outliers. The negative value indicates a general improvement\nin the new process. However, given the values are all zero for the old and new\ntimes, this seems to be a mistake or placeholder in the output.\n\n-----------------------------------------\n\n\n\nLet's first try to see what the different columns represent in the above\n`compare.py` benchmarking output:\n\n 1. **Benchmark:** The name of the function being benchmarked, along with the\n size of the input (after the slash).\n\n 2. **Time:** The average time per operation, across all iterations.\n\n 3. **CPU:** The average CPU time per operation, across all iterations.\n\n 4. **Iterations:** The number of iterations the benchmark was run to get a\n stable estimate.\n\n 5. **Time Old and Time New:** These represent the average time it takes for a\n function to run in two different scenarios or versions. For example, you\n might be comparing how fast a function runs before and after you make some\n changes to it.\n\n 6. **CPU Old and CPU New:** These show the average amount of CPU time that the\n function uses in two different scenarios or versions. This is similar to\n Time Old and Time New, but focuses on CPU usage instead of overall time.\n\nIn the comparison section, the relative differences in both time and CPU time\nare displayed for each input size.\n\n\nA statistically-significant difference is determined by a **p-value**, which is\na measure of the probability that the observed difference could have occurred\njust by random chance. A smaller p-value indicates stronger evidence against the\nnull hypothesis. \n\n**Therefore:**\n 1. If the p-value is less than the chosen significance level (alpha), we\n reject the null hypothesis and conclude the benchmarks are significantly\n different.\n 2. If the p-value is greater than or equal to alpha, we fail to reject the\n null hypothesis and treat the two benchmarks as similar.\n\n\n\nThe result of said the statistical test is additionally communicated through color coding:\n```diff\n+ Green:\n```\n The benchmarks are _**statistically different**_. This could mean the\n performance has either **significantly improved** or **significantly\n deteriorated**. You should look at the actual performance numbers to see which\n is the case.\n```diff\n- Red:\n```\n The benchmarks are _**statistically similar**_. This means the performance\n **hasn't significantly changed**.\n\nIn statistical terms, **'green'** means we reject the null hypothesis that\nthere's no difference in performance, and **'red'** means we fail to reject the\nnull hypothesis. This might seem counter-intuitive if you're expecting 'green'\nto mean 'improved performance' and 'red' to mean 'worsened performance'. \n```bash\n But remember, in this context:\n\n 'Success' means 'successfully finding a difference'.\n 'Failure' means 'failing to find a difference'.\n```\n\n\nAlso, please note that **even if** we determine that there **is** a\nstatistically-significant difference between the two measurements, it does not\n_necessarily_ mean that the actual benchmarks that were measured **are**\ndifferent, or vice versa, even if we determine that there is **no**\nstatistically-significant difference between the two measurements, it does not\nnecessarily mean that the actual benchmarks that were measured **are not**\ndifferent.\n\n\n\n### U test\n\nIf there is a sufficient repetition count of the benchmarks, the tool can do\na [U Test](https://en.wikipedia.org/wiki/Mann%E2%80%93Whitney_U_test), of the\nnull hypothesis that it is equally likely that a randomly selected value from\none sample will be less than or greater than a randomly selected value from a\nsecond sample.\n\nIf the calculated p-value is below this value is lower than the significance\nlevel alpha, then the result is said to be statistically significant and the\nnull hypothesis is rejected. Which in other words means that the two benchmarks\naren't identical.\n\n**WARNING**: requires **LARGE** (no less than 9) number of repetitions to be\nmeaningful!\n" }, { "path": "docs/user_guide.md", "content": "# User Guide\n\n## Command Line\n\n[Output Formats](#output-formats)\n\n[Output Files](#output-files)\n\n[Running Benchmarks](#running-benchmarks)\n\n[Running a Subset of Benchmarks](#running-a-subset-of-benchmarks)\n\n[Result Comparison](#result-comparison)\n\n[Extra Context](#extra-context)\n\n## Library\n\n[Runtime and Reporting Considerations](#runtime-and-reporting-considerations)\n\n[Setup/Teardown](#setupteardown)\n\n[Passing Arguments](#passing-arguments)\n\n[Custom Benchmark Name](#custom-benchmark-name)\n\n[Calculating Asymptotic Complexity](#asymptotic-complexity)\n\n[Templated Benchmarks](#templated-benchmarks)\n\n[Templated Benchmarks that take arguments](#templated-benchmarks-with-arguments)\n\n[Fixtures](#fixtures)\n\n[Custom Counters](#custom-counters)\n\n[Multithreaded Benchmarks](#multithreaded-benchmarks)\n\n[CPU Timers](#cpu-timers)\n\n[Manual Timing](#manual-timing)\n\n[Setting the Time Unit](#setting-the-time-unit)\n\n[Random Interleaving](random_interleaving.md)\n\n[User-Requested Performance Counters](perf_counters.md)\n\n[Preventing Optimization](#preventing-optimization)\n\n[Reporting Statistics](#reporting-statistics)\n\n[Custom Statistics](#custom-statistics)\n\n[Memory Usage](#memory-usage)\n\n[Using RegisterBenchmark](#using-register-benchmark)\n\n[Exiting with an Error](#exiting-with-an-error)\n\n[A Faster `KeepRunning` Loop](#a-faster-keep-running-loop)\n\n## Benchmarking Tips\n\n[Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling)\n\n[Reducing Variance in Benchmarks](reducing_variance.md)\n\n\n\n## Output Formats\n\nThe library supports multiple output formats. Use the\n`--benchmark_format=` flag (or set the\n`BENCHMARK_FORMAT=` environment variable) to set\nthe format type. `console` is the default format.\n\nThe Console format is intended to be a human readable format. By default\nthe format generates color output. Context is output on stderr and the\ntabular data on stdout. Example tabular output looks like:\n\n```\nBenchmark Time(ns) CPU(ns) Iterations\n----------------------------------------------------------------------\nBM_SetInsert/1024/1 28928 29349 23853 133.097kiB/s 33.2742k items/s\nBM_SetInsert/1024/8 32065 32913 21375 949.487kiB/s 237.372k items/s\nBM_SetInsert/1024/10 33157 33648 21431 1.13369MiB/s 290.225k items/s\n```\n\nThe JSON format outputs human readable json split into two top level attributes.\nThe `context` attribute contains information about the run in general, including\ninformation about the CPU and the date.\nThe `benchmarks` attribute contains a list of every benchmark run. Example json\noutput looks like:\n\n```json\n{\n \"context\": {\n \"date\": \"2015/03/17-18:40:25\",\n \"num_cpus\": 40,\n \"mhz_per_cpu\": 2801,\n \"cpu_scaling_enabled\": false,\n \"build_type\": \"debug\"\n },\n \"benchmarks\": [\n {\n \"name\": \"BM_SetInsert/1024/1\",\n \"iterations\": 94877,\n \"real_time\": 29275,\n \"cpu_time\": 29836,\n \"bytes_per_second\": 134066,\n \"items_per_second\": 33516\n },\n {\n \"name\": \"BM_SetInsert/1024/8\",\n \"iterations\": 21609,\n \"real_time\": 32317,\n \"cpu_time\": 32429,\n \"bytes_per_second\": 986770,\n \"items_per_second\": 246693\n },\n {\n \"name\": \"BM_SetInsert/1024/10\",\n \"iterations\": 21393,\n \"real_time\": 32724,\n \"cpu_time\": 33355,\n \"bytes_per_second\": 1199226,\n \"items_per_second\": 299807\n }\n ]\n}\n```\n\nThe CSV format outputs comma-separated values. The `context` is output on stderr\nand the CSV itself on stdout. Example CSV output looks like:\n\n```\nname,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label\n\"BM_SetInsert/1024/1\",65465,17890.7,8407.45,475768,118942,\n\"BM_SetInsert/1024/8\",116606,18810.1,9766.64,3.27646e+06,819115,\n\"BM_SetInsert/1024/10\",106365,17238.4,8421.53,4.74973e+06,1.18743e+06,\n```\n\n\n\n## Output Files\n\nWrite benchmark results to a file with the `--benchmark_out=` option\n(or set `BENCHMARK_OUT`). Specify the output format with\n`--benchmark_out_format={json|console|csv}` (or set\n`BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that the 'csv' reporter is\ndeprecated and the saved `.csv` file\n[is not parsable](https://github.com/google/benchmark/issues/794) by csv\nparsers.\n\nSpecifying `--benchmark_out` does not suppress the console output.\n\n\n\n## Running Benchmarks\n\nBenchmarks are executed by running the produced binaries. Benchmarks binaries,\nby default, accept options that may be specified either through their command\nline interface or by setting environment variables before execution. For every\n`--option_flag=` CLI switch, a corresponding environment variable\n`OPTION_FLAG=` exist and is used as default if set (CLI switches always\n prevails). A complete list of CLI options is available running benchmarks\n with the `--help` switch.\n\n### Dry runs\n\nTo confirm that benchmarks can run successfully without needing to wait for\nmultiple repetitions and iterations, the `--benchmark_dry_run` flag can be\nused. This will run the benchmarks as normal, but for 1 iteration and 1\nrepetition only.\n\n\n\n## Running a Subset of Benchmarks\n\nThe `--benchmark_filter=` option (or `BENCHMARK_FILTER=`\nenvironment variable) can be used to only run the benchmarks that match\nthe specified ``. For example:\n\n```bash\n$ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32\nRun on (1 X 2300 MHz CPU )\n2016-06-25 19:34:24\nBenchmark Time CPU Iterations\n----------------------------------------------------\nBM_memcpy/32 11 ns 11 ns 79545455\nBM_memcpy/32k 2181 ns 2185 ns 324074\nBM_memcpy/32 12 ns 12 ns 54687500\nBM_memcpy/32k 1834 ns 1837 ns 357143\n```\n\n## Disabling Benchmarks\n\nIt is possible to temporarily disable benchmarks by renaming the benchmark\nfunction to have the prefix \"DISABLED_\". This will cause the benchmark to\nbe skipped at runtime.\n\n\n\n## Result comparison\n\nIt is possible to compare the benchmarking results.\nSee [Additional Tooling Documentation](tools.md)\n\n\n\n## Extra Context\n\nSometimes it's useful to add extra context to the content printed before the\nresults. By default this section includes information about the CPU on which\nthe benchmarks are running. If you do want to add more context, you can use\nthe `benchmark_context` command line flag:\n\n```bash\n$ ./run_benchmarks --benchmark_context=pwd=`pwd`\nRun on (1 x 2300 MHz CPU)\npwd: /home/user/benchmark/\nBenchmark Time CPU Iterations\n----------------------------------------------------\nBM_memcpy/32 11 ns 11 ns 79545455\nBM_memcpy/32k 2181 ns 2185 ns 324074\n```\n\nYou can get the same effect with the API:\n\n```c++\n benchmark::AddCustomContext(\"foo\", \"bar\");\n```\n\nNote that attempts to add a second value with the same key will fail with an\nerror message.\n\n\n\n## Runtime and Reporting Considerations\n\nWhen the benchmark binary is executed, each benchmark function is run serially.\nThe number of iterations to run is determined dynamically by running the\nbenchmark a few times and measuring the time taken and ensuring that the\nultimate result will be statistically stable. As such, faster benchmark\nfunctions will be run for more iterations than slower benchmark functions, and\nthe number of iterations is thus reported.\n\nIn all cases, the number of iterations for which the benchmark is run is\ngoverned by the amount of time the benchmark takes. Concretely, the number of\niterations is at least one, not more than 1e9, until CPU time is greater than\nthe minimum time, or the wallclock time is 5x minimum time. The minimum time is\nset per benchmark by calling `MinTime` on the registered benchmark object.\n\nFurthermore warming up a benchmark might be necessary in order to get\nstable results because of e.g caching effects of the code under benchmark.\nWarming up means running the benchmark a given amount of time, before\nresults are actually taken into account. The amount of time for which\nthe warmup should be run can be set per benchmark by calling\n`MinWarmUpTime` on the registered benchmark object or for all benchmarks\nusing the `--benchmark_min_warmup_time` command-line option. Note that\n`MinWarmUpTime` will overwrite the value of `--benchmark_min_warmup_time`\nfor the single benchmark. How many iterations the warmup run of each\nbenchmark takes is determined the same way as described in the paragraph\nabove. Per default the warmup phase is set to 0 seconds and is therefore\ndisabled.\n\nAverage timings are then reported over the iterations run. If multiple\nrepetitions are requested using the `--benchmark_repetitions` command-line\noption, or at registration time, the benchmark function will be run several\ntimes and statistical results across these repetitions will also be reported.\n\nAs well as the per-benchmark entries, a preamble in the report will include\ninformation about the machine on which the benchmarks are run.\n\n\n\n## Setup/Teardown\n\nGlobal setup/teardown specific to each benchmark can be done by\npassing a callback to Setup/Teardown:\n\nThe setup/teardown callbacks will be invoked once for each benchmark. If the\nbenchmark is multi-threaded (will run in k threads), they will be invoked\nexactly once before each run with k threads.\n\nIf the benchmark uses different size groups of threads, the above will be true\nfor each size group.\n\nEg.,\n\n```c++\nstatic void DoSetup(const benchmark::State& state) {\n}\n\nstatic void DoTeardown(const benchmark::State& state) {\n}\n\nstatic void BM_func(benchmark::State& state) {...}\n\nBENCHMARK(BM_func)->Arg(1)->Arg(3)->Threads(16)->Threads(32)->Setup(DoSetup)->Teardown(DoTeardown);\n\n```\n\nIn this example, `DoSetup` and `DoTearDown` will be invoked 4 times each,\nspecifically, once for each of this family:\n - BM_func_Arg_1_Threads_16, BM_func_Arg_1_Threads_32\n - BM_func_Arg_3_Threads_16, BM_func_Arg_3_Threads_32\n\n\n\n## Passing Arguments\n\nSometimes a family of benchmarks can be implemented with just one routine that\ntakes an extra argument to specify which one of the family of benchmarks to\nrun. For example, the following code defines a family of benchmarks for\nmeasuring the speed of `memcpy()` calls of different lengths:\n\n```c++\nstatic void BM_memcpy(benchmark::State& state) {\n char* src = new char[state.range(0)];\n char* dst = new char[state.range(0)];\n memset(src, 'x', state.range(0));\n for (auto _ : state)\n memcpy(dst, src, state.range(0));\n state.SetBytesProcessed(int64_t(state.iterations()) *\n int64_t(state.range(0)));\n delete[] src;\n delete[] dst;\n}\nBENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(4<<10)->Arg(8<<10);\n```\n\nThe preceding code is quite repetitive, and can be replaced with the following\nshort-hand. The following invocation will pick a few appropriate arguments in\nthe specified range and will generate a benchmark for each such argument.\n\n```c++\nBENCHMARK(BM_memcpy)->Range(8, 8<<10);\n```\n\nBy default the arguments in the range are generated in multiples of eight and\nthe command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the\nrange multiplier is changed to multiples of two.\n\n```c++\nBENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10);\n```\n\nNow arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ].\n\nThe preceding code shows a method of defining a sparse range. The following\nexample shows a method of defining a dense range. It is then used to benchmark\nthe performance of `std::vector` initialization for uniformly increasing sizes.\n\n```c++\nstatic void BM_DenseRange(benchmark::State& state) {\n for(auto _ : state) {\n std::vector v(state.range(0), state.range(0));\n auto data = v.data();\n benchmark::DoNotOptimize(data);\n benchmark::ClobberMemory();\n }\n}\nBENCHMARK(BM_DenseRange)->DenseRange(0, 1024, 128);\n```\n\nNow arguments generated are [ 0, 128, 256, 384, 512, 640, 768, 896, 1024 ].\n\nYou might have a benchmark that depends on two or more inputs. For example, the\nfollowing code defines a family of benchmarks for measuring the speed of set\ninsertion.\n\n```c++\nstatic void BM_SetInsert(benchmark::State& state) {\n std::set data;\n for (auto _ : state) {\n state.PauseTiming();\n data = ConstructRandomSet(state.range(0));\n state.ResumeTiming();\n for (int j = 0; j < state.range(1); ++j)\n data.insert(RandomNumber());\n }\n}\nBENCHMARK(BM_SetInsert)\n ->Args({1<<10, 128})\n ->Args({2<<10, 128})\n ->Args({4<<10, 128})\n ->Args({8<<10, 128})\n ->Args({1<<10, 512})\n ->Args({2<<10, 512})\n ->Args({4<<10, 512})\n ->Args({8<<10, 512});\n```\n\nThe preceding code is quite repetitive, and can be replaced with the following\nshort-hand. The following macro will pick a few appropriate arguments in the\nproduct of the two specified ranges and will generate a benchmark for each such\npair.\n\n\n```c++\nBENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}});\n```\n\n\nSome benchmarks may require specific argument values that cannot be expressed\nwith `Ranges`. In this case, `ArgsProduct` offers the ability to generate a\nbenchmark input for each combination in the product of the supplied vectors.\n\n\n```c++\nBENCHMARK(BM_SetInsert)\n ->ArgsProduct({{1<<10, 3<<10, 8<<10}, {20, 40, 60, 80}})\n// would generate the same benchmark arguments as\nBENCHMARK(BM_SetInsert)\n ->Args({1<<10, 20})\n ->Args({3<<10, 20})\n ->Args({8<<10, 20})\n ->Args({3<<10, 40})\n ->Args({8<<10, 40})\n ->Args({1<<10, 40})\n ->Args({1<<10, 60})\n ->Args({3<<10, 60})\n ->Args({8<<10, 60})\n ->Args({1<<10, 80})\n ->Args({3<<10, 80})\n ->Args({8<<10, 80});\n```\n\n\nFor the most common scenarios, helper methods for creating a list of\nintegers for a given sparse or dense range are provided.\n\n```c++\nBENCHMARK(BM_SetInsert)\n ->ArgsProduct({\n benchmark::CreateRange(8, 128, /*multi=*/2),\n benchmark::CreateDenseRange(1, 4, /*step=*/1)\n })\n// would generate the same benchmark arguments as\nBENCHMARK(BM_SetInsert)\n ->ArgsProduct({\n {8, 16, 32, 64, 128},\n {1, 2, 3, 4}\n });\n```\n\nFor more complex patterns of inputs, passing a custom function to `Apply` allows\nprogrammatic specification of an arbitrary set of arguments on which to run the\nbenchmark. The following example enumerates a dense range on one parameter,\nand a sparse range on the second.\n\n```c++\nstatic void CustomArguments(benchmark::Benchmark* b) {\n for (int i = 0; i <= 10; ++i)\n for (int j = 32; j <= 1024*1024; j *= 8)\n b->Args({i, j});\n}\nBENCHMARK(BM_SetInsert)->Apply(CustomArguments);\n```\n\n### Passing Arbitrary Arguments to a Benchmark\n\nIt is possible to define a benchmark that takes an arbitrary number\nof extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)`\nmacro creates a benchmark that invokes `func` with the `benchmark::State` as\nthe first argument followed by the specified `args...`.\nThe `test_case_name` is appended to the name of the benchmark and\nshould describe the values passed.\n\n```c++\ntemplate \nvoid BM_takes_args(benchmark::State& state, Args&&... args) {\n auto args_tuple = std::make_tuple(std::move(args)...);\n for (auto _ : state) {\n std::cout << std::get<0>(args_tuple) << \": \" << std::get<1>(args_tuple)\n << '\\n';\n [...]\n }\n}\n// Registers a benchmark named \"BM_takes_args/int_string_test\" that passes\n// the specified values to `args`.\nBENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string(\"abc\"));\n\n// Registers the same benchmark \"BM_takes_args/int_test\" that passes\n// the specified values to `args`.\nBENCHMARK_CAPTURE(BM_takes_args, int_test, 42, 43);\n```\n\nNote that elements of `...args` may refer to global variables. Users should\navoid modifying global state inside of a benchmark.\n\n### Naming a Benchmark Without Capturing Arguments\n\nIf you only need to give a benchmark a custom name (without passing extra\narguments), use `BENCHMARK_NAMED(func, test_case_name)`. Unlike\n`BENCHMARK_CAPTURE`, this macro does not create a lambda, which avoids\ncompiler and linker scalability issues when registering thousands of\nbenchmarks.\n\n```c++\nvoid BM_Foo(benchmark::State& state) {\n for (auto _ : state) {}\n}\n// Registers a benchmark named \"BM_Foo/my_variant\"\nBENCHMARK_NAMED(BM_Foo, my_variant);\n```\n\nUse `BENCHMARK_CAPTURE` when you need to pass extra arguments; use\n`BENCHMARK_NAMED` when you only need the name.\n\n\n\n## Calculating Asymptotic Complexity (Big O)\n\nAsymptotic complexity might be calculated for a family of benchmarks. The\nfollowing code will calculate the coefficient for the high-order term in the\nrunning time and the normalized root-mean square error of string comparison.\n\n```c++\nstatic void BM_StringCompare(benchmark::State& state) {\n std::string s1(state.range(0), '-');\n std::string s2(state.range(0), '-');\n for (auto _ : state) {\n auto comparison_result = s1.compare(s2);\n benchmark::DoNotOptimize(comparison_result);\n }\n state.SetComplexityN(state.range(0));\n}\nBENCHMARK(BM_StringCompare)\n ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN);\n```\n\nAs shown in the following invocation, asymptotic complexity might also be\ncalculated automatically.\n\n```c++\nBENCHMARK(BM_StringCompare)\n ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity();\n```\n\nThe following code will specify asymptotic complexity with a lambda function,\nthat might be used to customize high-order term calculation.\n\n```c++\nBENCHMARK(BM_StringCompare)->RangeMultiplier(2)\n ->Range(1<<10, 1<<18)->Complexity([](benchmark::IterationCount n)->double{return n; });\n```\n\n\n\n## Custom Benchmark Name\n\nYou can change the benchmark's name as follows:\n\n```c++\nBENCHMARK(BM_memcpy)->Name(\"memcpy\")->RangeMultiplier(2)->Range(8, 8<<10);\n```\n\nThe invocation will execute the benchmark as before using `BM_memcpy` but changes\nthe prefix in the report to `memcpy`.\n\n\n\n## Templated Benchmarks\n\nThis example produces and consumes messages of size `sizeof(v)` `range_x`\ntimes. It also outputs throughput in the absence of multiprogramming.\n\n```c++\ntemplate void BM_Sequential(benchmark::State& state) {\n Q q;\n typename Q::value_type v;\n for (auto _ : state) {\n for (int i = state.range(0); i--; )\n q.push(v);\n for (int e = state.range(0); e--; )\n q.Wait(&v);\n }\n // actually messages, not bytes:\n state.SetBytesProcessed(\n static_cast(state.iterations())*state.range(0));\n}\n\n// You can use the BENCHMARK macro with template parameters:\nBENCHMARK(BM_Sequential>)->Range(1<<0, 1<<10);\n\n// Old, legacy verbose C++03 syntax:\nBENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10);\n\n```\n\nThree macros are provided for adding benchmark templates.\n\n```c++\n#define BENCHMARK(func<...>) // Takes any number of parameters.\n#define BENCHMARK_TEMPLATE1(func, arg1)\n#define BENCHMARK_TEMPLATE2(func, arg1, arg2)\n```\n\n\n\n## Templated Benchmarks that take arguments\n\nSometimes there is a need to template benchmarks, and provide arguments to them.\n\n```c++\ntemplate void BM_Sequential_With_Step(benchmark::State& state, int step) {\n Q q;\n typename Q::value_type v;\n for (auto _ : state) {\n for (int i = state.range(0); i-=step; )\n q.push(v);\n for (int e = state.range(0); e-=step; )\n q.Wait(&v);\n }\n // actually messages, not bytes:\n state.SetBytesProcessed(\n static_cast(state.iterations())*state.range(0));\n}\n\nBENCHMARK_TEMPLATE1_CAPTURE(BM_Sequential, WaitQueue, Step1, 1)->Range(1<<0, 1<<10);\n```\n\n\n\n## Fixtures\n\nFixture tests are created by first defining a type that derives from\n`::benchmark::Fixture` and then creating/registering the tests using the\nfollowing macros:\n\n* `BENCHMARK_F(ClassName, Method)`\n* `BENCHMARK_DEFINE_F(ClassName, Method)`\n* `BENCHMARK_REGISTER_F(ClassName, Method)`\n\nFor Example:\n\n```c++\nclass MyFixture : public benchmark::Fixture {\npublic:\n void SetUp(::benchmark::State& state) {\n }\n\n void TearDown(::benchmark::State& state) {\n }\n};\n\n// Defines and registers `FooTest` using the class `MyFixture`.\nBENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) {\n for (auto _ : st) {\n ...\n }\n}\n\n// Only defines `BarTest` using the class `MyFixture`.\nBENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) {\n for (auto _ : st) {\n ...\n }\n}\n// `BarTest` is NOT registered.\nBENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2);\n// `BarTest` is now registered.\n```\n\n### Templated Fixtures\n\nAlso you can create templated fixture by using the following macros:\n\n* `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)`\n* `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)`\n\nFor example:\n\n```c++\ntemplate\nclass MyFixture : public benchmark::Fixture {};\n\n// Defines and registers `IntTest` using the class template `MyFixture`.\nBENCHMARK_TEMPLATE_F(MyFixture, IntTest, int)(benchmark::State& st) {\n for (auto _ : st) {\n ...\n }\n}\n\n// Only defines `DoubleTest` using the class template `MyFixture`.\nBENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) {\n for (auto _ : st) {\n ...\n }\n}\n// `DoubleTest` is NOT registered.\nBENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2);\n// `DoubleTest` is now registered.\n```\n\nIf you want to use a method template for your fixtures,\nwhich you instantiate afterward, use the following macros:\n\n* `BENCHMARK_TEMPLATE_METHOD_F(ClassName, Method)`\n* `BENCHMARK_TEMPLATE_INSTANTIATE_F(ClassName, Method, ...)`\n\nWith these macros you can define one method for several instantiations.\nExample (using `MyFixture` from above):\n\n```c++\n// Defines `Test` using the class template `MyFixture`.\nBENCHMARK_TEMPLATE_METHOD_F(MyFixture, Test)(benchmark::State& st) {\n for (auto _ : st) {\n ...\n }\n}\n\n// Instantiates and registers the benchmark `MyFixture::Test`.\nBENCHMARK_TEMPLATE_INSTANTIATE_F(MyFixture, Test, int)->Threads(2);\n// Instantiates and registers the benchmark `MyFixture::Test`.\nBENCHMARK_TEMPLATE_INSTANTIATE_F(MyFixture, Test, double)->Threads(4);\n```\n\nInside the method definition of `BENCHMARK_TEMPLATE_METHOD_F` the type `Base` refers\nto the type of the instantiated fixture.\nAccesses to members of the fixture must be prefixed by `this->`.\n\n`BENCHMARK_TEMPLATE_METHOD_F`and `BENCHMARK_TEMPLATE_INSTANTIATE_F` can only be used,\nif the fixture does not use non-type template parameters.\nIf you want to pass values as template parameters, use e.g. `std::integral_constant`.\nFor example:\n\n```c++\ntemplate\nclass SizedFixture : public benchmark::Fixture {\n static constexpr auto Size = Sz::value;\n int myValue;\n};\n\nBENCHMARK_TEMPLATE_METHOD_F(SizedFixture, Test)(benchmark::State& st) {\n for (auto _ : st) {\n this->myValue = Base::Size;\n }\n}\n\nBENCHMARK_TEMPLATE_INSTANTIATE_F(SizedFixture, Test, std::integral_constant<5>)->Threads(2);\n```\n\n\n\n## Custom Counters\n\nYou can add your own counters with user-defined names. The example below\nwill add columns \"Foo\", \"Bar\" and \"Baz\" in its output:\n\n```c++\nstatic void UserCountersExample1(benchmark::State& state) {\n double numFoos = 0, numBars = 0, numBazs = 0;\n for (auto _ : state) {\n // ... count Foo,Bar,Baz events\n }\n state.counters[\"Foo\"] = numFoos;\n state.counters[\"Bar\"] = numBars;\n state.counters[\"Baz\"] = numBazs;\n}\n```\n\nThe `state.counters` object is a `std::map` with `std::string` keys\nand `Counter` values. The latter is a `double`-like class, via an implicit\nconversion to `double&`. Thus you can use all of the standard arithmetic\nassignment operators (`=,+=,-=,*=,/=`) to change the value of each counter.\n\nThe `Counter` constructor accepts three parameters: the value as a `double`\n; a bit flag which allows you to show counters as rates, and/or as per-thread\niteration, and/or as per-thread averages, and/or iteration invariants,\nand/or finally inverting the result; and a flag specifying the 'unit' - i.e.\nis 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024\n(`benchmark::Counter::OneK::kIs1024`)?\n\n```c++\n // sets a simple counter\n state.counters[\"Foo\"] = numFoos;\n\n // Set the counter as a rate. It will be presented divided\n // by the duration of the benchmark.\n // Meaning: per one second, how many 'foo's are processed?\n state.counters[\"FooRate\"] = Counter(numFoos, benchmark::Counter::kIsRate);\n\n // Set the counter as a rate. It will be presented divided\n // by the duration of the benchmark, and the result inverted.\n // Meaning: how many seconds it takes to process one 'foo'?\n state.counters[\"FooInvRate\"] = Counter(numFoos, benchmark::Counter::kIsRate | benchmark::Counter::kInvert);\n\n // Set the counter as a thread-average quantity. It will\n // be presented divided by the number of threads.\n state.counters[\"FooAvg\"] = Counter(numFoos, benchmark::Counter::kAvgThreads);\n\n // There's also a combined flag:\n state.counters[\"FooAvgRate\"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate);\n\n // This says that we process with the rate of state.range(0) bytes every iteration:\n state.counters[\"BytesProcessed\"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024);\n```\n\nYou can use `insert()` with `std::initializer_list`:\n\n\n```c++\n state.counters.insert({{\"Foo\", numFoos}, {\"Bar\", numBars}, {\"Baz\", numBazs}});\n // ... instead of:\n state.counters[\"Foo\"] = numFoos;\n state.counters[\"Bar\"] = numBars;\n state.counters[\"Baz\"] = numBazs;\n```\n\n\nIn multithreaded benchmarks, each counter is set on the calling thread only.\nWhen the benchmark finishes, the counters from each thread will be summed.\nCounters that are configured with `kIsRate`, will report the average rate across all threads, while `kAvgThreadsRate` counters will report the average rate per thread.\n\n### Counter Reporting\n\nWhen using the console reporter, by default, user counters are printed at\nthe end after the table, the same way as ``bytes_processed`` and\n``items_processed``. This is best for cases in which there are few counters,\nor where there are only a couple of lines per benchmark. Here's an example of\nthe default output:\n\n```\n------------------------------------------------------------------------------\nBenchmark Time CPU Iterations UserCounters...\n------------------------------------------------------------------------------\nBM_UserCounter/threads:8 2248 ns 10277 ns 68808 Bar=16 Bat=40 Baz=24 Foo=8\nBM_UserCounter/threads:1 9797 ns 9788 ns 71523 Bar=2 Bat=5 Baz=3 Foo=1024m\nBM_UserCounter/threads:2 4924 ns 9842 ns 71036 Bar=4 Bat=10 Baz=6 Foo=2\nBM_UserCounter/threads:4 2589 ns 10284 ns 68012 Bar=8 Bat=20 Baz=12 Foo=4\nBM_UserCounter/threads:8 2212 ns 10287 ns 68040 Bar=16 Bat=40 Baz=24 Foo=8\nBM_UserCounter/threads:16 1782 ns 10278 ns 68144 Bar=32 Bat=80 Baz=48 Foo=16\nBM_UserCounter/threads:32 1291 ns 10296 ns 68256 Bar=64 Bat=160 Baz=96 Foo=32\nBM_UserCounter/threads:4 2615 ns 10307 ns 68040 Bar=8 Bat=20 Baz=12 Foo=4\nBM_Factorial 26 ns 26 ns 26608979 40320\nBM_Factorial/real_time 26 ns 26 ns 26587936 40320\nBM_CalculatePiRange/1 16 ns 16 ns 45704255 0\nBM_CalculatePiRange/8 73 ns 73 ns 9520927 3.28374\nBM_CalculatePiRange/64 609 ns 609 ns 1140647 3.15746\nBM_CalculatePiRange/512 4900 ns 4901 ns 142696 3.14355\n```\n\nIf this doesn't suit you, you can print each counter as a table column by\npassing the flag `--benchmark_counters_tabular=true` to the benchmark\napplication. This is best for cases in which there are a lot of counters, or\na lot of lines per individual benchmark. Note that this will trigger a\nreprinting of the table header any time the counter set changes between\nindividual benchmarks. Here's an example of corresponding output when\n`--benchmark_counters_tabular=true` is passed:\n\n```\n---------------------------------------------------------------------------------------\nBenchmark Time CPU Iterations Bar Bat Baz Foo\n---------------------------------------------------------------------------------------\nBM_UserCounter/threads:8 2198 ns 9953 ns 70688 16 40 24 8\nBM_UserCounter/threads:1 9504 ns 9504 ns 73787 2 5 3 1\nBM_UserCounter/threads:2 4775 ns 9550 ns 72606 4 10 6 2\nBM_UserCounter/threads:4 2508 ns 9951 ns 70332 8 20 12 4\nBM_UserCounter/threads:8 2055 ns 9933 ns 70344 16 40 24 8\nBM_UserCounter/threads:16 1610 ns 9946 ns 70720 32 80 48 16\nBM_UserCounter/threads:32 1192 ns 9948 ns 70496 64 160 96 32\nBM_UserCounter/threads:4 2506 ns 9949 ns 70332 8 20 12 4\n--------------------------------------------------------------\nBenchmark Time CPU Iterations\n--------------------------------------------------------------\nBM_Factorial 26 ns 26 ns 26392245 40320\nBM_Factorial/real_time 26 ns 26 ns 26494107 40320\nBM_CalculatePiRange/1 15 ns 15 ns 45571597 0\nBM_CalculatePiRange/8 74 ns 74 ns 9450212 3.28374\nBM_CalculatePiRange/64 595 ns 595 ns 1173901 3.15746\nBM_CalculatePiRange/512 4752 ns 4752 ns 147380 3.14355\nBM_CalculatePiRange/4k 37970 ns 37972 ns 18453 3.14184\nBM_CalculatePiRange/32k 303733 ns 303744 ns 2305 3.14162\nBM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416\nBM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159\nBM_CalculatePi/threads:8 2255 ns 9943 ns 70936\n```\n\nNote above the additional header printed when the benchmark changes from\n``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does\nnot have the same counter set as ``BM_UserCounter``.\n\n\n\n## Multithreaded Benchmarks\n\nIn a multithreaded test (benchmark invoked by multiple threads simultaneously),\nit is guaranteed that none of the threads will start until all have reached\nthe start of the benchmark loop, and all will have finished before any thread\nexits the benchmark loop. (This behavior is also provided by the `KeepRunning()`\nAPI) As such, any global setup or teardown can be wrapped in a check against the thread\nindex:\n\n```c++\nstatic void BM_MultiThreaded(benchmark::State& state) {\n if (state.thread_index() == 0) {\n // Setup code here.\n }\n for (auto _ : state) {\n // Run the test as normal.\n }\n if (state.thread_index() == 0) {\n // Teardown code here.\n }\n}\nBENCHMARK(BM_MultiThreaded)->Threads(2);\n```\n\nTo run the benchmark across a range of thread counts, instead of `Threads`, use\n`ThreadRange`. This takes two parameters (`min_threads` and `max_threads`) and\nruns the benchmark once for values in the inclusive range. For example:\n\n```c++\nBENCHMARK(BM_MultiThreaded)->ThreadRange(1, 8);\n```\n\nwill run `BM_MultiThreaded` with thread counts 1, 2, 4, and 8.\n\nIf the benchmarked code itself uses threads and you want to compare it to\nsingle-threaded code, you may want to use real-time (\"wallclock\") measurements\nfor latency comparisons:\n\n```c++\nBENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime();\n```\n\nWithout `UseRealTime`, CPU time is used by default.\n\n### Manual Multithreaded Benchmarks\n\nGoogle/benchmark uses `std::thread` as multithreading environment per default.\nIf you want to use another multithreading environment (e.g. OpenMP), you can provide\na factory function to your benchmark using the `ThreadRunner` function.\nThe factory function takes the number of threads as argument and creates a custom class\nderived from `benchmark::ThreadRunnerBase`.\nThis custom class must override the function\n`void RunThreads(const std::function& fn)`.\n`RunThreads` is called by the main thread and spawns the requested number of threads.\nEach spawned thread must call `fn(thread_index)`, where `thread_index` is its own\nthread index. Before `RunThreads` returns, all spawned threads must be joined.\n```c++\nclass OpenMPThreadRunner : public benchmark::ThreadRunnerBase\n{\n OpenMPThreadRunner(int num_threads)\n : num_threads_(num_threads)\n {}\n\n void RunThreads(const std::function& fn) final\n {\n#pragma omp parallel num_threads(num_threads_)\n fn(omp_get_thread_num());\n }\n\nprivate:\n int num_threads_;\n};\n\nBENCHMARK(BM_MultiThreaded)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(1)->Threads(2)->Threads(4);\n```\nThe above example creates a parallel OpenMP region before it enters `BM_MultiThreaded`.\nThe actual benchmark code can remain the same and is therefore not tied to a specific\nthread runner. The measurement does not include the time for creating and joining the\nthreads.\n\n\n\n## CPU Timers\n\nBy default, the CPU timer only measures the time spent by the main thread.\nIf the benchmark itself uses threads internally, this measurement may not\nbe what you are looking for. Instead, there is a way to measure the total\nCPU usage of the process, by all the threads.\n\n```c++\nvoid callee(int i);\n\nstatic void MyMain(int size) {\n#pragma omp parallel for\n for(int i = 0; i < size; i++)\n callee(i);\n}\n\nstatic void BM_OpenMP(benchmark::State& state) {\n for (auto _ : state)\n MyMain(state.range(0));\n}\n\n// Measure the time spent by the main thread, use it to decide for how long to\n// run the benchmark loop. Depending on the internal implementation detail may\n// measure to anywhere from near-zero (the overhead spent before/after work\n// handoff to worker thread[s]) to the whole single-thread time.\nBENCHMARK(BM_OpenMP)->Range(8, 8<<10);\n\n// Measure the user-visible time, the wall clock (literally, the time that\n// has passed on the clock on the wall), use it to decide for how long to\n// run the benchmark loop. This will always be meaningful, and will match the\n// time spent by the main thread in single-threaded case, in general decreasing\n// with the number of internal threads doing the work.\nBENCHMARK(BM_OpenMP)->Range(8, 8<<10)->UseRealTime();\n\n// Measure the total CPU consumption, use it to decide for how long to\n// run the benchmark loop. This will always measure to no less than the\n// time spent by the main thread in single-threaded case.\nBENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime();\n\n// A mixture of the last two. Measure the total CPU consumption, but use the\n// wall clock to decide for how long to run the benchmark loop.\nBENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime()->UseRealTime();\n```\n\n### Controlling Timers\n\nNormally, the entire duration of the work loop (`for (auto _ : state) {}`)\nis measured. But sometimes, it is necessary to do some work inside of\nthat loop, every iteration, but without counting that time to the benchmark time.\nThat is possible, although it is not recommended, since it has high overhead.\n\n\n```c++\nstatic void BM_SetInsert_With_Timer_Control(benchmark::State& state) {\n std::set data;\n for (auto _ : state) {\n state.PauseTiming(); // Stop timers. They will not count until they are resumed.\n data = ConstructRandomSet(state.range(0)); // Do something that should not be measured\n state.ResumeTiming(); // And resume timers. They are now counting again.\n // The rest will be measured.\n for (int j = 0; j < state.range(1); ++j)\n data.insert(RandomNumber());\n }\n}\nBENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}});\n```\n\n\n\n\n## Manual Timing\n\nFor benchmarking something for which neither CPU time nor real-time are\ncorrect or accurate enough, completely manual timing is supported using\nthe `UseManualTime` function.\n\nWhen `UseManualTime` is used, the benchmarked code must call\n`SetIterationTime` once per iteration of the benchmark loop to\nreport the manually measured time.\n\nAn example use case for this is benchmarking GPU execution (e.g. OpenCL\nor CUDA kernels, OpenGL or Vulkan or Direct3D draw calls), which cannot\nbe accurately measured using CPU time or real-time. Instead, they can be\nmeasured accurately using a dedicated API, and these measurement results\ncan be reported back with `SetIterationTime`.\n\n```c++\nstatic void BM_ManualTiming(benchmark::State& state) {\n int microseconds = state.range(0);\n std::chrono::duration sleep_duration {\n static_cast(microseconds)\n };\n\n for (auto _ : state) {\n auto start = std::chrono::high_resolution_clock::now();\n // Simulate some useful workload with a sleep\n std::this_thread::sleep_for(sleep_duration);\n auto end = std::chrono::high_resolution_clock::now();\n\n auto elapsed_seconds =\n std::chrono::duration_cast>(\n end - start);\n\n state.SetIterationTime(elapsed_seconds.count());\n }\n}\nBENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime();\n```\n\n\n\n## Setting the Time Unit\n\nIf a benchmark runs a few milliseconds it may be hard to visually compare the\nmeasured times, since the output data is given in nanoseconds per default. In\norder to manually set the time unit, you can specify it manually:\n\n```c++\nBENCHMARK(BM_test)->Unit(benchmark::kMillisecond);\n```\n\nAdditionally the default time unit can be set globally with the\n`--benchmark_time_unit={ns|us|ms|s}` command line argument. The argument only\naffects benchmarks where the time unit is not set explicitly.\n\n\n\n## Preventing Optimization\n\nTo prevent a value or expression from being optimized away by the compiler\nthe `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()`\nfunctions can be used.\n\n```c++\nstatic void BM_test(benchmark::State& state) {\n for (auto _ : state) {\n int x = 0;\n for (int i=0; i < 64; ++i) {\n benchmark::DoNotOptimize(x += i);\n }\n }\n}\n```\n\n`DoNotOptimize()` forces the *result* of `` to be stored in either\nmemory or a register. For GNU based compilers it acts as read/write barrier\nfor global memory. More specifically it forces the compiler to flush pending\nwrites to memory and reload any other values as necessary.\n\nNote that `DoNotOptimize()` does not prevent optimizations on ``\nin any way. `` may even be removed entirely when the result is already\nknown. For example:\n\n```c++\n // Example 1: `` is removed entirely.\n int foo(int x) { return x + 42; }\n while (...) DoNotOptimize(foo(0)); // Optimized to DoNotOptimize(42);\n\n // Example 2: Result of '' is only reused.\n int bar(int) __attribute__((const));\n while (...) DoNotOptimize(bar(0)); // Optimized to:\n // int __result__ = bar(0);\n // while (...) DoNotOptimize(__result__);\n```\n\nThe second tool for preventing optimizations is `ClobberMemory()`. In essence\n`ClobberMemory()` forces the compiler to perform all pending writes to global\nmemory. Memory managed by block scope objects must be \"escaped\" using\n`DoNotOptimize(...)` before it can be clobbered. In the below example\n`ClobberMemory()` prevents the call to `v.push_back(42)` from being optimized\naway.\n\n```c++\nstatic void BM_vector_push_back(benchmark::State& state) {\n for (auto _ : state) {\n std::vector v;\n v.reserve(1);\n auto data = v.data(); // Allow v.data() to be clobbered. Pass as non-const\n benchmark::DoNotOptimize(data); // lvalue to avoid undesired compiler optimizations\n v.push_back(42);\n benchmark::ClobberMemory(); // Force 42 to be written to memory.\n }\n}\n```\n\nNote that `ClobberMemory()` is only available for GNU or MSVC based compilers.\n\n\n\n## Statistics: Reporting the Mean, Median and Standard Deviation / Coefficient of variation of Repeated Benchmarks\n\nBy default each benchmark is run once and that single result is reported.\nHowever benchmarks are often noisy and a single result may not be representative\nof the overall behavior. For this reason it's possible to repeatedly rerun the\nbenchmark.\n\nThe number of runs of each benchmark is specified globally by the\n`--benchmark_repetitions` flag or on a per benchmark basis by calling\n`Repetitions` on the registered benchmark object. When a benchmark is run more\nthan once the mean, median, standard deviation and coefficient of variation\nof the runs will be reported.\n\nAdditionally the `--benchmark_report_aggregates_only={true|false}`,\n`--benchmark_display_aggregates_only={true|false}` flags or\n`ReportAggregatesOnly(bool)`, `DisplayAggregatesOnly(bool)` functions can be\nused to change how repeated tests are reported. By default the result of each\nrepeated run is reported. When `report aggregates only` option is `true`,\nonly the aggregates (i.e. mean, median, standard deviation and coefficient\nof variation, maybe complexity measurements if they were requested) of the runs\nis reported, to both the reporters - standard output (console), and the file.\nHowever when only the `display aggregates only` option is `true`,\nonly the aggregates are displayed in the standard output, while the file\noutput still contains everything.\nCalling `ReportAggregatesOnly(bool)` / `DisplayAggregatesOnly(bool)` on a\nregistered benchmark object overrides the value of the appropriate flag for that\nbenchmark.\n\n\n\n## Custom Statistics\n\nWhile having these aggregates is nice, this may not be enough for everyone.\nFor example you may want to know what the largest observation is, e.g. because\nyou have some real-time constraints. This is easy. The following code will\nspecify a custom statistic to be calculated, defined by a lambda function.\n\n```c++\nvoid BM_spin_empty(benchmark::State& state) {\n for (auto _ : state) {\n for (int x = 0; x < state.range(0); ++x) {\n benchmark::DoNotOptimize(x);\n }\n }\n}\n\nBENCHMARK(BM_spin_empty)\n ->Repetitions(3) // or add option --benchmark_repetitions=3\n ->ComputeStatistics(\"max\", [](const std::vector& v) -> double {\n return *(std::max_element(std::begin(v), std::end(v)));\n })\n ->Arg(512);\n```\n\nWhile usually the statistics produce values in time units,\nyou can also produce percentages:\n\n```c++\nvoid BM_spin_empty(benchmark::State& state) {\n for (auto _ : state) {\n for (int x = 0; x < state.range(0); ++x) {\n benchmark::DoNotOptimize(x);\n }\n }\n}\n\nBENCHMARK(BM_spin_empty)\n ->Repetitions(3) // or add option --benchmark_repetitions=3\n ->ComputeStatistics(\"ratio\", [](const std::vector& v) -> double {\n return v.front() / v.back();\n }, benchmark::StatisticUnit::kPercentage)\n ->Arg(512);\n```\n\n\n\n## Memory Usage\n\nIt's often useful to also track memory usage for benchmarks, alongside CPU\nperformance. For this reason, benchmark offers the `RegisterMemoryManager`\nmethod that allows a custom `MemoryManager` to be injected.\n\nIf set, the `MemoryManager::Start` and `MemoryManager::Stop` methods will be\ncalled at the start and end of benchmark runs to allow user code to fill out\na report on the number of allocations, bytes used, etc.\n\nThis data will then be reported alongside other performance data, currently\nonly when using JSON output.\n\n\n\n## Profiling\n\nIt's often useful to also profile benchmarks in particular ways, in addition to\nCPU performance. For this reason, benchmark offers the `RegisterProfilerManager`\nmethod that allows a custom `ProfilerManager` to be injected.\n\nIf set, the `ProfilerManager::AfterSetupStart` and\n`ProfilerManager::BeforeTeardownStop` methods will be called at the start and\nend of a separate benchmark run to allow user code to collect and report\nuser-provided profile metrics.\n\nOutput collected from this profiling run must be reported separately.\n\n\n\n## Using RegisterBenchmark(name, fn, args...)\n\nThe `RegisterBenchmark(name, func, args...)` function provides an alternative\nway to create and register benchmarks.\n`RegisterBenchmark(name, func, args...)` creates, registers, and returns a\npointer to a new benchmark with the specified `name` that invokes\n`func(st, args...)` where `st` is a `benchmark::State` object.\n\nUnlike the `BENCHMARK` registration macros, which can only be used at the global\nscope, the `RegisterBenchmark` can be called anywhere. This allows for\nbenchmark tests to be registered programmatically.\n\nAdditionally `RegisterBenchmark` allows any callable object to be registered\nas a benchmark. Including capturing lambdas and function objects.\n\nFor Example:\n```c++\nauto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ };\n\nint main(int argc, char** argv) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n for (auto& test_input : { /* ... */ })\n benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input);\n benchmark::Initialize(&argc, argv);\n benchmark::RunSpecifiedBenchmarks();\n benchmark::Shutdown();\n}\n```\n\n\n\n## Exiting with an Error\n\nWhen errors caused by external influences, such as file I/O and network\ncommunication, occur within a benchmark the\n`State::SkipWithError(const std::string& msg)` function can be used to skip that run\nof benchmark and report the error. Note that only future iterations of the\n`KeepRunning()` are skipped. For the ranged-for version of the benchmark loop\nUsers must explicitly exit the loop, otherwise all iterations will be performed.\nUsers may explicitly return to exit the benchmark immediately.\n\nThe `SkipWithError(...)` function may be used at any point within the benchmark,\nincluding before and after the benchmark loop. Moreover, if `SkipWithError(...)`\nhas been used, it is not required to reach the benchmark loop and one may return\nfrom the benchmark function early.\n\nFor example:\n\n```c++\nstatic void BM_test(benchmark::State& state) {\n auto resource = GetResource();\n if (!resource.good()) {\n state.SkipWithError(\"Resource is not good!\");\n // KeepRunning() loop will not be entered.\n }\n while (state.KeepRunning()) {\n auto data = resource.read_data();\n if (!resource.good()) {\n state.SkipWithError(\"Failed to read data!\");\n break; // Needed to skip the rest of the iteration.\n }\n do_stuff(data);\n }\n}\n\nstatic void BM_test_ranged_fo(benchmark::State & state) {\n auto resource = GetResource();\n if (!resource.good()) {\n state.SkipWithError(\"Resource is not good!\");\n return; // Early return is allowed when SkipWithError() has been used.\n }\n for (auto _ : state) {\n auto data = resource.read_data();\n if (!resource.good()) {\n state.SkipWithError(\"Failed to read data!\");\n break; // REQUIRED to prevent all further iterations.\n }\n do_stuff(data);\n }\n}\n```\n\n\n## A Faster KeepRunning Loop\n\nA ranged-based for loop should be used in preference to\nthe `KeepRunning` loop for running the benchmarks. For example:\n\n```c++\nstatic void BM_Fast(benchmark::State &state) {\n for (auto _ : state) {\n FastOperation();\n }\n}\nBENCHMARK(BM_Fast);\n```\n\nThe reason the ranged-for loop is faster than using `KeepRunning`, is\nbecause `KeepRunning` requires a memory load and store of the iteration count\never iteration, whereas the ranged-for variant is able to keep the iteration count\nin a register.\n\nFor example, an empty inner loop of using the ranged-based for method looks like:\n\n```asm\n# Loop Init\n mov rbx, qword ptr [r14 + 104]\n call benchmark::State::StartKeepRunning()\n test rbx, rbx\n je .LoopEnd\n.LoopHeader: # =>This Inner Loop Header: Depth=1\n add rbx, -1\n jne .LoopHeader\n.LoopEnd:\n```\n\nCompared to an empty `KeepRunning` loop, which looks like:\n\n```asm\n.LoopHeader: # in Loop: Header=BB0_3 Depth=1\n cmp byte ptr [rbx], 1\n jne .LoopInit\n.LoopBody: # =>This Inner Loop Header: Depth=1\n mov rax, qword ptr [rbx + 8]\n lea rcx, [rax + 1]\n mov qword ptr [rbx + 8], rcx\n cmp rax, qword ptr [rbx + 104]\n jb .LoopHeader\n jmp .LoopEnd\n.LoopInit:\n mov rdi, rbx\n call benchmark::State::StartKeepRunning()\n jmp .LoopBody\n.LoopEnd:\n```\n\nUnless C++03 compatibility is required, the ranged-for variant of writing\nthe benchmark loop should be preferred.\n\n\n\n## Disabling CPU Frequency Scaling\n\nIf you see this error:\n\n```\n***WARNING*** CPU scaling is enabled, the benchmark real time measurements may\nbe noisy and will incur extra overhead.\n```\n\nyou might want to disable the CPU frequency scaling while running the\nbenchmark, as well as consider other ways to stabilize the performance of\nyour system while benchmarking.\n\nSee [Reducing Variance](reducing_variance.md) for more information.\n" }, { "path": "include/benchmark/benchmark.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_BENCHMARK_H_\n#define BENCHMARK_BENCHMARK_H_\n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/counter.h\"\n#include \"benchmark/macros.h\"\n#include \"benchmark/managers.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/statistics.h\"\n#include \"benchmark/sysinfo.h\"\n#include \"benchmark/types.h\"\n#include \"benchmark/utils.h\"\n\n#endif // BENCHMARK_BENCHMARK_H_\n" }, { "path": "include/benchmark/benchmark_api.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_BENCHMARK_API_H_\n#define BENCHMARK_BENCHMARK_API_H_\n\n#if defined(_MSC_VER)\n#pragma warning(push)\n#pragma warning(disable : 4251)\n#endif\n\n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/counter.h\"\n#include \"benchmark/macros.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/statistics.h\"\n#include \"benchmark/types.h\"\n\nnamespace benchmark {\n\nconst char kDefaultMinTimeStr[] = \"0.5s\";\n\nBENCHMARK_EXPORT void MaybeReenterWithoutASLR(int, char**);\n\nBENCHMARK_EXPORT std::string GetBenchmarkVersion();\n\nBENCHMARK_EXPORT void PrintDefaultHelp();\n\nBENCHMARK_EXPORT void Initialize(int* argc, char** argv,\n void (*HelperPrintf)() = PrintDefaultHelp);\nBENCHMARK_EXPORT void Shutdown();\n\nBENCHMARK_EXPORT bool ReportUnrecognizedArguments(int argc, char** argv);\n\nBENCHMARK_EXPORT std::string GetBenchmarkFilter();\n\nBENCHMARK_EXPORT void SetBenchmarkFilter(std::string value);\n\nBENCHMARK_EXPORT int32_t GetBenchmarkVerbosity();\n\nBENCHMARK_EXPORT BenchmarkReporter* CreateDefaultDisplayReporter();\n\nBENCHMARK_EXPORT size_t RunSpecifiedBenchmarks();\nBENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(std::string spec);\n\nBENCHMARK_EXPORT size_t\nRunSpecifiedBenchmarks(BenchmarkReporter* display_reporter);\nBENCHMARK_EXPORT size_t\nRunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::string spec);\n\nBENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(\n BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter);\nBENCHMARK_EXPORT size_t\nRunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,\n BenchmarkReporter* file_reporter, std::string spec);\n\nBENCHMARK_EXPORT TimeUnit GetDefaultTimeUnit();\n\nBENCHMARK_EXPORT void SetDefaultTimeUnit(TimeUnit unit);\n\nBENCHMARK_EXPORT\nvoid AddCustomContext(std::string key, std::string value);\n\nstruct ThreadRunnerBase {\n virtual ~ThreadRunnerBase() {}\n virtual void RunThreads(const std::function& fn) = 0;\n};\n\nusing threadrunner_factory =\n std::function(int)>;\n\nnamespace internal {\nclass BenchmarkFamilies;\nclass BenchmarkInstance;\n} // namespace internal\n\nclass BENCHMARK_EXPORT Benchmark {\n public:\n virtual ~Benchmark();\n\n Benchmark* Name(const std::string& name);\n Benchmark* Arg(int64_t x);\n Benchmark* Unit(TimeUnit unit);\n Benchmark* Range(int64_t start, int64_t limit);\n Benchmark* DenseRange(int64_t start, int64_t limit, int step = 1);\n Benchmark* Args(const std::vector& args);\n Benchmark* ArgPair(int64_t x, int64_t y) {\n std::vector args;\n args.push_back(x);\n args.push_back(y);\n return Args(args);\n }\n Benchmark* Ranges(const std::vector>& ranges);\n Benchmark* ArgsProduct(const std::vector>& arglists);\n Benchmark* ArgName(const std::string& name);\n Benchmark* ArgNames(const std::vector& names);\n Benchmark* RangePair(int64_t lo1, int64_t hi1, int64_t lo2, int64_t hi2) {\n std::vector> ranges;\n ranges.push_back(std::make_pair(lo1, hi1));\n ranges.push_back(std::make_pair(lo2, hi2));\n return Ranges(ranges);\n }\n Benchmark* Setup(callback_function&&);\n Benchmark* Setup(const callback_function&);\n Benchmark* Teardown(callback_function&&);\n Benchmark* Teardown(const callback_function&);\n Benchmark* Apply(const std::function&);\n Benchmark* RangeMultiplier(int multiplier);\n Benchmark* MinTime(double t);\n Benchmark* MinWarmUpTime(double t);\n Benchmark* Iterations(IterationCount n);\n Benchmark* Repetitions(int n);\n Benchmark* ReportAggregatesOnly(bool value = true);\n Benchmark* DisplayAggregatesOnly(bool value = true);\n Benchmark* MeasureProcessCPUTime();\n Benchmark* UseRealTime();\n Benchmark* UseManualTime();\n Benchmark* Complexity(BigO complexity = benchmark::oAuto);\n Benchmark* Complexity(BigOFunc* complexity);\n Benchmark* ComputeStatistics(const std::string& name,\n StatisticsFunc* statistics,\n StatisticUnit unit = kTime);\n Benchmark* Threads(int t);\n Benchmark* ThreadRange(int min_threads, int max_threads);\n Benchmark* DenseThreadRange(int min_threads, int max_threads, int stride = 1);\n Benchmark* ThreadPerCpu();\n Benchmark* ThreadRunner(threadrunner_factory&& factory);\n\n virtual void Run(State& state) = 0;\n\n TimeUnit GetTimeUnit() const;\n\n protected:\n explicit Benchmark(const std::string& name);\n void SetName(const std::string& name);\n\n public:\n const char* GetName() const;\n int ArgsCnt() const;\n const char* GetArgName(int arg) const;\n\n private:\n friend class internal::BenchmarkFamilies;\n friend class internal::BenchmarkInstance;\n\n std::string name_;\n internal::AggregationReportMode aggregation_report_mode_;\n std::vector arg_names_;\n std::vector> args_;\n\n TimeUnit time_unit_;\n bool use_default_time_unit_;\n\n int range_multiplier_;\n double min_time_;\n double min_warmup_time_;\n IterationCount iterations_;\n int repetitions_;\n bool measure_process_cpu_time_;\n bool use_real_time_;\n bool use_manual_time_;\n BigO complexity_;\n BigOFunc* complexity_lambda_;\n std::vector statistics_;\n std::vector thread_counts_;\n\n callback_function setup_;\n callback_function teardown_;\n\n threadrunner_factory threadrunner_;\n\n BENCHMARK_DISALLOW_COPY_AND_ASSIGN(Benchmark);\n};\n\nnamespace internal {\ntypedef BENCHMARK_DEPRECATED_MSG(\n \"Use ::benchmark::Benchmark instead\")::benchmark::Benchmark Benchmark;\ntypedef BENCHMARK_DEPRECATED_MSG(\n \"Use ::benchmark::threadrunner_factory instead\")::benchmark::\n threadrunner_factory threadrunner_factory;\n\ntypedef void(Function)(State&);\n\nBENCHMARK_EXPORT ::benchmark::Benchmark* RegisterBenchmarkInternal(\n std::unique_ptr<::benchmark::Benchmark>);\nBENCHMARK_EXPORT std::map*& GetGlobalContext();\nBENCHMARK_EXPORT void UseCharPointer(char const volatile*);\nBENCHMARK_EXPORT int InitializeStreams();\nBENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams();\n} // namespace internal\n\nBenchmark* RegisterBenchmark(const std::string& name, internal::Function* fn);\n\ntemplate \nBenchmark* RegisterBenchmark(const std::string& name, Lambda&& fn);\n\nBENCHMARK_EXPORT void ClearRegisteredBenchmarks();\n\nnamespace internal {\nclass BENCHMARK_EXPORT FunctionBenchmark : public benchmark::Benchmark {\n public:\n FunctionBenchmark(const std::string& name, Function* func)\n : Benchmark(name), func_(func) {}\n void Run(State& st) override;\n\n private:\n Function* func_;\n};\n\ntemplate \nclass LambdaBenchmark : public benchmark::Benchmark {\n public:\n void Run(State& st) override { lambda_(st); }\n template \n LambdaBenchmark(const std::string& name, OLambda&& lam)\n : Benchmark(name), lambda_(std::forward(lam)) {}\n\n private:\n LambdaBenchmark(LambdaBenchmark const&) = delete;\n Lambda lambda_;\n};\n} // namespace internal\n\ninline Benchmark* RegisterBenchmark(const std::string& name,\n internal::Function* fn) {\n return internal::RegisterBenchmarkInternal(\n ::benchmark::internal::make_unique(name,\n fn));\n}\n\ntemplate \nBenchmark* RegisterBenchmark(const std::string& name, Lambda&& fn) {\n using BenchType =\n internal::LambdaBenchmark::type>;\n return internal::RegisterBenchmarkInternal(\n ::benchmark::internal::make_unique(name,\n std::forward(fn)));\n}\n\ntemplate \nBenchmark* RegisterBenchmark(const std::string& name, Lambda&& fn,\n Args&&... args) {\n return benchmark::RegisterBenchmark(\n name, [=](benchmark::State& st) { fn(st, args...); });\n}\n\nclass Fixture : public Benchmark {\n public:\n Fixture() : Benchmark(\"\") {}\n void Run(State& st) override {\n this->SetUp(st);\n this->BenchmarkCase(st);\n this->TearDown(st);\n }\n virtual void SetUp(const State&) {}\n virtual void TearDown(const State&) {}\n virtual void SetUp(State& st) { SetUp(const_cast(st)); }\n virtual void TearDown(State& st) { TearDown(const_cast(st)); }\n\n protected:\n virtual void BenchmarkCase(State&) = 0;\n};\n\nBENCHMARK_EXPORT\nstd::vector CreateRange(int64_t lo, int64_t hi, int multi);\n\nBENCHMARK_EXPORT\nstd::vector CreateDenseRange(int64_t start, int64_t limit, int step);\n\n} // namespace benchmark\n\n#if defined(_MSC_VER)\n#pragma warning(pop)\n#endif\n\n#endif // BENCHMARK_BENCHMARK_API_H_\n" }, { "path": "include/benchmark/counter.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_COUNTER_H_\n#define BENCHMARK_COUNTER_H_\n\n#if defined(_MSC_VER)\n#pragma warning(push)\n#pragma warning(disable : 4251)\n#endif\n\n#include \n#include \n\n#include \"benchmark/macros.h\"\n#include \"benchmark/types.h\"\n\nnamespace benchmark {\n\nclass BENCHMARK_EXPORT Counter {\n public:\n enum Flags {\n kDefaults = 0,\n kIsRate = 1 << 0,\n kAvgThreads = 1 << 1,\n kAvgThreadsRate = kIsRate | kAvgThreads,\n kIsIterationInvariant = 1 << 2,\n kIsIterationInvariantRate = kIsRate | kIsIterationInvariant,\n kAvgIterations = 1 << 3,\n kAvgIterationsRate = kIsRate | kAvgIterations,\n kInvert = 1 << 31\n };\n\n enum OneK { kIs1000 = 1000, kIs1024 = 1024 };\n\n double value;\n Flags flags;\n OneK oneK;\n\n BENCHMARK_ALWAYS_INLINE\n Counter(double v = 0., Flags f = kDefaults, OneK k = kIs1000)\n : value(v), flags(f), oneK(k) {}\n\n BENCHMARK_ALWAYS_INLINE operator double const&() const { return value; }\n BENCHMARK_ALWAYS_INLINE operator double&() { return value; }\n};\n\nCounter::Flags inline operator|(const Counter::Flags& LHS,\n const Counter::Flags& RHS) {\n return static_cast(static_cast(LHS) |\n static_cast(RHS));\n}\n\nusing UserCounters = std::map;\n\nnamespace internal {\nvoid Finish(UserCounters* l, IterationCount iterations, double cpu_time,\n double num_threads);\nvoid Increment(UserCounters* l, UserCounters const& r);\nbool SameNames(UserCounters const& l, UserCounters const& r);\n} // namespace internal\n\n} // namespace benchmark\n\n#if defined(_MSC_VER)\n#pragma warning(pop)\n#endif\n\n#endif // BENCHMARK_COUNTER_H_\n" }, { "path": "include/benchmark/export.h", "content": "#ifndef BENCHMARK_EXPORT_H\n#define BENCHMARK_EXPORT_H\n\n#if defined(_WIN32)\n#define EXPORT_ATTR __declspec(dllexport)\n#define IMPORT_ATTR __declspec(dllimport)\n#define NO_EXPORT_ATTR\n#define DEPRECATED_ATTR __declspec(deprecated)\n#else // _WIN32\n#define EXPORT_ATTR __attribute__((visibility(\"default\")))\n#define IMPORT_ATTR __attribute__((visibility(\"default\")))\n#define NO_EXPORT_ATTR __attribute__((visibility(\"hidden\")))\n#define DEPRECATE_ATTR __attribute__((__deprecated__))\n#endif // _WIN32\n\n#ifdef BENCHMARK_STATIC_DEFINE\n#define BENCHMARK_EXPORT\n#define BENCHMARK_NO_EXPORT\n#else // BENCHMARK_STATIC_DEFINE\n#ifndef BENCHMARK_EXPORT\n#ifdef benchmark_EXPORTS\n/* We are building this library */\n#define BENCHMARK_EXPORT EXPORT_ATTR\n#else // benchmark_EXPORTS\n/* We are using this library */\n#define BENCHMARK_EXPORT IMPORT_ATTR\n#endif // benchmark_EXPORTS\n#endif // !BENCHMARK_EXPORT\n\n#ifndef BENCHMARK_NO_EXPORT\n#define BENCHMARK_NO_EXPORT NO_EXPORT_ATTR\n#endif // !BENCHMARK_NO_EXPORT\n#endif // BENCHMARK_STATIC_DEFINE\n\n#ifndef BENCHMARK_DEPRECATED\n#define BENCHMARK_DEPRECATED DEPRECATE_ATTR\n#endif // BENCHMARK_DEPRECATED\n\n#ifndef BENCHMARK_DEPRECATED_EXPORT\n#define BENCHMARK_DEPRECATED_EXPORT BENCHMARK_EXPORT BENCHMARK_DEPRECATED\n#endif // BENCHMARK_DEPRECATED_EXPORT\n\n#ifndef BENCHMARK_DEPRECATED_NO_EXPORT\n#define BENCHMARK_DEPRECATED_NO_EXPORT BENCHMARK_NO_EXPORT BENCHMARK_DEPRECATED\n#endif // BENCHMARK_DEPRECATED_EXPORT\n\n#endif /* BENCHMARK_EXPORT_H */\n" }, { "path": "include/benchmark/macros.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_MACROS_H_\n#define BENCHMARK_MACROS_H_\n\n#if defined(_MSC_VER)\n#include \n#endif\n\n#include \n\n#define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \\\n TypeName(const TypeName&) = delete; \\\n TypeName& operator=(const TypeName&) = delete\n\n#ifdef BENCHMARK_HAS_CXX17\n#define BENCHMARK_UNUSED [[maybe_unused]]\n#elif defined(__GNUC__) || defined(__clang__)\n#define BENCHMARK_UNUSED __attribute__((unused))\n#else\n#define BENCHMARK_UNUSED\n#endif\n\n#if defined(__clang__)\n#define BENCHMARK_DONT_OPTIMIZE __attribute__((optnone))\n#elif defined(__GNUC__) || defined(__GNUG__)\n#define BENCHMARK_DONT_OPTIMIZE __attribute__((optimize(0)))\n#else\n#define BENCHMARK_DONT_OPTIMIZE\n#endif\n\n#if defined(__GNUC__) || defined(__clang__)\n#define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline))\n#elif defined(_MSC_VER) && !defined(__clang__)\n#define BENCHMARK_ALWAYS_INLINE __forceinline\n#define __func__ __FUNCTION__\n#else\n#define BENCHMARK_ALWAYS_INLINE\n#endif\n\n#define BENCHMARK_INTERNAL_TOSTRING2(x) #x\n#define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x)\n\n#if (defined(__GNUC__) && !defined(__NVCC__) && !defined(__NVCOMPILER)) || \\\n defined(__clang__)\n#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y)\n#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg)))\n#define BENCHMARK_DISABLE_DEPRECATED_WARNING \\\n _Pragma(\"GCC diagnostic push\") \\\n _Pragma(\"GCC diagnostic ignored \\\"-Wdeprecated-declarations\\\"\")\n#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma(\"GCC diagnostic pop\")\n#elif defined(__NVCOMPILER)\n#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y)\n#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg)))\n#define BENCHMARK_DISABLE_DEPRECATED_WARNING \\\n _Pragma(\"diagnostic push\") \\\n _Pragma(\"diag_suppress deprecated_entity_with_custom_message\")\n#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma(\"diagnostic pop\")\n#elif defined(_MSC_VER)\n#define BENCHMARK_BUILTIN_EXPECT(x, y) x\n#define BENCHMARK_DEPRECATED_MSG(msg) __declspec(deprecated(msg))\n#define BENCHMARK_WARNING_MSG(msg) \\\n __pragma(message(__FILE__ \"(\" BENCHMARK_INTERNAL_TOSTRING( \\\n __LINE__) \") : warning note: \" msg))\n#define BENCHMARK_DISABLE_DEPRECATED_WARNING \\\n __pragma(warning(push)) __pragma(warning(disable : 4996))\n#define BENCHMARK_RESTORE_DEPRECATED_WARNING __pragma(warning(pop))\n#else\n#define BENCHMARK_BUILTIN_EXPECT(x, y) x\n#define BENCHMARK_DEPRECATED_MSG(msg)\n#define BENCHMARK_WARNING_MSG(msg) \\\n __pragma(message(__FILE__ \"(\" BENCHMARK_INTERNAL_TOSTRING( \\\n __LINE__) \") : warning note: \" msg))\n#define BENCHMARK_DISABLE_DEPRECATED_WARNING\n#define BENCHMARK_RESTORE_DEPRECATED_WARNING\n#endif\n\n#if defined(__GNUC__) && !defined(__clang__)\n#define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)\n#endif\n\n#ifndef __has_builtin\n#define __has_builtin(x) 0\n#endif\n\n#if defined(__GNUC__) || __has_builtin(__builtin_unreachable)\n#define BENCHMARK_UNREACHABLE() __builtin_unreachable()\n#elif defined(_MSC_VER)\n#define BENCHMARK_UNREACHABLE() __assume(false)\n#else\n#define BENCHMARK_UNREACHABLE() ((void)0)\n#endif\n\n#if defined(__GNUC__)\n#if defined(__i386__) || defined(__x86_64__)\n#define BENCHMARK_INTERNAL_CACHELINE_SIZE 64\n#elif defined(__powerpc64__)\n#define BENCHMARK_INTERNAL_CACHELINE_SIZE 128\n#elif defined(__aarch64__)\n#define BENCHMARK_INTERNAL_CACHELINE_SIZE 64\n#elif defined(__arm__)\n#if defined(__ARM_ARCH_5T__)\n#define BENCHMARK_INTERNAL_CACHELINE_SIZE 32\n#elif defined(__ARM_ARCH_7A__)\n#define BENCHMARK_INTERNAL_CACHELINE_SIZE 64\n#endif\n#endif\n#endif\n\n#ifndef BENCHMARK_INTERNAL_CACHELINE_SIZE\n#define BENCHMARK_INTERNAL_CACHELINE_SIZE 64\n#endif\n\n#if defined(__GNUC__)\n#define BENCHMARK_INTERNAL_CACHELINE_ALIGNED \\\n __attribute__((aligned(BENCHMARK_INTERNAL_CACHELINE_SIZE)))\n#elif defined(_MSC_VER)\n#define BENCHMARK_INTERNAL_CACHELINE_ALIGNED \\\n __declspec(align(BENCHMARK_INTERNAL_CACHELINE_SIZE))\n#else\n#define BENCHMARK_INTERNAL_CACHELINE_ALIGNED\n#endif\n\n#endif // BENCHMARK_MACROS_H_\n" }, { "path": "include/benchmark/managers.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_MANAGERS_H_\n#define BENCHMARK_MANAGERS_H_\n\n#include \n\n#include \n\n#include \"benchmark/macros.h\"\n#include \"benchmark/types.h\"\n\nnamespace benchmark {\n\nclass MemoryManager {\n public:\n static constexpr int64_t TombstoneValue = std::numeric_limits::max();\n\n struct Result {\n Result()\n : num_allocs(0),\n max_bytes_used(0),\n total_allocated_bytes(TombstoneValue),\n net_heap_growth(TombstoneValue),\n memory_iterations(0) {}\n\n int64_t num_allocs;\n int64_t max_bytes_used;\n int64_t total_allocated_bytes;\n int64_t net_heap_growth;\n IterationCount memory_iterations;\n };\n\n virtual ~MemoryManager() {}\n virtual void Start() = 0;\n virtual void Stop(Result& result) = 0;\n};\n\nBENCHMARK_EXPORT\nvoid RegisterMemoryManager(MemoryManager* memory_manager);\n\nclass ProfilerManager {\n public:\n virtual ~ProfilerManager() {}\n virtual void AfterSetupStart() = 0;\n virtual void BeforeTeardownStop() = 0;\n};\n\nBENCHMARK_EXPORT\nvoid RegisterProfilerManager(ProfilerManager* profiler_manager);\n\n} // namespace benchmark\n\n#endif // BENCHMARK_MANAGERS_H_\n" }, { "path": "include/benchmark/registration.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_REGISTRATION_H_\n#define BENCHMARK_REGISTRATION_H_\n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/macros.h\"\n\n#if defined(__clang__)\n#define BENCHMARK_DISABLE_COUNTER_WARNING \\\n _Pragma(\"GCC diagnostic push\") \\\n _Pragma(\"GCC diagnostic ignored \\\"-Wunknown-warning-option\\\"\") \\\n _Pragma(\"GCC diagnostic ignored \\\"-Wc2y-extensions\\\"\")\n#define BENCHMARK_RESTORE_COUNTER_WARNING _Pragma(\"GCC diagnostic pop\")\n#else\n#define BENCHMARK_DISABLE_COUNTER_WARNING\n#define BENCHMARK_RESTORE_COUNTER_WARNING\n#endif\n\nBENCHMARK_DISABLE_COUNTER_WARNING\n#if defined(__COUNTER__) && (__COUNTER__ + 1 == __COUNTER__ + 0)\n#define BENCHMARK_PRIVATE_UNIQUE_ID __COUNTER__\n#else\n#define BENCHMARK_PRIVATE_UNIQUE_ID __LINE__\n#endif\nBENCHMARK_RESTORE_COUNTER_WARNING\n\n#define BENCHMARK_PRIVATE_NAME(...) \\\n BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, \\\n __VA_ARGS__)\n\n#define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c)\n#define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c\n#define BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method) \\\n BaseClass##_##Method##_Benchmark\n\n#define BENCHMARK_PRIVATE_DECLARE(n) \\\n BENCHMARK_DISABLE_COUNTER_WARNING \\\n static ::benchmark::Benchmark const* const BENCHMARK_PRIVATE_NAME(n) \\\n BENCHMARK_RESTORE_COUNTER_WARNING BENCHMARK_UNUSED\n\n#define BENCHMARK(...) \\\n BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \\\n (::benchmark::internal::RegisterBenchmarkInternal( \\\n ::benchmark::internal::make_unique< \\\n ::benchmark::internal::FunctionBenchmark>( \\\n #__VA_ARGS__, \\\n static_cast<::benchmark::internal::Function*>(__VA_ARGS__))))\n\n#define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a))\n#define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->Args({(a1), (a2)})\n#define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t))\n#define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi))\n#define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \\\n BENCHMARK(n)->RangePair({{(l1), (h1)}, {(l2), (h2)}})\n\n#define BENCHMARK_CAPTURE(func, test_case_name, ...) \\\n BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \\\n (::benchmark::internal::RegisterBenchmarkInternal( \\\n ::benchmark::internal::make_unique< \\\n ::benchmark::internal::FunctionBenchmark>( \\\n #func \"/\" #test_case_name, \\\n [](::benchmark::State& st) { func(st, __VA_ARGS__); })))\n\n#define BENCHMARK_NAMED(func, test_case_name) \\\n BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \\\n (::benchmark::internal::RegisterBenchmarkInternal( \\\n ::benchmark::internal::make_unique< \\\n ::benchmark::internal::FunctionBenchmark>( \\\n #func \"/\" #test_case_name, \\\n static_cast<::benchmark::internal::Function*>(func))))\n\n#define BENCHMARK_TEMPLATE1(n, a) \\\n BENCHMARK_PRIVATE_DECLARE(n) = \\\n (::benchmark::internal::RegisterBenchmarkInternal( \\\n ::benchmark::internal::make_unique< \\\n ::benchmark::internal::FunctionBenchmark>( \\\n #n \"<\" #a \">\", \\\n static_cast<::benchmark::internal::Function*>(n))))\n\n#define BENCHMARK_TEMPLATE2(n, a, b) \\\n BENCHMARK_PRIVATE_DECLARE(n) = \\\n (::benchmark::internal::RegisterBenchmarkInternal( \\\n ::benchmark::internal::make_unique< \\\n ::benchmark::internal::FunctionBenchmark>( \\\n #n \"<\" #a \",\" #b \">\", \\\n static_cast<::benchmark::internal::Function*>(n))))\n\n#define BENCHMARK_TEMPLATE(n, ...) \\\n BENCHMARK_PRIVATE_DECLARE(n) = \\\n (::benchmark::internal::RegisterBenchmarkInternal( \\\n ::benchmark::internal::make_unique< \\\n ::benchmark::internal::FunctionBenchmark>( \\\n #n \"<\" #__VA_ARGS__ \">\", \\\n static_cast<::benchmark::internal::Function*>(n<__VA_ARGS__>))))\n\n#define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \\\n BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__)\n\n#define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \\\n BENCHMARK_PRIVATE_DECLARE(func) = \\\n (::benchmark::internal::RegisterBenchmarkInternal( \\\n ::benchmark::internal::make_unique< \\\n ::benchmark::internal::FunctionBenchmark>( \\\n #func \"<\" #a \",\" #b \">\" \\\n \"/\" #test_case_name, \\\n [](::benchmark::State& st) { func(st, __VA_ARGS__); })))\n\n#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \\\n class BaseClass##_##Method##_Benchmark : public BaseClass { \\\n public: \\\n BaseClass##_##Method##_Benchmark() { \\\n this->SetName(#BaseClass \"/\" #Method); \\\n } \\\n \\\n protected: \\\n void BenchmarkCase(::benchmark::State&) override; \\\n };\n\n#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \\\n class BaseClass##_##Method##_Benchmark : public BaseClass { \\\n public: \\\n BaseClass##_##Method##_Benchmark() { \\\n this->SetName(#BaseClass \"<\" #a \">/\" #Method); \\\n } \\\n \\\n protected: \\\n void BenchmarkCase(::benchmark::State&) override; \\\n };\n\n#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \\\n class BaseClass##_##Method##_Benchmark : public BaseClass { \\\n public: \\\n BaseClass##_##Method##_Benchmark() { \\\n this->SetName(#BaseClass \"<\" #a \",\" #b \">/\" #Method); \\\n } \\\n \\\n protected: \\\n void BenchmarkCase(::benchmark::State&) override; \\\n };\n\n#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, ...) \\\n class BaseClass##_##Method##_Benchmark : public BaseClass<__VA_ARGS__> { \\\n public: \\\n BaseClass##_##Method##_Benchmark() { \\\n this->SetName(#BaseClass \"<\" #__VA_ARGS__ \">/\" #Method); \\\n } \\\n \\\n protected: \\\n void BenchmarkCase(::benchmark::State&) override; \\\n };\n\n#define BENCHMARK_DEFINE_F(BaseClass, Method) \\\n BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \\\n void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase\n\n#define BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) \\\n BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \\\n void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase\n\n#define BENCHMARK_TEMPLATE2_DEFINE_F(BaseClass, Method, a, b) \\\n BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \\\n void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase\n\n#define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, ...) \\\n BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \\\n void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase\n\n#define BENCHMARK_REGISTER_F(BaseClass, Method) \\\n BENCHMARK_PRIVATE_REGISTER_F(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method))\n\n#define BENCHMARK_PRIVATE_REGISTER_F(TestName) \\\n BENCHMARK_PRIVATE_DECLARE(TestName) = \\\n (::benchmark::internal::RegisterBenchmarkInternal( \\\n ::benchmark::internal::make_unique()))\n\n#define BENCHMARK_TEMPLATE_PRIVATE_CONCAT_NAME_F(BaseClass, Method) \\\n BaseClass##_##Method##_BenchmarkTemplate\n\n#define BENCHMARK_TEMPLATE_METHOD_F(BaseClass, Method) \\\n template \\\n class BENCHMARK_TEMPLATE_PRIVATE_CONCAT_NAME_F(BaseClass, Method) \\\n : public BaseClass { \\\n protected: \\\n using Base = BaseClass; \\\n void BenchmarkCase(::benchmark::State&) override; \\\n }; \\\n template \\\n void BENCHMARK_TEMPLATE_PRIVATE_CONCAT_NAME_F( \\\n BaseClass, Method)::BenchmarkCase\n\n#define BENCHMARK_TEMPLATE_PRIVATE_INSTANTIATE_F(BaseClass, Method, \\\n UniqueName, ...) \\\n class UniqueName : public BENCHMARK_TEMPLATE_PRIVATE_CONCAT_NAME_F( \\\n BaseClass, Method)<__VA_ARGS__> { \\\n public: \\\n UniqueName() { this->SetName(#BaseClass \"<\" #__VA_ARGS__ \">/\" #Method); } \\\n }; \\\n BENCHMARK_PRIVATE_DECLARE(BaseClass##_##Method##_Benchmark) = \\\n (::benchmark::internal::RegisterBenchmarkInternal( \\\n ::benchmark::internal::make_unique()))\n\n#define BENCHMARK_TEMPLATE_INSTANTIATE_F(BaseClass, Method, ...) \\\n BENCHMARK_DISABLE_COUNTER_WARNING \\\n BENCHMARK_TEMPLATE_PRIVATE_INSTANTIATE_F( \\\n BaseClass, Method, BENCHMARK_PRIVATE_NAME(BaseClass##Method), \\\n __VA_ARGS__) \\\n BENCHMARK_RESTORE_COUNTER_WARNING\n\n#define BENCHMARK_F(BaseClass, Method) \\\n BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \\\n BENCHMARK_REGISTER_F(BaseClass, Method); \\\n void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase\n\n#define BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) \\\n BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \\\n BENCHMARK_REGISTER_F(BaseClass, Method); \\\n void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase\n\n#define BENCHMARK_TEMPLATE2_F(BaseClass, Method, a, b) \\\n BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \\\n BENCHMARK_REGISTER_F(BaseClass, Method); \\\n void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase\n\n#define BENCHMARK_TEMPLATE_F(BaseClass, Method, ...) \\\n BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \\\n void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase\n\n#define BENCHMARK_MAIN() \\\n int main(int argc, char** argv) { \\\n benchmark::MaybeReenterWithoutASLR(argc, argv); \\\n char arg0_default[] = \"benchmark\"; \\\n char* args_default = reinterpret_cast(arg0_default); \\\n if (!argv) { \\\n argc = 1; \\\n argv = &args_default; \\\n } \\\n ::benchmark::Initialize(&argc, argv); \\\n if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \\\n ::benchmark::RunSpecifiedBenchmarks(); \\\n ::benchmark::Shutdown(); \\\n return 0; \\\n } \\\n int main(int, char**)\n\n#endif // BENCHMARK_REGISTRATION_H_\n" }, { "path": "include/benchmark/reporter.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_REPORTER_H_\n#define BENCHMARK_REPORTER_H_\n\n#if defined(_MSC_VER)\n#pragma warning(push)\n// C4251: needs to have dll-interface to be used by clients of class\n#pragma warning(disable : 4251)\n#endif\n\n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/counter.h\"\n#include \"benchmark/macros.h\"\n#include \"benchmark/managers.h\"\n#include \"benchmark/statistics.h\"\n#include \"benchmark/sysinfo.h\"\n#include \"benchmark/types.h\"\n\nnamespace benchmark {\n\nstruct BENCHMARK_EXPORT BenchmarkName {\n std::string function_name;\n std::string args;\n std::string min_time;\n std::string min_warmup_time;\n std::string iterations;\n std::string repetitions;\n std::string time_type;\n std::string threads;\n\n std::string str() const;\n};\n\nclass BENCHMARK_EXPORT BenchmarkReporter {\n public:\n struct Context {\n CPUInfo const& cpu_info;\n SystemInfo const& sys_info;\n size_t name_field_width = 0;\n static const char* executable_name;\n Context();\n };\n\n struct BENCHMARK_EXPORT Run {\n static const int64_t no_repetition_index = -1;\n enum RunType { RT_Iteration, RT_Aggregate };\n\n Run()\n : run_type(RT_Iteration),\n aggregate_unit(kTime),\n skipped(internal::NotSkipped),\n iterations(1),\n threads(1),\n time_unit(kNanosecond),\n real_accumulated_time(0),\n cpu_accumulated_time(0),\n max_heapbytes_used(0),\n use_real_time_for_initial_big_o(false),\n complexity(oNone),\n complexity_lambda(),\n complexity_n(0),\n statistics(),\n report_big_o(false),\n report_rms(false),\n allocs_per_iter(0.0) {}\n\n std::string benchmark_name() const;\n BenchmarkName run_name;\n int64_t family_index;\n int64_t per_family_instance_index;\n RunType run_type;\n std::string aggregate_name;\n StatisticUnit aggregate_unit;\n std::string report_label;\n internal::Skipped skipped;\n std::string skip_message;\n\n IterationCount iterations;\n int64_t threads;\n int64_t repetition_index;\n int64_t repetitions;\n TimeUnit time_unit;\n double real_accumulated_time;\n double cpu_accumulated_time;\n\n double GetAdjustedRealTime() const;\n double GetAdjustedCPUTime() const;\n\n double max_heapbytes_used;\n bool use_real_time_for_initial_big_o;\n BigO complexity;\n BigOFunc* complexity_lambda;\n ComplexityN complexity_n;\n const std::vector* statistics;\n bool report_big_o;\n bool report_rms;\n UserCounters counters;\n MemoryManager::Result memory_result;\n double allocs_per_iter;\n };\n\n struct PerFamilyRunReports {\n PerFamilyRunReports() : num_runs_total(0), num_runs_done(0) {}\n int num_runs_total;\n int num_runs_done;\n std::vector Runs;\n };\n\n BenchmarkReporter();\n virtual bool ReportContext(const Context& context) = 0;\n virtual void ReportRunsConfig(double /*min_time*/,\n bool /*has_explicit_iters*/,\n IterationCount /*iters*/) {}\n virtual void ReportRuns(const std::vector& report) = 0;\n virtual void Finalize() {}\n\n void SetOutputStream(std::ostream* out) {\n assert(out);\n output_stream_ = out;\n }\n void SetErrorStream(std::ostream* err) {\n assert(err);\n error_stream_ = err;\n }\n std::ostream& GetOutputStream() const { return *output_stream_; }\n std::ostream& GetErrorStream() const { return *error_stream_; }\n virtual ~BenchmarkReporter();\n static void PrintBasicContext(std::ostream* out, Context const& context);\n\n private:\n std::ostream* output_stream_;\n std::ostream* error_stream_;\n};\n\nclass BENCHMARK_EXPORT ConsoleReporter : public BenchmarkReporter {\n public:\n enum OutputOptions {\n OO_None = 0,\n OO_Color = 1,\n OO_Tabular = 2,\n OO_ColorTabular = OO_Color | OO_Tabular,\n OO_Defaults = OO_ColorTabular\n };\n explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults)\n : output_options_(opts_), name_field_width_(0), printed_header_(false) {}\n\n bool ReportContext(const Context& context) override;\n void ReportRuns(const std::vector& reports) override;\n\n protected:\n virtual void PrintRunData(const Run& result);\n virtual void PrintHeader(const Run& run);\n\n OutputOptions output_options_;\n size_t name_field_width_;\n UserCounters prev_counters_;\n bool printed_header_;\n};\n\nclass BENCHMARK_EXPORT JSONReporter : public BenchmarkReporter {\n public:\n JSONReporter() : first_report_(true) {}\n bool ReportContext(const Context& context) override;\n void ReportRuns(const std::vector& reports) override;\n void Finalize() override;\n\n private:\n void PrintRunData(const Run& run);\n bool first_report_;\n};\n\nclass BENCHMARK_EXPORT BENCHMARK_DEPRECATED_MSG(\n \"The CSV Reporter will be removed in a future release\") CSVReporter\n : public BenchmarkReporter {\n public:\n CSVReporter() : printed_header_(false) {}\n bool ReportContext(const Context& context) override;\n void ReportRuns(const std::vector& reports) override;\n\n private:\n void PrintRunData(const Run& run);\n bool printed_header_;\n std::set user_counter_names_;\n};\n\ninline const char* GetTimeUnitString(TimeUnit unit) {\n switch (unit) {\n case kSecond:\n return \"s\";\n case kMillisecond:\n return \"ms\";\n case kMicrosecond:\n return \"us\";\n case kNanosecond:\n return \"ns\";\n }\n BENCHMARK_UNREACHABLE();\n}\n\ninline double GetTimeUnitMultiplier(TimeUnit unit) {\n switch (unit) {\n case kSecond:\n return 1;\n case kMillisecond:\n return 1e3;\n case kMicrosecond:\n return 1e6;\n case kNanosecond:\n return 1e9;\n }\n BENCHMARK_UNREACHABLE();\n}\n\n} // namespace benchmark\n\n#if defined(_MSC_VER)\n#pragma warning(pop)\n#endif\n\n#endif // BENCHMARK_REPORTER_H_\n" }, { "path": "include/benchmark/state.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_STATE_H_\n#define BENCHMARK_STATE_H_\n\n#if defined(_MSC_VER)\n#pragma warning(push)\n#pragma warning(disable : 4251 4324)\n#endif\n\n#include \n#include \n#include \n\n#include \"benchmark/counter.h\"\n#include \"benchmark/macros.h\"\n#include \"benchmark/statistics.h\"\n#include \"benchmark/types.h\"\n\nnamespace benchmark {\n\nnamespace internal {\nclass BenchmarkInstance;\nclass ThreadTimer;\nclass ThreadManager;\nclass PerfCountersMeasurement;\n} // namespace internal\n\nclass ProfilerManager;\n\nclass BENCHMARK_EXPORT BENCHMARK_INTERNAL_CACHELINE_ALIGNED State {\n public:\n struct StateIterator;\n friend struct StateIterator;\n\n inline BENCHMARK_ALWAYS_INLINE StateIterator begin();\n inline BENCHMARK_ALWAYS_INLINE StateIterator end();\n\n inline bool KeepRunning();\n\n inline bool KeepRunningBatch(IterationCount n);\n\n void PauseTiming();\n\n void ResumeTiming();\n\n void SkipWithMessage(const std::string& msg);\n\n void SkipWithError(const std::string& msg);\n\n bool skipped() const { return internal::NotSkipped != skipped_; }\n\n bool error_occurred() const { return internal::SkippedWithError == skipped_; }\n\n void SetIterationTime(double seconds);\n\n BENCHMARK_ALWAYS_INLINE\n void SetBytesProcessed(int64_t bytes) {\n counters[\"bytes_per_second\"] =\n Counter(static_cast(bytes), Counter::kIsRate, Counter::kIs1024);\n }\n\n BENCHMARK_ALWAYS_INLINE\n int64_t bytes_processed() const {\n if (counters.find(\"bytes_per_second\") != counters.end())\n return static_cast(counters.at(\"bytes_per_second\"));\n return 0;\n }\n\n BENCHMARK_ALWAYS_INLINE\n void SetComplexityN(ComplexityN complexity_n) {\n complexity_n_ = complexity_n;\n }\n\n BENCHMARK_ALWAYS_INLINE\n ComplexityN complexity_length_n() const { return complexity_n_; }\n\n BENCHMARK_ALWAYS_INLINE\n void SetItemsProcessed(int64_t items) {\n counters[\"items_per_second\"] =\n Counter(static_cast(items), benchmark::Counter::kIsRate);\n }\n\n BENCHMARK_ALWAYS_INLINE\n int64_t items_processed() const {\n if (counters.find(\"items_per_second\") != counters.end())\n return static_cast(counters.at(\"items_per_second\"));\n return 0;\n }\n\n void SetLabel(const std::string& label);\n\n BENCHMARK_ALWAYS_INLINE\n int64_t range(std::size_t pos = 0) const {\n assert(range_.size() > pos);\n return range_[pos];\n }\n\n BENCHMARK_DEPRECATED_MSG(\"use 'range(0)' instead\")\n int64_t range_x() const { return range(0); }\n\n BENCHMARK_DEPRECATED_MSG(\"use 'range(1)' instead\")\n int64_t range_y() const { return range(1); }\n\n BENCHMARK_ALWAYS_INLINE\n int threads() const { return threads_; }\n\n BENCHMARK_ALWAYS_INLINE\n int thread_index() const { return thread_index_; }\n\n BENCHMARK_ALWAYS_INLINE\n IterationCount iterations() const {\n if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) {\n return 0;\n }\n return max_iterations - total_iterations_ + batch_leftover_;\n }\n\n BENCHMARK_ALWAYS_INLINE\n std::string name() const { return name_; }\n\n size_t range_size() const { return range_.size(); }\n\n private:\n IterationCount total_iterations_;\n\n IterationCount batch_leftover_;\n\n public:\n const IterationCount max_iterations;\n\n private:\n bool started_;\n bool finished_;\n internal::Skipped skipped_;\n\n std::vector range_;\n\n ComplexityN complexity_n_;\n\n public:\n UserCounters counters;\n\n private:\n State(std::string name, IterationCount max_iters,\n const std::vector& ranges, int thread_i, int n_threads,\n internal::ThreadTimer* timer, internal::ThreadManager* manager,\n internal::PerfCountersMeasurement* perf_counters_measurement,\n ProfilerManager* profiler_manager);\n\n void StartKeepRunning();\n inline bool KeepRunningInternal(IterationCount n, bool is_batch);\n void FinishKeepRunning();\n\n const std::string name_;\n const int thread_index_;\n const int threads_;\n\n internal::ThreadTimer* const timer_;\n internal::ThreadManager* const manager_;\n internal::PerfCountersMeasurement* const perf_counters_measurement_;\n ProfilerManager* const profiler_manager_;\n\n friend class internal::BenchmarkInstance;\n};\n\ninline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() {\n return KeepRunningInternal(1, /*is_batch=*/false);\n}\n\ninline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(IterationCount n) {\n return KeepRunningInternal(n, /*is_batch=*/true);\n}\n\ninline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n,\n bool is_batch) {\n assert(n > 0);\n assert(is_batch || n == 1);\n if (BENCHMARK_BUILTIN_EXPECT(total_iterations_ >= n, true)) {\n total_iterations_ -= n;\n return true;\n }\n if (!started_) {\n StartKeepRunning();\n if (!skipped() && total_iterations_ >= n) {\n total_iterations_ -= n;\n return true;\n }\n }\n if (is_batch && total_iterations_ != 0) {\n batch_leftover_ = n - total_iterations_;\n total_iterations_ = 0;\n return true;\n }\n FinishKeepRunning();\n return false;\n}\n\nstruct State::StateIterator {\n struct BENCHMARK_UNUSED Value {};\n typedef std::forward_iterator_tag iterator_category;\n typedef Value value_type;\n typedef Value reference;\n typedef Value pointer;\n typedef std::ptrdiff_t difference_type;\n\n private:\n friend class State;\n BENCHMARK_ALWAYS_INLINE\n StateIterator() : cached_(0), parent_() {}\n\n BENCHMARK_ALWAYS_INLINE\n explicit StateIterator(State* st)\n : cached_(st->skipped() ? 0 : st->max_iterations), parent_(st) {}\n\n public:\n BENCHMARK_ALWAYS_INLINE\n Value operator*() const { return Value(); }\n\n BENCHMARK_ALWAYS_INLINE\n StateIterator& operator++() {\n assert(cached_ > 0);\n --cached_;\n return *this;\n }\n\n BENCHMARK_ALWAYS_INLINE\n bool operator!=(StateIterator const&) const {\n if (BENCHMARK_BUILTIN_EXPECT(cached_ != 0, true)) return true;\n parent_->FinishKeepRunning();\n return false;\n }\n\n private:\n IterationCount cached_;\n State* const parent_;\n};\n\ninline BENCHMARK_ALWAYS_INLINE State::StateIterator State::begin() {\n return StateIterator(this);\n}\ninline BENCHMARK_ALWAYS_INLINE State::StateIterator State::end() {\n StartKeepRunning();\n return StateIterator();\n}\n\n} // namespace benchmark\n\n#if defined(_MSC_VER)\n#pragma warning(pop)\n#endif\n\n#endif // BENCHMARK_STATE_H_\n" }, { "path": "include/benchmark/statistics.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_STATISTICS_H_\n#define BENCHMARK_STATISTICS_H_\n\n#include \n#include \n\n#include \"benchmark/types.h\"\n\nnamespace benchmark {\n\nenum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda };\n\ntypedef int64_t ComplexityN;\n\nenum StatisticUnit { kTime, kPercentage };\n\ntypedef double(BigOFunc)(ComplexityN);\n\ntypedef double(StatisticsFunc)(const std::vector&);\n\nnamespace internal {\nstruct Statistics {\n std::string name_;\n StatisticsFunc* compute_;\n StatisticUnit unit_;\n\n Statistics(const std::string& name, StatisticsFunc* compute,\n StatisticUnit unit = kTime)\n : name_(name), compute_(compute), unit_(unit) {}\n};\n\nenum AggregationReportMode : unsigned {\n ARM_Unspecified = 0,\n ARM_Default = 1U << 0U,\n ARM_FileReportAggregatesOnly = 1U << 1U,\n ARM_DisplayReportAggregatesOnly = 1U << 2U,\n ARM_ReportAggregatesOnly =\n ARM_FileReportAggregatesOnly | ARM_DisplayReportAggregatesOnly\n};\n\nenum Skipped : unsigned {\n NotSkipped = 0,\n SkippedWithMessage,\n SkippedWithError\n};\n\n} // namespace internal\n\n} // namespace benchmark\n\n#endif // BENCHMARK_STATISTICS_H_\n" }, { "path": "include/benchmark/sysinfo.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_SYSINFO_H_\n#define BENCHMARK_SYSINFO_H_\n\n#if defined(_MSC_VER)\n#pragma warning(push)\n#pragma warning(disable : 4251)\n#endif\n\n#include \n#include \n\n#include \"benchmark/macros.h\"\n\nnamespace benchmark {\n\nstruct BENCHMARK_EXPORT CPUInfo {\n struct CacheInfo {\n std::string type;\n int level;\n int size;\n int num_sharing;\n };\n\n enum Scaling { UNKNOWN, ENABLED, DISABLED };\n\n int num_cpus;\n Scaling scaling;\n double cycles_per_second;\n std::vector caches;\n std::vector load_avg;\n\n static const CPUInfo& Get();\n\n private:\n CPUInfo();\n BENCHMARK_DISALLOW_COPY_AND_ASSIGN(CPUInfo);\n};\n\nstruct BENCHMARK_EXPORT SystemInfo {\n enum class ASLR { UNKNOWN, ENABLED, DISABLED };\n\n std::string name;\n ASLR ASLRStatus;\n static const SystemInfo& Get();\n\n private:\n SystemInfo();\n BENCHMARK_DISALLOW_COPY_AND_ASSIGN(SystemInfo);\n};\n\n} // namespace benchmark\n\n#if defined(_MSC_VER)\n#pragma warning(pop)\n#endif\n\n#endif // BENCHMARK_SYSINFO_H_\n" }, { "path": "include/benchmark/types.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_TYPES_H_\n#define BENCHMARK_TYPES_H_\n\n#include \n\n#include \n#include \n#include \n\n#include \"benchmark/export.h\"\n\nnamespace benchmark {\n\nnamespace internal {\n#if (__cplusplus < 201402L || (defined(_MSC_VER) && _MSVC_LANG < 201402L))\ntemplate \nstd::unique_ptr make_unique(Args&&... args) {\n return std::unique_ptr(new T(std::forward(args)...));\n}\n#else\nusing ::std::make_unique;\n#endif\n} // namespace internal\n\nclass BenchmarkReporter;\nclass State;\n\nusing IterationCount = int64_t;\n\nusing callback_function = std::function;\n\nenum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond };\n\n} // namespace benchmark\n\n#endif // BENCHMARK_TYPES_H_\n" }, { "path": "include/benchmark/utils.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_UTILS_H_\n#define BENCHMARK_UTILS_H_\n\n#include \n#include \n#include \n\n#include \"benchmark/macros.h\"\n\nnamespace benchmark {\n\nnamespace internal {\nBENCHMARK_EXPORT void UseCharPointer(char const volatile*);\n}\n\n#if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \\\n defined(__EMSCRIPTEN__)\n#define BENCHMARK_HAS_NO_INLINE_ASSEMBLY\n#endif\n\ninline BENCHMARK_ALWAYS_INLINE void ClobberMemory() {\n std::atomic_signal_fence(std::memory_order_acq_rel);\n}\n\n#ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY\n#if !defined(__GNUC__) || defined(__llvm__) || defined(__INTEL_COMPILER)\ntemplate \nBENCHMARK_DEPRECATED_MSG(\n \"The const-ref version of this method can permit \"\n \"undesired compiler optimizations in benchmarks\")\ninline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) {\n asm volatile(\"\" : : \"r,m\"(value) : \"memory\");\n}\n\ntemplate \ninline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) {\n#if defined(__clang__)\n asm volatile(\"\" : \"+r,m\"(value) : : \"memory\");\n#else\n asm volatile(\"\" : \"+m,r\"(value) : : \"memory\");\n#endif\n}\n\ntemplate \ninline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) {\n#if defined(__clang__)\n asm volatile(\"\" : \"+r,m\"(value) : : \"memory\");\n#else\n asm volatile(\"\" : \"+m,r\"(value) : : \"memory\");\n#endif\n}\n#elif (__GNUC__ >= 5)\ntemplate \nBENCHMARK_DEPRECATED_MSG(\n \"The const-ref version of this method can permit \"\n \"undesired compiler optimizations in benchmarks\")\ninline BENCHMARK_ALWAYS_INLINE\n typename std::enable_if::value &&\n (sizeof(Tp) <= sizeof(Tp*))>::type\n DoNotOptimize(Tp const& value) {\n asm volatile(\"\" : : \"r,m\"(value) : \"memory\");\n}\n\ntemplate \nBENCHMARK_DEPRECATED_MSG(\n \"The const-ref version of this method can permit \"\n \"undesired compiler optimizations in benchmarks\")\ninline BENCHMARK_ALWAYS_INLINE\n typename std::enable_if::value ||\n (sizeof(Tp) > sizeof(Tp*))>::type\n DoNotOptimize(Tp const& value) {\n asm volatile(\"\" : : \"m\"(value) : \"memory\");\n}\n\ntemplate \ninline BENCHMARK_ALWAYS_INLINE\n typename std::enable_if::value &&\n (sizeof(Tp) <= sizeof(Tp*))>::type\n DoNotOptimize(Tp& value) {\n asm volatile(\"\" : \"+m,r\"(value) : : \"memory\");\n}\n\ntemplate \ninline BENCHMARK_ALWAYS_INLINE\n typename std::enable_if::value ||\n (sizeof(Tp) > sizeof(Tp*))>::type\n DoNotOptimize(Tp& value) {\n asm volatile(\"\" : \"+m\"(value) : : \"memory\");\n}\n\ntemplate \ninline BENCHMARK_ALWAYS_INLINE\n typename std::enable_if::value &&\n (sizeof(Tp) <= sizeof(Tp*))>::type\n DoNotOptimize(Tp&& value) {\n asm volatile(\"\" : \"+m,r\"(value) : : \"memory\");\n}\n\ntemplate \ninline BENCHMARK_ALWAYS_INLINE\n typename std::enable_if::value ||\n (sizeof(Tp) > sizeof(Tp*))>::type\n DoNotOptimize(Tp&& value) {\n asm volatile(\"\" : \"+m\"(value) : : \"memory\");\n}\n#endif\n\n#elif defined(_MSC_VER)\ntemplate \nBENCHMARK_DEPRECATED_MSG(\n \"The const-ref version of this method can permit \"\n \"undesired compiler optimizations in benchmarks\")\ninline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) {\n internal::UseCharPointer(&reinterpret_cast(value));\n _ReadWriteBarrier();\n}\n\ntemplate \ninline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) {\n internal::UseCharPointer(&reinterpret_cast(value));\n _ReadWriteBarrier();\n}\n\ntemplate \ninline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) {\n internal::UseCharPointer(&reinterpret_cast(value));\n _ReadWriteBarrier();\n}\n#else\ntemplate \ninline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) {\n internal::UseCharPointer(&reinterpret_cast(value));\n}\n#endif\n\n} // end namespace benchmark\n\n#endif // BENCHMARK_UTILS_H_\n" }, { "path": "pyproject.toml", "content": "[build-system]\nrequires = [\"setuptools\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = \"google_benchmark\"\ndescription = \"A library to benchmark code snippets.\"\nrequires-python = \">=3.10\"\nlicense = \"Apache-2.0\"\nkeywords = [\"benchmark\"]\n\nauthors = [{ name = \"Google\", email = \"benchmark-discuss@googlegroups.com\" }]\n\nclassifiers = [\n \"Development Status :: 4 - Beta\",\n \"Intended Audience :: Developers\",\n \"Intended Audience :: Science/Research\",\n \"Programming Language :: Python :: 3.10\",\n \"Programming Language :: Python :: 3.11\",\n \"Programming Language :: Python :: 3.12\",\n \"Programming Language :: Python :: 3.13\",\n \"Topic :: Software Development :: Testing\",\n \"Topic :: System :: Benchmark\",\n]\n\ndynamic = [\"readme\", \"version\"]\n\n[dependency-groups]\ndev = [\"pre-commit>=3.3.3\"]\n\n[project.urls]\nHomepage = \"https://github.com/google/benchmark\"\nDocumentation = \"https://github.com/google/benchmark/tree/main/docs\"\nRepository = \"https://github.com/google/benchmark.git\"\nDiscord = \"https://discord.gg/cz7UX7wKC2\"\n\n[tool.setuptools]\npackage-dir = { \"\" = \"bindings/python\" }\nzip-safe = false\n\n[tool.setuptools.packages.find]\nwhere = [\"bindings/python\"]\n\n[tool.setuptools.dynamic]\nreadme = { file = \"README.md\", content-type = \"text/markdown\" }\nversion = { attr = \"google_benchmark.__version__\" }\n\n[tool.mypy]\ncheck_untyped_defs = true\ndisallow_incomplete_defs = true\npretty = true\npython_version = \"3.11\"\nstrict_optional = false\nwarn_unreachable = true\n\n[[tool.mypy.overrides]]\nmodule = [\"yaml\"]\nignore_missing_imports = true\n\n[tool.ruff]\n# explicitly tell ruff the source directory to correctly identify first-party package.\nsrc = [\"bindings/python\"]\n\nline-length = 80\ntarget-version = \"py311\"\n\n[tool.ruff.lint]\n# Enable pycodestyle (`E`, `W`), Pyflakes (`F`), and isort (`I`) codes by default.\nselect = [\"ASYNC\", \"B\", \"C4\", \"C90\", \"E\", \"F\", \"I\", \"PERF\", \"PIE\", \"PT018\", \"RUF\", \"SIM\", \"UP\", \"W\"]\nignore = [\n \"PLW2901\", # redefined-loop-name\n \"UP031\", # printf-string-formatting\n]\n\n[tool.ruff.lint.isort]\ncombine-as-imports = true\n" }, { "path": "setup.py", "content": "import contextlib\nimport os\nimport platform\nimport re\nimport shutil\nimport sys\nfrom collections.abc import Generator\nfrom pathlib import Path\nfrom typing import Any\n\nimport setuptools\nfrom setuptools.command import build_ext\n\nIS_WINDOWS = platform.system() == \"Windows\"\nIS_MAC = platform.system() == \"Darwin\"\nIS_LINUX = platform.system() == \"Linux\"\n\n# hardcoded SABI-related options. Requires that each Python interpreter\n# (hermetic or not) participating is of the same major-minor version.\npy_limited_api = sys.version_info >= (3, 12)\noptions = {\"bdist_wheel\": {\"py_limited_api\": \"cp312\"}} if py_limited_api else {}\n\n\ndef is_cibuildwheel() -> bool:\n return os.getenv(\"CIBUILDWHEEL\") is not None\n\n\n@contextlib.contextmanager\ndef _maybe_patch_toolchains() -> Generator[None, None, None]:\n \"\"\"\n Patch rules_python toolchains to ignore root user error\n when run in a Docker container on Linux in cibuildwheel.\n \"\"\"\n\n def fmt_toolchain_args(matchobj):\n suffix = \"ignore_root_user_error = True\"\n callargs = matchobj.group(1)\n # toolchain def is broken over multiple lines\n if callargs.endswith(\"\\n\"):\n callargs = callargs + \" \" + suffix + \",\\n\"\n # toolchain def is on one line.\n else:\n callargs = callargs + \", \" + suffix\n return \"python.toolchain(\" + callargs + \")\"\n\n CIBW_LINUX = is_cibuildwheel() and IS_LINUX\n module_bazel = Path(\"MODULE.bazel\")\n content: str = module_bazel.read_text()\n try:\n if CIBW_LINUX:\n module_bazel.write_text(\n re.sub(\n r\"python.toolchain\\(([\\w\\\"\\s,.=]*)\\)\",\n fmt_toolchain_args,\n content,\n )\n )\n yield\n finally:\n if CIBW_LINUX:\n module_bazel.write_text(content)\n\n\nclass BazelExtension(setuptools.Extension):\n \"\"\"A C/C++ extension that is defined as a Bazel BUILD target.\"\"\"\n\n def __init__(self, name: str, bazel_target: str, **kwargs: Any):\n super().__init__(name=name, sources=[], **kwargs)\n\n self.bazel_target = bazel_target\n stripped_target = bazel_target.split(\"//\")[-1]\n self.relpath, self.target_name = stripped_target.split(\":\")\n\n\nclass BuildBazelExtension(build_ext.build_ext):\n \"\"\"A command that runs Bazel to build a C/C++ extension.\"\"\"\n\n def run(self):\n for ext in self.extensions:\n self.bazel_build(ext)\n # explicitly call `bazel shutdown` for graceful exit\n self.spawn([\"bazel\", \"shutdown\"])\n\n def copy_extensions_to_source(self):\n \"\"\"\n Copy generated extensions into the source tree.\n This is done in the ``bazel_build`` method, so it's not necessary to\n do again in the `build_ext` base class.\n \"\"\"\n\n def bazel_build(self, ext: BazelExtension) -> None: # noqa: C901\n \"\"\"Runs the bazel build to create the package.\"\"\"\n temp_path = Path(self.build_temp)\n\n # We round to the minor version, which makes rules_python\n # look up the latest available patch version internally.\n python_version = \"{}.{}\".format(*sys.version_info[:2])\n\n bazel_argv = [\n \"bazel\",\n \"run\",\n ext.bazel_target,\n f\"--symlink_prefix={temp_path / 'bazel-'}\",\n f\"--compilation_mode={'dbg' if self.debug else 'opt'}\",\n # C++17 is required by nanobind\n f\"--cxxopt={'/std:c++17' if IS_WINDOWS else '-std=c++17'}\",\n f\"--@rules_python//python/config_settings:python_version={python_version}\",\n ]\n\n if ext.py_limited_api:\n bazel_argv += [\"--@nanobind_bazel//:py-limited-api=cp312\"]\n\n if IS_WINDOWS:\n # Link with python*.lib.\n for library_dir in self.library_dirs:\n bazel_argv.append(\"--linkopt=/LIBPATH:\" + library_dir)\n elif IS_MAC:\n # C++17 needs macOS 10.14 at minimum\n bazel_argv.append(\"--macos_minimum_os=10.14\")\n\n with _maybe_patch_toolchains():\n self.spawn(bazel_argv)\n\n if IS_WINDOWS:\n suffix = \".pyd\"\n else:\n suffix = \".abi3.so\" if ext.py_limited_api else \".so\"\n\n # copy the Bazel build artifacts into setuptools' libdir,\n # from where the wheel is built.\n pkgname = \"google_benchmark\"\n pythonroot = Path(\"bindings\") / \"python\" / \"google_benchmark\"\n srcdir = temp_path / \"bazel-bin\" / pythonroot\n if not self.inplace:\n libdir = Path(self.build_lib) / pkgname\n else:\n build_py = self.get_finalized_command(\"build_py\")\n libdir = Path(build_py.get_package_dir(pkgname))\n\n for root, dirs, files in os.walk(srcdir, topdown=True):\n # exclude runfiles directories and children.\n dirs[:] = [d for d in dirs if \"runfiles\" not in d]\n\n for f in files:\n fp = Path(f)\n should_copy = False\n # we do not want the bare .so file included\n # when building for ABI3, so we require a\n # full and exact match on the file extension.\n if \"\".join(fp.suffixes) == suffix or fp.suffix == \".pyi\":\n should_copy = True\n elif Path(root) == srcdir and f == \"py.typed\":\n # copy py.typed, but only at the package root.\n should_copy = True\n\n if should_copy:\n shutil.copyfile(root / fp, libdir / fp)\n\n\nsetuptools.setup(\n cmdclass={\"build_ext\": BuildBazelExtension},\n package_data={\"google_benchmark\": [\"py.typed\", \"*.pyi\"]},\n ext_modules=[\n BazelExtension(\n name=\"google_benchmark._benchmark\",\n bazel_target=\"//bindings/python/google_benchmark:benchmark_stubgen\",\n py_limited_api=py_limited_api,\n )\n ],\n options=options,\n)\n" }, { "path": "src/CMakeLists.txt", "content": "#Allow the source files to find headers in src /\ninclude(GNUInstallDirs)\ninclude_directories(${PROJECT_SOURCE_DIR}/src)\n\nif (DEFINED BENCHMARK_CXX_LINKER_FLAGS)\n list(APPEND CMAKE_SHARED_LINKER_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS})\n list(APPEND CMAKE_MODULE_LINKER_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS})\nendif()\n\nfile(GLOB\n SOURCE_FILES\n *.cc\n ${PROJECT_SOURCE_DIR}/include/benchmark/*.h\n ${CMAKE_CURRENT_SOURCE_DIR}/*.h)\nfile(GLOB BENCHMARK_MAIN \"benchmark_main.cc\")\nforeach(item ${BENCHMARK_MAIN})\n list(REMOVE_ITEM SOURCE_FILES \"${item}\")\nendforeach()\n\nadd_library(benchmark ${SOURCE_FILES})\nadd_library(benchmark::benchmark ALIAS benchmark)\nset_target_properties(benchmark PROPERTIES\n OUTPUT_NAME \"benchmark\"\n VERSION ${GENERIC_LIB_VERSION}\n SOVERSION ${GENERIC_LIB_SOVERSION}\n)\ntarget_include_directories(benchmark PUBLIC\n $\n)\n\nset_property(\n SOURCE benchmark.cc\n APPEND\n PROPERTY COMPILE_DEFINITIONS\n BENCHMARK_VERSION=\"${VERSION}\"\n)\n\n# libpfm, if available\nif (PFM_FOUND)\n target_link_libraries(benchmark PRIVATE PFM::libpfm)\n target_compile_definitions(benchmark PRIVATE -DHAVE_LIBPFM)\n install(\n FILES \"${PROJECT_SOURCE_DIR}/cmake/Modules/FindPFM.cmake\"\n DESTINATION \"${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}\")\nendif()\n\n# pthread affinity, if available\nif(HAVE_PTHREAD_AFFINITY)\n target_compile_definitions(benchmark PRIVATE -DBENCHMARK_HAS_PTHREAD_AFFINITY)\nendif()\n\ntarget_link_libraries(benchmark PRIVATE ${BENCHMARK_CXX_LIBRARIES})\n\nif(HAVE_LIB_RT)\n target_link_libraries(benchmark PRIVATE rt)\nendif(HAVE_LIB_RT)\n\n\n# We need extra libraries on Windows\nif(${CMAKE_SYSTEM_NAME} MATCHES \"Windows\")\n target_link_libraries(benchmark PRIVATE shlwapi)\nendif()\n\n# We need extra libraries on Solaris\nif(${CMAKE_SYSTEM_NAME} MATCHES \"SunOS\")\n target_link_libraries(benchmark PRIVATE kstat)\nendif()\n\nif (NOT BUILD_SHARED_LIBS)\n target_compile_definitions(benchmark PUBLIC -DBENCHMARK_STATIC_DEFINE)\nendif()\n\n# Benchmark main library\nadd_library(benchmark_main \"benchmark_main.cc\")\nadd_library(benchmark::benchmark_main ALIAS benchmark_main)\nset_target_properties(benchmark_main PROPERTIES\n OUTPUT_NAME \"benchmark_main\"\n VERSION ${GENERIC_LIB_VERSION}\n SOVERSION ${GENERIC_LIB_SOVERSION}\n DEFINE_SYMBOL benchmark_EXPORTS\n)\ntarget_link_libraries(benchmark_main PUBLIC benchmark::benchmark)\n\nset(generated_dir \"${PROJECT_BINARY_DIR}\")\n\nset(version_config \"${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake\")\nset(project_config \"${generated_dir}/${PROJECT_NAME}Config.cmake\")\nset(pkg_config \"${generated_dir}/${PROJECT_NAME}.pc\")\nset(pkg_config_main \"${generated_dir}/${PROJECT_NAME}_main.pc\")\nset(targets_to_export benchmark benchmark_main)\nset(targets_export_name \"${PROJECT_NAME}Targets\")\n\nset(namespace \"${PROJECT_NAME}::\")\n\ninclude(CMakePackageConfigHelpers)\n\nconfigure_package_config_file (\n ${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in\n ${project_config}\n INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}\n NO_SET_AND_CHECK_MACRO\n NO_CHECK_REQUIRED_COMPONENTS_MACRO\n)\nwrite_basic_package_version_file(\n \"${version_config}\" VERSION ${GENERIC_LIB_VERSION} COMPATIBILITY SameMajorVersion\n)\n\n# Derive private link libraries from target\nif(NOT BUILD_SHARED_LIBS)\n get_target_property(LINK_LIBS benchmark LINK_LIBRARIES)\n if(LINK_LIBS)\n set(BENCHMARK_PRIVATE_LINK_LIBRARIES \"\")\n foreach(LIB IN LISTS LINK_LIBS)\n if(NOT TARGET \"${LIB}\" AND LIB MATCHES \"^[a-zA-Z0-9_.-]+$\")\n list(APPEND BENCHMARK_PRIVATE_LINK_LIBRARIES \"-l${LIB}\")\n endif()\n endforeach()\n string(JOIN \" \" BENCHMARK_PRIVATE_LINK_LIBRARIES ${BENCHMARK_PRIVATE_LINK_LIBRARIES})\n endif()\nendif()\n\nconfigure_file(\"${PROJECT_SOURCE_DIR}/cmake/benchmark.pc.in\" \"${pkg_config}\" @ONLY)\nconfigure_file(\"${PROJECT_SOURCE_DIR}/cmake/benchmark_main.pc.in\" \"${pkg_config_main}\" @ONLY)\n\nexport (\n TARGETS ${targets_to_export}\n NAMESPACE \"${namespace}\"\n FILE ${generated_dir}/${targets_export_name}.cmake\n)\n\nif (BENCHMARK_ENABLE_INSTALL)\n # Install target (will install the library to specified CMAKE_INSTALL_PREFIX variable)\n install(\n TARGETS ${targets_to_export}\n EXPORT ${targets_export_name}\n ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}\n LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}\n RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}\n INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})\n\n install(\n DIRECTORY \"${PROJECT_SOURCE_DIR}/include/benchmark\"\n \"${PROJECT_BINARY_DIR}/include/benchmark\"\n DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}\n FILES_MATCHING PATTERN \"*.*h\")\n\n install(\n FILES \"${project_config}\" \"${version_config}\"\n DESTINATION \"${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}\")\n\n install(\n FILES \"${pkg_config}\" \"${pkg_config_main}\"\n DESTINATION \"${CMAKE_INSTALL_LIBDIR}/pkgconfig\")\n\n install(\n EXPORT \"${targets_export_name}\"\n NAMESPACE \"${namespace}\"\n DESTINATION \"${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}\")\nendif()\n\nif (BENCHMARK_ENABLE_DOXYGEN)\n find_package(Doxygen REQUIRED)\n set(DOXYGEN_QUIET YES)\n set(DOXYGEN_RECURSIVE YES)\n set(DOXYGEN_GENERATE_HTML YES)\n set(DOXYGEN_GENERATE_MAN NO)\n set(DOXYGEN_MARKDOWN_SUPPORT YES)\n set(DOXYGEN_BUILTIN_STL_SUPPORT YES)\n set(DOXYGEN_EXTRACT_PACKAGE YES)\n set(DOXYGEN_EXTRACT_STATIC YES)\n set(DOXYGEN_SHOW_INCLUDE_FILES YES)\n set(DOXYGEN_BINARY_TOC YES)\n set(DOXYGEN_TOC_EXPAND YES)\n set(DOXYGEN_USE_MDFILE_AS_MAINPAGE \"index.md\")\n doxygen_add_docs(benchmark_doxygen\n docs\n include\n src\n ALL\n WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}\n COMMENT \"Building documentation with Doxygen.\")\n if (BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_DOCS)\n install(\n DIRECTORY \"${CMAKE_CURRENT_BINARY_DIR}/html/\"\n DESTINATION ${CMAKE_INSTALL_DOCDIR})\n endif()\nelse()\n if (BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_DOCS)\n install(\n DIRECTORY \"${PROJECT_SOURCE_DIR}/docs/\"\n DESTINATION ${CMAKE_INSTALL_DOCDIR})\n endif()\nendif()\n\nset(CMAKE_INSTALL_PYTOOLSDIR \"${CMAKE_INSTALL_DATADIR}/googlebenchmark/tools\" CACHE PATH \"\")\n\nif (BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_TOOLS)\n install(\n DIRECTORY \"${PROJECT_SOURCE_DIR}/tools/\"\n DESTINATION ${CMAKE_INSTALL_PYTOOLSDIR})\nendif()\n" }, { "path": "src/arraysize.h", "content": "#ifndef BENCHMARK_ARRAYSIZE_H_\n#define BENCHMARK_ARRAYSIZE_H_\n\n#include \"internal_macros.h\"\n\nnamespace benchmark {\nnamespace internal {\n// The arraysize(arr) macro returns the # of elements in an array arr.\n// The expression is a compile-time constant, and therefore can be\n// used in defining new arrays, for example. If you use arraysize on\n// a pointer by mistake, you will get a compile-time error.\n//\n\n// This template function declaration is used in defining arraysize.\n// Note that the function doesn't need an implementation, as we only\n// use its type.\ntemplate \nchar (&ArraySizeHelper(T (&array)[N]))[N];\n\n// That gcc wants both of these prototypes seems mysterious. VC, for\n// its part, can't decide which to use (another mystery). Matching of\n// template overloads: the final frontier.\n#ifndef COMPILER_MSVC\ntemplate \nchar (&ArraySizeHelper(const T (&array)[N]))[N];\n#endif\n\n#define arraysize(array) (sizeof(::benchmark::internal::ArraySizeHelper(array)))\n\n} // end namespace internal\n} // end namespace benchmark\n\n#endif // BENCHMARK_ARRAYSIZE_H_\n" }, { "path": "src/benchmark.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/types.h\"\n#include \"benchmark_api_internal.h\"\n#include \"benchmark_runner.h\"\n#include \"internal_macros.h\"\n\n#ifndef BENCHMARK_OS_WINDOWS\n#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \\\n !defined(BENCHMARK_OS_WASI)\n#include \n#endif\n#include \n#include \n#endif\n\n#ifdef BENCHMARK_OS_LINUX\n#include \n#endif\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"check.h\"\n#include \"colorprint.h\"\n#include \"commandlineflags.h\"\n#include \"complexity.h\"\n#include \"counter.h\"\n#include \"log.h\"\n#include \"mutex.h\"\n#include \"perf_counters.h\"\n#include \"re.h\"\n#include \"statistics.h\"\n#include \"string_util.h\"\n#include \"thread_manager.h\"\n#include \"thread_timer.h\"\n\nnamespace benchmark {\n// Print a list of benchmarks. This option overrides all other options.\nBM_DEFINE_bool(benchmark_list_tests, false);\n\n// A regular expression that specifies the set of benchmarks to execute. If\n// this flag is empty, or if this flag is the string \\\"all\\\", all benchmarks\n// linked into the binary are run.\nBM_DEFINE_string(benchmark_filter, \"\");\n\n// Specification of how long to run the benchmark.\n//\n// It can be either an exact number of iterations (specified as `x`),\n// or a minimum number of seconds (specified as `s`). If the latter\n// format (ie., min seconds) is used, the system may run the benchmark longer\n// until the results are considered significant.\n//\n// For backward compatibility, the `s` suffix may be omitted, in which case,\n// the specified number is interpreted as the number of seconds.\n//\n// For cpu-time based tests, this is the lower bound\n// on the total cpu time used by all threads that make up the test. For\n// real-time based tests, this is the lower bound on the elapsed time of the\n// benchmark execution, regardless of number of threads.\nBM_DEFINE_string(benchmark_min_time, kDefaultMinTimeStr);\n\n// Minimum number of seconds a benchmark should be run before results should be\n// taken into account. This e.g can be necessary for benchmarks of code which\n// needs to fill some form of cache before performance is of interest.\n// Note: results gathered within this period are discarded and not used for\n// reported result.\nBM_DEFINE_double(benchmark_min_warmup_time, 0.0);\n\n// The number of runs of each benchmark. If greater than 1, the mean and\n// standard deviation of the runs will be reported.\nBM_DEFINE_int32(benchmark_repetitions, 1);\n\n// If enabled, forces each benchmark to execute exactly one iteration and one\n// repetition, bypassing any configured\n// MinTime()/MinWarmUpTime()/Iterations()/Repetitions()\nBM_DEFINE_bool(benchmark_dry_run, false);\n\n// If set, enable random interleaving of repetitions of all benchmarks.\n// See http://github.com/google/benchmark/issues/1051 for details.\nBM_DEFINE_bool(benchmark_enable_random_interleaving, false);\n\n// Report the result of each benchmark repetitions. When 'true' is specified\n// only the mean, standard deviation, and other statistics are reported for\n// repeated benchmarks. Affects all reporters.\nBM_DEFINE_bool(benchmark_report_aggregates_only, false);\n\n// Display the result of each benchmark repetitions. When 'true' is specified\n// only the mean, standard deviation, and other statistics are displayed for\n// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects\n// the display reporter, but *NOT* file reporter, which will still contain\n// all the output.\nBM_DEFINE_bool(benchmark_display_aggregates_only, false);\n\n// The format to use for console output.\n// Valid values are 'console', 'json', or 'csv'.\nBM_DEFINE_string(benchmark_format, \"console\");\n\n// The format to use for file output.\n// Valid values are 'console', 'json', or 'csv'.\nBM_DEFINE_string(benchmark_out_format, \"json\");\n\n// The file to write additional output to.\nBM_DEFINE_string(benchmark_out, \"\");\n\n// Whether to use colors in the output. Valid values:\n// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if\n// the output is being sent to a terminal and the TERM environment variable is\n// set to a terminal type that supports colors.\nBM_DEFINE_string(benchmark_color, \"auto\");\n\n// Whether to use tabular format when printing user counters to the console.\n// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false.\nBM_DEFINE_bool(benchmark_counters_tabular, false);\n\n// List of additional perf counters to collect, in libpfm format. For more\n// information about libpfm: https://man7.org/linux/man-pages/man3/libpfm.3.html\nBM_DEFINE_string(benchmark_perf_counters, \"\");\n\n// Extra context to include in the output formatted as comma-separated key-value\n// pairs. Kept internal as it's only used for parsing from env/command line.\nBM_DEFINE_kvpairs(benchmark_context, {});\n\n// Set the default time unit to use for reports\n// Valid values are 'ns', 'us', 'ms' or 's'\nBM_DEFINE_string(benchmark_time_unit, \"\");\n\n// The level of verbose logging to output\nBM_DEFINE_int32(v, 0);\n\nnamespace internal {\n\n// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)\nstd::map* global_context = nullptr;\n\nBENCHMARK_EXPORT std::map*& GetGlobalContext() {\n return global_context;\n}\n\nnamespace {\n// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)\nvoid const volatile* volatile global_force_escape_pointer;\n} // namespace\n\n// FIXME: Verify if LTO still messes this up?\nvoid UseCharPointer(char const volatile* const v) {\n // We want to escape the pointer `v` so that the compiler can not eliminate\n // computations that produced it. To do that, we escape the pointer by storing\n // it into a volatile variable, since generally, volatile store, is not\n // something the compiler is allowed to elide.\n global_force_escape_pointer = reinterpret_cast(v);\n}\n\n} // namespace internal\n\nState::State(std::string name, IterationCount max_iters,\n const std::vector& ranges, int thread_i, int n_threads,\n internal::ThreadTimer* timer, internal::ThreadManager* manager,\n internal::PerfCountersMeasurement* perf_counters_measurement,\n ProfilerManager* profiler_manager)\n : total_iterations_(0),\n batch_leftover_(0),\n max_iterations(max_iters),\n started_(false),\n finished_(false),\n skipped_(internal::NotSkipped),\n range_(ranges),\n complexity_n_(0),\n name_(std::move(name)),\n thread_index_(thread_i),\n threads_(n_threads),\n timer_(timer),\n manager_(manager),\n perf_counters_measurement_(perf_counters_measurement),\n profiler_manager_(profiler_manager) {\n BM_CHECK(max_iterations != 0) << \"At least one iteration must be run\";\n BM_CHECK_LT(thread_index_, threads_)\n << \"thread_index must be less than threads\";\n\n // Add counters with correct flag now. If added with `counters[name]` in\n // `PauseTiming`, a new `Counter` will be inserted the first time, which\n // won't have the flag. Inserting them now also reduces the allocations\n // during the benchmark.\n if (perf_counters_measurement_ != nullptr) {\n for (const std::string& counter_name :\n perf_counters_measurement_->names()) {\n counters[counter_name] = Counter(0.0, Counter::kAvgIterations);\n }\n }\n\n // Note: The use of offsetof below is technically undefined until C++17\n // because State is not a standard layout type. However, all compilers\n // currently provide well-defined behavior as an extension (which is\n // demonstrated since constexpr evaluation must diagnose all undefined\n // behavior). However, GCC and Clang also warn about this use of offsetof,\n // which must be suppressed.\n#if defined(__INTEL_COMPILER)\n#pragma warning push\n#pragma warning(disable : 1875)\n#elif defined(__GNUC__) || defined(__clang__)\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Winvalid-offsetof\"\n#endif\n#if defined(__NVCC__)\n#pragma nv_diagnostic push\n#pragma nv_diag_suppress 1427\n#endif\n#if defined(__NVCOMPILER)\n#pragma diagnostic push\n#pragma diag_suppress offset_in_non_POD_nonstandard\n#endif\n // Offset tests to ensure commonly accessed data is on the first cache line.\n const int cache_line_size = 64;\n static_assert(\n offsetof(State, skipped_) <= (cache_line_size - sizeof(skipped_)), \"\");\n#if defined(__INTEL_COMPILER)\n#pragma warning pop\n#elif defined(__GNUC__) || defined(__clang__)\n#pragma GCC diagnostic pop\n#endif\n#if defined(__NVCC__)\n#pragma nv_diagnostic pop\n#endif\n#if defined(__NVCOMPILER)\n#pragma diagnostic pop\n#endif\n}\n\nvoid State::PauseTiming() {\n // Add in time accumulated so far\n BM_CHECK(started_ && !finished_ && !skipped());\n timer_->StopTimer();\n if (perf_counters_measurement_ != nullptr) {\n std::vector> measurements;\n if (!perf_counters_measurement_->Stop(measurements)) {\n BM_CHECK(false) << \"Perf counters read the value failed.\";\n }\n for (const auto& name_and_measurement : measurements) {\n const std::string& name = name_and_measurement.first;\n const double measurement = name_and_measurement.second;\n // Counter was inserted with `kAvgIterations` flag by the constructor.\n assert(counters.find(name) != counters.end());\n counters[name].value += measurement;\n }\n }\n}\n\nvoid State::ResumeTiming() {\n BM_CHECK(started_ && !finished_ && !skipped());\n timer_->StartTimer();\n if (perf_counters_measurement_ != nullptr) {\n perf_counters_measurement_->Start();\n }\n}\n\nvoid State::SkipWithMessage(const std::string& msg) {\n skipped_ = internal::SkippedWithMessage;\n {\n MutexLock l(manager_->GetBenchmarkMutex());\n if (internal::NotSkipped == manager_->results.skipped_) {\n manager_->results.skip_message_ = msg;\n manager_->results.skipped_ = skipped_;\n }\n }\n total_iterations_ = 0;\n if (timer_->running()) {\n timer_->StopTimer();\n }\n}\n\nvoid State::SkipWithError(const std::string& msg) {\n skipped_ = internal::SkippedWithError;\n {\n MutexLock l(manager_->GetBenchmarkMutex());\n if (internal::NotSkipped == manager_->results.skipped_) {\n manager_->results.skip_message_ = msg;\n manager_->results.skipped_ = skipped_;\n }\n }\n total_iterations_ = 0;\n if (timer_->running()) {\n timer_->StopTimer();\n }\n}\n\nvoid State::SetIterationTime(double seconds) {\n timer_->SetIterationTime(seconds);\n}\n\nvoid State::SetLabel(const std::string& label) {\n MutexLock l(manager_->GetBenchmarkMutex());\n manager_->results.report_label_ = label;\n}\n\nvoid State::StartKeepRunning() {\n BM_CHECK(!started_ && !finished_);\n started_ = true;\n total_iterations_ = skipped() ? 0 : max_iterations;\n if (BENCHMARK_BUILTIN_EXPECT(profiler_manager_ != nullptr, false)) {\n profiler_manager_->AfterSetupStart();\n }\n manager_->StartStopBarrier();\n if (!skipped()) {\n ResumeTiming();\n }\n}\n\nvoid State::FinishKeepRunning() {\n BM_CHECK(started_ && (!finished_ || skipped()));\n if (!skipped()) {\n PauseTiming();\n }\n // Total iterations has now wrapped around past 0. Fix this.\n total_iterations_ = 0;\n finished_ = true;\n manager_->StartStopBarrier();\n if (BENCHMARK_BUILTIN_EXPECT(profiler_manager_ != nullptr, false)) {\n profiler_manager_->BeforeTeardownStop();\n }\n}\n\nnamespace internal {\nnamespace {\n\n// Flushes streams after invoking reporter methods that write to them. This\n// ensures users get timely updates even when streams are not line-buffered.\nvoid FlushStreams(BenchmarkReporter* reporter) {\n if (reporter == nullptr) {\n return;\n }\n std::flush(reporter->GetOutputStream());\n std::flush(reporter->GetErrorStream());\n}\n\n// Reports in both display and file reporters.\nvoid Report(BenchmarkReporter* display_reporter,\n BenchmarkReporter* file_reporter, const RunResults& run_results) {\n auto report_one = [](BenchmarkReporter* reporter, bool aggregates_only,\n const RunResults& results) {\n assert(reporter);\n // If there are no aggregates, do output non-aggregates.\n aggregates_only &= !results.aggregates_only.empty();\n if (!aggregates_only) {\n reporter->ReportRuns(results.non_aggregates);\n }\n if (!results.aggregates_only.empty()) {\n reporter->ReportRuns(results.aggregates_only);\n }\n };\n\n report_one(display_reporter, run_results.display_report_aggregates_only,\n run_results);\n if (file_reporter != nullptr) {\n report_one(file_reporter, run_results.file_report_aggregates_only,\n run_results);\n }\n\n FlushStreams(display_reporter);\n FlushStreams(file_reporter);\n}\n\nvoid RunBenchmarks(const std::vector& benchmarks,\n BenchmarkReporter* display_reporter,\n BenchmarkReporter* file_reporter) {\n // Note the file_reporter can be null.\n BM_CHECK(display_reporter != nullptr);\n\n // Determine the width of the name field using a minimum width of 10.\n bool might_have_aggregates = FLAGS_benchmark_repetitions > 1;\n size_t name_field_width = 10;\n size_t stat_field_width = 0;\n for (const BenchmarkInstance& benchmark : benchmarks) {\n name_field_width =\n std::max(name_field_width, benchmark.name().str().size());\n might_have_aggregates |= benchmark.repetitions() > 1;\n\n for (const auto& Stat : benchmark.statistics()) {\n stat_field_width = std::max(stat_field_width, Stat.name_.size());\n }\n }\n if (might_have_aggregates) {\n name_field_width += 1 + stat_field_width;\n }\n\n // Print header here\n BenchmarkReporter::Context context;\n context.name_field_width = name_field_width;\n\n // Keep track of running times of all instances of each benchmark family.\n std::map\n per_family_reports;\n\n if (display_reporter->ReportContext(context) &&\n ((file_reporter == nullptr) || file_reporter->ReportContext(context))) {\n FlushStreams(display_reporter);\n FlushStreams(file_reporter);\n\n size_t num_repetitions_total = 0;\n\n // This perfcounters object needs to be created before the runners vector\n // below so it outlasts their lifetime.\n PerfCountersMeasurement perfcounters(\n StrSplit(FLAGS_benchmark_perf_counters, ','));\n\n // Vector of benchmarks to run\n std::vector runners;\n runners.reserve(benchmarks.size());\n\n // Count the number of benchmarks with threads to warn the user in case\n // performance counters are used.\n int benchmarks_with_threads = 0;\n\n // Loop through all benchmarks\n for (const BenchmarkInstance& benchmark : benchmarks) {\n BenchmarkReporter::PerFamilyRunReports* reports_for_family = nullptr;\n if (benchmark.complexity() != oNone) {\n reports_for_family = &per_family_reports[benchmark.family_index()];\n }\n benchmarks_with_threads += static_cast(benchmark.threads() > 1);\n runners.emplace_back(benchmark, &perfcounters, reports_for_family);\n int num_repeats_of_this_instance = runners.back().GetNumRepeats();\n num_repetitions_total +=\n static_cast(num_repeats_of_this_instance);\n if (reports_for_family != nullptr) {\n reports_for_family->num_runs_total += num_repeats_of_this_instance;\n }\n }\n assert(runners.size() == benchmarks.size() && \"Unexpected runner count.\");\n\n // The use of performance counters with threads would be unintuitive for\n // the average user so we need to warn them about this case\n if ((benchmarks_with_threads > 0) && (perfcounters.num_counters() > 0)) {\n GetErrorLogInstance()\n << \"***WARNING*** There are \" << benchmarks_with_threads\n << \" benchmarks with threads and \" << perfcounters.num_counters()\n << \" performance counters were requested. Beware counters will \"\n \"reflect the combined usage across all \"\n \"threads.\\n\";\n }\n\n std::vector repetition_indices;\n repetition_indices.reserve(num_repetitions_total);\n for (size_t runner_index = 0, num_runners = runners.size();\n runner_index != num_runners; ++runner_index) {\n const internal::BenchmarkRunner& runner = runners[runner_index];\n std::fill_n(std::back_inserter(repetition_indices),\n runner.GetNumRepeats(), runner_index);\n }\n assert(repetition_indices.size() == num_repetitions_total &&\n \"Unexpected number of repetition indexes.\");\n\n if (FLAGS_benchmark_enable_random_interleaving) {\n std::random_device rd;\n std::mt19937 g(rd());\n std::shuffle(repetition_indices.begin(), repetition_indices.end(), g);\n }\n\n for (size_t repetition_index : repetition_indices) {\n internal::BenchmarkRunner& runner = runners[repetition_index];\n runner.DoOneRepetition();\n if (runner.HasRepeatsRemaining()) {\n continue;\n }\n // FIXME: report each repetition separately, not all of them in bulk.\n\n display_reporter->ReportRunsConfig(\n runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters());\n if (file_reporter != nullptr) {\n file_reporter->ReportRunsConfig(\n runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters());\n }\n\n RunResults run_results = runner.GetResults();\n\n // Maybe calculate complexity report\n if (const auto* reports_for_family = runner.GetReportsForFamily()) {\n if (reports_for_family->num_runs_done ==\n reports_for_family->num_runs_total) {\n auto additional_run_stats = ComputeBigO(reports_for_family->Runs);\n run_results.aggregates_only.insert(run_results.aggregates_only.end(),\n additional_run_stats.begin(),\n additional_run_stats.end());\n per_family_reports.erase(\n static_cast(reports_for_family->Runs.front().family_index));\n }\n }\n\n Report(display_reporter, file_reporter, run_results);\n }\n }\n display_reporter->Finalize();\n if (file_reporter != nullptr) {\n file_reporter->Finalize();\n }\n FlushStreams(display_reporter);\n FlushStreams(file_reporter);\n}\n\n// Disable deprecated warnings temporarily because we need to reference\n// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations\nBENCHMARK_DISABLE_DEPRECATED_WARNING\n\nstd::unique_ptr CreateReporter(\n std::string const& name, ConsoleReporter::OutputOptions output_opts) {\n typedef std::unique_ptr PtrType;\n if (name == \"console\") {\n return PtrType(new ConsoleReporter(output_opts));\n }\n if (name == \"json\") {\n return PtrType(new JSONReporter());\n }\n if (name == \"csv\") {\n return PtrType(new CSVReporter());\n }\n std::cerr << \"Unexpected format: '\" << name << \"'\\n\";\n std::flush(std::cerr);\n std::exit(1);\n}\n\nBENCHMARK_RESTORE_DEPRECATED_WARNING\n\n} // end namespace\n\nbool IsZero(double n) {\n return std::abs(n) < std::numeric_limits::epsilon();\n}\n\nConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) {\n int output_opts = ConsoleReporter::OO_Defaults;\n auto is_benchmark_color = [force_no_color]() -> bool {\n if (force_no_color) {\n return false;\n }\n if (FLAGS_benchmark_color == \"auto\") {\n return IsColorTerminal();\n }\n return IsTruthyFlagValue(FLAGS_benchmark_color);\n };\n if (is_benchmark_color()) {\n output_opts |= ConsoleReporter::OO_Color;\n } else {\n output_opts &= ~ConsoleReporter::OO_Color;\n }\n if (FLAGS_benchmark_counters_tabular) {\n output_opts |= ConsoleReporter::OO_Tabular;\n } else {\n output_opts &= ~ConsoleReporter::OO_Tabular;\n }\n return static_cast(output_opts);\n}\n\n} // end namespace internal\n\nBenchmarkReporter* CreateDefaultDisplayReporter() {\n static auto* default_display_reporter =\n internal::CreateReporter(FLAGS_benchmark_format,\n internal::GetOutputOptions())\n .release();\n return default_display_reporter;\n}\n\nsize_t RunSpecifiedBenchmarks() {\n return RunSpecifiedBenchmarks(nullptr, nullptr, FLAGS_benchmark_filter);\n}\n\nsize_t RunSpecifiedBenchmarks(std::string spec) {\n return RunSpecifiedBenchmarks(nullptr, nullptr, std::move(spec));\n}\n\nsize_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter) {\n return RunSpecifiedBenchmarks(display_reporter, nullptr,\n FLAGS_benchmark_filter);\n}\n\nsize_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,\n std::string spec) {\n return RunSpecifiedBenchmarks(display_reporter, nullptr, std::move(spec));\n}\n\nsize_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,\n BenchmarkReporter* file_reporter) {\n return RunSpecifiedBenchmarks(display_reporter, file_reporter,\n FLAGS_benchmark_filter);\n}\n\nsize_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,\n BenchmarkReporter* file_reporter,\n std::string spec) {\n if (spec.empty() || spec == \"all\") {\n spec = \".\"; // Regexp that matches all benchmarks\n }\n\n // Setup the reporters\n std::ofstream output_file;\n std::unique_ptr default_display_reporter;\n std::unique_ptr default_file_reporter;\n if (display_reporter == nullptr) {\n default_display_reporter.reset(CreateDefaultDisplayReporter());\n display_reporter = default_display_reporter.get();\n }\n auto& Out = display_reporter->GetOutputStream();\n auto& Err = display_reporter->GetErrorStream();\n\n std::string const& fname = FLAGS_benchmark_out;\n if (fname.empty() && (file_reporter != nullptr)) {\n Err << \"A custom file reporter was provided but \"\n \"--benchmark_out= was not specified.\\n\";\n Out.flush();\n Err.flush();\n std::exit(1);\n }\n if (!fname.empty()) {\n output_file.open(fname);\n if (!output_file.is_open()) {\n Err << \"invalid file name: '\" << fname << \"'\\n\";\n Out.flush();\n Err.flush();\n std::exit(1);\n }\n if (file_reporter == nullptr) {\n default_file_reporter = internal::CreateReporter(\n FLAGS_benchmark_out_format, FLAGS_benchmark_counters_tabular\n ? ConsoleReporter::OO_Tabular\n : ConsoleReporter::OO_None);\n file_reporter = default_file_reporter.get();\n }\n file_reporter->SetOutputStream(&output_file);\n file_reporter->SetErrorStream(&output_file);\n }\n\n std::vector benchmarks;\n if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) {\n Out.flush();\n Err.flush();\n return 0;\n }\n\n if (benchmarks.empty()) {\n Err << \"Failed to match any benchmarks against regex: \" << spec << \"\\n\";\n Out.flush();\n Err.flush();\n return 0;\n }\n\n if (FLAGS_benchmark_list_tests) {\n for (auto const& benchmark : benchmarks) {\n Out << benchmark.name().str() << \"\\n\";\n }\n } else {\n internal::RunBenchmarks(benchmarks, display_reporter, file_reporter);\n }\n\n Out.flush();\n Err.flush();\n return benchmarks.size();\n}\n\nnamespace {\n// stores the time unit benchmarks use by default\nTimeUnit default_time_unit = kNanosecond;\n} // namespace\n\nTimeUnit GetDefaultTimeUnit() { return default_time_unit; }\n\nvoid SetDefaultTimeUnit(TimeUnit unit) { default_time_unit = unit; }\n\nstd::string GetBenchmarkFilter() { return FLAGS_benchmark_filter; }\n\nvoid SetBenchmarkFilter(std::string value) {\n FLAGS_benchmark_filter = std::move(value);\n}\n\nint32_t GetBenchmarkVerbosity() { return FLAGS_v; }\n\nvoid RegisterMemoryManager(MemoryManager* manager) {\n internal::memory_manager = manager;\n}\n\nvoid RegisterProfilerManager(ProfilerManager* manager) {\n // Don't allow overwriting an existing manager.\n if (manager != nullptr) {\n BM_CHECK_EQ(internal::profiler_manager, nullptr);\n }\n internal::profiler_manager = manager;\n}\n\nvoid AddCustomContext(std::string key, std::string value) {\n if (internal::global_context == nullptr) {\n internal::global_context = new std::map();\n }\n if (!internal::global_context->emplace(std::move(key), std::move(value))\n .second) {\n std::cerr << \"Failed to add custom context \\\"\" << key << \"\\\" as it already \"\n << \"exists with value \\\"\" << value << \"\\\"\\n\";\n }\n}\n\nnamespace internal {\n\nvoid (*HelperPrintf)();\n\nnamespace {\nvoid PrintUsageAndExit() {\n HelperPrintf();\n std::flush(std::cout);\n std::flush(std::cerr);\n std::exit(0);\n}\n\nvoid SetDefaultTimeUnitFromFlag(const std::string& time_unit_flag) {\n if (time_unit_flag == \"s\") {\n return SetDefaultTimeUnit(kSecond);\n }\n if (time_unit_flag == \"ms\") {\n return SetDefaultTimeUnit(kMillisecond);\n }\n if (time_unit_flag == \"us\") {\n return SetDefaultTimeUnit(kMicrosecond);\n }\n if (time_unit_flag == \"ns\") {\n return SetDefaultTimeUnit(kNanosecond);\n }\n if (!time_unit_flag.empty()) {\n PrintUsageAndExit();\n }\n}\n\nvoid ParseCommandLineFlags(int* argc, char** argv) {\n using namespace benchmark;\n BenchmarkReporter::Context::executable_name =\n ((argc != nullptr) && *argc > 0) ? argv[0] : \"unknown\";\n for (int i = 1; (argc != nullptr) && i < *argc; ++i) {\n if (ParseBoolFlag(argv[i], \"benchmark_list_tests\",\n &FLAGS_benchmark_list_tests) ||\n ParseStringFlag(argv[i], \"benchmark_filter\", &FLAGS_benchmark_filter) ||\n ParseStringFlag(argv[i], \"benchmark_min_time\",\n &FLAGS_benchmark_min_time) ||\n ParseDoubleFlag(argv[i], \"benchmark_min_warmup_time\",\n &FLAGS_benchmark_min_warmup_time) ||\n ParseInt32Flag(argv[i], \"benchmark_repetitions\",\n &FLAGS_benchmark_repetitions) ||\n ParseBoolFlag(argv[i], \"benchmark_dry_run\", &FLAGS_benchmark_dry_run) ||\n ParseBoolFlag(argv[i], \"benchmark_enable_random_interleaving\",\n &FLAGS_benchmark_enable_random_interleaving) ||\n ParseBoolFlag(argv[i], \"benchmark_report_aggregates_only\",\n &FLAGS_benchmark_report_aggregates_only) ||\n ParseBoolFlag(argv[i], \"benchmark_display_aggregates_only\",\n &FLAGS_benchmark_display_aggregates_only) ||\n ParseStringFlag(argv[i], \"benchmark_format\", &FLAGS_benchmark_format) ||\n ParseStringFlag(argv[i], \"benchmark_out\", &FLAGS_benchmark_out) ||\n ParseStringFlag(argv[i], \"benchmark_out_format\",\n &FLAGS_benchmark_out_format) ||\n ParseStringFlag(argv[i], \"benchmark_color\", &FLAGS_benchmark_color) ||\n ParseBoolFlag(argv[i], \"benchmark_counters_tabular\",\n &FLAGS_benchmark_counters_tabular) ||\n ParseStringFlag(argv[i], \"benchmark_perf_counters\",\n &FLAGS_benchmark_perf_counters) ||\n ParseKeyValueFlag(argv[i], \"benchmark_context\",\n &FLAGS_benchmark_context) ||\n ParseStringFlag(argv[i], \"benchmark_time_unit\",\n &FLAGS_benchmark_time_unit) ||\n ParseInt32Flag(argv[i], \"v\", &FLAGS_v)) {\n for (int j = i; j != *argc - 1; ++j) {\n argv[j] = argv[j + 1];\n }\n\n --(*argc);\n --i;\n } else if (IsFlag(argv[i], \"help\")) {\n PrintUsageAndExit();\n }\n }\n for (auto const* flag :\n {&FLAGS_benchmark_format, &FLAGS_benchmark_out_format}) {\n if (*flag != \"console\" && *flag != \"json\" && *flag != \"csv\") {\n PrintUsageAndExit();\n }\n }\n SetDefaultTimeUnitFromFlag(FLAGS_benchmark_time_unit);\n if (FLAGS_benchmark_color.empty()) {\n PrintUsageAndExit();\n }\n if (FLAGS_benchmark_dry_run) {\n AddCustomContext(\"dry_run\", \"true\");\n }\n for (const auto& kv : FLAGS_benchmark_context) {\n AddCustomContext(kv.first, kv.second);\n }\n}\n\n} // end namespace\n\nint InitializeStreams() {\n static std::ios_base::Init init;\n return 0;\n}\n\ntemplate \nstd::make_unsigned_t get_as_unsigned(T v) {\n using UnsignedT = std::make_unsigned_t;\n return static_cast(v);\n}\n\n} // end namespace internal\n\nvoid MaybeReenterWithoutASLR(int /*argc*/, char** argv) {\n // On e.g. Hexagon simulator, argv may be NULL.\n if (!argv) return;\n\n#ifdef BENCHMARK_OS_LINUX\n const auto curr_personality = personality(0xffffffff);\n\n // We should never fail to read-only query the current personality,\n // but let's be cautious.\n if (curr_personality == -1) return;\n\n // If ASLR is already disabled, we have nothing more to do.\n if (internal::get_as_unsigned(curr_personality) & ADDR_NO_RANDOMIZE) return;\n\n // Try to change the personality to disable ASLR.\n const auto proposed_personality =\n internal::get_as_unsigned(curr_personality) | ADDR_NO_RANDOMIZE;\n const auto prev_personality = personality(proposed_personality);\n\n // Have we failed to change the personality? That may happen.\n if (prev_personality == -1) return;\n\n // Make sure the parsona has been updated with the no-ASLR flag,\n // otherwise we will try to reenter infinitely.\n // This seems impossible, but can happen in some docker configurations.\n const auto new_personality = personality(0xffffffff);\n if ((internal::get_as_unsigned(new_personality) & ADDR_NO_RANDOMIZE) == 0)\n return;\n\n execv(argv[0], argv);\n // The exec() functions return only if an error has occurred,\n // in which case we want to just continue as-is.\n#else\n return;\n#endif\n}\n\nstd::string GetBenchmarkVersion() {\n#ifdef BENCHMARK_VERSION\n return {BENCHMARK_VERSION};\n#else\n return {\"\"};\n#endif\n}\n\nvoid PrintDefaultHelp() {\n fprintf(stdout,\n \"benchmark\"\n \" [--benchmark_list_tests={true|false}]\\n\"\n \" [--benchmark_filter=]\\n\"\n \" [--benchmark_min_time=`x` OR `s` ]\\n\"\n \" [--benchmark_min_warmup_time=]\\n\"\n \" [--benchmark_repetitions=]\\n\"\n \" [--benchmark_dry_run={true|false}]\\n\"\n \" [--benchmark_enable_random_interleaving={true|false}]\\n\"\n \" [--benchmark_report_aggregates_only={true|false}]\\n\"\n \" [--benchmark_display_aggregates_only={true|false}]\\n\"\n \" [--benchmark_format=]\\n\"\n \" [--benchmark_out=]\\n\"\n \" [--benchmark_out_format=]\\n\"\n \" [--benchmark_color={auto|true|false}]\\n\"\n \" [--benchmark_counters_tabular={true|false}]\\n\"\n#if defined HAVE_LIBPFM\n \" [--benchmark_perf_counters=,...]\\n\"\n#endif\n \" [--benchmark_context==,...]\\n\"\n \" [--benchmark_time_unit={ns|us|ms|s}]\\n\"\n \" [--v=]\\n\");\n}\n\nvoid Initialize(int* argc, char** argv, void (*HelperPrintf)()) {\n internal::HelperPrintf = HelperPrintf;\n internal::ParseCommandLineFlags(argc, argv);\n internal::LogLevel() = FLAGS_v;\n}\n\nvoid Shutdown() { delete internal::global_context; }\n\nbool ReportUnrecognizedArguments(int argc, char** argv) {\n for (int i = 1; i < argc; ++i) {\n fprintf(stderr, \"%s: error: unrecognized command-line flag: %s\\n\", argv[0],\n argv[i]);\n }\n return argc > 1;\n}\n\n} // end namespace benchmark\n" }, { "path": "src/benchmark_api_internal.cc", "content": "#include \"benchmark_api_internal.h\"\n\n#include \n\n#include \"string_util.h\"\n\nnamespace benchmark {\nnamespace internal {\n\nBenchmarkInstance::BenchmarkInstance(benchmark::Benchmark* benchmark,\n int family_idx,\n int per_family_instance_idx,\n const std::vector& args,\n int thread_count)\n : benchmark_(*benchmark),\n family_index_(family_idx),\n per_family_instance_index_(per_family_instance_idx),\n aggregation_report_mode_(benchmark_.aggregation_report_mode_),\n args_(args),\n time_unit_(benchmark_.GetTimeUnit()),\n measure_process_cpu_time_(benchmark_.measure_process_cpu_time_),\n use_real_time_(benchmark_.use_real_time_),\n use_manual_time_(benchmark_.use_manual_time_),\n complexity_(benchmark_.complexity_),\n complexity_lambda_(benchmark_.complexity_lambda_),\n statistics_(benchmark_.statistics_),\n repetitions_(benchmark_.repetitions_),\n min_time_(benchmark_.min_time_),\n min_warmup_time_(benchmark_.min_warmup_time_),\n iterations_(benchmark_.iterations_),\n threads_(thread_count),\n setup_(benchmark_.setup_),\n teardown_(benchmark_.teardown_) {\n name_.function_name = benchmark_.name_;\n\n size_t arg_i = 0;\n for (const auto& arg : args) {\n if (!name_.args.empty()) {\n name_.args += '/';\n }\n\n if (arg_i < benchmark->arg_names_.size()) {\n const auto& arg_name = benchmark_.arg_names_[arg_i];\n if (!arg_name.empty()) {\n name_.args += StrFormat(\"%s:\", arg_name.c_str());\n }\n }\n\n name_.args += StrFormat(\"%\" PRId64, arg);\n ++arg_i;\n }\n\n if (!IsZero(benchmark->min_time_)) {\n name_.min_time = StrFormat(\"min_time:%0.3f\", benchmark_.min_time_);\n }\n\n if (!IsZero(benchmark->min_warmup_time_)) {\n name_.min_warmup_time =\n StrFormat(\"min_warmup_time:%0.3f\", benchmark_.min_warmup_time_);\n }\n\n if (benchmark_.iterations_ != 0) {\n name_.iterations = StrFormat(\n \"iterations:%lu\", static_cast(benchmark_.iterations_));\n }\n\n if (benchmark_.repetitions_ != 0) {\n name_.repetitions = StrFormat(\"repeats:%d\", benchmark_.repetitions_);\n }\n\n if (benchmark_.measure_process_cpu_time_) {\n name_.time_type = \"process_time\";\n }\n\n if (benchmark_.use_manual_time_) {\n if (!name_.time_type.empty()) {\n name_.time_type += '/';\n }\n name_.time_type += \"manual_time\";\n } else if (benchmark_.use_real_time_) {\n if (!name_.time_type.empty()) {\n name_.time_type += '/';\n }\n name_.time_type += \"real_time\";\n }\n\n if (!benchmark_.thread_counts_.empty()) {\n name_.threads = StrFormat(\"threads:%d\", threads_);\n }\n}\n\nState BenchmarkInstance::Run(\n IterationCount iters, int thread_id, internal::ThreadTimer* timer,\n internal::ThreadManager* manager,\n internal::PerfCountersMeasurement* perf_counters_measurement,\n ProfilerManager* profiler_manager) const {\n State st(name_.function_name, iters, args_, thread_id, threads_, timer,\n manager, perf_counters_measurement, profiler_manager);\n benchmark_.Run(st);\n return st;\n}\n\nvoid BenchmarkInstance::Setup() const {\n if (setup_ != nullptr) {\n State st(name_.function_name, /*iters*/ 1, args_, /*thread_id*/ 0, threads_,\n nullptr, nullptr, nullptr, nullptr);\n setup_(st);\n }\n}\n\nvoid BenchmarkInstance::Teardown() const {\n if (teardown_ != nullptr) {\n State st(name_.function_name, /*iters*/ 1, args_, /*thread_id*/ 0, threads_,\n nullptr, nullptr, nullptr, nullptr);\n teardown_(st);\n }\n}\n} // namespace internal\n} // namespace benchmark\n" }, { "path": "src/benchmark_api_internal.h", "content": "#ifndef BENCHMARK_API_INTERNAL_H\n#define BENCHMARK_API_INTERNAL_H\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/sysinfo.h\"\n#include \"commandlineflags.h\"\n\nnamespace benchmark {\nnamespace internal {\n\n// Information kept per benchmark we may want to run\nclass BenchmarkInstance {\n public:\n BenchmarkInstance(benchmark::Benchmark* benchmark, int family_idx,\n int per_family_instance_idx,\n const std::vector& args, int thread_count);\n\n const BenchmarkName& name() const { return name_; }\n int family_index() const { return family_index_; }\n int per_family_instance_index() const { return per_family_instance_index_; }\n AggregationReportMode aggregation_report_mode() const {\n return aggregation_report_mode_;\n }\n TimeUnit time_unit() const { return time_unit_; }\n bool measure_process_cpu_time() const { return measure_process_cpu_time_; }\n bool use_real_time() const { return use_real_time_; }\n bool use_manual_time() const { return use_manual_time_; }\n BigO complexity() const { return complexity_; }\n BigOFunc* complexity_lambda() const { return complexity_lambda_; }\n const std::vector& statistics() const { return statistics_; }\n int repetitions() const { return repetitions_; }\n double min_time() const { return min_time_; }\n double min_warmup_time() const { return min_warmup_time_; }\n IterationCount iterations() const { return iterations_; }\n int threads() const { return threads_; }\n void Setup() const;\n void Teardown() const;\n const auto& GetUserThreadRunnerFactory() const {\n return benchmark_.threadrunner_;\n }\n\n State Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer,\n internal::ThreadManager* manager,\n internal::PerfCountersMeasurement* perf_counters_measurement,\n ProfilerManager* profiler_manager) const;\n\n private:\n BenchmarkName name_;\n benchmark::Benchmark& benchmark_;\n const int family_index_;\n const int per_family_instance_index_;\n AggregationReportMode aggregation_report_mode_;\n const std::vector& args_;\n TimeUnit time_unit_;\n bool measure_process_cpu_time_;\n bool use_real_time_;\n bool use_manual_time_;\n BigO complexity_;\n BigOFunc* complexity_lambda_;\n UserCounters counters_;\n const std::vector& statistics_;\n int repetitions_;\n double min_time_;\n double min_warmup_time_;\n IterationCount iterations_;\n int threads_; // Number of concurrent threads to us\n\n callback_function setup_;\n callback_function teardown_;\n};\n\nbool FindBenchmarksInternal(const std::string& re,\n std::vector* benchmarks,\n std::ostream* Err);\n\nbool IsZero(double n);\n\nBENCHMARK_EXPORT\nConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color = false);\n\n} // end namespace internal\n} // end namespace benchmark\n\n#endif // BENCHMARK_API_INTERNAL_H\n" }, { "path": "src/benchmark_main.cc", "content": "// Copyright 2018 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"benchmark/export.h\"\n#include \"benchmark/registration.h\"\n\nBENCHMARK_EXPORT int main(int /*argc*/, char** /*argv*/);\nBENCHMARK_MAIN();\n" }, { "path": "src/benchmark_name.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"benchmark/export.h\"\n#include \"benchmark/reporter.h\"\n\nnamespace benchmark {\n\nnamespace {\n\n// Compute the total size of a pack of std::strings\nsize_t size_impl() { return 0; }\n\ntemplate \nsize_t size_impl(const Head& head, const Tail&... tail) {\n return head.size() + size_impl(tail...);\n}\n\n// Join a pack of std::strings using a delimiter\n// TODO(dominic): use absl::StrJoin\nvoid join_impl(std::string& /*unused*/, char /*unused*/) {}\n\ntemplate \nvoid join_impl(std::string& s, const char delimiter, const Head& head,\n const Tail&... tail) {\n if (!s.empty() && !head.empty()) {\n s += delimiter;\n }\n\n s += head;\n\n join_impl(s, delimiter, tail...);\n}\n\ntemplate \nstd::string join(char delimiter, const Ts&... ts) {\n std::string s;\n s.reserve(sizeof...(Ts) + size_impl(ts...));\n join_impl(s, delimiter, ts...);\n return s;\n}\n} // namespace\n\nBENCHMARK_EXPORT\nstd::string BenchmarkName::str() const {\n return join('/', function_name, args, min_time, min_warmup_time, iterations,\n repetitions, time_type, threads);\n}\n} // namespace benchmark\n" }, { "path": "src/benchmark_register.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"benchmark_register.h\"\n\n#ifndef BENCHMARK_OS_WINDOWS\n#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \\\n !defined(BENCHMARK_OS_WASI)\n#include \n#endif\n#include \n#include \n#endif\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/statistics.h\"\n#include \"benchmark/types.h\"\n#include \"benchmark_api_internal.h\"\n#include \"check.h\"\n#include \"commandlineflags.h\"\n#include \"complexity.h\"\n#include \"internal_macros.h\"\n#include \"log.h\"\n#include \"mutex.h\"\n#include \"re.h\"\n#include \"statistics.h\"\n#include \"string_util.h\"\n#include \"timers.h\"\n\nnamespace benchmark {\n\nnamespace {\n// For non-dense Range, intermediate values are powers of kRangeMultiplier.\nconstexpr int kRangeMultiplier = 8;\n\n// The size of a benchmark family determines is the number of inputs to repeat\n// the benchmark on. If this is \"large\" then warn the user during configuration.\nconstexpr size_t kMaxFamilySize = 100;\n\nconstexpr char kDisabledPrefix[] = \"DISABLED_\";\n} // end namespace\n\nnamespace internal {\n\n//=============================================================================//\n// BenchmarkFamilies\n//=============================================================================//\n\n// Class for managing registered benchmarks. Note that each registered\n// benchmark identifies a family of related benchmarks to run.\nclass BenchmarkFamilies {\n public:\n static BenchmarkFamilies* GetInstance();\n\n // Registers a benchmark family and returns the index assigned to it.\n size_t AddBenchmark(std::unique_ptr family);\n\n // Clear all registered benchmark families.\n void ClearBenchmarks();\n\n // Extract the list of benchmark instances that match the specified\n // regular expression.\n bool FindBenchmarks(std::string spec,\n std::vector* benchmarks,\n std::ostream* Err);\n\n private:\n BenchmarkFamilies() {}\n\n std::vector> families_;\n Mutex mutex_;\n};\n\nBenchmarkFamilies* BenchmarkFamilies::GetInstance() {\n static BenchmarkFamilies instance;\n return &instance;\n}\n\nsize_t BenchmarkFamilies::AddBenchmark(\n std::unique_ptr family) {\n MutexLock l(mutex_);\n size_t index = families_.size();\n families_.push_back(std::move(family));\n return index;\n}\n\nvoid BenchmarkFamilies::ClearBenchmarks() {\n MutexLock l(mutex_);\n families_.clear();\n families_.shrink_to_fit();\n}\n\nbool BenchmarkFamilies::FindBenchmarks(\n std::string spec, std::vector* benchmarks,\n std::ostream* ErrStream) {\n BM_CHECK(ErrStream);\n auto& Err = *ErrStream;\n // Make regular expression out of command-line flag\n std::string error_msg;\n Regex re;\n bool is_negative_filter = false;\n if (spec[0] == '-') {\n spec.replace(0, 1, \"\");\n is_negative_filter = true;\n }\n if (!re.Init(spec, &error_msg)) {\n Err << \"Could not compile benchmark re: \" << error_msg << '\\n';\n return false;\n }\n\n // Special list of thread counts to use when none are specified\n const std::vector one_thread = {1};\n\n int next_family_index = 0;\n\n MutexLock l(mutex_);\n for (std::unique_ptr& family : families_) {\n int family_index = next_family_index;\n int per_family_instance_index = 0;\n\n // Family was deleted or benchmark doesn't match\n if (!family) {\n continue;\n }\n\n if (family->ArgsCnt() == -1) {\n family->Args({});\n }\n const std::vector* thread_counts =\n (family->thread_counts_.empty()\n ? &one_thread\n : &static_cast&>(family->thread_counts_));\n const size_t family_size = family->args_.size() * thread_counts->size();\n // The benchmark will be run at least 'family_size' different inputs.\n // If 'family_size' is very large warn the user.\n if (family_size > kMaxFamilySize) {\n Err << \"The number of inputs is very large. \" << family->name_\n << \" will be repeated at least \" << family_size << \" times.\\n\";\n }\n // reserve in the special case the regex \".\", since we know the final\n // family size. this doesn't take into account any disabled benchmarks\n // so worst case we reserve more than we need.\n if (spec == \".\") {\n benchmarks->reserve(benchmarks->size() + family_size);\n }\n\n for (auto const& args : family->args_) {\n for (int num_threads : *thread_counts) {\n BenchmarkInstance instance(family.get(), family_index,\n per_family_instance_index, args,\n num_threads);\n\n const auto full_name = instance.name().str();\n if (full_name.rfind(kDisabledPrefix, 0) != 0 &&\n ((re.Match(full_name) && !is_negative_filter) ||\n (!re.Match(full_name) && is_negative_filter))) {\n benchmarks->push_back(std::move(instance));\n\n ++per_family_instance_index;\n\n // Only bump the next family index once we've established that\n // at least one instance of this family will be run.\n if (next_family_index == family_index) {\n ++next_family_index;\n }\n }\n }\n }\n }\n return true;\n}\n\nbenchmark::Benchmark* RegisterBenchmarkInternal(\n std::unique_ptr bench) {\n benchmark::Benchmark* bench_ptr = bench.get();\n BenchmarkFamilies* families = BenchmarkFamilies::GetInstance();\n families->AddBenchmark(std::move(bench));\n return bench_ptr;\n}\n\n// FIXME: This function is a hack so that benchmark.cc can access\n// `BenchmarkFamilies`\nbool FindBenchmarksInternal(const std::string& re,\n std::vector* benchmarks,\n std::ostream* Err) {\n return BenchmarkFamilies::GetInstance()->FindBenchmarks(re, benchmarks, Err);\n}\n\n} // end namespace internal\n\n//=============================================================================//\n// Benchmark\n//=============================================================================//\n\nBenchmark::Benchmark(const std::string& name)\n : name_(name),\n aggregation_report_mode_(internal::ARM_Unspecified),\n time_unit_(GetDefaultTimeUnit()),\n use_default_time_unit_(true),\n range_multiplier_(kRangeMultiplier),\n min_time_(0),\n min_warmup_time_(0),\n iterations_(0),\n repetitions_(0),\n measure_process_cpu_time_(false),\n use_real_time_(false),\n use_manual_time_(false),\n complexity_(oNone),\n complexity_lambda_(nullptr) {\n ComputeStatistics(\"mean\", StatisticsMean);\n ComputeStatistics(\"median\", StatisticsMedian);\n ComputeStatistics(\"stddev\", StatisticsStdDev);\n ComputeStatistics(\"cv\", StatisticsCV, kPercentage);\n}\n\nBenchmark::~Benchmark() {}\n\nBenchmark* Benchmark::Name(const std::string& name) {\n SetName(name);\n return this;\n}\n\nBenchmark* Benchmark::Arg(int64_t x) {\n BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);\n args_.push_back({x});\n return this;\n}\n\nBenchmark* Benchmark::Unit(TimeUnit unit) {\n time_unit_ = unit;\n use_default_time_unit_ = false;\n return this;\n}\n\nBenchmark* Benchmark::Range(int64_t start, int64_t limit) {\n BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);\n std::vector arglist;\n internal::AddRange(&arglist, start, limit, range_multiplier_);\n\n for (int64_t i : arglist) {\n args_.push_back({i});\n }\n return this;\n}\n\nBenchmark* Benchmark::Ranges(\n const std::vector>& ranges) {\n BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(ranges.size()));\n std::vector> arglists(ranges.size());\n for (std::size_t i = 0; i < ranges.size(); i++) {\n internal::AddRange(&arglists[i], ranges[i].first, ranges[i].second,\n range_multiplier_);\n }\n\n ArgsProduct(arglists);\n\n return this;\n}\n\nBenchmark* Benchmark::ArgsProduct(\n const std::vector>& arglists) {\n BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(arglists.size()));\n\n std::vector indices(arglists.size());\n const std::size_t total = std::accumulate(\n std::begin(arglists), std::end(arglists), std::size_t{1},\n [](const std::size_t res, const std::vector& arglist) {\n return res * arglist.size();\n });\n std::vector args;\n args.reserve(arglists.size());\n for (std::size_t i = 0; i < total; i++) {\n for (std::size_t arg = 0; arg < arglists.size(); arg++) {\n args.push_back(arglists[arg][indices[arg]]);\n }\n args_.push_back(args);\n args.clear();\n\n std::size_t arg = 0;\n do {\n indices[arg] = (indices[arg] + 1) % arglists[arg].size();\n } while (indices[arg++] == 0 && arg < arglists.size());\n }\n\n return this;\n}\n\nBenchmark* Benchmark::ArgName(const std::string& name) {\n BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);\n arg_names_ = {name};\n return this;\n}\n\nBenchmark* Benchmark::ArgNames(const std::vector& names) {\n BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(names.size()));\n arg_names_ = names;\n return this;\n}\n\nBenchmark* Benchmark::DenseRange(int64_t start, int64_t limit, int step) {\n BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);\n BM_CHECK_LE(start, limit);\n for (int64_t arg = start; arg <= limit; arg += step) {\n args_.push_back({arg});\n }\n return this;\n}\n\nBenchmark* Benchmark::Args(const std::vector& args) {\n BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(args.size()));\n args_.push_back(args);\n return this;\n}\n\nBenchmark* Benchmark::Apply(\n const std::function& custom_arguments) {\n custom_arguments(this);\n return this;\n}\n\nBenchmark* Benchmark::Setup(callback_function&& setup) {\n BM_CHECK(setup != nullptr);\n setup_ = std::forward(setup);\n return this;\n}\n\nBenchmark* Benchmark::Setup(const callback_function& setup) {\n BM_CHECK(setup != nullptr);\n setup_ = setup;\n return this;\n}\n\nBenchmark* Benchmark::Teardown(callback_function&& teardown) {\n BM_CHECK(teardown != nullptr);\n teardown_ = std::forward(teardown);\n return this;\n}\n\nBenchmark* Benchmark::Teardown(const callback_function& teardown) {\n BM_CHECK(teardown != nullptr);\n teardown_ = teardown;\n return this;\n}\n\nBenchmark* Benchmark::RangeMultiplier(int multiplier) {\n BM_CHECK(multiplier > 1);\n range_multiplier_ = multiplier;\n return this;\n}\n\nBenchmark* Benchmark::MinTime(double t) {\n BM_CHECK(t > 0.0);\n BM_CHECK(iterations_ == 0);\n min_time_ = t;\n return this;\n}\n\nBenchmark* Benchmark::MinWarmUpTime(double t) {\n BM_CHECK(t >= 0.0);\n BM_CHECK(iterations_ == 0);\n min_warmup_time_ = t;\n return this;\n}\n\nBenchmark* Benchmark::Iterations(IterationCount n) {\n BM_CHECK(n > 0);\n BM_CHECK(internal::IsZero(min_time_));\n BM_CHECK(internal::IsZero(min_warmup_time_));\n iterations_ = n;\n return this;\n}\n\nBenchmark* Benchmark::Repetitions(int n) {\n BM_CHECK(n > 0);\n repetitions_ = n;\n return this;\n}\n\nBenchmark* Benchmark::ReportAggregatesOnly(bool value) {\n aggregation_report_mode_ =\n value ? internal::ARM_ReportAggregatesOnly : internal::ARM_Default;\n return this;\n}\n\nBenchmark* Benchmark::DisplayAggregatesOnly(bool value) {\n // If we were called, the report mode is no longer 'unspecified', in any case.\n using internal::AggregationReportMode;\n aggregation_report_mode_ = static_cast(\n aggregation_report_mode_ | internal::ARM_Default);\n\n if (value) {\n aggregation_report_mode_ = static_cast(\n aggregation_report_mode_ | internal::ARM_DisplayReportAggregatesOnly);\n } else {\n aggregation_report_mode_ = static_cast(\n aggregation_report_mode_ & ~internal::ARM_DisplayReportAggregatesOnly);\n }\n\n return this;\n}\n\nBenchmark* Benchmark::MeasureProcessCPUTime() {\n // Can be used together with UseRealTime() / UseManualTime().\n measure_process_cpu_time_ = true;\n return this;\n}\n\nBenchmark* Benchmark::UseRealTime() {\n BM_CHECK(!use_manual_time_)\n << \"Cannot set UseRealTime and UseManualTime simultaneously.\";\n use_real_time_ = true;\n return this;\n}\n\nBenchmark* Benchmark::UseManualTime() {\n BM_CHECK(!use_real_time_)\n << \"Cannot set UseRealTime and UseManualTime simultaneously.\";\n use_manual_time_ = true;\n return this;\n}\n\nBenchmark* Benchmark::Complexity(BigO complexity) {\n complexity_ = complexity;\n return this;\n}\n\nBenchmark* Benchmark::Complexity(BigOFunc* complexity) {\n complexity_lambda_ = complexity;\n complexity_ = oLambda;\n return this;\n}\n\nBenchmark* Benchmark::ComputeStatistics(const std::string& name,\n StatisticsFunc* statistics,\n StatisticUnit unit) {\n statistics_.emplace_back(name, statistics, unit);\n return this;\n}\n\nBenchmark* Benchmark::Threads(int t) {\n BM_CHECK_GT(t, 0);\n thread_counts_.push_back(t);\n return this;\n}\n\nBenchmark* Benchmark::ThreadRange(int min_threads, int max_threads) {\n BM_CHECK_GT(min_threads, 0);\n BM_CHECK_GE(max_threads, min_threads);\n\n internal::AddRange(&thread_counts_, min_threads, max_threads, 2);\n return this;\n}\n\nBenchmark* Benchmark::DenseThreadRange(int min_threads, int max_threads,\n int stride) {\n BM_CHECK_GT(min_threads, 0);\n BM_CHECK_GE(max_threads, min_threads);\n BM_CHECK_GE(stride, 1);\n\n for (auto i = min_threads; i < max_threads; i += stride) {\n thread_counts_.push_back(i);\n }\n thread_counts_.push_back(max_threads);\n return this;\n}\n\nBenchmark* Benchmark::ThreadPerCpu() {\n thread_counts_.push_back(CPUInfo::Get().num_cpus);\n return this;\n}\n\nBenchmark* Benchmark::ThreadRunner(threadrunner_factory&& factory) {\n threadrunner_ = std::move(factory);\n return this;\n}\n\nvoid Benchmark::SetName(const std::string& name) { name_ = name; }\n\nconst char* Benchmark::GetName() const { return name_.c_str(); }\n\nint Benchmark::ArgsCnt() const {\n if (args_.empty()) {\n if (arg_names_.empty()) {\n return -1;\n }\n return static_cast(arg_names_.size());\n }\n return static_cast(args_.front().size());\n}\n\nconst char* Benchmark::GetArgName(int arg) const {\n BM_CHECK_GE(arg, 0);\n size_t uarg = static_cast(arg);\n BM_CHECK_LT(uarg, arg_names_.size());\n return arg_names_[uarg].c_str();\n}\n\nTimeUnit Benchmark::GetTimeUnit() const {\n return use_default_time_unit_ ? GetDefaultTimeUnit() : time_unit_;\n}\n\nnamespace internal {\n\n//=============================================================================//\n// FunctionBenchmark\n//=============================================================================//\n\nvoid FunctionBenchmark::Run(State& st) { func_(st); }\n\n} // end namespace internal\n\nvoid ClearRegisteredBenchmarks() {\n internal::BenchmarkFamilies::GetInstance()->ClearBenchmarks();\n}\n\nstd::vector CreateRange(int64_t lo, int64_t hi, int multi) {\n std::vector args;\n internal::AddRange(&args, lo, hi, multi);\n return args;\n}\n\nstd::vector CreateDenseRange(int64_t start, int64_t limit, int step) {\n BM_CHECK_LE(start, limit);\n std::vector args;\n for (int64_t arg = start; arg <= limit; arg += step) {\n args.push_back(arg);\n }\n return args;\n}\n\n} // end namespace benchmark\n" }, { "path": "src/benchmark_register.h", "content": "#ifndef BENCHMARK_REGISTER_H\n#define BENCHMARK_REGISTER_H\n\n#include \n#include \n#include \n#include \n\n#include \"check.h\"\n\nnamespace benchmark {\nnamespace internal {\n\n// Append the powers of 'mult' in the closed interval [lo, hi].\n// Returns iterator to the start of the inserted range.\ntemplate \ntypename std::vector::iterator AddPowers(std::vector* dst, T lo, T hi,\n int mult) {\n BM_CHECK_GE(lo, 0);\n BM_CHECK_GE(hi, lo);\n BM_CHECK_GE(mult, 2);\n\n const size_t start_offset = dst->size();\n\n static const T kmax = std::numeric_limits::max();\n\n // Space out the values in multiples of \"mult\"\n for (T i = static_cast(1); i <= hi; i = static_cast(i * mult)) {\n if (i >= lo) {\n dst->push_back(i);\n }\n // Break the loop here since multiplying by\n // 'mult' would move outside of the range of T\n if (i > kmax / mult) break;\n }\n\n return dst->begin() + static_cast(start_offset);\n}\n\ntemplate \nvoid AddNegatedPowers(std::vector* dst, T lo, T hi, int mult) {\n // We negate lo and hi so we require that they cannot be equal to 'min'.\n BM_CHECK_GT(lo, std::numeric_limits::min());\n BM_CHECK_GT(hi, std::numeric_limits::min());\n BM_CHECK_GE(hi, lo);\n BM_CHECK_LE(hi, 0);\n\n // Add positive powers, then negate and reverse.\n // Casts necessary since small integers get promoted\n // to 'int' when negating.\n const auto lo_complement = static_cast(-lo);\n const auto hi_complement = static_cast(-hi);\n\n const auto it = AddPowers(dst, hi_complement, lo_complement, mult);\n\n std::for_each(it, dst->end(), [](T& t) { t = static_cast(t * -1); });\n std::reverse(it, dst->end());\n}\n\ntemplate \nvoid AddRange(std::vector* dst, T lo, T hi, int mult) {\n static_assert(std::is_integral::value && std::is_signed::value,\n \"Args type must be a signed integer\");\n\n BM_CHECK_GE(hi, lo);\n BM_CHECK_GE(mult, 2);\n\n // Add \"lo\"\n dst->push_back(lo);\n\n // Handle lo == hi as a special case, so we then know\n // lo < hi and so it is safe to add 1 to lo and subtract 1\n // from hi without falling outside of the range of T.\n if (lo == hi) return;\n\n // Ensure that lo_inner <= hi_inner below.\n if (lo + 1 == hi) {\n dst->push_back(hi);\n return;\n }\n\n // Add all powers of 'mult' in the range [lo+1, hi-1] (inclusive).\n const auto lo_inner = static_cast(lo + 1);\n const auto hi_inner = static_cast(hi - 1);\n\n // Insert negative values\n if (lo_inner < 0) {\n AddNegatedPowers(dst, lo_inner, std::min(hi_inner, T{-1}), mult);\n }\n\n // Treat 0 as a special case (see discussion on #762).\n if (lo < 0 && hi >= 0) {\n dst->push_back(0);\n }\n\n // Insert positive values\n if (hi_inner > 0) {\n AddPowers(dst, std::max(lo_inner, T{1}), hi_inner, mult);\n }\n\n // Add \"hi\" (if different from last value).\n if (hi != dst->back()) {\n dst->push_back(hi);\n }\n}\n\n} // namespace internal\n} // namespace benchmark\n\n#endif // BENCHMARK_REGISTER_H\n" }, { "path": "src/benchmark_runner.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"benchmark_runner.h\"\n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/managers.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/types.h\"\n#include \"benchmark_api_internal.h\"\n#include \"internal_macros.h\"\n\n#ifndef BENCHMARK_OS_WINDOWS\n#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \\\n !defined(BENCHMARK_OS_WASI)\n#include \n#endif\n#include \n#include \n#endif\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"check.h\"\n#include \"colorprint.h\"\n#include \"commandlineflags.h\"\n#include \"complexity.h\"\n#include \"counter.h\"\n#include \"log.h\"\n#include \"mutex.h\"\n#include \"perf_counters.h\"\n#include \"re.h\"\n#include \"statistics.h\"\n#include \"string_util.h\"\n#include \"thread_manager.h\"\n#include \"thread_timer.h\"\n\nnamespace benchmark {\n\nBM_DECLARE_bool(benchmark_dry_run);\nBM_DECLARE_string(benchmark_min_time);\nBM_DECLARE_double(benchmark_min_warmup_time);\nBM_DECLARE_int32(benchmark_repetitions);\nBM_DECLARE_bool(benchmark_report_aggregates_only);\nBM_DECLARE_bool(benchmark_display_aggregates_only);\nBM_DECLARE_string(benchmark_perf_counters);\n\nnamespace internal {\n\nMemoryManager* memory_manager = nullptr;\n\nProfilerManager* profiler_manager = nullptr;\n\nnamespace {\n\nconstexpr IterationCount kMaxIterations = 1000000000000;\nconst double kDefaultMinTime =\n std::strtod(::benchmark::kDefaultMinTimeStr, /*p_end*/ nullptr);\n\nBenchmarkReporter::Run CreateRunReport(\n const benchmark::internal::BenchmarkInstance& b,\n const internal::ThreadManager::Result& results,\n IterationCount memory_iterations,\n const MemoryManager::Result& memory_result, double seconds,\n int64_t repetition_index, int64_t repeats) {\n // Create report about this benchmark run.\n BenchmarkReporter::Run report;\n\n report.run_name = b.name();\n report.family_index = b.family_index();\n report.per_family_instance_index = b.per_family_instance_index();\n report.skipped = results.skipped_;\n report.skip_message = results.skip_message_;\n report.report_label = results.report_label_;\n // This is the total iterations across all threads.\n report.iterations = results.iterations;\n report.time_unit = b.time_unit();\n report.threads = b.threads();\n report.repetition_index = repetition_index;\n report.repetitions = repeats;\n\n if (report.skipped == 0u) {\n if (b.use_manual_time()) {\n report.real_accumulated_time = results.manual_time_used;\n } else {\n report.real_accumulated_time = results.real_time_used;\n }\n report.use_real_time_for_initial_big_o = b.use_manual_time();\n report.cpu_accumulated_time = results.cpu_time_used;\n report.complexity_n = results.complexity_n;\n report.complexity = b.complexity();\n report.complexity_lambda = b.complexity_lambda();\n report.statistics = &b.statistics();\n report.counters = results.counters;\n\n if (memory_iterations > 0) {\n report.memory_result = memory_result;\n report.allocs_per_iter =\n memory_iterations != 0\n ? static_cast(memory_result.num_allocs) /\n static_cast(memory_iterations)\n : 0;\n }\n\n // The CPU time is the total time taken by all thread. If we used that as\n // the denominator, we'd be calculating the rate per thread here. This is\n // why we have to divide the total cpu_time by the number of threads for\n // global counters to get a global rate.\n const double thread_seconds = seconds / b.threads();\n internal::Finish(&report.counters, results.iterations, thread_seconds,\n b.threads());\n }\n return report;\n}\n\n// Execute one thread of benchmark b for the specified number of iterations.\n// Adds the stats collected for the thread into manager->results.\nvoid RunInThread(const BenchmarkInstance* b, IterationCount iters,\n int thread_id, ThreadManager* manager,\n PerfCountersMeasurement* perf_counters_measurement,\n ProfilerManager* profiler_manager_) {\n internal::ThreadTimer timer(\n b->measure_process_cpu_time()\n ? internal::ThreadTimer::CreateProcessCpuTime()\n : internal::ThreadTimer::Create());\n\n State st = b->Run(iters, thread_id, &timer, manager,\n perf_counters_measurement, profiler_manager_);\n if (!(st.skipped() || st.iterations() >= st.max_iterations)) {\n st.SkipWithError(\n \"The benchmark didn't run, nor was it explicitly skipped. Please call \"\n \"'SkipWithXXX` in your benchmark as appropriate.\");\n }\n {\n MutexLock l(manager->GetBenchmarkMutex());\n internal::ThreadManager::Result& results = manager->results;\n results.iterations += st.iterations();\n results.cpu_time_used += timer.cpu_time_used();\n results.real_time_used += timer.real_time_used();\n results.manual_time_used += timer.manual_time_used();\n results.complexity_n += st.complexity_length_n();\n internal::Increment(&results.counters, st.counters);\n }\n manager->NotifyThreadComplete();\n}\n\ndouble ComputeMinTime(const benchmark::internal::BenchmarkInstance& b,\n const BenchTimeType& iters_or_time) {\n if (!IsZero(b.min_time())) {\n return b.min_time();\n }\n // If the flag was used to specify number of iters, then return the default\n // min_time.\n if (iters_or_time.tag == BenchTimeType::ITERS) {\n return kDefaultMinTime;\n }\n\n return iters_or_time.time;\n}\n\nIterationCount ComputeIters(const benchmark::internal::BenchmarkInstance& b,\n const BenchTimeType& iters_or_time) {\n if (b.iterations() != 0) {\n return b.iterations();\n }\n\n // We've already concluded that this flag is currently used to pass\n // iters but do a check here again anyway.\n BM_CHECK(iters_or_time.tag == BenchTimeType::ITERS);\n return iters_or_time.iters;\n}\n\nclass ThreadRunnerDefault : public ThreadRunnerBase {\n public:\n explicit ThreadRunnerDefault(int num_threads)\n : pool(static_cast(num_threads - 1)) {}\n\n void RunThreads(const std::function& fn) override final {\n // Run all but one thread in separate threads\n for (std::size_t ti = 0; ti < pool.size(); ++ti) {\n pool[ti] = std::thread(fn, static_cast(ti + 1));\n }\n // And run one thread here directly.\n // (If we were asked to run just one thread, we don't create new threads.)\n // Yes, we need to do this here *after* we start the separate threads.\n fn(0);\n\n // The main thread has finished. Now let's wait for the other threads.\n for (std::thread& thread : pool) {\n thread.join();\n }\n }\n\n private:\n std::vector pool;\n};\n\nstd::unique_ptr GetThreadRunner(\n const benchmark::threadrunner_factory& userThreadRunnerFactory,\n int num_threads) {\n return userThreadRunnerFactory\n ? userThreadRunnerFactory(num_threads)\n : std::make_unique(num_threads);\n}\n\n} // end namespace\n\nBenchTimeType ParseBenchMinTime(const std::string& value) {\n BenchTimeType ret = {};\n\n if (value.empty()) {\n ret.tag = BenchTimeType::TIME;\n ret.time = 0.0;\n return ret;\n }\n\n if (value.back() == 'x') {\n char* p_end = nullptr;\n // Reset errno before it's changed by strtol.\n errno = 0;\n IterationCount num_iters = std::strtol(value.c_str(), &p_end, 10);\n\n // After a valid parse, p_end should have been set to\n // point to the 'x' suffix.\n BM_CHECK(errno == 0 && p_end != nullptr && *p_end == 'x')\n << \"Malformed iters value passed to --benchmark_min_time: `\" << value\n << \"`. Expected --benchmark_min_time=x.\";\n\n ret.tag = BenchTimeType::ITERS;\n ret.iters = num_iters;\n return ret;\n }\n\n bool has_suffix = value.back() == 's';\n if (!has_suffix) {\n BM_VLOG(0) << \"Value passed to --benchmark_min_time should have a suffix. \"\n \"Eg., `30s` for 30-seconds.\";\n }\n\n char* p_end = nullptr;\n // Reset errno before it's changed by strtod.\n errno = 0;\n double min_time = std::strtod(value.c_str(), &p_end);\n\n // After a successful parse, p_end should point to the suffix 's',\n // or the end of the string if the suffix was omitted.\n BM_CHECK(errno == 0 && p_end != nullptr &&\n ((has_suffix && *p_end == 's') || *p_end == '\\0'))\n << \"Malformed seconds value passed to --benchmark_min_time: `\" << value\n << \"`. Expected --benchmark_min_time=x.\";\n\n ret.tag = BenchTimeType::TIME;\n ret.time = min_time;\n\n return ret;\n}\n\nBenchmarkRunner::BenchmarkRunner(\n const benchmark::internal::BenchmarkInstance& b_,\n PerfCountersMeasurement* pcm_,\n BenchmarkReporter::PerFamilyRunReports* reports_for_family_)\n : b(b_),\n reports_for_family(reports_for_family_),\n parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)),\n min_time(FLAGS_benchmark_dry_run\n ? 0\n : ComputeMinTime(b_, parsed_benchtime_flag)),\n min_warmup_time(\n FLAGS_benchmark_dry_run\n ? 0\n : ((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0)\n ? b.min_warmup_time()\n : FLAGS_benchmark_min_warmup_time)),\n warmup_done(FLAGS_benchmark_dry_run ? true : !(min_warmup_time > 0.0)),\n repeats(FLAGS_benchmark_dry_run\n ? 1\n : (b.repetitions() != 0 ? b.repetitions()\n : FLAGS_benchmark_repetitions)),\n has_explicit_iteration_count(b.iterations() != 0 ||\n parsed_benchtime_flag.tag ==\n BenchTimeType::ITERS),\n thread_runner(\n GetThreadRunner(b.GetUserThreadRunnerFactory(), b.threads())),\n iters(FLAGS_benchmark_dry_run\n ? 1\n : (has_explicit_iteration_count\n ? ComputeIters(b_, parsed_benchtime_flag)\n : 1)),\n perf_counters_measurement_ptr(pcm_) {\n run_results.display_report_aggregates_only =\n (FLAGS_benchmark_report_aggregates_only ||\n FLAGS_benchmark_display_aggregates_only);\n run_results.file_report_aggregates_only =\n FLAGS_benchmark_report_aggregates_only;\n if (b.aggregation_report_mode() != internal::ARM_Unspecified) {\n run_results.display_report_aggregates_only =\n ((b.aggregation_report_mode() &\n internal::ARM_DisplayReportAggregatesOnly) != 0u);\n run_results.file_report_aggregates_only =\n ((b.aggregation_report_mode() &\n internal::ARM_FileReportAggregatesOnly) != 0u);\n BM_CHECK(FLAGS_benchmark_perf_counters.empty() ||\n (perf_counters_measurement_ptr->num_counters() == 0))\n << \"Perf counters were requested but could not be set up.\";\n }\n}\n\nBenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() {\n BM_VLOG(2) << \"Running \" << b.name().str() << \" for \" << iters << \"\\n\";\n\n std::unique_ptr manager;\n manager.reset(new internal::ThreadManager(b.threads()));\n\n thread_runner->RunThreads([&](int thread_idx) {\n RunInThread(&b, iters, thread_idx, manager.get(),\n perf_counters_measurement_ptr, /*profiler_manager=*/nullptr);\n });\n\n IterationResults i;\n // Acquire the measurements/counters from the manager, UNDER THE LOCK!\n {\n MutexLock l(manager->GetBenchmarkMutex());\n i.results = manager->results;\n }\n\n // And get rid of the manager.\n manager.reset();\n\n BM_VLOG(2) << \"Ran in \" << i.results.cpu_time_used << \"/\"\n << i.results.real_time_used << \"\\n\";\n\n // By using KeepRunningBatch a benchmark can iterate more times than\n // requested, so take the iteration count from i.results.\n i.iters = i.results.iterations / b.threads();\n\n // Base decisions off of real time if requested by this benchmark.\n i.seconds = i.results.cpu_time_used;\n if (b.use_manual_time()) {\n i.seconds = i.results.manual_time_used;\n } else if (b.use_real_time()) {\n i.seconds = i.results.real_time_used;\n }\n\n return i;\n}\n\nIterationCount BenchmarkRunner::PredictNumItersNeeded(\n const IterationResults& i) const {\n // See how much iterations should be increased by.\n // Note: Avoid division by zero with max(seconds, 1ns).\n double multiplier = GetMinTimeToApply() * 1.4 / std::max(i.seconds, 1e-9);\n // If our last run was at least 10% of FLAGS_benchmark_min_time then we\n // use the multiplier directly.\n // Otherwise we use at most 10 times expansion.\n // NOTE: When the last run was at least 10% of the min time the max\n // expansion should be 14x.\n const bool is_significant = (i.seconds / GetMinTimeToApply()) > 0.1;\n multiplier = is_significant ? multiplier : 10.0;\n\n // So what seems to be the sufficiently-large iteration count? Round up.\n const IterationCount max_next_iters = static_cast(\n std::llround(std::max(multiplier * static_cast(i.iters),\n static_cast(i.iters) + 1.0)));\n // But we do have *some* limits though..\n const IterationCount next_iters = std::min(max_next_iters, kMaxIterations);\n\n BM_VLOG(3) << \"Next iters: \" << next_iters << \", \" << multiplier << \"\\n\";\n return next_iters; // round up before conversion to integer.\n}\n\nbool BenchmarkRunner::ShouldReportIterationResults(\n const IterationResults& i) const {\n // Determine if this run should be reported;\n // Either it has run for a sufficient amount of time\n // or because an error was reported.\n return (i.results.skipped_ != 0u) || FLAGS_benchmark_dry_run ||\n i.iters >= kMaxIterations || // Too many iterations already.\n i.seconds >=\n GetMinTimeToApply() || // The elapsed time is large enough.\n // CPU time is specified but the elapsed real time greatly exceeds\n // the minimum time.\n // Note that user provided timers are except from this test.\n ((i.results.real_time_used >= 5 * GetMinTimeToApply()) &&\n !b.use_manual_time());\n}\n\ndouble BenchmarkRunner::GetMinTimeToApply() const {\n // In order to reuse functionality to run and measure benchmarks for running\n // a warmup phase of the benchmark, we need a way of telling whether to apply\n // min_time or min_warmup_time. This function will figure out if we are in the\n // warmup phase and therefore need to apply min_warmup_time or if we already\n // in the benchmarking phase and min_time needs to be applied.\n return warmup_done ? min_time : min_warmup_time;\n}\n\nvoid BenchmarkRunner::FinishWarmUp(const IterationCount& i) {\n warmup_done = true;\n iters = i;\n}\n\nvoid BenchmarkRunner::RunWarmUp() {\n // Use the same mechanisms for warming up the benchmark as used for actually\n // running and measuring the benchmark.\n IterationResults i_warmup;\n // Dont use the iterations determined in the warmup phase for the actual\n // measured benchmark phase. While this may be a good starting point for the\n // benchmark and it would therefore get rid of the need to figure out how many\n // iterations are needed if min_time is set again, this may also be a complete\n // wrong guess since the warmup loops might be considerably slower (e.g\n // because of caching effects).\n const IterationCount i_backup = iters;\n\n for (;;) {\n b.Setup();\n i_warmup = DoNIterations();\n b.Teardown();\n\n const bool finish = ShouldReportIterationResults(i_warmup);\n\n if (finish) {\n FinishWarmUp(i_backup);\n break;\n }\n\n // Although we are running \"only\" a warmup phase where running enough\n // iterations at once without measuring time isn't as important as it is for\n // the benchmarking phase, we still do it the same way as otherwise it is\n // very confusing for the user to know how to choose a proper value for\n // min_warmup_time if a different approach on running it is used.\n iters = PredictNumItersNeeded(i_warmup);\n assert(iters > i_warmup.iters &&\n \"if we did more iterations than we want to do the next time, \"\n \"then we should have accepted the current iteration run.\");\n }\n}\n\nMemoryManager::Result BenchmarkRunner::RunMemoryManager(\n IterationCount memory_iterations) {\n memory_manager->Start();\n std::unique_ptr manager;\n manager.reset(new internal::ThreadManager(1));\n b.Setup();\n RunInThread(&b, memory_iterations, 0, manager.get(),\n perf_counters_measurement_ptr,\n /*profiler_manager=*/nullptr);\n manager.reset();\n b.Teardown();\n MemoryManager::Result memory_result;\n memory_manager->Stop(memory_result);\n memory_result.memory_iterations = memory_iterations;\n return memory_result;\n}\n\nvoid BenchmarkRunner::RunProfilerManager(IterationCount profile_iterations) {\n std::unique_ptr manager;\n manager.reset(new internal::ThreadManager(1));\n b.Setup();\n RunInThread(&b, profile_iterations, 0, manager.get(),\n /*perf_counters_measurement_ptr=*/nullptr,\n /*profiler_manager=*/profiler_manager);\n manager.reset();\n b.Teardown();\n}\n\nvoid BenchmarkRunner::DoOneRepetition() {\n assert(HasRepeatsRemaining() && \"Already done all repetitions?\");\n\n const bool is_the_first_repetition = num_repetitions_done == 0;\n\n // In case a warmup phase is requested by the benchmark, run it now.\n // After running the warmup phase the BenchmarkRunner should be in a state as\n // this warmup never happened except the fact that warmup_done is set. Every\n // other manipulation of the BenchmarkRunner instance would be a bug! Please\n // fix it.\n if (!warmup_done) {\n RunWarmUp();\n }\n\n IterationResults i;\n // We *may* be gradually increasing the length (iteration count)\n // of the benchmark until we decide the results are significant.\n // And once we do, we report those last results and exit.\n // Please do note that the if there are repetitions, the iteration count\n // is *only* calculated for the *first* repetition, and other repetitions\n // simply use that precomputed iteration count.\n for (;;) {\n b.Setup();\n i = DoNIterations();\n b.Teardown();\n\n // Do we consider the results to be significant?\n // If we are doing repetitions, and the first repetition was already done,\n // it has calculated the correct iteration time, so we have run that very\n // iteration count just now. No need to calculate anything. Just report.\n // Else, the normal rules apply.\n const bool results_are_significant = !is_the_first_repetition ||\n has_explicit_iteration_count ||\n ShouldReportIterationResults(i);\n // Good, let's report them!\n if (results_are_significant) {\n break;\n }\n\n // Nope, bad iteration. Let's re-estimate the hopefully-sufficient\n // iteration count, and run the benchmark again...\n\n iters = PredictNumItersNeeded(i);\n assert(iters > i.iters &&\n \"if we did more iterations than we want to do the next time, \"\n \"then we should have accepted the current iteration run.\");\n }\n\n // Produce memory measurements if requested.\n MemoryManager::Result memory_result;\n IterationCount memory_iterations = 0;\n if (memory_manager != nullptr) {\n // Only run a few iterations to reduce the impact of one-time\n // allocations in benchmarks that are not properly managed.\n memory_iterations = std::min(16, iters);\n memory_result = RunMemoryManager(memory_iterations);\n }\n\n if (profiler_manager != nullptr) {\n // We want to externally profile the benchmark for the same number of\n // iterations because, for example, if we're tracing the benchmark then we\n // want trace data to reasonably match PMU data.\n RunProfilerManager(iters);\n }\n\n // Ok, now actually report.\n BenchmarkReporter::Run report =\n CreateRunReport(b, i.results, memory_iterations, memory_result, i.seconds,\n num_repetitions_done, repeats);\n\n if (reports_for_family != nullptr) {\n ++reports_for_family->num_runs_done;\n if (report.skipped == 0u) {\n reports_for_family->Runs.push_back(report);\n }\n }\n\n run_results.non_aggregates.push_back(report);\n\n ++num_repetitions_done;\n}\n\nRunResults&& BenchmarkRunner::GetResults() {\n assert(!HasRepeatsRemaining() && \"Did not run all repetitions yet?\");\n\n // Calculate additional statistics over the repetitions of this instance.\n run_results.aggregates_only = ComputeStats(run_results.non_aggregates);\n\n return std::move(run_results);\n}\n\n} // end namespace internal\n\n} // end namespace benchmark\n" }, { "path": "src/benchmark_runner.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_RUNNER_H_\n#define BENCHMARK_RUNNER_H_\n\n#include \n#include \n#include \n\n#include \"benchmark_api_internal.h\"\n#include \"perf_counters.h\"\n#include \"thread_manager.h\"\n\nnamespace benchmark {\n\nnamespace internal {\n\nextern MemoryManager* memory_manager;\nextern ProfilerManager* profiler_manager;\n\nstruct RunResults {\n std::vector non_aggregates;\n std::vector aggregates_only;\n\n bool display_report_aggregates_only = false;\n bool file_report_aggregates_only = false;\n};\n\nstruct BENCHMARK_EXPORT BenchTimeType {\n enum { UNSPECIFIED, ITERS, TIME } tag;\n union {\n IterationCount iters;\n double time;\n };\n};\n\nBENCHMARK_EXPORT\nBenchTimeType ParseBenchMinTime(const std::string& value);\n\nclass BenchmarkRunner {\n public:\n BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_,\n benchmark::internal::PerfCountersMeasurement* pcm_,\n BenchmarkReporter::PerFamilyRunReports* reports_for_family);\n\n int GetNumRepeats() const { return repeats; }\n\n bool HasRepeatsRemaining() const {\n return GetNumRepeats() != num_repetitions_done;\n }\n\n void DoOneRepetition();\n\n RunResults&& GetResults();\n\n BenchmarkReporter::PerFamilyRunReports* GetReportsForFamily() const {\n return reports_for_family;\n }\n\n double GetMinTime() const { return min_time; }\n\n bool HasExplicitIters() const { return has_explicit_iteration_count; }\n\n IterationCount GetIters() const { return iters; }\n\n private:\n RunResults run_results;\n\n const benchmark::internal::BenchmarkInstance& b;\n BenchmarkReporter::PerFamilyRunReports* reports_for_family;\n\n BenchTimeType parsed_benchtime_flag;\n const double min_time;\n const double min_warmup_time;\n bool warmup_done;\n const int repeats;\n const bool has_explicit_iteration_count;\n\n int num_repetitions_done = 0;\n\n std::unique_ptr thread_runner;\n\n IterationCount iters; // preserved between repetitions!\n // So only the first repetition has to find/calculate it,\n // the other repetitions will just use that precomputed iteration count.\n\n PerfCountersMeasurement* const perf_counters_measurement_ptr = nullptr;\n\n struct IterationResults {\n internal::ThreadManager::Result results;\n IterationCount iters;\n double seconds;\n };\n IterationResults DoNIterations();\n\n MemoryManager::Result RunMemoryManager(IterationCount memory_iterations);\n\n void RunProfilerManager(IterationCount profile_iterations);\n\n IterationCount PredictNumItersNeeded(const IterationResults& i) const;\n\n bool ShouldReportIterationResults(const IterationResults& i) const;\n\n double GetMinTimeToApply() const;\n\n void FinishWarmUp(const IterationCount& i);\n\n void RunWarmUp();\n};\n\n} // namespace internal\n\n} // end namespace benchmark\n\n#endif // BENCHMARK_RUNNER_H_\n" }, { "path": "src/check.cc", "content": "#include \"check.h\"\n\nnamespace benchmark {\nnamespace internal {\n\nnamespace {\n// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)\nAbortHandlerT* handler = &std::abort;\n} // namespace\n\nBENCHMARK_EXPORT AbortHandlerT*& GetAbortHandler() { return handler; }\n\n} // namespace internal\n} // namespace benchmark\n" }, { "path": "src/check.h", "content": "#ifndef CHECK_H_\n#define CHECK_H_\n\n#include \n#include \n#include \n#include \n\n#include \"benchmark/export.h\"\n#include \"internal_macros.h\"\n#include \"log.h\"\n\n#if defined(__GNUC__) || defined(__clang__)\n#define BENCHMARK_NOEXCEPT noexcept\n#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x)\n#elif defined(_MSC_VER) && !defined(__clang__)\n#if _MSC_VER >= 1900\n#define BENCHMARK_NOEXCEPT noexcept\n#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x)\n#else\n#define BENCHMARK_NOEXCEPT\n#define BENCHMARK_NOEXCEPT_OP(x)\n#endif\n#define __func__ __FUNCTION__\n#else\n#define BENCHMARK_NOEXCEPT\n#define BENCHMARK_NOEXCEPT_OP(x)\n#endif\n\nnamespace benchmark {\nnamespace internal {\n\ntypedef void(AbortHandlerT)();\n\nBENCHMARK_EXPORT\nAbortHandlerT*& GetAbortHandler();\n\nBENCHMARK_NORETURN inline void CallAbortHandler() {\n GetAbortHandler()();\n std::flush(std::cout);\n std::flush(std::cerr);\n std::abort(); // fallback to enforce noreturn\n}\n\n// CheckHandler is the class constructed by failing BM_CHECK macros.\n// CheckHandler will log information about the failures and abort when it is\n// destructed.\nclass CheckHandler {\n public:\n CheckHandler(std::string_view check, std::string_view file,\n std::string_view func, int line)\n : log_(GetErrorLogInstance()) {\n log_ << file << \":\" << line << \": \" << func << \": Check `\" << check\n << \"' failed. \";\n }\n\n LogType& GetLog() { return log_; }\n\n#if defined(COMPILER_MSVC)\n#pragma warning(push)\n#pragma warning(disable : 4722)\n#endif\n BENCHMARK_NORETURN ~CheckHandler() BENCHMARK_NOEXCEPT_OP(false) {\n log_ << '\\n';\n CallAbortHandler();\n }\n#if defined(COMPILER_MSVC)\n#pragma warning(pop)\n#endif\n\n CheckHandler& operator=(const CheckHandler&) = delete;\n CheckHandler(const CheckHandler&) = delete;\n CheckHandler() = delete;\n\n private:\n LogType& log_;\n};\n\n} // end namespace internal\n} // end namespace benchmark\n\n// The BM_CHECK macro returns a std::ostream object that can have extra\n// information written to it.\n#ifndef NDEBUG\n#define BM_CHECK(b) \\\n (b ? ::benchmark::internal::GetNullLogInstance() \\\n : ::benchmark::internal::CheckHandler( \\\n std::string_view(#b), std::string_view(__FILE__), \\\n std::string_view(__func__), __LINE__) \\\n .GetLog())\n#else\n#define BM_CHECK(b) ::benchmark::internal::GetNullLogInstance()\n#endif\n\n// clang-format off\n// preserve whitespacing between operators for alignment\n#define BM_CHECK_EQ(a, b) BM_CHECK((a) == (b))\n#define BM_CHECK_NE(a, b) BM_CHECK((a) != (b))\n#define BM_CHECK_GE(a, b) BM_CHECK((a) >= (b))\n#define BM_CHECK_LE(a, b) BM_CHECK((a) <= (b))\n#define BM_CHECK_GT(a, b) BM_CHECK((a) > (b))\n#define BM_CHECK_LT(a, b) BM_CHECK((a) < (b))\n\n#define BM_CHECK_FLOAT_EQ(a, b, eps) BM_CHECK(std::fabs((a) - (b)) < (eps))\n#define BM_CHECK_FLOAT_NE(a, b, eps) BM_CHECK(std::fabs((a) - (b)) >= (eps))\n#define BM_CHECK_FLOAT_GE(a, b, eps) BM_CHECK((a) - (b) > -(eps))\n#define BM_CHECK_FLOAT_LE(a, b, eps) BM_CHECK((b) - (a) > -(eps))\n#define BM_CHECK_FLOAT_GT(a, b, eps) BM_CHECK((a) - (b) > (eps))\n#define BM_CHECK_FLOAT_LT(a, b, eps) BM_CHECK((b) - (a) > (eps))\n//clang-format on\n\n#endif // CHECK_H_\n" }, { "path": "src/colorprint.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"colorprint.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"check.h\"\n#include \"internal_macros.h\"\n\n#ifdef BENCHMARK_OS_WINDOWS\n#include \n#include \n#else\n#include \n#endif // BENCHMARK_OS_WINDOWS\n\nnamespace benchmark {\nnamespace {\n#ifdef BENCHMARK_OS_WINDOWS\ntypedef WORD PlatformColorCode;\n#else\ntypedef const char* PlatformColorCode;\n#endif\n\nPlatformColorCode GetPlatformColorCode(LogColor color) {\n#ifdef BENCHMARK_OS_WINDOWS\n switch (color) {\n case COLOR_RED:\n return FOREGROUND_RED;\n case COLOR_GREEN:\n return FOREGROUND_GREEN;\n case COLOR_YELLOW:\n return FOREGROUND_RED | FOREGROUND_GREEN;\n case COLOR_BLUE:\n return FOREGROUND_BLUE;\n case COLOR_MAGENTA:\n return FOREGROUND_BLUE | FOREGROUND_RED;\n case COLOR_CYAN:\n return FOREGROUND_BLUE | FOREGROUND_GREEN;\n case COLOR_WHITE: // fall through to default\n default:\n return 0;\n }\n#else\n switch (color) {\n case COLOR_RED:\n return \"1\";\n case COLOR_GREEN:\n return \"2\";\n case COLOR_YELLOW:\n return \"3\";\n case COLOR_BLUE:\n return \"4\";\n case COLOR_MAGENTA:\n return \"5\";\n case COLOR_CYAN:\n return \"6\";\n case COLOR_WHITE:\n return \"7\";\n default:\n return nullptr;\n };\n#endif\n}\n\n} // end namespace\n\nstd::string FormatString(const char* msg, va_list args) {\n // we might need a second shot at this, so pre-emptivly make a copy\n va_list args_cp;\n va_copy(args_cp, args);\n\n std::size_t size = 256;\n char local_buff[256];\n auto ret = vsnprintf(local_buff, size, msg, args_cp);\n\n va_end(args_cp);\n\n // currently there is no error handling for failure, so this is hack.\n BM_CHECK(ret >= 0);\n\n if (ret == 0) { // handle empty expansion\n return {};\n }\n if (static_cast(ret) < size) {\n return local_buff;\n }\n // we did not provide a long enough buffer on our first attempt.\n size = static_cast(ret) + 1; // + 1 for the null byte\n std::unique_ptr buff(new char[size]);\n va_list args_cp2;\n va_copy(args_cp2, args);\n ret = vsnprintf(buff.get(), size, msg, args_cp2);\n va_end(args_cp2);\n BM_CHECK(ret > 0 && (static_cast(ret)) < size);\n return buff.get();\n}\n\nstd::string FormatString(const char* msg, ...) {\n va_list args;\n va_start(args, msg);\n auto tmp = FormatString(msg, args);\n va_end(args);\n return tmp;\n}\n\nvoid ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...) {\n va_list args;\n va_start(args, fmt);\n ColorPrintf(out, color, fmt, args);\n va_end(args);\n}\n\nvoid ColorPrintf(std::ostream& out, LogColor color, const char* fmt,\n va_list args) {\n#ifdef BENCHMARK_OS_WINDOWS\n ((void)out); // suppress unused warning\n\n const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);\n\n // Gets the current text color.\n CONSOLE_SCREEN_BUFFER_INFO buffer_info;\n GetConsoleScreenBufferInfo(stdout_handle, &buffer_info);\n const WORD original_color_attrs = buffer_info.wAttributes;\n\n // We need to flush the stream buffers into the console before each\n // SetConsoleTextAttribute call lest it affect the text that is already\n // printed but has not yet reached the console.\n out.flush();\n\n const WORD original_background_attrs =\n original_color_attrs & (BACKGROUND_RED | BACKGROUND_GREEN |\n BACKGROUND_BLUE | BACKGROUND_INTENSITY);\n\n SetConsoleTextAttribute(stdout_handle, GetPlatformColorCode(color) |\n FOREGROUND_INTENSITY |\n original_background_attrs);\n out << FormatString(fmt, args);\n\n out.flush();\n // Restores the text and background color.\n SetConsoleTextAttribute(stdout_handle, original_color_attrs);\n#else\n const char* color_code = GetPlatformColorCode(color);\n if (color_code != nullptr) {\n out << FormatString(\"\\033[0;3%sm\", color_code);\n }\n out << FormatString(fmt, args) << \"\\033[m\";\n#endif\n}\n\nbool IsColorTerminal() {\n#if BENCHMARK_OS_WINDOWS\n // On Windows the TERM variable is usually not set, but the\n // console there does support colors.\n return 0 != _isatty(_fileno(stdout));\n#else\n // On non-Windows platforms, we rely on the TERM variable. This list of\n // supported TERM values is copied from Google Test:\n // .\n const char* const SUPPORTED_TERM_VALUES[] = {\n \"xterm\",\n \"xterm-color\",\n \"xterm-256color\",\n \"screen\",\n \"screen-256color\",\n \"tmux\",\n \"tmux-256color\",\n \"rxvt-unicode\",\n \"rxvt-unicode-256color\",\n \"linux\",\n \"cygwin\",\n \"xterm-kitty\",\n \"alacritty\",\n \"foot\",\n \"foot-extra\",\n \"wezterm\",\n };\n\n const char* const term = getenv(\"TERM\");\n\n bool term_supports_color = false;\n for (const char* candidate : SUPPORTED_TERM_VALUES) {\n if ((term != nullptr) && 0 == strcmp(term, candidate)) {\n term_supports_color = true;\n break;\n }\n }\n\n return 0 != isatty(fileno(stdout)) && term_supports_color;\n#endif // BENCHMARK_OS_WINDOWS\n}\n\n} // end namespace benchmark\n" }, { "path": "src/colorprint.h", "content": "#ifndef BENCHMARK_COLORPRINT_H_\n#define BENCHMARK_COLORPRINT_H_\n\n#include \n#include \n#include \n\n#include \"internal_macros.h\"\n\nnamespace benchmark {\nenum LogColor {\n COLOR_DEFAULT,\n COLOR_RED,\n COLOR_GREEN,\n COLOR_YELLOW,\n COLOR_BLUE,\n COLOR_MAGENTA,\n COLOR_CYAN,\n COLOR_WHITE\n};\n\nPRINTF_FORMAT_STRING_FUNC(1, 0)\nstd::string FormatString(const char* msg, va_list args);\nPRINTF_FORMAT_STRING_FUNC(1, 2) std::string FormatString(const char* msg, ...);\n\nPRINTF_FORMAT_STRING_FUNC(3, 0)\nvoid ColorPrintf(std::ostream& out, LogColor color, const char* fmt,\n va_list args);\nPRINTF_FORMAT_STRING_FUNC(3, 4)\nvoid ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...);\n\n// Returns true if stdout appears to be a terminal that supports colored\n// output, false otherwise.\nbool IsColorTerminal();\n\n} // end namespace benchmark\n\n#endif // BENCHMARK_COLORPRINT_H_\n" }, { "path": "src/commandlineflags.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"commandlineflags.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"../src/string_util.h\"\n\nnamespace benchmark {\nnamespace {\n\n// Parses 'str' for a 32-bit signed integer. If successful, writes\n// the result to *value and returns true; otherwise leaves *value\n// unchanged and returns false.\nbool ParseInt32(const std::string& src_text, const char* str, int32_t* value) {\n // Parses the environment variable as a decimal integer.\n char* end = nullptr;\n const long long_value = strtol(str, &end, 10); // NOLINT\n\n // Has strtol() consumed all characters in the string?\n if (*end != '\\0') {\n // No - an invalid character was encountered.\n std::cerr << src_text << \" is expected to be a 32-bit integer, \"\n << \"but actually has value \\\"\" << str << \"\\\".\\n\";\n return false;\n }\n\n // Is the parsed value in the range of an Int32?\n const int32_t result = static_cast(long_value);\n if (long_value == std::numeric_limits::max() ||\n long_value == std::numeric_limits::min() ||\n // The parsed value overflows as a long. (strtol() returns\n // LONG_MAX or LONG_MIN when the input overflows.)\n result != long_value\n // The parsed value overflows as an Int32.\n ) {\n std::cerr << src_text << \" is expected to be a 32-bit integer, \"\n << \"but actually has value \\\"\" << str << \"\\\", \"\n << \"which overflows.\\n\";\n return false;\n }\n\n *value = result;\n return true;\n}\n\n// Parses 'str' for a double. If successful, writes the result to *value and\n// returns true; otherwise leaves *value unchanged and returns false.\nbool ParseDouble(const std::string& src_text, const char* str, double* value) {\n // Parses the environment variable as a decimal integer.\n char* end = nullptr;\n const double double_value = strtod(str, &end); // NOLINT\n\n // Has strtol() consumed all characters in the string?\n if (*end != '\\0') {\n // No - an invalid character was encountered.\n std::cerr << src_text << \" is expected to be a double, \"\n << \"but actually has value \\\"\" << str << \"\\\".\\n\";\n return false;\n }\n\n *value = double_value;\n return true;\n}\n\n// Parses 'str' into KV pairs. If successful, writes the result to *value and\n// returns true; otherwise leaves *value unchanged and returns false.\nbool ParseKvPairs(const std::string& src_text, const char* str,\n std::map* value) {\n std::map kvs;\n for (const auto& kvpair : StrSplit(str, ',')) {\n const auto kv = StrSplit(kvpair, '=');\n if (kv.size() != 2) {\n std::cerr << src_text << \" is expected to be a comma-separated list of \"\n << \"= strings, but actually has value \\\"\" << str\n << \"\\\".\\n\";\n return false;\n }\n if (!kvs.emplace(kv[0], kv[1]).second) {\n std::cerr << src_text << \" is expected to contain unique keys but key \\\"\"\n << kv[0] << \"\\\" was repeated.\\n\";\n return false;\n }\n }\n\n *value = kvs;\n return true;\n}\n\n// Returns the name of the environment variable corresponding to the\n// given flag. For example, FlagToEnvVar(\"foo\") will return\n// \"BENCHMARK_FOO\" in the open-source version.\nstd::string FlagToEnvVar(const char* flag) {\n const std::string flag_str(flag);\n\n std::string env_var;\n for (size_t i = 0; i != flag_str.length(); ++i) {\n env_var += static_cast(::toupper(flag_str.c_str()[i]));\n }\n\n return env_var;\n}\n\n} // namespace\n\nBENCHMARK_EXPORT\nbool BoolFromEnv(const char* flag, bool default_val) {\n const std::string env_var = FlagToEnvVar(flag);\n const char* const value_str = getenv(env_var.c_str());\n return value_str == nullptr ? default_val : IsTruthyFlagValue(value_str);\n}\n\nBENCHMARK_EXPORT\nint32_t Int32FromEnv(const char* flag, int32_t default_val) {\n const std::string env_var = FlagToEnvVar(flag);\n const char* const value_str = getenv(env_var.c_str());\n int32_t value = default_val;\n if (value_str == nullptr ||\n !ParseInt32(std::string(\"Environment variable \") + env_var, value_str,\n &value)) {\n return default_val;\n }\n return value;\n}\n\nBENCHMARK_EXPORT\ndouble DoubleFromEnv(const char* flag, double default_val) {\n const std::string env_var = FlagToEnvVar(flag);\n const char* const value_str = getenv(env_var.c_str());\n double value = default_val;\n if (value_str == nullptr ||\n !ParseDouble(std::string(\"Environment variable \") + env_var, value_str,\n &value)) {\n return default_val;\n }\n return value;\n}\n\nBENCHMARK_EXPORT\nconst char* StringFromEnv(const char* flag, const char* default_val) {\n const std::string env_var = FlagToEnvVar(flag);\n const char* const value = getenv(env_var.c_str());\n return value == nullptr ? default_val : value;\n}\n\nBENCHMARK_EXPORT\nstd::map KvPairsFromEnv(\n const char* flag, std::map default_val) {\n const std::string env_var = FlagToEnvVar(flag);\n const char* const value_str = getenv(env_var.c_str());\n\n if (value_str == nullptr) {\n return default_val;\n }\n\n std::map value;\n if (!ParseKvPairs(\"Environment variable \" + env_var, value_str, &value)) {\n return default_val;\n }\n return value;\n}\n\nnamespace {\n\n// Parses a string as a command line flag. The string should have\n// the format \"--flag=value\". When def_optional is true, the \"=value\"\n// part can be omitted.\n//\n// Returns the value of the flag, or nullptr if the parsing failed.\nconst char* ParseFlagValue(const char* str, const char* flag,\n bool def_optional) {\n // str and flag must not be nullptr.\n if (str == nullptr || flag == nullptr) {\n return nullptr;\n }\n\n // The flag must start with \"--\".\n const std::string flag_str = std::string(\"--\") + std::string(flag);\n const size_t flag_len = flag_str.length();\n if (strncmp(str, flag_str.c_str(), flag_len) != 0) {\n return nullptr;\n }\n\n // Skips the flag name.\n const char* flag_end = str + flag_len;\n\n // When def_optional is true, it's OK to not have a \"=value\" part.\n if (def_optional && (flag_end[0] == '\\0')) {\n return flag_end;\n }\n\n // If def_optional is true and there are more characters after the\n // flag name, or if def_optional is false, there must be a '=' after\n // the flag name.\n if (flag_end[0] != '=') {\n return nullptr;\n }\n\n // Returns the string after \"=\".\n return flag_end + 1;\n}\n\n} // end namespace\n\nBENCHMARK_EXPORT\nbool ParseBoolFlag(const char* str, const char* flag, bool* value) {\n // Gets the value of the flag as a string.\n const char* const value_str = ParseFlagValue(str, flag, true);\n\n // Aborts if the parsing failed.\n if (value_str == nullptr) {\n return false;\n }\n\n // Converts the string value to a bool.\n *value = IsTruthyFlagValue(value_str);\n return true;\n}\n\nBENCHMARK_EXPORT\nbool ParseInt32Flag(const char* str, const char* flag, int32_t* value) {\n // Gets the value of the flag as a string.\n const char* const value_str = ParseFlagValue(str, flag, false);\n\n // Aborts if the parsing failed.\n if (value_str == nullptr) {\n return false;\n }\n\n // Sets *value to the value of the flag.\n return ParseInt32(std::string(\"The value of flag --\") + flag, value_str,\n value);\n}\n\nBENCHMARK_EXPORT\nbool ParseDoubleFlag(const char* str, const char* flag, double* value) {\n // Gets the value of the flag as a string.\n const char* const value_str = ParseFlagValue(str, flag, false);\n\n // Aborts if the parsing failed.\n if (value_str == nullptr) {\n return false;\n }\n\n // Sets *value to the value of the flag.\n return ParseDouble(std::string(\"The value of flag --\") + flag, value_str,\n value);\n}\n\nBENCHMARK_EXPORT\nbool ParseStringFlag(const char* str, const char* flag, std::string* value) {\n // Gets the value of the flag as a string.\n const char* const value_str = ParseFlagValue(str, flag, false);\n\n // Aborts if the parsing failed.\n if (value_str == nullptr) {\n return false;\n }\n\n *value = value_str;\n return true;\n}\n\nBENCHMARK_EXPORT\nbool ParseKeyValueFlag(const char* str, const char* flag,\n std::map* value) {\n const char* const value_str = ParseFlagValue(str, flag, false);\n\n if (value_str == nullptr) {\n return false;\n }\n\n for (const auto& kvpair : StrSplit(value_str, ',')) {\n const auto kv = StrSplit(kvpair, '=');\n if (kv.size() != 2) {\n return false;\n }\n value->emplace(kv[0], kv[1]);\n }\n\n return true;\n}\n\nBENCHMARK_EXPORT\nbool IsFlag(const char* str, const char* flag) {\n return (ParseFlagValue(str, flag, true) != nullptr);\n}\n\nBENCHMARK_EXPORT\nbool IsTruthyFlagValue(const std::string& value) {\n if (value.size() == 1) {\n char v = value[0];\n return isalnum(v) &&\n !(v == '0' || v == 'f' || v == 'F' || v == 'n' || v == 'N');\n }\n if (!value.empty()) {\n std::string value_lower(value);\n std::transform(value_lower.begin(), value_lower.end(), value_lower.begin(),\n [](char c) { return static_cast(::tolower(c)); });\n return !(value_lower == \"false\" || value_lower == \"no\" ||\n value_lower == \"off\");\n }\n return true;\n}\n\n} // end namespace benchmark\n" }, { "path": "src/commandlineflags.h", "content": "#ifndef BENCHMARK_COMMANDLINEFLAGS_H_\n#define BENCHMARK_COMMANDLINEFLAGS_H_\n\n#include \n#include \n#include \n\n#include \"benchmark/export.h\"\n\n// Macro for referencing flags.\n#define FLAG(name) FLAGS_##name\n\n// Macros for declaring flags.\n// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables)\n#define BM_DECLARE_bool(name) BENCHMARK_EXPORT extern bool FLAG(name)\n#define BM_DECLARE_int32(name) BENCHMARK_EXPORT extern int32_t FLAG(name)\n#define BM_DECLARE_double(name) BENCHMARK_EXPORT extern double FLAG(name)\n#define BM_DECLARE_string(name) BENCHMARK_EXPORT extern std::string FLAG(name)\n#define BM_DECLARE_kvpairs(name) \\\n BENCHMARK_EXPORT extern std::map FLAG(name)\n// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables)\n\n// Macros for defining flags.\n// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables)\n#define BM_DEFINE_bool(name, default_val) \\\n BENCHMARK_EXPORT bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val)\n#define BM_DEFINE_int32(name, default_val) \\\n BENCHMARK_EXPORT int32_t FLAG(name) = \\\n benchmark::Int32FromEnv(#name, default_val)\n#define BM_DEFINE_double(name, default_val) \\\n BENCHMARK_EXPORT double FLAG(name) = \\\n benchmark::DoubleFromEnv(#name, default_val)\n#define BM_DEFINE_string(name, default_val) \\\n BENCHMARK_EXPORT std::string FLAG(name) = \\\n benchmark::StringFromEnv(#name, default_val)\n#define BM_DEFINE_kvpairs(name, default_val) \\\n BENCHMARK_EXPORT std::map FLAG(name) = \\\n benchmark::KvPairsFromEnv(#name, default_val)\n// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables)\n\nnamespace benchmark {\n\n// Parses a bool from the environment variable corresponding to the given flag.\n//\n// If the variable exists, returns IsTruthyFlagValue() value; if not,\n// returns the given default value.\nBENCHMARK_EXPORT\nbool BoolFromEnv(const char* flag, bool default_val);\n\n// Parses an Int32 from the environment variable corresponding to the given\n// flag.\n//\n// If the variable exists, returns ParseInt32() value; if not, returns\n// the given default value.\nBENCHMARK_EXPORT\nint32_t Int32FromEnv(const char* flag, int32_t default_val);\n\n// Parses an Double from the environment variable corresponding to the given\n// flag.\n//\n// If the variable exists, returns ParseDouble(); if not, returns\n// the given default value.\nBENCHMARK_EXPORT\ndouble DoubleFromEnv(const char* flag, double default_val);\n\n// Parses a string from the environment variable corresponding to the given\n// flag.\n//\n// If variable exists, returns its value; if not, returns\n// the given default value.\nBENCHMARK_EXPORT\nconst char* StringFromEnv(const char* flag, const char* default_val);\n\n// Parses a set of kvpairs from the environment variable corresponding to the\n// given flag.\n//\n// If variable exists, returns its value; if not, returns\n// the given default value.\nBENCHMARK_EXPORT\nstd::map KvPairsFromEnv(\n const char* flag, std::map default_val);\n\n// Parses a string for a bool flag, in the form of either\n// \"--flag=value\" or \"--flag\".\n//\n// In the former case, the value is taken as true if it passes IsTruthyValue().\n//\n// In the latter case, the value is taken as true.\n//\n// On success, stores the value of the flag in *value, and returns\n// true. On failure, returns false without changing *value.\nBENCHMARK_EXPORT\nbool ParseBoolFlag(const char* str, const char* flag, bool* value);\n\n// Parses a string for an Int32 flag, in the form of \"--flag=value\".\n//\n// On success, stores the value of the flag in *value, and returns\n// true. On failure, returns false without changing *value.\nBENCHMARK_EXPORT\nbool ParseInt32Flag(const char* str, const char* flag, int32_t* value);\n\n// Parses a string for a Double flag, in the form of \"--flag=value\".\n//\n// On success, stores the value of the flag in *value, and returns\n// true. On failure, returns false without changing *value.\nBENCHMARK_EXPORT\nbool ParseDoubleFlag(const char* str, const char* flag, double* value);\n\n// Parses a string for a string flag, in the form of \"--flag=value\".\n//\n// On success, stores the value of the flag in *value, and returns\n// true. On failure, returns false without changing *value.\nBENCHMARK_EXPORT\nbool ParseStringFlag(const char* str, const char* flag, std::string* value);\n\n// Parses a string for a kvpairs flag in the form \"--flag=key=value,key=value\"\n//\n// On success, stores the value of the flag in *value and returns true. On\n// failure returns false, though *value may have been mutated.\nBENCHMARK_EXPORT\nbool ParseKeyValueFlag(const char* str, const char* flag,\n std::map* value);\n\n// Returns true if the string matches the flag.\nBENCHMARK_EXPORT\nbool IsFlag(const char* str, const char* flag);\n\n// Returns true unless value starts with one of: '0', 'f', 'F', 'n' or 'N', or\n// some non-alphanumeric character. Also returns false if the value matches\n// one of 'no', 'false', 'off' (case-insensitive). As a special case, also\n// returns true if value is the empty string.\nBENCHMARK_EXPORT\nbool IsTruthyFlagValue(const std::string& value);\n\n} // end namespace benchmark\n\n#endif // BENCHMARK_COMMANDLINEFLAGS_H_\n" }, { "path": "src/complexity.cc", "content": "// Copyright 2016 Ismael Jimenez Martinez. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Source project : https://github.com/ismaelJimenez/cpp.leastsq\n// Adapted to be used with google benchmark\n\n#include \"complexity.h\"\n\n#include \n\n#include \"benchmark/reporter.h\"\n#include \"benchmark/statistics.h\"\n#include \"benchmark/types.h\"\n#include \"check.h\"\n\nnamespace benchmark {\n\nnamespace {\n\n// Internal function to calculate the different scalability forms\nBigOFunc* FittingCurve(BigO complexity) {\n switch (complexity) {\n case oN:\n return [](IterationCount n) -> double { return static_cast(n); };\n case oNSquared:\n return [](IterationCount n) -> double { return std::pow(n, 2); };\n case oNCubed:\n return [](IterationCount n) -> double { return std::pow(n, 3); };\n case oLogN:\n return [](IterationCount n) -> double {\n return std::log2(static_cast(n));\n };\n case oNLogN:\n return [](IterationCount n) -> double {\n return static_cast(n) * std::log2(static_cast(n));\n };\n case o1:\n default:\n return [](IterationCount) { return 1.0; };\n }\n}\n\n} // end namespace\n\n// Function to return an string for the calculated complexity\nstd::string GetBigOString(BigO complexity) {\n switch (complexity) {\n case oN:\n return \"N\";\n case oNSquared:\n return \"N^2\";\n case oNCubed:\n return \"N^3\";\n case oLogN:\n return \"lgN\";\n case oNLogN:\n return \"NlgN\";\n case o1:\n return \"(1)\";\n default:\n return \"f(N)\";\n }\n}\n\nnamespace {\n\n// Find the coefficient for the high-order term in the running time, by\n// minimizing the sum of squares of relative error, for the fitting curve\n// given by the lambda expression.\n// - n : Vector containing the size of the benchmark tests.\n// - time : Vector containing the times for the benchmark tests.\n// - fitting_curve : lambda expression (e.g. [](ComplexityN n) {return n; };).\n\n// For a deeper explanation on the algorithm logic, please refer to\n// https://en.wikipedia.org/wiki/Least_squares#Least_squares,_regression_analysis_and_statistics\n\nLeastSq MinimalLeastSq(const std::vector& n,\n const std::vector& time,\n BigOFunc* fitting_curve) {\n double sigma_gn_squared = 0.0;\n double sigma_time = 0.0;\n double sigma_time_gn = 0.0;\n\n // Calculate least square fitting parameter\n for (size_t i = 0; i < n.size(); ++i) {\n double gn_i = fitting_curve(n[i]);\n sigma_gn_squared += gn_i * gn_i;\n sigma_time += time[i];\n sigma_time_gn += time[i] * gn_i;\n }\n\n LeastSq result;\n result.complexity = oLambda;\n\n // Calculate complexity.\n result.coef = sigma_time_gn / sigma_gn_squared;\n\n // Calculate RMS\n double rms = 0.0;\n for (size_t i = 0; i < n.size(); ++i) {\n double fit = result.coef * fitting_curve(n[i]);\n rms += std::pow((time[i] - fit), 2);\n }\n\n // Normalized RMS by the mean of the observed values\n double mean = sigma_time / static_cast(n.size());\n result.rms = std::sqrt(rms / static_cast(n.size())) / mean;\n\n return result;\n}\n\n// Find the coefficient for the high-order term in the running time, by\n// minimizing the sum of squares of relative error.\n// - n : Vector containing the size of the benchmark tests.\n// - time : Vector containing the times for the benchmark tests.\n// - complexity : If different than oAuto, the fitting curve will stick to\n// this one. If it is oAuto, it will be calculated the best\n// fitting curve.\nLeastSq MinimalLeastSq(const std::vector& n,\n const std::vector& time, const BigO complexity) {\n BM_CHECK_EQ(n.size(), time.size());\n BM_CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two\n // benchmark runs are given\n BM_CHECK_NE(complexity, oNone);\n\n LeastSq best_fit;\n\n if (complexity == oAuto) {\n std::vector fit_curves = {oLogN, oN, oNLogN, oNSquared, oNCubed};\n\n // Take o1 as default best fitting curve\n best_fit = MinimalLeastSq(n, time, FittingCurve(o1));\n best_fit.complexity = o1;\n\n // Compute all possible fitting curves and stick to the best one\n for (const auto& fit : fit_curves) {\n LeastSq current_fit = MinimalLeastSq(n, time, FittingCurve(fit));\n if (current_fit.rms < best_fit.rms) {\n best_fit = current_fit;\n best_fit.complexity = fit;\n }\n }\n } else {\n best_fit = MinimalLeastSq(n, time, FittingCurve(complexity));\n best_fit.complexity = complexity;\n }\n\n return best_fit;\n}\n\n} // end namespace\n\nstd::vector ComputeBigO(\n const std::vector& reports) {\n typedef BenchmarkReporter::Run Run;\n std::vector results;\n\n if (reports.size() < 2) {\n return results;\n }\n\n // Accumulators.\n std::vector n;\n std::vector real_time;\n std::vector cpu_time;\n\n // Populate the accumulators.\n for (const Run& run : reports) {\n BM_CHECK_GT(run.complexity_n, 0)\n << \"Did you forget to call SetComplexityN?\";\n n.push_back(run.complexity_n);\n real_time.push_back(run.real_accumulated_time /\n static_cast(run.iterations));\n cpu_time.push_back(run.cpu_accumulated_time /\n static_cast(run.iterations));\n }\n\n LeastSq result_cpu;\n LeastSq result_real;\n\n if (reports[0].complexity == oLambda) {\n result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda);\n result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda);\n } else {\n const BigO* InitialBigO = &reports[0].complexity;\n const bool use_real_time_for_initial_big_o =\n reports[0].use_real_time_for_initial_big_o;\n if (use_real_time_for_initial_big_o) {\n result_real = MinimalLeastSq(n, real_time, *InitialBigO);\n InitialBigO = &result_real.complexity;\n // The Big-O complexity for CPU time must have the same Big-O function!\n }\n result_cpu = MinimalLeastSq(n, cpu_time, *InitialBigO);\n InitialBigO = &result_cpu.complexity;\n if (!use_real_time_for_initial_big_o) {\n result_real = MinimalLeastSq(n, real_time, *InitialBigO);\n }\n }\n\n // Drop the 'args' when reporting complexity.\n auto run_name = reports[0].run_name;\n run_name.args.clear();\n\n // Get the data from the accumulator to BenchmarkReporter::Run's.\n Run big_o;\n big_o.run_name = run_name;\n big_o.family_index = reports[0].family_index;\n big_o.per_family_instance_index = reports[0].per_family_instance_index;\n big_o.run_type = BenchmarkReporter::Run::RT_Aggregate;\n big_o.repetitions = reports[0].repetitions;\n big_o.repetition_index = Run::no_repetition_index;\n big_o.threads = reports[0].threads;\n big_o.aggregate_name = \"BigO\";\n big_o.aggregate_unit = StatisticUnit::kTime;\n big_o.report_label = reports[0].report_label;\n big_o.iterations = 0;\n big_o.real_accumulated_time = result_real.coef;\n big_o.cpu_accumulated_time = result_cpu.coef;\n big_o.report_big_o = true;\n big_o.complexity = result_cpu.complexity;\n\n // All the time results are reported after being multiplied by the\n // time unit multiplier. But since RMS is a relative quantity it\n // should not be multiplied at all. So, here, we _divide_ it by the\n // multiplier so that when it is multiplied later the result is the\n // correct one.\n double multiplier = GetTimeUnitMultiplier(reports[0].time_unit);\n\n // Only add label to mean/stddev if it is same for all runs\n Run rms;\n rms.run_name = run_name;\n rms.family_index = reports[0].family_index;\n rms.per_family_instance_index = reports[0].per_family_instance_index;\n rms.run_type = BenchmarkReporter::Run::RT_Aggregate;\n rms.aggregate_name = \"RMS\";\n rms.aggregate_unit = StatisticUnit::kPercentage;\n rms.report_label = big_o.report_label;\n rms.iterations = 0;\n rms.repetition_index = Run::no_repetition_index;\n rms.repetitions = reports[0].repetitions;\n rms.threads = reports[0].threads;\n rms.real_accumulated_time = result_real.rms / multiplier;\n rms.cpu_accumulated_time = result_cpu.rms / multiplier;\n rms.report_rms = true;\n rms.complexity = result_cpu.complexity;\n // don't forget to keep the time unit, or we won't be able to\n // recover the correct value.\n rms.time_unit = reports[0].time_unit;\n\n results.push_back(big_o);\n results.push_back(rms);\n return results;\n}\n\n} // end namespace benchmark\n" }, { "path": "src/complexity.h", "content": "// Copyright 2016 Ismael Jimenez Martinez. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Source project : https://github.com/ismaelJimenez/cpp.leastsq\n// Adapted to be used with google benchmark\n\n#ifndef COMPLEXITY_H_\n#define COMPLEXITY_H_\n\n#include \n#include \n\n#include \"benchmark/reporter.h\"\n#include \"benchmark/statistics.h\"\n\nnamespace benchmark {\n\n// Return a vector containing the bigO and RMS information for the specified\n// list of reports. If 'reports.size() < 2' an empty vector is returned.\nstd::vector ComputeBigO(\n const std::vector& reports);\n\n// This data structure will contain the result returned by MinimalLeastSq\n// - coef : Estimated coefficient for the high-order term as\n// interpolated from data.\n// - rms : Normalized Root Mean Squared Error.\n// - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability\n// form has been provided to MinimalLeastSq this will return\n// the same value. In case BigO::oAuto has been selected, this\n// parameter will return the best fitting curve detected.\n\nstruct LeastSq {\n LeastSq() : coef(0.0), rms(0.0), complexity(oNone) {}\n\n double coef;\n double rms;\n BigO complexity;\n};\n\n// Function to return an string for the calculated complexity\nstd::string GetBigOString(BigO complexity);\n\n} // end namespace benchmark\n\n#endif // COMPLEXITY_H_\n" }, { "path": "src/console_reporter.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/export.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/types.h\"\n#include \"check.h\"\n#include \"colorprint.h\"\n#include \"commandlineflags.h\"\n#include \"complexity.h\"\n#include \"counter.h\"\n#include \"internal_macros.h\"\n#include \"string_util.h\"\n#include \"timers.h\"\n\nnamespace benchmark {\n\nBENCHMARK_EXPORT\nbool ConsoleReporter::ReportContext(const Context& context) {\n name_field_width_ = context.name_field_width;\n printed_header_ = false;\n prev_counters_.clear();\n\n PrintBasicContext(&GetErrorStream(), context);\n\n#ifdef BENCHMARK_OS_WINDOWS\n if ((output_options_ & OO_Color)) {\n auto stdOutBuf = std::cout.rdbuf();\n auto outStreamBuf = GetOutputStream().rdbuf();\n if (stdOutBuf != outStreamBuf) {\n GetErrorStream()\n << \"Color printing is only supported for stdout on windows.\"\n \" Disabling color printing\\n\";\n output_options_ = static_cast(output_options_ & ~OO_Color);\n }\n }\n#endif\n\n return true;\n}\n\nBENCHMARK_EXPORT\nvoid ConsoleReporter::PrintHeader(const Run& run) {\n std::string str =\n FormatString(\"%-*s %13s %15s %12s\", static_cast(name_field_width_),\n \"Benchmark\", \"Time\", \"CPU\", \"Iterations\");\n if (!run.counters.empty()) {\n if ((output_options_ & OO_Tabular) != 0) {\n for (auto const& c : run.counters) {\n str += FormatString(\" %10s\", c.first.c_str());\n }\n } else {\n str += \" UserCounters...\";\n }\n }\n std::string line = std::string(str.length(), '-');\n GetOutputStream() << line << \"\\n\" << str << \"\\n\" << line << \"\\n\";\n}\n\nBENCHMARK_EXPORT\nvoid ConsoleReporter::ReportRuns(const std::vector& reports) {\n for (const auto& run : reports) {\n // print the header:\n // --- if none was printed yet\n bool print_header = !printed_header_;\n // --- or if the format is tabular and this run\n // has different fields from the prev header\n print_header |= ((output_options_ & OO_Tabular) != 0) &&\n (!internal::SameNames(run.counters, prev_counters_));\n if (print_header) {\n printed_header_ = true;\n prev_counters_ = run.counters;\n PrintHeader(run);\n }\n // As an alternative to printing the headers like this, we could sort\n // the benchmarks by header and then print. But this would require\n // waiting for the full results before printing, or printing twice.\n PrintRunData(run);\n }\n}\n\nPRINTF_FORMAT_STRING_FUNC(3, 4)\nstatic void IgnoreColorPrint(std::ostream& out, LogColor /*unused*/,\n const char* fmt, ...) {\n va_list args;\n va_start(args, fmt);\n out << FormatString(fmt, args);\n va_end(args);\n}\n\nstatic std::string FormatTime(double time) {\n // For the time columns of the console printer 13 digits are reserved. One of\n // them is a space and max two of them are the time unit (e.g ns). That puts\n // us at 10 digits usable for the number.\n // Align decimal places...\n if (time < 1.0) {\n return FormatString(\"%10.3f\", time);\n }\n if (time < 10.0) {\n return FormatString(\"%10.2f\", time);\n }\n if (time < 100.0) {\n return FormatString(\"%10.1f\", time);\n }\n // Assuming the time is at max 9.9999e+99 and we have 10 digits for the\n // number, we get 10-1(.)-1(e)-1(sign)-2(exponent) = 5 digits to print.\n if (time > 9999999999 /*max 10 digit number*/) {\n return FormatString(\"%1.4e\", time);\n }\n return FormatString(\"%10.0f\", time);\n}\n\nBENCHMARK_EXPORT\nvoid ConsoleReporter::PrintRunData(const Run& result) {\n typedef void(PrinterFn)(std::ostream&, LogColor, const char*, ...);\n auto& Out = GetOutputStream();\n PrinterFn* printer = (output_options_ & OO_Color) != 0\n ? static_cast(ColorPrintf)\n : IgnoreColorPrint;\n auto name_color =\n (result.report_big_o || result.report_rms) ? COLOR_BLUE : COLOR_GREEN;\n printer(Out, name_color, \"%-*s \", static_cast(name_field_width_),\n result.benchmark_name().c_str());\n\n if (internal::SkippedWithError == result.skipped) {\n printer(Out, COLOR_RED, \"ERROR OCCURRED: \\'%s\\'\",\n result.skip_message.c_str());\n printer(Out, COLOR_DEFAULT, \"\\n\");\n return;\n }\n if (internal::SkippedWithMessage == result.skipped) {\n printer(Out, COLOR_WHITE, \"SKIPPED: \\'%s\\'\", result.skip_message.c_str());\n printer(Out, COLOR_DEFAULT, \"\\n\");\n return;\n }\n\n const double real_time = result.GetAdjustedRealTime();\n const double cpu_time = result.GetAdjustedCPUTime();\n const std::string real_time_str = FormatTime(real_time);\n const std::string cpu_time_str = FormatTime(cpu_time);\n\n if (result.report_big_o) {\n std::string big_o = GetBigOString(result.complexity);\n printer(Out, COLOR_YELLOW, \"%10.2f %-4s %10.2f %-4s \", real_time,\n big_o.c_str(), cpu_time, big_o.c_str());\n } else if (result.report_rms) {\n printer(Out, COLOR_YELLOW, \"%10.0f %-4s %10.0f %-4s \", real_time * 100, \"%\",\n cpu_time * 100, \"%\");\n } else if (result.run_type != Run::RT_Aggregate ||\n result.aggregate_unit == StatisticUnit::kTime) {\n const char* timeLabel = GetTimeUnitString(result.time_unit);\n printer(Out, COLOR_YELLOW, \"%s %-4s %s %-4s \", real_time_str.c_str(),\n timeLabel, cpu_time_str.c_str(), timeLabel);\n } else {\n assert(result.aggregate_unit == StatisticUnit::kPercentage);\n printer(Out, COLOR_YELLOW, \"%10.2f %-4s %10.2f %-4s \",\n (100. * result.real_accumulated_time), \"%\",\n (100. * result.cpu_accumulated_time), \"%\");\n }\n\n if (!result.report_big_o && !result.report_rms) {\n printer(Out, COLOR_CYAN, \"%10lld\", result.iterations);\n }\n\n for (const auto& c : result.counters) {\n const std::size_t cNameLen =\n std::max(static_cast(10), c.first.length());\n std::string s;\n const char* unit = \"\";\n if (result.run_type == Run::RT_Aggregate &&\n result.aggregate_unit == StatisticUnit::kPercentage) {\n s = StrFormat(\"%.2f\", 100. * c.second.value);\n unit = \"%\";\n } else {\n s = HumanReadableNumber(c.second.value, c.second.oneK);\n if ((c.second.flags & Counter::kIsRate) != 0) {\n unit = (c.second.flags & Counter::kInvert) != 0 ? \"s\" : \"/s\";\n }\n }\n if ((output_options_ & OO_Tabular) != 0) {\n printer(Out, COLOR_DEFAULT, \" %*s%s\",\n static_cast(cNameLen - strlen(unit)), s.c_str(), unit);\n } else {\n printer(Out, COLOR_DEFAULT, \" %s=%s%s\", c.first.c_str(), s.c_str(), unit);\n }\n }\n\n if (!result.report_label.empty()) {\n printer(Out, COLOR_DEFAULT, \" %s\", result.report_label.c_str());\n }\n\n printer(Out, COLOR_DEFAULT, \"\\n\");\n}\n\n} // end namespace benchmark\n" }, { "path": "src/counter.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"counter.h\"\n\nnamespace benchmark {\nnamespace internal {\n\nnamespace {\n\ndouble Finish(Counter const& c, IterationCount iterations, double cpu_time,\n double num_threads) {\n double v = c.value;\n if ((c.flags & Counter::kIsRate) != 0) {\n v /= cpu_time;\n }\n if ((c.flags & Counter::kAvgThreads) != 0) {\n v /= num_threads;\n }\n if ((c.flags & Counter::kIsIterationInvariant) != 0) {\n v *= static_cast(iterations);\n }\n if ((c.flags & Counter::kAvgIterations) != 0) {\n v /= static_cast(iterations);\n }\n\n if ((c.flags & Counter::kInvert) != 0) { // Invert is *always* last.\n v = 1.0 / v;\n }\n return v;\n}\n\n} // namespace\n\nvoid Finish(UserCounters* l, IterationCount iterations, double cpu_time,\n double num_threads) {\n for (auto& c : *l) {\n c.second.value = Finish(c.second, iterations, cpu_time, num_threads);\n }\n}\n\nvoid Increment(UserCounters* l, UserCounters const& r) {\n // add counters present in both or just in *l\n for (auto& c : *l) {\n auto it = r.find(c.first);\n if (it != r.end()) {\n c.second.value = c.second + it->second;\n }\n }\n // add counters present in r, but not in *l\n for (auto const& tc : r) {\n auto it = l->find(tc.first);\n if (it == l->end()) {\n (*l)[tc.first] = tc.second;\n }\n }\n}\n\nbool SameNames(UserCounters const& l, UserCounters const& r) {\n if (&l == &r) {\n return true;\n }\n if (l.size() != r.size()) {\n return false;\n }\n for (auto const& c : l) {\n if (r.find(c.first) == r.end()) {\n return false;\n }\n }\n return true;\n}\n\n} // end namespace internal\n} // end namespace benchmark\n" }, { "path": "src/counter.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_SRC_COUNTER_H_\n#define BENCHMARK_SRC_COUNTER_H_\n\n#include \"benchmark/counter.h\"\n#include \"benchmark/export.h\"\n#include \"benchmark/types.h\"\n\nnamespace benchmark {\n\nnamespace internal {\nvoid Finish(UserCounters* l, IterationCount iterations, double time,\n double num_threads);\nvoid Increment(UserCounters* l, UserCounters const& r);\nbool SameNames(UserCounters const& l, UserCounters const& r);\n} // end namespace internal\n\n} // end namespace benchmark\n\n#endif // BENCHMARK_SRC_COUNTER_H_\n" }, { "path": "src/csv_reporter.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \n#include \n#include \n\n#include \"benchmark/export.h\"\n#include \"benchmark/reporter.h\"\n#include \"check.h\"\n#include \"complexity.h\"\n\n// File format reference: http://edoceo.com/utilitas/csv-file-format.\n\nnamespace benchmark {\n\nnamespace {\nconst std::vector elements = {\n \"name\", \"iterations\", \"real_time\", \"cpu_time\",\n \"time_unit\", \"bytes_per_second\", \"items_per_second\", \"label\",\n \"error_occurred\", \"error_message\"};\n\nstd::string CsvEscape(const std::string& s) {\n std::string tmp;\n tmp.reserve(s.size() + 2);\n for (char c : s) {\n switch (c) {\n case '\"':\n tmp += \"\\\"\\\"\";\n break;\n default:\n tmp += c;\n break;\n }\n }\n return '\"' + tmp + '\"';\n}\n} // namespace\n\nBENCHMARK_EXPORT\nbool CSVReporter::ReportContext(const Context& context) {\n PrintBasicContext(&GetErrorStream(), context);\n return true;\n}\n\nBENCHMARK_EXPORT\nvoid CSVReporter::ReportRuns(const std::vector& reports) {\n std::ostream& Out = GetOutputStream();\n\n if (!printed_header_) {\n // save the names of all the user counters\n for (const auto& run : reports) {\n for (const auto& cnt : run.counters) {\n if (cnt.first == \"bytes_per_second\" ||\n cnt.first == \"items_per_second\") {\n continue;\n }\n user_counter_names_.insert(cnt.first);\n }\n }\n\n // print the header\n for (auto B = elements.begin(); B != elements.end();) {\n Out << *B++;\n if (B != elements.end()) {\n Out << \",\";\n }\n }\n for (auto B = user_counter_names_.begin();\n B != user_counter_names_.end();) {\n Out << \",\\\"\" << *B++ << \"\\\"\";\n }\n Out << \"\\n\";\n\n printed_header_ = true;\n } else {\n // check that all the current counters are saved in the name set\n for (const auto& run : reports) {\n for (const auto& cnt : run.counters) {\n if (cnt.first == \"bytes_per_second\" ||\n cnt.first == \"items_per_second\") {\n continue;\n }\n BM_CHECK(user_counter_names_.find(cnt.first) !=\n user_counter_names_.end())\n << \"All counters must be present in each run. \"\n << \"Counter named \\\"\" << cnt.first\n << \"\\\" was not in a run after being added to the header\";\n }\n }\n }\n\n // print results for each run\n for (const auto& run : reports) {\n PrintRunData(run);\n }\n}\n\nBENCHMARK_EXPORT\nvoid CSVReporter::PrintRunData(const Run& run) {\n std::ostream& Out = GetOutputStream();\n Out << CsvEscape(run.benchmark_name()) << \",\";\n if (run.skipped != 0u) {\n Out << std::string(elements.size() - 3, ',');\n Out << std::boolalpha << (internal::SkippedWithError == run.skipped) << \",\";\n Out << CsvEscape(run.skip_message) << \"\\n\";\n return;\n }\n\n // Do not print iteration on bigO and RMS report\n if (!run.report_big_o && !run.report_rms) {\n Out << run.iterations;\n }\n Out << \",\";\n\n if (run.run_type != Run::RT_Aggregate ||\n run.aggregate_unit == StatisticUnit::kTime) {\n Out << run.GetAdjustedRealTime() << \",\";\n Out << run.GetAdjustedCPUTime() << \",\";\n } else {\n assert(run.aggregate_unit == StatisticUnit::kPercentage);\n Out << run.real_accumulated_time << \",\";\n Out << run.cpu_accumulated_time << \",\";\n }\n\n // Do not print timeLabel on bigO and RMS report\n if (run.report_big_o) {\n Out << GetBigOString(run.complexity);\n } else if (!run.report_rms &&\n run.aggregate_unit != StatisticUnit::kPercentage) {\n Out << GetTimeUnitString(run.time_unit);\n }\n Out << \",\";\n\n if (run.counters.find(\"bytes_per_second\") != run.counters.end()) {\n Out << run.counters.at(\"bytes_per_second\");\n }\n Out << \",\";\n if (run.counters.find(\"items_per_second\") != run.counters.end()) {\n Out << run.counters.at(\"items_per_second\");\n }\n Out << \",\";\n if (!run.report_label.empty()) {\n Out << CsvEscape(run.report_label);\n }\n Out << \",,\"; // for error_occurred and error_message\n\n // Print user counters\n for (const auto& ucn : user_counter_names_) {\n auto it = run.counters.find(ucn);\n if (it == run.counters.end()) {\n Out << \",\";\n } else {\n Out << \",\" << it->second;\n }\n }\n Out << '\\n';\n}\n\n} // end namespace benchmark\n" }, { "path": "src/cycleclock.h", "content": "// ----------------------------------------------------------------------\n// CycleClock\n// A CycleClock tells you the current time in Cycles. The \"time\"\n// is actually time since power-on. This is like time() but doesn't\n// involve a system call and is much more precise.\n//\n// NOTE: Not all cpu/platform/kernel combinations guarantee that this\n// clock increments at a constant rate or is synchronized across all logical\n// cpus in a system.\n//\n// If you need the above guarantees, please consider using a different\n// API. There are efforts to provide an interface which provides a millisecond\n// granularity and implemented as a memory read. A memory read is generally\n// cheaper than the CycleClock for many architectures.\n//\n// Also, in some out of order CPU implementations, the CycleClock is not\n// serializing. So if you're trying to count at cycles granularity, your\n// data might be inaccurate due to out of order instruction execution.\n// ----------------------------------------------------------------------\n\n#ifndef BENCHMARK_CYCLECLOCK_H_\n#define BENCHMARK_CYCLECLOCK_H_\n\n#include \n\n#include \"benchmark/macros.h\"\n#include \"internal_macros.h\"\n\n#if defined(BENCHMARK_OS_MACOSX)\n#include \n#endif\n// For MSVC, we want to use '_asm rdtsc' when possible (since it works\n// with even ancient MSVC compilers), and when not possible the\n// __rdtsc intrinsic, declared in . Unfortunately, in some\n// environments, and have conflicting\n// declarations of some other intrinsics, breaking compilation.\n// Therefore, we simply declare __rdtsc ourselves. See also\n// http://connect.microsoft.com/VisualStudio/feedback/details/262047\n//\n// Note that MSVC defines the x64 preprocessor macros when building\n// for Arm64EC, despite it using Arm64 assembly instructions.\n#if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64) && \\\n !defined(_M_ARM64EC)\nextern \"C\" uint64_t __rdtsc();\n#pragma intrinsic(__rdtsc)\n#endif\n\n#if !defined(BENCHMARK_OS_WINDOWS) || defined(BENCHMARK_OS_MINGW)\n#include \n#include \n#endif\n\n#ifdef BENCHMARK_OS_EMSCRIPTEN\n#include \n#endif\n\nnamespace benchmark {\n// NOTE: only i386 and x86_64 have been well tested.\n// PPC, sparc, alpha, and ia64 are based on\n// http://peter.kuscsik.com/wordpress/?p=14\n// with modifications by m3b. See also\n// https://setisvn.ssl.berkeley.edu/svn/lib/fftw-3.0.1/kernel/cycle.h\nnamespace cycleclock {\n// This should return the number of cycles since power-on. Thread-safe.\ninline BENCHMARK_ALWAYS_INLINE int64_t Now() {\n#if defined(BENCHMARK_OS_MACOSX)\n // this goes at the top because we need ALL Macs, regardless of\n // architecture, to return the number of \"mach time units\" that\n // have passed since startup. See sysinfo.cc where\n // InitializeSystemInfo() sets the supposed cpu clock frequency of\n // macs to the number of mach time units per second, not actual\n // CPU clock frequency (which can change in the face of CPU\n // frequency scaling). Also note that when the Mac sleeps, this\n // counter pauses; it does not continue counting, nor does it\n // reset to zero.\n return static_cast(mach_absolute_time());\n#elif defined(BENCHMARK_OS_EMSCRIPTEN)\n // this goes above x86-specific code because old versions of Emscripten\n // define __x86_64__, although they have nothing to do with it.\n return static_cast(emscripten_get_now() * 1e+6);\n#elif defined(__i386__)\n int64_t ret;\n __asm__ volatile(\"rdtsc\" : \"=A\"(ret));\n return ret;\n\n// Note that Clang, like MSVC, defines the x64 preprocessor macros when building\n// for Arm64EC, despite it using Arm64 assembly instructions.\n#elif (defined(__x86_64__) || defined(__amd64__)) && !defined(__arm64ec__)\n uint64_t low, high;\n __asm__ volatile(\"rdtsc\" : \"=a\"(low), \"=d\"(high));\n return static_cast((high << 32) | low);\n#elif defined(__powerpc__) || defined(__ppc__)\n // This returns a time-base, which is not always precisely a cycle-count.\n#if defined(__powerpc64__) || defined(__ppc64__)\n int64_t tb;\n asm volatile(\"mfspr %0, 268\" : \"=r\"(tb));\n return tb;\n#else\n uint32_t tbl, tbu0, tbu1;\n asm volatile(\n \"mftbu %0\\n\"\n \"mftb %1\\n\"\n \"mftbu %2\"\n : \"=r\"(tbu0), \"=r\"(tbl), \"=r\"(tbu1));\n tbl &= -static_cast(tbu0 == tbu1);\n // high 32 bits in tbu1; low 32 bits in tbl (tbu0 is no longer needed)\n return (static_cast(tbu1) << 32) | tbl;\n#endif\n#elif defined(__sparc__)\n int64_t tick;\n asm(\".byte 0x83, 0x41, 0x00, 0x00\");\n asm(\"mov %%g1, %0\" : \"=r\"(tick));\n return tick;\n#elif defined(__ia64__)\n int64_t itc;\n asm(\"mov %0 = ar.itc\" : \"=r\"(itc));\n return itc;\n#elif defined(COMPILER_MSVC) && defined(_M_IX86)\n // Older MSVC compilers (like 7.x) don't seem to support the\n // __rdtsc intrinsic properly, so I prefer to use _asm instead\n // when I know it will work. Otherwise, I'll use __rdtsc and hope\n // the code is being compiled with a non-ancient compiler.\n _asm rdtsc\n#elif defined(COMPILER_MSVC) && (defined(_M_ARM64) || defined(_M_ARM64EC))\n // See // https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics\n // and https://reviews.llvm.org/D53115\n int64_t virtual_timer_value;\n virtual_timer_value = _ReadStatusReg(ARM64_CNTVCT);\n return virtual_timer_value;\n#elif defined(COMPILER_MSVC)\n return __rdtsc();\n#elif defined(BENCHMARK_OS_NACL)\n // Native Client validator on x86/x86-64 allows RDTSC instructions,\n // and this case is handled above. Native Client validator on ARM\n // rejects MRC instructions (used in the ARM-specific sequence below),\n // so we handle it here. Portable Native Client compiles to\n // architecture-agnostic bytecode, which doesn't provide any\n // cycle counter access mnemonics.\n\n // Native Client does not provide any API to access cycle counter.\n // Use clock_gettime(CLOCK_MONOTONIC, ...) instead of gettimeofday\n // because is provides nanosecond resolution (which is noticeable at\n // least for PNaCl modules running on x86 Mac & Linux).\n // Initialize to always return 0 if clock_gettime fails.\n struct timespec ts = {0, 0};\n clock_gettime(CLOCK_MONOTONIC, &ts);\n return static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec;\n#elif defined(__aarch64__) || defined(__arm64ec__)\n // System timer of ARMv8 runs at a different frequency than the CPU's.\n // The frequency is fixed, typically in the range 1-50MHz. It can be\n // read at CNTFRQ special register. We assume the OS has set up\n // the virtual timer properly.\n int64_t virtual_timer_value;\n asm volatile(\"mrs %0, cntvct_el0\" : \"=r\"(virtual_timer_value));\n return virtual_timer_value;\n#elif defined(__ARM_ARCH)\n // V6 is the earliest arch that has a standard cyclecount\n // Native Client validator doesn't allow MRC instructions.\n#if (__ARM_ARCH >= 6)\n uint32_t pmccntr;\n uint32_t pmuseren;\n uint32_t pmcntenset;\n // Read the user mode perf monitor counter access permissions.\n asm volatile(\"mrc p15, 0, %0, c9, c14, 0\" : \"=r\"(pmuseren));\n if (pmuseren & 1) { // Allows reading perfmon counters for user mode code.\n asm volatile(\"mrc p15, 0, %0, c9, c12, 1\" : \"=r\"(pmcntenset));\n if (pmcntenset & 0x80000000ul) { // Is it counting?\n asm volatile(\"mrc p15, 0, %0, c9, c13, 0\" : \"=r\"(pmccntr));\n // The counter is set up to count every 64th cycle\n return static_cast(pmccntr) * 64; // Should optimize to << 6\n }\n }\n#endif\n struct timeval tv;\n gettimeofday(&tv, nullptr);\n return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec;\n#elif defined(__mips__) || defined(__m68k__)\n // mips apparently only allows rdtsc for superusers, so we fall\n // back to gettimeofday. It's possible clock_gettime would be better.\n struct timeval tv;\n gettimeofday(&tv, nullptr);\n return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec;\n#elif defined(__loongarch__) || defined(__csky__)\n struct timeval tv;\n gettimeofday(&tv, nullptr);\n return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec;\n#elif defined(__s390__) // Covers both s390 and s390x.\n // Return the CPU clock.\n uint64_t tsc;\n#if defined(BENCHMARK_OS_ZOS)\n // z/OS HLASM syntax.\n asm(\" stck %0\" : \"=m\"(tsc) : : \"cc\");\n#else\n // Linux on Z syntax.\n asm(\"stck %0\" : \"=Q\"(tsc) : : \"cc\");\n#endif\n return tsc;\n#elif defined(__riscv) // RISC-V\n // Use RDTIME (and RDTIMEH on riscv32).\n // RDCYCLE is a privileged instruction since Linux 6.6.\n#if __riscv_xlen == 32\n uint32_t cycles_lo, cycles_hi0, cycles_hi1;\n // This asm also includes the PowerPC overflow handling strategy, as above.\n // Implemented in assembly because Clang insisted on branching.\n asm volatile(\n \"rdtimeh %0\\n\"\n \"rdtime %1\\n\"\n \"rdtimeh %2\\n\"\n \"sub %0, %0, %2\\n\"\n \"seqz %0, %0\\n\"\n \"sub %0, zero, %0\\n\"\n \"and %1, %1, %0\\n\"\n : \"=r\"(cycles_hi0), \"=r\"(cycles_lo), \"=r\"(cycles_hi1));\n return static_cast((static_cast(cycles_hi1) << 32) |\n cycles_lo);\n#else\n uint64_t cycles;\n asm volatile(\"rdtime %0\" : \"=r\"(cycles));\n return static_cast(cycles);\n#endif\n#elif defined(__e2k__) || defined(__elbrus__)\n struct timeval tv;\n gettimeofday(&tv, nullptr);\n return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec;\n#elif defined(__hexagon__)\n uint64_t pcycle;\n asm volatile(\"%0 = C15:14\" : \"=r\"(pcycle));\n return static_cast(pcycle);\n#elif defined(__alpha__)\n // Alpha has a cycle counter, the PCC register, but it is an unsigned 32-bit\n // integer and thus wraps every ~4s, making using it for tick counts\n // unreliable beyond this time range. The real-time clock is low-precision,\n // roughtly ~1ms, but it is the only option that can reasonable count\n // indefinitely.\n struct timeval tv;\n gettimeofday(&tv, nullptr);\n return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec;\n#elif defined(__hppa__) || defined(__linux__) || defined(BENCHMARK_OS_WASI)\n // Fallback for all other architectures with a recent Linux kernel, e.g.:\n // HP PA-RISC provides a user-readable clock counter (cr16), but\n // it's not syncronized across CPUs and only 32-bit wide when programs\n // are built as 32-bit binaries.\n // Same for SH-4 and possibly others.\n // Use clock_gettime(CLOCK_MONOTONIC, ...) instead of gettimeofday\n // because is provides nanosecond resolution.\n // Initialize to always return 0 if clock_gettime fails.\n struct timespec ts = {0, 0};\n clock_gettime(CLOCK_MONOTONIC, &ts);\n return static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec;\n#else\n // The soft failover to a generic implementation is automatic only for ARM.\n // For other platforms the developer is expected to make an attempt to create\n // a fast implementation and use generic version if nothing better is\n // available.\n#error You need to define CycleTimer for your OS and CPU\n#endif\n}\n} // end namespace cycleclock\n} // end namespace benchmark\n\n#endif // BENCHMARK_CYCLECLOCK_H_\n" }, { "path": "src/internal_macros.h", "content": "#ifndef BENCHMARK_INTERNAL_MACROS_H_\n#define BENCHMARK_INTERNAL_MACROS_H_\n\n/* Needed to detect STL */\n#include \n\n// clang-format off\n\n#ifndef __has_feature\n#define __has_feature(x) 0\n#endif\n\n#if defined(__clang__)\n #if !defined(COMPILER_CLANG)\n #define COMPILER_CLANG\n #endif\n#elif defined(_MSC_VER)\n #if !defined(COMPILER_MSVC)\n #define COMPILER_MSVC\n #endif\n#elif defined(__GNUC__)\n #if !defined(COMPILER_GCC)\n #define COMPILER_GCC\n #endif\n#endif\n\n#if __has_feature(cxx_attributes)\n #define BENCHMARK_NORETURN [[noreturn]]\n#elif defined(__GNUC__)\n #define BENCHMARK_NORETURN __attribute__((noreturn))\n#elif defined(COMPILER_MSVC)\n #define BENCHMARK_NORETURN __declspec(noreturn)\n#else\n #define BENCHMARK_NORETURN\n#endif\n\n#if defined(__CYGWIN__)\n #define BENCHMARK_OS_CYGWIN 1\n#elif defined(_WIN32)\n #define BENCHMARK_OS_WINDOWS 1\n // WINAPI_FAMILY_PARTITION is defined in winapifamily.h.\n // We include windows.h which implicitly includes winapifamily.h for compatibility.\n #ifndef NOMINMAX\n #define NOMINMAX\n #endif\n #include \n #if defined(WINAPI_FAMILY_PARTITION)\n #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)\n #define BENCHMARK_OS_WINDOWS_WIN32 1\n #elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP)\n #define BENCHMARK_OS_WINDOWS_RT 1\n #endif\n #endif\n #if defined(__MINGW32__)\n #define BENCHMARK_OS_MINGW 1\n #endif\n#elif defined(__APPLE__)\n #define BENCHMARK_OS_APPLE 1\n #include \"TargetConditionals.h\"\n #if defined(TARGET_OS_MAC)\n #define BENCHMARK_OS_MACOSX 1\n #if defined(TARGET_OS_IPHONE)\n #define BENCHMARK_OS_IOS 1\n #endif\n #endif\n#elif defined(__FreeBSD__)\n #define BENCHMARK_OS_FREEBSD 1\n#elif defined(__NetBSD__)\n #define BENCHMARK_OS_NETBSD 1\n#elif defined(__OpenBSD__)\n #define BENCHMARK_OS_OPENBSD 1\n#elif defined(__DragonFly__)\n #define BENCHMARK_OS_DRAGONFLY 1\n#elif defined(__linux__)\n #define BENCHMARK_OS_LINUX 1\n#elif defined(__native_client__)\n #define BENCHMARK_OS_NACL 1\n#elif defined(__EMSCRIPTEN__)\n #define BENCHMARK_OS_EMSCRIPTEN 1\n#elif defined(__wasi__)\n #define BENCHMARK_OS_WASI 1\n#elif defined(__rtems__)\n #define BENCHMARK_OS_RTEMS 1\n#elif defined(__Fuchsia__)\n#define BENCHMARK_OS_FUCHSIA 1\n#elif defined (__SVR4) && defined (__sun)\n#define BENCHMARK_OS_SOLARIS 1\n#elif defined(__QNX__)\n#define BENCHMARK_OS_QNX 1\n#elif defined(__MVS__)\n#define BENCHMARK_OS_ZOS 1\n#elif defined(__hexagon__)\n#define BENCHMARK_OS_QURT 1\n#endif\n\n#if defined(__ANDROID__) && defined(__GLIBCXX__)\n#define BENCHMARK_STL_ANDROID_GNUSTL 1\n#endif\n\n#if !__has_feature(cxx_exceptions) && !defined(__cpp_exceptions) \\\n && !defined(__EXCEPTIONS)\n #define BENCHMARK_HAS_NO_EXCEPTIONS\n#endif\n\n#if defined(COMPILER_CLANG) || defined(COMPILER_GCC)\n #define BENCHMARK_MAYBE_UNUSED __attribute__((unused))\n#else\n #define BENCHMARK_MAYBE_UNUSED\n#endif\n\n#if defined(__GNUC__) || defined(__clang__)\n#define PRINTF_FORMAT_STRING_FUNC(format_arg, first_idx) \\\n __attribute__((format(printf, format_arg, first_idx)))\n#elif defined(__MINGW32__)\n#define PRINTF_FORMAT_STRING_FUNC(format_arg, first_idx) \\\n __attribute__((format(__MINGW_PRINTF_FORMAT, format_arg, first_idx)))\n#else\n#define PRINTF_FORMAT_STRING_FUNC(format_arg, first_idx)\n#endif\n\n// clang-format on\n\n#endif // BENCHMARK_INTERNAL_MACROS_H_\n" }, { "path": "src/json_reporter.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \n#include \n#include \n#include // for setprecision\n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/export.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/types.h\"\n#include \"complexity.h\"\n#include \"string_util.h\"\n#include \"timers.h\"\n\nnamespace benchmark {\nnamespace {\n\nstd::string StrEscape(const std::string& s) {\n std::string tmp;\n tmp.reserve(s.size());\n for (char c : s) {\n switch (c) {\n case '\\b':\n tmp += \"\\\\b\";\n break;\n case '\\f':\n tmp += \"\\\\f\";\n break;\n case '\\n':\n tmp += \"\\\\n\";\n break;\n case '\\r':\n tmp += \"\\\\r\";\n break;\n case '\\t':\n tmp += \"\\\\t\";\n break;\n case '\\\\':\n tmp += \"\\\\\\\\\";\n break;\n case '\"':\n tmp += \"\\\\\\\"\";\n break;\n default:\n tmp += c;\n break;\n }\n }\n return tmp;\n}\n\nstd::string FormatKV(std::string const& key, std::string const& value) {\n return StrFormat(\"\\\"%s\\\": \\\"%s\\\"\", StrEscape(key).c_str(),\n StrEscape(value).c_str());\n}\n\nstd::string FormatKV(std::string const& key, const char* value) {\n return StrFormat(\"\\\"%s\\\": \\\"%s\\\"\", StrEscape(key).c_str(),\n StrEscape(value).c_str());\n}\n\nstd::string FormatKV(std::string const& key, bool value) {\n return StrFormat(\"\\\"%s\\\": %s\", StrEscape(key).c_str(),\n value ? \"true\" : \"false\");\n}\n\nstd::string FormatKV(std::string const& key, int64_t value) {\n std::stringstream ss;\n // We really want to just dump the integer as-is,\n // without the system locale interfering.\n ss << '\"' << StrEscape(key) << \"\\\": \" << std::to_string(value);\n return ss.str();\n}\n\nstd::string FormatKV(std::string const& key, int value) {\n return FormatKV(key, static_cast(value));\n}\n\nstd::string FormatKV(std::string const& key, double value) {\n std::stringstream ss;\n ss << '\"' << StrEscape(key) << \"\\\": \";\n\n if (std::isnan(value)) {\n ss << (value < 0 ? \"-\" : \"\") << \"NaN\";\n } else if (std::isinf(value)) {\n ss << (value < 0 ? \"-\" : \"\") << \"Infinity\";\n } else {\n const auto max_digits10 =\n std::numeric_limits::max_digits10;\n const auto max_fractional_digits10 = max_digits10 - 1;\n ss << std::scientific << std::setprecision(max_fractional_digits10)\n << value;\n }\n return ss.str();\n}\n\nint64_t RoundDouble(double v) { return std::lround(v); }\n\n} // end namespace\n\nbool JSONReporter::ReportContext(const Context& context) {\n std::ostream& out = GetOutputStream();\n\n out << \"{\\n\";\n std::string inner_indent(2, ' ');\n\n // Open context block and print context information.\n out << inner_indent << \"\\\"context\\\": {\\n\";\n std::string indent(4, ' ');\n\n std::string walltime_value = LocalDateTimeString();\n out << indent << FormatKV(\"date\", walltime_value) << \",\\n\";\n\n out << indent << FormatKV(\"host_name\", context.sys_info.name) << \",\\n\";\n\n if (Context::executable_name != nullptr) {\n out << indent << FormatKV(\"executable\", Context::executable_name) << \",\\n\";\n }\n\n CPUInfo const& info = context.cpu_info;\n out << indent << FormatKV(\"num_cpus\", static_cast(info.num_cpus))\n << \",\\n\";\n out << indent\n << FormatKV(\"mhz_per_cpu\",\n RoundDouble(info.cycles_per_second / 1000000.0))\n << \",\\n\";\n if (CPUInfo::Scaling::UNKNOWN != info.scaling) {\n out << indent\n << FormatKV(\"cpu_scaling_enabled\",\n info.scaling == CPUInfo::Scaling::ENABLED)\n << \",\\n\";\n }\n\n const SystemInfo& sysinfo = context.sys_info;\n if (SystemInfo::ASLR::UNKNOWN != sysinfo.ASLRStatus) {\n out << indent\n << FormatKV(\"aslr_enabled\",\n sysinfo.ASLRStatus == SystemInfo::ASLR::ENABLED)\n << \",\\n\";\n }\n\n out << indent << \"\\\"caches\\\": [\\n\";\n indent = std::string(6, ' ');\n std::string cache_indent(8, ' ');\n for (size_t i = 0; i < info.caches.size(); ++i) {\n const auto& CI = info.caches[i];\n out << indent << \"{\\n\";\n out << cache_indent << FormatKV(\"type\", CI.type) << \",\\n\";\n out << cache_indent << FormatKV(\"level\", static_cast(CI.level))\n << \",\\n\";\n out << cache_indent << FormatKV(\"size\", static_cast(CI.size))\n << \",\\n\";\n out << cache_indent\n << FormatKV(\"num_sharing\", static_cast(CI.num_sharing))\n << \"\\n\";\n out << indent << \"}\";\n if (i != info.caches.size() - 1) {\n out << \",\";\n }\n out << \"\\n\";\n }\n indent = std::string(4, ' ');\n out << indent << \"],\\n\";\n out << indent << \"\\\"load_avg\\\": [\";\n for (auto it = info.load_avg.begin(); it != info.load_avg.end();) {\n out << *it++;\n if (it != info.load_avg.end()) {\n out << \",\";\n }\n }\n out << \"],\\n\";\n\n out << indent << FormatKV(\"library_version\", GetBenchmarkVersion());\n out << \",\\n\";\n\n#if defined(NDEBUG)\n const char build_type[] = \"release\";\n#else\n const char build_type[] = \"debug\";\n#endif\n out << indent << FormatKV(\"library_build_type\", build_type);\n out << \",\\n\";\n\n // NOTE: our json schema is not strictly tied to the library version!\n out << indent << FormatKV(\"json_schema_version\", 1);\n\n std::map* global_context =\n internal::GetGlobalContext();\n\n if (global_context != nullptr) {\n for (const auto& kv : *global_context) {\n out << \",\\n\";\n out << indent << FormatKV(kv.first, kv.second);\n }\n }\n out << \"\\n\";\n\n // Close context block and open the list of benchmarks.\n out << inner_indent << \"},\\n\";\n out << inner_indent << \"\\\"benchmarks\\\": [\\n\";\n return true;\n}\n\nvoid JSONReporter::ReportRuns(std::vector const& reports) {\n if (reports.empty()) {\n return;\n }\n std::string indent(4, ' ');\n std::ostream& out = GetOutputStream();\n if (!first_report_) {\n out << \",\\n\";\n }\n first_report_ = false;\n\n for (auto it = reports.begin(); it != reports.end(); ++it) {\n out << indent << \"{\\n\";\n PrintRunData(*it);\n out << indent << '}';\n auto it_cp = it;\n if (++it_cp != reports.end()) {\n out << \",\\n\";\n }\n }\n}\n\nvoid JSONReporter::Finalize() {\n // Close the list of benchmarks and the top level object.\n GetOutputStream() << \"\\n ]\\n}\\n\";\n}\n\nvoid JSONReporter::PrintRunData(Run const& run) {\n std::string indent(6, ' ');\n std::ostream& out = GetOutputStream();\n out << indent << FormatKV(\"name\", run.benchmark_name()) << \",\\n\";\n out << indent << FormatKV(\"family_index\", run.family_index) << \",\\n\";\n out << indent\n << FormatKV(\"per_family_instance_index\", run.per_family_instance_index)\n << \",\\n\";\n out << indent << FormatKV(\"run_name\", run.run_name.str()) << \",\\n\";\n out << indent << FormatKV(\"run_type\", [&run]() -> const char* {\n switch (run.run_type) {\n case BenchmarkReporter::Run::RT_Iteration:\n return \"iteration\";\n case BenchmarkReporter::Run::RT_Aggregate:\n return \"aggregate\";\n }\n BENCHMARK_UNREACHABLE();\n }()) << \",\\n\";\n out << indent << FormatKV(\"repetitions\", run.repetitions) << \",\\n\";\n if (run.run_type != BenchmarkReporter::Run::RT_Aggregate) {\n out << indent << FormatKV(\"repetition_index\", run.repetition_index)\n << \",\\n\";\n }\n out << indent << FormatKV(\"threads\", run.threads) << \",\\n\";\n if (run.run_type == BenchmarkReporter::Run::RT_Aggregate) {\n out << indent << FormatKV(\"aggregate_name\", run.aggregate_name) << \",\\n\";\n out << indent << FormatKV(\"aggregate_unit\", [&run]() -> const char* {\n switch (run.aggregate_unit) {\n case StatisticUnit::kTime:\n return \"time\";\n case StatisticUnit::kPercentage:\n return \"percentage\";\n }\n BENCHMARK_UNREACHABLE();\n }()) << \",\\n\";\n }\n if (internal::SkippedWithError == run.skipped) {\n out << indent << FormatKV(\"error_occurred\", true) << \",\\n\";\n out << indent << FormatKV(\"error_message\", run.skip_message) << \",\\n\";\n } else if (internal::SkippedWithMessage == run.skipped) {\n out << indent << FormatKV(\"skipped\", true) << \",\\n\";\n out << indent << FormatKV(\"skip_message\", run.skip_message) << \",\\n\";\n }\n if (!run.report_big_o && !run.report_rms) {\n out << indent << FormatKV(\"iterations\", run.iterations) << \",\\n\";\n if (run.run_type != Run::RT_Aggregate ||\n run.aggregate_unit == StatisticUnit::kTime) {\n out << indent << FormatKV(\"real_time\", run.GetAdjustedRealTime())\n << \",\\n\";\n out << indent << FormatKV(\"cpu_time\", run.GetAdjustedCPUTime());\n } else {\n assert(run.aggregate_unit == StatisticUnit::kPercentage);\n out << indent << FormatKV(\"real_time\", run.real_accumulated_time)\n << \",\\n\";\n out << indent << FormatKV(\"cpu_time\", run.cpu_accumulated_time);\n }\n out << \",\\n\"\n << indent << FormatKV(\"time_unit\", GetTimeUnitString(run.time_unit));\n } else if (run.report_big_o) {\n out << indent << FormatKV(\"cpu_coefficient\", run.GetAdjustedCPUTime())\n << \",\\n\";\n out << indent << FormatKV(\"real_coefficient\", run.GetAdjustedRealTime())\n << \",\\n\";\n out << indent << FormatKV(\"big_o\", GetBigOString(run.complexity)) << \",\\n\";\n out << indent << FormatKV(\"time_unit\", GetTimeUnitString(run.time_unit));\n } else if (run.report_rms) {\n out << indent << FormatKV(\"rms\", run.GetAdjustedCPUTime());\n }\n\n for (const auto& c : run.counters) {\n out << \",\\n\" << indent << FormatKV(c.first, c.second);\n }\n\n if (run.memory_result.memory_iterations > 0) {\n const auto& memory_result = run.memory_result;\n out << \",\\n\" << indent << FormatKV(\"allocs_per_iter\", run.allocs_per_iter);\n out << \",\\n\"\n << indent << FormatKV(\"max_bytes_used\", memory_result.max_bytes_used);\n\n auto report_if_present = [&out, &indent](const std::string& label,\n int64_t val) {\n if (val != MemoryManager::TombstoneValue) {\n out << \",\\n\" << indent << FormatKV(label, val);\n }\n };\n\n report_if_present(\"total_allocated_bytes\",\n memory_result.total_allocated_bytes);\n report_if_present(\"net_heap_growth\", memory_result.net_heap_growth);\n }\n\n if (!run.report_label.empty()) {\n out << \",\\n\" << indent << FormatKV(\"label\", run.report_label);\n }\n out << '\\n';\n}\n\n} // end namespace benchmark\n" }, { "path": "src/log.h", "content": "#ifndef BENCHMARK_LOG_H_\n#define BENCHMARK_LOG_H_\n\n#include \n#include \n\nnamespace benchmark {\nnamespace internal {\n\ntypedef std::basic_ostream&(EndLType)(std::basic_ostream&);\n\nclass LogType {\n friend LogType& GetNullLogInstance();\n friend LogType& GetErrorLogInstance();\n\n // FIXME: Add locking to output.\n template \n friend LogType& operator<<(LogType&, Tp const&);\n friend LogType& operator<<(LogType&, EndLType*);\n\n private:\n LogType(std::ostream* out) : out_(out) {}\n std::ostream* out_;\n\n // NOTE: we could use BENCHMARK_DISALLOW_COPY_AND_ASSIGN but we shouldn't have\n // a dependency on benchmark.h from here.\n LogType(const LogType&) = delete;\n LogType& operator=(const LogType&) = delete;\n};\n\ntemplate \nLogType& operator<<(LogType& log, Tp const& value) {\n if (log.out_) {\n *log.out_ << value;\n }\n return log;\n}\n\ninline LogType& operator<<(LogType& log, EndLType* m) {\n if (log.out_) {\n *log.out_ << m;\n }\n return log;\n}\n\ninline int& LogLevel() {\n static int log_level = 0;\n return log_level;\n}\n\ninline LogType& GetNullLogInstance() {\n static LogType null_log(static_cast(nullptr));\n return null_log;\n}\n\ninline LogType& GetErrorLogInstance() {\n static LogType error_log(&std::clog);\n return error_log;\n}\n\ninline LogType& GetLogInstanceForLevel(int level) {\n if (level <= LogLevel()) {\n return GetErrorLogInstance();\n }\n return GetNullLogInstance();\n}\n\n} // end namespace internal\n} // end namespace benchmark\n\n// clang-format off\n#define BM_VLOG(x) \\\n (::benchmark::internal::GetLogInstanceForLevel(x) << \"-- LOG(\" << x << \"):\" \\\n \" \")\n// clang-format on\n#endif\n" }, { "path": "src/mutex.h", "content": "#ifndef BENCHMARK_MUTEX_H_\n#define BENCHMARK_MUTEX_H_\n\n#include \n#include \n\n#include \"check.h\"\n\n// Enable thread safety attributes only with clang.\n// The attributes can be safely erased when compiling with other compilers.\n#if defined(HAVE_THREAD_SAFETY_ATTRIBUTES)\n#define THREAD_ANNOTATION_ATTRIBUTE_(x) __attribute__((x))\n#else\n#define THREAD_ANNOTATION_ATTRIBUTE_(x) // no-op\n#endif\n\n#define CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE_(capability(x))\n\n#define SCOPED_CAPABILITY THREAD_ANNOTATION_ATTRIBUTE_(scoped_lockable)\n\n#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE_(guarded_by(x))\n\n#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE_(pt_guarded_by(x))\n\n#define ACQUIRED_BEFORE(...) \\\n THREAD_ANNOTATION_ATTRIBUTE_(acquired_before(__VA_ARGS__))\n\n#define ACQUIRED_AFTER(...) \\\n THREAD_ANNOTATION_ATTRIBUTE_(acquired_after(__VA_ARGS__))\n\n#define REQUIRES(...) \\\n THREAD_ANNOTATION_ATTRIBUTE_(requires_capability(__VA_ARGS__))\n\n#define REQUIRES_SHARED(...) \\\n THREAD_ANNOTATION_ATTRIBUTE_(requires_shared_capability(__VA_ARGS__))\n\n#define ACQUIRE(...) \\\n THREAD_ANNOTATION_ATTRIBUTE_(acquire_capability(__VA_ARGS__))\n\n#define ACQUIRE_SHARED(...) \\\n THREAD_ANNOTATION_ATTRIBUTE_(acquire_shared_capability(__VA_ARGS__))\n\n#define RELEASE(...) \\\n THREAD_ANNOTATION_ATTRIBUTE_(release_capability(__VA_ARGS__))\n\n#define RELEASE_SHARED(...) \\\n THREAD_ANNOTATION_ATTRIBUTE_(release_shared_capability(__VA_ARGS__))\n\n#define TRY_ACQUIRE(...) \\\n THREAD_ANNOTATION_ATTRIBUTE_(try_acquire_capability(__VA_ARGS__))\n\n#define TRY_ACQUIRE_SHARED(...) \\\n THREAD_ANNOTATION_ATTRIBUTE_(try_acquire_shared_capability(__VA_ARGS__))\n\n#define EXCLUDES(...) THREAD_ANNOTATION_ATTRIBUTE_(locks_excluded(__VA_ARGS__))\n\n#define ASSERT_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE_(assert_capability(x))\n\n#define ASSERT_SHARED_CAPABILITY(x) \\\n THREAD_ANNOTATION_ATTRIBUTE_(assert_shared_capability(x))\n\n#define RETURN_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE_(lock_returned(x))\n\n#define NO_THREAD_SAFETY_ANALYSIS \\\n THREAD_ANNOTATION_ATTRIBUTE_(no_thread_safety_analysis)\n\nnamespace benchmark {\n\ntypedef std::condition_variable Condition;\n\n// NOTE: Wrappers for std::mutex and std::unique_lock are provided so that\n// we can annotate them with thread safety attributes and use the\n// -Wthread-safety warning with clang. The standard library types cannot be\n// used directly because they do not provide the required annotations.\nclass CAPABILITY(\"mutex\") Mutex {\n public:\n Mutex() {}\n\n void lock() ACQUIRE() { mut_.lock(); }\n void unlock() RELEASE() { mut_.unlock(); }\n std::mutex& native_handle() { return mut_; }\n\n private:\n std::mutex mut_;\n};\n\nclass SCOPED_CAPABILITY MutexLock {\n typedef std::unique_lock MutexLockImp;\n\n public:\n MutexLock(Mutex& m) ACQUIRE(m) : ml_(m.native_handle()) {}\n ~MutexLock() RELEASE() {}\n MutexLockImp& native_handle() { return ml_; }\n\n private:\n MutexLockImp ml_;\n};\n\nclass Barrier {\n public:\n Barrier(int num_threads) : running_threads_(num_threads) {}\n\n // Called by each thread\n bool wait() EXCLUDES(lock_) {\n bool last_thread = false;\n {\n MutexLock ml(lock_);\n last_thread = createBarrier(ml);\n }\n if (last_thread) phase_condition_.notify_all();\n return last_thread;\n }\n\n void removeThread() EXCLUDES(lock_) {\n MutexLock ml(lock_);\n --running_threads_;\n if (entered_ != 0) phase_condition_.notify_all();\n }\n\n private:\n Mutex lock_;\n Condition phase_condition_;\n int running_threads_;\n\n // State for barrier management\n int phase_number_ = 0;\n int entered_ = 0; // Number of threads that have entered this barrier\n\n // Enter the barrier and wait until all other threads have also\n // entered the barrier. Returns iff this is the last thread to\n // enter the barrier.\n bool createBarrier(MutexLock& ml) REQUIRES(lock_) {\n BM_CHECK_LT(entered_, running_threads_);\n entered_++;\n if (entered_ < running_threads_) {\n // Wait for all threads to enter\n int phase_number_cp = phase_number_;\n auto cb = [this, phase_number_cp]() {\n return this->phase_number_ > phase_number_cp ||\n entered_ == running_threads_; // A thread has aborted in error\n };\n phase_condition_.wait(ml.native_handle(), cb);\n if (phase_number_ > phase_number_cp) return false;\n // else (running_threads_ == entered_) and we are the last thread.\n }\n // Last thread has reached the barrier\n phase_number_++;\n entered_ = 0;\n return true;\n }\n};\n\n} // end namespace benchmark\n\n#endif // BENCHMARK_MUTEX_H_\n" }, { "path": "src/perf_counters.cc", "content": "// Copyright 2021 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"perf_counters.h\"\n\n#include \n#include \n#include \n#include \n\n#if defined HAVE_LIBPFM\n#include \n#include \n#include \n#include \n\n#include \"perfmon/pfmlib.h\"\n#include \"perfmon/pfmlib_perf_event.h\"\n#endif\n\nnamespace benchmark {\nnamespace internal {\n\n#if defined HAVE_LIBPFM\n\nsize_t PerfCounterValues::Read(const std::vector& leaders) {\n // Create a pointer for multiple reads\n const size_t bufsize = values_.size() * sizeof(values_[0]);\n char* ptr = reinterpret_cast(values_.data());\n size_t size = bufsize;\n for (int lead : leaders) {\n auto read_bytes = ::read(lead, ptr, size);\n if (read_bytes >= ssize_t(sizeof(uint64_t))) {\n // Actual data bytes are all bytes minus initial padding\n std::size_t data_bytes =\n static_cast(read_bytes) - sizeof(uint64_t);\n // This should be very cheap since it's in hot cache\n std::memmove(ptr, ptr + sizeof(uint64_t), data_bytes);\n // Increment our counters\n ptr += data_bytes;\n size -= data_bytes;\n } else {\n int err = errno;\n GetErrorLogInstance() << \"Error reading lead \" << lead << \" errno:\" << err\n << \" \" << ::strerror(err) << \"\\n\";\n return 0;\n }\n }\n return (bufsize - size) / sizeof(uint64_t);\n}\n\nconst bool PerfCounters::kSupported = true;\n\n// Initializes libpfm only on the first call. Returns whether that single\n// initialization was successful.\nbool PerfCounters::Initialize() {\n // Function-scope static gets initialized only once on first call.\n static const bool success = []() {\n return pfm_initialize() == PFM_SUCCESS;\n }();\n return success;\n}\n\nbool PerfCounters::IsCounterSupported(const std::string& name) {\n Initialize();\n perf_event_attr attr;\n std::memset(&attr, 0, sizeof(attr));\n pfm_perf_encode_arg_t arg;\n std::memset(&arg, 0, sizeof(arg));\n arg.attr = &attr;\n const int mode = PFM_PLM3; // user mode only\n int ret = pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT_EXT,\n &arg);\n return (ret == PFM_SUCCESS);\n}\n\nstatic std::optional> QueryCPUPMUTypes() {\n std::vector types;\n DIR* dir = opendir(\"/sys/bus/event_source/devices\");\n if (!dir) {\n return std::nullopt;\n }\n while (dirent* ent = readdir(dir)) {\n std::string_view name_str = ent->d_name;\n auto node_path = [&](const char* node) {\n return std::string(\"/sys/bus/event_source/devices/\") + ent->d_name + \"/\" +\n node;\n };\n struct stat st;\n if (name_str == \"cpu\" || name_str == \"cpum_cf\" ||\n stat(node_path(\"cpus\").c_str(), &st) == 0 || errno != ENOENT) {\n int type_fd = open(node_path(\"type\").c_str(), O_RDONLY);\n if (type_fd < 0) {\n closedir(dir);\n return std::nullopt;\n }\n char type_str[32] = {};\n ssize_t res = read(type_fd, type_str, sizeof(type_str) - 1);\n close(type_fd);\n if (res < 0) {\n closedir(dir);\n return std::nullopt;\n }\n uint64_t type;\n if (sscanf(type_str, \"%\" PRIu64, &type) != 1) {\n closedir(dir);\n return std::nullopt;\n }\n types.push_back(type);\n }\n }\n closedir(dir);\n return types;\n}\n\nstatic std::vector GetPMUTypesForEvent(const perf_event_attr& attr) {\n // Replicate generic hardware events on all CPU PMUs.\n if (attr.type == PERF_TYPE_HARDWARE && attr.config < PERF_COUNT_HW_MAX) {\n if (auto types = QueryCPUPMUTypes()) {\n return *types;\n }\n }\n return {0};\n}\n\nPerfCounters PerfCounters::Create(\n const std::vector& counter_names) {\n if (!counter_names.empty()) {\n Initialize();\n }\n\n // Valid counters will populate these arrays but we start empty\n std::vector valid_names;\n std::vector counter_ids;\n std::vector leader_ids;\n\n // Resize to the maximum possible\n valid_names.reserve(counter_names.size());\n counter_ids.reserve(counter_names.size());\n\n const int kCounterMode = PFM_PLM3; // user mode only\n\n // Group leads will be assigned on demand. The idea is that once we cannot\n // create a counter descriptor, the reason is that this group has maxed out\n // so we set the group_id again to -1 and retry - giving the algorithm a\n // chance to create a new group leader to hold the next set of counters.\n int group_id = -1;\n\n // Loop through all performance counters\n for (size_t i = 0; i < counter_names.size(); ++i) {\n // we are about to push into the valid names vector\n // check if we did not reach the maximum\n if (valid_names.size() == PerfCounterValues::kMaxCounters) {\n // Log a message if we maxed out and stop adding\n GetErrorLogInstance()\n << counter_names.size() << \" counters were requested. The maximum is \"\n << PerfCounterValues::kMaxCounters << \" and \" << valid_names.size()\n << \" were already added. All remaining counters will be ignored\\n\";\n // stop the loop and return what we have already\n break;\n }\n\n // Check if this name is empty\n const auto& name = counter_names[i];\n if (name.empty()) {\n GetErrorLogInstance()\n << \"A performance counter name was the empty string\\n\";\n continue;\n }\n\n // Here first means first in group, ie the group leader\n const bool is_first = (group_id < 0);\n\n // This struct will be populated by libpfm from the counter string\n // and then fed into the syscall perf_event_open\n struct perf_event_attr attr {};\n attr.size = sizeof(attr);\n\n // This is the input struct to libpfm.\n pfm_perf_encode_arg_t arg{};\n arg.attr = &attr;\n const int pfm_get = pfm_get_os_event_encoding(name.c_str(), kCounterMode,\n PFM_OS_PERF_EVENT, &arg);\n if (pfm_get != PFM_SUCCESS) {\n GetErrorLogInstance()\n << \"Unknown performance counter name: \" << name << \"\\n\";\n continue;\n }\n\n // We then proceed to populate the remaining fields in our attribute struct\n // Note: the man page for perf_event_create suggests inherit = true and\n // read_format = PERF_FORMAT_GROUP don't work together, but that's not the\n // case.\n attr.disabled = is_first;\n attr.inherit = true;\n attr.pinned = is_first;\n attr.exclude_kernel = true;\n attr.exclude_user = false;\n attr.exclude_hv = true;\n\n // Read all counters in a group in one read.\n attr.read_format = PERF_FORMAT_GROUP; //| PERF_FORMAT_TOTAL_TIME_ENABLED |\n // PERF_FORMAT_TOTAL_TIME_RUNNING;\n\n uint64_t base_config = attr.config;\n for (uint64_t pmu : GetPMUTypesForEvent(attr)) {\n attr.config = (pmu << PERF_PMU_TYPE_SHIFT) | base_config;\n int id = -1;\n while (id < 0) {\n static constexpr size_t kNrOfSyscallRetries = 5;\n // Retry syscall as it was interrupted often (b/64774091).\n for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries;\n ++num_retries) {\n id = perf_event_open(&attr, 0, -1, group_id, 0);\n if (id >= 0 || errno != EINTR) {\n break;\n }\n }\n if (id < 0) {\n // If the file descriptor is negative we might have reached a limit\n // in the current group. Set the group_id to -1 and retry\n if (group_id >= 0) {\n // Create a new group\n group_id = -1;\n } else {\n // At this point we have already retried to set a new group id and\n // failed. We then give up.\n break;\n }\n }\n }\n\n // We failed to get a new file descriptor. We might have reached a hard\n // hardware limit that cannot be resolved even with group multiplexing\n if (id < 0) {\n GetErrorLogInstance() << \"***WARNING** Failed to get a file descriptor \"\n \"for performance counter \"\n << name << \". Ignoring\\n\";\n\n // We give up on this counter but try to keep going\n // as the others would be fine\n continue;\n }\n if (group_id < 0) {\n // This is a leader, store and assign it to the current file descriptor\n leader_ids.push_back(id);\n group_id = id;\n }\n // This is a valid counter, add it to our descriptor's list\n counter_ids.push_back(id);\n valid_names.push_back(name);\n }\n }\n\n // Loop through all group leaders activating them\n // There is another option of starting ALL counters in a process but\n // that would be far reaching an intrusion. If the user is using PMCs\n // by themselves then this would have a side effect on them. It is\n // friendlier to loop through all groups individually.\n for (int lead : leader_ids) {\n if (ioctl(lead, PERF_EVENT_IOC_ENABLE) != 0) {\n // This should never happen but if it does, we give up on the\n // entire batch as recovery would be a mess.\n GetErrorLogInstance() << \"***WARNING*** Failed to start counters. \"\n \"Clearing out all counters.\\n\";\n\n // Close all performance counters\n for (int id : counter_ids) {\n ::close(id);\n }\n\n // Return an empty object so our internal state is still good and\n // the process can continue normally without impact\n return NoCounters();\n }\n }\n\n return PerfCounters(std::move(valid_names), std::move(counter_ids),\n std::move(leader_ids));\n}\n\nvoid PerfCounters::CloseCounters() const {\n if (counter_ids_.empty()) {\n return;\n }\n for (int lead : leader_ids_) {\n ioctl(lead, PERF_EVENT_IOC_DISABLE);\n }\n for (int fd : counter_ids_) {\n close(fd);\n }\n}\n#else // defined HAVE_LIBPFM\nsize_t PerfCounterValues::Read(const std::vector&) { return 0; }\n\nconst bool PerfCounters::kSupported = false;\n\nbool PerfCounters::Initialize() { return false; }\n\nbool PerfCounters::IsCounterSupported(const std::string&) { return false; }\n\nPerfCounters PerfCounters::Create(\n const std::vector& counter_names) {\n if (!counter_names.empty()) {\n GetErrorLogInstance() << \"Performance counters not supported.\\n\";\n }\n return NoCounters();\n}\n\nvoid PerfCounters::CloseCounters() const {}\n#endif // defined HAVE_LIBPFM\n\nPerfCountersMeasurement::PerfCountersMeasurement(\n const std::vector& counter_names)\n : start_values_(counter_names.size()), end_values_(counter_names.size()) {\n counters_ = PerfCounters::Create(counter_names);\n}\n\nPerfCounters& PerfCounters::operator=(PerfCounters&& other) noexcept {\n if (this != &other) {\n CloseCounters();\n\n counter_ids_ = std::move(other.counter_ids_);\n leader_ids_ = std::move(other.leader_ids_);\n counter_names_ = std::move(other.counter_names_);\n }\n return *this;\n}\n} // namespace internal\n} // namespace benchmark\n" }, { "path": "src/perf_counters.h", "content": "// Copyright 2021 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_PERF_COUNTERS_H\n#define BENCHMARK_PERF_COUNTERS_H\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/export.h\"\n#include \"benchmark/macros.h\"\n#include \"benchmark/utils.h\"\n#include \"check.h\"\n#include \"log.h\"\n#include \"mutex.h\"\n\n#ifndef BENCHMARK_OS_WINDOWS\n#include \n#endif\n\n#if defined(_MSC_VER)\n#pragma warning(push)\n// C4251: needs to have dll-interface to be used by clients of class\n#pragma warning(disable : 4251)\n#endif\n\nnamespace benchmark {\nnamespace internal {\n\n// Typically, we can only read a small number of counters. There is also a\n// padding preceding counter values, when reading multiple counters with one\n// syscall (which is desirable). PerfCounterValues abstracts these details.\n// The implementation ensures the storage is inlined, and allows 0-based\n// indexing into the counter values.\n// The object is used in conjunction with a PerfCounters object, by passing it\n// to Snapshot(). The Read() method relocates individual reads, discarding\n// the initial padding from each group leader in the values buffer such that\n// all user accesses through the [] operator are correct.\nclass BENCHMARK_EXPORT PerfCounterValues {\n public:\n explicit PerfCounterValues(size_t nr_counters) : nr_counters_(nr_counters) {\n BM_CHECK_LE(nr_counters_, kMaxCounters);\n }\n\n // We are reading correctly now so the values don't need to skip padding\n uint64_t operator[](size_t pos) const { return values_[pos]; }\n\n // Increased the maximum to 32 only since the buffer\n // is std::array<> backed\n static constexpr size_t kMaxCounters = 32;\n\n private:\n friend class PerfCounters;\n // Get the byte buffer in which perf counters can be captured.\n // This is used by PerfCounters::Read\n std::pair get_data_buffer() {\n return {reinterpret_cast(values_.data()),\n sizeof(uint64_t) * (kPadding + nr_counters_)};\n }\n\n // This reading is complex and as the goal of this class is to\n // abstract away the intrincacies of the reading process, this is\n // a better place for it\n size_t Read(const std::vector& leaders);\n\n // Move the padding to 2 due to the reading algorithm (1st padding plus a\n // current read padding)\n static constexpr size_t kPadding = 2;\n std::array values_;\n const size_t nr_counters_;\n};\n\n// Collect PMU counters. The object, once constructed, is ready to be used by\n// calling read(). PMU counter collection is enabled from the time create() is\n// called, to obtain the object, until the object's destructor is called.\nclass BENCHMARK_EXPORT PerfCounters final {\n public:\n // True iff this platform supports performance counters.\n static const bool kSupported;\n\n // Returns an empty object\n static PerfCounters NoCounters() { return PerfCounters(); }\n\n ~PerfCounters() { CloseCounters(); }\n PerfCounters() = default;\n PerfCounters(PerfCounters&&) = default;\n PerfCounters(const PerfCounters&) = delete;\n PerfCounters& operator=(PerfCounters&&) noexcept;\n PerfCounters& operator=(const PerfCounters&) = delete;\n\n // Platform-specific implementations may choose to do some library\n // initialization here.\n static bool Initialize();\n\n // Check if the given counter is supported, if the app wants to\n // check before passing\n static bool IsCounterSupported(const std::string& name);\n\n // Return a PerfCounters object ready to read the counters with the names\n // specified. The values are user-mode only. The counter name format is\n // implementation and OS specific.\n // In case of failure, this method will in the worst case return an\n // empty object whose state will still be valid.\n static PerfCounters Create(const std::vector& counter_names);\n\n // Take a snapshot of the current value of the counters into the provided\n // valid PerfCounterValues storage. The values are populated such that:\n // names()[i]'s value is (*values)[i]\n BENCHMARK_ALWAYS_INLINE bool Snapshot(PerfCounterValues* values) const {\n#ifndef BENCHMARK_OS_WINDOWS\n assert(values != nullptr);\n return values->Read(leader_ids_) == counter_ids_.size();\n#else\n (void)values;\n return false;\n#endif\n }\n\n const std::vector& names() const { return counter_names_; }\n size_t num_counters() const { return counter_names_.size(); }\n\n private:\n PerfCounters(const std::vector& counter_names,\n std::vector&& counter_ids, std::vector&& leader_ids)\n : counter_ids_(std::move(counter_ids)),\n leader_ids_(std::move(leader_ids)),\n counter_names_(counter_names) {}\n\n void CloseCounters() const;\n\n std::vector counter_ids_;\n std::vector leader_ids_;\n std::vector counter_names_;\n};\n\n// Typical usage of the above primitives.\nclass BENCHMARK_EXPORT PerfCountersMeasurement final {\n public:\n PerfCountersMeasurement(const std::vector& counter_names);\n\n size_t num_counters() const { return counters_.num_counters(); }\n\n const std::vector& names() const { return counters_.names(); }\n\n BENCHMARK_ALWAYS_INLINE bool Start() {\n if (num_counters() == 0) return true;\n // Tell the compiler to not move instructions above/below where we take\n // the snapshot.\n ClobberMemory();\n valid_read_ &= counters_.Snapshot(&start_values_);\n ClobberMemory();\n\n return valid_read_;\n }\n\n BENCHMARK_ALWAYS_INLINE bool Stop(\n std::vector>& measurements) {\n if (num_counters() == 0) return true;\n // Tell the compiler to not move instructions above/below where we take\n // the snapshot.\n ClobberMemory();\n valid_read_ &= counters_.Snapshot(&end_values_);\n ClobberMemory();\n\n for (size_t i = 0; i < counters_.names().size(); ++i) {\n double measurement = static_cast(end_values_[i]) -\n static_cast(start_values_[i]);\n measurements.push_back({counters_.names()[i], measurement});\n }\n\n return valid_read_;\n }\n\n private:\n PerfCounters counters_;\n bool valid_read_ = true;\n PerfCounterValues start_values_;\n PerfCounterValues end_values_;\n};\n\n} // namespace internal\n} // namespace benchmark\n\n#if defined(_MSC_VER)\n#pragma warning(pop)\n#endif\n\n#endif // BENCHMARK_PERF_COUNTERS_H\n" }, { "path": "src/re.h", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef BENCHMARK_RE_H_\n#define BENCHMARK_RE_H_\n\n#include \n\n#include \"internal_macros.h\"\n\n// clang-format off\n\n#if !defined(HAVE_STD_REGEX) && \\\n !defined(HAVE_GNU_POSIX_REGEX) && \\\n !defined(HAVE_POSIX_REGEX)\n // No explicit regex selection; detect based on builtin hints.\n #if defined(BENCHMARK_OS_LINUX) || defined(BENCHMARK_OS_APPLE)\n #define HAVE_POSIX_REGEX 1\n #elif __cplusplus >= 199711L\n #define HAVE_STD_REGEX 1\n #endif\n#endif\n\n// Prefer C regex libraries when compiling w/o exceptions so that we can\n// correctly report errors.\n#if defined(BENCHMARK_HAS_NO_EXCEPTIONS) && \\\n defined(HAVE_STD_REGEX) && \\\n (defined(HAVE_GNU_POSIX_REGEX) || defined(HAVE_POSIX_REGEX))\n #undef HAVE_STD_REGEX\n#endif\n\n#if defined(HAVE_STD_REGEX)\n #include \n#elif defined(HAVE_GNU_POSIX_REGEX)\n #include \n#elif defined(HAVE_POSIX_REGEX)\n #include \n#else\n#error No regular expression backend was found!\n#endif\n\n// clang-format on\n\n#include \n\n#include \"check.h\"\n\nnamespace benchmark {\n\n// A wrapper around the POSIX regular expression API that provides automatic\n// cleanup\nclass Regex {\n public:\n Regex() : init_(false) {}\n\n ~Regex();\n\n // Compile a regular expression matcher from spec. Returns true on success.\n //\n // On failure (and if error is not nullptr), error is populated with a human\n // readable error message if an error occurs.\n bool Init(const std::string& spec, std::string* error);\n\n // Returns whether str matches the compiled regular expression.\n bool Match(const std::string& str);\n\n private:\n bool init_;\n// Underlying regular expression object\n#if defined(HAVE_STD_REGEX)\n std::regex re_;\n#elif defined(HAVE_POSIX_REGEX) || defined(HAVE_GNU_POSIX_REGEX)\n regex_t re_;\n#else\n#error No regular expression backend implementation available\n#endif\n};\n\n#if defined(HAVE_STD_REGEX)\n\ninline bool Regex::Init(const std::string& spec, std::string* error) {\n#ifdef BENCHMARK_HAS_NO_EXCEPTIONS\n ((void)error); // suppress unused warning\n#else\n try {\n#endif\n re_ = std::regex(spec, std::regex_constants::extended);\n init_ = true;\n#ifndef BENCHMARK_HAS_NO_EXCEPTIONS\n}\ncatch (const std::regex_error& e) {\n if (error) {\n *error = e.what();\n }\n}\n#endif\nreturn init_;\n}\n\ninline Regex::~Regex() {}\n\ninline bool Regex::Match(const std::string& str) {\n if (!init_) {\n return false;\n }\n return std::regex_search(str, re_);\n}\n\n#else\ninline bool Regex::Init(const std::string& spec, std::string* error) {\n int ec = regcomp(&re_, spec.c_str(), REG_EXTENDED | REG_NOSUB);\n if (ec != 0) {\n if (error) {\n size_t needed = regerror(ec, &re_, nullptr, 0);\n std::vector errbuf(needed);\n regerror(ec, &re_, errbuf.data(), needed);\n\n // regerror returns the number of bytes necessary to null terminate\n // the string, so we move that when assigning to error.\n BM_CHECK_NE(needed, 0);\n error->assign(errbuf.data(), needed - 1);\n }\n\n return false;\n }\n\n init_ = true;\n return true;\n}\n\ninline Regex::~Regex() {\n if (init_) {\n regfree(&re_);\n }\n}\n\ninline bool Regex::Match(const std::string& str) {\n if (!init_) {\n return false;\n }\n return regexec(&re_, str.c_str(), 0, nullptr, 0) == 0;\n}\n#endif\n\n} // end namespace benchmark\n\n#endif // BENCHMARK_RE_H_\n" }, { "path": "src/reporter.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"benchmark/reporter.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/sysinfo.h\"\n#include \"check.h\"\n#include \"string_util.h\"\n#include \"timers.h\"\n\nnamespace benchmark {\n\nBenchmarkReporter::BenchmarkReporter()\n : output_stream_(&std::cout), error_stream_(&std::cerr) {}\n\nBenchmarkReporter::~BenchmarkReporter() {}\n\nvoid BenchmarkReporter::PrintBasicContext(std::ostream* out,\n Context const& context) {\n BM_CHECK(out) << \"cannot be null\";\n auto& Out = *out;\n\n#ifndef BENCHMARK_OS_QURT\n // Date/time information is not available on QuRT.\n // Attempting to get it via this call cause the binary to crash.\n Out << LocalDateTimeString() << \"\\n\";\n#endif\n\n if (benchmark::BenchmarkReporter::Context::executable_name != nullptr) {\n Out << \"Running \" << benchmark::BenchmarkReporter::Context::executable_name\n << \"\\n\";\n }\n\n const CPUInfo& info = context.cpu_info;\n Out << \"Run on (\" << info.num_cpus << \" X \"\n << (info.cycles_per_second / 1000000.0) << \" MHz CPU \"\n << ((info.num_cpus > 1) ? \"s\" : \"\") << \")\\n\";\n if (!info.caches.empty()) {\n Out << \"CPU Caches:\\n\";\n for (const auto& CInfo : info.caches) {\n Out << \" L\" << CInfo.level << \" \" << CInfo.type << \" \"\n << (CInfo.size / 1024) << \" KiB\";\n if (CInfo.num_sharing != 0) {\n Out << \" (x\" << (info.num_cpus / CInfo.num_sharing) << \")\";\n }\n Out << \"\\n\";\n }\n }\n if (!info.load_avg.empty()) {\n Out << \"Load Average: \";\n for (auto It = info.load_avg.begin(); It != info.load_avg.end();) {\n Out << StrFormat(\"%.2f\", *It++);\n if (It != info.load_avg.end()) {\n Out << \", \";\n }\n }\n Out << \"\\n\";\n }\n\n std::map* global_context =\n internal::GetGlobalContext();\n\n if (global_context != nullptr) {\n for (const auto& kv : *global_context) {\n Out << kv.first << \": \" << kv.second << \"\\n\";\n }\n }\n\n if (CPUInfo::Scaling::ENABLED == info.scaling) {\n Out << \"***WARNING*** CPU scaling is enabled, the benchmark \"\n \"real time measurements may be noisy and will incur extra \"\n \"overhead.\\n\";\n }\n\n const SystemInfo& sysinfo = context.sys_info;\n if (SystemInfo::ASLR::ENABLED == sysinfo.ASLRStatus) {\n Out << \"***WARNING*** ASLR is enabled, the results may have unreproducible \"\n \"noise in them.\\n\";\n }\n\n#ifndef NDEBUG\n Out << \"***WARNING*** Library was built as DEBUG. Timings may be \"\n \"affected.\\n\";\n#endif\n}\n\n// No initializer because it's already initialized to NULL.\nconst char* BenchmarkReporter::Context::executable_name;\n\nBenchmarkReporter::Context::Context()\n : cpu_info(CPUInfo::Get()), sys_info(SystemInfo::Get()) {}\n\nstd::string BenchmarkReporter::Run::benchmark_name() const {\n std::string name = run_name.str();\n if (run_type == RT_Aggregate) {\n name += \"_\" + aggregate_name;\n }\n return name;\n}\n\ndouble BenchmarkReporter::Run::GetAdjustedRealTime() const {\n double new_time = real_accumulated_time * GetTimeUnitMultiplier(time_unit);\n if (iterations != 0) {\n new_time /= static_cast(iterations);\n }\n return new_time;\n}\n\ndouble BenchmarkReporter::Run::GetAdjustedCPUTime() const {\n double new_time = cpu_accumulated_time * GetTimeUnitMultiplier(time_unit);\n if (iterations != 0) {\n new_time /= static_cast(iterations);\n }\n return new_time;\n}\n\n} // end namespace benchmark\n" }, { "path": "src/statistics.cc", "content": "// Copyright 2016 Ismael Jimenez Martinez. All rights reserved.\n// Copyright 2017 Roman Lebedev. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"statistics.h\"\n\n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/reporter.h\"\n#include \"benchmark/statistics.h\"\n#include \"benchmark/types.h\"\n#include \"check.h\"\n\nnamespace benchmark {\n\nconst auto StatisticsSum = [](const std::vector& v) {\n return std::accumulate(v.begin(), v.end(), 0.0);\n};\n\ndouble StatisticsMean(const std::vector& v) {\n if (v.empty()) {\n return 0.0;\n }\n return StatisticsSum(v) * (1.0 / static_cast(v.size()));\n}\n\ndouble StatisticsMedian(const std::vector& v) {\n if (v.size() < 3) {\n return StatisticsMean(v);\n }\n std::vector copy(v);\n\n auto center = copy.begin() + v.size() / 2;\n std::nth_element(copy.begin(), center, copy.end());\n\n // Did we have an odd number of samples? If yes, then center is the median.\n // If not, then we are looking for the average between center and the value\n // before. Instead of resorting, we just look for the max value before it,\n // which is not necessarily the element immediately preceding `center` Since\n // `copy` is only partially sorted by `nth_element`.\n if (v.size() % 2 == 1) {\n return *center;\n }\n auto center2 = std::max_element(copy.begin(), center);\n return (*center + *center2) / 2.0;\n}\n\n// Return the sum of the squares of this sample set\nconst auto SumSquares = [](const std::vector& v) {\n return std::inner_product(v.begin(), v.end(), v.begin(), 0.0);\n};\n\nconst auto Sqr = [](const double dat) { return dat * dat; };\nconst auto Sqrt = [](const double dat) {\n // Avoid NaN due to imprecision in the calculations\n if (dat < 0.0) {\n return 0.0;\n }\n return std::sqrt(dat);\n};\n\ndouble StatisticsStdDev(const std::vector& v) {\n const auto mean = StatisticsMean(v);\n if (v.empty()) {\n return mean;\n }\n\n // Sample standard deviation is undefined for n = 1\n if (v.size() == 1) {\n return 0.0;\n }\n\n const double avg_squares =\n SumSquares(v) * (1.0 / static_cast(v.size()));\n return Sqrt(static_cast(v.size()) /\n (static_cast(v.size()) - 1.0) *\n (avg_squares - Sqr(mean)));\n}\n\ndouble StatisticsCV(const std::vector& v) {\n if (v.size() < 2) {\n return 0.0;\n }\n\n const auto stddev = StatisticsStdDev(v);\n const auto mean = StatisticsMean(v);\n\n if (std::fpclassify(mean) == FP_ZERO) {\n return 0.0;\n }\n\n return stddev / mean;\n}\n\nstd::vector ComputeStats(\n const std::vector& reports) {\n typedef BenchmarkReporter::Run Run;\n std::vector results;\n\n auto error_count = std::count_if(reports.begin(), reports.end(),\n [](Run const& run) { return run.skipped; });\n\n if (reports.size() - static_cast(error_count) < 2) {\n // We don't report aggregated data if there was a single run.\n return results;\n }\n\n // Accumulators.\n std::vector real_accumulated_time_stat;\n std::vector cpu_accumulated_time_stat;\n\n real_accumulated_time_stat.reserve(reports.size());\n cpu_accumulated_time_stat.reserve(reports.size());\n\n // All repetitions should be run with the same number of iterations so we\n // can take this information from the first benchmark.\n const IterationCount run_iterations = reports.front().iterations;\n // create stats for user counters\n struct CounterStat {\n Counter c;\n std::vector s;\n };\n std::map counter_stats;\n for (Run const& r : reports) {\n for (auto const& cnt : r.counters) {\n auto it = counter_stats.find(cnt.first);\n if (it == counter_stats.end()) {\n it = counter_stats\n .emplace(cnt.first,\n CounterStat{cnt.second, std::vector{}})\n .first;\n it->second.s.reserve(reports.size());\n } else {\n BM_CHECK_EQ(it->second.c.flags, cnt.second.flags);\n }\n }\n }\n\n // Populate the accumulators.\n for (Run const& run : reports) {\n BM_CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name());\n BM_CHECK_EQ(run_iterations, run.iterations);\n if (run.skipped != 0u) {\n continue;\n }\n real_accumulated_time_stat.emplace_back(run.real_accumulated_time);\n cpu_accumulated_time_stat.emplace_back(run.cpu_accumulated_time);\n // user counters\n for (auto const& cnt : run.counters) {\n auto it = counter_stats.find(cnt.first);\n BM_CHECK_NE(it, counter_stats.end());\n it->second.s.emplace_back(cnt.second);\n }\n }\n\n // Only add label if it is same for all runs\n std::string report_label = reports[0].report_label;\n for (std::size_t i = 1; i < reports.size(); i++) {\n if (reports[i].report_label != report_label) {\n report_label = \"\";\n break;\n }\n }\n\n const double iteration_rescale_factor =\n static_cast(reports.size()) / static_cast(run_iterations);\n\n for (const auto& Stat : *reports[0].statistics) {\n // Get the data from the accumulator to BenchmarkReporter::Run's.\n Run data;\n data.run_name = reports[0].run_name;\n data.family_index = reports[0].family_index;\n data.per_family_instance_index = reports[0].per_family_instance_index;\n data.run_type = BenchmarkReporter::Run::RT_Aggregate;\n data.threads = reports[0].threads;\n data.repetitions = reports[0].repetitions;\n data.repetition_index = Run::no_repetition_index;\n data.aggregate_name = Stat.name_;\n data.aggregate_unit = Stat.unit_;\n data.report_label = report_label;\n\n // It is incorrect to say that an aggregate is computed over\n // run's iterations, because those iterations already got averaged.\n // Similarly, if there are N repetitions with 1 iterations each,\n // an aggregate will be computed over N measurements, not 1.\n // Thus it is best to simply use the count of separate reports.\n data.iterations = static_cast(reports.size());\n\n data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat);\n data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat);\n\n if (data.aggregate_unit == StatisticUnit::kTime) {\n // We will divide these times by data.iterations when reporting, but the\n // data.iterations is not necessarily the scale of these measurements,\n // because in each repetition, these timers are sum over all the iters.\n // And if we want to say that the stats are over N repetitions and not\n // M iterations, we need to multiply these by (N/M).\n data.real_accumulated_time *= iteration_rescale_factor;\n data.cpu_accumulated_time *= iteration_rescale_factor;\n }\n\n data.time_unit = reports[0].time_unit;\n\n // user counters\n for (auto const& kv : counter_stats) {\n // Do *NOT* rescale the custom counters. They are already properly scaled.\n const auto uc_stat = Stat.compute_(kv.second.s);\n auto c = Counter(uc_stat, counter_stats[kv.first].c.flags,\n counter_stats[kv.first].c.oneK);\n data.counters[kv.first] = c;\n }\n\n results.push_back(data);\n }\n\n return results;\n}\n\n} // end namespace benchmark\n" }, { "path": "src/statistics.h", "content": "// Copyright 2016 Ismael Jimenez Martinez. All rights reserved.\n// Copyright 2017 Roman Lebedev. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#ifndef STATISTICS_H_\n#define STATISTICS_H_\n\n#include \n\n#include \"benchmark/export.h\"\n#include \"benchmark/reporter.h\"\n\nnamespace benchmark {\n\n// Return a vector containing the mean, median and standard deviation\n// information (and any user-specified info) for the specified list of reports.\n// If 'reports' contains less than two non-errored runs an empty vector is\n// returned\nBENCHMARK_EXPORT\nstd::vector ComputeStats(\n const std::vector& reports);\n\nBENCHMARK_EXPORT\ndouble StatisticsMean(const std::vector& v);\nBENCHMARK_EXPORT\ndouble StatisticsMedian(const std::vector& v);\nBENCHMARK_EXPORT\ndouble StatisticsStdDev(const std::vector& v);\nBENCHMARK_EXPORT\ndouble StatisticsCV(const std::vector& v);\n\n} // end namespace benchmark\n\n#endif // STATISTICS_H_\n" }, { "path": "src/string_util.cc", "content": "#include \"string_util.h\"\n\n#include \n#ifdef BENCHMARK_STL_ANDROID_GNUSTL\n#include \n#endif\n#include \n#include \n#include \n#include \n#include \n\n#include \"arraysize.h\"\n#include \"benchmark/types.h\"\n\nnamespace benchmark {\nnamespace {\n// kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta.\nconst char* const kBigSIUnits[] = {\"k\", \"M\", \"G\", \"T\", \"P\", \"E\", \"Z\", \"Y\"};\n// Kibi, Mebi, Gibi, Tebi, Pebi, Exbi, Zebi, Yobi.\nconst char* const kBigIECUnits[] = {\"Ki\", \"Mi\", \"Gi\", \"Ti\",\n \"Pi\", \"Ei\", \"Zi\", \"Yi\"};\n// milli, micro, nano, pico, femto, atto, zepto, yocto.\nconst char* const kSmallSIUnits[] = {\"m\", \"u\", \"n\", \"p\", \"f\", \"a\", \"z\", \"y\"};\n\n// We require that all three arrays have the same size.\nstatic_assert(arraysize(kBigSIUnits) == arraysize(kBigIECUnits),\n \"SI and IEC unit arrays must be the same size\");\nstatic_assert(arraysize(kSmallSIUnits) == arraysize(kBigSIUnits),\n \"Small SI and Big SI unit arrays must be the same size\");\n\nconst int64_t kUnitsSize = arraysize(kBigSIUnits);\n\nstd::pair ToExponentAndMantissa(double val, int precision,\n double one_k) {\n std::string mantissa;\n int64_t exponent = 0;\n if (val < 0) {\n mantissa = \"-\";\n val = -val;\n } else {\n mantissa.clear();\n }\n\n // Adjust threshold so that it never excludes things which can't be rendered\n // in 'precision' digits.\n const double adjusted_threshold =\n std::max(1.0, 1.0 / std::pow(10.0, precision));\n const double big_threshold = (adjusted_threshold * one_k) - 1;\n const double small_threshold = adjusted_threshold;\n // Values in ]simple_threshold,small_threshold[ will be printed as-is\n const double simple_threshold = 0.01;\n\n auto format_mantissa = [&](double v) { mantissa += StrFormat(\"%g\", v); };\n\n // Positive powers\n if (val > big_threshold) {\n double scaled = val;\n for (size_t i = 0; i < arraysize(kBigSIUnits); ++i) {\n scaled /= one_k;\n if (scaled <= big_threshold) {\n format_mantissa(scaled);\n exponent = static_cast(i + 1);\n return std::make_pair(mantissa, exponent);\n }\n }\n format_mantissa(val);\n exponent = 0;\n return std::make_pair(mantissa, exponent);\n }\n\n // Negative powers\n if (val < small_threshold) {\n if (val < simple_threshold) {\n double scaled = val;\n for (size_t i = 0; i < arraysize(kSmallSIUnits); ++i) {\n scaled *= one_k;\n if (scaled >= small_threshold) {\n format_mantissa(scaled);\n exponent = -static_cast(i + 1);\n return std::make_pair(mantissa, exponent);\n }\n }\n }\n format_mantissa(val);\n exponent = 0;\n return std::make_pair(mantissa, exponent);\n }\n\n format_mantissa(val);\n exponent = 0;\n return std::make_pair(mantissa, exponent);\n}\n\nstd::string ExponentToPrefix(int64_t exponent, bool iec) {\n if (exponent == 0) {\n return {};\n }\n\n const int64_t index = (exponent > 0 ? exponent - 1 : -exponent - 1);\n if (index >= kUnitsSize) {\n return {};\n }\n\n const char* const* array =\n (exponent > 0 ? (iec ? kBigIECUnits : kBigSIUnits) : kSmallSIUnits);\n\n return std::string(array[index]);\n}\n\nstd::string ToBinaryStringFullySpecified(double value, int precision,\n Counter::OneK one_k) {\n auto [mantissa, exponent] = ToExponentAndMantissa(\n value, precision, one_k == Counter::kIs1024 ? 1024.0 : 1000.0);\n return mantissa + ExponentToPrefix(exponent, one_k == Counter::kIs1024);\n}\n\nPRINTF_FORMAT_STRING_FUNC(1, 0)\nstd::string StrFormatImp(const char* msg, va_list args) {\n // we might need a second shot at this, so pre-emptivly make a copy\n va_list args_cp;\n va_copy(args_cp, args);\n\n // Use std::array for first attempt to avoid one memory allocation guess what\n // the size might be\n std::array local_buff = {};\n\n // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation\n // in the android-ndk\n auto ret = vsnprintf(local_buff.data(), local_buff.size(), msg, args_cp);\n\n va_end(args_cp);\n\n // handle empty expansion\n if (ret == 0) {\n return {};\n }\n if (static_cast(ret) < local_buff.size()) {\n return std::string(local_buff.data());\n }\n\n // we did not provide a long enough buffer on our first attempt.\n // add 1 to size to account for null-byte in size cast to prevent overflow\n std::size_t size = static_cast(ret) + 1;\n auto buff_ptr = std::unique_ptr(new char[size]);\n // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation\n // in the android-ndk\n va_list args_cp2;\n va_copy(args_cp2, args);\n vsnprintf(buff_ptr.get(), size, msg, args_cp2);\n va_end(args_cp2);\n return std::string(buff_ptr.get());\n}\n\n} // end namespace\n\nstd::string HumanReadableNumber(double n, Counter::OneK one_k) {\n return ToBinaryStringFullySpecified(n, 1, one_k);\n}\n\nstd::string StrFormat(const char* format, ...) {\n va_list args;\n va_start(args, format);\n std::string tmp = StrFormatImp(format, args);\n va_end(args);\n return tmp;\n}\n\nstd::vector StrSplit(const std::string& str, char delim) {\n if (str.empty()) {\n return {};\n }\n std::vector ret;\n size_t first = 0;\n size_t next = str.find(delim);\n for (; next != std::string::npos;\n first = next + 1, next = str.find(delim, first)) {\n ret.push_back(str.substr(first, next - first));\n }\n ret.push_back(str.substr(first));\n return ret;\n}\n\n#ifdef BENCHMARK_STL_ANDROID_GNUSTL\n/*\n * GNU STL in Android NDK lacks support for some C++11 functions, including\n * stoul, stoi, stod. We reimplement them here using C functions strtoul,\n * strtol, strtod. Note that reimplemented functions are in benchmark::\n * namespace, not std:: namespace.\n */\nunsigned long stoul(const std::string& str, size_t* pos, int base) {\n /* Record previous errno */\n const int oldErrno = errno;\n errno = 0;\n\n const char* strStart = str.c_str();\n char* strEnd = const_cast(strStart);\n const unsigned long result = strtoul(strStart, &strEnd, base);\n\n const int strtoulErrno = errno;\n /* Restore previous errno */\n errno = oldErrno;\n\n /* Check for errors and return */\n if (strtoulErrno == ERANGE) {\n throw std::out_of_range(\"stoul failed: \" + str +\n \" is outside of range of unsigned long\");\n } else if (strEnd == strStart || strtoulErrno != 0) {\n throw std::invalid_argument(\"stoul failed: \" + str + \" is not an integer\");\n }\n if (pos != nullptr) {\n *pos = static_cast(strEnd - strStart);\n }\n return result;\n}\n\nint stoi(const std::string& str, size_t* pos, int base) {\n /* Record previous errno */\n const int oldErrno = errno;\n errno = 0;\n\n const char* strStart = str.c_str();\n char* strEnd = const_cast(strStart);\n const long result = strtol(strStart, &strEnd, base);\n\n const int strtolErrno = errno;\n /* Restore previous errno */\n errno = oldErrno;\n\n /* Check for errors and return */\n if (strtolErrno == ERANGE || long(int(result)) != result) {\n throw std::out_of_range(\"stoul failed: \" + str +\n \" is outside of range of int\");\n } else if (strEnd == strStart || strtolErrno != 0) {\n throw std::invalid_argument(\"stoul failed: \" + str + \" is not an integer\");\n }\n if (pos != nullptr) {\n *pos = static_cast(strEnd - strStart);\n }\n return int(result);\n}\n\ndouble stod(const std::string& str, size_t* pos) {\n /* Record previous errno */\n const int oldErrno = errno;\n errno = 0;\n\n const char* strStart = str.c_str();\n char* strEnd = const_cast(strStart);\n const double result = strtod(strStart, &strEnd);\n\n /* Restore previous errno */\n const int strtodErrno = errno;\n errno = oldErrno;\n\n /* Check for errors and return */\n if (strtodErrno == ERANGE) {\n throw std::out_of_range(\"stoul failed: \" + str +\n \" is outside of range of int\");\n } else if (strEnd == strStart || strtodErrno != 0) {\n throw std::invalid_argument(\"stoul failed: \" + str + \" is not an integer\");\n }\n if (pos != nullptr) {\n *pos = static_cast(strEnd - strStart);\n }\n return result;\n}\n#endif\n\n} // end namespace benchmark\n" }, { "path": "src/string_util.h", "content": "#ifndef BENCHMARK_STRING_UTIL_H_\n#define BENCHMARK_STRING_UTIL_H_\n\n#include \n#include \n#include \n#include \n\n#include \"benchmark/counter.h\"\n#include \"benchmark/export.h\"\n#include \"check.h\"\n\nnamespace benchmark {\n\nBENCHMARK_EXPORT\nstd::string HumanReadableNumber(double n, Counter::OneK one_k);\n\nBENCHMARK_EXPORT\nstd::string StrFormat(const char* format, ...) PRINTF_FORMAT_STRING_FUNC(1, 2);\n\ninline std::ostream& StrCatImp(std::ostream& out) BENCHMARK_NOEXCEPT {\n return out;\n}\n\ntemplate \ninline std::ostream& StrCatImp(std::ostream& out, First&& f, Rest&&... rest) {\n out << std::forward(f);\n return StrCatImp(out, std::forward(rest)...);\n}\n\ntemplate \ninline std::string StrCat(Args&&... args) {\n std::ostringstream ss;\n StrCatImp(ss, std::forward(args)...);\n return ss.str();\n}\n\nBENCHMARK_EXPORT\nstd::vector StrSplit(const std::string& str, char delim);\n\n// Disable lint checking for this block since it re-implements C functions.\n// NOLINTBEGIN\n#ifdef BENCHMARK_STL_ANDROID_GNUSTL\n/*\n * GNU STL in Android NDK lacks support for some C++11 functions, including\n * stoul, stoi, stod. We reimplement them here using C functions strtoul,\n * strtol, strtod. Note that reimplemented functions are in benchmark::\n * namespace, not std:: namespace.\n */\nunsigned long stoul(const std::string& str, size_t* pos = nullptr,\n int base = 10);\nint stoi(const std::string& str, size_t* pos = nullptr, int base = 10);\ndouble stod(const std::string& str, size_t* pos = nullptr);\n#else\nusing std::stod; // NOLINT(misc-unused-using-decls)\nusing std::stoi; // NOLINT(misc-unused-using-decls)\nusing std::stoul; // NOLINT(misc-unused-using-decls)\n#endif\n// NOLINTEND\n\n} // end namespace benchmark\n\n#endif // BENCHMARK_STRING_UTIL_H_\n" }, { "path": "src/sysinfo.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"internal_macros.h\"\n\n#ifdef BENCHMARK_OS_WINDOWS\n#if !defined(WINVER) || WINVER < 0x0600\n#undef WINVER\n#define WINVER 0x0600\n#endif // WINVER handling\n#include \n#undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA\n#include \n#include \n\n#include \n#else\n#include \n#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \\\n !defined(BENCHMARK_OS_WASI)\n#include \n#endif\n#include \n#include // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD\n#include \n#if defined BENCHMARK_OS_FREEBSD || defined BENCHMARK_OS_MACOSX || \\\n defined BENCHMARK_OS_NETBSD || defined BENCHMARK_OS_OPENBSD || \\\n defined BENCHMARK_OS_DRAGONFLY\n#define BENCHMARK_HAS_SYSCTL\n#include \n#endif\n#endif\n#if defined(BENCHMARK_OS_SOLARIS)\n#include \n#include \n#endif\n#if defined(BENCHMARK_OS_QNX)\n#include \n#endif\n#if defined(BENCHMARK_OS_QURT)\n#include \n#endif\n#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY)\n#if defined(BENCHMARK_OS_FREEBSD)\n#include \n#endif\n#include \n#endif\n\n#if defined(BENCHMARK_OS_LINUX)\n#include \n#endif\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/export.h\"\n#include \"benchmark/sysinfo.h\"\n#include \"benchmark/utils.h\"\n#include \"check.h\"\n#include \"cycleclock.h\"\n#include \"log.h\"\n#include \"string_util.h\"\n#include \"timers.h\"\n\nnamespace benchmark {\nnamespace {\n\nvoid PrintImp(std::ostream& out) { out << '\\n'; }\n\ntemplate \nvoid PrintImp(std::ostream& out, First&& f, Rest&&... rest) {\n out << std::forward(f);\n PrintImp(out, std::forward(rest)...);\n}\n\ntemplate \nBENCHMARK_NORETURN void PrintErrorAndDie(Args&&... args) {\n PrintImp(std::cerr, std::forward(args)...);\n std::cerr << std::flush;\n std::exit(EXIT_FAILURE);\n}\n\n#ifdef BENCHMARK_HAS_SYSCTL\n\n/// ValueUnion - A type used to correctly alias the byte-for-byte output of\n/// `sysctl` with the result type it's to be interpreted as.\nstruct ValueUnion {\n union DataT {\n int32_t int32_value;\n int64_t int64_value;\n // For correct aliasing of union members from bytes.\n char bytes[8];\n };\n using DataPtr = std::unique_ptr;\n\n // The size of the data union member + its trailing array size.\n std::size_t size;\n DataPtr buff;\n\n public:\n ValueUnion() : size(0), buff(nullptr, &std::free) {}\n\n explicit ValueUnion(std::size_t buff_size)\n : size(sizeof(DataT) + buff_size),\n buff(::new(std::malloc(size)) DataT(), &std::free) {}\n\n ValueUnion(ValueUnion&& other) = default;\n\n explicit operator bool() const { return bool(buff); }\n\n char* data() const { return buff->bytes; }\n\n std::string GetAsString() const { return std::string(data()); }\n\n int64_t GetAsInteger() const {\n if (size == sizeof(buff->int32_value))\n return buff->int32_value;\n else if (size == sizeof(buff->int64_value))\n return buff->int64_value;\n BENCHMARK_UNREACHABLE();\n }\n\n template \n std::array GetAsArray() {\n const int arr_size = sizeof(T) * N;\n BM_CHECK_LE(arr_size, size);\n std::array arr;\n std::memcpy(arr.data(), data(), arr_size);\n return arr;\n }\n};\n\nValueUnion GetSysctlImp(std::string const& name) {\n#if defined BENCHMARK_OS_OPENBSD\n int mib[2];\n\n mib[0] = CTL_HW;\n if ((name == \"hw.ncpuonline\") || (name == \"hw.cpuspeed\")) {\n ValueUnion buff(sizeof(int));\n\n if (name == \"hw.ncpuonline\") {\n mib[1] = HW_NCPUONLINE;\n } else {\n mib[1] = HW_CPUSPEED;\n }\n\n if (sysctl(mib, 2, buff.data(), &buff.size, nullptr, 0) == -1) {\n return ValueUnion();\n }\n return buff;\n }\n return ValueUnion();\n#else\n std::size_t cur_buff_size = 0;\n if (sysctlbyname(name.c_str(), nullptr, &cur_buff_size, nullptr, 0) == -1)\n return ValueUnion();\n\n ValueUnion buff(cur_buff_size);\n if (sysctlbyname(name.c_str(), buff.data(), &buff.size, nullptr, 0) == 0)\n return buff;\n return ValueUnion();\n#endif\n}\n\nBENCHMARK_MAYBE_UNUSED\nbool GetSysctl(std::string const& name, std::string* out) {\n out->clear();\n auto buff = GetSysctlImp(name);\n if (!buff) return false;\n out->assign(buff.data());\n return true;\n}\n\ntemplate ::value>::type>\nbool GetSysctl(std::string const& name, Tp* out) {\n *out = 0;\n auto buff = GetSysctlImp(name);\n if (!buff) return false;\n *out = static_cast(buff.GetAsInteger());\n return true;\n}\n\ntemplate \nbool GetSysctl(std::string const& name, std::array* out) {\n auto buff = GetSysctlImp(name);\n if (!buff) return false;\n *out = buff.GetAsArray();\n return true;\n}\n#endif\n\ntemplate \nbool ReadFromFile(std::string const& fname, ArgT* arg) {\n *arg = ArgT();\n std::ifstream f(fname.c_str());\n if (!f.is_open()) {\n return false;\n }\n f >> *arg;\n return f.good();\n}\n\nCPUInfo::Scaling CpuScaling(int num_cpus) {\n // We don't have a valid CPU count, so don't even bother.\n if (num_cpus <= 0) {\n return CPUInfo::Scaling::UNKNOWN;\n }\n#if defined(BENCHMARK_OS_QNX)\n return CPUInfo::Scaling::UNKNOWN;\n#elif !defined(BENCHMARK_OS_WINDOWS)\n // On Linux, the CPUfreq subsystem exposes CPU information as files on the\n // local file system. If reading the exported files fails, then we may not be\n // running on Linux, so we silently ignore all the read errors.\n std::string res;\n for (int cpu = 0; cpu < num_cpus; ++cpu) {\n std::string governor_file =\n StrCat(\"/sys/devices/system/cpu/cpu\", cpu, \"/cpufreq/scaling_governor\");\n if (ReadFromFile(governor_file, &res) && res != \"performance\") {\n return CPUInfo::Scaling::ENABLED;\n }\n }\n return CPUInfo::Scaling::DISABLED;\n#else\n return CPUInfo::Scaling::UNKNOWN;\n#endif\n}\n\nint CountSetBitsInCPUMap(std::string val) {\n auto CountBits = [](std::string part) {\n using CPUMask = std::bitset;\n part = \"0x\" + part;\n CPUMask mask(benchmark::stoul(part, nullptr, 16));\n return static_cast(mask.count());\n };\n std::size_t pos = 0;\n int total = 0;\n while ((pos = val.find(',')) != std::string::npos) {\n total += CountBits(val.substr(0, pos));\n val = val.substr(pos + 1);\n }\n if (!val.empty()) {\n total += CountBits(val);\n }\n return total;\n}\n\nBENCHMARK_MAYBE_UNUSED\nstd::vector GetCacheSizesFromKVFS() {\n std::vector res;\n std::string dir = \"/sys/devices/system/cpu/cpu0/cache/\";\n int idx = 0;\n while (true) {\n CPUInfo::CacheInfo info;\n std::string fpath = StrCat(dir, \"index\", idx++, \"/\");\n std::ifstream f(StrCat(fpath, \"size\").c_str());\n if (!f.is_open()) {\n break;\n }\n std::string suffix;\n f >> info.size;\n if (f.fail()) {\n PrintErrorAndDie(\"Failed while reading file '\", fpath, \"size'\");\n }\n if (f.good()) {\n f >> suffix;\n if (f.bad()) {\n PrintErrorAndDie(\n \"Invalid cache size format: failed to read size suffix\");\n } else if (f && suffix != \"K\") {\n PrintErrorAndDie(\"Invalid cache size format: Expected bytes \", suffix);\n } else if (suffix == \"K\") {\n info.size *= 1024;\n }\n }\n if (!ReadFromFile(StrCat(fpath, \"type\"), &info.type)) {\n PrintErrorAndDie(\"Failed to read from file \", fpath, \"type\");\n }\n if (!ReadFromFile(StrCat(fpath, \"level\"), &info.level)) {\n PrintErrorAndDie(\"Failed to read from file \", fpath, \"level\");\n }\n std::string map_str;\n if (!ReadFromFile(StrCat(fpath, \"shared_cpu_map\"), &map_str)) {\n PrintErrorAndDie(\"Failed to read from file \", fpath, \"shared_cpu_map\");\n }\n info.num_sharing = CountSetBitsInCPUMap(map_str);\n res.push_back(info);\n }\n\n return res;\n}\n\n#ifdef BENCHMARK_OS_MACOSX\nstd::vector GetCacheSizesMacOSX() {\n std::vector res;\n std::array cache_counts{{0, 0, 0, 0}};\n GetSysctl(\"hw.cacheconfig\", &cache_counts);\n\n struct {\n std::string name;\n std::string type;\n int level;\n int num_sharing;\n } cases[] = {{\"hw.l1dcachesize\", \"Data\", 1, cache_counts[1]},\n {\"hw.l1icachesize\", \"Instruction\", 1, cache_counts[1]},\n {\"hw.l2cachesize\", \"Unified\", 2, cache_counts[2]},\n {\"hw.l3cachesize\", \"Unified\", 3, cache_counts[3]}};\n for (auto& c : cases) {\n int val;\n if (!GetSysctl(c.name, &val)) continue;\n CPUInfo::CacheInfo info;\n info.type = c.type;\n info.level = c.level;\n info.size = val;\n info.num_sharing = c.num_sharing;\n res.push_back(std::move(info));\n }\n return res;\n}\n#elif defined(BENCHMARK_OS_WINDOWS)\nstd::vector GetCacheSizesWindows() {\n std::vector res;\n DWORD buffer_size = 0;\n using PInfo = SYSTEM_LOGICAL_PROCESSOR_INFORMATION;\n using CInfo = CACHE_DESCRIPTOR;\n\n using UPtr = std::unique_ptr;\n GetLogicalProcessorInformation(nullptr, &buffer_size);\n UPtr buff(static_cast(std::malloc(buffer_size)), &std::free);\n if (!GetLogicalProcessorInformation(buff.get(), &buffer_size)) {\n PrintErrorAndDie(\"Failed during call to GetLogicalProcessorInformation: \",\n GetLastError());\n }\n\n PInfo* it = buff.get();\n PInfo* end = buff.get() + (buffer_size / sizeof(PInfo));\n\n for (; it != end; ++it) {\n if (it->Relationship != RelationCache) {\n continue;\n }\n using BitSet = std::bitset;\n BitSet b(it->ProcessorMask);\n // To prevent duplicates, only consider caches where CPU 0 is specified\n if (!b.test(0)) continue;\n const CInfo& cache = it->Cache;\n CPUInfo::CacheInfo C;\n C.num_sharing = static_cast(b.count());\n C.level = cache.Level;\n C.size = static_cast(cache.Size);\n C.type = \"Unknown\";\n switch (cache.Type) {\n// Windows SDK version >= 10.0.26100.0\n#ifdef NTDDI_WIN11_GE\n case CacheUnknown:\n break;\n#endif\n case CacheUnified:\n C.type = \"Unified\";\n break;\n case CacheInstruction:\n C.type = \"Instruction\";\n break;\n case CacheData:\n C.type = \"Data\";\n break;\n case CacheTrace:\n C.type = \"Trace\";\n break;\n }\n res.push_back(C);\n }\n return res;\n}\n#elif BENCHMARK_OS_QNX\nstd::vector GetCacheSizesQNX() {\n std::vector res;\n struct cacheattr_entry* cache = SYSPAGE_ENTRY(cacheattr);\n uint32_t const elsize = SYSPAGE_ELEMENT_SIZE(cacheattr);\n int num = SYSPAGE_ENTRY_SIZE(cacheattr) / elsize;\n for (int i = 0; i < num; ++i) {\n CPUInfo::CacheInfo info;\n switch (cache->flags) {\n case CACHE_FLAG_INSTR:\n info.type = \"Instruction\";\n info.level = 1;\n break;\n case CACHE_FLAG_DATA:\n info.type = \"Data\";\n info.level = 1;\n break;\n case CACHE_FLAG_UNIFIED:\n info.type = \"Unified\";\n info.level = 2;\n break;\n case CACHE_FLAG_SHARED:\n info.type = \"Shared\";\n info.level = 3;\n break;\n default:\n continue;\n break;\n }\n info.size = cache->line_size * cache->num_lines;\n info.num_sharing = 0;\n res.push_back(std::move(info));\n cache = SYSPAGE_ARRAY_ADJ_OFFSET(cacheattr, cache, elsize);\n }\n return res;\n}\n#endif\n\nstd::vector GetCacheSizes() {\n#ifdef BENCHMARK_OS_MACOSX\n return GetCacheSizesMacOSX();\n#elif defined(BENCHMARK_OS_WINDOWS)\n return GetCacheSizesWindows();\n#elif defined(BENCHMARK_OS_QNX)\n return GetCacheSizesQNX();\n#elif defined(BENCHMARK_OS_QURT) || defined(BENCHMARK_OS_EMSCRIPTEN) || \\\n defined(BENCHMARK_OS_WASI)\n return std::vector();\n#else\n return GetCacheSizesFromKVFS();\n#endif\n}\n\nstd::string GetSystemName() {\n#if defined(BENCHMARK_OS_WINDOWS)\n std::string str;\n static constexpr int COUNT = MAX_COMPUTERNAME_LENGTH + 1;\n TCHAR hostname[COUNT] = {'\\0'};\n DWORD DWCOUNT = COUNT;\n if (!GetComputerName(hostname, &DWCOUNT)) return std::string(\"\");\n#ifndef UNICODE\n str = std::string(hostname, DWCOUNT);\n#else\n // `WideCharToMultiByte` returns `0` when conversion fails.\n int len = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, hostname,\n DWCOUNT, NULL, 0, NULL, NULL);\n str.resize(len);\n WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, hostname, DWCOUNT, &str[0],\n static_cast(str.size()), NULL, NULL);\n#endif\n return str;\n#elif defined(BENCHMARK_OS_QURT)\n std::string str = \"Hexagon DSP\";\n qurt_arch_version_t arch_version_struct;\n if (qurt_sysenv_get_arch_version(&arch_version_struct) == QURT_EOK) {\n str += \" v\";\n str += std::to_string(arch_version_struct.arch_version);\n }\n return str;\n#elif defined(BENCHMARK_OS_WASI)\n return std::string(\"wasi\");\n#else\n#ifndef HOST_NAME_MAX\n#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac doesn't have HOST_NAME_MAX defined\n#define HOST_NAME_MAX 64\n#elif defined(BENCHMARK_OS_NACL)\n#define HOST_NAME_MAX 64\n#elif defined(BENCHMARK_OS_QNX)\n#define HOST_NAME_MAX 154\n#elif defined(BENCHMARK_OS_RTEMS)\n#define HOST_NAME_MAX 256\n#elif defined(BENCHMARK_OS_SOLARIS)\n#define HOST_NAME_MAX MAXHOSTNAMELEN\n#elif defined(BENCHMARK_OS_ZOS)\n#define HOST_NAME_MAX _POSIX_HOST_NAME_MAX\n#else\n#pragma message(\"HOST_NAME_MAX not defined. using 64\")\n#define HOST_NAME_MAX 64\n#endif\n#endif // def HOST_NAME_MAX\n char hostname[HOST_NAME_MAX];\n int retVal = gethostname(hostname, HOST_NAME_MAX);\n return retVal != 0 ? std::string() : std::string(hostname);\n#endif // Catch-all POSIX block.\n}\n\nSystemInfo::ASLR GetASLR() {\n#ifdef BENCHMARK_OS_LINUX\n const auto curr_personality = personality(0xffffffff);\n return (curr_personality & ADDR_NO_RANDOMIZE) ? SystemInfo::ASLR::DISABLED\n : SystemInfo::ASLR::ENABLED;\n#else\n // FIXME: support detecting ASLR on other OS.\n return SystemInfo::ASLR::UNKNOWN;\n#endif\n}\n\nint GetNumCPUsImpl() {\n#ifdef BENCHMARK_OS_WINDOWS\n SYSTEM_INFO sysinfo;\n // Use memset as opposed to = {} to avoid GCC missing initializer false\n // positives.\n std::memset(&sysinfo, 0, sizeof(SYSTEM_INFO));\n GetSystemInfo(&sysinfo);\n // number of logical processors in the current group\n return static_cast(sysinfo.dwNumberOfProcessors);\n#elif defined(BENCHMARK_OS_QNX)\n return static_cast(_syspage_ptr->num_cpu);\n#elif defined(BENCHMARK_OS_QURT)\n qurt_sysenv_max_hthreads_t hardware_threads;\n if (qurt_sysenv_get_max_hw_threads(&hardware_threads) != QURT_EOK) {\n hardware_threads.max_hthreads = 1;\n }\n return static_cast(hardware_threads.max_hthreads);\n#elif defined(BENCHMARK_HAS_SYSCTL)\n // *BSD, macOS\n int num_cpu = -1;\n constexpr auto* hwncpu =\n#if defined BENCHMARK_OS_MACOSX\n \"hw.logicalcpu\";\n#elif defined(HW_NCPUONLINE)\n \"hw.ncpuonline\";\n#else\n \"hw.ncpu\";\n#endif\n if (GetSysctl(hwncpu, &num_cpu)) return num_cpu;\n PrintErrorAndDie(\"Err: \", strerror(errno));\n#elif defined(_SC_NPROCESSORS_ONLN)\n // Linux, Solaris, AIX, Haiku, WASM, etc.\n // Returns -1 in case of a failure.\n int num_cpu = static_cast(sysconf(_SC_NPROCESSORS_ONLN));\n if (num_cpu < 0) {\n PrintErrorAndDie(\"sysconf(_SC_NPROCESSORS_ONLN) failed with error: \",\n strerror(errno));\n }\n return num_cpu;\n#else\n // Fallback, no other API exists.\n return -1;\n#endif\n BENCHMARK_UNREACHABLE();\n}\n\nint GetNumCPUs() {\n int num_cpus = GetNumCPUsImpl();\n if (num_cpus < 1) {\n std::cerr << \"Unable to extract number of CPUs.\\n\";\n // There must be at least one CPU on which we're running.\n num_cpus = 1;\n }\n return num_cpus;\n}\n\nclass ThreadAffinityGuard final {\n public:\n ThreadAffinityGuard() : reset_affinity(SetAffinity()) {\n if (!reset_affinity) {\n std::cerr << \"***WARNING*** Failed to set thread affinity. Estimated CPU \"\n \"frequency may be incorrect.\\n\";\n }\n }\n\n ~ThreadAffinityGuard() {\n if (!reset_affinity) {\n return;\n }\n\n#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY)\n int ret = pthread_setaffinity_np(self, sizeof(previous_affinity),\n &previous_affinity);\n if (ret == 0) {\n return;\n }\n#elif defined(BENCHMARK_OS_WINDOWS_WIN32)\n DWORD_PTR ret = SetThreadAffinityMask(self, previous_affinity);\n if (ret != 0) {\n return;\n }\n#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY\n PrintErrorAndDie(\"Failed to reset thread affinity\");\n }\n\n ThreadAffinityGuard(ThreadAffinityGuard&&) = delete;\n ThreadAffinityGuard(const ThreadAffinityGuard&) = delete;\n ThreadAffinityGuard& operator=(ThreadAffinityGuard&&) = delete;\n ThreadAffinityGuard& operator=(const ThreadAffinityGuard&) = delete;\n\n private:\n bool SetAffinity() {\n#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY)\n int ret = 0;\n self = pthread_self();\n ret = pthread_getaffinity_np(self, sizeof(previous_affinity),\n &previous_affinity);\n if (ret != 0) {\n return false;\n }\n\n cpu_set_t affinity;\n memcpy(&affinity, &previous_affinity, sizeof(affinity));\n\n bool is_first_cpu = true;\n\n for (int i = 0; i < CPU_SETSIZE; ++i) {\n if (CPU_ISSET(i, &affinity)) {\n if (is_first_cpu) {\n is_first_cpu = false;\n } else {\n CPU_CLR(i, &affinity);\n }\n }\n }\n\n if (is_first_cpu) {\n return false;\n }\n\n ret = pthread_setaffinity_np(self, sizeof(affinity), &affinity);\n return ret == 0;\n#elif defined(BENCHMARK_OS_WINDOWS_WIN32)\n self = GetCurrentThread();\n DWORD_PTR mask = static_cast(1) << GetCurrentProcessorNumber();\n previous_affinity = SetThreadAffinityMask(self, mask);\n return previous_affinity != 0;\n#else\n return false;\n#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY\n }\n\n#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY)\n pthread_t self{};\n cpu_set_t previous_affinity{};\n#elif defined(BENCHMARK_OS_WINDOWS_WIN32)\n HANDLE self;\n DWORD_PTR previous_affinity;\n#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY\n bool reset_affinity;\n};\n\ndouble GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) {\n // Currently, scaling is only used on linux path here,\n // suppress diagnostics about it being unused on other paths.\n (void)scaling;\n\n#if defined BENCHMARK_OS_LINUX || defined BENCHMARK_OS_CYGWIN\n long freq = 0;\n\n // If the kernel is exporting the tsc frequency use that. There are issues\n // where cpuinfo_max_freq cannot be relied on because the BIOS may be\n // exporintg an invalid p-state (on x86) or p-states may be used to put the\n // processor in a new mode (turbo mode). Essentially, those frequencies\n // cannot always be relied upon. The same reasons apply to /proc/cpuinfo as\n // well.\n if (ReadFromFile(\"/sys/devices/system/cpu/cpu0/tsc_freq_khz\", &freq)\n // If CPU scaling is disabled, use the *current* frequency.\n // Note that we specifically don't want to read cpuinfo_cur_freq,\n // because it is only readable by root.\n || (scaling == CPUInfo::Scaling::DISABLED &&\n ReadFromFile(\"/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq\",\n &freq))\n // Otherwise, if CPU scaling may be in effect, we want to use\n // the *maximum* frequency, not whatever CPU speed some random processor\n // happens to be using now.\n || ReadFromFile(\"/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq\",\n &freq)) {\n // The value is in kHz (as the file name suggests). For example, on a\n // 2GHz warpstation, the file contains the value \"2000000\".\n return static_cast(freq) * 1000.0;\n }\n\n const double error_value = -1;\n double bogo_clock = error_value;\n\n std::ifstream f(\"/proc/cpuinfo\");\n if (!f.is_open()) {\n std::cerr << \"failed to open /proc/cpuinfo\\n\";\n return error_value;\n }\n\n auto StartsWithKey = [](std::string const& Value, std::string const& Key) {\n if (Key.size() > Value.size()) {\n return false;\n }\n auto Cmp = [&](char X, char Y) {\n return std::tolower(X) == std::tolower(Y);\n };\n return std::equal(Key.begin(), Key.end(), Value.begin(), Cmp);\n };\n\n std::string ln;\n while (std::getline(f, ln)) {\n if (ln.empty()) {\n continue;\n }\n std::size_t split_idx = ln.find(':');\n std::string value;\n if (split_idx != std::string::npos) {\n value = ln.substr(split_idx + 1);\n }\n // When parsing the \"cpu MHz\" and \"bogomips\" (fallback) entries, we only\n // accept positive values. Some environments (virtual machines) report zero,\n // which would cause infinite looping in WallTime_Init.\n if (StartsWithKey(ln, \"cpu MHz\")) {\n if (!value.empty()) {\n double cycles_per_second = benchmark::stod(value) * 1000000.0;\n if (cycles_per_second > 0) {\n return cycles_per_second;\n }\n }\n } else if (StartsWithKey(ln, \"bogomips\")) {\n if (!value.empty()) {\n bogo_clock = benchmark::stod(value) * 1000000.0;\n if (bogo_clock < 0.0) {\n bogo_clock = error_value;\n }\n }\n }\n }\n if (f.bad()) {\n std::cerr << \"Failure reading /proc/cpuinfo\\n\";\n return error_value;\n }\n if (!f.eof()) {\n std::cerr << \"Failed to read to end of /proc/cpuinfo\\n\";\n return error_value;\n }\n f.close();\n // If we found the bogomips clock, but nothing better, we'll use it (but\n // we're not happy about it); otherwise, fallback to the rough estimation\n // below.\n if (bogo_clock >= 0.0) {\n return bogo_clock;\n }\n\n#elif defined BENCHMARK_HAS_SYSCTL\n constexpr auto* freqStr =\n#if defined(BENCHMARK_OS_FREEBSD) || defined(BENCHMARK_OS_NETBSD)\n \"machdep.tsc_freq\";\n#elif defined BENCHMARK_OS_OPENBSD\n \"hw.cpuspeed\";\n#elif defined BENCHMARK_OS_DRAGONFLY\n \"hw.tsc_frequency\";\n#else\n \"hw.cpufrequency\";\n#endif\n unsigned long long hz = 0;\n#if defined BENCHMARK_OS_OPENBSD\n if (GetSysctl(freqStr, &hz)) {\n return static_cast(hz * 1000000);\n }\n#else\n if (GetSysctl(freqStr, &hz)) {\n return static_cast(hz);\n }\n#endif\n fprintf(stderr, \"Unable to determine clock rate from sysctl: %s: %s\\n\",\n freqStr, strerror(errno));\n fprintf(stderr,\n \"This does not affect benchmark measurements, only the \"\n \"metadata output.\\n\");\n\n#elif defined BENCHMARK_OS_WINDOWS_WIN32\n // In NT, read MHz from the registry. If we fail to do so or we're in win9x\n // then make a crude estimate.\n DWORD data, data_size = sizeof(data);\n if (IsWindowsXPOrGreater() &&\n SUCCEEDED(\n SHGetValueA(HKEY_LOCAL_MACHINE,\n \"HARDWARE\\\\DESCRIPTION\\\\System\\\\CentralProcessor\\\\0\",\n \"~MHz\", nullptr, &data, &data_size))) {\n return static_cast(static_cast(data) *\n static_cast(1000 * 1000)); // was mhz\n }\n#elif defined(BENCHMARK_OS_SOLARIS)\n kstat_ctl_t* kc = kstat_open();\n if (!kc) {\n std::cerr << \"failed to open /dev/kstat\\n\";\n return -1;\n }\n kstat_t* ksp = kstat_lookup(kc, const_cast(\"cpu_info\"), -1,\n const_cast(\"cpu_info0\"));\n if (!ksp) {\n std::cerr << \"failed to lookup in /dev/kstat\\n\";\n return -1;\n }\n if (kstat_read(kc, ksp, NULL) < 0) {\n std::cerr << \"failed to read from /dev/kstat\\n\";\n return -1;\n }\n kstat_named_t* knp = (kstat_named_t*)kstat_data_lookup(\n ksp, const_cast(\"current_clock_Hz\"));\n if (!knp) {\n std::cerr << \"failed to lookup data in /dev/kstat\\n\";\n return -1;\n }\n if (knp->data_type != KSTAT_DATA_UINT64) {\n std::cerr << \"current_clock_Hz is of unexpected data type: \"\n << knp->data_type << \"\\n\";\n return -1;\n }\n double clock_hz = knp->value.ui64;\n kstat_close(kc);\n return clock_hz;\n#elif defined(BENCHMARK_OS_QNX)\n return static_cast(\n static_cast(SYSPAGE_ENTRY(cpuinfo)->speed) *\n static_cast(1000 * 1000));\n#elif defined(BENCHMARK_OS_QURT)\n // QuRT doesn't provide any API to query Hexagon frequency.\n return 1000000000;\n#endif\n // If we've fallen through, attempt to roughly estimate the CPU clock rate.\n\n // Make sure to use the same cycle counter when starting and stopping the\n // cycle timer. We just pin the current thread to a cpu in the previous\n // affinity set.\n ThreadAffinityGuard affinity_guard;\n\n static constexpr double estimate_time_s = 1.0;\n const double start_time = ChronoClockNow();\n const auto start_ticks = cycleclock::Now();\n\n // Impose load instead of calling sleep() to make sure the cycle counter\n // works.\n using PRNG = std::minstd_rand;\n using Result = PRNG::result_type;\n PRNG rng(static_cast(start_ticks));\n\n Result state = 0;\n\n do {\n static constexpr size_t batch_size = 10000;\n rng.discard(batch_size);\n state += rng();\n\n } while (ChronoClockNow() - start_time < estimate_time_s);\n\n DoNotOptimize(state);\n\n const auto end_ticks = cycleclock::Now();\n const double end_time = ChronoClockNow();\n\n return static_cast(end_ticks - start_ticks) / (end_time - start_time);\n // Reset the affinity of current thread when the lifetime of affinity_guard\n // ends.\n}\n\nstd::vector GetLoadAvg() {\n#if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \\\n defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \\\n defined BENCHMARK_OS_OPENBSD || defined BENCHMARK_OS_DRAGONFLY) && \\\n !(defined(__ANDROID__) && __ANDROID_API__ < 29)\n static constexpr int kMaxSamples = 3;\n std::vector res(kMaxSamples, 0.0);\n const auto nelem = getloadavg(res.data(), kMaxSamples);\n if (nelem < 1) {\n res.clear();\n } else {\n res.resize(static_cast(nelem));\n }\n return res;\n#else\n return {};\n#endif\n}\n\n} // end namespace\n\nconst CPUInfo& CPUInfo::Get() {\n static const CPUInfo* info = new CPUInfo();\n return *info;\n}\n\nCPUInfo::CPUInfo()\n : num_cpus(GetNumCPUs()),\n scaling(CpuScaling(num_cpus)),\n cycles_per_second(GetCPUCyclesPerSecond(scaling)),\n caches(GetCacheSizes()),\n load_avg(GetLoadAvg()) {}\n\nconst SystemInfo& SystemInfo::Get() {\n static const SystemInfo* info = new SystemInfo();\n return *info;\n}\n\nSystemInfo::SystemInfo() : name(GetSystemName()), ASLRStatus(GetASLR()) {}\n} // end namespace benchmark\n" }, { "path": "src/thread_manager.h", "content": "#ifndef BENCHMARK_THREAD_MANAGER_H\n#define BENCHMARK_THREAD_MANAGER_H\n\n#include \n\n#include \"benchmark/counter.h\"\n#include \"benchmark/statistics.h\"\n#include \"benchmark/types.h\"\n#include \"mutex.h\"\n\nnamespace benchmark {\nnamespace internal {\n\nclass ThreadManager {\n public:\n explicit ThreadManager(int num_threads) : start_stop_barrier_(num_threads) {}\n\n Mutex& GetBenchmarkMutex() const RETURN_CAPABILITY(benchmark_mutex_) {\n return benchmark_mutex_;\n }\n\n bool StartStopBarrier() { return start_stop_barrier_.wait(); }\n\n void NotifyThreadComplete() { start_stop_barrier_.removeThread(); }\n\n struct Result {\n IterationCount iterations = 0;\n double real_time_used = 0;\n double cpu_time_used = 0;\n double manual_time_used = 0;\n int64_t complexity_n = 0;\n std::string report_label_;\n std::string skip_message_;\n internal::Skipped skipped_ = internal::NotSkipped;\n UserCounters counters;\n };\n GUARDED_BY(GetBenchmarkMutex()) Result results;\n\n private:\n mutable Mutex benchmark_mutex_;\n Barrier start_stop_barrier_;\n};\n\n} // namespace internal\n} // namespace benchmark\n\n#endif // BENCHMARK_THREAD_MANAGER_H\n" }, { "path": "src/thread_timer.h", "content": "#ifndef BENCHMARK_THREAD_TIMER_H\n#define BENCHMARK_THREAD_TIMER_H\n\n#include \"check.h\"\n#include \"timers.h\"\n\nnamespace benchmark {\nnamespace internal {\n\nclass ThreadTimer {\n explicit ThreadTimer(bool measure_process_cpu_time_)\n : measure_process_cpu_time(measure_process_cpu_time_) {}\n\n public:\n static ThreadTimer Create() {\n return ThreadTimer(/*measure_process_cpu_time_=*/false);\n }\n static ThreadTimer CreateProcessCpuTime() {\n return ThreadTimer(/*measure_process_cpu_time_=*/true);\n }\n\n // Called by each thread\n void StartTimer() {\n running_ = true;\n start_real_time_ = ChronoClockNow();\n start_cpu_time_ = ReadCpuTimerOfChoice();\n }\n\n // Called by each thread\n void StopTimer() {\n BM_CHECK(running_);\n running_ = false;\n real_time_used_ += ChronoClockNow() - start_real_time_;\n // Floating point error can result in the subtraction producing a negative\n // time. Guard against that.\n cpu_time_used_ +=\n std::max(ReadCpuTimerOfChoice() - start_cpu_time_, 0);\n }\n\n // Called by each thread\n void SetIterationTime(double seconds) { manual_time_used_ += seconds; }\n\n bool running() const { return running_; }\n\n // REQUIRES: timer is not running\n double real_time_used() const {\n BM_CHECK(!running_);\n return real_time_used_;\n }\n\n // REQUIRES: timer is not running\n double cpu_time_used() const {\n BM_CHECK(!running_);\n return cpu_time_used_;\n }\n\n // REQUIRES: timer is not running\n double manual_time_used() const {\n BM_CHECK(!running_);\n return manual_time_used_;\n }\n\n private:\n double ReadCpuTimerOfChoice() const {\n if (measure_process_cpu_time) return ProcessCPUUsage();\n return ThreadCPUUsage();\n }\n\n // should the thread, or the process, time be measured?\n const bool measure_process_cpu_time;\n\n bool running_ = false; // Is the timer running\n double start_real_time_ = 0; // If running_\n double start_cpu_time_ = 0; // If running_\n\n // Accumulated time so far (does not contain current slice if running_)\n double real_time_used_ = 0;\n double cpu_time_used_ = 0;\n // Manually set iteration time. User sets this with SetIterationTime(seconds).\n double manual_time_used_ = 0;\n};\n\n} // namespace internal\n} // namespace benchmark\n\n#endif // BENCHMARK_THREAD_TIMER_H\n" }, { "path": "src/timers.cc", "content": "// Copyright 2015 Google Inc. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \"timers.h\"\n\n#include \"internal_macros.h\"\n\n#ifdef BENCHMARK_OS_WINDOWS\n#include \n#undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA\n#include \n#include \n#else\n#include \n#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \\\n !defined(BENCHMARK_OS_WASI)\n#include \n#endif\n#include \n#include // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD\n#include \n#if defined BENCHMARK_OS_FREEBSD || defined BENCHMARK_OS_DRAGONFLY || \\\n defined BENCHMARK_OS_MACOSX\n#include \n#endif\n#if defined(BENCHMARK_OS_MACOSX)\n#include \n#include \n#include \n#endif\n#if defined(BENCHMARK_OS_QURT)\n#include \n#endif\n#endif\n\n#ifdef BENCHMARK_OS_EMSCRIPTEN\n#include \n#endif\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"check.h\"\n#include \"log.h\"\n#include \"string_util.h\"\n\nnamespace benchmark {\n\n// Suppress unused warnings on helper functions.\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Wunused-function\"\n#endif\n#if defined(__NVCOMPILER)\n#pragma diag_suppress declared_but_not_referenced\n#endif\n\nnamespace {\n#if defined(BENCHMARK_OS_WINDOWS)\ndouble MakeTime(FILETIME const& kernel_time, FILETIME const& user_time) {\n ULARGE_INTEGER kernel;\n ULARGE_INTEGER user;\n kernel.HighPart = kernel_time.dwHighDateTime;\n kernel.LowPart = kernel_time.dwLowDateTime;\n user.HighPart = user_time.dwHighDateTime;\n user.LowPart = user_time.dwLowDateTime;\n return (static_cast(kernel.QuadPart) +\n static_cast(user.QuadPart)) *\n 1e-7;\n}\n#elif !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \\\n !defined(BENCHMARK_OS_WASI)\ndouble MakeTime(struct rusage const& ru) {\n return (static_cast(ru.ru_utime.tv_sec) +\n static_cast(ru.ru_utime.tv_usec) * 1e-6 +\n static_cast(ru.ru_stime.tv_sec) +\n static_cast(ru.ru_stime.tv_usec) * 1e-6);\n}\n#endif\n#if defined(BENCHMARK_OS_MACOSX)\ndouble MakeTime(thread_basic_info_data_t const& info) {\n return (static_cast(info.user_time.seconds) +\n static_cast(info.user_time.microseconds) * 1e-6 +\n static_cast(info.system_time.seconds) +\n static_cast(info.system_time.microseconds) * 1e-6);\n}\n#endif\n#if defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_THREAD_CPUTIME_ID)\ndouble MakeTime(struct timespec const& ts) {\n return static_cast(ts.tv_sec) +\n (static_cast(ts.tv_nsec) * 1e-9);\n}\n#endif\n\nBENCHMARK_NORETURN void DiagnoseAndExit(const char* msg) {\n std::cerr << \"ERROR: \" << msg << '\\n';\n std::flush(std::cerr);\n std::exit(EXIT_FAILURE);\n}\n\n} // end namespace\n\ndouble ProcessCPUUsage() {\n#if defined(BENCHMARK_OS_WINDOWS)\n HANDLE proc = GetCurrentProcess();\n FILETIME creation_time;\n FILETIME exit_time;\n FILETIME kernel_time;\n FILETIME user_time;\n if (GetProcessTimes(proc, &creation_time, &exit_time, &kernel_time,\n &user_time))\n return MakeTime(kernel_time, user_time);\n DiagnoseAndExit(\"GetProccessTimes() failed\");\n#elif defined(BENCHMARK_OS_QURT)\n // Note that qurt_timer_get_ticks() is no longer documented as of SDK 5.3.0,\n // and doesn't appear to work on at least some devices (eg Samsung S22),\n // so let's use the actually-documented and apparently-equivalent\n // qurt_sysclock_get_hw_ticks() call instead.\n return static_cast(\n qurt_timer_timetick_to_us(qurt_sysclock_get_hw_ticks())) *\n 1.0e-6;\n#elif defined(BENCHMARK_OS_EMSCRIPTEN)\n // clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) returns 0 on Emscripten.\n // Use Emscripten-specific API. Reported CPU time would be exactly the\n // same as total time, but this is ok because there aren't long-latency\n // synchronous system calls in Emscripten.\n return emscripten_get_now() * 1e-3;\n#elif defined(BENCHMARK_OS_WASI)\n // WASI lacks CLOCK_PROCESS_CPUTIME_ID and getrusage; use monotonic clock.\n struct timespec ts {};\n if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) {\n return static_cast(ts.tv_sec) +\n (static_cast(ts.tv_nsec) * 1e-9);\n }\n DiagnoseAndExit(\"clock_gettime(CLOCK_MONOTONIC, ...) failed\");\n#elif defined(CLOCK_PROCESS_CPUTIME_ID) && !defined(BENCHMARK_OS_MACOSX)\n // FIXME We want to use clock_gettime, but its not available in MacOS 10.11.\n // See https://github.com/google/benchmark/pull/292\n struct timespec spec {};\n if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &spec) == 0) {\n return MakeTime(spec);\n }\n DiagnoseAndExit(\"clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) failed\");\n#else\n struct rusage ru;\n if (getrusage(RUSAGE_SELF, &ru) == 0) return MakeTime(ru);\n DiagnoseAndExit(\"getrusage(RUSAGE_SELF, ...) failed\");\n#endif\n}\n\ndouble ThreadCPUUsage() {\n#if defined(BENCHMARK_OS_WINDOWS)\n HANDLE this_thread = GetCurrentThread();\n FILETIME creation_time;\n FILETIME exit_time;\n FILETIME kernel_time;\n FILETIME user_time;\n GetThreadTimes(this_thread, &creation_time, &exit_time, &kernel_time,\n &user_time);\n return MakeTime(kernel_time, user_time);\n#elif defined(BENCHMARK_OS_QURT)\n // Note that qurt_timer_get_ticks() is no longer documented as of SDK 5.3.0,\n // and doesn't appear to work on at least some devices (eg Samsung S22),\n // so let's use the actually-documented and apparently-equivalent\n // qurt_sysclock_get_hw_ticks() call instead.\n return static_cast(\n qurt_timer_timetick_to_us(qurt_sysclock_get_hw_ticks())) *\n 1.0e-6;\n#elif defined(BENCHMARK_OS_MACOSX)\n // FIXME We want to use clock_gettime, but its not available in MacOS 10.11.\n // See https://github.com/google/benchmark/pull/292\n mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT;\n thread_basic_info_data_t info;\n mach_port_t thread = pthread_mach_thread_np(pthread_self());\n if (thread_info(thread, THREAD_BASIC_INFO,\n reinterpret_cast(&info),\n &count) == KERN_SUCCESS) {\n return MakeTime(info);\n }\n DiagnoseAndExit(\"ThreadCPUUsage() failed when evaluating thread_info\");\n#elif defined(BENCHMARK_OS_EMSCRIPTEN)\n // Emscripten doesn't support traditional threads\n return ProcessCPUUsage();\n#elif defined(BENCHMARK_OS_RTEMS)\n // RTEMS doesn't support CLOCK_THREAD_CPUTIME_ID. See\n // https://github.com/RTEMS/rtems/blob/master/cpukit/posix/src/clockgettime.c\n return ProcessCPUUsage();\n#elif defined(BENCHMARK_OS_ZOS)\n // z/OS doesn't support CLOCK_THREAD_CPUTIME_ID.\n return ProcessCPUUsage();\n#elif defined(BENCHMARK_OS_WASI)\n // WASI doesn't support per-thread CPU timing; fall back to process time.\n return ProcessCPUUsage();\n#elif defined(BENCHMARK_OS_SOLARIS)\n struct rusage ru;\n if (getrusage(RUSAGE_LWP, &ru) == 0) return MakeTime(ru);\n DiagnoseAndExit(\"getrusage(RUSAGE_LWP, ...) failed\");\n#elif defined(CLOCK_THREAD_CPUTIME_ID)\n struct timespec ts {};\n if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0) {\n return MakeTime(ts);\n }\n DiagnoseAndExit(\"clock_gettime(CLOCK_THREAD_CPUTIME_ID, ...) failed\");\n#else\n#error Per-thread timing is not available on your system.\n#endif\n}\n\nstd::string LocalDateTimeString() {\n // Write the local time in RFC3339 format yyyy-mm-ddTHH:MM:SS+/-HH:MM.\n typedef std::chrono::system_clock Clock;\n std::time_t now = Clock::to_time_t(Clock::now());\n const std::size_t kTzOffsetLen = 6;\n const std::size_t kTimestampLen = 19;\n\n std::size_t tz_len = 0;\n std::size_t timestamp_len = 0;\n long int offset_minutes = 0;\n char tz_offset_sign = '+';\n // tz_offset is set in one of three ways:\n // * strftime with %z - This either returns empty or the ISO 8601 time. The\n // maximum length an\n // ISO 8601 string can be is 7 (e.g. -03:30, plus trailing zero).\n // * snprintf with %c%02li:%02li - The maximum length is 41 (one for %c, up to\n // 19 for %02li,\n // one for :, up to 19 %02li, plus trailing zero).\n // * A fixed string of \"-00:00\". The maximum length is 7 (-00:00, plus\n // trailing zero).\n //\n // Thus, the maximum size this needs to be is 41.\n char tz_offset[41];\n // Long enough buffer to avoid format-overflow warnings\n char storage[128];\n\n#if defined(BENCHMARK_OS_WINDOWS)\n std::tm* timeinfo_p = ::localtime(&now);\n#else\n std::tm timeinfo{};\n std::tm* timeinfo_p = &timeinfo;\n ::localtime_r(&now, &timeinfo);\n#endif\n\n tz_len = std::strftime(tz_offset, sizeof(tz_offset), \"%z\", timeinfo_p);\n\n if (tz_len < kTzOffsetLen && tz_len > 1) {\n // Timezone offset was written. strftime writes offset as +HHMM or -HHMM,\n // RFC3339 specifies an offset as +HH:MM or -HH:MM. To convert, we parse\n // the offset as an integer, then reprint it to a string.\n\n offset_minutes = ::strtol(tz_offset, NULL, 10);\n if (offset_minutes < 0) {\n offset_minutes *= -1;\n tz_offset_sign = '-';\n }\n\n tz_len = static_cast(\n ::snprintf(tz_offset, sizeof(tz_offset), \"%c%02li:%02li\",\n tz_offset_sign, offset_minutes / 100, offset_minutes % 100));\n BM_CHECK(tz_len == kTzOffsetLen);\n ((void)tz_len); // Prevent unused variable warning in optimized build.\n } else {\n // Unknown offset. RFC3339 specifies that unknown local offsets should be\n // written as UTC time with -00:00 timezone.\n#if defined(BENCHMARK_OS_WINDOWS)\n // Potential race condition if another thread calls localtime or gmtime.\n timeinfo_p = ::gmtime(&now);\n#else\n ::gmtime_r(&now, &timeinfo);\n#endif\n\n strncpy(tz_offset, \"-00:00\", kTzOffsetLen + 1);\n }\n\n timestamp_len =\n std::strftime(storage, sizeof(storage), \"%Y-%m-%dT%H:%M:%S\", timeinfo_p);\n BM_CHECK(timestamp_len == kTimestampLen);\n // Prevent unused variable warning in optimized build.\n ((void)kTimestampLen);\n\n std::strncat(storage, tz_offset, sizeof(storage) - timestamp_len - 1);\n return std::string(storage);\n}\n\n} // end namespace benchmark\n" }, { "path": "src/timers.h", "content": "#ifndef BENCHMARK_TIMERS_H\n#define BENCHMARK_TIMERS_H\n\n#include \n#include \n\nnamespace benchmark {\n\n// Return the CPU usage of the current process\ndouble ProcessCPUUsage();\n\n// Return the CPU usage of the children of the current process\ndouble ChildrenCPUUsage();\n\n// Return the CPU usage of the current thread\ndouble ThreadCPUUsage();\n\n#if defined(BENCHMARK_OS_QURT)\n\n// std::chrono::now() can return 0 on some Hexagon devices;\n// this reads the value of a 56-bit, 19.2MHz hardware counter\n// and converts it to seconds. Unlike std::chrono, this doesn't\n// return an absolute time, but since ChronoClockNow() is only used\n// to compute elapsed time, this shouldn't matter.\nstruct QuRTClock {\n typedef uint64_t rep;\n typedef std::ratio<1, 19200000> period;\n typedef std::chrono::duration duration;\n typedef std::chrono::time_point time_point;\n static const bool is_steady = false;\n\n static time_point now() {\n unsigned long long count;\n asm volatile(\" %0 = c31:30 \" : \"=r\"(count));\n return time_point(static_cast(count));\n }\n};\n\n#else\n\n#if defined(HAVE_STEADY_CLOCK)\ntemplate \nstruct ChooseSteadyClock {\n typedef std::chrono::high_resolution_clock type;\n};\n\ntemplate <>\nstruct ChooseSteadyClock {\n typedef std::chrono::steady_clock type;\n};\n#endif // HAVE_STEADY_CLOCK\n\n#endif\n\nstruct ChooseClockType {\n#if defined(BENCHMARK_OS_QURT)\n typedef QuRTClock type;\n#elif defined(HAVE_STEADY_CLOCK)\n typedef ChooseSteadyClock<>::type type;\n#else\n typedef std::chrono::high_resolution_clock type;\n#endif\n};\n\ninline double ChronoClockNow() {\n typedef ChooseClockType::type ClockType;\n using FpSeconds = std::chrono::duration;\n return FpSeconds(ClockType::now().time_since_epoch()).count();\n}\n\nstd::string LocalDateTimeString();\n\n} // end namespace benchmark\n\n#endif // BENCHMARK_TIMERS_H\n" }, { "path": "test/AssemblyTests.cmake", "content": "set(CLANG_SUPPORTED_VERSION \"5.0.0\")\nset(GCC_SUPPORTED_VERSION \"5.5.0\")\n\nif (CMAKE_CXX_COMPILER_ID MATCHES \"Clang\")\n if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL ${CLANG_SUPPORTED_VERSION})\n message (WARNING\n \"Unsupported Clang version \" ${CMAKE_CXX_COMPILER_VERSION}\n \". Expected is \" ${CLANG_SUPPORTED_VERSION}\n \". Assembly tests may be broken.\")\n endif()\nelseif(CMAKE_CXX_COMPILER_ID MATCHES \"GNU\")\n if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL ${GCC_SUPPORTED_VERSION})\n message (WARNING\n \"Unsupported GCC version \" ${CMAKE_CXX_COMPILER_VERSION}\n \". Expected is \" ${GCC_SUPPORTED_VERSION}\n \". Assembly tests may be broken.\")\n endif()\nelse()\n message (WARNING \"Unsupported compiler. Assembly tests may be broken.\")\nendif()\n\ninclude(split_list)\n\nset(ASM_TEST_FLAGS \"\")\ncheck_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG)\nif (BENCHMARK_HAS_O3_FLAG)\n list(APPEND ASM_TEST_FLAGS -O3)\nendif()\n\ncheck_cxx_compiler_flag(-g0 BENCHMARK_HAS_G0_FLAG)\nif (BENCHMARK_HAS_G0_FLAG)\n list(APPEND ASM_TEST_FLAGS -g0)\nendif()\n\ncheck_cxx_compiler_flag(-fno-stack-protector BENCHMARK_HAS_FNO_STACK_PROTECTOR_FLAG)\nif (BENCHMARK_HAS_FNO_STACK_PROTECTOR_FLAG)\n list(APPEND ASM_TEST_FLAGS -fno-stack-protector)\nendif()\n\nsplit_list(ASM_TEST_FLAGS)\nstring(TOUPPER \"${CMAKE_CXX_COMPILER_ID}\" ASM_TEST_COMPILER)\n\nmacro(add_filecheck_test name)\n cmake_parse_arguments(ARG \"\" \"\" \"CHECK_PREFIXES\" ${ARGV})\n add_library(${name} OBJECT ${name}.cc)\n target_link_libraries(${name} PRIVATE benchmark::benchmark)\n set_target_properties(${name} PROPERTIES COMPILE_FLAGS \"-S ${ASM_TEST_FLAGS}\")\n set(ASM_OUTPUT_FILE \"${CMAKE_CURRENT_BINARY_DIR}/${name}.s\")\n add_custom_target(copy_${name} ALL\n COMMAND ${PROJECT_SOURCE_DIR}/tools/strip_asm.py\n $\n ${ASM_OUTPUT_FILE}\n BYPRODUCTS ${ASM_OUTPUT_FILE})\n add_dependencies(copy_${name} ${name})\n if (NOT ARG_CHECK_PREFIXES)\n set(ARG_CHECK_PREFIXES \"CHECK\")\n endif()\n foreach(prefix ${ARG_CHECK_PREFIXES})\n add_test(NAME run_${name}_${prefix}\n COMMAND\n ${LLVM_FILECHECK_EXE} ${name}.cc\n --input-file=${ASM_OUTPUT_FILE}\n --check-prefixes=CHECK,CHECK-${ASM_TEST_COMPILER}\n WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})\n endforeach()\nendmacro()\n\n" }, { "path": "test/BUILD", "content": "load(\"@rules_cc//cc:defs.bzl\", \"cc_library\", \"cc_test\")\n\nplatform(\n name = \"windows\",\n constraint_values = [\n \"@platforms//os:windows\",\n ],\n)\n\nTEST_COPTS = [\n \"-pedantic\",\n \"-pedantic-errors\",\n \"-std=c++17\",\n \"-Wall\",\n \"-Wconversion\",\n \"-Wextra\",\n \"-Wshadow\",\n # \"-Wshorten-64-to-32\",\n \"-Wfloat-equal\",\n \"-fstrict-aliasing\",\n ## assert() are used a lot in tests upstream, which may be optimised out leading to\n ## unused-variable warning.\n \"-Wno-unused-variable\",\n \"-Werror=old-style-cast\",\n]\n\nTEST_MSVC_OPTS = [\n \"/std:c++17\",\n]\n\n# Some of the issues with DoNotOptimize only occur when optimization is enabled\nPER_SRC_COPTS = {\n \"donotoptimize_test.cc\": [\"-O3\"],\n}\n\nTEST_ARGS = [\"--benchmark_min_time=0.01s\"]\n\nPER_SRC_TEST_ARGS = {\n \"user_counters_tabular_test.cc\": [\"--benchmark_counters_tabular=true\"],\n \"repetitions_test.cc\": [\" --benchmark_repetitions=3\"],\n \"spec_arg_test.cc\": [\"--benchmark_filter=BM_NotChosen\"],\n \"spec_arg_verbosity_test.cc\": [\"--v=42\"],\n \"complexity_test.cc\": [\"--benchmark_min_time=1000000x\"],\n}\n\ncc_library(\n name = \"output_test_helper\",\n testonly = 1,\n srcs = [\"output_test_helper.cc\"],\n hdrs = [\"output_test.h\"],\n copts = select({\n \"//:windows\": TEST_MSVC_OPTS,\n \"//conditions:default\": TEST_COPTS,\n }),\n deps = [\n \"//:benchmark\",\n \"//:benchmark_internal_headers\",\n ],\n)\n\n# Tests that use gtest. These rely on `gtest_main`.\n[\n cc_test(\n name = test_src[:-len(\".cc\")],\n size = \"small\",\n srcs = [test_src],\n copts = select({\n \"//:windows\": TEST_MSVC_OPTS,\n \"//conditions:default\": TEST_COPTS,\n }) + PER_SRC_COPTS.get(test_src, []),\n deps = [\n \"//:benchmark\",\n \"//:benchmark_internal_headers\",\n \"@com_google_googletest//:gtest\",\n \"@com_google_googletest//:gtest_main\",\n ],\n )\n for test_src in glob([\"*_gtest.cc\"])\n]\n\n# Tests that do not use gtest. These have their own `main` defined.\n[\n cc_test(\n name = test_src[:-len(\".cc\")],\n size = \"small\",\n srcs = [test_src],\n args = TEST_ARGS + PER_SRC_TEST_ARGS.get(test_src, []),\n copts = select({\n \"//:windows\": TEST_MSVC_OPTS,\n \"//conditions:default\": TEST_COPTS,\n }) + PER_SRC_COPTS.get(test_src, []),\n deps = [\n \":output_test_helper\",\n \"//:benchmark\",\n \"//:benchmark_internal_headers\",\n ],\n # FIXME: Add support for assembly tests to bazel.\n # See Issue #556\n # https://github.com/google/benchmark/issues/556\n )\n for test_src in glob(\n [\"*_test.cc\"],\n exclude = [\n \"*_assembly_test.cc\",\n \"cxx11_test.cc\",\n \"link_main_test.cc\",\n ],\n )\n]\n\ncc_test(\n name = \"cxx11_test\",\n size = \"small\",\n srcs = [\"cxx11_test.cc\"],\n copts = TEST_COPTS + [\"-std=c++11\"],\n target_compatible_with = select({\n \"//:windows\": [\"@platforms//:incompatible\"],\n \"//conditions:default\": [],\n }),\n deps = [\n \":output_test_helper\",\n \"//:benchmark_main\",\n ],\n)\n\ncc_test(\n name = \"link_main_test\",\n size = \"small\",\n srcs = [\"link_main_test.cc\"],\n copts = select({\n \"//:windows\": TEST_MSVC_OPTS,\n \"//conditions:default\": TEST_COPTS,\n }),\n deps = [\"//:benchmark_main\"],\n)\n" }, { "path": "test/CMakeLists.txt", "content": "#Enable the tests\n\nset(THREADS_PREFER_PTHREAD_FLAG ON)\n\nfind_package(Threads REQUIRED)\ninclude(CheckCXXCompilerFlag)\n\nadd_cxx_compiler_flag(-Wno-unused-variable)\n\n# NOTE: Some tests use `` to perform the test. Therefore we must\n# strip -DNDEBUG from the default CMake flags in DEBUG mode.\nstring(TOUPPER \"${CMAKE_BUILD_TYPE}\" uppercase_CMAKE_BUILD_TYPE)\nif( NOT uppercase_CMAKE_BUILD_TYPE STREQUAL \"DEBUG\" )\n add_definitions( -UNDEBUG )\n add_definitions(-DTEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS)\n # Also remove /D NDEBUG to avoid MSVC warnings about conflicting defines.\n foreach (flags_var_to_scrub\n CMAKE_CXX_FLAGS_RELEASE\n CMAKE_CXX_FLAGS_RELWITHDEBINFO\n CMAKE_CXX_FLAGS_MINSIZEREL\n CMAKE_C_FLAGS_RELEASE\n CMAKE_C_FLAGS_RELWITHDEBINFO\n CMAKE_C_FLAGS_MINSIZEREL)\n string (REGEX REPLACE \"(^| )[/-]D *NDEBUG($| )\" \" \"\n \"${flags_var_to_scrub}\" \"${${flags_var_to_scrub}}\")\n endforeach()\nendif()\n\nif (NOT BUILD_SHARED_LIBS)\n add_definitions(-DBENCHMARK_STATIC_DEFINE)\nendif()\n\ncheck_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG)\nset(BENCHMARK_O3_FLAG \"\")\nif (BENCHMARK_HAS_O3_FLAG)\n set(BENCHMARK_O3_FLAG \"-O3\")\nendif()\n\n# NOTE: These flags must be added after find_package(Threads REQUIRED) otherwise\n# they will break the configuration check.\nif (DEFINED BENCHMARK_CXX_LINKER_FLAGS)\n list(APPEND CMAKE_EXE_LINKER_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS})\nendif()\n\nadd_library(output_test_helper STATIC output_test_helper.cc output_test.h)\ntarget_link_libraries(output_test_helper PRIVATE benchmark::benchmark)\n\nmacro(compile_benchmark_test name)\n add_executable(${name} \"${name}.cc\")\n target_link_libraries(${name} benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT})\n if(\"${CMAKE_CXX_COMPILER_ID}\" STREQUAL \"NVHPC\")\n target_compile_options( ${name} PRIVATE --diag_suppress partial_override )\n endif()\nendmacro(compile_benchmark_test)\n\nmacro(compile_benchmark_test_with_main name)\n add_executable(${name} \"${name}.cc\")\n target_link_libraries(${name} benchmark::benchmark_main)\nendmacro(compile_benchmark_test_with_main)\n\nmacro(compile_output_test name)\n add_executable(${name} \"${name}.cc\" output_test.h)\n target_link_libraries(${name} output_test_helper benchmark::benchmark_main\n ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})\nendmacro(compile_output_test)\n\nmacro(benchmark_add_test)\n add_test(${ARGV})\n if(WIN32 AND BUILD_SHARED_LIBS)\n cmake_parse_arguments(TEST \"\" \"NAME\" \"\" ${ARGN})\n set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION \"PATH=path_list_prepend:$\")\n endif()\nendmacro(benchmark_add_test)\n\n# Demonstration executable\n\ncompile_benchmark_test_with_main(cxx11_test)\nif(DEFINED MSVC)\n # MSVC does not really support C++11.\n set_property(TARGET cxx11_test PROPERTY CXX_STANDARD 14)\nelse()\n set_property(TARGET cxx11_test PROPERTY CXX_STANDARD 11)\nendif()\nset_property(TARGET cxx11_test PROPERTY CXX_STANDARD_REQUIRED ON)\nset_property(TARGET cxx11_test PROPERTY CXX_EXTENSIONS OFF)\nbenchmark_add_test(NAME cxx11_test COMMAND cxx11_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(benchmark_test)\nbenchmark_add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(spec_arg_test)\nbenchmark_add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen)\n\ncompile_benchmark_test(spec_arg_verbosity_test)\nbenchmark_add_test(NAME spec_arg_verbosity COMMAND spec_arg_verbosity_test --v=42)\n\ncompile_benchmark_test(benchmark_setup_teardown_test)\nbenchmark_add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test)\n\ncompile_benchmark_test(filter_test)\nmacro(add_filter_test name filter expect)\n benchmark_add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01s --benchmark_filter=${filter} ${expect})\n benchmark_add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect})\nendmacro(add_filter_test)\n\ncompile_benchmark_test(benchmark_min_time_flag_time_test)\nbenchmark_add_test(NAME min_time_flag_time COMMAND benchmark_min_time_flag_time_test)\n\ncompile_benchmark_test(benchmark_min_time_flag_iters_test)\nbenchmark_add_test(NAME min_time_flag_iters COMMAND benchmark_min_time_flag_iters_test)\n\nadd_filter_test(filter_simple \"Foo\" 3)\nadd_filter_test(filter_simple_negative \"-Foo\" 2)\nadd_filter_test(filter_suffix \"BM_.*\" 4)\nadd_filter_test(filter_suffix_negative \"-BM_.*\" 1)\nadd_filter_test(filter_regex_all \".*\" 5)\nadd_filter_test(filter_regex_all_negative \"-.*\" 0)\nadd_filter_test(filter_regex_blank \"\" 5)\nadd_filter_test(filter_regex_blank_negative \"-\" 0)\nadd_filter_test(filter_regex_none \"monkey\" 0)\nadd_filter_test(filter_regex_none_negative \"-monkey\" 5)\nadd_filter_test(filter_regex_wildcard \".*Foo.*\" 3)\nadd_filter_test(filter_regex_wildcard_negative \"-.*Foo.*\" 2)\nadd_filter_test(filter_regex_begin \"^BM_.*\" 4)\nadd_filter_test(filter_regex_begin_negative \"-^BM_.*\" 1)\nadd_filter_test(filter_regex_begin2 \"^N\" 1)\nadd_filter_test(filter_regex_begin2_negative \"-^N\" 4)\nadd_filter_test(filter_regex_end \".*Ba$\" 1)\nadd_filter_test(filter_regex_end_negative \"-.*Ba$\" 4)\n\ncompile_benchmark_test(options_test)\nbenchmark_add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(basic_test)\nbenchmark_add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01s)\n\ncompile_output_test(repetitions_test)\nbenchmark_add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01s --benchmark_repetitions=3)\n\ncompile_benchmark_test(diagnostics_test)\nbenchmark_add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(skip_with_error_test)\nbenchmark_add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(donotoptimize_test)\n# Enable errors for deprecated deprecations (DoNotOptimize(Tp const& value)).\ncheck_cxx_compiler_flag(-Werror=deprecated-declarations BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG)\nif (BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG)\n target_compile_options (donotoptimize_test PRIVATE \"-Werror=deprecated-declarations\")\nendif()\n# Some of the issues with DoNotOptimize only occur when optimization is enabled\ncheck_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG)\nif (BENCHMARK_HAS_O3_FLAG)\n set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS \"-O3\")\nendif()\nbenchmark_add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(fixture_test)\nbenchmark_add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(register_benchmark_test)\nbenchmark_add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(map_test)\nbenchmark_add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(multiple_ranges_test)\nbenchmark_add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(args_product_test)\nbenchmark_add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test_with_main(link_main_test)\nbenchmark_add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01s)\n\ncompile_output_test(reporter_output_test)\nbenchmark_add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01s)\n\ncompile_output_test(templated_fixture_test)\nbenchmark_add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01s)\n\ncompile_output_test(templated_fixture_method_test)\nbenchmark_add_test(NAME templated_fixture_method_test COMMAND templated_fixture_method_test --benchmark_min_time=0.01s)\n\ncompile_output_test(user_counters_test)\nbenchmark_add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.2s)\n\ncompile_output_test(user_counters_threads_test)\nbenchmark_add_test(NAME user_counters_threads_test COMMAND user_counters_threads_test --benchmark_min_time=0.2s)\n\ncompile_output_test(perf_counters_test)\nbenchmark_add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01s --benchmark_perf_counters=CYCLES,INSTRUCTIONS)\n\ncompile_output_test(internal_threading_test)\nbenchmark_add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01s)\n\ncompile_output_test(manual_threading_test)\nbenchmark_add_test(NAME manual_threading_test COMMAND manual_threading_test --benchmark_min_time=0.01s)\n\ncompile_output_test(report_aggregates_only_test)\nbenchmark_add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01s)\n\ncompile_output_test(display_aggregates_only_test)\nbenchmark_add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01s)\n\ncompile_output_test(user_counters_tabular_test)\nbenchmark_add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.2s)\n\ncompile_output_test(user_counters_thousands_test)\nbenchmark_add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01s)\n\ncompile_output_test(memory_manager_test)\nbenchmark_add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01s)\n\ncompile_output_test(profiler_manager_test)\nbenchmark_add_test(NAME profiler_manager_test COMMAND profiler_manager_test --benchmark_min_time=0.01s)\n\ncompile_benchmark_test(profiler_manager_iterations_test)\nbenchmark_add_test(NAME profiler_manager_iterations COMMAND profiler_manager_iterations_test)\n\ncompile_output_test(complexity_test)\nbenchmark_add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=1000000x)\n\ncompile_output_test(locale_impermeability_test)\nbenchmark_add_test(NAME locale_impermeability_test COMMAND locale_impermeability_test)\n\n###############################################################################\n# GoogleTest Unit Tests\n###############################################################################\n\nif (BENCHMARK_ENABLE_GTEST_TESTS)\n macro(compile_gtest name)\n add_executable(${name} \"${name}.cc\")\n target_link_libraries(${name} benchmark::benchmark\n gmock_main ${CMAKE_THREAD_LIBS_INIT})\n endmacro(compile_gtest)\n\n macro(add_gtest name)\n compile_gtest(${name})\n benchmark_add_test(NAME ${name} COMMAND ${name})\n if(WIN32 AND BUILD_SHARED_LIBS)\n set_tests_properties(${name} PROPERTIES\n ENVIRONMENT_MODIFICATION \"PATH=path_list_prepend:$;PATH=path_list_prepend:$\"\n )\n endif()\n endmacro()\n\n add_gtest(benchmark_gtest)\n add_gtest(benchmark_name_gtest)\n add_gtest(benchmark_random_interleaving_gtest)\n add_gtest(commandlineflags_gtest)\n add_gtest(statistics_gtest)\n add_gtest(string_util_gtest)\n add_gtest(perf_counters_gtest)\n add_gtest(time_unit_gtest)\n add_gtest(min_time_parse_gtest)\n add_gtest(profiler_manager_gtest)\n add_gtest(benchmark_setup_teardown_cb_types_gtest)\n add_gtest(memory_results_gtest)\nendif(BENCHMARK_ENABLE_GTEST_TESTS)\n\n###############################################################################\n# Assembly Unit Tests\n###############################################################################\n\nif (BENCHMARK_ENABLE_ASSEMBLY_TESTS)\n if (NOT LLVM_FILECHECK_EXE)\n message(FATAL_ERROR \"LLVM FileCheck is required when including this file\")\n endif()\n include(AssemblyTests.cmake)\n add_filecheck_test(donotoptimize_assembly_test)\n add_filecheck_test(state_assembly_test)\n add_filecheck_test(clobber_memory_assembly_test)\nendif()\n\n\n\n###############################################################################\n# Code Coverage Configuration\n###############################################################################\n\n# Add the coverage command(s)\nif(CMAKE_BUILD_TYPE)\n string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)\nendif()\nif (${CMAKE_BUILD_TYPE_LOWER} MATCHES \"coverage\")\n find_program(GCOV gcov)\n find_program(LCOV lcov)\n find_program(GENHTML genhtml)\n find_program(CTEST ctest)\n if (GCOV AND LCOV AND GENHTML AND CTEST AND HAVE_CXX_FLAG_COVERAGE)\n add_custom_command(\n OUTPUT ${CMAKE_BINARY_DIR}/lcov/index.html\n COMMAND ${LCOV} -q -z -d .\n COMMAND ${LCOV} -q --no-external -c -b \"${CMAKE_SOURCE_DIR}\" -d . -o before.lcov -i\n COMMAND ${CTEST} --force-new-ctest-process\n COMMAND ${LCOV} -q --no-external -c -b \"${CMAKE_SOURCE_DIR}\" -d . -o after.lcov\n COMMAND ${LCOV} -q -a before.lcov -a after.lcov --output-file final.lcov\n COMMAND ${LCOV} -q -r final.lcov \"'${CMAKE_SOURCE_DIR}/test/*'\" -o final.lcov\n COMMAND ${GENHTML} final.lcov -o lcov --demangle-cpp --sort -p \"${CMAKE_BINARY_DIR}\" -t benchmark\n DEPENDS filter_test benchmark_test options_test basic_test fixture_test complexity_test\n WORKING_DIRECTORY ${CMAKE_BINARY_DIR}\n COMMENT \"Running LCOV\"\n )\n add_custom_target(coverage\n DEPENDS ${CMAKE_BINARY_DIR}/lcov/index.html\n COMMENT \"LCOV report at lcov/index.html\"\n )\n message(STATUS \"Coverage command added\")\n else()\n if (HAVE_CXX_FLAG_COVERAGE)\n set(CXX_FLAG_COVERAGE_MESSAGE supported)\n else()\n set(CXX_FLAG_COVERAGE_MESSAGE unavailable)\n endif()\n message(WARNING\n \"Coverage not available:\\n\"\n \" gcov: ${GCOV}\\n\"\n \" lcov: ${LCOV}\\n\"\n \" genhtml: ${GENHTML}\\n\"\n \" ctest: ${CTEST}\\n\"\n \" --coverage flag: ${CXX_FLAG_COVERAGE_MESSAGE}\")\n endif()\nendif()\n" }, { "path": "test/args_product_test.cc", "content": "#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n\nclass ArgsProductFixture : public ::benchmark::Fixture {\n public:\n ArgsProductFixture()\n : expectedValues({{0, 100, 2000, 30000},\n {1, 15, 3, 8},\n {1, 15, 3, 9},\n {1, 15, 7, 8},\n {1, 15, 7, 9},\n {1, 15, 10, 8},\n {1, 15, 10, 9},\n {2, 15, 3, 8},\n {2, 15, 3, 9},\n {2, 15, 7, 8},\n {2, 15, 7, 9},\n {2, 15, 10, 8},\n {2, 15, 10, 9},\n {4, 5, 6, 11}}) {}\n\n void SetUp(const ::benchmark::State& state) override {\n std::vector ranges = {state.range(0), state.range(1),\n state.range(2), state.range(3)};\n\n assert(expectedValues.find(ranges) != expectedValues.end());\n\n actualValues.insert(ranges);\n }\n\n // NOTE: This is not TearDown as we want to check after _all_ runs are\n // complete.\n ~ArgsProductFixture() override {\n if (actualValues != expectedValues) {\n std::cout << \"EXPECTED\\n\";\n for (const auto& v : expectedValues) {\n std::cout << \"{\";\n for (int64_t iv : v) {\n std::cout << iv << \", \";\n }\n std::cout << \"}\\n\";\n }\n std::cout << \"ACTUAL\\n\";\n for (const auto& v : actualValues) {\n std::cout << \"{\";\n for (int64_t iv : v) {\n std::cout << iv << \", \";\n }\n std::cout << \"}\\n\";\n }\n }\n }\n\n std::set> expectedValues;\n std::set> actualValues;\n};\n\nBENCHMARK_DEFINE_F(ArgsProductFixture, Empty)(benchmark::State& state) {\n for (auto _ : state) {\n int64_t product =\n state.range(0) * state.range(1) * state.range(2) * state.range(3);\n for (int64_t x = 0; x < product; x++) {\n benchmark::DoNotOptimize(x);\n }\n }\n}\n\nBENCHMARK_REGISTER_F(ArgsProductFixture, Empty)\n ->Args({0, 100, 2000, 30000})\n ->ArgsProduct({{1, 2}, {15}, {3, 7, 10}, {8, 9}})\n ->Args({4, 5, 6, 11});\n\nBENCHMARK_MAIN();\n" }, { "path": "test/basic_test.cc", "content": "\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/types.h\"\n#include \"benchmark/utils.h\"\n\n#define BASIC_BENCHMARK_TEST(x) BENCHMARK(x)->Arg(8)->Arg(512)->Arg(8192)\n\nnamespace {\nvoid BM_empty(benchmark::State& state) {\n for (auto _ : state) {\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n}\nBENCHMARK(BM_empty);\nBENCHMARK(BM_empty)->ThreadPerCpu();\n\nvoid BM_spin_empty(benchmark::State& state) {\n for (auto _ : state) {\n for (auto x = 0; x < state.range(0); ++x) {\n benchmark::DoNotOptimize(x);\n }\n }\n}\nBASIC_BENCHMARK_TEST(BM_spin_empty);\nBASIC_BENCHMARK_TEST(BM_spin_empty)->ThreadPerCpu();\n\nvoid BM_spin_pause_before(benchmark::State& state) {\n for (auto i = 0; i < state.range(0); ++i) {\n benchmark::DoNotOptimize(i);\n }\n for (auto _ : state) {\n for (auto i = 0; i < state.range(0); ++i) {\n benchmark::DoNotOptimize(i);\n }\n }\n}\nBASIC_BENCHMARK_TEST(BM_spin_pause_before);\nBASIC_BENCHMARK_TEST(BM_spin_pause_before)->ThreadPerCpu();\n\nvoid BM_spin_pause_during(benchmark::State& state) {\n for (auto _ : state) {\n state.PauseTiming();\n for (auto i = 0; i < state.range(0); ++i) {\n benchmark::DoNotOptimize(i);\n }\n state.ResumeTiming();\n for (auto i = 0; i < state.range(0); ++i) {\n benchmark::DoNotOptimize(i);\n }\n }\n}\nBASIC_BENCHMARK_TEST(BM_spin_pause_during);\nBASIC_BENCHMARK_TEST(BM_spin_pause_during)->ThreadPerCpu();\n\nvoid BM_pause_during(benchmark::State& state) {\n for (auto _ : state) {\n state.PauseTiming();\n state.ResumeTiming();\n }\n}\nBENCHMARK(BM_pause_during);\nBENCHMARK(BM_pause_during)->ThreadPerCpu();\nBENCHMARK(BM_pause_during)->UseRealTime();\nBENCHMARK(BM_pause_during)->UseRealTime()->ThreadPerCpu();\n\nvoid BM_spin_pause_after(benchmark::State& state) {\n for (auto _ : state) {\n for (auto i = 0; i < state.range(0); ++i) {\n benchmark::DoNotOptimize(i);\n }\n }\n for (auto i = 0; i < state.range(0); ++i) {\n benchmark::DoNotOptimize(i);\n }\n}\nBASIC_BENCHMARK_TEST(BM_spin_pause_after);\nBASIC_BENCHMARK_TEST(BM_spin_pause_after)->ThreadPerCpu();\n\nvoid BM_spin_pause_before_and_after(benchmark::State& state) {\n for (auto i = 0; i < state.range(0); ++i) {\n benchmark::DoNotOptimize(i);\n }\n for (auto _ : state) {\n for (auto i = 0; i < state.range(0); ++i) {\n benchmark::DoNotOptimize(i);\n }\n }\n for (auto i = 0; i < state.range(0); ++i) {\n benchmark::DoNotOptimize(i);\n }\n}\nBASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after);\nBASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after)->ThreadPerCpu();\n\nvoid BM_empty_stop_start(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_empty_stop_start);\nBENCHMARK(BM_empty_stop_start)->ThreadPerCpu();\n\nvoid BM_KeepRunning(benchmark::State& state) {\n benchmark::IterationCount iter_count = 0;\n assert(iter_count == state.iterations());\n while (state.KeepRunning()) {\n ++iter_count;\n }\n assert(iter_count == state.iterations());\n}\nBENCHMARK(BM_KeepRunning);\n\nvoid BM_KeepRunningBatch(benchmark::State& state) {\n // Choose a batch size >1000 to skip the typical runs with iteration\n // targets of 10, 100 and 1000. If these are not actually skipped the\n // bug would be detectable as consecutive runs with the same iteration\n // count. Below we assert that this does not happen.\n const benchmark::IterationCount batch_size = 1009;\n\n static benchmark::IterationCount prior_iter_count = 0;\n benchmark::IterationCount iter_count = 0;\n while (state.KeepRunningBatch(batch_size)) {\n iter_count += batch_size;\n }\n assert(state.iterations() == iter_count);\n\n // Verify that the iteration count always increases across runs (see\n // comment above).\n assert(iter_count == batch_size // max_iterations == 1\n || iter_count > prior_iter_count); // max_iterations > batch_size\n prior_iter_count = iter_count;\n}\n// Register with a fixed repetition count to establish the invariant that\n// the iteration count should always change across runs. This overrides\n// the --benchmark_repetitions command line flag, which would otherwise\n// cause this test to fail if set > 1.\nBENCHMARK(BM_KeepRunningBatch)->Repetitions(1);\n\nvoid BM_RangedFor(benchmark::State& state) {\n benchmark::IterationCount iter_count = 0;\n for (auto _ : state) {\n ++iter_count;\n }\n assert(iter_count == state.max_iterations);\n}\nBENCHMARK(BM_RangedFor);\n\ntemplate \nvoid BM_OneTemplateFunc(benchmark::State& state) {\n auto arg = state.range(0);\n T sum = 0;\n for (auto _ : state) {\n sum += static_cast(arg);\n }\n}\nBENCHMARK(BM_OneTemplateFunc)->Arg(1);\nBENCHMARK(BM_OneTemplateFunc)->Arg(1);\n\ntemplate \nvoid BM_TwoTemplateFunc(benchmark::State& state) {\n auto arg = state.range(0);\n A sum = 0;\n B prod = 1;\n for (auto _ : state) {\n sum += static_cast(arg);\n prod *= static_cast(arg);\n }\n}\nBENCHMARK(BM_TwoTemplateFunc)->Arg(1);\nBENCHMARK(BM_TwoTemplateFunc)->Arg(1);\n\n// Ensure that StateIterator provides all the necessary typedefs required to\n// instantiate std::iterator_traits.\nstatic_assert(\n std::is_same::value_type,\n typename benchmark::State::StateIterator::value_type>::value,\n \"\");\n} // end namespace\n\nBENCHMARK_MAIN();\n" }, { "path": "test/benchmark_gtest.cc", "content": "#include \n#include \n#include \n\n#include \"../src/benchmark_register.h\"\n#include \"benchmark/benchmark_api.h\"\n#include \"gmock/gmock.h\"\n#include \"gtest/gtest.h\"\n\nnamespace benchmark {\nnamespace internal {\n\nnamespace {\n\nTEST(AddRangeTest, Simple) {\n std::vector dst;\n AddRange(&dst, 1, 2, 2);\n EXPECT_THAT(dst, testing::ElementsAre(1, 2));\n}\n\nTEST(AddRangeTest, Simple64) {\n std::vector dst;\n AddRange(&dst, static_cast(1), static_cast(2), 2);\n EXPECT_THAT(dst, testing::ElementsAre(1, 2));\n}\n\nTEST(AddRangeTest, Advanced) {\n std::vector dst;\n AddRange(&dst, 5, 15, 2);\n EXPECT_THAT(dst, testing::ElementsAre(5, 8, 15));\n}\n\nTEST(AddRangeTest, Advanced64) {\n std::vector dst;\n AddRange(&dst, static_cast(5), static_cast(15), 2);\n EXPECT_THAT(dst, testing::ElementsAre(5, 8, 15));\n}\n\nTEST(AddRangeTest, FullRange8) {\n std::vector dst;\n AddRange(&dst, int8_t{1}, std::numeric_limits::max(), 8);\n EXPECT_THAT(\n dst, testing::ElementsAre(int8_t{1}, int8_t{8}, int8_t{64}, int8_t{127}));\n}\n\nTEST(AddRangeTest, FullRange64) {\n std::vector dst;\n AddRange(&dst, int64_t{1}, std::numeric_limits::max(), 1024);\n EXPECT_THAT(\n dst, testing::ElementsAre(1LL, 1024LL, 1048576LL, 1073741824LL,\n 1099511627776LL, 1125899906842624LL,\n 1152921504606846976LL, 9223372036854775807LL));\n}\n\nTEST(AddRangeTest, NegativeRanges) {\n std::vector dst;\n AddRange(&dst, -8, 0, 2);\n EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1, 0));\n}\n\nTEST(AddRangeTest, StrictlyNegative) {\n std::vector dst;\n AddRange(&dst, -8, -1, 2);\n EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1));\n}\n\nTEST(AddRangeTest, SymmetricNegativeRanges) {\n std::vector dst;\n AddRange(&dst, -8, 8, 2);\n EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1, 0, 1, 2, 4, 8));\n}\n\nTEST(AddRangeTest, SymmetricNegativeRangesOddMult) {\n std::vector dst;\n AddRange(&dst, -30, 32, 5);\n EXPECT_THAT(dst, testing::ElementsAre(-30, -25, -5, -1, 0, 1, 5, 25, 32));\n}\n\nTEST(AddRangeTest, NegativeRangesAsymmetric) {\n std::vector dst;\n AddRange(&dst, -3, 5, 2);\n EXPECT_THAT(dst, testing::ElementsAre(-3, -2, -1, 0, 1, 2, 4, 5));\n}\n\nTEST(AddRangeTest, NegativeRangesLargeStep) {\n // Always include -1, 0, 1 when crossing zero.\n std::vector dst;\n AddRange(&dst, -8, 8, 10);\n EXPECT_THAT(dst, testing::ElementsAre(-8, -1, 0, 1, 8));\n}\n\nTEST(AddRangeTest, ZeroOnlyRange) {\n std::vector dst;\n AddRange(&dst, 0, 0, 2);\n EXPECT_THAT(dst, testing::ElementsAre(0));\n}\n\nTEST(AddRangeTest, ZeroStartingRange) {\n std::vector dst;\n AddRange(&dst, 0, 2, 2);\n EXPECT_THAT(dst, testing::ElementsAre(0, 1, 2));\n}\n\nTEST(AddRangeTest, NegativeRange64) {\n std::vector dst;\n AddRange(&dst, -4, 4, 2);\n EXPECT_THAT(dst, testing::ElementsAre(-4, -2, -1, 0, 1, 2, 4));\n}\n\nTEST(AddRangeTest, NegativeRangePreservesExistingOrder) {\n // If elements already exist in the range, ensure we don't change\n // their ordering by adding negative values.\n std::vector dst = {1, 2, 3};\n AddRange(&dst, -2, 2, 2);\n EXPECT_THAT(dst, testing::ElementsAre(1, 2, 3, -2, -1, 0, 1, 2));\n}\n\nTEST(AddRangeTest, FullNegativeRange64) {\n std::vector dst;\n const auto min = std::numeric_limits::min();\n const auto max = std::numeric_limits::max();\n AddRange(&dst, min, max, 1024);\n EXPECT_THAT(\n dst, testing::ElementsAreArray(std::vector{\n min, -1152921504606846976LL, -1125899906842624LL,\n -1099511627776LL, -1073741824LL, -1048576LL, -1024LL, -1LL, 0LL,\n 1LL, 1024LL, 1048576LL, 1073741824LL, 1099511627776LL,\n 1125899906842624LL, 1152921504606846976LL, max}));\n}\n\nTEST(AddRangeTest, Simple8) {\n std::vector dst;\n AddRange(&dst, int8_t{1}, int8_t{8}, int8_t{2});\n EXPECT_THAT(dst,\n testing::ElementsAre(int8_t{1}, int8_t{2}, int8_t{4}, int8_t{8}));\n}\n\nTEST(AddCustomContext, Simple) {\n std::map*& global_context = GetGlobalContext();\n EXPECT_THAT(global_context, nullptr);\n\n AddCustomContext(\"foo\", \"bar\");\n AddCustomContext(\"baz\", \"qux\");\n\n EXPECT_THAT(*global_context,\n testing::UnorderedElementsAre(testing::Pair(\"foo\", \"bar\"),\n testing::Pair(\"baz\", \"qux\")));\n\n delete global_context;\n global_context = nullptr;\n}\n\nTEST(AddCustomContext, DuplicateKey) {\n std::map*& global_context = GetGlobalContext();\n EXPECT_THAT(global_context, nullptr);\n\n AddCustomContext(\"foo\", \"bar\");\n AddCustomContext(\"foo\", \"qux\");\n\n EXPECT_THAT(*global_context,\n testing::UnorderedElementsAre(testing::Pair(\"foo\", \"bar\")));\n\n delete global_context;\n global_context = nullptr;\n}\n\n} // namespace\n} // namespace internal\n} // namespace benchmark\n" }, { "path": "test/benchmark_min_time_flag_iters_test.cc", "content": "#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n\n// Tests that we can specify the number of iterations with\n// --benchmark_min_time=x.\nnamespace {\n\nclass TestReporter : public benchmark::ConsoleReporter {\n public:\n bool ReportContext(const Context& context) override {\n return ConsoleReporter::ReportContext(context);\n };\n\n void ReportRuns(const std::vector& report) override {\n assert(report.size() == 1);\n iter_nums_.push_back(report[0].iterations);\n ConsoleReporter::ReportRuns(report);\n };\n\n TestReporter() {}\n\n ~TestReporter() override {}\n\n const std::vector& GetIters() const {\n return iter_nums_;\n }\n\n private:\n std::vector iter_nums_;\n};\n\nvoid BM_MyBench(benchmark::State& state) {\n for (auto s : state) {\n }\n}\n} // end namespace\n\nBENCHMARK(BM_MyBench);\n\nint main(int argc, char** argv) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n\n // Make a fake argv and append the new --benchmark_min_time= to it.\n int fake_argc = argc + 1;\n std::vector fake_argv(static_cast(fake_argc));\n for (size_t i = 0; i < static_cast(argc); ++i) {\n fake_argv[i] = argv[i];\n }\n fake_argv[static_cast(argc)] = \"--benchmark_min_time=4x\";\n\n benchmark::Initialize(&fake_argc, const_cast(fake_argv.data()));\n\n TestReporter test_reporter;\n const size_t returned_count =\n benchmark::RunSpecifiedBenchmarks(&test_reporter, \"BM_MyBench\");\n assert(returned_count == 1);\n\n // Check the executed iters.\n const std::vector iters = test_reporter.GetIters();\n assert(!iters.empty() && iters[0] == 4);\n\n return 0;\n}\n" }, { "path": "test/benchmark_min_time_flag_time_test.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n\n// Tests that we can specify the min time with\n// --benchmark_min_time= (no suffix needed) OR\n// --benchmark_min_time=s\nnamespace {\n\n// This is from benchmark.h\ntypedef int64_t IterationCount;\n\nclass TestReporter : public benchmark::ConsoleReporter {\n public:\n bool ReportContext(const Context& context) override {\n return ConsoleReporter::ReportContext(context);\n };\n\n void ReportRuns(const std::vector& report) override {\n assert(report.size() == 1);\n ConsoleReporter::ReportRuns(report);\n };\n\n void ReportRunsConfig(double min_time, bool /* has_explicit_iters */,\n IterationCount /* iters */) override {\n min_times_.push_back(min_time);\n }\n\n TestReporter() {}\n\n ~TestReporter() override {}\n\n const std::vector& GetMinTimes() const { return min_times_; }\n\n private:\n std::vector min_times_;\n};\n\nbool AlmostEqual(double a, double b) {\n return std::fabs(a - b) < std::numeric_limits::epsilon();\n}\n\nvoid DoTestHelper(int* argc, const char** argv, double expected) {\n benchmark::Initialize(argc, const_cast(argv));\n\n TestReporter test_reporter;\n const size_t returned_count =\n benchmark::RunSpecifiedBenchmarks(&test_reporter, \"BM_MyBench\");\n assert(returned_count == 1);\n\n // Check the min_time\n const std::vector& min_times = test_reporter.GetMinTimes();\n assert(!min_times.empty() && AlmostEqual(min_times[0], expected));\n}\n\nvoid BM_MyBench(benchmark::State& state) {\n for (auto s : state) {\n }\n}\nBENCHMARK(BM_MyBench);\n\n} // end namespace\n\nint main(int argc, char** argv) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n\n // Make a fake argv and append the new --benchmark_min_time= to it.\n int fake_argc = argc + 1;\n std::vector fake_argv(static_cast(fake_argc));\n\n for (size_t i = 0; i < static_cast(argc); ++i) {\n fake_argv[i] = argv[i];\n }\n\n const char* no_suffix = \"--benchmark_min_time=4\";\n const char* with_suffix = \"--benchmark_min_time=4.0s\";\n double expected = 4.0;\n\n fake_argv[static_cast(argc)] = no_suffix;\n DoTestHelper(&fake_argc, fake_argv.data(), expected);\n\n fake_argv[static_cast(argc)] = with_suffix;\n DoTestHelper(&fake_argc, fake_argv.data(), expected);\n\n return 0;\n}\n" }, { "path": "test/benchmark_name_gtest.cc", "content": "#include \"benchmark/reporter.h\"\n#include \"gtest/gtest.h\"\n\nnamespace {\n\nusing namespace benchmark;\nusing namespace benchmark::internal;\n\nTEST(BenchmarkNameTest, Empty) {\n const auto name = BenchmarkName();\n EXPECT_EQ(name.str(), std::string());\n}\n\nTEST(BenchmarkNameTest, FunctionName) {\n auto name = BenchmarkName();\n name.function_name = \"function_name\";\n EXPECT_EQ(name.str(), \"function_name\");\n}\n\nTEST(BenchmarkNameTest, FunctionNameAndArgs) {\n auto name = BenchmarkName();\n name.function_name = \"function_name\";\n name.args = \"some_args:3/4/5\";\n EXPECT_EQ(name.str(), \"function_name/some_args:3/4/5\");\n}\n\nTEST(BenchmarkNameTest, MinTime) {\n auto name = BenchmarkName();\n name.function_name = \"function_name\";\n name.args = \"some_args:3/4\";\n name.min_time = \"min_time:3.4s\";\n EXPECT_EQ(name.str(), \"function_name/some_args:3/4/min_time:3.4s\");\n}\n\nTEST(BenchmarkNameTest, MinWarmUpTime) {\n auto name = BenchmarkName();\n name.function_name = \"function_name\";\n name.args = \"some_args:3/4\";\n name.min_warmup_time = \"min_warmup_time:3.5s\";\n EXPECT_EQ(name.str(), \"function_name/some_args:3/4/min_warmup_time:3.5s\");\n}\n\nTEST(BenchmarkNameTest, Iterations) {\n auto name = BenchmarkName();\n name.function_name = \"function_name\";\n name.min_time = \"min_time:3.4s\";\n name.iterations = \"iterations:42\";\n EXPECT_EQ(name.str(), \"function_name/min_time:3.4s/iterations:42\");\n}\n\nTEST(BenchmarkNameTest, Repetitions) {\n auto name = BenchmarkName();\n name.function_name = \"function_name\";\n name.min_time = \"min_time:3.4s\";\n name.repetitions = \"repetitions:24\";\n EXPECT_EQ(name.str(), \"function_name/min_time:3.4s/repetitions:24\");\n}\n\nTEST(BenchmarkNameTest, TimeType) {\n auto name = BenchmarkName();\n name.function_name = \"function_name\";\n name.min_time = \"min_time:3.4s\";\n name.time_type = \"hammer_time\";\n EXPECT_EQ(name.str(), \"function_name/min_time:3.4s/hammer_time\");\n}\n\nTEST(BenchmarkNameTest, Threads) {\n auto name = BenchmarkName();\n name.function_name = \"function_name\";\n name.min_time = \"min_time:3.4s\";\n name.threads = \"threads:256\";\n EXPECT_EQ(name.str(), \"function_name/min_time:3.4s/threads:256\");\n}\n\nTEST(BenchmarkNameTest, TestEmptyFunctionName) {\n auto name = BenchmarkName();\n name.args = \"first:3/second:4\";\n name.threads = \"threads:22\";\n EXPECT_EQ(name.str(), \"first:3/second:4/threads:22\");\n}\n\n} // end namespace\n" }, { "path": "test/benchmark_random_interleaving_gtest.cc", "content": "#include \n#include \n#include \n\n#include \"../src/commandlineflags.h\"\n#include \"../src/string_util.h\"\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n#include \"gmock/gmock.h\"\n#include \"gtest/gtest.h\"\n\nnamespace benchmark {\n\nBM_DECLARE_bool(benchmark_enable_random_interleaving);\nBM_DECLARE_string(benchmark_filter);\nBM_DECLARE_int32(benchmark_repetitions);\n\nnamespace internal {\nnamespace {\n\nclass EventQueue : public std::queue {\n public:\n void Put(const std::string& event) { push(event); }\n\n void Clear() {\n while (!empty()) {\n pop();\n }\n }\n\n std::string Get() {\n std::string event = front();\n pop();\n return event;\n }\n};\n\n// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)\nEventQueue* const queue = new EventQueue();\n\nclass NullReporter : public BenchmarkReporter {\n public:\n bool ReportContext(const Context& /*context*/) override { return true; }\n void ReportRuns(const std::vector& /* report */) override {}\n};\n\nclass BenchmarkTest : public testing::Test {\n public:\n static void SetupHook(int /* num_threads */) { queue->push(\"Setup\"); }\n\n static void TeardownHook(int /* num_threads */) { queue->push(\"Teardown\"); }\n\n static void Execute(const std::string& pattern) {\n queue->Clear();\n\n std::unique_ptr reporter(new NullReporter());\n FLAGS_benchmark_filter = pattern;\n RunSpecifiedBenchmarks(reporter.get());\n\n queue->Put(\"DONE\"); // End marker\n }\n};\n\nvoid BM_Match1(benchmark::State& state) {\n const int64_t arg = state.range(0);\n\n for (auto _ : state) {\n }\n queue->Put(StrFormat(\"BM_Match1/%d\", static_cast(arg)));\n}\nBENCHMARK(BM_Match1)\n ->Iterations(100)\n ->Arg(1)\n ->Arg(2)\n ->Arg(3)\n ->Range(10, 80)\n ->Args({90})\n ->Args({100});\n\nTEST_F(BenchmarkTest, Match1) {\n Execute(\"BM_Match1\");\n ASSERT_EQ(\"BM_Match1/1\", queue->Get());\n ASSERT_EQ(\"BM_Match1/2\", queue->Get());\n ASSERT_EQ(\"BM_Match1/3\", queue->Get());\n ASSERT_EQ(\"BM_Match1/10\", queue->Get());\n ASSERT_EQ(\"BM_Match1/64\", queue->Get());\n ASSERT_EQ(\"BM_Match1/80\", queue->Get());\n ASSERT_EQ(\"BM_Match1/90\", queue->Get());\n ASSERT_EQ(\"BM_Match1/100\", queue->Get());\n ASSERT_EQ(\"DONE\", queue->Get());\n}\n\nTEST_F(BenchmarkTest, Match1WithRepetition) {\n FLAGS_benchmark_repetitions = 2;\n\n Execute(\"BM_Match1/(64|80)\");\n ASSERT_EQ(\"BM_Match1/64\", queue->Get());\n ASSERT_EQ(\"BM_Match1/64\", queue->Get());\n ASSERT_EQ(\"BM_Match1/80\", queue->Get());\n ASSERT_EQ(\"BM_Match1/80\", queue->Get());\n ASSERT_EQ(\"DONE\", queue->Get());\n}\n\nTEST_F(BenchmarkTest, Match1WithRandomInterleaving) {\n FLAGS_benchmark_enable_random_interleaving = true;\n FLAGS_benchmark_repetitions = 100;\n\n std::map element_count;\n std::map interleaving_count;\n Execute(\"BM_Match1/(64|80)\");\n for (int i = 0; i < 100; ++i) {\n std::vector interleaving;\n interleaving.push_back(queue->Get());\n interleaving.push_back(queue->Get());\n element_count[interleaving[0]]++;\n element_count[interleaving[1]]++;\n interleaving_count[StrFormat(\"%s,%s\", interleaving[0].c_str(),\n interleaving[1].c_str())]++;\n }\n EXPECT_EQ(element_count[\"BM_Match1/64\"], 100) << \"Unexpected repetitions.\";\n EXPECT_EQ(element_count[\"BM_Match1/80\"], 100) << \"Unexpected repetitions.\";\n EXPECT_GE(interleaving_count.size(), 2) << \"Interleaving was not randomized.\";\n ASSERT_EQ(\"DONE\", queue->Get());\n}\n\n} // namespace\n} // namespace internal\n} // namespace benchmark\n" }, { "path": "test/benchmark_setup_teardown_cb_types_gtest.cc", "content": "#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/types.h\"\n#include \"gtest/gtest.h\"\n\nusing benchmark::Benchmark;\nusing benchmark::BenchmarkReporter;\nusing benchmark::callback_function;\nusing benchmark::ClearRegisteredBenchmarks;\nusing benchmark::RegisterBenchmark;\nusing benchmark::RunSpecifiedBenchmarks;\nusing benchmark::State;\n\nstatic int functor_called = 0;\nstruct Functor {\n void operator()(const benchmark::State& /*unused*/) { functor_called++; }\n};\n\nclass NullReporter : public BenchmarkReporter {\n public:\n bool ReportContext(const Context& /*context*/) override { return true; }\n void ReportRuns(const std::vector& /* report */) override {}\n};\n\nclass BenchmarkTest : public testing::Test {\n public:\n Benchmark* bm;\n NullReporter null_reporter;\n\n int setup_calls;\n int teardown_calls;\n\n void SetUp() override {\n setup_calls = 0;\n teardown_calls = 0;\n functor_called = 0;\n\n bm = RegisterBenchmark(\"BM\", [](State& st) {\n for (auto _ : st) {\n }\n });\n bm->Iterations(1);\n }\n\n void TearDown() override { ClearRegisteredBenchmarks(); }\n};\n\n// Test that Setup/Teardown can correctly take a lambda expressions\nTEST_F(BenchmarkTest, LambdaTestCopy) {\n auto setup_lambda = [this](const State&) { setup_calls++; };\n auto teardown_lambda = [this](const State&) { teardown_calls++; };\n bm->Setup(setup_lambda);\n bm->Teardown(teardown_lambda);\n RunSpecifiedBenchmarks(&null_reporter);\n EXPECT_EQ(setup_calls, 1);\n EXPECT_EQ(teardown_calls, 1);\n}\n\n// Test that Setup/Teardown can correctly take a lambda expressions\nTEST_F(BenchmarkTest, LambdaTestMove) {\n auto setup_lambda = [this](const State&) { setup_calls++; };\n auto teardown_lambda = [this](const State&) { teardown_calls++; };\n bm->Setup(std::move(setup_lambda));\n bm->Teardown(std::move(teardown_lambda));\n RunSpecifiedBenchmarks(&null_reporter);\n EXPECT_EQ(setup_calls, 1);\n EXPECT_EQ(teardown_calls, 1);\n}\n\n// Test that Setup/Teardown can correctly take std::function\nTEST_F(BenchmarkTest, CallbackFunctionCopy) {\n callback_function setup_lambda = [this](const State&) { setup_calls++; };\n callback_function teardown_lambda = [this](const State&) {\n teardown_calls++;\n };\n bm->Setup(setup_lambda);\n bm->Teardown(teardown_lambda);\n RunSpecifiedBenchmarks(&null_reporter);\n EXPECT_EQ(setup_calls, 1);\n EXPECT_EQ(teardown_calls, 1);\n}\n\n// Test that Setup/Teardown can correctly take std::function\nTEST_F(BenchmarkTest, CallbackFunctionMove) {\n callback_function setup_lambda = [this](const State&) { setup_calls++; };\n callback_function teardown_lambda = [this](const State&) {\n teardown_calls++;\n };\n bm->Setup(std::move(setup_lambda));\n bm->Teardown(std::move(teardown_lambda));\n RunSpecifiedBenchmarks(&null_reporter);\n EXPECT_EQ(setup_calls, 1);\n EXPECT_EQ(teardown_calls, 1);\n}\n\n// Test that Setup/Teardown can correctly take functors\nTEST_F(BenchmarkTest, FunctorCopy) {\n Functor func;\n bm->Setup(func);\n bm->Teardown(func);\n RunSpecifiedBenchmarks(&null_reporter);\n EXPECT_EQ(functor_called, 2);\n}\n\n// Test that Setup/Teardown can correctly take functors\nTEST_F(BenchmarkTest, FunctorMove) {\n Functor func1;\n Functor func2;\n bm->Setup(std::move(func1));\n bm->Teardown(std::move(func2));\n RunSpecifiedBenchmarks(&null_reporter);\n EXPECT_EQ(functor_called, 2);\n}\n\n// Test that Setup/Teardown can not take nullptr\nTEST_F(BenchmarkTest, NullptrTest) {\n#if GTEST_HAS_DEATH_TEST\n // Tests only runnable in debug mode (when BM_CHECK is enabled).\n#ifndef NDEBUG\n#ifndef TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS\n EXPECT_DEATH(bm->Setup(nullptr), \"setup != nullptr\");\n EXPECT_DEATH(bm->Teardown(nullptr), \"teardown != nullptr\");\n#else\n GTEST_SKIP() << \"Test skipped because BM_CHECK is disabled\";\n#endif\n#endif\n#endif\n}\n" }, { "path": "test/benchmark_setup_teardown_test.cc", "content": "#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n\n// Test that Setup() and Teardown() are called exactly once\n// for each benchmark run (single-threaded).\nnamespace {\nnamespace singlethreaded {\nstatic int setup_call = 0;\nstatic int teardown_call = 0;\n} // namespace singlethreaded\n} // namespace\nstatic void DoSetup1(const benchmark::State& state) {\n ++singlethreaded::setup_call;\n\n // Setup/Teardown should never be called with any thread_idx != 0.\n assert(state.thread_index() == 0);\n}\n\nstatic void DoTeardown1(const benchmark::State& state) {\n ++singlethreaded::teardown_call;\n assert(state.thread_index() == 0);\n}\n\nstatic void BM_with_setup(benchmark::State& state) {\n for (auto s : state) {\n }\n}\nBENCHMARK(BM_with_setup)\n ->Arg(1)\n ->Arg(3)\n ->Arg(5)\n ->Arg(7)\n ->Iterations(100)\n ->Setup(DoSetup1)\n ->Teardown(DoTeardown1);\n\n// Test that Setup() and Teardown() are called once for each group of threads.\nnamespace {\nnamespace concurrent {\nstatic std::atomic setup_call(0);\nstatic std::atomic teardown_call(0);\nstatic std::atomic func_call(0);\n} // namespace concurrent\n\nvoid DoSetup2(const benchmark::State& state) {\n concurrent::setup_call.fetch_add(1, std::memory_order_acquire);\n assert(state.thread_index() == 0);\n}\n\nvoid DoTeardown2(const benchmark::State& state) {\n concurrent::teardown_call.fetch_add(1, std::memory_order_acquire);\n assert(state.thread_index() == 0);\n}\n\nvoid BM_concurrent(benchmark::State& state) {\n for (auto s : state) {\n }\n concurrent::func_call.fetch_add(1, std::memory_order_acquire);\n}\n\nBENCHMARK(BM_concurrent)\n ->Setup(DoSetup2)\n ->Teardown(DoTeardown2)\n ->Iterations(100)\n ->Threads(5)\n ->Threads(10)\n ->Threads(15);\n\n// Testing interaction with Fixture::Setup/Teardown\nnamespace fixture_interaction {\nint setup = 0;\nint fixture_setup = 0;\n} // namespace fixture_interaction\n\n#define FIXTURE_BECHMARK_NAME MyFixture\n\nclass FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture {\n public:\n void SetUp(const ::benchmark::State& /*unused*/) override {\n fixture_interaction::fixture_setup++;\n }\n\n ~FIXTURE_BECHMARK_NAME() override {}\n};\n\nBENCHMARK_F(FIXTURE_BECHMARK_NAME, BM_WithFixture)(benchmark::State& st) {\n for (auto _ : st) {\n }\n}\n\nvoid DoSetupWithFixture(const benchmark::State& /*unused*/) {\n fixture_interaction::setup++;\n}\n\nBENCHMARK_REGISTER_F(FIXTURE_BECHMARK_NAME, BM_WithFixture)\n ->Arg(1)\n ->Arg(3)\n ->Arg(5)\n ->Arg(7)\n ->Setup(DoSetupWithFixture)\n ->Repetitions(1)\n ->Iterations(100);\n\n// Testing repetitions.\nnamespace repetitions {\nint setup = 0;\n}\n\nvoid DoSetupWithRepetitions(const benchmark::State& /*unused*/) {\n repetitions::setup++;\n}\nvoid BM_WithRep(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\n\nBENCHMARK(BM_WithRep)\n ->Arg(1)\n ->Arg(3)\n ->Arg(5)\n ->Arg(7)\n ->Setup(DoSetupWithRepetitions)\n ->Iterations(100)\n ->Repetitions(4);\n} // namespace\n\nint main(int argc, char** argv) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n\n benchmark::Initialize(&argc, argv);\n\n size_t ret = benchmark::RunSpecifiedBenchmarks(\".\");\n assert(ret > 0);\n\n // Setup/Teardown is called once for each arg group (1,3,5,7).\n assert(singlethreaded::setup_call == 4);\n assert(singlethreaded::teardown_call == 4);\n\n // 3 group of threads calling this function (3,5,10).\n assert(concurrent::setup_call.load(std::memory_order_relaxed) == 3);\n assert(concurrent::teardown_call.load(std::memory_order_relaxed) == 3);\n assert((5 + 10 + 15) ==\n concurrent::func_call.load(std::memory_order_relaxed));\n\n // Setup is called 4 times, once for each arg group (1,3,5,7)\n assert(fixture_interaction::setup == 4);\n // Fixture::Setup is called every time the bm routine is run.\n // The exact number is indeterministic, so we just assert that\n // it's more than setup.\n assert(fixture_interaction::fixture_setup > fixture_interaction::setup);\n\n // Setup is call once for each repetition * num_arg = 4 * 4 = 16.\n assert(repetitions::setup == 16);\n\n return 0;\n}\n" }, { "path": "test/benchmark_test.cc", "content": "#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n\n#if defined(__GNUC__)\n#define BENCHMARK_NOINLINE __attribute__((noinline))\n#else\n#define BENCHMARK_NOINLINE\n#endif\n\nnamespace {\n\nint BENCHMARK_NOINLINE Factorial(int n) {\n return (n == 1) ? 1 : n * Factorial(n - 1);\n}\n\ndouble CalculatePi(int depth) {\n double pi = 0.0;\n for (int i = 0; i < depth; ++i) {\n double numerator = static_cast(((i % 2) * 2) - 1);\n double denominator = static_cast((2 * i) - 1);\n pi += numerator / denominator;\n }\n return (pi - 1.0) * 4;\n}\n\nstd::set ConstructRandomSet(int64_t size) {\n std::set s;\n for (int i = 0; i < size; ++i) {\n s.insert(s.end(), i);\n }\n return s;\n}\n\n// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables)\nstd::mutex test_vector_mu;\nstd::optional> test_vector;\n// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables)\n\nvoid BM_Factorial(benchmark::State& state) {\n int fac_42 = 0;\n for (auto _ : state) {\n fac_42 = Factorial(8);\n }\n // Prevent compiler optimizations\n std::stringstream ss;\n ss << fac_42;\n state.SetLabel(ss.str());\n}\nBENCHMARK(BM_Factorial);\nBENCHMARK(BM_Factorial)->UseRealTime();\n\nvoid BM_CalculatePiRange(benchmark::State& state) {\n double pi = 0.0;\n for (auto _ : state) {\n pi = CalculatePi(static_cast(state.range(0)));\n }\n std::stringstream ss;\n ss << pi;\n state.SetLabel(ss.str());\n}\nBENCHMARK_RANGE(BM_CalculatePiRange, 1, 1024 * 1024);\n\nvoid BM_CalculatePi(benchmark::State& state) {\n static const int depth = 1024;\n for (auto _ : state) {\n double pi = CalculatePi(static_cast(depth));\n benchmark::DoNotOptimize(pi);\n }\n}\nBENCHMARK(BM_CalculatePi)->Threads(8);\nBENCHMARK(BM_CalculatePi)->ThreadRange(1, 32);\nBENCHMARK(BM_CalculatePi)->ThreadPerCpu();\n\nvoid BM_SetInsert(benchmark::State& state) {\n std::set data;\n for (auto _ : state) {\n state.PauseTiming();\n data = ConstructRandomSet(state.range(0));\n state.ResumeTiming();\n for (int j = 0; j < state.range(1); ++j) {\n data.insert(rand());\n }\n }\n state.SetItemsProcessed(state.iterations() * state.range(1));\n state.SetBytesProcessed(state.iterations() * state.range(1) *\n static_cast(sizeof(int)));\n}\n\n// Test many inserts at once to reduce the total iterations needed. Otherwise,\n// the slower, non-timed part of each iteration will make the benchmark take\n// forever.\nBENCHMARK(BM_SetInsert)->Ranges({{1 << 10, 8 << 10}, {128, 512}});\n\ntemplate \nvoid BM_Sequential(benchmark::State& state) {\n ValueType v = 42;\n for (auto _ : state) {\n Container c;\n for (int64_t i = state.range(0); --i;) {\n c.push_back(v);\n }\n }\n const int64_t items_processed = state.iterations() * state.range(0);\n state.SetItemsProcessed(items_processed);\n state.SetBytesProcessed(items_processed * static_cast(sizeof(v)));\n}\nBENCHMARK_TEMPLATE2(BM_Sequential, std::vector, int)\n ->Range(1 << 0, 1 << 10);\nBENCHMARK_TEMPLATE(BM_Sequential, std::list)->Range(1 << 0, 1 << 10);\n// Test the variadic version of BENCHMARK_TEMPLATE in C++11 and beyond.\nBENCHMARK_TEMPLATE(BM_Sequential, std::vector, int)->Arg(512);\n\nvoid BM_StringCompare(benchmark::State& state) {\n size_t len = static_cast(state.range(0));\n std::string s1(len, '-');\n std::string s2(len, '-');\n for (auto _ : state) {\n auto comp = s1.compare(s2);\n benchmark::DoNotOptimize(comp);\n }\n}\nBENCHMARK(BM_StringCompare)->Range(1, 1 << 20);\n\nvoid BM_SetupTeardown(benchmark::State& state) {\n if (state.thread_index() == 0) {\n // No need to lock test_vector_mu here as this is running single-threaded.\n test_vector = std::vector();\n }\n int i = 0;\n for (auto _ : state) {\n std::lock_guard l(test_vector_mu);\n if (i % 2 == 0) {\n test_vector->push_back(i);\n } else {\n test_vector->pop_back();\n }\n ++i;\n }\n if (state.thread_index() == 0) {\n test_vector.reset();\n }\n}\nBENCHMARK(BM_SetupTeardown)->ThreadPerCpu();\n\nvoid BM_LongTest(benchmark::State& state) {\n double tracker = 0.0;\n for (auto _ : state) {\n for (int i = 0; i < state.range(0); ++i) {\n benchmark::DoNotOptimize(tracker += i);\n }\n }\n}\nBENCHMARK(BM_LongTest)->Range(1 << 16, 1 << 28);\n\nvoid BM_ParallelMemset(benchmark::State& state) {\n int64_t size = state.range(0) / static_cast(sizeof(int));\n int thread_size = static_cast(size) / state.threads();\n int from = thread_size * state.thread_index();\n int to = from + thread_size;\n\n if (state.thread_index() == 0) {\n test_vector = std::vector(static_cast(size));\n }\n\n for (auto _ : state) {\n for (int i = from; i < to; i++) {\n // No need to lock test_vector_mu as ranges\n // do not overlap between threads.\n benchmark::DoNotOptimize(test_vector->at(static_cast(i)) = 1);\n }\n }\n\n if (state.thread_index() == 0) {\n test_vector.reset();\n }\n}\nBENCHMARK(BM_ParallelMemset)->Arg(10 << 20)->ThreadRange(1, 4);\n\nvoid BM_ManualTiming(benchmark::State& state) {\n int64_t slept_for = 0;\n int64_t microseconds = state.range(0);\n std::chrono::duration sleep_duration{\n static_cast(microseconds)};\n\n for (auto _ : state) {\n auto start = std::chrono::high_resolution_clock::now();\n // Simulate some useful workload with a sleep\n std::this_thread::sleep_for(\n std::chrono::duration_cast(sleep_duration));\n auto end = std::chrono::high_resolution_clock::now();\n\n auto elapsed =\n std::chrono::duration_cast>(end - start);\n\n state.SetIterationTime(elapsed.count());\n slept_for += microseconds;\n }\n state.SetItemsProcessed(slept_for);\n}\nBENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseRealTime();\nBENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseManualTime();\n\ntemplate \nvoid BM_with_args(benchmark::State& state, Args&&...) {\n for (auto _ : state) {\n }\n}\nBENCHMARK_CAPTURE(BM_with_args, int_test, 42, 43, 44);\nBENCHMARK_CAPTURE(BM_with_args, string_and_pair_test, std::string(\"abc\"),\n std::pair(42, 3.8));\n\nvoid BM_non_template_args(benchmark::State& state, int, double) {\n while (state.KeepRunning()) {\n }\n}\nBENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0);\n\ntemplate \nvoid BM_template2_capture(benchmark::State& state, ExtraArgs&&... extra_args) {\n static_assert(std::is_same::value, \"\");\n static_assert(std::is_same::value, \"\");\n static_assert(std::is_same::value, \"\");\n unsigned int dummy[sizeof...(ExtraArgs)] = {extra_args...};\n assert(dummy[0] == 42);\n for (auto _ : state) {\n }\n}\nBENCHMARK_TEMPLATE2_CAPTURE(BM_template2_capture, void, char*, foo, 42U);\nBENCHMARK_CAPTURE((BM_template2_capture), foo, 42U);\n\ntemplate \nvoid BM_template1_capture(benchmark::State& state, ExtraArgs&&... extra_args) {\n static_assert(std::is_same::value, \"\");\n static_assert(std::is_same::value, \"\");\n unsigned long dummy[sizeof...(ExtraArgs)] = {extra_args...};\n assert(dummy[0] == 24);\n for (auto _ : state) {\n }\n}\nBENCHMARK_TEMPLATE1_CAPTURE(BM_template1_capture, void, foo, 24UL);\nBENCHMARK_CAPTURE(BM_template1_capture, foo, 24UL);\n\nvoid BM_DenseThreadRanges(benchmark::State& st) {\n switch (st.range(0)) {\n case 1:\n assert(st.threads() == 1 || st.threads() == 2 || st.threads() == 3);\n break;\n case 2:\n assert(st.threads() == 1 || st.threads() == 3 || st.threads() == 4);\n break;\n case 3:\n assert(st.threads() == 5 || st.threads() == 8 || st.threads() == 11 ||\n st.threads() == 14);\n break;\n default:\n assert(false && \"Invalid test case number\");\n }\n while (st.KeepRunning()) {\n }\n}\nBENCHMARK(BM_DenseThreadRanges)->Arg(1)->DenseThreadRange(1, 3);\nBENCHMARK(BM_DenseThreadRanges)->Arg(2)->DenseThreadRange(1, 4, 2);\nBENCHMARK(BM_DenseThreadRanges)->Arg(3)->DenseThreadRange(5, 14, 3);\n\nvoid BM_BenchmarkName(benchmark::State& state) {\n for (auto _ : state) {\n }\n\n // Check that the benchmark name is passed correctly to `state`.\n assert(\"BM_BenchmarkName\" == state.name());\n}\nBENCHMARK(BM_BenchmarkName);\n\n// regression test for #1446\ntemplate \nvoid BM_templated_test(benchmark::State& state) {\n for (auto _ : state) {\n type created_string;\n benchmark::DoNotOptimize(created_string);\n }\n}\n\nconst auto BM_templated_test_double = BM_templated_test>;\nBENCHMARK(BM_templated_test_double);\n} // end namespace\n\nBENCHMARK_MAIN();\n" }, { "path": "test/clobber_memory_assembly_test.cc", "content": "#include \"benchmark/macros.h\"\n#include \"benchmark/utils.h\"\n\n#ifdef __clang__\n#pragma clang diagnostic ignored \"-Wreturn-type\"\n#endif\nBENCHMARK_DISABLE_DEPRECATED_WARNING\n\nextern \"C\" {\n\nextern int ExternInt;\nextern int ExternInt2;\nextern int ExternInt3;\n}\n\n// CHECK-LABEL: test_basic:\nextern \"C\" void test_basic() {\n int x;\n benchmark::DoNotOptimize(&x);\n x = 101;\n benchmark::ClobberMemory();\n // CHECK: leaq [[DEST:[^,]+]], %rax\n // CHECK: movl $101, [[DEST]]\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_redundant_store:\nextern \"C\" void test_redundant_store() {\n ExternInt = 3;\n benchmark::ClobberMemory();\n ExternInt = 51;\n // CHECK-DAG: ExternInt\n // CHECK-DAG: movl $3\n // CHECK: movl $51\n}\n\n// CHECK-LABEL: test_redundant_read:\nextern \"C\" void test_redundant_read() {\n int x;\n benchmark::DoNotOptimize(&x);\n x = ExternInt;\n benchmark::ClobberMemory();\n x = ExternInt2;\n // CHECK: leaq [[DEST:[^,]+]], %rax\n // CHECK: ExternInt(%rip)\n // CHECK: movl %eax, [[DEST]]\n // CHECK-NOT: ExternInt2\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_redundant_read2:\nextern \"C\" void test_redundant_read2() {\n int x;\n benchmark::DoNotOptimize(&x);\n x = ExternInt;\n benchmark::ClobberMemory();\n x = ExternInt2;\n benchmark::ClobberMemory();\n // CHECK: leaq [[DEST:[^,]+]], %rax\n // CHECK: ExternInt(%rip)\n // CHECK: movl %eax, [[DEST]]\n // CHECK: ExternInt2(%rip)\n // CHECK: movl %eax, [[DEST]]\n // CHECK: ret\n}\n" }, { "path": "test/commandlineflags_gtest.cc", "content": "#include \n\n#include \"../src/commandlineflags.h\"\n#include \"../src/internal_macros.h\"\n#include \"gmock/gmock.h\"\n#include \"gtest/gtest.h\"\n\nnamespace benchmark {\nnamespace {\n\n#if defined(BENCHMARK_OS_WINDOWS)\nint setenv(const char* name, const char* value, int overwrite) {\n if (!overwrite) {\n // NOTE: getenv_s is far superior but not available under mingw.\n char* env_value = getenv(name);\n if (env_value == nullptr) {\n return -1;\n }\n }\n return _putenv_s(name, value);\n}\n\nint unsetenv(const char* name) { return _putenv_s(name, \"\"); }\n\n#endif // BENCHMARK_OS_WINDOWS\n\nTEST(BoolFromEnv, Default) {\n ASSERT_EQ(unsetenv(\"NOT_IN_ENV\"), 0);\n EXPECT_EQ(BoolFromEnv(\"not_in_env\", true), true);\n}\n\nTEST(BoolFromEnv, False) {\n ASSERT_EQ(setenv(\"IN_ENV\", \"0\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"N\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"n\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"NO\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"No\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"no\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"F\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"f\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"FALSE\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"False\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"false\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"OFF\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"Off\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"off\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", true), false);\n unsetenv(\"IN_ENV\");\n}\n\nTEST(BoolFromEnv, True) {\n ASSERT_EQ(setenv(\"IN_ENV\", \"1\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"Y\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"y\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"YES\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"Yes\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"yes\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"T\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"t\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"TRUE\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"True\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"true\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"ON\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"On\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n ASSERT_EQ(setenv(\"IN_ENV\", \"on\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n\n#ifndef BENCHMARK_OS_WINDOWS\n ASSERT_EQ(setenv(\"IN_ENV\", \"\", 1), 0);\n EXPECT_EQ(BoolFromEnv(\"in_env\", false), true);\n unsetenv(\"IN_ENV\");\n#endif\n}\n\nTEST(Int32FromEnv, NotInEnv) {\n ASSERT_EQ(unsetenv(\"NOT_IN_ENV\"), 0);\n EXPECT_EQ(Int32FromEnv(\"not_in_env\", 42), 42);\n}\n\nTEST(Int32FromEnv, InvalidInteger) {\n ASSERT_EQ(setenv(\"IN_ENV\", \"foo\", 1), 0);\n EXPECT_EQ(Int32FromEnv(\"in_env\", 42), 42);\n unsetenv(\"IN_ENV\");\n}\n\nTEST(Int32FromEnv, ValidInteger) {\n ASSERT_EQ(setenv(\"IN_ENV\", \"42\", 1), 0);\n EXPECT_EQ(Int32FromEnv(\"in_env\", 64), 42);\n unsetenv(\"IN_ENV\");\n}\n\nTEST(DoubleFromEnv, NotInEnv) {\n ASSERT_EQ(unsetenv(\"NOT_IN_ENV\"), 0);\n EXPECT_EQ(DoubleFromEnv(\"not_in_env\", 0.51), 0.51);\n}\n\nTEST(DoubleFromEnv, InvalidReal) {\n ASSERT_EQ(setenv(\"IN_ENV\", \"foo\", 1), 0);\n EXPECT_EQ(DoubleFromEnv(\"in_env\", 0.51), 0.51);\n unsetenv(\"IN_ENV\");\n}\n\nTEST(DoubleFromEnv, ValidReal) {\n ASSERT_EQ(setenv(\"IN_ENV\", \"0.51\", 1), 0);\n EXPECT_EQ(DoubleFromEnv(\"in_env\", 0.71), 0.51);\n unsetenv(\"IN_ENV\");\n}\n\nTEST(StringFromEnv, Default) {\n ASSERT_EQ(unsetenv(\"NOT_IN_ENV\"), 0);\n EXPECT_STREQ(StringFromEnv(\"not_in_env\", \"foo\"), \"foo\");\n}\n\nTEST(StringFromEnv, Valid) {\n ASSERT_EQ(setenv(\"IN_ENV\", \"foo\", 1), 0);\n EXPECT_STREQ(StringFromEnv(\"in_env\", \"bar\"), \"foo\");\n unsetenv(\"IN_ENV\");\n}\n\nTEST(KvPairsFromEnv, Default) {\n ASSERT_EQ(unsetenv(\"NOT_IN_ENV\"), 0);\n EXPECT_THAT(KvPairsFromEnv(\"not_in_env\", {{\"foo\", \"bar\"}}),\n testing::ElementsAre(testing::Pair(\"foo\", \"bar\")));\n}\n\nTEST(KvPairsFromEnv, MalformedReturnsDefault) {\n ASSERT_EQ(setenv(\"IN_ENV\", \"foo\", 1), 0);\n EXPECT_THAT(KvPairsFromEnv(\"in_env\", {{\"foo\", \"bar\"}}),\n testing::ElementsAre(testing::Pair(\"foo\", \"bar\")));\n unsetenv(\"IN_ENV\");\n}\n\nTEST(KvPairsFromEnv, Single) {\n ASSERT_EQ(setenv(\"IN_ENV\", \"foo=bar\", 1), 0);\n EXPECT_THAT(KvPairsFromEnv(\"in_env\", {}),\n testing::ElementsAre(testing::Pair(\"foo\", \"bar\")));\n unsetenv(\"IN_ENV\");\n}\n\nTEST(KvPairsFromEnv, Multiple) {\n ASSERT_EQ(setenv(\"IN_ENV\", \"foo=bar,baz=qux\", 1), 0);\n EXPECT_THAT(KvPairsFromEnv(\"in_env\", {}),\n testing::UnorderedElementsAre(testing::Pair(\"foo\", \"bar\"),\n testing::Pair(\"baz\", \"qux\")));\n unsetenv(\"IN_ENV\");\n}\n\n} // namespace\n} // namespace benchmark\n" }, { "path": "test/complexity_test.cc", "content": "#undef NDEBUG\n#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/statistics.h\"\n#include \"benchmark/types.h\"\n#include \"benchmark/utils.h\"\n#include \"output_test.h\"\n\nnamespace {\n\n#define ADD_COMPLEXITY_CASES(...) \\\n const int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__)\n\nint AddComplexityTest(const std::string& test_name,\n const std::string& big_o_test_name,\n const std::string& rms_test_name,\n const std::string& big_o, int family_index) {\n SetSubstitutions({{\"%name\", test_name},\n {\"%bigo_name\", big_o_test_name},\n {\"%rms_name\", rms_test_name},\n {\"%bigo_str\", \"[ ]* %float \" + big_o},\n {\"%bigo\", big_o},\n {\"%rms\", \"[ ]*[0-9]+ %\"}});\n AddCases(\n TC_ConsoleOut,\n {{\"^%bigo_name %bigo_str %bigo_str[ ]*$\"},\n {\"^%bigo_name\", MR_Not}, // Assert we we didn't only matched a name.\n {\"^%rms_name %rms %rms[ ]*$\", MR_Next}});\n AddCases(\n TC_JSONOut,\n {{\"\\\"name\\\": \\\"%bigo_name\\\",$\"},\n {\"\\\"family_index\\\": \" + std::to_string(family_index) + \",$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"%name\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": %int,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"BigO\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"cpu_coefficient\\\": %float,$\", MR_Next},\n {\"\\\"real_coefficient\\\": %float,$\", MR_Next},\n {\"\\\"big_o\\\": \\\"%bigo\\\",$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next},\n {\"\\\"name\\\": \\\"%rms_name\\\",$\"},\n {\"\\\"family_index\\\": \" + std::to_string(family_index) + \",$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"%name\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": %int,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"RMS\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"percentage\\\",$\", MR_Next},\n {\"\\\"rms\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\n AddCases(TC_CSVOut, {{\"^\\\"%bigo_name\\\",,%float,%float,%bigo,,,,,$\"},\n {\"^\\\"%bigo_name\\\"\", MR_Not},\n {\"^\\\"%rms_name\\\",,%float,%float,,,,,,$\", MR_Next}});\n return 0;\n}\n\n// ========================================================================= //\n// --------------------------- Testing BigO O(1) --------------------------- //\n// ========================================================================= //\n\nvoid BM_Complexity_O1(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n benchmark::DoNotOptimize(state.iterations());\n double tmp = static_cast(state.iterations());\n benchmark::DoNotOptimize(tmp);\n for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) {\n benchmark::DoNotOptimize(state.iterations());\n tmp *= static_cast(state.iterations());\n benchmark::DoNotOptimize(tmp);\n }\n\n // always 1ns per iteration\n state.SetIterationTime(42 * 1e-9);\n }\n state.SetComplexityN(state.range(0));\n}\nBENCHMARK(BM_Complexity_O1)\n ->Range(1, 1 << 18)\n ->UseManualTime()\n ->Complexity(benchmark::o1);\nBENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->UseManualTime()->Complexity();\nBENCHMARK(BM_Complexity_O1)\n ->Range(1, 1 << 18)\n ->UseManualTime()\n ->Complexity([](benchmark::IterationCount) { return 1.0; });\n\nconstexpr char one_test_name[] = \"BM_Complexity_O1/manual_time\";\nconstexpr char big_o_1_test_name[] = \"BM_Complexity_O1/manual_time_BigO\";\nconstexpr char rms_o_1_test_name[] = \"BM_Complexity_O1/manual_time_RMS\";\nconstexpr char enum_auto_big_o_1[] = \"\\\\([0-9]+\\\\)\";\nconstexpr char lambda_big_o_1[] = \"f\\\\(N\\\\)\";\n\n// Add enum tests\nADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,\n enum_auto_big_o_1, /*family_index=*/0);\n\n// Add auto tests\nADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,\n enum_auto_big_o_1, /*family_index=*/1);\n\n// Add lambda tests\nADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,\n lambda_big_o_1, /*family_index=*/2);\n\n// ========================================================================= //\n// --------------------------- Testing BigO O(N) --------------------------- //\n// ========================================================================= //\n\nvoid BM_Complexity_O_N(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n benchmark::DoNotOptimize(state.iterations());\n double tmp = static_cast(state.iterations());\n benchmark::DoNotOptimize(tmp);\n for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) {\n benchmark::DoNotOptimize(state.iterations());\n tmp *= static_cast(state.iterations());\n benchmark::DoNotOptimize(tmp);\n }\n\n // 1ns per iteration per entry\n state.SetIterationTime(static_cast(state.range(0)) * 42 * 1e-9);\n }\n state.SetComplexityN(state.range(0));\n}\nBENCHMARK(BM_Complexity_O_N)\n ->RangeMultiplier(2)\n ->Range(1 << 10, 1 << 20)\n ->UseManualTime()\n ->Complexity(benchmark::oN);\nBENCHMARK(BM_Complexity_O_N)\n ->RangeMultiplier(2)\n ->Range(1 << 10, 1 << 20)\n ->UseManualTime()\n ->Complexity();\nBENCHMARK(BM_Complexity_O_N)\n ->RangeMultiplier(2)\n ->Range(1 << 10, 1 << 20)\n ->UseManualTime()\n ->Complexity([](benchmark::IterationCount n) -> double {\n return static_cast(n);\n });\n\nconstexpr char n_test_name[] = \"BM_Complexity_O_N/manual_time\";\nconstexpr char big_o_n_test_name[] = \"BM_Complexity_O_N/manual_time_BigO\";\nconstexpr char rms_o_n_test_name[] = \"BM_Complexity_O_N/manual_time_RMS\";\nconstexpr char enum_auto_big_o_n[] = \"N\";\nconstexpr char lambda_big_o_n[] = \"f\\\\(N\\\\)\";\n\n// Add enum tests\nADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,\n enum_auto_big_o_n, /*family_index=*/3);\n\n// Add auto tests\nADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,\n enum_auto_big_o_n, /*family_index=*/4);\n\n// Add lambda tests\nADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,\n lambda_big_o_n, /*family_index=*/5);\n\n// ========================================================================= //\n// ------------------------- Testing BigO O(NlgN) ------------------------- //\n// ========================================================================= //\n\nconst double kLog2E = 1.44269504088896340736;\nvoid BM_Complexity_O_N_log_N(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n benchmark::DoNotOptimize(state.iterations());\n double tmp = static_cast(state.iterations());\n benchmark::DoNotOptimize(tmp);\n for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) {\n benchmark::DoNotOptimize(state.iterations());\n tmp *= static_cast(state.iterations());\n benchmark::DoNotOptimize(tmp);\n }\n\n state.SetIterationTime(static_cast(state.range(0)) * kLog2E *\n std::log(state.range(0)) * 42 * 1e-9);\n }\n state.SetComplexityN(state.range(0));\n}\nBENCHMARK(BM_Complexity_O_N_log_N)\n ->RangeMultiplier(2)\n ->Range(1 << 10, 1U << 24)\n ->UseManualTime()\n ->Complexity(benchmark::oNLogN);\nBENCHMARK(BM_Complexity_O_N_log_N)\n ->RangeMultiplier(2)\n ->Range(1 << 10, 1U << 24)\n ->UseManualTime()\n ->Complexity();\nBENCHMARK(BM_Complexity_O_N_log_N)\n ->RangeMultiplier(2)\n ->Range(1 << 10, 1U << 24)\n ->UseManualTime()\n ->Complexity([](benchmark::IterationCount n) {\n return kLog2E * static_cast(n) * std::log(static_cast(n));\n });\n\nconstexpr char n_lg_n_test_name[] = \"BM_Complexity_O_N_log_N/manual_time\";\nconstexpr char big_o_n_lg_n_test_name[] =\n \"BM_Complexity_O_N_log_N/manual_time_BigO\";\nconstexpr char rms_o_n_lg_n_test_name[] =\n \"BM_Complexity_O_N_log_N/manual_time_RMS\";\nconstexpr char enum_auto_big_o_n_lg_n[] = \"NlgN\";\nconstexpr char lambda_big_o_n_lg_n[] = \"f\\\\(N\\\\)\";\n\n// Add enum tests\nADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,\n rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n,\n /*family_index=*/6);\n\n// NOTE: auto big-o is wron.g\nADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,\n rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n,\n /*family_index=*/7);\n\n//// Add lambda tests\nADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,\n rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n,\n /*family_index=*/8);\n\n// ========================================================================= //\n// -------- Testing formatting of Complexity with captured args ------------ //\n// ========================================================================= //\n\nvoid BM_ComplexityCaptureArgs(benchmark::State& state, int n) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n benchmark::DoNotOptimize(state.iterations());\n double tmp = static_cast(state.iterations());\n benchmark::DoNotOptimize(tmp);\n for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) {\n benchmark::DoNotOptimize(state.iterations());\n tmp *= static_cast(state.iterations());\n benchmark::DoNotOptimize(tmp);\n }\n\n state.SetIterationTime(static_cast(state.range(0)) * 42 * 1e-9);\n }\n state.SetComplexityN(n);\n}\n\nBENCHMARK_CAPTURE(BM_ComplexityCaptureArgs, capture_test, 100)\n ->UseManualTime()\n ->Complexity(benchmark::oN)\n ->Ranges({{1, 2}, {3, 4}});\n\nconst std::string complexity_capture_name =\n \"BM_ComplexityCaptureArgs/capture_test/manual_time\";\n\nADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + \"_BigO\",\n complexity_capture_name + \"_RMS\", \"N\",\n /*family_index=*/9);\n} // end namespace\n\n// ========================================================================= //\n// --------------------------- TEST CASES END ------------------------------ //\n// ========================================================================= //\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n RunOutputTests(argc, argv);\n}\n" }, { "path": "test/cxx11_test.cc", "content": "#include \"benchmark/benchmark_api.h\"\n\n#if defined(_MSC_VER)\n#if _MSVC_LANG != 201402L\n// MSVC, even in C++11 mode, dooes not claim to be in C++11 mode.\n#error \"Trying to compile C++11 test with wrong C++ standard\"\n#endif // _MSVC_LANG\n#else // Non-MSVC\n#if __cplusplus != 201103L\n#error \"Trying to compile C++11 test with wrong C++ standard\"\n#endif // Non-MSVC\n#endif\n" }, { "path": "test/diagnostics_test.cc", "content": "// Testing:\n// State::PauseTiming()\n// State::ResumeTiming()\n// Test that CHECK's within these function diagnose when they are called\n// outside of the KeepRunning() loop.\n//\n// NOTE: Users should NOT include or use src/check.h. This is only done in\n// order to test library internals.\n\n#include \n#include \n\n#include \"../src/check.h\"\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n\n#if defined(__GNUC__) && !defined(__EXCEPTIONS)\n#define TEST_HAS_NO_EXCEPTIONS\n#endif\n\nnamespace {\nvoid TestHandler() {\n#ifndef TEST_HAS_NO_EXCEPTIONS\n throw std::logic_error(\"\");\n#else\n std::abort();\n#endif\n}\n\nvoid try_invalid_pause_resume(benchmark::State& state) {\n#if !defined(TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS) && \\\n !defined(TEST_HAS_NO_EXCEPTIONS)\n try {\n state.PauseTiming();\n std::abort();\n } catch (std::logic_error const&) {\n }\n try {\n state.ResumeTiming();\n std::abort();\n } catch (std::logic_error const&) {\n }\n#else\n (void)state; // avoid unused warning\n#endif\n}\n\nvoid BM_diagnostic_test(benchmark::State& state) {\n static bool called_once = false;\n\n if (!called_once) {\n try_invalid_pause_resume(state);\n }\n\n for (auto _ : state) {\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n\n if (!called_once) {\n try_invalid_pause_resume(state);\n }\n\n called_once = true;\n}\nBENCHMARK(BM_diagnostic_test);\n\nvoid BM_diagnostic_test_keep_running(benchmark::State& state) {\n static bool called_once = false;\n\n if (!called_once) {\n try_invalid_pause_resume(state);\n }\n\n while (state.KeepRunning()) {\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n\n if (!called_once) {\n try_invalid_pause_resume(state);\n }\n\n called_once = true;\n}\nBENCHMARK(BM_diagnostic_test_keep_running);\n} // end namespace\n\nint main(int argc, char* argv[]) {\n#ifdef NDEBUG\n // This test is exercising functionality for debug builds, which are not\n // available in release builds. Skip the test if we are in that environment\n // to avoid a test failure.\n std::cout << \"Diagnostic test disabled in release build\\n\";\n (void)argc;\n (void)argv;\n#else\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n benchmark::internal::GetAbortHandler() = &TestHandler;\n benchmark::Initialize(&argc, argv);\n benchmark::RunSpecifiedBenchmarks();\n#endif\n}\n" }, { "path": "test/display_aggregates_only_test.cc", "content": "\n#undef NDEBUG\n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"output_test.h\"\n\n// Ok this test is super ugly. We want to check what happens with the file\n// reporter in the presence of DisplayAggregatesOnly().\n// We do not care about console output, the normal tests check that already.\n\nnamespace {\nvoid BM_SummaryRepeat(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_SummaryRepeat)->Repetitions(3)->DisplayAggregatesOnly();\n} // end namespace\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n\n const std::string output = GetFileReporterOutput(argc, argv);\n\n if (SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3\") != 7 ||\n SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3\\\"\") != 3 ||\n SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_mean\\\"\") != 1 ||\n SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_median\\\"\") !=\n 1 ||\n SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_stddev\\\"\") !=\n 1 ||\n SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_cv\\\"\") != 1) {\n std::cout << \"Precondition mismatch. Expected to only find 8 \"\n \"occurrences of \\\"BM_SummaryRepeat/repeats:3\\\" substring:\\n\"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3\\\", \"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3\\\", \"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3\\\", \"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_mean\\\", \"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_median\\\", \"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_stddev\\\", \"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_cv\\\"\\nThe entire \"\n \"output:\\n\";\n std::cout << output;\n return 1;\n }\n\n return 0;\n}\n" }, { "path": "test/donotoptimize_assembly_test.cc", "content": "#include \"benchmark/macros.h\"\n#include \"benchmark/utils.h\"\n\n#ifdef __clang__\n#pragma clang diagnostic ignored \"-Wreturn-type\"\n#pragma clang diagnostic ignored \"-Wmissing-prototypes\"\n#endif\nBENCHMARK_DISABLE_DEPRECATED_WARNING\n\nextern \"C\" {\n\nextern int ExternInt;\nextern int ExternInt2;\nextern int ExternInt3;\nextern int BigArray[2049];\n\nconst int ConstBigArray[2049]{};\n\ninline int Add42(int x) { return x + 42; }\n\nstruct NotTriviallyCopyable {\n NotTriviallyCopyable();\n explicit NotTriviallyCopyable(int x) : value(x) {}\n NotTriviallyCopyable(NotTriviallyCopyable const&);\n int value;\n};\n\nstruct Large {\n int value;\n int data[2];\n};\n\nstruct ExtraLarge {\n int arr[2049];\n};\n}\n\nextern ExtraLarge ExtraLargeObj;\nconst ExtraLarge ConstExtraLargeObj{};\n\n// CHECK-LABEL: test_with_rvalue:\nextern \"C\" void test_with_rvalue() {\n benchmark::DoNotOptimize(Add42(0));\n // CHECK: movl $42, %eax\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_large_rvalue:\nextern \"C\" void test_with_large_rvalue() {\n benchmark::DoNotOptimize(Large{ExternInt, {ExternInt, ExternInt}});\n // CHECK: ExternInt(%rip)\n // CHECK: movl %eax, -{{[0-9]+}}(%[[REG:[a-z]+]]\n // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]])\n // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]])\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_non_trivial_rvalue:\nextern \"C\" void test_with_non_trivial_rvalue() {\n benchmark::DoNotOptimize(NotTriviallyCopyable(ExternInt));\n // CHECK: mov{{l|q}} ExternInt(%rip)\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_lvalue:\nextern \"C\" void test_with_lvalue() {\n int x = 101;\n benchmark::DoNotOptimize(x);\n // CHECK-GNU: movl $101, %eax\n // CHECK-CLANG: movl $101, -{{[0-9]+}}(%[[REG:[a-z]+]])\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_large_lvalue:\nextern \"C\" void test_with_large_lvalue() {\n Large L{ExternInt, {ExternInt, ExternInt}};\n benchmark::DoNotOptimize(L);\n // CHECK: ExternInt(%rip)\n // CHECK: movl %eax, -{{[0-9]+}}(%[[REG:[a-z]+]])\n // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]])\n // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]])\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_extra_large_lvalue_with_op:\nextern \"C\" void test_with_extra_large_lvalue_with_op() {\n ExtraLargeObj.arr[16] = 42;\n benchmark::DoNotOptimize(ExtraLargeObj);\n // CHECK: movl $42, ExtraLargeObj+64(%rip)\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_big_array_with_op\nextern \"C\" void test_with_big_array_with_op() {\n BigArray[16] = 42;\n benchmark::DoNotOptimize(BigArray);\n // CHECK: movl $42, BigArray+64(%rip)\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_non_trivial_lvalue:\nextern \"C\" void test_with_non_trivial_lvalue() {\n NotTriviallyCopyable NTC(ExternInt);\n benchmark::DoNotOptimize(NTC);\n // CHECK: ExternInt(%rip)\n // CHECK: movl %eax, -{{[0-9]+}}(%[[REG:[a-z]+]])\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_const_lvalue:\nextern \"C\" void test_with_const_lvalue() {\n const int x = 123;\n benchmark::DoNotOptimize(x);\n // CHECK: movl $123, %eax\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_large_const_lvalue:\nextern \"C\" void test_with_large_const_lvalue() {\n const Large L{ExternInt, {ExternInt, ExternInt}};\n benchmark::DoNotOptimize(L);\n // CHECK: ExternInt(%rip)\n // CHECK: movl %eax, -{{[0-9]+}}(%[[REG:[a-z]+]])\n // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]])\n // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]])\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_const_extra_large_obj:\nextern \"C\" void test_with_const_extra_large_obj() {\n benchmark::DoNotOptimize(ConstExtraLargeObj);\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_const_big_array\nextern \"C\" void test_with_const_big_array() {\n benchmark::DoNotOptimize(ConstBigArray);\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_with_non_trivial_const_lvalue:\nextern \"C\" void test_with_non_trivial_const_lvalue() {\n const NotTriviallyCopyable Obj(ExternInt);\n benchmark::DoNotOptimize(Obj);\n // CHECK: mov{{q|l}} ExternInt(%rip)\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_div_by_two:\nextern \"C\" int test_div_by_two(int input) {\n int divisor = 2;\n benchmark::DoNotOptimize(divisor);\n return input / divisor;\n // CHECK: movl $2, [[DEST:.*]]\n // CHECK: idivl [[DEST]]\n // CHECK: ret\n}\n\n// CHECK-LABEL: test_inc_integer:\nextern \"C\" int test_inc_integer() {\n int x = 0;\n for (int i = 0; i < 5; ++i) benchmark::DoNotOptimize(++x);\n // CHECK: movl $1, [[DEST:.*]]\n // CHECK: {{(addl \\$1,|incl)}} [[DEST]]\n // CHECK: {{(addl \\$1,|incl)}} [[DEST]]\n // CHECK: {{(addl \\$1,|incl)}} [[DEST]]\n // CHECK: {{(addl \\$1,|incl)}} [[DEST]]\n // CHECK-CLANG: movl [[DEST]], %eax\n // CHECK: ret\n return x;\n}\n\n// CHECK-LABEL: test_pointer_rvalue\nextern \"C\" void test_pointer_rvalue() {\n // CHECK: movl $42, [[DEST:.*]]\n // CHECK: leaq [[DEST]], %rax\n // CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z]+]])\n // CHECK: ret\n int x = 42;\n benchmark::DoNotOptimize(&x);\n}\n\n// CHECK-LABEL: test_pointer_const_lvalue:\nextern \"C\" void test_pointer_const_lvalue() {\n // CHECK: movl $42, [[DEST:.*]]\n // CHECK: leaq [[DEST]], %rax\n // CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z]+]])\n // CHECK: ret\n int x = 42;\n int* const xp = &x;\n benchmark::DoNotOptimize(xp);\n}\n\n// CHECK-LABEL: test_pointer_lvalue:\nextern \"C\" void test_pointer_lvalue() {\n // CHECK: movl $42, [[DEST:.*]]\n // CHECK: leaq [[DEST]], %rax\n // CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z+]+]])\n // CHECK: ret\n int x = 42;\n int* xp = &x;\n benchmark::DoNotOptimize(xp);\n}\n" }, { "path": "test/donotoptimize_test.cc", "content": "#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/utils.h\"\n\nnamespace {\n#if defined(__GNUC__)\nstd::int64_t double_up(std::int64_t x) __attribute__((const));\n#endif\nstd::int64_t double_up(const std::int64_t x) { return x * 2; }\n} // namespace\n\n// Using DoNotOptimize on types like BitRef seem to cause a lot of problems\n// with the inline assembly on both GCC and Clang.\nstruct BitRef {\n int index;\n unsigned char& byte;\n\n public:\n static BitRef Make() {\n static unsigned char arr[2] = {};\n BitRef b(1, arr[0]);\n return b;\n }\n\n private:\n BitRef(int i, unsigned char& b) : index(i), byte(b) {}\n};\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n\n // this test verifies compilation of DoNotOptimize() for some types\n\n char buffer1[1] = \"\";\n benchmark::DoNotOptimize(buffer1);\n\n char buffer2[2] = \"\";\n benchmark::DoNotOptimize(buffer2);\n\n char buffer3[3] = \"\";\n benchmark::DoNotOptimize(buffer3);\n\n char buffer8[8] = \"\";\n benchmark::DoNotOptimize(buffer8);\n\n char buffer20[20] = \"\";\n benchmark::DoNotOptimize(buffer20);\n\n char buffer1024[1024] = \"\";\n benchmark::DoNotOptimize(buffer1024);\n char* bptr = &buffer1024[0];\n benchmark::DoNotOptimize(bptr);\n\n int x = 123;\n benchmark::DoNotOptimize(x);\n int* xp = &x;\n benchmark::DoNotOptimize(xp);\n benchmark::DoNotOptimize(x += 42);\n\n std::int64_t y = double_up(x);\n benchmark::DoNotOptimize(y);\n\n // These tests are to e\n BitRef lval = BitRef::Make();\n benchmark::DoNotOptimize(lval);\n\n // Check that accept rvalue.\n benchmark::DoNotOptimize(BitRef::Make());\n}\n" }, { "path": "test/filter_test.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n\nnamespace {\n\nclass TestReporter : public benchmark::ConsoleReporter {\n public:\n bool ReportContext(const Context& context) override {\n return ConsoleReporter::ReportContext(context);\n };\n\n void ReportRuns(const std::vector& report) override {\n ++count_;\n max_family_index_ = std::max(max_family_index_, report[0].family_index);\n ConsoleReporter::ReportRuns(report);\n };\n\n TestReporter() : count_(0), max_family_index_(0) {}\n\n ~TestReporter() override {}\n\n int GetCount() const { return count_; }\n\n int64_t GetMaxFamilyIndex() const { return max_family_index_; }\n\n private:\n mutable int count_;\n mutable int64_t max_family_index_;\n};\n\nvoid NoPrefix(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(NoPrefix);\n\nvoid BM_Foo(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_Foo);\n\nvoid BM_Bar(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_Bar);\n\nvoid BM_FooBar(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_FooBar);\n\nvoid BM_FooBa(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_FooBa);\n} // end namespace\n\nint main(int argc, char** argv) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n\n bool list_only = false;\n for (int i = 0; i < argc; ++i) {\n list_only |= std::string(argv[i]).find(\"--benchmark_list_tests\") !=\n std::string::npos;\n }\n\n benchmark::Initialize(&argc, argv);\n\n TestReporter test_reporter;\n const int64_t returned_count =\n static_cast(benchmark::RunSpecifiedBenchmarks(&test_reporter));\n\n if (argc == 2) {\n // Make sure we ran all of the tests\n std::stringstream ss(argv[1]);\n int64_t expected_return = 0;\n ss >> expected_return;\n\n if (returned_count != expected_return) {\n std::cerr << \"ERROR: Expected \" << expected_return\n << \" tests to match the filter but returned_count = \"\n << returned_count << '\\n';\n return -1;\n }\n\n const int64_t expected_reports = list_only ? 0 : expected_return;\n const int64_t reports_count = test_reporter.GetCount();\n if (reports_count != expected_reports) {\n std::cerr << \"ERROR: Expected \" << expected_reports\n << \" tests to be run but reported_count = \" << reports_count\n << '\\n';\n return -1;\n }\n\n const int64_t max_family_index = test_reporter.GetMaxFamilyIndex();\n const int64_t num_families = reports_count == 0 ? 0 : 1 + max_family_index;\n if (num_families != expected_reports) {\n std::cerr << \"ERROR: Expected \" << expected_reports\n << \" test families to be run but num_families = \"\n << num_families << '\\n';\n return -1;\n }\n }\n\n return 0;\n}\n" }, { "path": "test/fixture_test.cc", "content": "\n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n\n#define FIXTURE_BECHMARK_NAME MyFixture\n\nclass FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture {\n public:\n void SetUp(const ::benchmark::State& state) override {\n if (state.thread_index() == 0) {\n assert(data.get() == nullptr);\n data.reset(new int(42));\n }\n }\n\n void TearDown(const ::benchmark::State& state) override {\n if (state.thread_index() == 0) {\n assert(data.get() != nullptr);\n data.reset();\n }\n }\n\n ~FIXTURE_BECHMARK_NAME() override { assert(data == nullptr); }\n\n std::unique_ptr data;\n};\n\nBENCHMARK_F(FIXTURE_BECHMARK_NAME, Foo)(benchmark::State& st) {\n assert(data.get() != nullptr);\n assert(*data == 42);\n for (auto _ : st) {\n }\n}\n\nBENCHMARK_DEFINE_F(FIXTURE_BECHMARK_NAME, Bar)(benchmark::State& st) {\n if (st.thread_index() == 0) {\n assert(data.get() != nullptr);\n assert(*data == 42);\n }\n for (auto _ : st) {\n assert(data.get() != nullptr);\n assert(*data == 42);\n }\n st.SetItemsProcessed(st.range(0));\n}\nBENCHMARK_REGISTER_F(FIXTURE_BECHMARK_NAME, Bar)->Arg(42);\nBENCHMARK_REGISTER_F(FIXTURE_BECHMARK_NAME, Bar)->Arg(42)->ThreadPerCpu();\n\nBENCHMARK_MAIN();\n" }, { "path": "test/internal_threading_test.cc", "content": "\n#undef NDEBUG\n\n#include \n#include \n\n#include \"../src/timers.h\"\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/counter.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"output_test.h\"\n\nnamespace {\nconst std::chrono::duration time_frame(50);\nconst double time_frame_in_sec(\n std::chrono::duration_cast>>(\n time_frame)\n .count());\n\nvoid MyBusySpinwait() {\n const auto start = benchmark::ChronoClockNow();\n\n while (true) {\n const auto now = benchmark::ChronoClockNow();\n const auto elapsed = now - start;\n\n if (std::chrono::duration(elapsed) >=\n time_frame) {\n return;\n }\n }\n}\n\n// ========================================================================= //\n// --------------------------- TEST CASES BEGIN ---------------------------- //\n// ========================================================================= //\n\n// ========================================================================= //\n// BM_MainThread\n\nvoid BM_MainThread(benchmark::State& state) {\n for (auto _ : state) {\n MyBusySpinwait();\n state.SetIterationTime(time_frame_in_sec);\n }\n state.counters[\"invtime\"] =\n benchmark::Counter{1, benchmark::Counter::kIsRate};\n}\n\nBENCHMARK(BM_MainThread)->Iterations(1)->Threads(1);\nBENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->UseRealTime();\nBENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->UseManualTime();\nBENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->MeasureProcessCPUTime();\nBENCHMARK(BM_MainThread)\n ->Iterations(1)\n ->Threads(1)\n ->MeasureProcessCPUTime()\n ->UseRealTime();\nBENCHMARK(BM_MainThread)\n ->Iterations(1)\n ->Threads(1)\n ->MeasureProcessCPUTime()\n ->UseManualTime();\n\nBENCHMARK(BM_MainThread)->Iterations(1)->Threads(2);\nBENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->UseRealTime();\nBENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->UseManualTime();\nBENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->MeasureProcessCPUTime();\nBENCHMARK(BM_MainThread)\n ->Iterations(1)\n ->Threads(2)\n ->MeasureProcessCPUTime()\n ->UseRealTime();\nBENCHMARK(BM_MainThread)\n ->Iterations(1)\n ->Threads(2)\n ->MeasureProcessCPUTime()\n ->UseManualTime();\n\n// ========================================================================= //\n// BM_WorkerThread\n\nvoid BM_WorkerThread(benchmark::State& state) {\n for (auto _ : state) {\n std::thread Worker(&MyBusySpinwait);\n Worker.join();\n state.SetIterationTime(time_frame_in_sec);\n }\n state.counters[\"invtime\"] =\n benchmark::Counter{1, benchmark::Counter::kIsRate};\n}\n\nBENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1);\nBENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->UseRealTime();\nBENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->UseManualTime();\nBENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->MeasureProcessCPUTime();\nBENCHMARK(BM_WorkerThread)\n ->Iterations(1)\n ->Threads(1)\n ->MeasureProcessCPUTime()\n ->UseRealTime();\nBENCHMARK(BM_WorkerThread)\n ->Iterations(1)\n ->Threads(1)\n ->MeasureProcessCPUTime()\n ->UseManualTime();\n\nBENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2);\nBENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->UseRealTime();\nBENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->UseManualTime();\nBENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->MeasureProcessCPUTime();\nBENCHMARK(BM_WorkerThread)\n ->Iterations(1)\n ->Threads(2)\n ->MeasureProcessCPUTime()\n ->UseRealTime();\nBENCHMARK(BM_WorkerThread)\n ->Iterations(1)\n ->Threads(2)\n ->MeasureProcessCPUTime()\n ->UseManualTime();\n\n// ========================================================================= //\n// BM_MainThreadAndWorkerThread\n\nvoid BM_MainThreadAndWorkerThread(benchmark::State& state) {\n for (auto _ : state) {\n std::thread Worker(&MyBusySpinwait);\n MyBusySpinwait();\n Worker.join();\n state.SetIterationTime(time_frame_in_sec);\n }\n state.counters[\"invtime\"] =\n benchmark::Counter{1, benchmark::Counter::kIsRate};\n}\n\nBENCHMARK(BM_MainThreadAndWorkerThread)->Iterations(1)->Threads(1);\nBENCHMARK(BM_MainThreadAndWorkerThread)\n ->Iterations(1)\n ->Threads(1)\n ->UseRealTime();\nBENCHMARK(BM_MainThreadAndWorkerThread)\n ->Iterations(1)\n ->Threads(1)\n ->UseManualTime();\nBENCHMARK(BM_MainThreadAndWorkerThread)\n ->Iterations(1)\n ->Threads(1)\n ->MeasureProcessCPUTime();\nBENCHMARK(BM_MainThreadAndWorkerThread)\n ->Iterations(1)\n ->Threads(1)\n ->MeasureProcessCPUTime()\n ->UseRealTime();\nBENCHMARK(BM_MainThreadAndWorkerThread)\n ->Iterations(1)\n ->Threads(1)\n ->MeasureProcessCPUTime()\n ->UseManualTime();\n\nBENCHMARK(BM_MainThreadAndWorkerThread)->Iterations(1)->Threads(2);\nBENCHMARK(BM_MainThreadAndWorkerThread)\n ->Iterations(1)\n ->Threads(2)\n ->UseRealTime();\nBENCHMARK(BM_MainThreadAndWorkerThread)\n ->Iterations(1)\n ->Threads(2)\n ->UseManualTime();\nBENCHMARK(BM_MainThreadAndWorkerThread)\n ->Iterations(1)\n ->Threads(2)\n ->MeasureProcessCPUTime();\nBENCHMARK(BM_MainThreadAndWorkerThread)\n ->Iterations(1)\n ->Threads(2)\n ->MeasureProcessCPUTime()\n ->UseRealTime();\nBENCHMARK(BM_MainThreadAndWorkerThread)\n ->Iterations(1)\n ->Threads(2)\n ->MeasureProcessCPUTime()\n ->UseManualTime();\n} // end namespace\n\n// ========================================================================= //\n// ---------------------------- TEST CASES END ----------------------------- //\n// ========================================================================= //\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n\n RunOutputTests(argc, argv);\n}\n" }, { "path": "test/link_main_test.cc", "content": "#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n\nnamespace {\nvoid BM_empty(benchmark::State& state) {\n for (auto _ : state) {\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n}\nBENCHMARK(BM_empty);\n} // end namespace\n" }, { "path": "test/locale_impermeability_test.cc", "content": "#undef NDEBUG\n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"output_test.h\"\n\nnamespace {\nvoid BM_ostream(benchmark::State& state) {\n#if !defined(__MINGW64__) || defined(__clang__)\n // GCC-based versions of MINGW64 do not support locale manipulations,\n // don't run the test under them.\n std::locale::global(std::locale(\"en_US.UTF-8\"));\n#endif\n while (state.KeepRunning()) {\n state.SetIterationTime(1e-6);\n }\n}\nBENCHMARK(BM_ostream)->UseManualTime()->Iterations(1000000);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_ostream/iterations:1000000/manual_time\"\n \" %console_report$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_ostream/iterations:1000000/manual_time\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \"\n \"\\\"BM_ostream/iterations:1000000/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": 1000000,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_ostream/iterations:1000000/\"\n \"manual_time\\\",1000000,%float,%float,ns,,,,,$\"}});\n} // end namespace\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n RunOutputTests(argc, argv);\n}\n" }, { "path": "test/manual_threading_test.cc", "content": "\n#include \n#undef NDEBUG\n\n#include \n#include \n\n#include \"../src/timers.h\"\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/counter.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n\nnamespace {\n\nconst std::chrono::duration time_frame(50);\nconst double time_frame_in_sec(\n std::chrono::duration_cast>>(\n time_frame)\n .count());\n\nvoid MyBusySpinwait() {\n const auto start = benchmark::ChronoClockNow();\n\n while (true) {\n const auto now = benchmark::ChronoClockNow();\n const auto elapsed = now - start;\n\n if (std::chrono::duration(elapsed) >=\n time_frame) {\n return;\n }\n }\n}\n\nint numRunThreadsCalled_ = 0;\n\nclass ManualThreadRunner : public benchmark::ThreadRunnerBase {\n public:\n explicit ManualThreadRunner(int num_threads)\n : pool(static_cast(num_threads - 1)) {}\n\n void RunThreads(const std::function& fn) final {\n for (std::size_t ti = 0; ti < pool.size(); ++ti) {\n pool[ti] = std::thread(fn, static_cast(ti + 1));\n }\n\n fn(0);\n\n for (std::thread& thread : pool) {\n thread.join();\n }\n\n ++numRunThreadsCalled_;\n }\n\n private:\n std::vector pool;\n};\n\n// ========================================================================= //\n// --------------------------- TEST CASES BEGIN ---------------------------- //\n// ========================================================================= //\n\n// ========================================================================= //\n// BM_ManualThreading\n// Creation of threads is done before the start of the measurement,\n// joining after the finish of the measurement.\nvoid BM_ManualThreading(benchmark::State& state) {\n for (auto _ : state) {\n MyBusySpinwait();\n state.SetIterationTime(time_frame_in_sec);\n }\n state.counters[\"invtime\"] =\n benchmark::Counter{1, benchmark::Counter::kIsRate};\n}\n\n} // end namespace\n\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(1);\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(1)\n ->UseRealTime();\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(1)\n ->UseManualTime();\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(1)\n ->MeasureProcessCPUTime();\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(1)\n ->MeasureProcessCPUTime()\n ->UseRealTime();\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(1)\n ->MeasureProcessCPUTime()\n ->UseManualTime();\n\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(2);\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(2)\n ->UseRealTime();\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(2)\n ->UseManualTime();\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(2)\n ->MeasureProcessCPUTime();\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(2)\n ->MeasureProcessCPUTime()\n ->UseRealTime();\nBENCHMARK(BM_ManualThreading)\n ->Iterations(1)\n ->ThreadRunner([](int num_threads) {\n return std::make_unique(num_threads);\n })\n ->Threads(2)\n ->MeasureProcessCPUTime()\n ->UseManualTime();\n\n// ========================================================================= //\n// ---------------------------- TEST CASES END ----------------------------- //\n// ========================================================================= //\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n benchmark::Initialize(&argc, argv);\n benchmark::RunSpecifiedBenchmarks();\n benchmark::Shutdown();\n assert(numRunThreadsCalled_ > 0);\n}\n" }, { "path": "test/map_test.cc", "content": "#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n\nnamespace {\n\nstd::map ConstructRandomMap(int size) {\n std::map m;\n for (int i = 0; i < size; ++i) {\n m.insert(std::make_pair(std::rand() % size, std::rand() % size));\n }\n return m;\n}\n\n// Basic version.\nvoid BM_MapLookup(benchmark::State& state) {\n const int size = static_cast(state.range(0));\n std::map m;\n for (auto _ : state) {\n state.PauseTiming();\n m = ConstructRandomMap(size);\n state.ResumeTiming();\n for (int i = 0; i < size; ++i) {\n auto it = m.find(std::rand() % size);\n benchmark::DoNotOptimize(it);\n }\n }\n state.SetItemsProcessed(state.iterations() * size);\n}\nBENCHMARK(BM_MapLookup)->Range(1 << 3, 1 << 12);\n} // namespace\n\n// Using fixtures.\nclass MapFixture : public ::benchmark::Fixture {\n public:\n void SetUp(const ::benchmark::State& st) override {\n m = ConstructRandomMap(static_cast(st.range(0)));\n }\n\n void TearDown(const ::benchmark::State& /*unused*/) override { m.clear(); }\n\n std::map m;\n};\n\nBENCHMARK_DEFINE_F(MapFixture, Lookup)(benchmark::State& state) {\n const int size = static_cast(state.range(0));\n for (auto _ : state) {\n for (int i = 0; i < size; ++i) {\n auto it = m.find(std::rand() % size);\n benchmark::DoNotOptimize(it);\n }\n }\n state.SetItemsProcessed(state.iterations() * size);\n}\nBENCHMARK_REGISTER_F(MapFixture, Lookup)->Range(1 << 3, 1 << 12);\n\nBENCHMARK_MAIN();\n" }, { "path": "test/memory_manager_test.cc", "content": "#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/managers.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n#include \"output_test.h\"\n\nnamespace {\nclass TestMemoryManager : public benchmark::MemoryManager {\n void Start() override {}\n void Stop(Result& result) override {\n result.num_allocs = 42;\n result.max_bytes_used = 42000;\n }\n};\n\nvoid BM_empty(benchmark::State& state) {\n for (auto _ : state) {\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n}\nBENCHMARK(BM_empty);\n} // end namespace\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_empty %console_report$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_empty\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_empty\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"allocs_per_iter\\\": %float,$\", MR_Next},\n {\"\\\"max_bytes_used\\\": 42000$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_empty\\\",%csv_report$\"}});\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n std::unique_ptr mm(new TestMemoryManager());\n\n benchmark::RegisterMemoryManager(mm.get());\n RunOutputTests(argc, argv);\n benchmark::RegisterMemoryManager(nullptr);\n}\n" }, { "path": "test/memory_results_gtest.cc", "content": "#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/managers.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n#include \"gtest/gtest.h\"\n\nnamespace {\n\nusing benchmark::Benchmark;\nusing benchmark::ClearRegisteredBenchmarks;\nusing benchmark::ConsoleReporter;\nusing benchmark::MemoryManager;\nusing benchmark::RegisterBenchmark;\nusing benchmark::RunSpecifiedBenchmarks;\nusing benchmark::State;\n\nconstexpr int N_REPETITIONS = 100;\nconstexpr int N_ITERATIONS = 1;\n\nint num_allocs = 0;\nint max_bytes_used = 0;\nint total_allocated_bytes = 0;\nint net_heap_growth = 0;\n\nvoid reset() {\n num_allocs = 0;\n max_bytes_used = 0;\n total_allocated_bytes = 0;\n net_heap_growth = 0;\n}\nclass TestMemoryManager : public MemoryManager {\n void Start() override {}\n void Stop(Result& result) override {\n result.num_allocs = num_allocs;\n result.net_heap_growth = net_heap_growth;\n result.max_bytes_used = max_bytes_used;\n result.total_allocated_bytes = total_allocated_bytes;\n\n num_allocs += 1;\n max_bytes_used += 2;\n net_heap_growth += 4;\n total_allocated_bytes += 10;\n }\n};\n\nclass TestReporter : public ConsoleReporter {\n public:\n TestReporter() = default;\n virtual ~TestReporter() = default;\n\n bool ReportContext(const Context& /*unused*/) override { return true; }\n\n void PrintHeader(const Run&) override {}\n void PrintRunData(const Run& run) override {\n if (run.repetition_index == -1) return;\n if (!run.memory_result.memory_iterations) return;\n\n store.push_back(run.memory_result);\n }\n\n std::vector store;\n};\n\nclass MemoryResultsTest : public testing::Test {\n public:\n Benchmark* bm;\n TestReporter reporter;\n\n void SetUp() override {\n bm = RegisterBenchmark(\"BM\", [](State& st) {\n for (auto _ : st) {\n }\n });\n bm->Repetitions(N_REPETITIONS);\n bm->Iterations(N_ITERATIONS);\n reset();\n }\n void TearDown() override { ClearRegisteredBenchmarks(); }\n};\n\nTEST_F(MemoryResultsTest, NoMMTest) {\n RunSpecifiedBenchmarks(&reporter);\n EXPECT_EQ(reporter.store.size(), 0);\n}\n\nTEST_F(MemoryResultsTest, ResultsTest) {\n auto mm = std::make_unique();\n RegisterMemoryManager(mm.get());\n\n RunSpecifiedBenchmarks(&reporter);\n EXPECT_EQ(reporter.store.size(), N_REPETITIONS);\n\n for (size_t i = 0; i < reporter.store.size(); i++) {\n EXPECT_EQ(reporter.store[i].num_allocs, static_cast(i));\n EXPECT_EQ(reporter.store[i].max_bytes_used, static_cast(i) * 2);\n EXPECT_EQ(reporter.store[i].net_heap_growth, static_cast(i) * 4);\n EXPECT_EQ(reporter.store[i].total_allocated_bytes,\n static_cast(i) * 10);\n }\n}\n\n} // namespace\n" }, { "path": "test/min_time_parse_gtest.cc", "content": "#include \"../src/benchmark_runner.h\"\n#include \"gtest/gtest.h\"\n\nnamespace {\n\nTEST(ParseMinTimeTest, InvalidInput) {\n#if GTEST_HAS_DEATH_TEST\n // Tests only runnable in debug mode (when BM_CHECK is enabled).\n#ifndef NDEBUG\n#ifndef TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS\n ASSERT_DEATH_IF_SUPPORTED(\n { benchmark::internal::ParseBenchMinTime(\"abc\"); },\n \"Malformed seconds value passed to --benchmark_min_time: `abc`\");\n\n ASSERT_DEATH_IF_SUPPORTED(\n { benchmark::internal::ParseBenchMinTime(\"123ms\"); },\n \"Malformed seconds value passed to --benchmark_min_time: `123ms`\");\n\n ASSERT_DEATH_IF_SUPPORTED(\n { benchmark::internal::ParseBenchMinTime(\"1z\"); },\n \"Malformed seconds value passed to --benchmark_min_time: `1z`\");\n\n ASSERT_DEATH_IF_SUPPORTED(\n { benchmark::internal::ParseBenchMinTime(\"1hs\"); },\n \"Malformed seconds value passed to --benchmark_min_time: `1hs`\");\n#endif\n#endif\n#endif\n}\n} // namespace\n" }, { "path": "test/multiple_ranges_test.cc", "content": "#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n\nnamespace {\nclass MultipleRangesFixture : public ::benchmark::Fixture {\n public:\n MultipleRangesFixture()\n : expectedValues({{1, 3, 5},\n {1, 3, 8},\n {1, 3, 15},\n {2, 3, 5},\n {2, 3, 8},\n {2, 3, 15},\n {1, 4, 5},\n {1, 4, 8},\n {1, 4, 15},\n {2, 4, 5},\n {2, 4, 8},\n {2, 4, 15},\n {1, 7, 5},\n {1, 7, 8},\n {1, 7, 15},\n {2, 7, 5},\n {2, 7, 8},\n {2, 7, 15},\n {7, 6, 3}}) {}\n\n void SetUp(const ::benchmark::State& state) override {\n std::vector ranges = {state.range(0), state.range(1),\n state.range(2)};\n\n assert(expectedValues.find(ranges) != expectedValues.end());\n\n actualValues.insert(ranges);\n }\n\n // NOTE: This is not TearDown as we want to check after _all_ runs are\n // complete.\n ~MultipleRangesFixture() override {\n if (actualValues != expectedValues) {\n std::cout << \"EXPECTED\\n\";\n for (const auto& v : expectedValues) {\n std::cout << \"{\";\n for (int64_t iv : v) {\n std::cout << iv << \", \";\n }\n std::cout << \"}\\n\";\n }\n std::cout << \"ACTUAL\\n\";\n for (const auto& v : actualValues) {\n std::cout << \"{\";\n for (int64_t iv : v) {\n std::cout << iv << \", \";\n }\n std::cout << \"}\\n\";\n }\n }\n }\n\n std::set> expectedValues;\n std::set> actualValues;\n};\n\nBENCHMARK_DEFINE_F(MultipleRangesFixture, Empty)(benchmark::State& state) {\n for (auto _ : state) {\n int64_t product = state.range(0) * state.range(1) * state.range(2);\n for (int64_t x = 0; x < product; x++) {\n benchmark::DoNotOptimize(x);\n }\n }\n}\n\nBENCHMARK_REGISTER_F(MultipleRangesFixture, Empty)\n ->RangeMultiplier(2)\n ->Ranges({{1, 2}, {3, 7}, {5, 15}})\n ->Args({7, 6, 3});\n\nvoid BM_CheckDefaultArgument(benchmark::State& state) {\n // Test that the 'range()' without an argument is the same as 'range(0)'.\n assert(state.range() == state.range(0));\n assert(state.range() != state.range(1));\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_CheckDefaultArgument)->Ranges({{1, 5}, {6, 10}});\n\nvoid BM_MultipleRanges(benchmark::State& st) {\n for (auto _ : st) {\n }\n}\nBENCHMARK(BM_MultipleRanges)->Ranges({{5, 5}, {6, 6}});\n} // end namespace\n\nBENCHMARK_MAIN();\n" }, { "path": "test/options_test.cc", "content": "#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/types.h\"\n\n#if defined(NDEBUG)\n#undef NDEBUG\n#endif\n#include \n\nnamespace {\nvoid BM_basic(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\n\nvoid BM_basic_slow(benchmark::State& state) {\n std::chrono::milliseconds sleep_duration(state.range(0));\n for (auto _ : state) {\n std::this_thread::sleep_for(\n std::chrono::duration_cast(sleep_duration));\n }\n}\n\nBENCHMARK(BM_basic);\nBENCHMARK(BM_basic)->Arg(42);\nBENCHMARK(BM_basic_slow)->Arg(10)->Unit(benchmark::kNanosecond);\nBENCHMARK(BM_basic_slow)->Arg(100)->Unit(benchmark::kMicrosecond);\nBENCHMARK(BM_basic_slow)->Arg(1000)->Unit(benchmark::kMillisecond);\nBENCHMARK(BM_basic_slow)->Arg(1000)->Unit(benchmark::kSecond);\nBENCHMARK(BM_basic)->Range(1, 8);\nBENCHMARK(BM_basic)->RangeMultiplier(2)->Range(1, 8);\nBENCHMARK(BM_basic)->DenseRange(10, 15);\nBENCHMARK(BM_basic)->Args({42, 42});\nBENCHMARK(BM_basic)->Ranges({{64, 512}, {64, 512}});\nBENCHMARK(BM_basic)->MinTime(0.7);\nBENCHMARK(BM_basic)->MinWarmUpTime(0.8);\nBENCHMARK(BM_basic)->MinTime(0.1)->MinWarmUpTime(0.2);\nBENCHMARK(BM_basic)->UseRealTime();\nBENCHMARK(BM_basic)->ThreadRange(2, 4);\nBENCHMARK(BM_basic)->ThreadPerCpu();\nBENCHMARK(BM_basic)->Repetitions(3);\nBENCHMARK(BM_basic)\n ->RangeMultiplier(std::numeric_limits::max())\n ->Range(std::numeric_limits::min(),\n std::numeric_limits::max());\n\n// Negative ranges\nBENCHMARK(BM_basic)->Range(-64, -1);\nBENCHMARK(BM_basic)->RangeMultiplier(4)->Range(-8, 8);\nBENCHMARK(BM_basic)->DenseRange(-2, 2, 1);\nBENCHMARK(BM_basic)->Ranges({{-64, 1}, {-8, -1}});\n\nvoid CustomArgs(benchmark::Benchmark* b) {\n for (int i = 0; i < 10; ++i) {\n b->Arg(i);\n }\n}\n\nBENCHMARK(BM_basic)->Apply(CustomArgs);\n\nvoid BM_explicit_iteration_count(benchmark::State& state) {\n // Test that benchmarks specified with an explicit iteration count are\n // only run once.\n static bool invoked_before = false;\n assert(!invoked_before);\n invoked_before = true;\n\n // Test that the requested iteration count is respected.\n assert(state.max_iterations == 42);\n for (auto _ : state) {\n }\n assert(state.iterations() == state.max_iterations);\n assert(state.iterations() == 42);\n}\nBENCHMARK(BM_explicit_iteration_count)->Iterations(42);\n} // end namespace\n\nBENCHMARK_MAIN();\n" }, { "path": "test/output_test.h", "content": "#ifndef TEST_OUTPUT_TEST_H\n#define TEST_OUTPUT_TEST_H\n\n#undef NDEBUG\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"../src/re.h\"\n#include \"benchmark/benchmark_api.h\"\n\n#define CONCAT2(x, y) x##y\n#define CONCAT(x, y) CONCAT2(x, y)\n\n#define ADD_CASES(...) \\\n const int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)\n\n#define SET_SUBSTITUTIONS(...) \\\n const int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)\n\nenum MatchRules : uint8_t {\n MR_Default, // Skip non-matching lines until a match is found.\n MR_Next, // Match must occur on the next line.\n MR_Not // No line between the current position and the next match matches\n // the regex\n};\n\nstruct TestCase {\n TestCase(std::string re, int rule = MR_Default);\n\n std::string regex_str;\n int match_rule;\n std::string substituted_regex;\n std::shared_ptr regex;\n};\n\nenum TestCaseID : uint8_t {\n TC_ConsoleOut,\n TC_ConsoleErr,\n TC_JSONOut,\n TC_JSONErr,\n TC_CSVOut,\n TC_CSVErr,\n\n TC_NumID // PRIVATE\n};\n\n// Add a list of test cases to be run against the output specified by\n// 'ID'\nint AddCases(TestCaseID ID, std::initializer_list il);\n\n// Add or set a list of substitutions to be performed on constructed regex's\n// See 'output_test_helper.cc' for a list of default substitutions.\nint SetSubstitutions(\n std::initializer_list> il);\n\n// Run all output tests.\nvoid RunOutputTests(int argc, char* argv[]);\n\n// Count the number of 'pat' substrings in the 'haystack' string.\nint SubstrCnt(const std::string& haystack, const std::string& pat);\n\n// Run registered benchmarks with file reporter enabled, and return the content\n// outputted by the file reporter.\nstd::string GetFileReporterOutput(int argc, char* argv[]);\n\n// ========================================================================= //\n// ------------------------- Results checking ------------------------------ //\n// ========================================================================= //\n\n// Call this macro to register a benchmark for checking its results. This\n// should be all that's needed. It subscribes a function to check the (CSV)\n// results of a benchmark. This is done only after verifying that the output\n// strings are really as expected.\n// bm_name_pattern: a name or a regex pattern which will be matched against\n// all the benchmark names. Matching benchmarks\n// will be the subject of a call to checker_function\n// checker_function: should be of type ResultsCheckFn (see below)\n#define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \\\n const size_t CONCAT(dummy, __LINE__) = \\\n AddChecker(bm_name_pattern, checker_function)\n\nstruct Results;\ntypedef std::function ResultsCheckFn;\n\nsize_t AddChecker(const std::string& bm_name_pattern, const ResultsCheckFn& fn);\n\n// Class holding the results of a benchmark.\n// It is passed in calls to checker functions.\nstruct Results {\n // the benchmark name\n std::string name;\n // the benchmark fields\n std::map values;\n\n Results(const std::string& n) : name(n) {}\n\n int NumThreads() const;\n\n double NumIterations() const;\n\n typedef enum : uint8_t { kCpuTime, kRealTime } BenchmarkTime;\n\n // get cpu_time or real_time in seconds\n double GetTime(BenchmarkTime which) const;\n\n // get the real_time duration of the benchmark in seconds.\n // it is better to use fuzzy float checks for this, as the float\n // ASCII formatting is lossy.\n double DurationRealTime() const {\n return NumIterations() * GetTime(kRealTime);\n }\n // get the cpu_time duration of the benchmark in seconds\n double DurationCPUTime() const { return NumIterations() * GetTime(kCpuTime); }\n\n // get the string for a result by name, or nullptr if the name\n // is not found\n const std::string* Get(const std::string& entry_name) const {\n auto it = values.find(entry_name);\n if (it == values.end()) return nullptr;\n return &it->second;\n }\n\n // get a result by name, parsed as a specific type.\n // NOTE: for counters, use GetCounterAs instead.\n template \n T GetAs(const std::string& entry_name) const;\n\n // counters are written as doubles, so they have to be read first\n // as a double, and only then converted to the asked type.\n template \n T GetCounterAs(const std::string& entry_name) const {\n double dval = GetAs(entry_name);\n T tval = static_cast(dval);\n return tval;\n }\n};\n\ntemplate \nT Results::GetAs(const std::string& entry_name) const {\n auto* sv = Get(entry_name);\n BM_CHECK(sv != nullptr && !sv->empty());\n std::stringstream ss;\n ss << *sv;\n T out;\n ss >> out;\n BM_CHECK(!ss.fail());\n return out;\n}\n\n//----------------------------------\n// Macros to help in result checking. Do not use them with arguments causing\n// side-effects.\n\n// clang-format off\n\n#define CHECK_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value) \\\n CONCAT(BM_CHECK_, relationship) \\\n (entry.getfn< var_type >(var_name), (value)) << \"\\n\" \\\n << __FILE__ << \":\" << __LINE__ << \": \" << (entry).name << \":\\n\" \\\n << __FILE__ << \":\" << __LINE__ << \": \" \\\n << \"expected (\" << #var_type << \")\" << (var_name) \\\n << \"=\" << (entry).getfn< var_type >(var_name) \\\n << \" to be \" #relationship \" to \" << (value) << \"\\n\"\n\n// check with tolerance. eps_factor is the tolerance window, which is\n// interpreted relative to value (eg, 0.1 means 10% of value).\n#define CHECK_FLOAT_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value, eps_factor) \\\n CONCAT(BM_CHECK_FLOAT_, relationship) \\\n (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << \"\\n\" \\\n << __FILE__ << \":\" << __LINE__ << \": \" << (entry).name << \":\\n\" \\\n << __FILE__ << \":\" << __LINE__ << \": \" \\\n << \"expected (\" << #var_type << \")\" << (var_name) \\\n << \"=\" << (entry).getfn< var_type >(var_name) \\\n << \" to be \" #relationship \" to \" << (value) << \"\\n\" \\\n << __FILE__ << \":\" << __LINE__ << \": \" \\\n << \"with tolerance of \" << (eps_factor) * (value) \\\n << \" (\" << (eps_factor)*100. << \"%), \" \\\n << \"but delta was \" << ((entry).getfn< var_type >(var_name) - (value)) \\\n << \" (\" << (((entry).getfn< var_type >(var_name) - (value)) \\\n / \\\n ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \\\n << \"%)\"\n\n#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \\\n CHECK_RESULT_VALUE_IMPL(entry, GetAs, var_type, var_name, relationship, value)\n\n#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \\\n CHECK_RESULT_VALUE_IMPL(entry, GetCounterAs, var_type, var_name, relationship, value)\n\n#define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \\\n CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetAs, double, var_name, relationship, value, eps_factor)\n\n#define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \\\n CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetCounterAs, double, var_name, relationship, value, eps_factor)\n\n// clang-format on\n\n// ========================================================================= //\n// --------------------------- Misc Utilities ------------------------------ //\n// ========================================================================= //\n\nnamespace {\n\nconst char* const dec_re = \"[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?\";\n\n} // end namespace\n\n#endif // TEST_OUTPUT_TEST_H\n" }, { "path": "test/output_test_helper.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"../src/benchmark_api_internal.h\"\n#include \"../src/check.h\" // NOTE: check.h is for internal use only!\n#include \"../src/log.h\" // NOTE: log.h is for internal use only\n#include \"../src/re.h\" // NOTE: re.h is for internal use only\n#include \"output_test.h\"\n\n// ========================================================================= //\n// ------------------------------ Internals -------------------------------- //\n// ========================================================================= //\nnamespace internal {\nnamespace {\n\nusing TestCaseList = std::vector;\n\n// Use a vector because the order elements are added matters during iteration.\n// std::map/unordered_map don't guarantee that.\n// For example:\n// SetSubstitutions({{\"%HelloWorld\", \"Hello\"}, {\"%Hello\", \"Hi\"}});\n// Substitute(\"%HelloWorld\") // Always expands to Hello.\nusing SubMap = std::vector>;\n\nTestCaseList& GetTestCaseList(TestCaseID ID) {\n // Uses function-local statics to ensure initialization occurs\n // before first use.\n static TestCaseList lists[TC_NumID];\n return lists[ID];\n}\n\nSubMap& GetSubstitutions() {\n // Don't use 'dec_re' from header because it may not yet be initialized.\n // clang-format off\n static std::string safe_dec_re = \"[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?\";\n static std::string time_re = \"([0-9]+[.])?[0-9]+\";\n static std::string percentage_re = \"[0-9]+[.][0-9]{2}\";\n static SubMap map = {\n {\"%float\", \"[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?\"},\n // human-readable float\n {\"%hrfloat\", \"[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kKMGTPEZYmunpfazy]?i?\"},\n {\"%percentage\", percentage_re},\n {\"%int\", \"[ ]*[0-9]+\"},\n {\" %s \", \"[ ]+\"},\n {\"%time\", \"[ ]*\" + time_re + \"[ ]+ns\"},\n {\"%console_report\", \"[ ]*\" + time_re + \"[ ]+ns [ ]*\" + time_re + \"[ ]+ns [ ]*[0-9]+\"},\n {\"%console_percentage_report\", \"[ ]*\" + percentage_re + \"[ ]+% [ ]*\" + percentage_re + \"[ ]+% [ ]*[0-9]+\"},\n {\"%console_us_report\", \"[ ]*\" + time_re + \"[ ]+us [ ]*\" + time_re + \"[ ]+us [ ]*[0-9]+\"},\n {\"%console_ms_report\", \"[ ]*\" + time_re + \"[ ]+ms [ ]*\" + time_re + \"[ ]+ms [ ]*[0-9]+\"},\n {\"%console_s_report\", \"[ ]*\" + time_re + \"[ ]+s [ ]*\" + time_re + \"[ ]+s [ ]*[0-9]+\"},\n {\"%console_time_only_report\", \"[ ]*\" + time_re + \"[ ]+ns [ ]*\" + time_re + \"[ ]+ns\"},\n {\"%console_us_report\", \"[ ]*\" + time_re + \"[ ]+us [ ]*\" + time_re + \"[ ]+us [ ]*[0-9]+\"},\n {\"%console_us_time_only_report\", \"[ ]*\" + time_re + \"[ ]+us [ ]*\" + time_re + \"[ ]+us\"},\n {\"%csv_header\",\n \"name,iterations,real_time,cpu_time,time_unit,bytes_per_second,\"\n \"items_per_second,label,error_occurred,error_message\"},\n {\"%csv_report\", \"[0-9]+,\" + safe_dec_re + \",\" + safe_dec_re + \",ns,,,,,\"},\n {\"%csv_us_report\", \"[0-9]+,\" + safe_dec_re + \",\" + safe_dec_re + \",us,,,,,\"},\n {\"%csv_ms_report\", \"[0-9]+,\" + safe_dec_re + \",\" + safe_dec_re + \",ms,,,,,\"},\n {\"%csv_s_report\", \"[0-9]+,\" + safe_dec_re + \",\" + safe_dec_re + \",s,,,,,\"},\n {\"%csv_cv_report\", \"[0-9]+,\" + safe_dec_re + \",\" + safe_dec_re + \",,,,,,\"},\n {\"%csv_bytes_report\",\n \"[0-9]+,\" + safe_dec_re + \",\" + safe_dec_re + \",ns,\" + safe_dec_re + \",,,,\"},\n {\"%csv_items_report\",\n \"[0-9]+,\" + safe_dec_re + \",\" + safe_dec_re + \",ns,,\" + safe_dec_re + \",,,\"},\n {\"%csv_bytes_items_report\",\n \"[0-9]+,\" + safe_dec_re + \",\" + safe_dec_re + \",ns,\" + safe_dec_re +\n \",\" + safe_dec_re + \",,,\"},\n {\"%csv_label_report_begin\", \"[0-9]+,\" + safe_dec_re + \",\" + safe_dec_re + \",ns,,,\"},\n {\"%csv_label_report_end\", \",,\"}};\n // clang-format on\n return map;\n}\n\nstd::string PerformSubstitutions(std::string source) {\n SubMap const& subs = GetSubstitutions();\n using SizeT = std::string::size_type;\n for (auto const& KV : subs) {\n SizeT pos = 0;\n SizeT next_start = 0;\n while ((pos = source.find(KV.first, next_start)) != std::string::npos) {\n next_start = pos + KV.second.size();\n source.replace(pos, KV.first.size(), KV.second);\n }\n }\n return source;\n}\n\nvoid CheckCase(std::stringstream& remaining_output, TestCase const& TC,\n TestCaseList const& not_checks) {\n std::string first_line;\n bool on_first = true;\n std::string line;\n while (!remaining_output.eof()) {\n BM_CHECK(remaining_output.good());\n std::getline(remaining_output, line);\n if (on_first) {\n first_line = line;\n on_first = false;\n }\n for (const auto& NC : not_checks) {\n BM_CHECK(!NC.regex->Match(line))\n << \"Unexpected match for line \\\"\" << line << \"\\\" for MR_Not regex \\\"\"\n << NC.regex_str << \"\\\"\"\n << \"\\n actual regex string \\\"\" << TC.substituted_regex << \"\\\"\"\n << \"\\n started matching near: \" << first_line;\n }\n if (TC.regex->Match(line)) {\n return;\n }\n BM_CHECK(TC.match_rule != MR_Next)\n << \"Expected line \\\"\" << line << \"\\\" to match regex \\\"\" << TC.regex_str\n << \"\\\"\"\n << \"\\n actual regex string \\\"\" << TC.substituted_regex << \"\\\"\"\n << \"\\n started matching near: \" << first_line;\n }\n BM_CHECK(remaining_output.eof() == false)\n << \"End of output reached before match for regex \\\"\" << TC.regex_str\n << \"\\\" was found\"\n << \"\\n actual regex string \\\"\" << TC.substituted_regex << \"\\\"\"\n << \"\\n started matching near: \" << first_line;\n}\n\nvoid CheckCases(TestCaseList const& checks, std::stringstream& output) {\n std::vector not_checks;\n for (size_t i = 0; i < checks.size(); ++i) {\n const auto& TC = checks[i];\n if (TC.match_rule == MR_Not) {\n not_checks.push_back(TC);\n continue;\n }\n CheckCase(output, TC, not_checks);\n not_checks.clear();\n }\n}\n\nclass TestReporter : public benchmark::BenchmarkReporter {\n public:\n TestReporter(std::vector reps)\n : reporters_(std::move(reps)) {}\n\n bool ReportContext(const Context& context) override {\n bool last_ret = false;\n bool first = true;\n for (auto* rep : reporters_) {\n bool new_ret = rep->ReportContext(context);\n BM_CHECK(first || new_ret == last_ret)\n << \"Reports return different values for ReportContext\";\n first = false;\n last_ret = new_ret;\n }\n (void)first;\n return last_ret;\n }\n\n void ReportRuns(const std::vector& report) override {\n for (auto* rep : reporters_) {\n rep->ReportRuns(report);\n }\n }\n void Finalize() override {\n for (auto* rep : reporters_) {\n rep->Finalize();\n }\n }\n\n private:\n std::vector reporters_;\n};\n} // namespace\n\n} // end namespace internal\n\n// ========================================================================= //\n// -------------------------- Results checking ----------------------------- //\n// ========================================================================= //\n\nnamespace internal {\n\n// Utility class to manage subscribers for checking benchmark results.\n// It works by parsing the CSV output to read the results.\nclass ResultsChecker {\n public:\n struct PatternAndFn : public TestCase { // reusing TestCase for its regexes\n PatternAndFn(const std::string& rx, ResultsCheckFn fn_)\n : TestCase(rx), fn(std::move(fn_)) {}\n ResultsCheckFn fn;\n };\n\n std::vector check_patterns;\n std::vector results;\n std::vector field_names;\n\n void Add(const std::string& entry_pattern, const ResultsCheckFn& fn);\n\n void CheckResults(std::stringstream& output);\n\n private:\n void SetHeader_(const std::string& csv_header);\n void SetValues_(const std::string& entry_csv_line);\n\n std::vector SplitCsv_(const std::string& line) const;\n};\n\nnamespace {\n// store the static ResultsChecker in a function to prevent initialization\n// order problems\nResultsChecker& GetResultsChecker() {\n static ResultsChecker rc;\n return rc;\n}\n} // end namespace\n\n// add a results checker for a benchmark\nvoid ResultsChecker::Add(const std::string& entry_pattern,\n const ResultsCheckFn& fn) {\n check_patterns.emplace_back(entry_pattern, fn);\n}\n\n// check the results of all subscribed benchmarks\nvoid ResultsChecker::CheckResults(std::stringstream& output) {\n // first reset the stream to the start\n {\n auto start = std::stringstream::pos_type(0);\n // clear before calling tellg()\n output.clear();\n // seek to zero only when needed\n if (output.tellg() > start) {\n output.seekg(start);\n }\n // and just in case\n output.clear();\n }\n // now go over every line and publish it to the ResultsChecker\n std::string line;\n bool on_first = true;\n while (!output.eof()) {\n BM_CHECK(output.good());\n std::getline(output, line);\n if (on_first) {\n SetHeader_(line); // this is important\n on_first = false;\n continue;\n }\n SetValues_(line);\n }\n // finally we can call the subscribed check functions\n for (const auto& p : check_patterns) {\n BM_VLOG(2) << \"--------------------------------\\n\";\n BM_VLOG(2) << \"checking for benchmarks matching \" << p.regex_str << \"...\\n\";\n for (const auto& r : results) {\n if (!p.regex->Match(r.name)) {\n BM_VLOG(2) << p.regex_str << \" is not matched by \" << r.name << \"\\n\";\n continue;\n }\n BM_VLOG(2) << p.regex_str << \" is matched by \" << r.name << \"\\n\";\n BM_VLOG(1) << \"Checking results of \" << r.name << \": ... \\n\";\n p.fn(r);\n BM_VLOG(1) << \"Checking results of \" << r.name << \": OK.\\n\";\n }\n }\n}\n\n// prepare for the names in this header\nvoid ResultsChecker::SetHeader_(const std::string& csv_header) {\n field_names = SplitCsv_(csv_header);\n}\n\n// set the values for a benchmark\nvoid ResultsChecker::SetValues_(const std::string& entry_csv_line) {\n if (entry_csv_line.empty()) {\n return;\n } // some lines are empty\n BM_CHECK(!field_names.empty());\n auto vals = SplitCsv_(entry_csv_line);\n BM_CHECK_EQ(vals.size(), field_names.size());\n results.emplace_back(vals[0]); // vals[0] is the benchmark name\n auto& entry = results.back();\n for (size_t i = 1, e = vals.size(); i < e; ++i) {\n entry.values[field_names[i]] = vals[i];\n }\n}\n\n// a quick'n'dirty csv splitter (eliminating quotes)\nstd::vector ResultsChecker::SplitCsv_(\n const std::string& line) const {\n std::vector out;\n if (line.empty()) {\n return out;\n }\n if (!field_names.empty()) {\n out.reserve(field_names.size());\n }\n size_t prev = 0;\n size_t pos = line.find_first_of(',');\n size_t curr = pos;\n while (pos != std::string::npos) {\n BM_CHECK(curr > 0);\n if (line[prev] == '\"') {\n ++prev;\n }\n if (line[curr - 1] == '\"') {\n --curr;\n }\n out.push_back(line.substr(prev, curr - prev));\n prev = pos + 1;\n pos = line.find_first_of(',', pos + 1);\n curr = pos;\n }\n curr = line.size();\n if (line[prev] == '\"') {\n ++prev;\n }\n if (line[curr - 1] == '\"') {\n --curr;\n }\n out.push_back(line.substr(prev, curr - prev));\n return out;\n}\n\n} // end namespace internal\n\nsize_t AddChecker(const std::string& bm_name, const ResultsCheckFn& fn) {\n auto& rc = internal::GetResultsChecker();\n rc.Add(bm_name, fn);\n return rc.results.size();\n}\n\nint Results::NumThreads() const {\n auto pos = name.find(\"/threads:\");\n if (pos == std::string::npos) {\n return 1;\n }\n auto end = name.find('/', pos + 9);\n std::stringstream ss;\n ss << name.substr(pos + 9, end);\n int num = 1;\n ss >> num;\n BM_CHECK(!ss.fail());\n return num;\n}\n\ndouble Results::NumIterations() const { return GetAs(\"iterations\"); }\n\ndouble Results::GetTime(BenchmarkTime which) const {\n BM_CHECK(which == kCpuTime || which == kRealTime);\n const char* which_str = which == kCpuTime ? \"cpu_time\" : \"real_time\";\n double val = GetAs(which_str);\n const auto* unit = Get(\"time_unit\");\n BM_CHECK(unit);\n if (*unit == \"ns\") {\n return val * 1.e-9;\n }\n if (*unit == \"us\") {\n return val * 1.e-6;\n }\n if (*unit == \"ms\") {\n return val * 1.e-3;\n }\n if (*unit == \"s\") {\n return val;\n }\n BM_CHECK(1 == 0) << \"unknown time unit: \" << *unit;\n return 0;\n}\n\n// ========================================================================= //\n// -------------------------- Public API Definitions------------------------ //\n// ========================================================================= //\n\nTestCase::TestCase(std::string re, int rule)\n : regex_str(std::move(re)),\n match_rule(rule),\n substituted_regex(internal::PerformSubstitutions(regex_str)),\n regex(std::make_shared()) {\n std::string err_str;\n regex->Init(substituted_regex, &err_str);\n BM_CHECK(err_str.empty())\n << \"Could not construct regex \\\"\" << substituted_regex << \"\\\"\"\n << \"\\n originally \\\"\" << regex_str << \"\\\"\"\n << \"\\n got error: \" << err_str;\n}\n\nint AddCases(TestCaseID ID, std::initializer_list il) {\n auto& L = internal::GetTestCaseList(ID);\n L.insert(L.end(), il);\n return 0;\n}\n\nint SetSubstitutions(\n std::initializer_list> il) {\n auto& subs = internal::GetSubstitutions();\n for (auto KV : il) {\n bool exists = false;\n KV.second = internal::PerformSubstitutions(KV.second);\n for (auto& EKV : subs) {\n if (EKV.first == KV.first) {\n EKV.second = std::move(KV.second);\n exists = true;\n break;\n }\n }\n if (!exists) {\n subs.push_back(std::move(KV));\n }\n }\n return 0;\n}\n\n// Disable deprecated warnings temporarily because we need to reference\n// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations\nBENCHMARK_DISABLE_DEPRECATED_WARNING\n\nvoid RunOutputTests(int argc, char* argv[]) {\n using internal::GetTestCaseList;\n benchmark::Initialize(&argc, argv);\n auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true);\n benchmark::ConsoleReporter CR(options);\n benchmark::JSONReporter JR;\n benchmark::CSVReporter CSVR;\n struct ReporterTest {\n std::string name;\n std::vector& output_cases;\n std::vector& error_cases;\n benchmark::BenchmarkReporter& reporter;\n std::stringstream out_stream;\n std::stringstream err_stream;\n\n ReporterTest(const std::string& n, std::vector& out_tc,\n std::vector& err_tc,\n benchmark::BenchmarkReporter& br)\n : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {\n reporter.SetOutputStream(&out_stream);\n reporter.SetErrorStream(&err_stream);\n }\n } TestCases[] = {\n {std::string(\"ConsoleReporter\"), GetTestCaseList(TC_ConsoleOut),\n GetTestCaseList(TC_ConsoleErr), CR},\n {std::string(\"JSONReporter\"), GetTestCaseList(TC_JSONOut),\n GetTestCaseList(TC_JSONErr), JR},\n {std::string(\"CSVReporter\"), GetTestCaseList(TC_CSVOut),\n GetTestCaseList(TC_CSVErr), CSVR},\n };\n\n // Create the test reporter and run the benchmarks.\n std::cout << \"Running benchmarks...\\n\";\n internal::TestReporter test_rep({&CR, &JR, &CSVR});\n benchmark::RunSpecifiedBenchmarks(&test_rep);\n\n for (auto& rep_test : TestCases) {\n std::string msg =\n std::string(\"\\nTesting \") + rep_test.name + std::string(\" Output\\n\");\n std::string banner(msg.size() - 1, '-');\n std::cout << banner << msg << banner << \"\\n\";\n\n std::cerr << rep_test.err_stream.str();\n std::cout << rep_test.out_stream.str();\n\n internal::CheckCases(rep_test.error_cases, rep_test.err_stream);\n internal::CheckCases(rep_test.output_cases, rep_test.out_stream);\n\n std::cout << \"\\n\";\n }\n\n // now that we know the output is as expected, we can dispatch\n // the checks to subscribees.\n auto& csv = TestCases[2];\n // would use == but gcc spits a warning\n BM_CHECK(csv.name == std::string(\"CSVReporter\"));\n internal::GetResultsChecker().CheckResults(csv.out_stream);\n}\n\nBENCHMARK_RESTORE_DEPRECATED_WARNING\n\nint SubstrCnt(const std::string& haystack, const std::string& pat) {\n if (pat.length() == 0) {\n return 0;\n }\n int count = 0;\n for (size_t offset = haystack.find(pat); offset != std::string::npos;\n offset = haystack.find(pat, offset + pat.length())) {\n ++count;\n }\n return count;\n}\n\nnamespace {\nchar ToHex(int ch) {\n return ch < 10 ? static_cast('0' + ch)\n : static_cast('a' + (ch - 10));\n}\n\nchar RandomHexChar() {\n static std::mt19937 rd{std::random_device{}()};\n static std::uniform_int_distribution mrand{0, 15};\n return ToHex(mrand(rd));\n}\n\nstd::string GetRandomFileName() {\n std::string model = \"test.%%%%%%\";\n for (auto& ch : model) {\n if (ch == '%') {\n ch = RandomHexChar();\n }\n }\n return model;\n}\n\nbool FileExists(std::string const& name) {\n std::ifstream in(name.c_str());\n return in.good();\n}\n\nstd::string GetTempFileName() {\n // This function attempts to avoid race conditions where two tests\n // create the same file at the same time. However, it still introduces races\n // similar to tmpnam.\n int retries = 3;\n while (--retries != 0) {\n std::string name = GetRandomFileName();\n if (!FileExists(name)) {\n return name;\n }\n }\n std::cerr << \"Failed to create unique temporary file name\\n\";\n std::flush(std::cerr);\n std::exit(1);\n}\n} // end namespace\n\nstd::string GetFileReporterOutput(int argc, char* argv[]) {\n std::vector new_argv(argv, argv + argc);\n assert(static_cast(argc) == new_argv.size());\n\n std::string tmp_file_name = GetTempFileName();\n std::cout << \"Will be using this as the tmp file: \" << tmp_file_name << '\\n';\n\n std::string tmp = \"--benchmark_out=\";\n tmp += tmp_file_name;\n new_argv.emplace_back(const_cast(tmp.c_str()));\n\n argc = static_cast(new_argv.size());\n\n benchmark::Initialize(&argc, new_argv.data());\n benchmark::RunSpecifiedBenchmarks();\n\n // Read the output back from the file, and delete the file.\n std::ifstream tmp_stream(tmp_file_name);\n std::string output = std::string((std::istreambuf_iterator(tmp_stream)),\n std::istreambuf_iterator());\n std::remove(tmp_file_name.c_str());\n\n return output;\n}\n" }, { "path": "test/overload_test.cc", "content": "#include \"benchmark/macros.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n\nnamespace {\n// Simulate an overloaded function name.\n// This version does nothing and is just here to create ambiguity for\n// MyOverloadedBenchmark.\nBENCHMARK_UNUSED void MyOverloadedBenchmark() {}\n\n// This is the actual benchmark function we want to register.\n// It has the signature void(benchmark::State&) required by the library.\nvoid MyOverloadedBenchmark(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\n\n// This macro invocation should compile correctly if benchmark.h\n// contains the fix (using static_cast), but would fail to compile\n// if the benchmark name were ambiguous (e.g., when using + or no cast\n// with an overloaded function).\nBENCHMARK(MyOverloadedBenchmark);\n\n// Also test BENCHMARK_TEMPLATE with an overloaded name.\ntemplate \nvoid MyTemplatedOverloadedBenchmark() {}\n\ntemplate \nvoid MyTemplatedOverloadedBenchmark(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\n\nBENCHMARK_TEMPLATE(MyTemplatedOverloadedBenchmark, 1);\n} // end namespace\n\nBENCHMARK_MAIN();\n" }, { "path": "test/perf_counters_gtest.cc", "content": "#include \n#include \n\n#include \"../src/perf_counters.h\"\n#include \"gmock/gmock.h\"\n#include \"gtest/gtest.h\"\n\n#ifndef GTEST_SKIP\nstruct MsgHandler {\n void operator=(std::ostream&) {}\n};\n#define GTEST_SKIP() return MsgHandler() = std::cout\n#endif\n\nusing benchmark::internal::PerfCounters;\nusing benchmark::internal::PerfCountersMeasurement;\nusing benchmark::internal::PerfCounterValues;\nusing ::testing::AllOf;\nusing ::testing::Gt;\nusing ::testing::Lt;\n\nnamespace {\nconst char kGenericPerfEvent1[] = \"CYCLES\";\nconst char kGenericPerfEvent2[] = \"INSTRUCTIONS\";\n\nTEST(PerfCountersTest, Init) {\n EXPECT_EQ(PerfCounters::Initialize(), PerfCounters::kSupported);\n}\n\n// Generic events will have as many counters as there are CPU PMUs, and each\n// will have the same name. In order to make these tests independent of the\n// number of CPU PMUs in the system, we uniquify the counter names before\n// testing them.\nstatic std::set UniqueCounterNames(const PerfCounters& pc) {\n std::set names{pc.names().begin(), pc.names().end()};\n return names;\n}\n\nTEST(PerfCountersTest, OneCounter) {\n if (!PerfCounters::kSupported) {\n GTEST_SKIP() << \"Performance counters not supported.\\n\";\n }\n EXPECT_TRUE(PerfCounters::Initialize());\n EXPECT_EQ(\n UniqueCounterNames(PerfCounters::Create({kGenericPerfEvent1})).size(), 1);\n}\n\nTEST(PerfCountersTest, NegativeTest) {\n if (!PerfCounters::kSupported) {\n EXPECT_FALSE(PerfCounters::Initialize());\n return;\n }\n EXPECT_TRUE(PerfCounters::Initialize());\n // Safety checks\n // Create() will always create a valid object, even if passed no or\n // wrong arguments as the new behavior is to warn and drop unsupported\n // counters\n EXPECT_EQ(PerfCounters::Create({}).num_counters(), 0);\n EXPECT_EQ(PerfCounters::Create({\"\"}).num_counters(), 0);\n EXPECT_EQ(PerfCounters::Create({\"not a counter name\"}).num_counters(), 0);\n {\n // Try sneaking in a bad egg to see if it is filtered out. The\n // number of counters has to be two, not zero\n auto counter =\n PerfCounters::Create({kGenericPerfEvent2, \"\", kGenericPerfEvent1});\n auto names = UniqueCounterNames(counter);\n EXPECT_EQ(names.size(), 2);\n EXPECT_EQ(names,\n std::set({kGenericPerfEvent2, kGenericPerfEvent1}));\n }\n {\n // Try sneaking in an outrageous counter, like a fat finger mistake\n auto counter = PerfCounters::Create(\n {kGenericPerfEvent2, \"not a counter name\", kGenericPerfEvent1});\n auto names = UniqueCounterNames(counter);\n EXPECT_EQ(names.size(), 2);\n EXPECT_EQ(names,\n std::set({kGenericPerfEvent2, kGenericPerfEvent1}));\n }\n {\n // Finally try a golden input - it should like both of them\n EXPECT_EQ(UniqueCounterNames(PerfCounters::Create(\n {kGenericPerfEvent1, kGenericPerfEvent2}))\n .size(),\n 2);\n }\n {\n // Add a bad apple in the end of the chain to check the edges\n auto counter = PerfCounters::Create(\n {kGenericPerfEvent1, kGenericPerfEvent2, \"bad event name\"});\n auto names = UniqueCounterNames(counter);\n EXPECT_EQ(names.size(), 2);\n EXPECT_EQ(names,\n std::set({kGenericPerfEvent1, kGenericPerfEvent2}));\n }\n}\n\nstatic std::map SnapshotAndCombine(\n PerfCounters& counters) {\n PerfCounterValues values(counters.num_counters());\n std::map value_map;\n\n if (counters.Snapshot(&values)) {\n for (size_t i = 0; i != counters.num_counters(); ++i) {\n value_map[counters.names()[i]] += values[i];\n }\n }\n return value_map;\n}\n\nTEST(PerfCountersTest, Read1Counter) {\n if (!PerfCounters::kSupported) {\n GTEST_SKIP() << \"Test skipped because libpfm is not supported.\\n\";\n }\n EXPECT_TRUE(PerfCounters::Initialize());\n auto counters = PerfCounters::Create({kGenericPerfEvent1});\n auto values1 = SnapshotAndCombine(counters);\n EXPECT_EQ(values1.size(), 1);\n EXPECT_GT(values1.begin()->second, 0);\n auto values2 = SnapshotAndCombine(counters);\n EXPECT_EQ(values2.size(), 1);\n EXPECT_GT(values2.begin()->second, 0);\n EXPECT_GT(values2.begin()->second, values1.begin()->second);\n}\n\nTEST(PerfCountersTest, Read1CounterEachCPU) {\n if (!PerfCounters::kSupported) {\n GTEST_SKIP() << \"Test skipped because libpfm is not supported.\\n\";\n }\n#ifdef __linux__\n EXPECT_TRUE(PerfCounters::Initialize());\n\n cpu_set_t saved_set;\n if (sched_getaffinity(0, sizeof(saved_set), &saved_set) != 0) {\n // This can happen e.g. if there are more than CPU_SETSIZE CPUs.\n GTEST_SKIP() << \"Could not save CPU affinity mask.\\n\";\n }\n\n for (size_t cpu = 0; cpu != CPU_SETSIZE; ++cpu) {\n cpu_set_t set;\n CPU_ZERO(&set);\n CPU_SET(cpu, &set);\n if (sched_setaffinity(0, sizeof(set), &set) != 0) {\n break;\n }\n\n auto counters = PerfCounters::Create({kGenericPerfEvent1});\n auto values1 = SnapshotAndCombine(counters);\n EXPECT_EQ(values1.size(), 1);\n EXPECT_GT(values1.begin()->second, 0);\n auto values2 = SnapshotAndCombine(counters);\n EXPECT_EQ(values2.size(), 1);\n EXPECT_GT(values2.begin()->second, 0);\n EXPECT_GT(values2.begin()->second, values1.begin()->second);\n }\n\n EXPECT_EQ(sched_setaffinity(0, sizeof(saved_set), &saved_set), 0);\n#else\n GTEST_SKIP() << \"Test skipped on non-Linux.\\n\";\n#endif\n}\n\nTEST(PerfCountersTest, Read2Counters) {\n if (!PerfCounters::kSupported) {\n GTEST_SKIP() << \"Test skipped because libpfm is not supported.\\n\";\n }\n EXPECT_TRUE(PerfCounters::Initialize());\n auto counters =\n PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2});\n auto values1 = SnapshotAndCombine(counters);\n EXPECT_EQ(values1.size(), 2);\n for (auto& kv : values1) {\n EXPECT_GT(kv.second, 0);\n }\n auto values2 = SnapshotAndCombine(counters);\n EXPECT_EQ(values1.size(), 2);\n for (auto& kv : values2) {\n EXPECT_GT(kv.second, 0);\n EXPECT_GT(kv.second, values1[kv.first]);\n }\n}\n\nTEST(PerfCountersTest, ReopenExistingCounters) {\n // This test works in recent and old Intel hardware, Pixel 3, and Pixel 6.\n // However we cannot make assumptions beyond 2 HW counters due to Pixel 6.\n if (!PerfCounters::kSupported) {\n GTEST_SKIP() << \"Test skipped because libpfm is not supported.\\n\";\n }\n EXPECT_TRUE(PerfCounters::Initialize());\n std::vector kMetrics({kGenericPerfEvent1});\n std::vector counters(2);\n for (auto& counter : counters) {\n counter = PerfCounters::Create(kMetrics);\n }\n PerfCounterValues values(counters[0].num_counters());\n EXPECT_TRUE(counters[0].Snapshot(&values));\n EXPECT_TRUE(counters[1].Snapshot(&values));\n}\n\nTEST(PerfCountersTest, CreateExistingMeasurements) {\n // The test works (i.e. causes read to fail) for the assumptions\n // about hardware capabilities (i.e. small number (2) hardware\n // counters) at this date,\n // the same as previous test ReopenExistingCounters.\n if (!PerfCounters::kSupported) {\n GTEST_SKIP() << \"Test skipped because libpfm is not supported.\\n\";\n }\n EXPECT_TRUE(PerfCounters::Initialize());\n\n // This means we will try 10 counters but we can only guarantee\n // for sure at this time that only 3 will work. Perhaps in the future\n // we could use libpfm to query for the hardware limits on this\n // particular platform.\n const int kMaxCounters = 10;\n const int kMinValidCounters = 2;\n\n // Let's use a ubiquitous counter that is guaranteed to work\n // on all platforms\n const std::vector kMetrics{\"cycles\"};\n\n // Cannot create a vector of actual objects because the\n // copy constructor of PerfCounters is deleted - and so is\n // implicitly deleted on PerfCountersMeasurement too\n std::vector>\n perf_counter_measurements;\n\n perf_counter_measurements.reserve(kMaxCounters);\n for (int j = 0; j < kMaxCounters; ++j) {\n perf_counter_measurements.emplace_back(\n new PerfCountersMeasurement(kMetrics));\n }\n\n std::vector> measurements;\n\n // Start all counters together to see if they hold\n size_t max_counters = kMaxCounters;\n for (size_t i = 0; i < kMaxCounters; ++i) {\n auto& counter(*perf_counter_measurements[i]);\n std::set names{counter.names().begin(), counter.names().end()};\n EXPECT_EQ(names.size(), 1);\n if (!counter.Start()) {\n max_counters = i;\n break;\n };\n }\n\n ASSERT_GE(max_counters, kMinValidCounters);\n\n // Start all together\n for (size_t i = 0; i < max_counters; ++i) {\n auto& counter(*perf_counter_measurements[i]);\n EXPECT_TRUE(counter.Stop(measurements) || (i >= kMinValidCounters));\n }\n\n // Start/stop individually\n for (size_t i = 0; i < max_counters; ++i) {\n auto& counter(*perf_counter_measurements[i]);\n measurements.clear();\n counter.Start();\n EXPECT_TRUE(counter.Stop(measurements) || (i >= kMinValidCounters));\n }\n}\n\n// We try to do some meaningful work here but the compiler\n// insists in optimizing away our loop so we had to add a\n// no-optimize macro. In case it fails, we added some entropy\n// to this pool as well.\n\nBENCHMARK_DONT_OPTIMIZE size_t do_work() {\n static std::mt19937 rd{std::random_device{}()};\n static std::uniform_int_distribution mrand(0, 10);\n const size_t kNumLoops = 1000000;\n size_t sum = 0;\n for (size_t j = 0; j < kNumLoops; ++j) {\n sum += mrand(rd);\n }\n benchmark::DoNotOptimize(sum);\n return sum;\n}\n\nvoid measure(size_t threadcount, std::map* before,\n std::map* after) {\n BM_CHECK_NE(before, nullptr);\n BM_CHECK_NE(after, nullptr);\n std::vector threads(threadcount);\n auto work = [&]() { BM_CHECK(do_work() > 1000); };\n\n // We need to first set up the counters, then start the threads, so the\n // threads would inherit the counters. But later, we need to first destroy\n // the thread pool (so all the work finishes), then measure the counters. So\n // the scopes overlap, and we need to explicitly control the scope of the\n // threadpool.\n auto counters =\n PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2});\n for (auto& t : threads) {\n t = std::thread(work);\n }\n *before = SnapshotAndCombine(counters);\n for (auto& t : threads) {\n t.join();\n }\n *after = SnapshotAndCombine(counters);\n}\n\nTEST(PerfCountersTest, MultiThreaded) {\n if (!PerfCounters::kSupported) {\n GTEST_SKIP() << \"Test skipped because libpfm is not supported.\";\n }\n EXPECT_TRUE(PerfCounters::Initialize());\n std::map before, after;\n\n // Notice that this test will work even if we taskset it to a single CPU\n // In this case the threads will run sequentially\n // Start two threads and measure the number of combined cycles and\n // instructions\n measure(2, &before, &after);\n std::vector Elapsed2Threads{\n static_cast(after[kGenericPerfEvent1] -\n before[kGenericPerfEvent1]),\n static_cast(after[kGenericPerfEvent2] -\n before[kGenericPerfEvent2])};\n\n // Start four threads and measure the number of combined cycles and\n // instructions\n measure(4, &before, &after);\n std::vector Elapsed4Threads{\n static_cast(after[kGenericPerfEvent1] -\n before[kGenericPerfEvent1]),\n static_cast(after[kGenericPerfEvent2] -\n before[kGenericPerfEvent2])};\n\n // The following expectations fail (at least on a beefy workstation with lots\n // of cpus) - it seems that in some circumstances the runtime of 4 threads\n // can even be better than with 2.\n // So instead of expecting 4 threads to be slower, let's just make sure they\n // do not differ too much in general (one is not more than 10x than the\n // other).\n EXPECT_THAT(Elapsed4Threads[0] / Elapsed2Threads[0], AllOf(Gt(0.1), Lt(10)));\n EXPECT_THAT(Elapsed4Threads[1] / Elapsed2Threads[1], AllOf(Gt(0.1), Lt(10)));\n}\n\nTEST(PerfCountersTest, HardwareLimits) {\n // The test works (i.e. causes read to fail) for the assumptions\n // about hardware capabilities (i.e. small number (3-4) hardware\n // counters) at this date,\n // the same as previous test ReopenExistingCounters.\n if (!PerfCounters::kSupported) {\n GTEST_SKIP() << \"Test skipped because libpfm is not supported.\\n\";\n }\n EXPECT_TRUE(PerfCounters::Initialize());\n\n // Taken from `perf list`, but focusses only on those HW events that actually\n // were reported when running `sudo perf stat -a sleep 10`, intersected over\n // several platforms. All HW events listed in the first command not reported\n // in the second seem to not work. This is sad as we don't really get to test\n // the grouping here (groups can contain up to 6 members)...\n std::vector counter_names{\n \"cycles\", // leader\n \"instructions\", //\n \"branch-misses\", //\n };\n\n // In the off-chance that some of these values are not supported,\n // we filter them out so the test will complete without failure\n // albeit it might not actually test the grouping on that platform\n std::vector valid_names;\n for (const std::string& name : counter_names) {\n if (PerfCounters::IsCounterSupported(name)) {\n valid_names.push_back(name);\n }\n }\n PerfCountersMeasurement counter(valid_names);\n\n std::vector> measurements;\n\n counter.Start();\n EXPECT_TRUE(counter.Stop(measurements));\n}\n\n} // namespace\n" }, { "path": "test/perf_counters_test.cc", "content": "#include \n#undef NDEBUG\n\n#include \"../src/commandlineflags.h\"\n#include \"../src/perf_counters.h\"\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n#include \"output_test.h\"\n\nnamespace benchmark {\n\nBM_DECLARE_string(benchmark_perf_counters);\n\n} // namespace benchmark\nnamespace {\n\nvoid BM_Simple(benchmark::State& state) {\n for (auto _ : state) {\n auto iterations = double(state.iterations()) * double(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n}\nBENCHMARK(BM_Simple);\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_Simple\\\",$\"}});\n\nconst int kIters = 1000000;\n\nvoid BM_WithoutPauseResume(benchmark::State& state) {\n int n = 0;\n\n for (auto _ : state) {\n for (auto i = 0; i < kIters; ++i) {\n n = 1 - n;\n benchmark::DoNotOptimize(n);\n }\n }\n}\n\nBENCHMARK(BM_WithoutPauseResume);\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_WithoutPauseResume\\\",$\"}});\n\nvoid BM_WithPauseResume(benchmark::State& state) {\n int m = 0, n = 0;\n\n for (auto _ : state) {\n for (auto i = 0; i < kIters; ++i) {\n n = 1 - n;\n benchmark::DoNotOptimize(n);\n }\n\n state.PauseTiming();\n for (auto j = 0; j < kIters; ++j) {\n m = 1 - m;\n benchmark::DoNotOptimize(m);\n }\n state.ResumeTiming();\n }\n}\n\nBENCHMARK(BM_WithPauseResume);\n\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_WithPauseResume\\\",$\"}});\n\nstatic void CheckSimple(Results const& e) {\n CHECK_COUNTER_VALUE(e, double, \"CYCLES\", GT, 0);\n}\n\ndouble withoutPauseResumeInstrCount = 0.0;\ndouble withPauseResumeInstrCount = 0.0;\n\nvoid SaveInstrCountWithoutResume(Results const& e) {\n withoutPauseResumeInstrCount = e.GetAs(\"INSTRUCTIONS\");\n}\n\nvoid SaveInstrCountWithResume(Results const& e) {\n withPauseResumeInstrCount = e.GetAs(\"INSTRUCTIONS\");\n}\n\nCHECK_BENCHMARK_RESULTS(\"BM_Simple\", &CheckSimple);\nCHECK_BENCHMARK_RESULTS(\"BM_WithoutPauseResume\", &SaveInstrCountWithoutResume);\nCHECK_BENCHMARK_RESULTS(\"BM_WithPauseResume\", &SaveInstrCountWithResume);\n} // end namespace\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n if (!benchmark::internal::PerfCounters::kSupported) {\n return 0;\n }\n benchmark::FLAGS_benchmark_perf_counters = \"CYCLES,INSTRUCTIONS\";\n benchmark::internal::PerfCounters::Initialize();\n RunOutputTests(argc, argv);\n\n BM_CHECK_GT(withPauseResumeInstrCount, kIters);\n BM_CHECK_GT(withoutPauseResumeInstrCount, kIters);\n BM_CHECK_LT(withPauseResumeInstrCount, 1.5 * withoutPauseResumeInstrCount);\n}\n" }, { "path": "test/profiler_manager_gtest.cc", "content": "#include \n\n#include \"benchmark/managers.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n#include \"gtest/gtest.h\"\n\nnamespace {\n\nclass TestProfilerManager : public benchmark::ProfilerManager {\n public:\n void AfterSetupStart() override { ++start_called; }\n void BeforeTeardownStop() override { ++stop_called; }\n\n int start_called = 0;\n int stop_called = 0;\n};\n\nvoid BM_empty(benchmark::State& state) {\n for (auto _ : state) {\n auto iterations = state.iterations();\n benchmark::DoNotOptimize(iterations);\n }\n}\nBENCHMARK(BM_empty);\n\nTEST(ProfilerManager, ReregisterManager) {\n#if GTEST_HAS_DEATH_TEST\n // Tests only runnable in debug mode (when BM_CHECK is enabled).\n#ifndef NDEBUG\n#ifndef TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS\n ASSERT_DEATH_IF_SUPPORTED(\n {\n std::unique_ptr pm(new TestProfilerManager());\n benchmark::RegisterProfilerManager(pm.get());\n benchmark::RegisterProfilerManager(pm.get());\n },\n \"RegisterProfilerManager\");\n#endif\n#endif\n#endif\n}\n\n} // namespace\n" }, { "path": "test/profiler_manager_iterations_test.cc", "content": "#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/managers.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n\n// Tests that we can specify the number of profiler iterations with\n// --benchmark_min_time=x.\nnamespace {\n\nint iteration_count = 0;\nint end_profiler_iteration_count = 0;\n\nclass TestProfilerManager : public benchmark::ProfilerManager {\n void AfterSetupStart() override { iteration_count = 0; }\n void BeforeTeardownStop() override {\n end_profiler_iteration_count = iteration_count;\n }\n};\n\nclass NullReporter : public benchmark::BenchmarkReporter {\n public:\n bool ReportContext(const Context& /*context*/) override { return true; }\n void ReportRuns(const std::vector& /* report */) override {}\n};\n\nvoid BM_MyBench(benchmark::State& state) {\n for (auto s : state) {\n ++iteration_count;\n }\n}\nBENCHMARK(BM_MyBench);\n} // end namespace\n\nint main(int argc, char** argv) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n // Make a fake argv and append the new --benchmark_profiler_iterations=\n // to it.\n int fake_argc = argc + 1;\n std::vector fake_argv(static_cast(fake_argc));\n for (size_t i = 0; i < static_cast(argc); ++i) {\n fake_argv[i] = argv[i];\n }\n fake_argv[static_cast(argc)] = \"--benchmark_min_time=4x\";\n\n std::unique_ptr pm(new TestProfilerManager());\n benchmark::RegisterProfilerManager(pm.get());\n\n benchmark::Initialize(&fake_argc, const_cast(fake_argv.data()));\n\n NullReporter null_reporter;\n const size_t returned_count =\n benchmark::RunSpecifiedBenchmarks(&null_reporter, \"BM_MyBench\");\n assert(returned_count == 1);\n\n // Check the executed iters.\n assert(end_profiler_iteration_count == 4);\n\n benchmark::RegisterProfilerManager(nullptr);\n return 0;\n}\n" }, { "path": "test/profiler_manager_test.cc", "content": "// FIXME: WIP\n\n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/managers.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n#include \"output_test.h\"\n\nnamespace {\nclass TestProfilerManager : public benchmark::ProfilerManager {\n public:\n void AfterSetupStart() override { ++start_called; }\n void BeforeTeardownStop() override { ++stop_called; }\n\n int start_called = 0;\n int stop_called = 0;\n};\n\nvoid BM_empty(benchmark::State& state) {\n for (auto _ : state) {\n auto iterations = state.iterations();\n benchmark::DoNotOptimize(iterations);\n }\n}\nBENCHMARK(BM_empty);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_empty %console_report$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_empty\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_empty\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_empty\\\",%csv_report$\"}});\n} // end namespace\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n std::unique_ptr pm(new TestProfilerManager());\n\n benchmark::RegisterProfilerManager(pm.get());\n RunOutputTests(argc, argv);\n benchmark::RegisterProfilerManager(nullptr);\n\n assert(pm->start_called == 1);\n assert(pm->stop_called == 1);\n}\n" }, { "path": "test/register_benchmark_test.cc", "content": "\n#undef NDEBUG\n#include \n#include \n\n#include \"../src/check.h\" // NOTE: check.h is for internal use only!\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n\nnamespace {\n\nclass TestReporter : public benchmark::ConsoleReporter {\n public:\n void ReportRuns(const std::vector& report) override {\n all_runs_.insert(all_runs_.end(), begin(report), end(report));\n ConsoleReporter::ReportRuns(report);\n }\n\n std::vector all_runs_;\n};\n\nstruct TestCase {\n const std::string name;\n const std::string label;\n // Note: not explicit as we rely on it being converted through ADD_CASES.\n TestCase(const std::string& xname) : TestCase(xname, \"\") {}\n TestCase(const std::string& xname, const std::string& xlabel)\n : name(xname), label(xlabel) {}\n\n typedef benchmark::BenchmarkReporter::Run Run;\n\n void CheckRun(Run const& run) const {\n // clang-format off\n BM_CHECK(name == run.benchmark_name()) << \"expected \" << name << \" got \"\n << run.benchmark_name();\n if (!label.empty()) {\n BM_CHECK(run.report_label == label) << \"expected \" << label << \" got \"\n << run.report_label;\n } else {\n BM_CHECK(run.report_label.empty());\n }\n // clang-format on\n }\n};\n\nstd::vector ExpectedResults;\n\nint AddCases(std::initializer_list const& v) {\n for (const auto& N : v) {\n ExpectedResults.push_back(N);\n }\n return 0;\n}\n\n#define CONCAT(x, y) CONCAT2(x, y)\n#define CONCAT2(x, y) x##y\n#define ADD_CASES(...) \\\n const int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__})\n\nusing ReturnVal = benchmark::Benchmark const* const;\n\n//----------------------------------------------------------------------------//\n// Test RegisterBenchmark with no additional arguments\n//----------------------------------------------------------------------------//\nvoid BM_function(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_function);\nReturnVal dummy = benchmark::RegisterBenchmark(\n \"BM_function_manual_registration\", BM_function);\nADD_CASES({\"BM_function\"}, {\"BM_function_manual_registration\"});\n\n//----------------------------------------------------------------------------//\n// Test RegisterBenchmark with additional arguments\n// Note: GCC <= 4.8 do not support this form of RegisterBenchmark because they\n// reject the variadic pack expansion of lambda captures.\n//----------------------------------------------------------------------------//\n\nvoid BM_extra_args(benchmark::State& st, const char* label) {\n for (auto _ : st) {\n }\n st.SetLabel(label);\n}\nint RegisterFromFunction() {\n std::pair cases[] = {\n {\"test1\", \"One\"}, {\"test2\", \"Two\"}, {\"test3\", \"Three\"}};\n for (auto const& c : cases) {\n benchmark::RegisterBenchmark(c.first, &BM_extra_args, c.second);\n }\n return 0;\n}\nconst int dummy2 = RegisterFromFunction();\nADD_CASES({\"test1\", \"One\"}, {\"test2\", \"Two\"}, {\"test3\", \"Three\"});\n\n//----------------------------------------------------------------------------//\n// Test RegisterBenchmark with DISABLED_ benchmark\n//----------------------------------------------------------------------------//\nvoid DISABLED_BM_function(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(DISABLED_BM_function);\nReturnVal dummy3 = benchmark::RegisterBenchmark(\"DISABLED_BM_function_manual\",\n DISABLED_BM_function);\n// No need to add cases because we don't expect them to run.\n\n//----------------------------------------------------------------------------//\n// Test BENCHMARK_NAMED: verifies name format \"func/test_case_name\" and that\n// chaining (e.g. ->Threads()) works, without introducing a lambda.\n//----------------------------------------------------------------------------//\nvoid BM_named(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK_NAMED(BM_named, variant_a);\nBENCHMARK_NAMED(BM_named, variant_b);\nBENCHMARK_NAMED(BM_named, variant_c)->Threads(2);\nADD_CASES({\"BM_named/variant_a\"}, {\"BM_named/variant_b\"},\n {\"BM_named/variant_c/threads:2\"});\n\n//----------------------------------------------------------------------------//\n// Test RegisterBenchmark with different callable types\n//----------------------------------------------------------------------------//\n\nstruct CustomFixture {\n void operator()(benchmark::State& st) {\n for (auto _ : st) {\n }\n }\n};\n\nvoid TestRegistrationAtRuntime() {\n {\n CustomFixture fx;\n benchmark::RegisterBenchmark(\"custom_fixture\", fx);\n AddCases({std::string(\"custom_fixture\")});\n }\n {\n const char* x = \"42\";\n auto capturing_lam = [=](benchmark::State& st) {\n for (auto _ : st) {\n }\n st.SetLabel(x);\n };\n benchmark::RegisterBenchmark(\"lambda_benchmark\", capturing_lam);\n AddCases({{\"lambda_benchmark\", x}});\n }\n}\n\n// Test that all benchmarks, registered at either during static init or runtime,\n// are run and the results are passed to the reported.\nvoid RunTestOne() {\n TestRegistrationAtRuntime();\n\n TestReporter test_reporter;\n benchmark::RunSpecifiedBenchmarks(&test_reporter);\n\n typedef benchmark::BenchmarkReporter::Run Run;\n auto EB = ExpectedResults.begin();\n\n for (Run const& run : test_reporter.all_runs_) {\n assert(EB != ExpectedResults.end());\n EB->CheckRun(run);\n ++EB;\n }\n assert(EB == ExpectedResults.end());\n}\n\n// Test that ClearRegisteredBenchmarks() clears all previously registered\n// benchmarks.\n// Also test that new benchmarks can be registered and ran afterwards.\nvoid RunTestTwo() {\n assert(!ExpectedResults.empty() &&\n \"must have at least one registered benchmark\");\n ExpectedResults.clear();\n benchmark::ClearRegisteredBenchmarks();\n\n TestReporter test_reporter;\n size_t num_ran = benchmark::RunSpecifiedBenchmarks(&test_reporter);\n assert(num_ran == 0);\n assert(test_reporter.all_runs_.begin() == test_reporter.all_runs_.end());\n\n TestRegistrationAtRuntime();\n num_ran = benchmark::RunSpecifiedBenchmarks(&test_reporter);\n assert(num_ran == ExpectedResults.size());\n\n typedef benchmark::BenchmarkReporter::Run Run;\n auto EB = ExpectedResults.begin();\n\n for (Run const& run : test_reporter.all_runs_) {\n assert(EB != ExpectedResults.end());\n EB->CheckRun(run);\n ++EB;\n }\n assert(EB == ExpectedResults.end());\n}\n} // end namespace\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n benchmark::Initialize(&argc, argv);\n\n RunTestOne();\n RunTestTwo();\n}\n" }, { "path": "test/repetitions_test.cc", "content": "\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"output_test.h\"\n\nnamespace {\n// ========================================================================= //\n// ------------------------ Testing Basic Output --------------------------- //\n// ========================================================================= //\n\nvoid BM_ExplicitRepetitions(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_ExplicitRepetitions)->Repetitions(2);\n\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_ExplicitRepetitions/repeats:2 %console_report$\"}});\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_ExplicitRepetitions/repeats:2 %console_report$\"}});\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_ExplicitRepetitions/repeats:2_mean %console_report$\"}});\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_ExplicitRepetitions/repeats:2_median %console_report$\"}});\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_ExplicitRepetitions/repeats:2_stddev %console_report$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_ExplicitRepetitions/repeats:2\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ExplicitRepetitions/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_ExplicitRepetitions/repeats:2\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ExplicitRepetitions/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"repetition_index\\\": 1,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_ExplicitRepetitions/repeats:2_mean\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ExplicitRepetitions/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_ExplicitRepetitions/repeats:2_median\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ExplicitRepetitions/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_ExplicitRepetitions/repeats:2_stddev\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ExplicitRepetitions/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_ExplicitRepetitions/repeats:2\\\",%csv_report$\"}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_ExplicitRepetitions/repeats:2\\\",%csv_report$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_ExplicitRepetitions/repeats:2_mean\\\",%csv_report$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_ExplicitRepetitions/repeats:2_median\\\",%csv_report$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_ExplicitRepetitions/repeats:2_stddev\\\",%csv_report$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Basic Output --------------------------- //\n// ========================================================================= //\n\nvoid BM_ImplicitRepetitions(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_ImplicitRepetitions);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_ImplicitRepetitions %console_report$\"}});\nADD_CASES(TC_ConsoleOut, {{\"^BM_ImplicitRepetitions %console_report$\"}});\nADD_CASES(TC_ConsoleOut, {{\"^BM_ImplicitRepetitions %console_report$\"}});\nADD_CASES(TC_ConsoleOut, {{\"^BM_ImplicitRepetitions_mean %console_report$\"}});\nADD_CASES(TC_ConsoleOut, {{\"^BM_ImplicitRepetitions_median %console_report$\"}});\nADD_CASES(TC_ConsoleOut, {{\"^BM_ImplicitRepetitions_stddev %console_report$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_ImplicitRepetitions\\\",$\"},\n {\"\\\"family_index\\\": 1,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ImplicitRepetitions\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_ImplicitRepetitions\\\",$\"},\n {\"\\\"family_index\\\": 1,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ImplicitRepetitions\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 1,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_ImplicitRepetitions\\\",$\"},\n {\"\\\"family_index\\\": 1,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ImplicitRepetitions\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_ImplicitRepetitions_mean\\\",$\"},\n {\"\\\"family_index\\\": 1,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ImplicitRepetitions\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_ImplicitRepetitions_median\\\",$\"},\n {\"\\\"family_index\\\": 1,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ImplicitRepetitions\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_ImplicitRepetitions_stddev\\\",$\"},\n {\"\\\"family_index\\\": 1,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_ImplicitRepetitions\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_ImplicitRepetitions\\\",%csv_report$\"}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_ImplicitRepetitions\\\",%csv_report$\"}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_ImplicitRepetitions_mean\\\",%csv_report$\"}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_ImplicitRepetitions_median\\\",%csv_report$\"}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_ImplicitRepetitions_stddev\\\",%csv_report$\"}});\n} // end namespace\n\n// ========================================================================= //\n// --------------------------- TEST CASES END ------------------------------ //\n// ========================================================================= //\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n RunOutputTests(argc, argv);\n}\n" }, { "path": "test/report_aggregates_only_test.cc", "content": "\n#undef NDEBUG\n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"output_test.h\"\n\nnamespace {\n// Ok this test is super ugly. We want to check what happens with the file\n// reporter in the presence of ReportAggregatesOnly().\n// We do not care about console output, the normal tests check that already.\n\nvoid BM_SummaryRepeat(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();\n} // end namespace\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n const std::string output = GetFileReporterOutput(argc, argv);\n\n if (SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3\") != 4 ||\n SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_mean\\\"\") != 1 ||\n SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_median\\\"\") !=\n 1 ||\n SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_stddev\\\"\") !=\n 1 ||\n SubstrCnt(output, \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_cv\\\"\") != 1) {\n std::cout << \"Precondition mismatch. Expected to only find four \"\n \"occurrences of \\\"BM_SummaryRepeat/repeats:3\\\" substring:\\n\"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_mean\\\", \"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_median\\\", \"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_stddev\\\", \"\n \"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_cv\\\"\\nThe entire \"\n \"output:\\n\";\n std::cout << output;\n return 1;\n }\n\n return 0;\n}\n" }, { "path": "test/reporter_output_test.cc", "content": "#undef NDEBUG\n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/statistics.h\"\n#include \"benchmark/sysinfo.h\"\n#include \"benchmark/types.h\"\n#include \"benchmark/utils.h\"\n#include \"output_test.h\"\n\nnamespace {\n// ========================================================================= //\n// ---------------------- Testing Prologue Output -------------------------- //\n// ========================================================================= //\n\nADD_CASES(TC_ConsoleOut, {{\"^[-]+$\", MR_Next},\n {\"^Benchmark %s Time %s CPU %s Iterations$\", MR_Next},\n {\"^[-]+$\", MR_Next}});\nint AddContextCases() {\n AddCases(TC_ConsoleErr,\n {\n {\"^%int-%int-%intT%int:%int:%int[-+]%int:%int$\", MR_Default},\n {\"Running .*(/|\\\\\\\\)reporter_output_test(\\\\.exe)?$\", MR_Next},\n {\"Run on \\\\(%int X %float MHz CPU s?\\\\)\", MR_Next},\n });\n AddCases(TC_JSONOut,\n {{\"^\\\\{\", MR_Default},\n {\"\\\"context\\\":\", MR_Next},\n {\"\\\"date\\\": \\\"\", MR_Next},\n {\"\\\"host_name\\\":\", MR_Next},\n {\"\\\"executable\\\": \\\".*(/|\\\\\\\\)reporter_output_test(\\\\.exe)?\\\",\",\n MR_Next},\n {\"\\\"num_cpus\\\": %int,$\", MR_Next},\n {\"\\\"mhz_per_cpu\\\": %float,$\", MR_Next},\n {\"\\\"caches\\\": \\\\[$\", MR_Default}});\n auto const& Info = benchmark::CPUInfo::Get();\n auto const& Caches = Info.caches;\n if (!Caches.empty()) {\n AddCases(TC_ConsoleErr, {{\"CPU Caches:$\", MR_Next}});\n }\n for (size_t I = 0; I < Caches.size(); ++I) {\n std::string num_caches_str =\n Caches[I].num_sharing != 0 ? \" \\\\(x%int\\\\)$\" : \"$\";\n AddCases(TC_ConsoleErr,\n {{\"L%int (Data|Instruction|Unified) %int KiB\" + num_caches_str,\n MR_Next}});\n AddCases(TC_JSONOut, {{\"\\\\{$\", MR_Next},\n {\"\\\"type\\\": \\\"\", MR_Next},\n {\"\\\"level\\\": %int,$\", MR_Next},\n {\"\\\"size\\\": %int,$\", MR_Next},\n {\"\\\"num_sharing\\\": %int$\", MR_Next},\n {\"}[,]{0,1}$\", MR_Next}});\n }\n AddCases(TC_JSONOut, {{\"],$\"}});\n auto const& LoadAvg = Info.load_avg;\n if (!LoadAvg.empty()) {\n AddCases(TC_ConsoleErr,\n {{\"Load Average: (%float, ){0,2}%float$\", MR_Next}});\n }\n AddCases(TC_JSONOut, {{\"\\\"load_avg\\\": \\\\[(%float,?){0,3}],$\", MR_Next}});\n AddCases(TC_JSONOut, {{\"\\\"library_version\\\": \\\".*\\\",$\", MR_Next}});\n AddCases(TC_JSONOut, {{\"\\\"library_build_type\\\": \\\".*\\\",$\", MR_Next}});\n AddCases(TC_JSONOut, {{\"\\\"json_schema_version\\\": 1$\", MR_Next}});\n return 0;\n}\nconst int dummy_register = AddContextCases();\nADD_CASES(TC_CSVOut, {{\"%csv_header\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Basic Output --------------------------- //\n// ========================================================================= //\n\nvoid BM_basic(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_basic);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_basic %console_report$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_basic\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_basic\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_basic\\\",%csv_report$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Bytes per Second Output ---------------- //\n// ========================================================================= //\n\nvoid BM_bytes_per_second(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n state.SetBytesProcessed(1);\n}\nBENCHMARK(BM_bytes_per_second);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_bytes_per_second %console_report \"\n \"bytes_per_second=%float[kM]{0,1}/s$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_bytes_per_second\\\",$\"},\n {\"\\\"family_index\\\": 1,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_bytes_per_second\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bytes_per_second\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_bytes_per_second\\\",%csv_bytes_report$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Items per Second Output ---------------- //\n// ========================================================================= //\n\nvoid BM_items_per_second(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n state.SetItemsProcessed(1);\n}\nBENCHMARK(BM_items_per_second);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_items_per_second %console_report \"\n \"items_per_second=%float[kM]{0,1}/s$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_items_per_second\\\",$\"},\n {\"\\\"family_index\\\": 2,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_items_per_second\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"items_per_second\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_items_per_second\\\",%csv_items_report$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Label Output --------------------------- //\n// ========================================================================= //\n\nvoid BM_label(benchmark::State& state) {\n for (auto _ : state) {\n }\n state.SetLabel(\"some label\");\n}\nBENCHMARK(BM_label);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_label %console_report some label$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_label\\\",$\"},\n {\"\\\"family_index\\\": 3,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_label\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"label\\\": \\\"some label\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_label\\\",%csv_label_report_begin\\\"some \"\n \"label\\\"%csv_label_report_end$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Time Label Output ---------------------- //\n// ========================================================================= //\n\nvoid BM_time_label_nanosecond(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_time_label_nanosecond %console_report$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_time_label_nanosecond\\\",$\"},\n {\"\\\"family_index\\\": 4,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_time_label_nanosecond\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_time_label_nanosecond\\\",%csv_report$\"}});\n\nvoid BM_time_label_microsecond(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_time_label_microsecond %console_us_report$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_time_label_microsecond\\\",$\"},\n {\"\\\"family_index\\\": 5,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_time_label_microsecond\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"us\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_time_label_microsecond\\\",%csv_us_report$\"}});\n\nvoid BM_time_label_millisecond(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_time_label_millisecond %console_ms_report$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_time_label_millisecond\\\",$\"},\n {\"\\\"family_index\\\": 6,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_time_label_millisecond\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ms\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_time_label_millisecond\\\",%csv_ms_report$\"}});\n\nvoid BM_time_label_second(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond);\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_time_label_second %console_s_report$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_time_label_second\\\",$\"},\n {\"\\\"family_index\\\": 7,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_time_label_second\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"s\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_time_label_second\\\",%csv_s_report$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Error Output --------------------------- //\n// ========================================================================= //\n\nvoid BM_error(benchmark::State& state) {\n state.SkipWithError(\"message\");\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_error);\nADD_CASES(TC_ConsoleOut, {{\"^BM_error[ ]+ERROR OCCURRED: 'message'$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_error\\\",$\"},\n {\"\\\"family_index\\\": 8,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_error\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"error_occurred\\\": true,$\", MR_Next},\n {\"\\\"error_message\\\": \\\"message\\\",$\", MR_Next}});\n\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_error\\\",,,,,,,,true,\\\"message\\\"$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing No Arg Name Output -----------------------\n// //\n// ========================================================================= //\n\nvoid BM_no_arg_name(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_no_arg_name)->Arg(3);\nADD_CASES(TC_ConsoleOut, {{\"^BM_no_arg_name/3 %console_report$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_no_arg_name/3\\\",$\"},\n {\"\\\"family_index\\\": 9,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_no_arg_name/3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_no_arg_name/3\\\",%csv_report$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Arg Name Output ------------------------ //\n// ========================================================================= //\n\nvoid BM_arg_name(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_arg_name)->ArgName(\"first\")->Arg(3);\nADD_CASES(TC_ConsoleOut, {{\"^BM_arg_name/first:3 %console_report$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_arg_name/first:3\\\",$\"},\n {\"\\\"family_index\\\": 10,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_arg_name/first:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_arg_name/first:3\\\",%csv_report$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Arg Names Output ----------------------- //\n// ========================================================================= //\n\nvoid BM_arg_names(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({\"first\", \"\", \"third\"});\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_arg_names/first:2/5/third:4 %console_report$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_arg_names/first:2/5/third:4\\\",$\"},\n {\"\\\"family_index\\\": 11,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_arg_names/first:2/5/third:4\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_arg_names/first:2/5/third:4\\\",%csv_report$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Name Output ---------------------------- //\n// ========================================================================= //\n\nvoid BM_name(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_name)->Name(\"BM_custom_name\");\n\nADD_CASES(TC_ConsoleOut, {{\"^BM_custom_name %console_report$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_custom_name\\\",$\"},\n {\"\\\"family_index\\\": 12,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_custom_name\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\"$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_custom_name\\\",%csv_report$\"}});\n\n// ========================================================================= //\n// ------------------------ Testing Big Args Output ------------------------ //\n// ========================================================================= //\n\nvoid BM_BigArgs(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);\nADD_CASES(TC_ConsoleOut, {{\"^BM_BigArgs/1073741824 %console_report$\"},\n {\"^BM_BigArgs/2147483648 %console_report$\"}});\n\n// ========================================================================= //\n// ----------------------- Testing Complexity Output ----------------------- //\n// ========================================================================= //\n\nvoid BM_Complexity_O1(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n state.SetComplexityN(state.range(0));\n}\nBENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);\nSET_SUBSTITUTIONS({{\"%bigOStr\", \"[ ]* %float \\\\([0-9]+\\\\)\"},\n {\"%RMS\", \"[ ]*[0-9]+ %\"}});\nADD_CASES(TC_ConsoleOut, {{\"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$\"},\n {\"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$\"}});\n\n// ========================================================================= //\n// ----------------------- Testing Aggregate Output ------------------------ //\n// ========================================================================= //\n\n// Test that non-aggregate data is printed by default\nvoid BM_Repeat(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\n// need two repetitions min to be able to output any aggregate output\nBENCHMARK(BM_Repeat)->Repetitions(2);\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_Repeat/repeats:2 %console_report$\"},\n {\"^BM_Repeat/repeats:2 %console_report$\"},\n {\"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$\"},\n {\"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$\"},\n {\"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_Repeat/repeats:2\\\",$\"},\n {\"\\\"family_index\\\": 15,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:2\\\"\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:2\\\",$\"},\n {\"\\\"family_index\\\": 15,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"repetition_index\\\": 1,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:2_mean\\\",$\"},\n {\"\\\"family_index\\\": 15,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:2_median\\\",$\"},\n {\"\\\"family_index\\\": 15,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:2_stddev\\\",$\"},\n {\"\\\"family_index\\\": 15,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 2,$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Repeat/repeats:2\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:2\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:2_mean\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:2_median\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:2_stddev\\\",%csv_report$\"}});\n// but for two repetitions, mean and median is the same, so let's repeat..\nBENCHMARK(BM_Repeat)->Repetitions(3);\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_Repeat/repeats:3 %console_report$\"},\n {\"^BM_Repeat/repeats:3 %console_report$\"},\n {\"^BM_Repeat/repeats:3 %console_report$\"},\n {\"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$\"},\n {\"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$\"},\n {\"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_Repeat/repeats:3\\\",$\"},\n {\"\\\"family_index\\\": 16,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:3\\\",$\"},\n {\"\\\"family_index\\\": 16,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 1,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:3\\\",$\"},\n {\"\\\"family_index\\\": 16,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:3_mean\\\",$\"},\n {\"\\\"family_index\\\": 16,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:3_median\\\",$\"},\n {\"\\\"family_index\\\": 16,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:3_stddev\\\",$\"},\n {\"\\\"family_index\\\": 16,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Repeat/repeats:3\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:3\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:3\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:3_mean\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:3_median\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:3_stddev\\\",%csv_report$\"}});\n// median differs between even/odd number of repetitions, so just to be sure\nBENCHMARK(BM_Repeat)->Repetitions(4);\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_Repeat/repeats:4 %console_report$\"},\n {\"^BM_Repeat/repeats:4 %console_report$\"},\n {\"^BM_Repeat/repeats:4 %console_report$\"},\n {\"^BM_Repeat/repeats:4 %console_report$\"},\n {\"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$\"},\n {\"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$\"},\n {\"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_Repeat/repeats:4\\\",$\"},\n {\"\\\"family_index\\\": 17,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:4\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 4,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:4\\\",$\"},\n {\"\\\"family_index\\\": 17,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:4\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 4,$\", MR_Next},\n {\"\\\"repetition_index\\\": 1,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:4\\\",$\"},\n {\"\\\"family_index\\\": 17,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:4\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 4,$\", MR_Next},\n {\"\\\"repetition_index\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:4\\\",$\"},\n {\"\\\"family_index\\\": 17,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:4\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 4,$\", MR_Next},\n {\"\\\"repetition_index\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:4_mean\\\",$\"},\n {\"\\\"family_index\\\": 17,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:4\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 4,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 4,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:4_median\\\",$\"},\n {\"\\\"family_index\\\": 17,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:4\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 4,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 4,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_Repeat/repeats:4_stddev\\\",$\"},\n {\"\\\"family_index\\\": 17,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Repeat/repeats:4\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 4,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 4,$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Repeat/repeats:4\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:4\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:4\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:4\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:4_mean\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:4_median\\\",%csv_report$\"},\n {\"^\\\"BM_Repeat/repeats:4_stddev\\\",%csv_report$\"}});\n\n// Test that a non-repeated test still prints non-aggregate results even when\n// only-aggregate reports have been requested\nvoid BM_RepeatOnce(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();\nADD_CASES(TC_ConsoleOut, {{\"^BM_RepeatOnce/repeats:1 %console_report$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_RepeatOnce/repeats:1\\\",$\"},\n {\"\\\"family_index\\\": 18,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_RepeatOnce/repeats:1\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_RepeatOnce/repeats:1\\\",%csv_report$\"}});\n\n// Test that non-aggregate data is not reported\nvoid BM_SummaryRepeat(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();\nADD_CASES(\n TC_ConsoleOut,\n {{\".*BM_SummaryRepeat/repeats:3 \", MR_Not},\n {\"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$\"},\n {\"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$\"},\n {\"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$\"}});\nADD_CASES(TC_JSONOut,\n {{\".*BM_SummaryRepeat/repeats:3 \", MR_Not},\n {\"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_mean\\\",$\"},\n {\"\\\"family_index\\\": 19,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_SummaryRepeat/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_median\\\",$\"},\n {\"\\\"family_index\\\": 19,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_SummaryRepeat/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_SummaryRepeat/repeats:3_stddev\\\",$\"},\n {\"\\\"family_index\\\": 19,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_SummaryRepeat/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\".*BM_SummaryRepeat/repeats:3 \", MR_Not},\n {\"^\\\"BM_SummaryRepeat/repeats:3_mean\\\",%csv_report$\"},\n {\"^\\\"BM_SummaryRepeat/repeats:3_median\\\",%csv_report$\"},\n {\"^\\\"BM_SummaryRepeat/repeats:3_stddev\\\",%csv_report$\"}});\n\n// Test that non-aggregate data is not displayed.\n// NOTE: this test is kinda bad. we are only testing the display output.\n// But we don't check that the file output still contains everything...\nvoid BM_SummaryDisplay(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();\nADD_CASES(\n TC_ConsoleOut,\n {{\".*BM_SummaryDisplay/repeats:2 \", MR_Not},\n {\"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$\"},\n {\"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$\"},\n {\"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$\"}});\nADD_CASES(TC_JSONOut,\n {{\".*BM_SummaryDisplay/repeats:2 \", MR_Not},\n {\"\\\"name\\\": \\\"BM_SummaryDisplay/repeats:2_mean\\\",$\"},\n {\"\\\"family_index\\\": 20,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_SummaryDisplay/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_SummaryDisplay/repeats:2_median\\\",$\"},\n {\"\\\"family_index\\\": 20,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_SummaryDisplay/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_SummaryDisplay/repeats:2_stddev\\\",$\"},\n {\"\\\"family_index\\\": 20,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_SummaryDisplay/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 2,$\", MR_Next}});\nADD_CASES(TC_CSVOut,\n {{\".*BM_SummaryDisplay/repeats:2 \", MR_Not},\n {\"^\\\"BM_SummaryDisplay/repeats:2_mean\\\",%csv_report$\"},\n {\"^\\\"BM_SummaryDisplay/repeats:2_median\\\",%csv_report$\"},\n {\"^\\\"BM_SummaryDisplay/repeats:2_stddev\\\",%csv_report$\"}});\n\n// Test repeats with custom time unit.\nvoid BM_RepeatTimeUnit(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_RepeatTimeUnit)\n ->Repetitions(3)\n ->ReportAggregatesOnly()\n ->Unit(benchmark::kMicrosecond);\nADD_CASES(\n TC_ConsoleOut,\n {{\".*BM_RepeatTimeUnit/repeats:3 \", MR_Not},\n {\"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$\"},\n {\"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ \"\n \"]*3$\"},\n {\"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ \"\n \"]*3$\"}});\nADD_CASES(TC_JSONOut,\n {{\".*BM_RepeatTimeUnit/repeats:3 \", MR_Not},\n {\"\\\"name\\\": \\\"BM_RepeatTimeUnit/repeats:3_mean\\\",$\"},\n {\"\\\"family_index\\\": 21,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_RepeatTimeUnit/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"us\\\",?$\"},\n {\"\\\"name\\\": \\\"BM_RepeatTimeUnit/repeats:3_median\\\",$\"},\n {\"\\\"family_index\\\": 21,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_RepeatTimeUnit/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"us\\\",?$\"},\n {\"\\\"name\\\": \\\"BM_RepeatTimeUnit/repeats:3_stddev\\\",$\"},\n {\"\\\"family_index\\\": 21,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_RepeatTimeUnit/repeats:3\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"us\\\",?$\"}});\nADD_CASES(TC_CSVOut,\n {{\".*BM_RepeatTimeUnit/repeats:3 \", MR_Not},\n {\"^\\\"BM_RepeatTimeUnit/repeats:3_mean\\\",%csv_us_report$\"},\n {\"^\\\"BM_RepeatTimeUnit/repeats:3_median\\\",%csv_us_report$\"},\n {\"^\\\"BM_RepeatTimeUnit/repeats:3_stddev\\\",%csv_us_report$\"}});\n\n// ========================================================================= //\n// -------------------- Testing user-provided statistics ------------------- //\n// ========================================================================= //\n\nconst auto UserStatistics = [](const std::vector& v) {\n return v.back();\n};\nvoid BM_UserStats(benchmark::State& state) {\n for (auto _ : state) {\n state.SetIterationTime(150 / 10e8);\n }\n}\n// clang-format off\nBENCHMARK(BM_UserStats)\n ->Repetitions(3)\n ->Iterations(5)\n ->UseManualTime()\n ->ComputeStatistics(\"\", UserStatistics);\n// clang-format on\n\n// check that user-provided stats is calculated, and is after the default-ones\n// empty string as name is intentional, it would sort before anything else\nADD_CASES(TC_ConsoleOut, {{\"^BM_UserStats/iterations:5/repeats:3/manual_time [ \"\n \"]* 150 ns %time [ ]*5$\"},\n {\"^BM_UserStats/iterations:5/repeats:3/manual_time [ \"\n \"]* 150 ns %time [ ]*5$\"},\n {\"^BM_UserStats/iterations:5/repeats:3/manual_time [ \"\n \"]* 150 ns %time [ ]*5$\"},\n {\"^BM_UserStats/iterations:5/repeats:3/\"\n \"manual_time_mean [ ]* 150 ns %time [ ]*3$\"},\n {\"^BM_UserStats/iterations:5/repeats:3/\"\n \"manual_time_median [ ]* 150 ns %time [ ]*3$\"},\n {\"^BM_UserStats/iterations:5/repeats:3/\"\n \"manual_time_stddev [ ]* 0.000 ns %time [ ]*3$\"},\n {\"^BM_UserStats/iterations:5/repeats:3/manual_time_ \"\n \"[ ]* 150 ns %time [ ]*3$\"}});\nADD_CASES(\n TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",$\"},\n {\"\\\"family_index\\\": 22,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": 5,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",$\"},\n {\"\\\"family_index\\\": 22,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 1,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": 5,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",$\"},\n {\"\\\"family_index\\\": 22,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": 5,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\\\",$\"},\n {\"\\\"family_index\\\": 22,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time_median\\\",$\"},\n {\"\\\"family_index\\\": 22,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\\\",$\"},\n {\"\\\"family_index\\\": 22,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time_\\\",$\"},\n {\"\\\"family_index\\\": 22,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next}});\nADD_CASES(\n TC_CSVOut,\n {{\"^\\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",%csv_report$\"},\n {\"^\\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",%csv_report$\"},\n {\"^\\\"BM_UserStats/iterations:5/repeats:3/manual_time\\\",%csv_report$\"},\n {\"^\\\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\\\",%csv_report$\"},\n {\"^\\\"BM_UserStats/iterations:5/repeats:3/\"\n \"manual_time_median\\\",%csv_report$\"},\n {\"^\\\"BM_UserStats/iterations:5/repeats:3/\"\n \"manual_time_stddev\\\",%csv_report$\"},\n {\"^\\\"BM_UserStats/iterations:5/repeats:3/manual_time_\\\",%csv_report$\"}});\n\n// ========================================================================= //\n// ------------- Testing relative standard deviation statistics ------------ //\n// ========================================================================= //\n\nconst auto UserPercentStatistics = [](const std::vector&) {\n return 1. / 100.;\n};\nvoid BM_UserPercentStats(benchmark::State& state) {\n for (auto _ : state) {\n state.SetIterationTime(150 / 10e8);\n }\n}\n// clang-format off\nBENCHMARK(BM_UserPercentStats)\n ->Repetitions(3)\n ->Iterations(5)\n ->UseManualTime()\n ->Unit(benchmark::TimeUnit::kNanosecond)\n ->ComputeStatistics(\"\", UserPercentStatistics, benchmark::StatisticUnit::kPercentage);\n// clang-format on\n\n// check that UserPercent-provided stats is calculated, and is after the\n// default-ones empty string as name is intentional, it would sort before\n// anything else\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ \"\n \"]* 150 ns %time [ ]*5$\"},\n {\"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ \"\n \"]* 150 ns %time [ ]*5$\"},\n {\"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ \"\n \"]* 150 ns %time [ ]*5$\"},\n {\"^BM_UserPercentStats/iterations:5/repeats:3/\"\n \"manual_time_mean [ ]* 150 ns %time [ ]*3$\"},\n {\"^BM_UserPercentStats/iterations:5/repeats:3/\"\n \"manual_time_median [ ]* 150 ns %time [ ]*3$\"},\n {\"^BM_UserPercentStats/iterations:5/repeats:3/\"\n \"manual_time_stddev [ ]* 0.000 ns %time [ ]*3$\"},\n {\"^BM_UserPercentStats/iterations:5/repeats:3/manual_time_ \"\n \"[ ]* 1.00 % [ ]* 1.00 %[ ]*3$\"}});\nADD_CASES(\n TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\\\",$\"},\n {\"\\\"family_index\\\": 23,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": 5,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\\\",$\"},\n {\"\\\"family_index\\\": 23,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 1,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": 5,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next},\n {\"\\\"name\\\": \\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\\\",$\"},\n {\"\\\"family_index\\\": 23,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"repetition_index\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": 5,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next},\n {\"\\\"name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_mean\\\",$\"},\n {\"\\\"family_index\\\": 23,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next},\n {\"\\\"name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_median\\\",$\"},\n {\"\\\"family_index\\\": 23,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.5(0)*e\\\\+(0)*2,$\", MR_Next},\n {\"\\\"name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_stddev\\\",$\"},\n {\"\\\"family_index\\\": 23,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_\\\",$\"},\n {\"\\\"family_index\\\": 23,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \"\n \"\\\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 3,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"percentage\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 3,$\", MR_Next},\n {\"\\\"real_time\\\": 1\\\\.(0)*e-(0)*2,$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_UserPercentStats/iterations:5/repeats:3/\"\n \"manual_time\\\",%csv_report$\"},\n {\"^\\\"BM_UserPercentStats/iterations:5/repeats:3/\"\n \"manual_time\\\",%csv_report$\"},\n {\"^\\\"BM_UserPercentStats/iterations:5/repeats:3/\"\n \"manual_time\\\",%csv_report$\"},\n {\"^\\\"BM_UserPercentStats/iterations:5/repeats:3/\"\n \"manual_time_mean\\\",%csv_report$\"},\n {\"^\\\"BM_UserPercentStats/iterations:5/repeats:3/\"\n \"manual_time_median\\\",%csv_report$\"},\n {\"^\\\"BM_UserPercentStats/iterations:5/repeats:3/\"\n \"manual_time_stddev\\\",%csv_report$\"},\n {\"^\\\"BM_UserPercentStats/iterations:5/repeats:3/\"\n \"manual_time_\\\",%csv_cv_report$\"}});\n\n// ========================================================================= //\n// ------------------------- Testing StrEscape JSON ------------------------ //\n// ========================================================================= //\n#if 0 // enable when csv testing code correctly handles multi-line fields\nvoid BM_JSON_Format(benchmark::State& state) {\n state.SkipWithError(\"val\\b\\f\\n\\r\\t\\\\\\\"with\\\"es,capes\");\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_JSON_Format);\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_JSON_Format\\\",$\"},\n {\"\\\"family_index\\\": 23,$\", MR_Next},\n{\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_JSON_Format\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"error_occurred\\\": true,$\", MR_Next},\n {R\"(\"error_message\": \"val\\\\b\\\\f\\\\n\\\\r\\\\t\\\\\\\\\\\\\"with\\\\\"es,capes\",$)\", MR_Next}});\n#endif\n// ========================================================================= //\n// -------------------------- Testing CsvEscape ---------------------------- //\n// ========================================================================= //\n\nvoid BM_CSV_Format(benchmark::State& state) {\n state.SkipWithError(\"\\\"freedom\\\"\");\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_CSV_Format);\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_CSV_Format\\\",,,,,,,,true,\\\"\\\"\\\"freedom\\\"\\\"\\\"$\"}});\n} // end namespace\n\n// ========================================================================= //\n// --------------------------- TEST CASES END ------------------------------ //\n// ========================================================================= //\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n RunOutputTests(argc, argv);\n}\n" }, { "path": "test/skip_with_error_test.cc", "content": "\n#undef NDEBUG\n#include \n#include \n\n#include \"../src/check.h\" // NOTE: check.h is for internal use only!\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n\nnamespace {\n\nclass TestReporter : public benchmark::ConsoleReporter {\n public:\n bool ReportContext(const Context& context) override {\n return ConsoleReporter::ReportContext(context);\n };\n\n void ReportRuns(const std::vector& report) override {\n all_runs_.insert(all_runs_.end(), begin(report), end(report));\n ConsoleReporter::ReportRuns(report);\n }\n\n TestReporter() {}\n ~TestReporter() override {}\n\n mutable std::vector all_runs_;\n};\n\nstruct TestCase {\n std::string name;\n bool error_occurred;\n std::string error_message;\n\n typedef benchmark::BenchmarkReporter::Run Run;\n\n void CheckRun(Run const& run) const {\n BM_CHECK(name == run.benchmark_name())\n << \"expected \" << name << \" got \" << run.benchmark_name();\n BM_CHECK_EQ(error_occurred,\n benchmark::internal::SkippedWithError == run.skipped);\n BM_CHECK(error_message == run.skip_message);\n if (error_occurred) {\n // BM_CHECK(run.iterations == 0);\n } else {\n BM_CHECK(run.iterations != 0);\n }\n }\n};\n\n// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)\nstd::vector ExpectedResults;\n\nint AddCases(const std::string& base_name,\n std::initializer_list const& v) {\n for (auto TC : v) {\n TC.name = base_name + TC.name;\n ExpectedResults.push_back(std::move(TC));\n }\n return 0;\n}\n\n#define CONCAT(x, y) CONCAT2(x, y)\n#define CONCAT2(x, y) x##y\n#define ADD_CASES(...) const int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)\n\nvoid BM_error_no_running(benchmark::State& state) {\n state.SkipWithError(\"error message\");\n}\nBENCHMARK(BM_error_no_running);\nADD_CASES(\"BM_error_no_running\", {{\"\", true, \"error message\"}});\n\nvoid BM_error_before_running(benchmark::State& state) {\n state.SkipWithError(\"error message\");\n while (state.KeepRunning()) {\n assert(false);\n }\n}\nBENCHMARK(BM_error_before_running);\nADD_CASES(\"BM_error_before_running\", {{\"\", true, \"error message\"}});\n\nvoid BM_error_before_running_batch(benchmark::State& state) {\n state.SkipWithError(\"error message\");\n while (state.KeepRunningBatch(17)) {\n assert(false);\n }\n}\nBENCHMARK(BM_error_before_running_batch);\nADD_CASES(\"BM_error_before_running_batch\", {{\"\", true, \"error message\"}});\n\nvoid BM_error_before_running_range_for(benchmark::State& state) {\n state.SkipWithError(\"error message\");\n for (auto _ : state) {\n assert(false);\n }\n}\nBENCHMARK(BM_error_before_running_range_for);\nADD_CASES(\"BM_error_before_running_range_for\", {{\"\", true, \"error message\"}});\n\nvoid BM_error_during_running(benchmark::State& state) {\n int first_iter = 1;\n while (state.KeepRunning()) {\n if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) {\n assert(first_iter);\n first_iter = 0;\n state.SkipWithError(\"error message\");\n } else {\n state.PauseTiming();\n state.ResumeTiming();\n }\n }\n}\nBENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);\nADD_CASES(\"BM_error_during_running\", {{\"/1/threads:1\", true, \"error message\"},\n {\"/1/threads:2\", true, \"error message\"},\n {\"/1/threads:4\", true, \"error message\"},\n {\"/1/threads:8\", true, \"error message\"},\n {\"/2/threads:1\", false, \"\"},\n {\"/2/threads:2\", false, \"\"},\n {\"/2/threads:4\", false, \"\"},\n {\"/2/threads:8\", false, \"\"}});\n\nvoid BM_error_during_running_ranged_for(benchmark::State& state) {\n assert(state.max_iterations > 3 && \"test requires at least a few iterations\");\n bool first_iter = true;\n // NOTE: Users should not write the for loop explicitly.\n for (auto It = state.begin(), End = state.end(); It != End; ++It) {\n if (state.range(0) == 1) {\n assert(first_iter);\n first_iter = false;\n (void)first_iter;\n state.SkipWithError(\"error message\");\n // Test the unfortunate but documented behavior that the ranged-for loop\n // doesn't automatically terminate when SkipWithError is set.\n assert(++It != End);\n break; // Required behavior\n }\n }\n}\nBENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5);\nADD_CASES(\"BM_error_during_running_ranged_for\",\n {{\"/1/iterations:5\", true, \"error message\"},\n {\"/2/iterations:5\", false, \"\"}});\n\nvoid BM_error_after_running(benchmark::State& state) {\n for (auto _ : state) {\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n if (state.thread_index() <= (state.threads() / 2)) {\n state.SkipWithError(\"error message\");\n }\n}\nBENCHMARK(BM_error_after_running)->ThreadRange(1, 8);\nADD_CASES(\"BM_error_after_running\", {{\"/threads:1\", true, \"error message\"},\n {\"/threads:2\", true, \"error message\"},\n {\"/threads:4\", true, \"error message\"},\n {\"/threads:8\", true, \"error message\"}});\n\nvoid BM_error_while_paused(benchmark::State& state) {\n bool first_iter = true;\n while (state.KeepRunning()) {\n if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) {\n assert(first_iter);\n first_iter = false;\n state.PauseTiming();\n state.SkipWithError(\"error message\");\n } else {\n state.PauseTiming();\n state.ResumeTiming();\n }\n }\n}\nBENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);\nADD_CASES(\"BM_error_while_paused\", {{\"/1/threads:1\", true, \"error message\"},\n {\"/1/threads:2\", true, \"error message\"},\n {\"/1/threads:4\", true, \"error message\"},\n {\"/1/threads:8\", true, \"error message\"},\n {\"/2/threads:1\", false, \"\"},\n {\"/2/threads:2\", false, \"\"},\n {\"/2/threads:4\", false, \"\"},\n {\"/2/threads:8\", false, \"\"}});\n\nvoid BM_malformed(benchmark::State& /*unused*/) {\n // NOTE: empty body wanted. No thing else.\n}\nBENCHMARK(BM_malformed);\nADD_CASES(\"BM_malformed\",\n {{\"\", true,\n \"The benchmark didn't run, nor was it explicitly skipped. Please \"\n \"call 'SkipWithXXX` in your benchmark as appropriate.\"}});\n} // end namespace\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n benchmark::Initialize(&argc, argv);\n\n TestReporter test_reporter;\n benchmark::RunSpecifiedBenchmarks(&test_reporter);\n\n typedef benchmark::BenchmarkReporter::Run Run;\n auto EB = ExpectedResults.begin();\n\n for (Run const& run : test_reporter.all_runs_) {\n assert(EB != ExpectedResults.end());\n EB->CheckRun(run);\n ++EB;\n }\n assert(EB == ExpectedResults.end());\n\n return 0;\n}\n" }, { "path": "test/spec_arg_test.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/reporter.h\"\n#include \"benchmark/state.h\"\n\n// Tests that we can override benchmark-spec value from FLAGS_benchmark_filter\n// with argument to RunSpecifiedBenchmarks(...).\n\nnamespace {\n\nclass TestReporter : public benchmark::ConsoleReporter {\n public:\n bool ReportContext(const Context& context) override {\n return ConsoleReporter::ReportContext(context);\n };\n\n void ReportRuns(const std::vector& report) override {\n assert(report.size() == 1);\n matched_functions.push_back(report[0].run_name.function_name);\n ConsoleReporter::ReportRuns(report);\n };\n\n TestReporter() {}\n\n ~TestReporter() override {}\n\n const std::vector& GetMatchedFunctions() const {\n return matched_functions;\n }\n\n private:\n std::vector matched_functions;\n};\n\nvoid BM_NotChosen(benchmark::State& state) {\n assert(false && \"SHOULD NOT BE CALLED\");\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_NotChosen);\n\nvoid BM_Chosen(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_Chosen);\n\n} // end namespace\n\nint main(int argc, char** argv) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n\n const std::string flag = \"BM_NotChosen\";\n\n // Verify that argv specify --benchmark_filter=BM_NotChosen.\n bool found = false;\n for (int i = 0; i < argc; ++i) {\n if (strcmp(\"--benchmark_filter=BM_NotChosen\", argv[i]) == 0) {\n found = true;\n break;\n }\n }\n assert(found);\n\n benchmark::Initialize(&argc, argv);\n\n // Check that the current flag value is reported accurately via the\n // GetBenchmarkFilter() function.\n if (flag != benchmark::GetBenchmarkFilter()) {\n std::cerr\n << \"Seeing different value for flags. GetBenchmarkFilter() returns [\"\n << benchmark::GetBenchmarkFilter() << \"] expected flag=[\" << flag\n << \"]\\n\";\n return 1;\n }\n TestReporter test_reporter;\n const char* const spec = \"BM_Chosen\";\n const size_t returned_count =\n benchmark::RunSpecifiedBenchmarks(&test_reporter, spec);\n assert(returned_count == 1);\n const std::vector matched_functions =\n test_reporter.GetMatchedFunctions();\n assert(matched_functions.size() == 1);\n if (strcmp(spec, matched_functions.front().c_str()) != 0) {\n std::cerr << \"Expected benchmark [\" << spec << \"] to run, but got [\"\n << matched_functions.front() << \"]\\n\";\n return 2;\n }\n\n // Test that SetBenchmarkFilter works.\n const std::string golden_value = \"golden_value\";\n benchmark::SetBenchmarkFilter(golden_value);\n std::string current_value = benchmark::GetBenchmarkFilter();\n if (golden_value != current_value) {\n std::cerr << \"Expected [\" << golden_value\n << \"] for --benchmark_filter but got [\" << current_value << \"]\\n\";\n return 3;\n }\n return 0;\n}\n" }, { "path": "test/spec_arg_verbosity_test.cc", "content": "#include \n\n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n\nnamespace {\n// Tests that the user specified verbosity level can be get.\nvoid BM_Verbosity(benchmark::State& state) {\n for (auto _ : state) {\n }\n}\nBENCHMARK(BM_Verbosity);\n} // end namespace\n\nint main(int argc, char** argv) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n\n const int32_t flagv = 42;\n\n // Verify that argv specify --v=42.\n bool found = false;\n for (int i = 0; i < argc; ++i) {\n if (strcmp(\"--v=42\", argv[i]) == 0) {\n found = true;\n break;\n }\n }\n if (!found) {\n std::cerr << \"This test requires '--v=42' to be passed as a command-line \"\n << \"argument.\\n\";\n return 1;\n }\n\n benchmark::Initialize(&argc, argv);\n\n // Check that the current flag value is reported accurately via the\n // GetBenchmarkVerbosity() function.\n if (flagv != benchmark::GetBenchmarkVerbosity()) {\n std::cerr\n << \"Seeing different value for flags. GetBenchmarkVerbosity() returns [\"\n << benchmark::GetBenchmarkVerbosity() << \"] expected flag=[\" << flagv\n << \"]\\n\";\n return 1;\n }\n return 0;\n}\n" }, { "path": "test/state_assembly_test.cc", "content": "#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n\n#ifdef __clang__\n#pragma clang diagnostic ignored \"-Wreturn-type\"\n#pragma clang diagnostic ignored \"-Wmissing-prototypes\"\n#endif\n\n// clang-format off\nextern \"C\" {\n extern int ExternInt;\n benchmark::State& GetState();\n void Fn();\n}\n// clang-format on\n\nusing benchmark::State;\n\n// CHECK-LABEL: test_for_auto_loop:\nextern \"C\" int test_for_auto_loop() {\n State& S = GetState();\n int x = 42;\n // CHECK: \t[[CALL:call(q)*]]\t_ZN9benchmark5State16StartKeepRunningEv\n // CHECK-NEXT: testq %rbx, %rbx\n // CHECK-NEXT: je [[LOOP_END:.*]]\n\n for (auto _ : S) {\n // CHECK: .L[[LOOP_HEAD:[a-zA-Z0-9_]+]]:\n // CHECK-GNU-NEXT: subq $1, %rbx\n // CHECK-CLANG-NEXT: {{(addq \\$1, %rax|incq %rax|addq \\$-1, %rbx)}}\n // CHECK-NEXT: jne .L[[LOOP_HEAD]]\n benchmark::DoNotOptimize(x);\n }\n // CHECK: [[LOOP_END]]:\n // CHECK: [[CALL]]\t_ZN9benchmark5State17FinishKeepRunningEv\n\n // CHECK: movl $101, %eax\n // CHECK: ret\n return 101;\n}\n\n// CHECK-LABEL: test_while_loop:\nextern \"C\" int test_while_loop() {\n State& S = GetState();\n int x = 42;\n\n // CHECK: j{{(e|mp)}} .L[[LOOP_HEADER:[a-zA-Z0-9_]+]]\n // CHECK-NEXT: .L[[LOOP_BODY:[a-zA-Z0-9_]+]]:\n while (S.KeepRunning()) {\n // CHECK-GNU-NEXT: subq $1, %[[IREG:[a-z]+]]\n // CHECK-CLANG-NEXT: {{(addq \\$-1,|decq)}} %[[IREG:[a-z]+]]\n // CHECK: movq %[[IREG]], [[DEST:.*]]\n benchmark::DoNotOptimize(x);\n }\n // CHECK-DAG: movq [[DEST]], %[[IREG]]\n // CHECK-DAG: testq %[[IREG]], %[[IREG]]\n // CHECK-DAG: jne .L[[LOOP_BODY]]\n // CHECK-DAG: .L[[LOOP_HEADER]]:\n\n // CHECK: cmpb $0\n // CHECK-NEXT: jne .L[[LOOP_END:[a-zA-Z0-9_]+]]\n // CHECK: [[CALL:call(q)*]] _ZN9benchmark5State16StartKeepRunningEv\n\n // CHECK: .L[[LOOP_END]]:\n // CHECK: [[CALL]] _ZN9benchmark5State17FinishKeepRunningEv\n\n // CHECK: movl $101, %eax\n // CHECK: ret\n return 101;\n}\n" }, { "path": "test/statistics_gtest.cc", "content": "//===---------------------------------------------------------------------===//\n// statistics_test - Unit tests for src/statistics.cc\n//===---------------------------------------------------------------------===//\n\n#include \"../src/statistics.h\"\n#include \"gtest/gtest.h\"\n\nnamespace {\nTEST(StatisticsTest, Mean) {\n EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({42, 42, 42, 42}), 42.0);\n EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1, 2, 3, 4}), 2.5);\n EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1, 2, 5, 10, 10, 14}), 7.0);\n}\n\nTEST(StatisticsTest, Median) {\n EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({42, 42, 42, 42}), 42.0);\n EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1, 2, 3, 4}), 2.5);\n EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1, 2, 5, 10, 10}), 5.0);\n}\n\nTEST(StatisticsTest, StdDev) {\n EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({101, 101, 101, 101}), 0.0);\n EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({1, 2, 3}), 1.0);\n EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({2.5, 2.4, 3.3, 4.2, 5.1}),\n 1.151086443322134);\n}\n\nTEST(StatisticsTest, CV) {\n EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({101, 101, 101, 101}), 0.0);\n EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({1, 2, 3}), 1. / 2.);\n ASSERT_NEAR(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}),\n 0.32888184094918121, 1e-15);\n}\n\n} // end namespace\n" }, { "path": "test/string_util_gtest.cc", "content": "//===---------------------------------------------------------------------===//\n// string_util_test - Unit tests for src/string_util.cc\n//===---------------------------------------------------------------------===//\n\n#include \n\n#include \"../src/internal_macros.h\"\n#include \"../src/string_util.h\"\n#include \"gmock/gmock.h\"\n#include \"gtest/gtest.h\"\n\nnamespace {\nTEST(StringUtilTest, stoul) {\n {\n size_t pos = 0;\n EXPECT_EQ(0UL, benchmark::stoul(\"0\", &pos));\n EXPECT_EQ(1UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(7UL, benchmark::stoul(\"7\", &pos));\n EXPECT_EQ(1UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(135UL, benchmark::stoul(\"135\", &pos));\n EXPECT_EQ(3UL, pos);\n }\n#if ULONG_MAX == 0xFFFFFFFFul\n {\n size_t pos = 0;\n EXPECT_EQ(0xFFFFFFFFul, benchmark::stoul(\"4294967295\", &pos));\n EXPECT_EQ(10ul, pos);\n }\n#elif ULONG_MAX == 0xFFFFFFFFFFFFFFFFul\n {\n size_t pos = 0;\n EXPECT_EQ(0xFFFFFFFFFFFFFFFFUL,\n benchmark::stoul(\"18446744073709551615\", &pos));\n EXPECT_EQ(20UL, pos);\n }\n#endif\n {\n size_t pos = 0;\n EXPECT_EQ(10UL, benchmark::stoul(\"1010\", &pos, 2));\n EXPECT_EQ(4UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(520UL, benchmark::stoul(\"1010\", &pos, 8));\n EXPECT_EQ(4UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(1010UL, benchmark::stoul(\"1010\", &pos, 10));\n EXPECT_EQ(4UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(4112UL, benchmark::stoul(\"1010\", &pos, 16));\n EXPECT_EQ(4UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(0xBEEFUL, benchmark::stoul(\"BEEF\", &pos, 16));\n EXPECT_EQ(4UL, pos);\n }\n#ifndef BENCHMARK_HAS_NO_EXCEPTIONS\n {\n ASSERT_THROW(std::ignore = benchmark::stoul(\"this is a test\"),\n std::invalid_argument);\n }\n#endif\n}\n\nTEST(StringUtilTest, stoi) {\n {\n size_t pos = 0;\n EXPECT_EQ(0, benchmark::stoi(\"0\", &pos));\n EXPECT_EQ(1UL, pos);\n } // namespace\n {\n size_t pos = 0;\n EXPECT_EQ(-17, benchmark::stoi(\"-17\", &pos));\n EXPECT_EQ(3UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(1357, benchmark::stoi(\"1357\", &pos));\n EXPECT_EQ(4UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(10, benchmark::stoi(\"1010\", &pos, 2));\n EXPECT_EQ(4UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(520, benchmark::stoi(\"1010\", &pos, 8));\n EXPECT_EQ(4UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(1010, benchmark::stoi(\"1010\", &pos, 10));\n EXPECT_EQ(4UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(4112, benchmark::stoi(\"1010\", &pos, 16));\n EXPECT_EQ(4UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(0xBEEF, benchmark::stoi(\"BEEF\", &pos, 16));\n EXPECT_EQ(4UL, pos);\n }\n#ifndef BENCHMARK_HAS_NO_EXCEPTIONS\n {\n ASSERT_THROW(std::ignore = benchmark::stoi(\"this is a test\"),\n std::invalid_argument);\n }\n#endif\n}\n\nTEST(StringUtilTest, stod) {\n {\n size_t pos = 0;\n EXPECT_EQ(0.0, benchmark::stod(\"0\", &pos));\n EXPECT_EQ(1UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(-84.0, benchmark::stod(\"-84\", &pos));\n EXPECT_EQ(3UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(1234.0, benchmark::stod(\"1234\", &pos));\n EXPECT_EQ(4UL, pos);\n }\n {\n size_t pos = 0;\n EXPECT_EQ(1.5, benchmark::stod(\"1.5\", &pos));\n EXPECT_EQ(3UL, pos);\n }\n {\n size_t pos = 0;\n /* Note: exactly representable as double */\n EXPECT_EQ(-1.25e+9, benchmark::stod(\"-1.25e+9\", &pos));\n EXPECT_EQ(8UL, pos);\n }\n#ifndef BENCHMARK_HAS_NO_EXCEPTIONS\n {\n ASSERT_THROW(std::ignore = benchmark::stod(\"this is a test\"),\n std::invalid_argument);\n }\n#endif\n}\n\nTEST(StringUtilTest, StrSplit) {\n EXPECT_EQ(benchmark::StrSplit(\"\", ','), std::vector{});\n EXPECT_EQ(benchmark::StrSplit(\"hello\", ','),\n std::vector({\"hello\"}));\n EXPECT_EQ(benchmark::StrSplit(\"hello,there,is,more\", ','),\n std::vector({\"hello\", \"there\", \"is\", \"more\"}));\n}\n\nusing HumanReadableFixture = ::testing::TestWithParam<\n std::tuple>;\n\nINSTANTIATE_TEST_SUITE_P(\n HumanReadableTests, HumanReadableFixture,\n ::testing::Values(\n std::make_tuple(0.0, benchmark::Counter::kIs1024, \"0\"),\n std::make_tuple(999.0, benchmark::Counter::kIs1024, \"999\"),\n std::make_tuple(1000.0, benchmark::Counter::kIs1024, \"1000\"),\n std::make_tuple(1024.0, benchmark::Counter::kIs1024, \"1Ki\"),\n std::make_tuple(1000 * 1000.0, benchmark::Counter::kIs1024,\n \"976\\\\.56.Ki\"),\n std::make_tuple(1024 * 1024.0, benchmark::Counter::kIs1024, \"1Mi\"),\n std::make_tuple(1000 * 1000 * 1000.0, benchmark::Counter::kIs1024,\n \"953\\\\.674Mi\"),\n std::make_tuple(1024 * 1024 * 1024.0, benchmark::Counter::kIs1024,\n \"1Gi\"),\n std::make_tuple(0.0, benchmark::Counter::kIs1000, \"0\"),\n std::make_tuple(999.0, benchmark::Counter::kIs1000, \"999\"),\n std::make_tuple(1000.0, benchmark::Counter::kIs1000, \"1k\"),\n std::make_tuple(1024.0, benchmark::Counter::kIs1000, \"1.024k\"),\n std::make_tuple(1000 * 1000.0, benchmark::Counter::kIs1000, \"1M\"),\n std::make_tuple(1024 * 1024.0, benchmark::Counter::kIs1000,\n \"1\\\\.04858M\"),\n std::make_tuple(1000 * 1000 * 1000.0, benchmark::Counter::kIs1000,\n \"1G\"),\n std::make_tuple(1024 * 1024 * 1024.0, benchmark::Counter::kIs1000,\n \"1\\\\.07374G\")));\n\nTEST_P(HumanReadableFixture, HumanReadableNumber) {\n std::string str = benchmark::HumanReadableNumber(std::get<0>(GetParam()),\n std::get<1>(GetParam()));\n ASSERT_THAT(str, ::testing::MatchesRegex(std::get<2>(GetParam())));\n}\n\n} // end namespace\n" }, { "path": "test/templated_fixture_method_test.cc", "content": "\n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n\ntemplate \nclass MyFixture : public ::benchmark::Fixture {\n public:\n MyFixture() : data(0) {}\n\n T data;\n\n using type = T;\n};\n\nBENCHMARK_TEMPLATE_METHOD_F(MyFixture, Foo)(benchmark::State& st) {\n for (auto _ : st) {\n this->data += typename Base::type(1);\n }\n}\n\nBENCHMARK_TEMPLATE_INSTANTIATE_F(MyFixture, Foo, int);\nBENCHMARK_TEMPLATE_INSTANTIATE_F(MyFixture, Foo, double);\n\nBENCHMARK_MAIN();\n" }, { "path": "test/templated_fixture_test.cc", "content": "\n#include \n#include \n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n\ntemplate \nclass MyFixture : public ::benchmark::Fixture {\n public:\n MyFixture() : data(0) {}\n\n T data;\n};\n\nBENCHMARK_TEMPLATE_F(MyFixture, Foo, int)(benchmark::State& st) {\n for (auto _ : st) {\n data += 1;\n }\n}\n\nBENCHMARK_TEMPLATE_DEFINE_F(MyFixture, Bar, double)(benchmark::State& st) {\n for (auto _ : st) {\n data += 1.0;\n }\n}\nBENCHMARK_REGISTER_F(MyFixture, Bar);\n\nBENCHMARK_MAIN();\n" }, { "path": "test/time_unit_gtest.cc", "content": "#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/state.h\"\n#include \"gtest/gtest.h\"\n\nnamespace benchmark {\nnamespace internal {\n\nnamespace {\n\nclass DummyBenchmark : public benchmark::Benchmark {\n public:\n DummyBenchmark() : Benchmark(\"dummy\") {}\n void Run(State& /*state*/) override {}\n};\n\nTEST(DefaultTimeUnitTest, TimeUnitIsNotSet) {\n DummyBenchmark benchmark;\n EXPECT_EQ(benchmark.GetTimeUnit(), kNanosecond);\n}\n\nTEST(DefaultTimeUnitTest, DefaultIsSet) {\n DummyBenchmark benchmark;\n EXPECT_EQ(benchmark.GetTimeUnit(), kNanosecond);\n SetDefaultTimeUnit(kMillisecond);\n EXPECT_EQ(benchmark.GetTimeUnit(), kMillisecond);\n}\n\nTEST(DefaultTimeUnitTest, DefaultAndExplicitUnitIsSet) {\n DummyBenchmark benchmark;\n benchmark.Unit(kMillisecond);\n SetDefaultTimeUnit(kMicrosecond);\n\n EXPECT_EQ(benchmark.GetTimeUnit(), kMillisecond);\n}\n\n} // namespace\n} // namespace internal\n} // namespace benchmark\n" }, { "path": "test/user_counters_tabular_test.cc", "content": "\n#undef NDEBUG\n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/counter.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n#include \"output_test.h\"\n\nnamespace {\n// @todo: this checks the full output at once; the rule for\n// CounterSet1 was failing because it was not matching \"^[-]+$\".\n// @todo: check that the counters are vertically aligned.\nADD_CASES(TC_ConsoleOut,\n {\n // keeping these lines long improves readability, so:\n // clang-format off\n {\"^[-]+$\", MR_Next},\n {\"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$\", MR_Next},\n {\"^[-]+$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:1_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:1_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:1_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:1_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:2_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:2_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:2_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_Counters_Tabular/repeats:2/threads:2_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$\", MR_Next},\n {\"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$\", MR_Next},\n {\"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$\", MR_Next},\n {\"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$\", MR_Next},\n {\"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$\", MR_Next},\n {\"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$\", MR_Next},\n {\"^[-]+$\", MR_Next},\n {\"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$\", MR_Next},\n {\"^[-]+$\", MR_Next},\n {\"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^[-]+$\", MR_Next},\n {\"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$\", MR_Next},\n {\"^[-]+$\", MR_Next},\n {\"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\", MR_Next},\n {\"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$\"},\n // clang-format on\n });\nADD_CASES(TC_CSVOut, {{\"%csv_header,\"\n \"\\\"Bar\\\",\\\"Bat\\\",\\\"Baz\\\",\\\"Foo\\\",\\\"Frob\\\",\\\"Lob\\\"\"}});\n\n// ========================================================================= //\n// ------------------------- Tabular Counters Output ----------------------- //\n// ========================================================================= //\n\nvoid BM_Counters_Tabular(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters.insert({\n {\"Foo\", {1, bm::Counter::kAvgThreads}},\n {\"Bar\", {2, bm::Counter::kAvgThreads}},\n {\"Baz\", {4, bm::Counter::kAvgThreads}},\n {\"Bat\", {8, bm::Counter::kAvgThreads}},\n {\"Frob\", {16, bm::Counter::kAvgThreads}},\n {\"Lob\", {32, bm::Counter::kAvgThreads}},\n });\n}\nBENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 2)->Repetitions(2);\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"repetition_index\\\": 1,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1_mean\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1_median\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1_stddev\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1_cv\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:1\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"cv\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"percentage\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\n\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:2\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 1,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:2\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 2,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:2\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 1,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:2\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"repetition_index\\\": 1,$\", MR_Next},\n {\"\\\"threads\\\": 2,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:2_median\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 1,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:2\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 2,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:2_stddev\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 1,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:2\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 2,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:2_cv\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 1,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Tabular/repeats:2/threads:2\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 2,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"cv\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"percentage\\\",$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:1\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:1\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:1_mean\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:1_median\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:1_stddev\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:1_cv\\\",%csv_cv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:2\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:2\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:2_mean\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:2_median\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:2_stddev\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Tabular/repeats:2/threads:2_cv\\\",%csv_cv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckTabular(Results const& e) {\n CHECK_COUNTER_VALUE(e, int, \"Foo\", EQ, 1);\n CHECK_COUNTER_VALUE(e, int, \"Bar\", EQ, 2);\n CHECK_COUNTER_VALUE(e, int, \"Baz\", EQ, 4);\n CHECK_COUNTER_VALUE(e, int, \"Bat\", EQ, 8);\n CHECK_COUNTER_VALUE(e, int, \"Frob\", EQ, 16);\n CHECK_COUNTER_VALUE(e, int, \"Lob\", EQ, 32);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_Tabular/repeats:2/threads:1$\",\n &CheckTabular);\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_Tabular/repeats:2/threads:2$\",\n &CheckTabular);\n\n// ========================================================================= //\n// -------------------- Tabular+Rate Counters Output ----------------------- //\n// ========================================================================= //\n\nvoid BM_CounterRates_Tabular(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters.insert({\n {\"Foo\", {1, bm::Counter::kAvgThreadsRate}},\n {\"Bar\", {2, bm::Counter::kAvgThreadsRate}},\n {\"Baz\", {4, bm::Counter::kAvgThreadsRate}},\n {\"Bat\", {8, bm::Counter::kAvgThreadsRate}},\n {\"Frob\", {16, bm::Counter::kAvgThreadsRate}},\n {\"Lob\", {32, bm::Counter::kAvgThreadsRate}},\n });\n}\nBENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_CounterRates_Tabular/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 1,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_CounterRates_Tabular/threads:%int\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float,$\", MR_Next},\n {\"\\\"Frob\\\": %float,$\", MR_Next},\n {\"\\\"Lob\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_CounterRates_Tabular/threads:%int\\\",%csv_report,\"\n \"%float,%float,%float,%float,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckTabularRate(Results const& e) {\n double t = e.DurationCPUTime() / e.NumThreads();\n CHECK_FLOAT_COUNTER_VALUE(e, \"Foo\", EQ, 1. / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"Bar\", EQ, 2. / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"Baz\", EQ, 4. / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"Bat\", EQ, 8. / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"Frob\", EQ, 16. / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"Lob\", EQ, 32. / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_CounterRates_Tabular/threads:%int\",\n &CheckTabularRate);\n\n// ========================================================================= //\n// ------------------------- Tabular Counters Output ----------------------- //\n// ========================================================================= //\n\n// set only some of the counters\nvoid BM_CounterSet0_Tabular(benchmark::State& state) {\n for (auto _ : state) {\n }\n namespace bm = benchmark;\n state.counters.insert({\n {\"Foo\", {10, bm::Counter::kAvgThreads}},\n {\"Bar\", {20, bm::Counter::kAvgThreads}},\n {\"Baz\", {40, bm::Counter::kAvgThreads}},\n });\n}\nBENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_CounterSet0_Tabular/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 2,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_CounterSet0_Tabular/threads:%int\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_CounterSet0_Tabular/threads:%int\\\",%csv_report,\"\n \"%float,,%float,%float,,\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckSet0(Results const& e) {\n CHECK_COUNTER_VALUE(e, int, \"Foo\", EQ, 10);\n CHECK_COUNTER_VALUE(e, int, \"Bar\", EQ, 20);\n CHECK_COUNTER_VALUE(e, int, \"Baz\", EQ, 40);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_CounterSet0_Tabular\", &CheckSet0);\n\n// again.\nvoid BM_CounterSet1_Tabular(benchmark::State& state) {\n for (auto _ : state) {\n }\n namespace bm = benchmark;\n state.counters.insert({\n {\"Foo\", {15, bm::Counter::kAvgThreads}},\n {\"Bar\", {25, bm::Counter::kAvgThreads}},\n {\"Baz\", {45, bm::Counter::kAvgThreads}},\n });\n}\nBENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_CounterSet1_Tabular/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 3,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_CounterSet1_Tabular/threads:%int\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bar\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_CounterSet1_Tabular/threads:%int\\\",%csv_report,\"\n \"%float,,%float,%float,,\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckSet1(Results const& e) {\n CHECK_COUNTER_VALUE(e, int, \"Foo\", EQ, 15);\n CHECK_COUNTER_VALUE(e, int, \"Bar\", EQ, 25);\n CHECK_COUNTER_VALUE(e, int, \"Baz\", EQ, 45);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_CounterSet1_Tabular/threads:%int\", &CheckSet1);\n\n// ========================================================================= //\n// ------------------------- Tabular Counters Output ----------------------- //\n// ========================================================================= //\n\n// set only some of the counters, different set now.\nvoid BM_CounterSet2_Tabular(benchmark::State& state) {\n for (auto _ : state) {\n }\n namespace bm = benchmark;\n state.counters.insert({\n {\"Foo\", {10, bm::Counter::kAvgThreads}},\n {\"Bat\", {30, bm::Counter::kAvgThreads}},\n {\"Baz\", {40, bm::Counter::kAvgThreads}},\n });\n}\nBENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_CounterSet2_Tabular/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 4,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_CounterSet2_Tabular/threads:%int\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"Bat\\\": %float,$\", MR_Next},\n {\"\\\"Baz\\\": %float,$\", MR_Next},\n {\"\\\"Foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_CounterSet2_Tabular/threads:%int\\\",%csv_report,\"\n \",%float,%float,%float,,\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckSet2(Results const& e) {\n CHECK_COUNTER_VALUE(e, int, \"Foo\", EQ, 10);\n CHECK_COUNTER_VALUE(e, int, \"Bat\", EQ, 30);\n CHECK_COUNTER_VALUE(e, int, \"Baz\", EQ, 40);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_CounterSet2_Tabular\", &CheckSet2);\n} // end namespace\n\n// ========================================================================= //\n// --------------------------- TEST CASES END ------------------------------ //\n// ========================================================================= //\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n RunOutputTests(argc, argv);\n}\n" }, { "path": "test/user_counters_test.cc", "content": "\n#undef NDEBUG\n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/counter.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n#include \"output_test.h\"\n\n// ========================================================================= //\n// ---------------------- Testing Prologue Output -------------------------- //\n// ========================================================================= //\n\n// clang-format off\n\nADD_CASES(TC_ConsoleOut,\n {{\"^[-]+$\", MR_Next},\n {\"^Benchmark %s Time %s CPU %s Iterations UserCounters...$\", MR_Next},\n {\"^[-]+$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"%csv_header,\\\"bar\\\",\\\"foo\\\"\"}});\n\n// clang-format on\n\n// ========================================================================= //\n// ------------------------- Simple Counters Output ------------------------ //\n// ========================================================================= //\nnamespace {\nvoid BM_Counters_Simple(benchmark::State& state) {\n for (auto _ : state) {\n }\n state.counters[\"foo\"] = 1;\n state.counters[\"bar\"] = 2 * static_cast(state.iterations());\n}\nBENCHMARK(BM_Counters_Simple);\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_Counters_Simple\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Simple\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_Simple\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckSimple(Results const& e) {\n double its = e.NumIterations();\n CHECK_COUNTER_VALUE(e, int, \"foo\", EQ, 1);\n // check that the value of bar is within 0.1% of the expected value\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. * its, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_Simple\", &CheckSimple);\n} // end namespace\n\n// ========================================================================= //\n// --------------------- Counters+Items+Bytes/s Output --------------------- //\n// ========================================================================= //\n\nnamespace {\nint num_calls1 = 0;\nvoid BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n state.counters[\"foo\"] = 1;\n state.counters[\"bar\"] = ++num_calls1;\n state.SetBytesProcessed(364);\n state.SetItemsProcessed(150);\n}\nBENCHMARK(BM_Counters_WithBytesAndItemsPSec);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_WithBytesAndItemsPSec %console_report \"\n \"bar=%hrfloat bytes_per_second=%hrfloat/s \"\n \"foo=%hrfloat items_per_second=%hrfloat/s$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_WithBytesAndItemsPSec\\\",$\"},\n {\"\\\"family_index\\\": 1,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_WithBytesAndItemsPSec\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"bytes_per_second\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float,$\", MR_Next},\n {\"\\\"items_per_second\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_WithBytesAndItemsPSec\\\",\"\n \"%csv_bytes_items_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckBytesAndItemsPSec(Results const& e) {\n double t = e.DurationCPUTime(); // this (and not real time) is the time used\n CHECK_COUNTER_VALUE(e, int, \"foo\", EQ, 1);\n CHECK_COUNTER_VALUE(e, int, \"bar\", EQ, num_calls1);\n // check that the values are within 0.1% of the expected values\n CHECK_FLOAT_RESULT_VALUE(e, \"bytes_per_second\", EQ, 364. / t, 0.001);\n CHECK_FLOAT_RESULT_VALUE(e, \"items_per_second\", EQ, 150. / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_WithBytesAndItemsPSec\",\n &CheckBytesAndItemsPSec);\n} // end namespace\n\n// ========================================================================= //\n// ------------------------- Rate Counters Output -------------------------- //\n// ========================================================================= //\nnamespace {\nvoid BM_Counters_Rate(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kIsRate};\n state.counters[\"bar\"] = bm::Counter{2, bm::Counter::kIsRate};\n}\nBENCHMARK(BM_Counters_Rate);\nADD_CASES(\n TC_ConsoleOut,\n {{\"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_Counters_Rate\\\",$\"},\n {\"\\\"family_index\\\": 2,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Rate\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_Rate\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckRate(Results const& e) {\n double t = e.DurationCPUTime(); // this (and not real time) is the time used\n // check that the values are within 0.1% of the expected values\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, 1. / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_Rate\", &CheckRate);\n} // end namespace\n\n// ========================================================================= //\n// ----------------------- Inverted Counters Output ------------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Invert(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{0.0001, bm::Counter::kInvert};\n state.counters[\"bar\"] = bm::Counter{10000, bm::Counter::kInvert};\n}\nBENCHMARK(BM_Invert);\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_Invert %console_report bar=%hrfloatu foo=%hrfloatk$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_Invert\\\",$\"},\n {\"\\\"family_index\\\": 3,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Invert\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Invert\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckInvert(Results const& e) {\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, 10000, 0.0001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 0.0001, 0.0001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Invert\", &CheckInvert);\n} // end namespace\n\n// ========================================================================= //\n// --------------------- InvertedRate Counters Output ---------------------- //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_InvertedRate(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] =\n bm::Counter{1, bm::Counter::kIsRate | bm::Counter::kInvert};\n state.counters[\"bar\"] =\n bm::Counter{8192, bm::Counter::kIsRate | bm::Counter::kInvert};\n}\nBENCHMARK(BM_Counters_InvertedRate);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_InvertedRate %console_report \"\n \"bar=%hrfloats foo=%hrfloats$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_InvertedRate\\\",$\"},\n {\"\\\"family_index\\\": 4,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_InvertedRate\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_InvertedRate\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckInvertedRate(Results const& e) {\n double t = e.DurationCPUTime(); // this (and not real time) is the time used\n // check that the values are within 0.1% of the expected values\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, t / 8192.0, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_InvertedRate\", &CheckInvertedRate);\n} // end namespace\n\n// ========================================================================= //\n// ------------------------- Thread Counters Output ------------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_Threads(benchmark::State& state) {\n for (auto _ : state) {\n }\n state.counters[\"foo\"] = 1;\n state.counters[\"bar\"] = 2;\n}\nBENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_Threads/threads:%int %console_report \"\n \"bar=%hrfloat foo=%hrfloat$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Threads/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 5,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Threads/threads:%int\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(\n TC_CSVOut,\n {{\"^\\\"BM_Counters_Threads/threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckThreads(Results const& e) {\n CHECK_COUNTER_VALUE(e, int, \"foo\", EQ, e.NumThreads());\n CHECK_COUNTER_VALUE(e, int, \"bar\", EQ, 2 * e.NumThreads());\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_Threads/threads:%int\", &CheckThreads);\n} // end namespace\n\n// ========================================================================= //\n// ---------------------- ThreadAvg Counters Output ------------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_AvgThreads(benchmark::State& state) {\n for (auto _ : state) {\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kAvgThreads};\n state.counters[\"bar\"] = bm::Counter{2, bm::Counter::kAvgThreads};\n}\nBENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_AvgThreads/threads:%int \"\n \"%console_report bar=%hrfloat foo=%hrfloat$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_AvgThreads/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 6,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_AvgThreads/threads:%int\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(\n TC_CSVOut,\n {{\"^\\\"BM_Counters_AvgThreads/threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckAvgThreads(Results const& e) {\n CHECK_COUNTER_VALUE(e, int, \"foo\", EQ, 1);\n CHECK_COUNTER_VALUE(e, int, \"bar\", EQ, 2);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_AvgThreads/threads:%int\",\n &CheckAvgThreads);\n} // end namespace\n\n// ========================================================================= //\n// ---------------------- ThreadAvg Counters Output ------------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_AvgThreadsRate(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kAvgThreadsRate};\n state.counters[\"bar\"] = bm::Counter{2, bm::Counter::kAvgThreadsRate};\n}\nBENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_AvgThreadsRate/threads:%int \"\n \"%console_report bar=%hrfloat/s foo=%hrfloat/s$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_AvgThreadsRate/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 7,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_AvgThreadsRate/threads:%int\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_AvgThreadsRate/\"\n \"threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckAvgThreadsRate(Results const& e) {\n // this (and not real time) is the time used\n double t = e.DurationCPUTime() / e.NumThreads();\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, 1. / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_AvgThreadsRate/threads:%int\",\n &CheckAvgThreadsRate);\n} // end namespace\n\n// ========================================================================= //\n// ------------------- IterationInvariant Counters Output ------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_IterationInvariant(benchmark::State& state) {\n for (auto _ : state) {\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kIsIterationInvariant};\n state.counters[\"bar\"] = bm::Counter{2, bm::Counter::kIsIterationInvariant};\n}\nBENCHMARK(BM_Counters_IterationInvariant);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_IterationInvariant %console_report \"\n \"bar=%hrfloat foo=%hrfloat$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_IterationInvariant\\\",$\"},\n {\"\\\"family_index\\\": 8,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_IterationInvariant\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_IterationInvariant\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckIterationInvariant(Results const& e) {\n double its = e.NumIterations();\n // check that the values are within 0.1% of the expected value\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, its, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. * its, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_IterationInvariant\",\n &CheckIterationInvariant);\n} // end namespace\n\n// ========================================================================= //\n// ----------------- IterationInvariantRate Counters Output ---------------- //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_kIsIterationInvariantRate(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] =\n bm::Counter{1, bm::Counter::kIsIterationInvariantRate};\n state.counters[\"bar\"] =\n bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kIsIterationInvariant};\n}\nBENCHMARK(BM_Counters_kIsIterationInvariantRate);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_kIsIterationInvariantRate \"\n \"%console_report bar=%hrfloat/s foo=%hrfloat/s$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_kIsIterationInvariantRate\\\",$\"},\n {\"\\\"family_index\\\": 9,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_kIsIterationInvariantRate\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_kIsIterationInvariantRate\\\",%csv_report,\"\n \"%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckIsIterationInvariantRate(Results const& e) {\n double its = e.NumIterations();\n double t = e.DurationCPUTime(); // this (and not real time) is the time used\n // check that the values are within 0.1% of the expected values\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, its * 1. / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, its * 2. / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_kIsIterationInvariantRate\",\n &CheckIsIterationInvariantRate);\n} // end namespace\n\n// ========================================================================= //\n// --------------------- AvgIterations Counters Output --------------------- //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_AvgIterations(benchmark::State& state) {\n for (auto _ : state) {\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kAvgIterations};\n state.counters[\"bar\"] = bm::Counter{2, bm::Counter::kAvgIterations};\n}\nBENCHMARK(BM_Counters_AvgIterations);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_AvgIterations %console_report \"\n \"bar=%hrfloat foo=%hrfloat$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_AvgIterations\\\",$\"},\n {\"\\\"family_index\\\": 10,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_AvgIterations\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_AvgIterations\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckAvgIterations(Results const& e) {\n double its = e.NumIterations();\n // check that the values are within 0.1% of the expected value\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, 1. / its, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. / its, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_AvgIterations\", &CheckAvgIterations);\n} // end namespace\n\n// ========================================================================= //\n// ------------------- AvgIterationsRate Counters Output ------------------- //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_kAvgIterationsRate(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kAvgIterationsRate};\n state.counters[\"bar\"] =\n bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kAvgIterations};\n}\nBENCHMARK(BM_Counters_kAvgIterationsRate);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_kAvgIterationsRate \"\n \"%console_report bar=%hrfloat/s foo=%hrfloat/s$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_kAvgIterationsRate\\\",$\"},\n {\"\\\"family_index\\\": 11,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_kAvgIterationsRate\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_kAvgIterationsRate\\\",%csv_report,\"\n \"%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckAvgIterationsRate(Results const& e) {\n double its = e.NumIterations();\n double t = e.DurationCPUTime(); // this (and not real time) is the time used\n // check that the values are within 0.1% of the expected values\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, 1. / its / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. / its / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_kAvgIterationsRate\",\n &CheckAvgIterationsRate);\n} // end namespace\n\n// ========================================================================= //\n// --------------------------- TEST CASES END ------------------------------ //\n// ========================================================================= //\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n RunOutputTests(argc, argv);\n}\n" }, { "path": "test/user_counters_thousands_test.cc", "content": "\n#undef NDEBUG\n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/counter.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"output_test.h\"\n\nnamespace {\n// ========================================================================= //\n// ------------------------ Thousands Customisation ------------------------ //\n// ========================================================================= //\n\nvoid BM_Counters_Thousands(benchmark::State& state) {\n for (auto _ : state) {\n }\n namespace bm = benchmark;\n state.counters.insert({\n {\"t0_1000000DefaultBase\",\n bm::Counter(1000 * 1000, bm::Counter::kDefaults)},\n {\"t1_1000000Base1000\", bm::Counter(1000 * 1000, bm::Counter::kDefaults,\n bm::Counter::OneK::kIs1000)},\n {\"t2_1000000Base1024\", bm::Counter(1000 * 1000, bm::Counter::kDefaults,\n bm::Counter::OneK::kIs1024)},\n {\"t3_1048576Base1000\", bm::Counter(1024 * 1024, bm::Counter::kDefaults,\n bm::Counter::OneK::kIs1000)},\n {\"t4_1048576Base1024\", bm::Counter(1024 * 1024, bm::Counter::kDefaults,\n bm::Counter::OneK::kIs1024)},\n });\n}\nBENCHMARK(BM_Counters_Thousands)->Repetitions(2);\nADD_CASES(\n TC_ConsoleOut,\n {\n {\"^BM_Counters_Thousands/repeats:2 %console_report \"\n \"t0_1000000DefaultBase=1M \"\n \"t1_1000000Base1000=1M t2_1000000Base1024=976.56[23]Ki \"\n \"t3_1048576Base1000=1.04858M t4_1048576Base1024=1Mi$\"},\n {\"^BM_Counters_Thousands/repeats:2 %console_report \"\n \"t0_1000000DefaultBase=1M \"\n \"t1_1000000Base1000=1M t2_1000000Base1024=976.56[23]Ki \"\n \"t3_1048576Base1000=1.04858M t4_1048576Base1024=1Mi$\"},\n {\"^BM_Counters_Thousands/repeats:2_mean %console_report \"\n \"t0_1000000DefaultBase=1M t1_1000000Base1000=1M \"\n \"t2_1000000Base1024=976.56[23]Ki t3_1048576Base1000=1.04858M \"\n \"t4_1048576Base1024=1Mi$\"},\n {\"^BM_Counters_Thousands/repeats:2_median %console_report \"\n \"t0_1000000DefaultBase=1M t1_1000000Base1000=1M \"\n \"t2_1000000Base1024=976.56[23]Ki t3_1048576Base1000=1.04858M \"\n \"t4_1048576Base1024=1Mi$\"},\n {\"^BM_Counters_Thousands/repeats:2_stddev %console_time_only_report [ \"\n \"]*2 t0_1000000DefaultBase=0 t1_1000000Base1000=0 \"\n \"t2_1000000Base1024=0 t3_1048576Base1000=0 t4_1048576Base1024=0$\"},\n });\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Thousands/repeats:2\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Thousands/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"t0_1000000DefaultBase\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t1_1000000Base1000\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t2_1000000Base1024\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t3_1048576Base1000\\\": 1\\\\.048576(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t4_1048576Base1024\\\": 1\\\\.048576(0)*e\\\\+(0)*6$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Thousands/repeats:2\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Thousands/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"repetition_index\\\": 1,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"t0_1000000DefaultBase\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t1_1000000Base1000\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t2_1000000Base1024\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t3_1048576Base1000\\\": 1\\\\.048576(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t4_1048576Base1024\\\": 1\\\\.048576(0)*e\\\\+(0)*6$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Thousands/repeats:2_mean\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Thousands/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"mean\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 2,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"t0_1000000DefaultBase\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t1_1000000Base1000\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t2_1000000Base1024\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t3_1048576Base1000\\\": 1\\\\.048576(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t4_1048576Base1024\\\": 1\\\\.048576(0)*e\\\\+(0)*6$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Thousands/repeats:2_median\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Thousands/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"median\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 2,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"t0_1000000DefaultBase\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t1_1000000Base1000\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t2_1000000Base1024\\\": 1\\\\.(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t3_1048576Base1000\\\": 1\\\\.048576(0)*e\\\\+(0)*6,$\", MR_Next},\n {\"\\\"t4_1048576Base1024\\\": 1\\\\.048576(0)*e\\\\+(0)*6$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Thousands/repeats:2_stddev\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Thousands/repeats:2\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"aggregate\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 2,$\", MR_Next},\n {\"\\\"threads\\\": 1,$\", MR_Next},\n {\"\\\"aggregate_name\\\": \\\"stddev\\\",$\", MR_Next},\n {\"\\\"aggregate_unit\\\": \\\"time\\\",$\", MR_Next},\n {\"\\\"iterations\\\": 2,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"t0_1000000DefaultBase\\\": 0\\\\.(0)*e\\\\+(0)*,$\", MR_Next},\n {\"\\\"t1_1000000Base1000\\\": 0\\\\.(0)*e\\\\+(0)*,$\", MR_Next},\n {\"\\\"t2_1000000Base1024\\\": 0\\\\.(0)*e\\\\+(0)*,$\", MR_Next},\n {\"\\\"t3_1048576Base1000\\\": 0\\\\.(0)*e\\\\+(0)*,$\", MR_Next},\n {\"\\\"t4_1048576Base1024\\\": 0\\\\.(0)*e\\\\+(0)*$\", MR_Next},\n {\"}\", MR_Next}});\n\nADD_CASES(\n TC_CSVOut,\n {{\"^\\\"BM_Counters_Thousands/\"\n \"repeats:2\\\",%csv_report,1e\\\\+(0)*6,1e\\\\+(0)*6,1e\\\\+(0)*6,1\\\\.04858e\\\\+(\"\n \"0)*6,1\\\\.04858e\\\\+(0)*6$\"},\n {\"^\\\"BM_Counters_Thousands/\"\n \"repeats:2\\\",%csv_report,1e\\\\+(0)*6,1e\\\\+(0)*6,1e\\\\+(0)*6,1\\\\.04858e\\\\+(\"\n \"0)*6,1\\\\.04858e\\\\+(0)*6$\"},\n {\"^\\\"BM_Counters_Thousands/\"\n \"repeats:2_mean\\\",%csv_report,1e\\\\+(0)*6,1e\\\\+(0)*6,1e\\\\+(0)*6,1\\\\.\"\n \"04858e\\\\+(0)*6,1\\\\.04858e\\\\+(0)*6$\"},\n {\"^\\\"BM_Counters_Thousands/\"\n \"repeats:2_median\\\",%csv_report,1e\\\\+(0)*6,1e\\\\+(0)*6,1e\\\\+(0)*6,1\\\\.\"\n \"04858e\\\\+(0)*6,1\\\\.04858e\\\\+(0)*6$\"},\n {\"^\\\"BM_Counters_Thousands/repeats:2_stddev\\\",%csv_report,0,0,0,0,0$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckThousands(Results const& e) {\n if (e.name != \"BM_Counters_Thousands/repeats:2\") {\n return; // Do not check the aggregates!\n }\n\n // check that the values are within 0.01% of the expected values\n CHECK_FLOAT_COUNTER_VALUE(e, \"t0_1000000DefaultBase\", EQ, 1000 * 1000,\n 0.0001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"t1_1000000Base1000\", EQ, 1000 * 1000, 0.0001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"t2_1000000Base1024\", EQ, 1000 * 1000, 0.0001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"t3_1048576Base1000\", EQ, 1024 * 1024, 0.0001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"t4_1048576Base1024\", EQ, 1024 * 1024, 0.0001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_Thousands\", &CheckThousands);\n} // end namespace\n\n// ========================================================================= //\n// --------------------------- TEST CASES END ------------------------------ //\n// ========================================================================= //\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n RunOutputTests(argc, argv);\n}\n" }, { "path": "test/user_counters_threads_test.cc", "content": "\n#undef NDEBUG\n\n#include \"benchmark/benchmark_api.h\"\n#include \"benchmark/registration.h\"\n#include \"benchmark/state.h\"\n#include \"benchmark/utils.h\"\n#include \"output_test.h\"\n\n// ========================================================================= //\n// ---------------------- Testing Prologue Output -------------------------- //\n// ========================================================================= //\n\n// clang-format off\n\nADD_CASES(TC_ConsoleOut,\n {{\"^[-]+$\", MR_Next},\n {\"^Benchmark %s Time %s CPU %s Iterations UserCounters...$\", MR_Next},\n {\"^[-]+$\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"%csv_header,\\\"bar\\\",\\\"foo\\\"\"}});\n\n// clang-format on\n\n// ========================================================================= //\n// ------------------------- Simple Counters Output ------------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_Simple(benchmark::State& state) {\n for (auto _ : state) {\n }\n state.counters[\"foo\"] = 1;\n state.counters[\"bar\"] = 2 * static_cast(state.iterations());\n}\nBENCHMARK(BM_Counters_Simple)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_Simple/threads:%int %console_report \"\n \"bar=%hrfloat foo=%hrfloat$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Simple/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 0,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Simple/threads:%int\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(\n TC_CSVOut,\n {{\"^\\\"BM_Counters_Simple/threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckSimple(Results const& e) {\n double its = e.NumIterations();\n CHECK_COUNTER_VALUE(e, int, \"foo\", EQ, 1 * e.NumThreads());\n // check that the value of bar is within 0.1% of the expected value\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. * its, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_Simple/threads:%int\", &CheckSimple);\n} // end namespace\n\n// ========================================================================= //\n// --------------------- Counters+Items+Bytes/s Output --------------------- //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n state.counters[\"foo\"] = 1;\n state.SetBytesProcessed(364);\n state.SetItemsProcessed(150);\n}\nBENCHMARK(BM_Counters_WithBytesAndItemsPSec)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_Counters_WithBytesAndItemsPSec/threads:%int %console_report \"\n \"bytes_per_second=%hrfloat/s \"\n \"foo=%hrfloat items_per_second=%hrfloat/s$\"}});\nADD_CASES(\n TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_WithBytesAndItemsPSec/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 1,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_WithBytesAndItemsPSec/threads:%int\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bytes_per_second\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float,$\", MR_Next},\n {\"\\\"items_per_second\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_WithBytesAndItemsPSec/threads:%int\\\",\"\n \"%csv_bytes_items_report,,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckBytesAndItemsPSec(Results const& e) {\n // this (and not real time) is the time used\n double t = e.DurationCPUTime() / e.NumThreads();\n CHECK_COUNTER_VALUE(e, int, \"foo\", EQ, 1 * e.NumThreads());\n // check that the values are within 0.1% of the expected values\n CHECK_FLOAT_RESULT_VALUE(e, \"bytes_per_second\", EQ,\n (364. * e.NumThreads()) / t, 0.001);\n CHECK_FLOAT_RESULT_VALUE(e, \"items_per_second\", EQ,\n (150. * e.NumThreads()) / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_WithBytesAndItemsPSec/threads:%int\",\n &CheckBytesAndItemsPSec);\n} // end namespace\n\n// ========================================================================= //\n// ------------------------- Rate Counters Output -------------------------- //\n// ========================================================================= //\nnamespace {\nvoid BM_Counters_Rate(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kIsRate};\n state.counters[\"bar\"] = bm::Counter{2, bm::Counter::kIsRate};\n}\nBENCHMARK(BM_Counters_Rate)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_Rate/threads:%int %console_report \"\n \"bar=%hrfloat/s foo=%hrfloat/s$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Rate/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 2,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Rate/threads:%int\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_Rate/threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckRate(Results const& e) {\n // this (and not real time) is the time used\n double t = e.DurationCPUTime() / e.NumThreads();\n // check that the values are within 0.1% of the expected values\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, (1. * e.NumThreads()) / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, (2. * e.NumThreads()) / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_Rate/threads:%int\", &CheckRate);\n} // end namespace\n\n// ========================================================================= //\n// ----------------------- Inverted Counters Output ------------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Invert(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{0.0001, bm::Counter::kInvert};\n state.counters[\"bar\"] = bm::Counter{10000, bm::Counter::kInvert};\n}\nBENCHMARK(BM_Invert)->ThreadRange(1, 8);\nADD_CASES(\n TC_ConsoleOut,\n {{\"^BM_Invert/threads:%int %console_report bar=%hrfloatu foo=%hrfloatk$\"}});\nADD_CASES(TC_JSONOut, {{\"\\\"name\\\": \\\"BM_Invert/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 3,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Invert/threads:%int\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Invert/threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckInvert(Results const& e) {\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, 1. / (0.0001 * e.NumThreads()),\n 0.0001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 1. / (10000 * e.NumThreads()),\n 0.0001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Invert/threads:%int\", &CheckInvert);\n} // end namespace\n\n// ========================================================================= //\n// --------------------- InvertedRate Counters Output ---------------------- //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_InvertedRate(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] =\n bm::Counter{1, bm::Counter::kIsRate | bm::Counter::kInvert};\n state.counters[\"bar\"] =\n bm::Counter{8192, bm::Counter::kIsRate | bm::Counter::kInvert};\n}\nBENCHMARK(BM_Counters_InvertedRate)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_Counters_InvertedRate/threads:%int %console_report \"\n \"bar=%hrfloats foo=%hrfloats$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_InvertedRate/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 4,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_InvertedRate/threads:%int\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_InvertedRate/\"\n \"threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckInvertedRate(Results const& e) {\n // this (and not real time) is the time used\n double t = e.DurationCPUTime() / e.NumThreads();\n // check that the values are within 0.1% of the expected values\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, t / (e.NumThreads()), 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, t / (8192.0 * e.NumThreads()), 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_InvertedRate/threads:%int\",\n &CheckInvertedRate);\n} // end namespace\n\n// ========================================================================= //\n// ------------------------- Thread Counters Output ------------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_Threads(benchmark::State& state) {\n for (auto _ : state) {\n }\n state.counters[\"foo\"] = 1;\n state.counters[\"bar\"] = 2;\n}\nBENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_Threads/threads:%int %console_report \"\n \"bar=%hrfloat foo=%hrfloat$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_Threads/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 5,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_Threads/threads:%int\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(\n TC_CSVOut,\n {{\"^\\\"BM_Counters_Threads/threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckThreads(Results const& e) {\n CHECK_COUNTER_VALUE(e, int, \"foo\", EQ, e.NumThreads());\n CHECK_COUNTER_VALUE(e, int, \"bar\", EQ, 2 * e.NumThreads());\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_Threads/threads:%int\", &CheckThreads);\n} // end namespace\n\n// ========================================================================= //\n// ---------------------- ThreadAvg Counters Output ------------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_AvgThreads(benchmark::State& state) {\n for (auto _ : state) {\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kAvgThreads};\n state.counters[\"bar\"] = bm::Counter{2, bm::Counter::kAvgThreads};\n}\nBENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_AvgThreads/threads:%int \"\n \"%console_report bar=%hrfloat foo=%hrfloat$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_AvgThreads/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 6,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_AvgThreads/threads:%int\\\",$\", MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(\n TC_CSVOut,\n {{\"^\\\"BM_Counters_AvgThreads/threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckAvgThreads(Results const& e) {\n CHECK_COUNTER_VALUE(e, int, \"foo\", EQ, 1);\n CHECK_COUNTER_VALUE(e, int, \"bar\", EQ, 2);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_AvgThreads/threads:%int\",\n &CheckAvgThreads);\n} // end namespace\n\n// ========================================================================= //\n// ---------------------- ThreadAvg Counters Output ------------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_AvgThreadsRate(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kAvgThreadsRate};\n state.counters[\"bar\"] = bm::Counter{2, bm::Counter::kAvgThreadsRate};\n}\nBENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_AvgThreadsRate/threads:%int \"\n \"%console_report bar=%hrfloat/s foo=%hrfloat/s$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_AvgThreadsRate/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 7,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_AvgThreadsRate/threads:%int\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_AvgThreadsRate/\"\n \"threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckAvgThreadsRate(Results const& e) {\n // this (and not real time) is the time used\n double t = e.DurationCPUTime() / e.NumThreads();\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, 1. / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_AvgThreadsRate/threads:%int\",\n &CheckAvgThreadsRate);\n} // end namespace\n\n// ========================================================================= //\n// ------------------- IterationInvariant Counters Output ------------------ //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_IterationInvariant(benchmark::State& state) {\n for (auto _ : state) {\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kIsIterationInvariant};\n state.counters[\"bar\"] = bm::Counter{2, bm::Counter::kIsIterationInvariant};\n}\nBENCHMARK(BM_Counters_IterationInvariant)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_Counters_IterationInvariant/threads:%int %console_report \"\n \"bar=%hrfloat foo=%hrfloat$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_IterationInvariant/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 8,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_IterationInvariant/threads:%int\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_IterationInvariant/\"\n \"threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckIterationInvariant(Results const& e) {\n double its = e.NumIterations();\n // check that the values are within 0.1% of the expected value\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, its * e.NumThreads(), 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. * its * e.NumThreads(), 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_IterationInvariant/threads:%int\",\n &CheckIterationInvariant);\n} // end namespace\n\n// ========================================================================= //\n// ----------------- IterationInvariantRate Counters Output ---------------- //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_kIsIterationInvariantRate(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] =\n bm::Counter{1, bm::Counter::kIsIterationInvariantRate};\n state.counters[\"bar\"] =\n bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kIsIterationInvariant};\n}\nBENCHMARK(BM_Counters_kIsIterationInvariantRate)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_Counters_kIsIterationInvariantRate/threads:%int \"\n \"%console_report bar=%hrfloat/s foo=%hrfloat/s$\"}});\nADD_CASES(\n TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_kIsIterationInvariantRate/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 9,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_kIsIterationInvariantRate/threads:%int\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(\n TC_CSVOut,\n {{\"^\\\"BM_Counters_kIsIterationInvariantRate/threads:%int\\\",%csv_report,\"\n \"%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckIsIterationInvariantRate(Results const& e) {\n double its = e.NumIterations();\n // this (and not real time) is the time used\n double t = e.DurationCPUTime() / e.NumThreads();\n // check that the values are within 0.1% of the expected values\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, its * 1. * e.NumThreads() / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, its * 2. * e.NumThreads() / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_kIsIterationInvariantRate/threads:%int\",\n &CheckIsIterationInvariantRate);\n} // end namespace\n\n// ========================================================================= //\n// --------------------- AvgIterations Counters Output --------------------- //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_AvgIterations(benchmark::State& state) {\n for (auto _ : state) {\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kAvgIterations};\n state.counters[\"bar\"] = bm::Counter{2, bm::Counter::kAvgIterations};\n}\nBENCHMARK(BM_Counters_AvgIterations)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut,\n {{\"^BM_Counters_AvgIterations/threads:%int %console_report \"\n \"bar=%hrfloat foo=%hrfloat$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_AvgIterations/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 10,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_AvgIterations/threads:%int\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut, {{\"^\\\"BM_Counters_AvgIterations/\"\n \"threads:%int\\\",%csv_report,%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckAvgIterations(Results const& e) {\n double its = e.NumIterations();\n // check that the values are within 0.1% of the expected value\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, 1. * e.NumThreads() / its, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. * e.NumThreads() / its, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_AvgIterations/threads:%int\",\n &CheckAvgIterations);\n} // end namespace\n\n// ========================================================================= //\n// ------------------- AvgIterationsRate Counters Output ------------------- //\n// ========================================================================= //\n\nnamespace {\nvoid BM_Counters_kAvgIterationsRate(benchmark::State& state) {\n for (auto _ : state) {\n // This test requires a non-zero CPU time to avoid divide-by-zero\n auto iterations = static_cast(state.iterations()) *\n static_cast(state.iterations());\n benchmark::DoNotOptimize(iterations);\n }\n namespace bm = benchmark;\n state.counters[\"foo\"] = bm::Counter{1, bm::Counter::kAvgIterationsRate};\n state.counters[\"bar\"] =\n bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kAvgIterations};\n}\nBENCHMARK(BM_Counters_kAvgIterationsRate)->ThreadRange(1, 8);\nADD_CASES(TC_ConsoleOut, {{\"^BM_Counters_kAvgIterationsRate/threads:%int \"\n \"%console_report bar=%hrfloat/s foo=%hrfloat/s$\"}});\nADD_CASES(TC_JSONOut,\n {{\"\\\"name\\\": \\\"BM_Counters_kAvgIterationsRate/threads:%int\\\",$\"},\n {\"\\\"family_index\\\": 11,$\", MR_Next},\n {\"\\\"per_family_instance_index\\\": 0,$\", MR_Next},\n {\"\\\"run_name\\\": \\\"BM_Counters_kAvgIterationsRate/threads:%int\\\",$\",\n MR_Next},\n {\"\\\"run_type\\\": \\\"iteration\\\",$\", MR_Next},\n {\"\\\"repetitions\\\": 1,$\", MR_Next},\n {\"\\\"repetition_index\\\": 0,$\", MR_Next},\n {\"\\\"threads\\\": %int,$\", MR_Next},\n {\"\\\"iterations\\\": %int,$\", MR_Next},\n {\"\\\"real_time\\\": %float,$\", MR_Next},\n {\"\\\"cpu_time\\\": %float,$\", MR_Next},\n {\"\\\"time_unit\\\": \\\"ns\\\",$\", MR_Next},\n {\"\\\"bar\\\": %float,$\", MR_Next},\n {\"\\\"foo\\\": %float$\", MR_Next},\n {\"}\", MR_Next}});\nADD_CASES(TC_CSVOut,\n {{\"^\\\"BM_Counters_kAvgIterationsRate/threads:%int\\\",%csv_report,\"\n \"%float,%float$\"}});\n// VS2013 does not allow this function to be passed as a lambda argument\n// to CHECK_BENCHMARK_RESULTS()\nvoid CheckAvgIterationsRate(Results const& e) {\n double its = e.NumIterations();\n // this (and not real time) is the time used\n double t = e.DurationCPUTime() / e.NumThreads();\n // check that the values are within 0.1% of the expected values\n CHECK_FLOAT_COUNTER_VALUE(e, \"foo\", EQ, 1. * e.NumThreads() / its / t, 0.001);\n CHECK_FLOAT_COUNTER_VALUE(e, \"bar\", EQ, 2. * e.NumThreads() / its / t, 0.001);\n}\nCHECK_BENCHMARK_RESULTS(\"BM_Counters_kAvgIterationsRate/threads:%int\",\n &CheckAvgIterationsRate);\n} // end namespace\n\n// ========================================================================= //\n// --------------------------- TEST CASES END ------------------------------ //\n// ========================================================================= //\n\nint main(int argc, char* argv[]) {\n benchmark::MaybeReenterWithoutASLR(argc, argv);\n RunOutputTests(argc, argv);\n}\n" }, { "path": "tools/BUILD.bazel", "content": "load(\"@rules_python//python:defs.bzl\", \"py_binary\", \"py_library\")\nload(\"@tools_pip_deps//:requirements.bzl\", \"requirement\")\n\npy_library(\n name = \"gbench\",\n srcs = glob([\"gbench/*.py\"]),\n deps = [\n requirement(\"numpy\"),\n requirement(\"scipy\"),\n ],\n)\n\npy_binary(\n name = \"compare\",\n srcs = [\"compare.py\"],\n imports = [\".\"],\n python_version = \"PY3\",\n deps = [\n \":gbench\",\n ],\n)\n" }, { "path": "tools/compare.py", "content": "#!/usr/bin/env python3\n\n# type: ignore\n\n\"\"\"\ncompare.py - versatile benchmark output compare tool\n\"\"\"\n\nimport argparse\nimport json\nimport os\nimport sys\nimport unittest\nfrom argparse import ArgumentParser\n\nimport gbench\nfrom gbench import report, util\n\n\ndef check_inputs(in1, in2, flags):\n \"\"\"\n Perform checking on the user provided inputs and diagnose any abnormalities\n \"\"\"\n in1_kind, _ = util.classify_input_file(in1)\n in2_kind, _ = util.classify_input_file(in2)\n output_file = util.find_benchmark_flag(\"--benchmark_out=\", flags)\n output_type = util.find_benchmark_flag(\"--benchmark_out_format=\", flags)\n if (\n in1_kind == util.IT_Executable\n and in2_kind == util.IT_Executable\n and output_file\n ):\n print(\n (\n \"WARNING: '--benchmark_out=%s' will be passed to both \"\n \"benchmarks causing it to be overwritten\"\n )\n % output_file\n )\n if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON:\n # When both sides are JSON the only supported flag is\n # --benchmark_filter=\n for flag in util.remove_benchmark_flags(\"--benchmark_filter=\", flags):\n print(\n \"WARNING: passing %s has no effect since both \"\n \"inputs are JSON\" % flag\n )\n if output_type is not None and output_type != \"json\":\n print(\n (\n \"ERROR: passing '--benchmark_out_format=%s' to 'compare.py`\"\n \" is not supported.\"\n )\n % output_type\n )\n sys.exit(1)\n\n\ndef create_parser():\n parser = ArgumentParser(\n description=\"versatile benchmark output compare tool\"\n )\n\n parser.add_argument(\n \"-a\",\n \"--display_aggregates_only\",\n dest=\"display_aggregates_only\",\n action=\"store_true\",\n help=\"If there are repetitions, by default, we display everything - the\"\n \" actual runs, and the aggregates computed. Sometimes, it is \"\n \"desirable to only view the aggregates. E.g. when there are a lot \"\n \"of repetitions. Do note that only the display is affected. \"\n \"Internally, all the actual runs are still used, e.g. for U test.\",\n )\n\n parser.add_argument(\n \"--no-color\",\n dest=\"color\",\n default=True,\n action=\"store_false\",\n help=\"Do not use colors in the terminal output\",\n )\n\n parser.add_argument(\n \"-d\",\n \"--dump_to_json\",\n dest=\"dump_to_json\",\n help=(\n \"Additionally, dump benchmark comparison output to this file in\"\n \" JSON format.\"\n ),\n )\n\n utest = parser.add_argument_group()\n utest.add_argument(\n \"--no-utest\",\n dest=\"utest\",\n default=True,\n action=\"store_false\",\n help=(\n \"The tool can do a two-tailed Mann-Whitney U test with the null\"\n \" hypothesis that it is equally likely that a randomly selected\"\n \" value from one sample will be less than or greater than a\"\n \" randomly selected value from a second sample.\\nWARNING: requires\"\n f\" **LARGE** (no less than {report.UTEST_OPTIMAL_REPETITIONS})\"\n \" number of repetitions to be meaningful!\\nThe test is being done\"\n f\" by default, if at least {report.UTEST_MIN_REPETITIONS}\"\n \" repetitions were done.\\nThis option can disable the U Test.\"\n ),\n )\n alpha_default = 0.05\n utest.add_argument(\n \"--alpha\",\n dest=\"utest_alpha\",\n default=alpha_default,\n type=float,\n help=(\n \"significance level alpha. if the calculated p-value is below this\"\n \" value, then the result is said to be statistically significant\"\n \" and the null hypothesis is rejected.\\n(default: %0.4f)\"\n )\n % alpha_default,\n )\n\n subparsers = parser.add_subparsers(\n help=\"This tool has multiple modes of operation:\", dest=\"mode\"\n )\n\n parser_a = subparsers.add_parser(\n \"benchmarks\",\n help=(\n \"The most simple use-case, compare all the output of these two\"\n \" benchmarks\"\n ),\n )\n baseline = parser_a.add_argument_group(\"baseline\", \"The benchmark baseline\")\n baseline.add_argument(\n \"test_baseline\",\n metavar=\"test_baseline\",\n type=argparse.FileType(\"r\"),\n nargs=1,\n help=\"A benchmark executable or JSON output file\",\n )\n contender = parser_a.add_argument_group(\n \"contender\", \"The benchmark that will be compared against the baseline\"\n )\n contender.add_argument(\n \"test_contender\",\n metavar=\"test_contender\",\n type=argparse.FileType(\"r\"),\n nargs=1,\n help=\"A benchmark executable or JSON output file\",\n )\n parser_a.add_argument(\n \"benchmark_options\",\n metavar=\"benchmark_options\",\n nargs=argparse.REMAINDER,\n help=\"Arguments to pass when running benchmark executables\",\n )\n\n parser_b = subparsers.add_parser(\n \"filters\", help=\"Compare filter one with the filter two of benchmark\"\n )\n baseline = parser_b.add_argument_group(\"baseline\", \"The benchmark baseline\")\n baseline.add_argument(\n \"test\",\n metavar=\"test\",\n type=argparse.FileType(\"r\"),\n nargs=1,\n help=\"A benchmark executable or JSON output file\",\n )\n baseline.add_argument(\n \"filter_baseline\",\n metavar=\"filter_baseline\",\n type=str,\n nargs=1,\n help=\"The first filter, that will be used as baseline\",\n )\n contender = parser_b.add_argument_group(\n \"contender\", \"The benchmark that will be compared against the baseline\"\n )\n contender.add_argument(\n \"filter_contender\",\n metavar=\"filter_contender\",\n type=str,\n nargs=1,\n help=\"The second filter, that will be compared against the baseline\",\n )\n parser_b.add_argument(\n \"benchmark_options\",\n metavar=\"benchmark_options\",\n nargs=argparse.REMAINDER,\n help=\"Arguments to pass when running benchmark executables\",\n )\n\n parser_c = subparsers.add_parser(\n \"benchmarksfiltered\",\n help=(\n \"Compare filter one of first benchmark with filter two of the\"\n \" second benchmark\"\n ),\n )\n baseline = parser_c.add_argument_group(\"baseline\", \"The benchmark baseline\")\n baseline.add_argument(\n \"test_baseline\",\n metavar=\"test_baseline\",\n type=argparse.FileType(\"r\"),\n nargs=1,\n help=\"A benchmark executable or JSON output file\",\n )\n baseline.add_argument(\n \"filter_baseline\",\n metavar=\"filter_baseline\",\n type=str,\n nargs=1,\n help=\"The first filter, that will be used as baseline\",\n )\n contender = parser_c.add_argument_group(\n \"contender\", \"The benchmark that will be compared against the baseline\"\n )\n contender.add_argument(\n \"test_contender\",\n metavar=\"test_contender\",\n type=argparse.FileType(\"r\"),\n nargs=1,\n help=(\n \"The second benchmark executable or JSON output file, that will be\"\n \" compared against the baseline\"\n ),\n )\n contender.add_argument(\n \"filter_contender\",\n metavar=\"filter_contender\",\n type=str,\n nargs=1,\n help=\"The second filter, that will be compared against the baseline\",\n )\n parser_c.add_argument(\n \"benchmark_options\",\n metavar=\"benchmark_options\",\n nargs=argparse.REMAINDER,\n help=\"Arguments to pass when running benchmark executables\",\n )\n\n return parser\n\n\ndef main():\n # Parse the command line flags\n parser = create_parser()\n args, unknown_args = parser.parse_known_args()\n if args.mode is None:\n parser.print_help()\n exit(1)\n assert not unknown_args\n benchmark_options = args.benchmark_options\n\n if args.mode == \"benchmarks\":\n test_baseline = args.test_baseline[0].name\n test_contender = args.test_contender[0].name\n filter_baseline = \"\"\n filter_contender = \"\"\n\n # NOTE: if test_baseline == test_contender, you are analyzing the stdev\n\n description = \"Comparing %s to %s\" % (test_baseline, test_contender)\n elif args.mode == \"filters\":\n test_baseline = args.test[0].name\n test_contender = args.test[0].name\n filter_baseline = args.filter_baseline[0]\n filter_contender = args.filter_contender[0]\n\n # NOTE: if filter_baseline == filter_contender, you are analyzing the\n # stdev\n\n description = \"Comparing %s to %s (from %s)\" % (\n filter_baseline,\n filter_contender,\n args.test[0].name,\n )\n elif args.mode == \"benchmarksfiltered\":\n test_baseline = args.test_baseline[0].name\n test_contender = args.test_contender[0].name\n filter_baseline = args.filter_baseline[0]\n filter_contender = args.filter_contender[0]\n\n # NOTE: if test_baseline == test_contender and\n # filter_baseline == filter_contender, you are analyzing the stdev\n\n description = \"Comparing %s (from %s) to %s (from %s)\" % (\n filter_baseline,\n test_baseline,\n filter_contender,\n test_contender,\n )\n else:\n # should never happen\n print(\"Unrecognized mode of operation: '%s'\" % args.mode)\n parser.print_help()\n exit(1)\n\n check_inputs(test_baseline, test_contender, benchmark_options)\n\n if args.display_aggregates_only:\n benchmark_options += [\"--benchmark_display_aggregates_only=true\"]\n\n options_baseline = []\n options_contender = []\n\n if filter_baseline and filter_contender:\n options_baseline = [\"--benchmark_filter=%s\" % filter_baseline]\n options_contender = [\"--benchmark_filter=%s\" % filter_contender]\n\n # Run the benchmarks and report the results\n json1 = json1_orig = gbench.util.sort_benchmark_results(\n gbench.util.run_or_load_benchmark(\n test_baseline, benchmark_options + options_baseline\n )\n )\n json2 = json2_orig = gbench.util.sort_benchmark_results(\n gbench.util.run_or_load_benchmark(\n test_contender, benchmark_options + options_contender\n )\n )\n\n # Now, filter the benchmarks so that the difference report can work\n if filter_baseline and filter_contender:\n replacement = \"[%s vs. %s]\" % (filter_baseline, filter_contender)\n json1 = gbench.report.filter_benchmark(\n json1_orig, filter_baseline, replacement\n )\n json2 = gbench.report.filter_benchmark(\n json2_orig, filter_contender, replacement\n )\n\n diff_report = gbench.report.get_difference_report(json1, json2, args.utest)\n output_lines = gbench.report.print_difference_report(\n diff_report,\n args.display_aggregates_only,\n args.utest,\n args.utest_alpha,\n args.color,\n )\n print(description)\n for ln in output_lines:\n print(ln)\n\n # Optionally, diff and output to JSON\n if args.dump_to_json is not None:\n with open(args.dump_to_json, \"w\") as f_json:\n json.dump(diff_report, f_json, indent=1)\n\n\nclass TestParser(unittest.TestCase):\n def setUp(self):\n self.parser = create_parser()\n testInputs = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"gbench\", \"Inputs\"\n )\n self.testInput0 = os.path.join(testInputs, \"test1_run1.json\")\n self.testInput1 = os.path.join(testInputs, \"test1_run2.json\")\n\n def test_benchmarks_basic(self):\n parsed = self.parser.parse_args(\n [\"benchmarks\", self.testInput0, self.testInput1]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.mode, \"benchmarks\")\n self.assertEqual(parsed.test_baseline[0].name, self.testInput0)\n self.assertEqual(parsed.test_contender[0].name, self.testInput1)\n self.assertFalse(parsed.benchmark_options)\n\n def test_benchmarks_basic_without_utest(self):\n parsed = self.parser.parse_args(\n [\"--no-utest\", \"benchmarks\", self.testInput0, self.testInput1]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertFalse(parsed.utest)\n self.assertEqual(parsed.utest_alpha, 0.05)\n self.assertEqual(parsed.mode, \"benchmarks\")\n self.assertEqual(parsed.test_baseline[0].name, self.testInput0)\n self.assertEqual(parsed.test_contender[0].name, self.testInput1)\n self.assertFalse(parsed.benchmark_options)\n\n def test_benchmarks_basic_display_aggregates_only(self):\n parsed = self.parser.parse_args(\n [\"-a\", \"benchmarks\", self.testInput0, self.testInput1]\n )\n self.assertTrue(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.mode, \"benchmarks\")\n self.assertEqual(parsed.test_baseline[0].name, self.testInput0)\n self.assertEqual(parsed.test_contender[0].name, self.testInput1)\n self.assertFalse(parsed.benchmark_options)\n\n def test_benchmarks_basic_with_utest_alpha(self):\n parsed = self.parser.parse_args(\n [\"--alpha=0.314\", \"benchmarks\", self.testInput0, self.testInput1]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.utest_alpha, 0.314)\n self.assertEqual(parsed.mode, \"benchmarks\")\n self.assertEqual(parsed.test_baseline[0].name, self.testInput0)\n self.assertEqual(parsed.test_contender[0].name, self.testInput1)\n self.assertFalse(parsed.benchmark_options)\n\n def test_benchmarks_basic_without_utest_with_utest_alpha(self):\n parsed = self.parser.parse_args(\n [\n \"--no-utest\",\n \"--alpha=0.314\",\n \"benchmarks\",\n self.testInput0,\n self.testInput1,\n ]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertFalse(parsed.utest)\n self.assertEqual(parsed.utest_alpha, 0.314)\n self.assertEqual(parsed.mode, \"benchmarks\")\n self.assertEqual(parsed.test_baseline[0].name, self.testInput0)\n self.assertEqual(parsed.test_contender[0].name, self.testInput1)\n self.assertFalse(parsed.benchmark_options)\n\n def test_benchmarks_with_remainder(self):\n parsed = self.parser.parse_args(\n [\"benchmarks\", self.testInput0, self.testInput1, \"d\"]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.mode, \"benchmarks\")\n self.assertEqual(parsed.test_baseline[0].name, self.testInput0)\n self.assertEqual(parsed.test_contender[0].name, self.testInput1)\n self.assertEqual(parsed.benchmark_options, [\"d\"])\n\n def test_benchmarks_with_remainder_after_doubleminus(self):\n parsed = self.parser.parse_args(\n [\"benchmarks\", self.testInput0, self.testInput1, \"--\", \"e\"]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.mode, \"benchmarks\")\n self.assertEqual(parsed.test_baseline[0].name, self.testInput0)\n self.assertEqual(parsed.test_contender[0].name, self.testInput1)\n self.assertEqual(parsed.benchmark_options, [\"e\"])\n\n def test_filters_basic(self):\n parsed = self.parser.parse_args([\"filters\", self.testInput0, \"c\", \"d\"])\n self.assertFalse(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.mode, \"filters\")\n self.assertEqual(parsed.test[0].name, self.testInput0)\n self.assertEqual(parsed.filter_baseline[0], \"c\")\n self.assertEqual(parsed.filter_contender[0], \"d\")\n self.assertFalse(parsed.benchmark_options)\n\n def test_filters_with_remainder(self):\n parsed = self.parser.parse_args(\n [\"filters\", self.testInput0, \"c\", \"d\", \"e\"]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.mode, \"filters\")\n self.assertEqual(parsed.test[0].name, self.testInput0)\n self.assertEqual(parsed.filter_baseline[0], \"c\")\n self.assertEqual(parsed.filter_contender[0], \"d\")\n self.assertEqual(parsed.benchmark_options, [\"e\"])\n\n def test_filters_with_remainder_after_doubleminus(self):\n parsed = self.parser.parse_args(\n [\"filters\", self.testInput0, \"c\", \"d\", \"--\", \"f\"]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.mode, \"filters\")\n self.assertEqual(parsed.test[0].name, self.testInput0)\n self.assertEqual(parsed.filter_baseline[0], \"c\")\n self.assertEqual(parsed.filter_contender[0], \"d\")\n self.assertEqual(parsed.benchmark_options, [\"f\"])\n\n def test_benchmarksfiltered_basic(self):\n parsed = self.parser.parse_args(\n [\"benchmarksfiltered\", self.testInput0, \"c\", self.testInput1, \"e\"]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.mode, \"benchmarksfiltered\")\n self.assertEqual(parsed.test_baseline[0].name, self.testInput0)\n self.assertEqual(parsed.filter_baseline[0], \"c\")\n self.assertEqual(parsed.test_contender[0].name, self.testInput1)\n self.assertEqual(parsed.filter_contender[0], \"e\")\n self.assertFalse(parsed.benchmark_options)\n\n def test_benchmarksfiltered_with_remainder(self):\n parsed = self.parser.parse_args(\n [\n \"benchmarksfiltered\",\n self.testInput0,\n \"c\",\n self.testInput1,\n \"e\",\n \"f\",\n ]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.mode, \"benchmarksfiltered\")\n self.assertEqual(parsed.test_baseline[0].name, self.testInput0)\n self.assertEqual(parsed.filter_baseline[0], \"c\")\n self.assertEqual(parsed.test_contender[0].name, self.testInput1)\n self.assertEqual(parsed.filter_contender[0], \"e\")\n self.assertEqual(parsed.benchmark_options[0], \"f\")\n\n def test_benchmarksfiltered_with_remainder_after_doubleminus(self):\n parsed = self.parser.parse_args(\n [\n \"benchmarksfiltered\",\n self.testInput0,\n \"c\",\n self.testInput1,\n \"e\",\n \"--\",\n \"g\",\n ]\n )\n self.assertFalse(parsed.display_aggregates_only)\n self.assertTrue(parsed.utest)\n self.assertEqual(parsed.mode, \"benchmarksfiltered\")\n self.assertEqual(parsed.test_baseline[0].name, self.testInput0)\n self.assertEqual(parsed.filter_baseline[0], \"c\")\n self.assertEqual(parsed.test_contender[0].name, self.testInput1)\n self.assertEqual(parsed.filter_contender[0], \"e\")\n self.assertEqual(parsed.benchmark_options[0], \"g\")\n\n\nif __name__ == \"__main__\":\n # unittest.main()\n main()\n\n# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4\n# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;\n# kate: indent-mode python; remove-trailing-spaces modified;\n" }, { "path": "tools/gbench/Inputs/test1_run1.json", "content": "{\n \"context\": {\n \"date\": \"2016-08-02 17:44:46\",\n \"num_cpus\": 4,\n \"mhz_per_cpu\": 4228,\n \"cpu_scaling_enabled\": false,\n \"library_build_type\": \"release\"\n },\n \"benchmarks\": [\n {\n \"name\": \"BM_SameTimes\",\n \"iterations\": 1000,\n \"real_time\": 10,\n \"cpu_time\": 10,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_2xFaster\",\n \"iterations\": 1000,\n \"real_time\": 50,\n \"cpu_time\": 50,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_2xSlower\",\n \"iterations\": 1000,\n \"real_time\": 50,\n \"cpu_time\": 50,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_1PercentFaster\",\n \"iterations\": 1000,\n \"real_time\": 100,\n \"cpu_time\": 100,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_1PercentSlower\",\n \"iterations\": 1000,\n \"real_time\": 100,\n \"cpu_time\": 100,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_10PercentFaster\",\n \"iterations\": 1000,\n \"real_time\": 100,\n \"cpu_time\": 100,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_10PercentSlower\",\n \"iterations\": 1000,\n \"real_time\": 100,\n \"cpu_time\": 100,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_100xSlower\",\n \"iterations\": 1000,\n \"real_time\": 100,\n \"cpu_time\": 100,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_100xFaster\",\n \"iterations\": 1000,\n \"real_time\": 10000,\n \"cpu_time\": 10000,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_10PercentCPUToTime\",\n \"iterations\": 1000,\n \"real_time\": 100,\n \"cpu_time\": 100,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_ThirdFaster\",\n \"iterations\": 1000,\n \"real_time\": 100,\n \"cpu_time\": 100,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"MyComplexityTest_BigO\",\n \"run_name\": \"MyComplexityTest\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"BigO\",\n \"cpu_coefficient\": 4.2749856294592886e+00,\n \"real_coefficient\": 6.4789275289789780e+00,\n \"big_o\": \"N\",\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"MyComplexityTest_RMS\",\n \"run_name\": \"MyComplexityTest\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"RMS\",\n \"rms\": 4.5097802512472874e-03\n },\n {\n \"name\": \"BM_NotBadTimeUnit\",\n \"iterations\": 1000,\n \"real_time\": 0.4,\n \"cpu_time\": 0.5,\n \"time_unit\": \"s\"\n },\n {\n \"name\": \"BM_DifferentTimeUnit\",\n \"iterations\": 1,\n \"real_time\": 1,\n \"cpu_time\": 1,\n \"time_unit\": \"s\"\n },\n {\n \"name\": \"BM_hasLabel\",\n \"label\": \"a label\",\n \"iterations\": 1,\n \"real_time\": 1,\n \"cpu_time\": 1,\n \"time_unit\": \"s\"\n }\n ]\n}\n" }, { "path": "tools/gbench/Inputs/test1_run2.json", "content": "{\n \"context\": {\n \"date\": \"2016-08-02 17:44:46\",\n \"num_cpus\": 4,\n \"mhz_per_cpu\": 4228,\n \"cpu_scaling_enabled\": false,\n \"library_build_type\": \"release\"\n },\n \"benchmarks\": [\n {\n \"name\": \"BM_SameTimes\",\n \"iterations\": 1000,\n \"real_time\": 10,\n \"cpu_time\": 10,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_2xFaster\",\n \"iterations\": 1000,\n \"real_time\": 25,\n \"cpu_time\": 25,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_2xSlower\",\n \"iterations\": 20833333,\n \"real_time\": 100,\n \"cpu_time\": 100,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_1PercentFaster\",\n \"iterations\": 1000,\n \"real_time\": 98.9999999,\n \"cpu_time\": 98.9999999,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_1PercentSlower\",\n \"iterations\": 1000,\n \"real_time\": 100.9999999,\n \"cpu_time\": 100.9999999,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_10PercentFaster\",\n \"iterations\": 1000,\n \"real_time\": 90,\n \"cpu_time\": 90,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_10PercentSlower\",\n \"iterations\": 1000,\n \"real_time\": 110,\n \"cpu_time\": 110,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_100xSlower\",\n \"iterations\": 1000,\n \"real_time\": 1.0000e+04,\n \"cpu_time\": 1.0000e+04,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_100xFaster\",\n \"iterations\": 1000,\n \"real_time\": 100,\n \"cpu_time\": 100,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_10PercentCPUToTime\",\n \"iterations\": 1000,\n \"real_time\": 110,\n \"cpu_time\": 90,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_ThirdFaster\",\n \"iterations\": 1000,\n \"real_time\": 66.665,\n \"cpu_time\": 66.664,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"MyComplexityTest_BigO\",\n \"run_name\": \"MyComplexityTest\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"BigO\",\n \"cpu_coefficient\": 5.6215779594361486e+00,\n \"real_coefficient\": 5.6288314793554610e+00,\n \"big_o\": \"N\",\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"MyComplexityTest_RMS\",\n \"run_name\": \"MyComplexityTest\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"RMS\",\n \"rms\": 3.3128901852342174e-03\n },\n {\n \"name\": \"BM_NotBadTimeUnit\",\n \"iterations\": 1000,\n \"real_time\": 0.04,\n \"cpu_time\": 0.6,\n \"time_unit\": \"s\"\n },\n {\n \"name\": \"BM_DifferentTimeUnit\",\n \"iterations\": 1,\n \"real_time\": 1,\n \"cpu_time\": 1,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_hasLabel\",\n \"label\": \"a label\",\n \"iterations\": 1,\n \"real_time\": 1,\n \"cpu_time\": 1,\n \"time_unit\": \"s\"\n }\n ]\n}\n" }, { "path": "tools/gbench/Inputs/test2_run.json", "content": "{\n \"context\": {\n \"date\": \"2016-08-02 17:44:46\",\n \"num_cpus\": 4,\n \"mhz_per_cpu\": 4228,\n \"cpu_scaling_enabled\": false,\n \"library_build_type\": \"release\"\n },\n \"benchmarks\": [\n {\n \"name\": \"BM_Hi\",\n \"iterations\": 1234,\n \"real_time\": 42,\n \"cpu_time\": 24,\n \"time_unit\": \"ms\"\n },\n {\n \"name\": \"BM_Zero\",\n \"iterations\": 1000,\n \"real_time\": 10,\n \"cpu_time\": 10,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_Zero/4\",\n \"iterations\": 4000,\n \"real_time\": 40,\n \"cpu_time\": 40,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"Prefix/BM_Zero\",\n \"iterations\": 2000,\n \"real_time\": 20,\n \"cpu_time\": 20,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"Prefix/BM_Zero/3\",\n \"iterations\": 3000,\n \"real_time\": 30,\n \"cpu_time\": 30,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_One\",\n \"iterations\": 5000,\n \"real_time\": 5,\n \"cpu_time\": 5,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_One/4\",\n \"iterations\": 2000,\n \"real_time\": 20,\n \"cpu_time\": 20,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"Prefix/BM_One\",\n \"iterations\": 1000,\n \"real_time\": 10,\n \"cpu_time\": 10,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"Prefix/BM_One/3\",\n \"iterations\": 1500,\n \"real_time\": 15,\n \"cpu_time\": 15,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_Bye\",\n \"iterations\": 5321,\n \"real_time\": 11,\n \"cpu_time\": 63,\n \"time_unit\": \"ns\"\n }\n ]\n}\n" }, { "path": "tools/gbench/Inputs/test3_run0.json", "content": "{\n \"context\": {\n \"date\": \"2016-08-02 17:44:46\",\n \"num_cpus\": 4,\n \"mhz_per_cpu\": 4228,\n \"cpu_scaling_enabled\": false,\n \"library_build_type\": \"release\"\n },\n \"benchmarks\": [\n {\n \"name\": \"BM_One\",\n \"run_type\": \"aggregate\",\n \"iterations\": 1000,\n \"real_time\": 10,\n \"cpu_time\": 100,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_Two\",\n \"iterations\": 1000,\n \"real_time\": 9,\n \"cpu_time\": 90,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_Two\",\n \"iterations\": 1000,\n \"real_time\": 8,\n \"cpu_time\": 86,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"short\",\n \"run_type\": \"aggregate\",\n \"iterations\": 1000,\n \"real_time\": 8,\n \"cpu_time\": 80,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"short\",\n \"run_type\": \"aggregate\",\n \"iterations\": 1000,\n \"real_time\": 8,\n \"cpu_time\": 77,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"medium\",\n \"run_type\": \"iteration\",\n \"iterations\": 1000,\n \"real_time\": 8,\n \"cpu_time\": 80,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"medium\",\n \"run_type\": \"iteration\",\n \"iterations\": 1000,\n \"real_time\": 9,\n \"cpu_time\": 82,\n \"time_unit\": \"ns\"\n }\n ]\n}\n" }, { "path": "tools/gbench/Inputs/test3_run1.json", "content": "{\n \"context\": {\n \"date\": \"2016-08-02 17:44:46\",\n \"num_cpus\": 4,\n \"mhz_per_cpu\": 4228,\n \"cpu_scaling_enabled\": false,\n \"library_build_type\": \"release\"\n },\n \"benchmarks\": [\n {\n \"name\": \"BM_One\",\n \"iterations\": 1000,\n \"real_time\": 9,\n \"cpu_time\": 110,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_Two\",\n \"run_type\": \"aggregate\",\n \"iterations\": 1000,\n \"real_time\": 10,\n \"cpu_time\": 89,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"BM_Two\",\n \"iterations\": 1000,\n \"real_time\": 7,\n \"cpu_time\": 72,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"short\",\n \"run_type\": \"aggregate\",\n \"iterations\": 1000,\n \"real_time\": 7,\n \"cpu_time\": 75,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"short\",\n \"run_type\": \"aggregate\",\n \"iterations\": 762,\n \"real_time\": 4.54,\n \"cpu_time\": 66.6,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"short\",\n \"run_type\": \"iteration\",\n \"iterations\": 1000,\n \"real_time\": 800,\n \"cpu_time\": 1,\n \"time_unit\": \"ns\"\n },\n {\n \"name\": \"medium\",\n \"run_type\": \"iteration\",\n \"iterations\": 1200,\n \"real_time\": 5,\n \"cpu_time\": 53,\n \"time_unit\": \"ns\"\n }\n ]\n}\n" }, { "path": "tools/gbench/Inputs/test4_run.json", "content": "{\n \"benchmarks\": [\n {\n \"name\": \"99 family 0 instance 0 repetition 0\",\n \"run_type\": \"iteration\",\n \"family_index\": 0,\n \"per_family_instance_index\": 0,\n \"repetition_index\": 0\n },\n {\n \"name\": \"98 family 0 instance 0 repetition 1\",\n \"run_type\": \"iteration\",\n \"family_index\": 0,\n \"per_family_instance_index\": 0,\n \"repetition_index\": 1\n },\n {\n \"name\": \"97 family 0 instance 0 aggregate\",\n \"run_type\": \"aggregate\",\n \"family_index\": 0,\n \"per_family_instance_index\": 0,\n \"aggregate_name\": \"9 aggregate\"\n },\n\n\n {\n \"name\": \"96 family 0 instance 1 repetition 0\",\n \"run_type\": \"iteration\",\n \"family_index\": 0,\n \"per_family_instance_index\": 1,\n \"repetition_index\": 0\n },\n {\n \"name\": \"95 family 0 instance 1 repetition 1\",\n \"run_type\": \"iteration\",\n \"family_index\": 0,\n \"per_family_instance_index\": 1,\n \"repetition_index\": 1\n },\n {\n \"name\": \"94 family 0 instance 1 aggregate\",\n \"run_type\": \"aggregate\",\n \"family_index\": 0,\n \"per_family_instance_index\": 1,\n \"aggregate_name\": \"9 aggregate\"\n },\n\n\n\n\n {\n \"name\": \"93 family 1 instance 0 repetition 0\",\n \"run_type\": \"iteration\",\n \"family_index\": 1,\n \"per_family_instance_index\": 0,\n \"repetition_index\": 0\n },\n {\n \"name\": \"92 family 1 instance 0 repetition 1\",\n \"run_type\": \"iteration\",\n \"family_index\": 1,\n \"per_family_instance_index\": 0,\n \"repetition_index\": 1\n },\n {\n \"name\": \"91 family 1 instance 0 aggregate\",\n \"run_type\": \"aggregate\",\n \"family_index\": 1,\n \"per_family_instance_index\": 0,\n \"aggregate_name\": \"9 aggregate\"\n },\n\n\n {\n \"name\": \"90 family 1 instance 1 repetition 0\",\n \"run_type\": \"iteration\",\n \"family_index\": 1,\n \"per_family_instance_index\": 1,\n \"repetition_index\": 0\n },\n {\n \"name\": \"89 family 1 instance 1 repetition 1\",\n \"run_type\": \"iteration\",\n \"family_index\": 1,\n \"per_family_instance_index\": 1,\n \"repetition_index\": 1\n },\n {\n \"name\": \"88 family 1 instance 1 aggregate\",\n \"run_type\": \"aggregate\",\n \"family_index\": 1,\n \"per_family_instance_index\": 1,\n \"aggregate_name\": \"9 aggregate\"\n }\n ]\n}\n" }, { "path": "tools/gbench/Inputs/test4_run0.json", "content": "{\n \"context\": {\n \"date\": \"2016-08-02 17:44:46\",\n \"num_cpus\": 4,\n \"mhz_per_cpu\": 4228,\n \"cpu_scaling_enabled\": false,\n \"library_build_type\": \"release\"\n },\n \"benchmarks\": [\n {\n \"name\": \"whocares\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"zz\",\n \"aggregate_unit\": \"percentage\",\n \"iterations\": 1000,\n \"real_time\": 0.01,\n \"cpu_time\": 0.10,\n \"time_unit\": \"ns\"\n }\n ]\n}\n" }, { "path": "tools/gbench/Inputs/test4_run1.json", "content": "{\n \"context\": {\n \"date\": \"2016-08-02 17:44:46\",\n \"num_cpus\": 4,\n \"mhz_per_cpu\": 4228,\n \"cpu_scaling_enabled\": false,\n \"library_build_type\": \"release\"\n },\n \"benchmarks\": [\n {\n \"name\": \"whocares\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"zz\",\n \"aggregate_unit\": \"percentage\",\n \"iterations\": 1000,\n \"real_time\": 0.005,\n \"cpu_time\": 0.15,\n \"time_unit\": \"ns\"\n }\n ]\n}\n" }, { "path": "tools/gbench/Inputs/test5_run0.json", "content": "{\n \"context\": {\n \"date\": \"2016-08-02 17:44:46\",\n \"num_cpus\": 4,\n \"mhz_per_cpu\": 4228,\n \"cpu_scaling_enabled\": false,\n \"library_build_type\": \"release\"\n },\n \"benchmarks\": [\n {\n \"name\": \"BM_ManyRepetitions\",\n \"iterations\": 1000,\n \"real_time\": 1,\n \"cpu_time\": 1000,\n \"time_unit\": \"s\"\n }\n ]\n}\n" }, { "path": "tools/gbench/Inputs/test5_run1.json", "content": "{\n \"context\": {\n \"date\": \"2016-08-02 17:44:46\",\n \"num_cpus\": 4,\n \"mhz_per_cpu\": 4228,\n \"cpu_scaling_enabled\": false,\n \"library_build_type\": \"release\"\n },\n \"benchmarks\": [\n {\n \"name\": \"BM_ManyRepetitions\",\n \"iterations\": 1000,\n \"real_time\": 1000,\n \"cpu_time\": 1,\n \"time_unit\": \"s\"\n }\n ]\n}\n" }, { "path": "tools/gbench/__init__.py", "content": "\"\"\"Google Benchmark tooling\"\"\"\n\n__author__ = \"Eric Fiselier\"\n__email__ = \"eric@efcs.ca\"\n__versioninfo__ = (0, 5, 0)\n__version__ = \".\".join(str(v) for v in __versioninfo__) + \"dev\"\n\n__all__ = [] # type: ignore\n" }, { "path": "tools/gbench/report.py", "content": "# type: ignore\n\n\"\"\"\nreport.py - Utilities for reporting statistics about benchmark results\n\"\"\"\n\nimport copy\nimport os\nimport random\nimport re\nimport unittest\n\nfrom numpy import array\nfrom scipy.stats import gmean, mannwhitneyu\n\n\nclass BenchmarkColor:\n def __init__(self, name, code):\n self.name = name\n self.code = code\n\n def __repr__(self):\n return \"%s%r\" % (self.__class__.__name__, (self.name, self.code))\n\n def __format__(self, format):\n return self.code\n\n\n# Benchmark Colors Enumeration\nBC_NONE = BenchmarkColor(\"NONE\", \"\")\nBC_MAGENTA = BenchmarkColor(\"MAGENTA\", \"\\033[95m\")\nBC_CYAN = BenchmarkColor(\"CYAN\", \"\\033[96m\")\nBC_OKBLUE = BenchmarkColor(\"OKBLUE\", \"\\033[94m\")\nBC_OKGREEN = BenchmarkColor(\"OKGREEN\", \"\\033[32m\")\nBC_HEADER = BenchmarkColor(\"HEADER\", \"\\033[92m\")\nBC_WARNING = BenchmarkColor(\"WARNING\", \"\\033[93m\")\nBC_WHITE = BenchmarkColor(\"WHITE\", \"\\033[97m\")\nBC_FAIL = BenchmarkColor(\"FAIL\", \"\\033[91m\")\nBC_ENDC = BenchmarkColor(\"ENDC\", \"\\033[0m\")\nBC_BOLD = BenchmarkColor(\"BOLD\", \"\\033[1m\")\nBC_UNDERLINE = BenchmarkColor(\"UNDERLINE\", \"\\033[4m\")\n\nUTEST_MIN_REPETITIONS = 2\nUTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better.\nUTEST_COL_NAME = \"_pvalue\"\n\n_TIME_UNIT_TO_SECONDS_MULTIPLIER = {\n \"s\": 1.0,\n \"ms\": 1e-3,\n \"us\": 1e-6,\n \"ns\": 1e-9,\n}\n\n\ndef color_format(use_color, fmt_str, *args, **kwargs):\n \"\"\"\n Return the result of 'fmt_str.format(*args, **kwargs)' after transforming\n 'args' and 'kwargs' according to the value of 'use_color'. If 'use_color'\n is False then all color codes in 'args' and 'kwargs' are replaced with\n the empty string.\n \"\"\"\n assert use_color is True or use_color is False\n if not use_color:\n args = [\n arg if not isinstance(arg, BenchmarkColor) else BC_NONE\n for arg in args\n ]\n kwargs = {\n key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE\n for key, arg in kwargs.items()\n }\n return fmt_str.format(*args, **kwargs)\n\n\ndef find_longest_name(benchmark_list):\n \"\"\"\n Return the length of the longest benchmark name in a given list of\n benchmark JSON objects\n \"\"\"\n longest_name = 1\n for bc in benchmark_list:\n if len(bc[\"name\"]) > longest_name:\n longest_name = len(bc[\"name\"])\n return longest_name\n\n\ndef calculate_change(old_val, new_val):\n \"\"\"\n Return a float representing the decimal change between old_val and new_val.\n \"\"\"\n if old_val == 0 and new_val == 0:\n return 0.0\n if old_val == 0:\n return float(new_val - old_val) / (float(old_val + new_val) / 2)\n return float(new_val - old_val) / abs(old_val)\n\n\ndef filter_benchmark(json_orig, family, replacement=\"\"):\n \"\"\"\n Apply a filter to the json, and only leave the 'family' of benchmarks.\n \"\"\"\n regex = re.compile(family)\n filtered = {}\n filtered[\"benchmarks\"] = []\n for be in json_orig[\"benchmarks\"]:\n if not regex.search(be[\"name\"]):\n continue\n filteredbench = copy.deepcopy(be) # Do NOT modify the old name!\n filteredbench[\"name\"] = regex.sub(replacement, filteredbench[\"name\"])\n filtered[\"benchmarks\"].append(filteredbench)\n return filtered\n\n\ndef get_unique_benchmark_names(json):\n \"\"\"\n While *keeping* the order, give all the unique 'names' used for benchmarks.\n \"\"\"\n seen = set()\n uniqued = [\n x[\"name\"]\n for x in json[\"benchmarks\"]\n if x[\"name\"] not in seen and (seen.add(x[\"name\"]) or True)\n ]\n return uniqued\n\n\ndef intersect(list1, list2):\n \"\"\"\n Given two lists, get a new list consisting of the elements only contained\n in *both of the input lists*, while preserving the ordering.\n \"\"\"\n return [x for x in list1 if x in list2]\n\n\ndef is_potentially_comparable_benchmark(x):\n return \"time_unit\" in x and \"real_time\" in x and \"cpu_time\" in x\n\n\ndef partition_benchmarks(json1, json2):\n \"\"\"\n While preserving the ordering, find benchmarks with the same names in\n both of the inputs, and group them.\n (i.e. partition/filter into groups with common name)\n \"\"\"\n json1_unique_names = get_unique_benchmark_names(json1)\n json2_unique_names = get_unique_benchmark_names(json2)\n names = intersect(json1_unique_names, json2_unique_names)\n partitions = []\n for name in names:\n time_unit = None\n # Pick the time unit from the first entry of the lhs benchmark.\n # We should be careful not to crash with unexpected input.\n for x in json1[\"benchmarks\"]:\n if x[\"name\"] == name and is_potentially_comparable_benchmark(x):\n time_unit = x[\"time_unit\"]\n break\n if time_unit is None:\n continue\n # Filter by name and time unit.\n # All the repetitions are assumed to be comparable.\n lhs = [\n x\n for x in json1[\"benchmarks\"]\n if x[\"name\"] == name and x[\"time_unit\"] == time_unit\n ]\n rhs = [\n x\n for x in json2[\"benchmarks\"]\n if x[\"name\"] == name and x[\"time_unit\"] == time_unit\n ]\n partitions.append([lhs, rhs])\n return partitions\n\n\ndef get_timedelta_field_as_seconds(benchmark, field_name):\n \"\"\"\n Get value of field_name field of benchmark, which is time with time unit\n time_unit, as time in seconds.\n \"\"\"\n timedelta = benchmark[field_name]\n time_unit = benchmark.get(\"time_unit\", \"s\")\n return timedelta * _TIME_UNIT_TO_SECONDS_MULTIPLIER.get(time_unit)\n\n\ndef calculate_geomean(json):\n \"\"\"\n Extract all real/cpu times from all the benchmarks as seconds,\n and calculate their geomean.\n \"\"\"\n times = []\n for benchmark in json[\"benchmarks\"]:\n if \"run_type\" in benchmark and benchmark[\"run_type\"] == \"aggregate\":\n continue\n times.append(\n [\n get_timedelta_field_as_seconds(benchmark, \"real_time\"),\n get_timedelta_field_as_seconds(benchmark, \"cpu_time\"),\n ]\n )\n return gmean(times) if times else array([])\n\n\ndef extract_field(partition, field_name):\n # The count of elements may be different. We want *all* of them.\n lhs = [x[field_name] for x in partition[0]]\n rhs = [x[field_name] for x in partition[1]]\n return [lhs, rhs]\n\n\ndef calc_utest(timings_cpu, timings_time):\n min_rep_cnt = min(\n len(timings_time[0]),\n len(timings_time[1]),\n len(timings_cpu[0]),\n len(timings_cpu[1]),\n )\n\n # Does *everything* has at least UTEST_MIN_REPETITIONS repetitions?\n if min_rep_cnt < UTEST_MIN_REPETITIONS:\n return False, None, None\n\n time_pvalue = mannwhitneyu(\n timings_time[0], timings_time[1], alternative=\"two-sided\"\n ).pvalue\n cpu_pvalue = mannwhitneyu(\n timings_cpu[0], timings_cpu[1], alternative=\"two-sided\"\n ).pvalue\n\n return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue\n\n\ndef print_utest(bc_name, utest, utest_alpha, first_col_width, use_color=True):\n def get_utest_color(pval):\n return BC_FAIL if pval >= utest_alpha else BC_OKGREEN\n\n # Check if we failed miserably with minimum required repetitions for utest\n if (\n not utest[\"have_optimal_repetitions\"]\n and utest[\"cpu_pvalue\"] is None\n and utest[\"time_pvalue\"] is None\n ):\n return []\n\n dsc = \"U Test, Repetitions: {} vs {}\".format(\n utest[\"nr_of_repetitions\"], utest[\"nr_of_repetitions_other\"]\n )\n dsc_color = BC_OKGREEN\n\n # We still got some results to show but issue a warning about it.\n if not utest[\"have_optimal_repetitions\"]:\n dsc_color = BC_WARNING\n dsc += (\n f\". WARNING: Results unreliable! {UTEST_OPTIMAL_REPETITIONS}+\"\n \" repetitions recommended.\"\n )\n\n special_str = \"{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}{endc}{} {}\"\n\n return [\n color_format(\n use_color,\n special_str,\n BC_HEADER,\n f\"{bc_name}{UTEST_COL_NAME}\",\n first_col_width,\n get_utest_color(utest[\"time_pvalue\"]),\n utest[\"time_pvalue\"],\n get_utest_color(utest[\"cpu_pvalue\"]),\n utest[\"cpu_pvalue\"],\n dsc_color,\n dsc,\n endc=BC_ENDC,\n )\n ]\n\n\ndef get_difference_report(json1, json2, utest=False):\n \"\"\"\n Calculate and report the difference between each test of two benchmarks\n runs specified as 'json1' and 'json2'. Output is another json containing\n relevant details for each test run.\n \"\"\"\n assert utest is True or utest is False\n\n diff_report = []\n partitions = partition_benchmarks(json1, json2)\n for partition in partitions:\n benchmark_name = partition[0][0][\"name\"]\n label = partition[0][0].get(\"label\", \"\")\n time_unit = partition[0][0][\"time_unit\"]\n measurements = []\n utest_results = {}\n # Careful, we may have different repetition count.\n for i in range(min(len(partition[0]), len(partition[1]))):\n bn = partition[0][i]\n other_bench = partition[1][i]\n measurements.append(\n {\n \"real_time\": bn[\"real_time\"],\n \"cpu_time\": bn[\"cpu_time\"],\n \"real_time_other\": other_bench[\"real_time\"],\n \"cpu_time_other\": other_bench[\"cpu_time\"],\n \"time\": calculate_change(\n bn[\"real_time\"], other_bench[\"real_time\"]\n ),\n \"cpu\": calculate_change(\n bn[\"cpu_time\"], other_bench[\"cpu_time\"]\n ),\n }\n )\n\n # After processing the whole partition, if requested, do the U test.\n if utest:\n timings_cpu = extract_field(partition, \"cpu_time\")\n timings_time = extract_field(partition, \"real_time\")\n have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(\n timings_cpu, timings_time\n )\n if cpu_pvalue is not None and time_pvalue is not None:\n utest_results = {\n \"have_optimal_repetitions\": have_optimal_repetitions,\n \"cpu_pvalue\": cpu_pvalue,\n \"time_pvalue\": time_pvalue,\n \"nr_of_repetitions\": len(timings_cpu[0]),\n \"nr_of_repetitions_other\": len(timings_cpu[1]),\n }\n\n # Store only if we had any measurements for given benchmark.\n # E.g. partition_benchmarks will filter out the benchmarks having\n # time units which are not compatible with other time units in the\n # benchmark suite.\n if measurements:\n run_type = partition[0][0].get(\"run_type\", \"\")\n aggregate_name = (\n partition[0][0][\"aggregate_name\"]\n if run_type == \"aggregate\"\n and \"aggregate_name\" in partition[0][0]\n else \"\"\n )\n diff_report.append(\n {\n \"name\": benchmark_name,\n \"label\": label,\n \"measurements\": measurements,\n \"time_unit\": time_unit,\n \"run_type\": run_type,\n \"aggregate_name\": aggregate_name,\n \"utest\": utest_results,\n }\n )\n\n lhs_gmean = calculate_geomean(json1)\n rhs_gmean = calculate_geomean(json2)\n if lhs_gmean.any() and rhs_gmean.any():\n diff_report.append(\n {\n \"name\": \"OVERALL_GEOMEAN\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"real_time\": lhs_gmean[0],\n \"cpu_time\": lhs_gmean[1],\n \"real_time_other\": rhs_gmean[0],\n \"cpu_time_other\": rhs_gmean[1],\n \"time\": calculate_change(lhs_gmean[0], rhs_gmean[0]),\n \"cpu\": calculate_change(lhs_gmean[1], rhs_gmean[1]),\n }\n ],\n \"time_unit\": \"s\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"geomean\",\n \"utest\": {},\n }\n )\n\n return diff_report\n\n\ndef print_difference_report(\n json_diff_report,\n include_aggregates_only=False,\n utest=False,\n utest_alpha=0.05,\n use_color=True,\n):\n \"\"\"\n Calculate and report the difference between each test of two benchmarks\n runs specified as 'json1' and 'json2'.\n \"\"\"\n assert utest is True or utest is False\n\n def get_color(res):\n if res > 0.05:\n return BC_FAIL\n elif res > -0.07:\n return BC_WHITE\n else:\n return BC_CYAN\n\n first_col_width = find_longest_name(json_diff_report)\n first_col_width = max(first_col_width, len(\"Benchmark\"))\n first_col_width += len(UTEST_COL_NAME)\n fmt_str = (\n \"{:<{}s}Time CPU Time Old Time New CPU Old\"\n \" CPU New\"\n )\n first_line = fmt_str.format(\"Benchmark\", 12 + first_col_width)\n output_strs = [first_line, \"-\" * len(first_line)]\n\n fmt_str = (\n \"{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}\"\n \"{endc}{:14.0f}{:14.0f}\"\n )\n for benchmark in json_diff_report:\n # *If* we were asked to only include aggregates,\n # and if it is non-aggregate, then don't print it.\n if (\n not include_aggregates_only\n or \"run_type\" not in benchmark\n or benchmark[\"run_type\"] == \"aggregate\"\n ):\n for measurement in benchmark[\"measurements\"]:\n output_strs += [\n color_format(\n use_color,\n fmt_str,\n BC_HEADER,\n benchmark[\"name\"],\n first_col_width,\n get_color(measurement[\"time\"]),\n measurement[\"time\"],\n get_color(measurement[\"cpu\"]),\n measurement[\"cpu\"],\n measurement[\"real_time\"],\n measurement[\"real_time_other\"],\n measurement[\"cpu_time\"],\n measurement[\"cpu_time_other\"],\n endc=BC_ENDC,\n )\n ]\n\n # After processing the measurements, if requested and\n # if applicable (e.g. u-test exists for given benchmark),\n # print the U test.\n if utest and benchmark[\"utest\"]:\n output_strs += print_utest(\n benchmark[\"name\"],\n benchmark[\"utest\"],\n utest_alpha=utest_alpha,\n first_col_width=first_col_width,\n use_color=use_color,\n )\n\n return output_strs\n\n\n###############################################################################\n# Unit tests\n\n\nclass TestGetUniqueBenchmarkNames(unittest.TestCase):\n def load_results(self):\n import json\n\n testInputs = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"Inputs\"\n )\n testOutput = os.path.join(testInputs, \"test3_run0.json\")\n with open(testOutput) as f:\n json = json.load(f)\n return json\n\n def test_basic(self):\n expect_lines = [\n \"BM_One\",\n \"BM_Two\",\n \"short\", # These two are not sorted\n \"medium\", # These two are not sorted\n ]\n json = self.load_results()\n output_lines = get_unique_benchmark_names(json)\n print(\"\\n\")\n print(\"\\n\".join(output_lines))\n self.assertEqual(len(output_lines), len(expect_lines))\n for i, output_line in enumerate(output_lines):\n self.assertEqual(expect_lines[i], output_line)\n\n\nclass TestReportDifference(unittest.TestCase):\n @classmethod\n def setUpClass(cls):\n def load_results():\n import json\n\n testInputs = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"Inputs\"\n )\n testOutput1 = os.path.join(testInputs, \"test1_run1.json\")\n testOutput2 = os.path.join(testInputs, \"test1_run2.json\")\n with open(testOutput1) as f:\n json1 = json.load(f)\n with open(testOutput2) as f:\n json2 = json.load(f)\n return json1, json2\n\n json1, json2 = load_results()\n cls.json_diff_report = get_difference_report(json1, json2)\n\n def test_json_diff_report_pretty_printing(self):\n expect_lines = [\n [\"BM_SameTimes\", \"+0.0000\", \"+0.0000\", \"10\", \"10\", \"10\", \"10\"],\n [\"BM_2xFaster\", \"-0.5000\", \"-0.5000\", \"50\", \"25\", \"50\", \"25\"],\n [\"BM_2xSlower\", \"+1.0000\", \"+1.0000\", \"50\", \"100\", \"50\", \"100\"],\n [\n \"BM_1PercentFaster\",\n \"-0.0100\",\n \"-0.0100\",\n \"100\",\n \"99\",\n \"100\",\n \"99\",\n ],\n [\n \"BM_1PercentSlower\",\n \"+0.0100\",\n \"+0.0100\",\n \"100\",\n \"101\",\n \"100\",\n \"101\",\n ],\n [\n \"BM_10PercentFaster\",\n \"-0.1000\",\n \"-0.1000\",\n \"100\",\n \"90\",\n \"100\",\n \"90\",\n ],\n [\n \"BM_10PercentSlower\",\n \"+0.1000\",\n \"+0.1000\",\n \"100\",\n \"110\",\n \"100\",\n \"110\",\n ],\n [\n \"BM_100xSlower\",\n \"+99.0000\",\n \"+99.0000\",\n \"100\",\n \"10000\",\n \"100\",\n \"10000\",\n ],\n [\n \"BM_100xFaster\",\n \"-0.9900\",\n \"-0.9900\",\n \"10000\",\n \"100\",\n \"10000\",\n \"100\",\n ],\n [\n \"BM_10PercentCPUToTime\",\n \"+0.1000\",\n \"-0.1000\",\n \"100\",\n \"110\",\n \"100\",\n \"90\",\n ],\n [\"BM_ThirdFaster\", \"-0.3333\", \"-0.3334\", \"100\", \"67\", \"100\", \"67\"],\n [\"BM_NotBadTimeUnit\", \"-0.9000\", \"+0.2000\", \"0\", \"0\", \"0\", \"1\"],\n [\"BM_hasLabel\", \"+0.0000\", \"+0.0000\", \"1\", \"1\", \"1\", \"1\"],\n [\"OVERALL_GEOMEAN\", \"-0.8113\", \"-0.7779\", \"0\", \"0\", \"0\", \"0\"],\n ]\n output_lines_with_header = print_difference_report(\n self.json_diff_report, use_color=False\n )\n output_lines = output_lines_with_header[2:]\n print(\"\\n\")\n print(\"\\n\".join(output_lines_with_header))\n self.assertEqual(len(output_lines), len(expect_lines))\n for i, output_line in enumerate(output_lines):\n parts = [x for x in output_line.split(\" \") if x]\n self.assertEqual(len(parts), 7)\n self.assertEqual(expect_lines[i], parts)\n\n def test_json_diff_report_output(self):\n expected_output = [\n {\n \"name\": \"BM_SameTimes\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": 0.0000,\n \"cpu\": 0.0000,\n \"real_time\": 10,\n \"real_time_other\": 10,\n \"cpu_time\": 10,\n \"cpu_time_other\": 10,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_2xFaster\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": -0.5000,\n \"cpu\": -0.5000,\n \"real_time\": 50,\n \"real_time_other\": 25,\n \"cpu_time\": 50,\n \"cpu_time_other\": 25,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_2xSlower\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": 1.0000,\n \"cpu\": 1.0000,\n \"real_time\": 50,\n \"real_time_other\": 100,\n \"cpu_time\": 50,\n \"cpu_time_other\": 100,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_1PercentFaster\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": -0.0100,\n \"cpu\": -0.0100,\n \"real_time\": 100,\n \"real_time_other\": 98.9999999,\n \"cpu_time\": 100,\n \"cpu_time_other\": 98.9999999,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_1PercentSlower\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": 0.0100,\n \"cpu\": 0.0100,\n \"real_time\": 100,\n \"real_time_other\": 101,\n \"cpu_time\": 100,\n \"cpu_time_other\": 101,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_10PercentFaster\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": -0.1000,\n \"cpu\": -0.1000,\n \"real_time\": 100,\n \"real_time_other\": 90,\n \"cpu_time\": 100,\n \"cpu_time_other\": 90,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_10PercentSlower\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": 0.1000,\n \"cpu\": 0.1000,\n \"real_time\": 100,\n \"real_time_other\": 110,\n \"cpu_time\": 100,\n \"cpu_time_other\": 110,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_100xSlower\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": 99.0000,\n \"cpu\": 99.0000,\n \"real_time\": 100,\n \"real_time_other\": 10000,\n \"cpu_time\": 100,\n \"cpu_time_other\": 10000,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_100xFaster\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": -0.9900,\n \"cpu\": -0.9900,\n \"real_time\": 10000,\n \"real_time_other\": 100,\n \"cpu_time\": 10000,\n \"cpu_time_other\": 100,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_10PercentCPUToTime\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": 0.1000,\n \"cpu\": -0.1000,\n \"real_time\": 100,\n \"real_time_other\": 110,\n \"cpu_time\": 100,\n \"cpu_time_other\": 90,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_ThirdFaster\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": -0.3333,\n \"cpu\": -0.3334,\n \"real_time\": 100,\n \"real_time_other\": 67,\n \"cpu_time\": 100,\n \"cpu_time_other\": 67,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_NotBadTimeUnit\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"time\": -0.9000,\n \"cpu\": 0.2000,\n \"real_time\": 0.4,\n \"real_time_other\": 0.04,\n \"cpu_time\": 0.5,\n \"cpu_time_other\": 0.6,\n }\n ],\n \"time_unit\": \"s\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_hasLabel\",\n \"label\": \"a label\",\n \"measurements\": [\n {\n \"time\": 0.0000,\n \"cpu\": 0.0000,\n \"real_time\": 1,\n \"real_time_other\": 1,\n \"cpu_time\": 1,\n \"cpu_time_other\": 1,\n }\n ],\n \"time_unit\": \"s\",\n \"utest\": {},\n },\n {\n \"name\": \"OVERALL_GEOMEAN\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"real_time\": 3.1622776601683826e-06,\n \"cpu_time\": 3.2130844755623912e-06,\n \"real_time_other\": 1.9768988699420897e-07,\n \"cpu_time_other\": 2.397447755209533e-07,\n \"time\": -0.8112976497120911,\n \"cpu\": -0.7778551721181174,\n }\n ],\n \"time_unit\": \"s\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"geomean\",\n \"utest\": {},\n },\n ]\n self.assertEqual(len(self.json_diff_report), len(expected_output))\n for out, expected in zip(\n self.json_diff_report, expected_output, strict=True\n ):\n self.assertEqual(out[\"name\"], expected[\"name\"])\n self.assertEqual(out[\"label\"], expected[\"label\"])\n self.assertEqual(out[\"time_unit\"], expected[\"time_unit\"])\n assert_utest(self, out, expected)\n assert_measurements(self, out, expected)\n\n\nclass TestReportDifferenceBetweenFamilies(unittest.TestCase):\n @classmethod\n def setUpClass(cls):\n def load_result():\n import json\n\n testInputs = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"Inputs\"\n )\n testOutput = os.path.join(testInputs, \"test2_run.json\")\n with open(testOutput) as f:\n json = json.load(f)\n return json\n\n json = load_result()\n json1 = filter_benchmark(json, \"BM_Z.ro\", \".\")\n json2 = filter_benchmark(json, \"BM_O.e\", \".\")\n cls.json_diff_report = get_difference_report(json1, json2)\n\n def test_json_diff_report_pretty_printing(self):\n expect_lines = [\n [\".\", \"-0.5000\", \"-0.5000\", \"10\", \"5\", \"10\", \"5\"],\n [\"./4\", \"-0.5000\", \"-0.5000\", \"40\", \"20\", \"40\", \"20\"],\n [\"Prefix/.\", \"-0.5000\", \"-0.5000\", \"20\", \"10\", \"20\", \"10\"],\n [\"Prefix/./3\", \"-0.5000\", \"-0.5000\", \"30\", \"15\", \"30\", \"15\"],\n [\"OVERALL_GEOMEAN\", \"-0.5000\", \"-0.5000\", \"0\", \"0\", \"0\", \"0\"],\n ]\n output_lines_with_header = print_difference_report(\n self.json_diff_report, use_color=False\n )\n output_lines = output_lines_with_header[2:]\n print(\"\\n\")\n print(\"\\n\".join(output_lines_with_header))\n self.assertEqual(len(output_lines), len(expect_lines))\n for i, output_line in enumerate(output_lines):\n parts = [x for x in output_line.split(\" \") if x]\n self.assertEqual(len(parts), 7)\n self.assertEqual(expect_lines[i], parts)\n\n def test_json_diff_report(self):\n expected_output = [\n {\n \"name\": \".\",\n \"measurements\": [\n {\n \"time\": -0.5,\n \"cpu\": -0.5,\n \"real_time\": 10,\n \"real_time_other\": 5,\n \"cpu_time\": 10,\n \"cpu_time_other\": 5,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"./4\",\n \"measurements\": [\n {\n \"time\": -0.5,\n \"cpu\": -0.5,\n \"real_time\": 40,\n \"real_time_other\": 20,\n \"cpu_time\": 40,\n \"cpu_time_other\": 20,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"Prefix/.\",\n \"measurements\": [\n {\n \"time\": -0.5,\n \"cpu\": -0.5,\n \"real_time\": 20,\n \"real_time_other\": 10,\n \"cpu_time\": 20,\n \"cpu_time_other\": 10,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"Prefix/./3\",\n \"measurements\": [\n {\n \"time\": -0.5,\n \"cpu\": -0.5,\n \"real_time\": 30,\n \"real_time_other\": 15,\n \"cpu_time\": 30,\n \"cpu_time_other\": 15,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"OVERALL_GEOMEAN\",\n \"measurements\": [\n {\n \"real_time\": 2.213363839400641e-08,\n \"cpu_time\": 2.213363839400641e-08,\n \"real_time_other\": 1.1066819197003185e-08,\n \"cpu_time_other\": 1.1066819197003185e-08,\n \"time\": -0.5000000000000009,\n \"cpu\": -0.5000000000000009,\n }\n ],\n \"time_unit\": \"s\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"geomean\",\n \"utest\": {},\n },\n ]\n self.assertEqual(len(self.json_diff_report), len(expected_output))\n for out, expected in zip(\n self.json_diff_report, expected_output, strict=True\n ):\n self.assertEqual(out[\"name\"], expected[\"name\"])\n self.assertEqual(out[\"time_unit\"], expected[\"time_unit\"])\n assert_utest(self, out, expected)\n assert_measurements(self, out, expected)\n\n\nclass TestReportDifferenceWithUTest(unittest.TestCase):\n @classmethod\n def setUpClass(cls):\n def load_results():\n import json\n\n testInputs = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"Inputs\"\n )\n testOutput1 = os.path.join(testInputs, \"test3_run0.json\")\n testOutput2 = os.path.join(testInputs, \"test3_run1.json\")\n with open(testOutput1) as f:\n json1 = json.load(f)\n with open(testOutput2) as f:\n json2 = json.load(f)\n return json1, json2\n\n json1, json2 = load_results()\n cls.json_diff_report = get_difference_report(json1, json2, utest=True)\n\n def test_json_diff_report_pretty_printing(self):\n expect_lines = [\n [\"BM_One\", \"-0.1000\", \"+0.1000\", \"10\", \"9\", \"100\", \"110\"],\n [\"BM_Two\", \"+0.1111\", \"-0.0111\", \"9\", \"10\", \"90\", \"89\"],\n [\"BM_Two\", \"-0.1250\", \"-0.1628\", \"8\", \"7\", \"86\", \"72\"],\n [\n \"BM_Two_pvalue\",\n \"1.0000\",\n \"0.6667\",\n \"U\",\n \"Test,\",\n \"Repetitions:\",\n \"2\",\n \"vs\",\n \"2.\",\n \"WARNING:\",\n \"Results\",\n \"unreliable!\",\n \"9+\",\n \"repetitions\",\n \"recommended.\",\n ],\n [\"short\", \"-0.1250\", \"-0.0625\", \"8\", \"7\", \"80\", \"75\"],\n [\"short\", \"-0.4325\", \"-0.1351\", \"8\", \"5\", \"77\", \"67\"],\n [\n \"short_pvalue\",\n \"0.7671\",\n \"0.2000\",\n \"U\",\n \"Test,\",\n \"Repetitions:\",\n \"2\",\n \"vs\",\n \"3.\",\n \"WARNING:\",\n \"Results\",\n \"unreliable!\",\n \"9+\",\n \"repetitions\",\n \"recommended.\",\n ],\n [\"medium\", \"-0.3750\", \"-0.3375\", \"8\", \"5\", \"80\", \"53\"],\n [\"OVERALL_GEOMEAN\", \"+1.6405\", \"-0.6985\", \"0\", \"0\", \"0\", \"0\"],\n ]\n output_lines_with_header = print_difference_report(\n self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False\n )\n output_lines = output_lines_with_header[2:]\n print(\"\\n\")\n print(\"\\n\".join(output_lines_with_header))\n self.assertEqual(len(output_lines), len(expect_lines))\n for i, output_line in enumerate(output_lines):\n parts = [x for x in output_line.split(\" \") if x]\n self.assertEqual(expect_lines[i], parts)\n\n def test_json_diff_report_pretty_printing_aggregates_only(self):\n expect_lines = [\n [\"BM_One\", \"-0.1000\", \"+0.1000\", \"10\", \"9\", \"100\", \"110\"],\n [\n \"BM_Two_pvalue\",\n \"1.0000\",\n \"0.6667\",\n \"U\",\n \"Test,\",\n \"Repetitions:\",\n \"2\",\n \"vs\",\n \"2.\",\n \"WARNING:\",\n \"Results\",\n \"unreliable!\",\n \"9+\",\n \"repetitions\",\n \"recommended.\",\n ],\n [\"short\", \"-0.1250\", \"-0.0625\", \"8\", \"7\", \"80\", \"75\"],\n [\"short\", \"-0.4325\", \"-0.1351\", \"8\", \"5\", \"77\", \"67\"],\n [\n \"short_pvalue\",\n \"0.7671\",\n \"0.2000\",\n \"U\",\n \"Test,\",\n \"Repetitions:\",\n \"2\",\n \"vs\",\n \"3.\",\n \"WARNING:\",\n \"Results\",\n \"unreliable!\",\n \"9+\",\n \"repetitions\",\n \"recommended.\",\n ],\n [\"OVERALL_GEOMEAN\", \"+1.6405\", \"-0.6985\", \"0\", \"0\", \"0\", \"0\"],\n ]\n output_lines_with_header = print_difference_report(\n self.json_diff_report,\n include_aggregates_only=True,\n utest=True,\n utest_alpha=0.05,\n use_color=False,\n )\n output_lines = output_lines_with_header[2:]\n print(\"\\n\")\n print(\"\\n\".join(output_lines_with_header))\n self.assertEqual(len(output_lines), len(expect_lines))\n for i, output_line in enumerate(output_lines):\n parts = [x for x in output_line.split(\" \") if x]\n self.assertEqual(expect_lines[i], parts)\n\n def test_json_diff_report(self):\n expected_output = [\n {\n \"name\": \"BM_One\",\n \"measurements\": [\n {\n \"time\": -0.1,\n \"cpu\": 0.1,\n \"real_time\": 10,\n \"real_time_other\": 9,\n \"cpu_time\": 100,\n \"cpu_time_other\": 110,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_Two\",\n \"measurements\": [\n {\n \"time\": 0.1111111111111111,\n \"cpu\": -0.011111111111111112,\n \"real_time\": 9,\n \"real_time_other\": 10,\n \"cpu_time\": 90,\n \"cpu_time_other\": 89,\n },\n {\n \"time\": -0.125,\n \"cpu\": -0.16279069767441862,\n \"real_time\": 8,\n \"real_time_other\": 7,\n \"cpu_time\": 86,\n \"cpu_time_other\": 72,\n },\n ],\n \"time_unit\": \"ns\",\n \"utest\": {\n \"have_optimal_repetitions\": False,\n \"cpu_pvalue\": 0.6666666666666666,\n \"time_pvalue\": 1.0,\n },\n },\n {\n \"name\": \"short\",\n \"measurements\": [\n {\n \"time\": -0.125,\n \"cpu\": -0.0625,\n \"real_time\": 8,\n \"real_time_other\": 7,\n \"cpu_time\": 80,\n \"cpu_time_other\": 75,\n },\n {\n \"time\": -0.4325,\n \"cpu\": -0.13506493506493514,\n \"real_time\": 8,\n \"real_time_other\": 4.54,\n \"cpu_time\": 77,\n \"cpu_time_other\": 66.6,\n },\n ],\n \"time_unit\": \"ns\",\n \"utest\": {\n \"have_optimal_repetitions\": False,\n \"cpu_pvalue\": 0.2,\n \"time_pvalue\": 0.7670968684102772,\n },\n },\n {\n \"name\": \"medium\",\n \"measurements\": [\n {\n \"time\": -0.375,\n \"cpu\": -0.3375,\n \"real_time\": 8,\n \"real_time_other\": 5,\n \"cpu_time\": 80,\n \"cpu_time_other\": 53,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"OVERALL_GEOMEAN\",\n \"measurements\": [\n {\n \"real_time\": 8.48528137423858e-09,\n \"cpu_time\": 8.441336246629233e-08,\n \"real_time_other\": 2.2405267593145244e-08,\n \"cpu_time_other\": 2.5453661413660466e-08,\n \"time\": 1.6404861082353634,\n \"cpu\": -0.6984640740519662,\n }\n ],\n \"time_unit\": \"s\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"geomean\",\n \"utest\": {},\n },\n ]\n self.assertEqual(len(self.json_diff_report), len(expected_output))\n for out, expected in zip(\n self.json_diff_report, expected_output, strict=True\n ):\n self.assertEqual(out[\"name\"], expected[\"name\"])\n self.assertEqual(out[\"time_unit\"], expected[\"time_unit\"])\n assert_utest(self, out, expected)\n assert_measurements(self, out, expected)\n\n\nclass TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(\n unittest.TestCase\n):\n @classmethod\n def setUpClass(cls):\n def load_results():\n import json\n\n testInputs = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"Inputs\"\n )\n testOutput1 = os.path.join(testInputs, \"test3_run0.json\")\n testOutput2 = os.path.join(testInputs, \"test3_run1.json\")\n with open(testOutput1) as f:\n json1 = json.load(f)\n with open(testOutput2) as f:\n json2 = json.load(f)\n return json1, json2\n\n json1, json2 = load_results()\n cls.json_diff_report = get_difference_report(json1, json2, utest=True)\n\n def test_json_diff_report_pretty_printing(self):\n expect_lines = [\n [\"BM_One\", \"-0.1000\", \"+0.1000\", \"10\", \"9\", \"100\", \"110\"],\n [\"BM_Two\", \"+0.1111\", \"-0.0111\", \"9\", \"10\", \"90\", \"89\"],\n [\"BM_Two\", \"-0.1250\", \"-0.1628\", \"8\", \"7\", \"86\", \"72\"],\n [\n \"BM_Two_pvalue\",\n \"1.0000\",\n \"0.6667\",\n \"U\",\n \"Test,\",\n \"Repetitions:\",\n \"2\",\n \"vs\",\n \"2.\",\n \"WARNING:\",\n \"Results\",\n \"unreliable!\",\n \"9+\",\n \"repetitions\",\n \"recommended.\",\n ],\n [\"short\", \"-0.1250\", \"-0.0625\", \"8\", \"7\", \"80\", \"75\"],\n [\"short\", \"-0.4325\", \"-0.1351\", \"8\", \"5\", \"77\", \"67\"],\n [\n \"short_pvalue\",\n \"0.7671\",\n \"0.2000\",\n \"U\",\n \"Test,\",\n \"Repetitions:\",\n \"2\",\n \"vs\",\n \"3.\",\n \"WARNING:\",\n \"Results\",\n \"unreliable!\",\n \"9+\",\n \"repetitions\",\n \"recommended.\",\n ],\n [\"medium\", \"-0.3750\", \"-0.3375\", \"8\", \"5\", \"80\", \"53\"],\n [\"OVERALL_GEOMEAN\", \"+1.6405\", \"-0.6985\", \"0\", \"0\", \"0\", \"0\"],\n ]\n output_lines_with_header = print_difference_report(\n self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False\n )\n output_lines = output_lines_with_header[2:]\n print(\"\\n\")\n print(\"\\n\".join(output_lines_with_header))\n self.assertEqual(len(output_lines), len(expect_lines))\n for i, output_line in enumerate(output_lines):\n parts = [x for x in output_line.split(\" \") if x]\n self.assertEqual(expect_lines[i], parts)\n\n def test_json_diff_report(self):\n expected_output = [\n {\n \"name\": \"BM_One\",\n \"measurements\": [\n {\n \"time\": -0.1,\n \"cpu\": 0.1,\n \"real_time\": 10,\n \"real_time_other\": 9,\n \"cpu_time\": 100,\n \"cpu_time_other\": 110,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n },\n {\n \"name\": \"BM_Two\",\n \"measurements\": [\n {\n \"time\": 0.1111111111111111,\n \"cpu\": -0.011111111111111112,\n \"real_time\": 9,\n \"real_time_other\": 10,\n \"cpu_time\": 90,\n \"cpu_time_other\": 89,\n },\n {\n \"time\": -0.125,\n \"cpu\": -0.16279069767441862,\n \"real_time\": 8,\n \"real_time_other\": 7,\n \"cpu_time\": 86,\n \"cpu_time_other\": 72,\n },\n ],\n \"time_unit\": \"ns\",\n \"utest\": {\n \"have_optimal_repetitions\": False,\n \"cpu_pvalue\": 0.6666666666666666,\n \"time_pvalue\": 1.0,\n },\n },\n {\n \"name\": \"short\",\n \"measurements\": [\n {\n \"time\": -0.125,\n \"cpu\": -0.0625,\n \"real_time\": 8,\n \"real_time_other\": 7,\n \"cpu_time\": 80,\n \"cpu_time_other\": 75,\n },\n {\n \"time\": -0.4325,\n \"cpu\": -0.13506493506493514,\n \"real_time\": 8,\n \"real_time_other\": 4.54,\n \"cpu_time\": 77,\n \"cpu_time_other\": 66.6,\n },\n ],\n \"time_unit\": \"ns\",\n \"utest\": {\n \"have_optimal_repetitions\": False,\n \"cpu_pvalue\": 0.2,\n \"time_pvalue\": 0.7670968684102772,\n },\n },\n {\n \"name\": \"medium\",\n \"measurements\": [\n {\n \"real_time_other\": 5,\n \"cpu_time\": 80,\n \"time\": -0.375,\n \"real_time\": 8,\n \"cpu_time_other\": 53,\n \"cpu\": -0.3375,\n }\n ],\n \"utest\": {},\n \"time_unit\": \"ns\",\n \"aggregate_name\": \"\",\n },\n {\n \"name\": \"OVERALL_GEOMEAN\",\n \"measurements\": [\n {\n \"real_time\": 8.48528137423858e-09,\n \"cpu_time\": 8.441336246629233e-08,\n \"real_time_other\": 2.2405267593145244e-08,\n \"cpu_time_other\": 2.5453661413660466e-08,\n \"time\": 1.6404861082353634,\n \"cpu\": -0.6984640740519662,\n }\n ],\n \"time_unit\": \"s\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"geomean\",\n \"utest\": {},\n },\n ]\n self.assertEqual(len(self.json_diff_report), len(expected_output))\n for out, expected in zip(\n self.json_diff_report, expected_output, strict=True\n ):\n self.assertEqual(out[\"name\"], expected[\"name\"])\n self.assertEqual(out[\"time_unit\"], expected[\"time_unit\"])\n assert_utest(self, out, expected)\n assert_measurements(self, out, expected)\n\n\nclass TestReportDifferenceForPercentageAggregates(unittest.TestCase):\n @classmethod\n def setUpClass(cls):\n def load_results():\n import json\n\n testInputs = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"Inputs\"\n )\n testOutput1 = os.path.join(testInputs, \"test4_run0.json\")\n testOutput2 = os.path.join(testInputs, \"test4_run1.json\")\n with open(testOutput1) as f:\n json1 = json.load(f)\n with open(testOutput2) as f:\n json2 = json.load(f)\n return json1, json2\n\n json1, json2 = load_results()\n cls.json_diff_report = get_difference_report(json1, json2, utest=True)\n\n def test_json_diff_report_pretty_printing(self):\n expect_lines = [[\"whocares\", \"-0.5000\", \"+0.5000\", \"0\", \"0\", \"0\", \"0\"]]\n output_lines_with_header = print_difference_report(\n self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False\n )\n output_lines = output_lines_with_header[2:]\n print(\"\\n\")\n print(\"\\n\".join(output_lines_with_header))\n self.assertEqual(len(output_lines), len(expect_lines))\n for i, output_line in enumerate(output_lines):\n parts = [x for x in output_line.split(\" \") if x]\n self.assertEqual(expect_lines[i], parts)\n\n def test_json_diff_report(self):\n expected_output = [\n {\n \"name\": \"whocares\",\n \"measurements\": [\n {\n \"time\": -0.5,\n \"cpu\": 0.5,\n \"real_time\": 0.01,\n \"real_time_other\": 0.005,\n \"cpu_time\": 0.10,\n \"cpu_time_other\": 0.15,\n }\n ],\n \"time_unit\": \"ns\",\n \"utest\": {},\n }\n ]\n self.assertEqual(len(self.json_diff_report), len(expected_output))\n for out, expected in zip(\n self.json_diff_report, expected_output, strict=True\n ):\n self.assertEqual(out[\"name\"], expected[\"name\"])\n self.assertEqual(out[\"time_unit\"], expected[\"time_unit\"])\n assert_utest(self, out, expected)\n assert_measurements(self, out, expected)\n\n\nclass TestReportSorting(unittest.TestCase):\n @classmethod\n def setUpClass(cls):\n def load_result():\n import json\n\n testInputs = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"Inputs\"\n )\n testOutput = os.path.join(testInputs, \"test4_run.json\")\n with open(testOutput) as f:\n json = json.load(f)\n return json\n\n cls.json = load_result()\n\n def test_json_diff_report_pretty_printing(self):\n import util\n\n expected_names = [\n \"99 family 0 instance 0 repetition 0\",\n \"98 family 0 instance 0 repetition 1\",\n \"97 family 0 instance 0 aggregate\",\n \"96 family 0 instance 1 repetition 0\",\n \"95 family 0 instance 1 repetition 1\",\n \"94 family 0 instance 1 aggregate\",\n \"93 family 1 instance 0 repetition 0\",\n \"92 family 1 instance 0 repetition 1\",\n \"91 family 1 instance 0 aggregate\",\n \"90 family 1 instance 1 repetition 0\",\n \"89 family 1 instance 1 repetition 1\",\n \"88 family 1 instance 1 aggregate\",\n ]\n\n for _n in range(len(self.json[\"benchmarks\"]) ** 2):\n random.shuffle(self.json[\"benchmarks\"])\n sorted_benchmarks = util.sort_benchmark_results(self.json)[\n \"benchmarks\"\n ]\n self.assertEqual(len(expected_names), len(sorted_benchmarks))\n for out, expected in zip(\n sorted_benchmarks, expected_names, strict=True\n ):\n self.assertEqual(out[\"name\"], expected)\n\n\nclass TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly2(\n unittest.TestCase\n):\n @classmethod\n def setUpClass(cls):\n def load_results():\n import json\n\n testInputs = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"Inputs\"\n )\n testOutput1 = os.path.join(testInputs, \"test5_run0.json\")\n testOutput2 = os.path.join(testInputs, \"test5_run1.json\")\n with open(testOutput1) as f:\n json1 = json.load(f)\n json1[\"benchmarks\"] = [\n json1[\"benchmarks\"][0] for i in range(1000)\n ]\n with open(testOutput2) as f:\n json2 = json.load(f)\n json2[\"benchmarks\"] = [\n json2[\"benchmarks\"][0] for i in range(1000)\n ]\n return json1, json2\n\n json1, json2 = load_results()\n cls.json_diff_report = get_difference_report(json1, json2, utest=True)\n\n def test_json_diff_report_pretty_printing(self):\n expect_line = [\n \"BM_ManyRepetitions_pvalue\",\n \"0.0000\",\n \"0.0000\",\n \"U\",\n \"Test,\",\n \"Repetitions:\",\n \"1000\",\n \"vs\",\n \"1000\",\n ]\n output_lines_with_header = print_difference_report(\n self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False\n )\n output_lines = output_lines_with_header[2:]\n found = False\n for output_line in output_lines:\n parts = [x for x in output_line.split(\" \") if x]\n found = expect_line == parts\n if found:\n break\n self.assertTrue(found)\n\n def test_json_diff_report(self):\n expected_output = [\n {\n \"name\": \"BM_ManyRepetitions\",\n \"label\": \"\",\n \"time_unit\": \"s\",\n \"run_type\": \"\",\n \"aggregate_name\": \"\",\n \"utest\": {\n \"have_optimal_repetitions\": True,\n \"cpu_pvalue\": 0.0,\n \"time_pvalue\": 0.0,\n \"nr_of_repetitions\": 1000,\n \"nr_of_repetitions_other\": 1000,\n },\n },\n {\n \"name\": \"OVERALL_GEOMEAN\",\n \"label\": \"\",\n \"measurements\": [\n {\n \"real_time\": 1.0,\n \"cpu_time\": 1000.000000000069,\n \"real_time_other\": 1000.000000000069,\n \"cpu_time_other\": 1.0,\n \"time\": 999.000000000069,\n \"cpu\": -0.9990000000000001,\n }\n ],\n \"time_unit\": \"s\",\n \"run_type\": \"aggregate\",\n \"aggregate_name\": \"geomean\",\n \"utest\": {},\n },\n ]\n self.assertEqual(len(self.json_diff_report), len(expected_output))\n for out, expected in zip(\n self.json_diff_report, expected_output, strict=True\n ):\n self.assertEqual(out[\"name\"], expected[\"name\"])\n self.assertEqual(out[\"time_unit\"], expected[\"time_unit\"])\n assert_utest(self, out, expected)\n\n\ndef assert_utest(unittest_instance, lhs, rhs):\n if lhs[\"utest\"]:\n unittest_instance.assertAlmostEqual(\n lhs[\"utest\"][\"cpu_pvalue\"], rhs[\"utest\"][\"cpu_pvalue\"]\n )\n unittest_instance.assertAlmostEqual(\n lhs[\"utest\"][\"time_pvalue\"], rhs[\"utest\"][\"time_pvalue\"]\n )\n unittest_instance.assertEqual(\n lhs[\"utest\"][\"have_optimal_repetitions\"],\n rhs[\"utest\"][\"have_optimal_repetitions\"],\n )\n else:\n # lhs is empty. assert if rhs is not.\n unittest_instance.assertEqual(lhs[\"utest\"], rhs[\"utest\"])\n\n\ndef assert_measurements(unittest_instance, lhs, rhs):\n for m1, m2 in zip(lhs[\"measurements\"], rhs[\"measurements\"], strict=False):\n unittest_instance.assertEqual(m1[\"real_time\"], m2[\"real_time\"])\n unittest_instance.assertEqual(m1[\"cpu_time\"], m2[\"cpu_time\"])\n # m1['time'] and m1['cpu'] hold values which are being calculated,\n # and therefore we must use almost-equal pattern.\n unittest_instance.assertAlmostEqual(m1[\"time\"], m2[\"time\"], places=4)\n unittest_instance.assertAlmostEqual(m1[\"cpu\"], m2[\"cpu\"], places=4)\n\n\nif __name__ == \"__main__\":\n unittest.main()\n\n# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4\n# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;\n# kate: indent-mode python; remove-trailing-spaces modified;\n" }, { "path": "tools/gbench/util.py", "content": "\"\"\"util.py - General utilities for running, loading, and processing\nbenchmarks\n\"\"\"\n\nimport json\nimport os\nimport re\nimport subprocess\nimport sys\nimport tempfile\n\n# Input file type enumeration\nIT_Invalid = 0\nIT_JSON = 1\nIT_Executable = 2\n\n_num_magic_bytes = 2 if sys.platform.startswith(\"win\") else 4\n\n\ndef is_executable_file(filename):\n \"\"\"\n Return 'True' if 'filename' names a valid file which is likely\n an executable. A file is considered an executable if it starts with the\n magic bytes for a EXE, Mach O, or ELF file.\n \"\"\"\n if not os.path.isfile(filename):\n return False\n with open(filename, mode=\"rb\") as f:\n magic_bytes = f.read(_num_magic_bytes)\n if sys.platform == \"darwin\":\n return magic_bytes in [\n b\"\\xfe\\xed\\xfa\\xce\", # MH_MAGIC\n b\"\\xce\\xfa\\xed\\xfe\", # MH_CIGAM\n b\"\\xfe\\xed\\xfa\\xcf\", # MH_MAGIC_64\n b\"\\xcf\\xfa\\xed\\xfe\", # MH_CIGAM_64\n b\"\\xca\\xfe\\xba\\xbe\", # FAT_MAGIC\n b\"\\xbe\\xba\\xfe\\xca\", # FAT_CIGAM\n ]\n elif sys.platform.startswith(\"win\"):\n return magic_bytes == b\"MZ\"\n else:\n return magic_bytes == b\"\\x7fELF\"\n\n\ndef is_json_file(filename):\n \"\"\"\n Returns 'True' if 'filename' names a valid JSON output file.\n 'False' otherwise.\n \"\"\"\n try:\n with open(filename) as f:\n json.load(f)\n return True\n except BaseException:\n pass\n return False\n\n\ndef classify_input_file(filename):\n \"\"\"\n Return a tuple (type, msg) where 'type' specifies the classified type\n of 'filename'. If 'type' is 'IT_Invalid' then 'msg' is a human readable\n string representing the error.\n \"\"\"\n ftype = IT_Invalid\n err_msg = None\n if not os.path.exists(filename):\n err_msg = \"'%s' does not exist\" % filename\n elif not os.path.isfile(filename):\n err_msg = \"'%s' does not name a file\" % filename\n elif is_executable_file(filename):\n ftype = IT_Executable\n elif is_json_file(filename):\n ftype = IT_JSON\n else:\n err_msg = (\n \"'%s' does not name a valid benchmark executable or JSON file\"\n % filename\n )\n return ftype, err_msg\n\n\ndef check_input_file(filename):\n \"\"\"\n Classify the file named by 'filename' and return the classification.\n If the file is classified as 'IT_Invalid' print an error message and exit\n the program.\n \"\"\"\n ftype, msg = classify_input_file(filename)\n if ftype == IT_Invalid:\n print(\"Invalid input file: %s\" % msg)\n sys.exit(1)\n return ftype\n\n\ndef find_benchmark_flag(prefix, benchmark_flags):\n \"\"\"\n Search the specified list of flags for a flag matching `` and\n if it is found return the arg it specifies. If specified more than once the\n last value is returned. If the flag is not found None is returned.\n \"\"\"\n assert prefix.startswith(\"--\")\n assert prefix.endswith(\"=\")\n result = None\n for f in benchmark_flags:\n if f.startswith(prefix):\n result = f[len(prefix) :]\n return result\n\n\ndef remove_benchmark_flags(prefix, benchmark_flags):\n \"\"\"\n Return a new list containing the specified benchmark_flags except those\n with the specified prefix.\n \"\"\"\n assert prefix.startswith(\"--\")\n assert prefix.endswith(\"=\")\n return [f for f in benchmark_flags if not f.startswith(prefix)]\n\n\ndef load_benchmark_results(fname, benchmark_filter):\n \"\"\"\n Read benchmark output from a file and return the JSON object.\n\n Apply benchmark_filter, a regular expression, with nearly the same\n semantics of the --benchmark_filter argument. May be None.\n Note: the Python regular expression engine is used instead of the\n one used by the C++ code, which may produce different results\n in complex cases.\n\n REQUIRES: 'fname' names a file containing JSON benchmark output.\n \"\"\"\n\n def benchmark_wanted(benchmark):\n if benchmark_filter is None:\n return True\n name = benchmark.get(\"run_name\", None) or benchmark[\"name\"]\n return re.search(benchmark_filter, name) is not None\n\n with open(fname) as f:\n results = json.load(f)\n if \"json_schema_version\" in results.get(\"context\", {}):\n json_schema_version = results[\"context\"][\"json_schema_version\"]\n if json_schema_version != 1:\n print(\n f\"In {fname}, got unnsupported JSON schema version:\"\n f\" {json_schema_version}, expected 1\"\n )\n sys.exit(1)\n if \"benchmarks\" in results:\n results[\"benchmarks\"] = list(\n filter(benchmark_wanted, results[\"benchmarks\"])\n )\n return results\n\n\ndef sort_benchmark_results(result):\n benchmarks = result[\"benchmarks\"]\n\n # From inner key to the outer key!\n benchmarks = sorted(\n benchmarks,\n key=lambda benchmark: benchmark.get(\"repetition_index\", -1),\n )\n benchmarks = sorted(\n benchmarks,\n key=lambda benchmark: 1\n if \"run_type\" in benchmark and benchmark[\"run_type\"] == \"aggregate\"\n else 0,\n )\n benchmarks = sorted(\n benchmarks,\n key=lambda benchmark: benchmark.get(\"per_family_instance_index\", -1),\n )\n benchmarks = sorted(\n benchmarks,\n key=lambda benchmark: benchmark.get(\"family_index\", -1),\n )\n\n result[\"benchmarks\"] = benchmarks\n return result\n\n\ndef run_benchmark(exe_name, benchmark_flags):\n \"\"\"\n Run a benchmark specified by 'exe_name' with the specified\n 'benchmark_flags'. The benchmark is run directly as a subprocess to preserve\n real time console output.\n RETURNS: A JSON object representing the benchmark output\n \"\"\"\n output_name = find_benchmark_flag(\"--benchmark_out=\", benchmark_flags)\n is_temp_output = False\n if output_name is None:\n is_temp_output = True\n thandle, output_name = tempfile.mkstemp()\n os.close(thandle)\n benchmark_flags = [\n *list(benchmark_flags),\n \"--benchmark_out=%s\" % output_name,\n ]\n\n cmd = [exe_name, *benchmark_flags]\n print(\"RUNNING: %s\" % \" \".join(cmd))\n exitCode = subprocess.call(cmd)\n if exitCode != 0:\n print(\"TEST FAILED...\")\n sys.exit(exitCode)\n json_res = load_benchmark_results(output_name, None)\n if is_temp_output:\n os.unlink(output_name)\n return json_res\n\n\ndef run_or_load_benchmark(filename, benchmark_flags):\n \"\"\"\n Get the results for a specified benchmark. If 'filename' specifies\n an executable benchmark then the results are generated by running the\n benchmark. Otherwise 'filename' must name a valid JSON output file,\n which is loaded and the result returned.\n \"\"\"\n ftype = check_input_file(filename)\n if ftype == IT_JSON:\n benchmark_filter = find_benchmark_flag(\n \"--benchmark_filter=\", benchmark_flags\n )\n return load_benchmark_results(filename, benchmark_filter)\n if ftype == IT_Executable:\n return run_benchmark(filename, benchmark_flags)\n raise ValueError(\"Unknown file type %s\" % ftype)\n" }, { "path": "tools/libpfm.BUILD.bazel", "content": "\"\"\"Build rule for libpfm, which is required to collect performance counters for BENCHMARK_ENABLE_LIBPFM builds.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\n\nAARCH32_SRCS_COMMON = [\n \"lib/pfmlib_arm.c\",\n \"lib/pfmlib_arm_armv7_pmuv1.c\",\n \"lib/pfmlib_arm_armv6.c\",\n \"lib/pfmlib_arm_armv8.c\",\n \"lib/pfmlib_tx2_unc_perf_event.c\",\n]\n\nAARCH32_SRCS_LINUX = [\n \"lib/pfmlib_arm_perf_event.c\",\n]\n\nAARCH64_SRCS_COMMON = [\n \"lib/pfmlib_arm.c\",\n \"lib/pfmlib_arm_armv8.c\",\n \"lib/pfmlib_tx2_unc_perf_event.c\",\n]\n\nAARCH64_SRCS_LINUX = [\n \"lib/pfmlib_arm_perf_event.c\",\n]\n\nMIPS_SRCS_COMMON = [\n \"lib/pfmlib_mips.c\",\n \"lib/pfmlib_mips_74k.c\",\n]\n\nMIPS_SRCS_LINUX = [\n \"lib/pfmlib_mips_perf_event.c\",\n]\n\nPOWERPC_SRCS_COMMON = [\n \"lib/pfmlib_powerpc.c\",\n \"lib/pfmlib_power4.c\",\n \"lib/pfmlib_ppc970.c\",\n \"lib/pfmlib_power5.c\",\n \"lib/pfmlib_power6.c\",\n \"lib/pfmlib_power7.c\",\n \"lib/pfmlib_torrent.c\",\n \"lib/pfmlib_power8.c\",\n \"lib/pfmlib_power9.c\",\n \"lib/pfmlib_powerpc_nest.c\",\n]\n\nPOWERPC_SRCS_LINUX = [\n \"lib/pfmlib_powerpc_perf_event.c\",\n]\n\nS390X_SRCS_COMMON = [\n \"lib/pfmlib_s390x_cpumf.c\",\n]\n\nS390X_SRCS_LINUX = [\n \"lib/pfmlib_s390x_perf_event.c\",\n]\n\nX86_64_SRCS_COMMON = [\n \"lib/pfmlib_amd64.c\",\n \"lib/pfmlib_intel_core.c\",\n \"lib/pfmlib_intel_x86.c\",\n \"lib/pfmlib_intel_x86_arch.c\",\n \"lib/pfmlib_intel_atom.c\",\n \"lib/pfmlib_intel_nhm_unc.c\",\n \"lib/pfmlib_intel_nhm.c\",\n \"lib/pfmlib_intel_wsm.c\",\n \"lib/pfmlib_intel_snb.c\",\n \"lib/pfmlib_intel_snb_unc.c\",\n \"lib/pfmlib_intel_ivb.c\",\n \"lib/pfmlib_intel_ivb_unc.c\",\n \"lib/pfmlib_intel_hsw.c\",\n \"lib/pfmlib_intel_bdw.c\",\n \"lib/pfmlib_intel_skl.c\",\n \"lib/pfmlib_intel_icl.c\",\n \"lib/pfmlib_intel_rapl.c\",\n \"lib/pfmlib_intel_snbep_unc.c\",\n \"lib/pfmlib_intel_snbep_unc_cbo.c\",\n \"lib/pfmlib_intel_snbep_unc_ha.c\",\n \"lib/pfmlib_intel_snbep_unc_imc.c\",\n \"lib/pfmlib_intel_snbep_unc_pcu.c\",\n \"lib/pfmlib_intel_snbep_unc_qpi.c\",\n \"lib/pfmlib_intel_snbep_unc_ubo.c\",\n \"lib/pfmlib_intel_snbep_unc_r2pcie.c\",\n \"lib/pfmlib_intel_snbep_unc_r3qpi.c\",\n \"lib/pfmlib_intel_ivbep_unc_cbo.c\",\n \"lib/pfmlib_intel_ivbep_unc_ha.c\",\n \"lib/pfmlib_intel_ivbep_unc_imc.c\",\n \"lib/pfmlib_intel_ivbep_unc_pcu.c\",\n \"lib/pfmlib_intel_ivbep_unc_qpi.c\",\n \"lib/pfmlib_intel_ivbep_unc_ubo.c\",\n \"lib/pfmlib_intel_ivbep_unc_r2pcie.c\",\n \"lib/pfmlib_intel_ivbep_unc_r3qpi.c\",\n \"lib/pfmlib_intel_ivbep_unc_irp.c\",\n \"lib/pfmlib_intel_hswep_unc_cbo.c\",\n \"lib/pfmlib_intel_hswep_unc_ha.c\",\n \"lib/pfmlib_intel_hswep_unc_imc.c\",\n \"lib/pfmlib_intel_hswep_unc_pcu.c\",\n \"lib/pfmlib_intel_hswep_unc_qpi.c\",\n \"lib/pfmlib_intel_hswep_unc_ubo.c\",\n \"lib/pfmlib_intel_hswep_unc_r2pcie.c\",\n \"lib/pfmlib_intel_hswep_unc_r3qpi.c\",\n \"lib/pfmlib_intel_hswep_unc_irp.c\",\n \"lib/pfmlib_intel_hswep_unc_sbo.c\",\n \"lib/pfmlib_intel_bdx_unc_cbo.c\",\n \"lib/pfmlib_intel_bdx_unc_ubo.c\",\n \"lib/pfmlib_intel_bdx_unc_sbo.c\",\n \"lib/pfmlib_intel_bdx_unc_ha.c\",\n \"lib/pfmlib_intel_bdx_unc_imc.c\",\n \"lib/pfmlib_intel_bdx_unc_irp.c\",\n \"lib/pfmlib_intel_bdx_unc_pcu.c\",\n \"lib/pfmlib_intel_bdx_unc_qpi.c\",\n \"lib/pfmlib_intel_bdx_unc_r2pcie.c\",\n \"lib/pfmlib_intel_bdx_unc_r3qpi.c\",\n \"lib/pfmlib_intel_skx_unc_cha.c\",\n \"lib/pfmlib_intel_skx_unc_iio.c\",\n \"lib/pfmlib_intel_skx_unc_imc.c\",\n \"lib/pfmlib_intel_skx_unc_irp.c\",\n \"lib/pfmlib_intel_skx_unc_m2m.c\",\n \"lib/pfmlib_intel_skx_unc_m3upi.c\",\n \"lib/pfmlib_intel_skx_unc_pcu.c\",\n \"lib/pfmlib_intel_skx_unc_ubo.c\",\n \"lib/pfmlib_intel_skx_unc_upi.c\",\n \"lib/pfmlib_intel_knc.c\",\n \"lib/pfmlib_intel_slm.c\",\n \"lib/pfmlib_intel_tmt.c\",\n \"lib/pfmlib_intel_knl.c\",\n \"lib/pfmlib_intel_knl_unc_imc.c\",\n \"lib/pfmlib_intel_knl_unc_edc.c\",\n \"lib/pfmlib_intel_knl_unc_cha.c\",\n \"lib/pfmlib_intel_knl_unc_m2pcie.c\",\n \"lib/pfmlib_intel_glm.c\",\n \"lib/pfmlib_intel_netburst.c\",\n \"lib/pfmlib_amd64_k7.c\",\n \"lib/pfmlib_amd64_k8.c\",\n \"lib/pfmlib_amd64_fam10h.c\",\n \"lib/pfmlib_amd64_fam11h.c\",\n \"lib/pfmlib_amd64_fam12h.c\",\n \"lib/pfmlib_amd64_fam14h.c\",\n \"lib/pfmlib_amd64_fam15h.c\",\n \"lib/pfmlib_amd64_fam17h.c\",\n \"lib/pfmlib_amd64_fam16h.c\",\n]\n\nX86_SRCS_COMMON = X86_64_SRCS_COMMON + [\n \"lib/pfmlib_intel_coreduo.c\",\n \"lib/pfmlib_intel_p6.c\",\n]\n\nfilegroup(\n name = \"cpu_srcs\",\n srcs = select({\n \"@platforms//cpu:x86_32\": X86_SRCS_COMMON,\n \"@platforms//cpu:x86_64\": X86_64_SRCS_COMMON,\n \"@platforms//cpu:aarch32\": AARCH32_SRCS_COMMON,\n \"@platforms//cpu:aarch64\": AARCH64_SRCS_COMMON,\n \"@platforms//cpu:mips64\": MIPS_SRCS_COMMON,\n \"@platforms//cpu:ppc32\": POWERPC_SRCS_COMMON,\n \"@platforms//cpu:ppc64le\": POWERPC_SRCS_COMMON,\n \"@platforms//cpu:ppc\": POWERPC_SRCS_COMMON,\n \"@platforms//cpu:s390x\": S390X_SRCS_COMMON,\n \"//conditions:default\": [],\n }),\n)\n\nfilegroup(\n name = \"linux_srcs\",\n srcs = select({\n \"@platforms//cpu:aarch32\": AARCH32_SRCS_LINUX,\n \"@platforms//cpu:aarch64\": AARCH64_SRCS_LINUX,\n \"@platforms//cpu:mips64\": MIPS_SRCS_LINUX,\n \"@platforms//cpu:ppc32\": POWERPC_SRCS_LINUX,\n \"@platforms//cpu:ppc64le\": POWERPC_SRCS_LINUX,\n \"@platforms//cpu:ppc\": POWERPC_SRCS_LINUX,\n \"@platforms//cpu:s390x\": S390X_SRCS_LINUX,\n \"//conditions:default\": [],\n }),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = [\n \"lib/pfmlib_common.c\",\n \"lib/pfmlib_perf_event.c\",\n \"lib/pfmlib_perf_event_pmu.c\",\n \"lib/pfmlib_perf_event_priv.h\",\n \"lib/pfmlib_perf_event_raw.c\",\n \"lib/pfmlib_torrent.c\",\n \"lib/pfmlib_tx2_unc_perf_event.c\",\n \":cpu_srcs\",\n ] + select({\n \"@platforms//os:linux\": [\":linux_srcs\"],\n \"//conditions:default\": [],\n }),\n)\n\ncc_library(\n name = \"pfm\",\n srcs = [\n \":srcs\",\n ],\n hdrs = glob([\n \"include/perfmon/*.h\",\n ]),\n copts = [\n \"-Wno-format-truncation\",\n \"-Wno-use-after-free\",\n \"-fPIC\",\n \"-D_REENTRANT\",\n \"-fvisibility=hidden\",\n ] + select({\n \"@platforms//cpu:aarch32\": [\"-DCONFIG_PFMLIB_ARCH_ARM\"],\n \"@platforms//cpu:aarch64\": [\"-DCONFIG_PFMLIB_ARCH_ARM64\"],\n \"@platforms//cpu:mips64\": [\"-DCONFIG_PFMLIB_ARCH_MIPS\"],\n \"@platforms//cpu:ppc32\": [\"-DCONFIG_PFMLIB_ARCH_POWERPC\"],\n \"@platforms//cpu:ppc64le\": [\"-DCONFIG_PFMLIB_ARCH_POWERPC\"],\n \"@platforms//cpu:ppc\": [\"-DCONFIG_PFMLIB_ARCH_POWERPC\"],\n \"@platforms//cpu:s390x\": [\"-DCONFIG_PFMLIB_ARCH_S390X\"],\n \"//conditions:default\": [],\n }),\n includes = [\n \"include\",\n \"lib\",\n ],\n strip_include_prefix = \"include\",\n textual_hdrs = glob([\n \"lib/**/*.h\",\n ]),\n visibility = [\n \"//visibility:public\",\n ],\n)\n\nalias(\n name = \"libpfm\",\n actual = \":pfm\",\n visibility = [\n \"//visibility:public\",\n ],\n)\n" }, { "path": "tools/requirements.txt", "content": "numpy == 2.4.3\nscipy == 1.17.1\n" }, { "path": "tools/strip_asm.py", "content": "#!/usr/bin/env python3\n\n\"\"\"\nstrip_asm.py - Cleanup ASM output for the specified file\n\"\"\"\n\nimport os\nimport re\nimport sys\nfrom argparse import ArgumentParser\n\n\ndef find_used_labels(asm):\n found = set()\n label_re = re.compile(r\"\\s*j[a-z]+\\s+\\.L([a-zA-Z0-9][a-zA-Z0-9_]*)\")\n for line in asm.splitlines():\n m = label_re.match(line)\n if m:\n found.add(\".L%s\" % m.group(1))\n return found\n\n\ndef normalize_labels(asm):\n decls = set()\n label_decl = re.compile(\"^[.]{0,1}L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)\")\n for line in asm.splitlines():\n m = label_decl.match(line)\n if m:\n decls.add(m.group(0))\n if len(decls) == 0:\n return asm\n needs_dot = next(iter(decls))[0] != \".\"\n if not needs_dot:\n return asm\n for ld in decls:\n asm = re.sub(r\"(^|\\s+)\" + ld + r\"(?=:|\\s)\", \"\\\\1.\" + ld, asm)\n return asm\n\n\ndef transform_labels(asm):\n asm = normalize_labels(asm)\n used_decls = find_used_labels(asm)\n new_asm = \"\"\n label_decl = re.compile(r\"^\\.L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)\")\n for line in asm.splitlines():\n m = label_decl.match(line)\n if not m or m.group(0) in used_decls:\n new_asm += line\n new_asm += \"\\n\"\n return new_asm\n\n\ndef is_identifier(tk):\n if len(tk) == 0:\n return False\n first = tk[0]\n if not first.isalpha() and first != \"_\":\n return False\n for i in range(1, len(tk)):\n c = tk[i]\n if not c.isalnum() and c != \"_\":\n return False\n return True\n\n\ndef process_identifiers(line):\n \"\"\"\n process_identifiers - process all identifiers and modify them to have\n consistent names across all platforms; specifically across ELF and MachO.\n For example, MachO inserts an additional understore at the beginning of\n names. This function removes that.\n \"\"\"\n parts = re.split(r\"([a-zA-Z0-9_]+)\", line)\n new_line = \"\"\n for tk in parts:\n if is_identifier(tk) and (\n tk.startswith(\"__Z\")\n or (\n tk.startswith(\"_\")\n and len(tk) > 1\n and tk[1].isalpha()\n and tk[1] != \"Z\"\n )\n ):\n tk = tk[1:]\n new_line += tk\n return new_line\n\n\ndef process_asm(asm):\n \"\"\"\n Strip the ASM of unwanted directives and lines\n \"\"\"\n new_contents = \"\"\n asm = transform_labels(asm)\n\n # TODO: Add more things we want to remove\n discard_regexes = [\n re.compile(r\"\\s+\\..*$\"), # directive\n re.compile(r\"\\s*#(NO_APP|APP)$\"), # inline ASM\n re.compile(r\"\\s*#.*$\"), # comment line\n re.compile(\n r\"\\s*\\.globa?l\\s*([.a-zA-Z_][a-zA-Z0-9$_.]*)\"\n ), # global directive\n re.compile(\n r\"\\s*\\.(string|asciz|ascii|[1248]?byte|short|word|long|quad|value|zero)\"\n ),\n ]\n keep_regexes: list[re.Pattern] = []\n fn_label_def = re.compile(\"^[a-zA-Z_][a-zA-Z0-9_.]*:\")\n for line in asm.splitlines():\n # Remove Mach-O attribute\n line = line.replace(\"@GOTPCREL\", \"\")\n add_line = True\n for reg in discard_regexes:\n if reg.match(line) is not None:\n add_line = False\n break\n for reg in keep_regexes:\n if reg.match(line) is not None:\n add_line = True\n break\n if add_line:\n if fn_label_def.match(line) and len(new_contents) != 0:\n new_contents += \"\\n\"\n line = process_identifiers(line)\n new_contents += line\n new_contents += \"\\n\"\n return new_contents\n\n\ndef main():\n parser = ArgumentParser(description=\"generate a stripped assembly file\")\n parser.add_argument(\n \"input\",\n metavar=\"input\",\n type=str,\n nargs=1,\n help=\"An input assembly file\",\n )\n parser.add_argument(\n \"out\", metavar=\"output\", type=str, nargs=1, help=\"The output file\"\n )\n args, _ = parser.parse_known_args()\n input = args.input[0]\n output = args.out[0]\n if not os.path.isfile(input):\n print(\"ERROR: input file '%s' does not exist\" % input)\n sys.exit(1)\n\n with open(input) as f:\n contents = f.read()\n new_contents = process_asm(contents)\n with open(output, \"w\") as f:\n f.write(new_contents)\n\n\nif __name__ == \"__main__\":\n main()\n\n# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4\n# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;\n# kate: indent-mode python; remove-trailing-spaces modified;\n" } ]