[ { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: cargo\n directory: \"/\"\n schedule:\n interval: monthly\n time: \"04:00\"\n timezone: Europe/Berlin\n open-pull-requests-limit: 2\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"daily\"\n" }, { "path": ".github/workflows/CICD.yml", "content": "name: CICD\n\nenv:\n CICD_INTERMEDIATES_DIR: \"_cicd-intermediates\"\n MSRV_FEATURES: \"\"\n\non:\n workflow_dispatch:\n pull_request:\n push:\n branches:\n - master\n tags:\n - '*'\n\njobs:\n crate_metadata:\n name: Extract crate metadata\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v5\n - name: Extract crate information\n id: crate_metadata\n run: |\n cargo metadata --no-deps --format-version 1 | jq -r '\n .packages[0] |\n [\n \"name=\" + .name,\n \"version=\" + .version,\n \"maintainer=\" + (.authors[0] // \"\"),\n \"homepage=\" + (.homepage // \"\"),\n \"msrv=\" + (.rust_version // \"\"),\n \"bin-name=\" + ( (.targets[] | select(.kind[0] == \"bin\") | .name) // .name )\n ] |\n join(\"\\n\")\n ' | tee -a $GITHUB_OUTPUT\n outputs:\n name: ${{ steps.crate_metadata.outputs.name }}\n version: ${{ steps.crate_metadata.outputs.version }}\n maintainer: ${{ steps.crate_metadata.outputs.maintainer }}\n homepage: ${{ steps.crate_metadata.outputs.homepage }}\n msrv: ${{ steps.crate_metadata.outputs.msrv }}\n bin-name: ${{ steps.crate_metadata.outputs.bin-name }}\n\n ensure_cargo_fmt:\n name: Ensure 'cargo fmt' has been run\n runs-on: ubuntu-24.04\n steps:\n - uses: dtolnay/rust-toolchain@stable\n with:\n components: rustfmt\n - uses: actions/checkout@v5\n - run: cargo fmt -- --check\n\n min_version:\n name: Minimum supported rust version\n runs-on: ubuntu-24.04\n needs: crate_metadata\n steps:\n - name: Checkout source code\n uses: actions/checkout@v5\n\n - name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }})\n uses: dtolnay/rust-toolchain@master\n with:\n toolchain: ${{ needs.crate_metadata.outputs.msrv }}\n components: clippy\n - name: Run clippy (on minimum supported rust version to prevent warnings we can't fix)\n run: cargo clippy --locked --all-targets ${{ env.MSRV_FEATURES }}\n - name: Run tests\n run: cargo test --locked ${{ env.MSRV_FEATURES }}\n\n build:\n name: ${{ matrix.job.target }} (${{ matrix.job.os }})\n runs-on: ${{ matrix.job.os }}\n needs: crate_metadata\n strategy:\n fail-fast: false\n matrix:\n job:\n - { target: aarch64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }\n - { target: arm-unknown-linux-gnueabihf , os: ubuntu-24.04, use-cross: true }\n - { target: arm-unknown-linux-musleabihf, os: ubuntu-24.04, use-cross: true }\n - { target: i686-pc-windows-msvc , os: windows-2022 }\n - { target: i686-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }\n - { target: i686-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }\n - { target: x86_64-apple-darwin , os: macos-15 }\n - { target: aarch64-apple-darwin , os: macos-15 }\n # - { target: x86_64-pc-windows-gnu , os: windows-2022 }\n - { target: x86_64-pc-windows-msvc , os: windows-2022 }\n - { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }\n - { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }\n env:\n BUILD_CMD: cargo\n steps:\n - name: Checkout source code\n uses: actions/checkout@v5\n\n - name: Install prerequisites\n shell: bash\n run: |\n case ${{ matrix.job.target }} in\n arm-unknown-linux-*) sudo apt-get -y update ; sudo apt-get -y install gcc-arm-linux-gnueabihf ;;\n aarch64-unknown-linux-gnu) sudo apt-get -y update ; sudo apt-get -y install gcc-aarch64-linux-gnu ;;\n esac\n\n - name: Install Rust toolchain\n uses: dtolnay/rust-toolchain@stable\n with:\n targets: ${{ matrix.job.target }}\n\n - name: Install cross\n if: matrix.job.use-cross\n uses: taiki-e/install-action@v2\n with:\n tool: cross\n\n - name: Overwrite build command env variable\n if: matrix.job.use-cross\n shell: bash\n run: echo \"BUILD_CMD=cross\" >> $GITHUB_ENV\n\n - name: Show version information (Rust, cargo, GCC)\n shell: bash\n run: |\n set -x\n gcc --version || true\n rustup -V\n rustup toolchain list\n rustup default\n cargo -V\n rustc -V\n\n - name: Build\n shell: bash\n run: $BUILD_CMD build --locked --release --target=${{ matrix.job.target }}\n\n - name: Set binary name & path\n id: bin\n shell: bash\n run: |\n # Figure out suffix of binary\n EXE_suffix=\"\"\n case ${{ matrix.job.target }} in\n *-pc-windows-*) EXE_suffix=\".exe\" ;;\n esac;\n\n # Setup paths\n BIN_NAME=\"${{ needs.crate_metadata.outputs.bin-name }}${EXE_suffix}\"\n BIN_PATH=\"target/${{ matrix.job.target }}/release/${BIN_NAME}\"\n\n # Let subsequent steps know where to find the binary\n echo \"BIN_PATH=${BIN_PATH}\" >> $GITHUB_OUTPUT\n echo \"BIN_NAME=${BIN_NAME}\" >> $GITHUB_OUTPUT\n\n - name: Set testing options\n id: test-options\n shell: bash\n run: |\n # test only library unit tests and binary for arm-type targets\n unset CARGO_TEST_OPTIONS\n unset CARGO_TEST_OPTIONS ; case ${{ matrix.job.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS=\"--bin ${{ steps.bin.outputs.BIN_NAME }}\" ;; esac;\n echo \"CARGO_TEST_OPTIONS=${CARGO_TEST_OPTIONS}\" >> $GITHUB_OUTPUT\n\n - name: Run tests\n shell: bash\n run: $BUILD_CMD test --locked --target=${{ matrix.job.target }} ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS}}\n\n - name: Create tarball\n id: package\n shell: bash\n run: |\n PKG_suffix=\".tar.gz\" ; case ${{ matrix.job.target }} in *-pc-windows-*) PKG_suffix=\".zip\" ;; esac;\n PKG_BASENAME=${{ needs.crate_metadata.outputs.name }}-v${{ needs.crate_metadata.outputs.version }}-${{ matrix.job.target }}\n PKG_NAME=${PKG_BASENAME}${PKG_suffix}\n echo \"PKG_NAME=${PKG_NAME}\" >> $GITHUB_OUTPUT\n\n PKG_STAGING=\"${{ env.CICD_INTERMEDIATES_DIR }}/package\"\n ARCHIVE_DIR=\"${PKG_STAGING}/${PKG_BASENAME}/\"\n mkdir -p \"${ARCHIVE_DIR}\"\n mkdir -p \"${ARCHIVE_DIR}/autocomplete\"\n\n # Binary\n cp \"${{ steps.bin.outputs.BIN_PATH }}\" \"$ARCHIVE_DIR\"\n\n # README, LICENSE and CHANGELOG files\n cp \"README.md\" \"LICENSE-MIT\" \"LICENSE-APACHE\" \"CHANGELOG.md\" \"$ARCHIVE_DIR\"\n\n # Man page\n cp 'doc/${{ needs.crate_metadata.outputs.name }}.1' \"$ARCHIVE_DIR\"\n\n # Autocompletion files\n cp 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'${{ needs.crate_metadata.outputs.name }}.bash' \"$ARCHIVE_DIR/autocomplete/\"\n cp 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'${{ needs.crate_metadata.outputs.name }}.fish' \"$ARCHIVE_DIR/autocomplete/\"\n cp 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'_${{ needs.crate_metadata.outputs.name }}.ps1' \"$ARCHIVE_DIR/autocomplete/\"\n cp 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'_${{ needs.crate_metadata.outputs.name }}' \"$ARCHIVE_DIR/autocomplete/\"\n\n # base compressed package\n pushd \"${PKG_STAGING}/\" >/dev/null\n case ${{ matrix.job.target }} in\n *-pc-windows-*) 7z -y a \"${PKG_NAME}\" \"${PKG_BASENAME}\"/* | tail -2 ;;\n *) tar czf \"${PKG_NAME}\" \"${PKG_BASENAME}\"/* ;;\n esac;\n popd >/dev/null\n\n # Let subsequent steps know where to find the compressed package\n echo \"PKG_PATH=${PKG_STAGING}/${PKG_NAME}\" >> $GITHUB_OUTPUT\n\n - name: Create Debian package\n id: debian-package\n shell: bash\n if: startsWith(matrix.job.os, 'ubuntu')\n run: |\n COPYRIGHT_YEARS=\"2018 - \"$(date \"+%Y\")\n DPKG_STAGING=\"${{ env.CICD_INTERMEDIATES_DIR }}/debian-package\"\n DPKG_DIR=\"${DPKG_STAGING}/dpkg\"\n mkdir -p \"${DPKG_DIR}\"\n\n DPKG_BASENAME=${{ needs.crate_metadata.outputs.name }}\n DPKG_CONFLICTS=${{ needs.crate_metadata.outputs.name }}-musl\n case ${{ matrix.job.target }} in *-musl*) DPKG_BASENAME=${{ needs.crate_metadata.outputs.name }}-musl ; DPKG_CONFLICTS=${{ needs.crate_metadata.outputs.name }} ;; esac;\n DPKG_VERSION=${{ needs.crate_metadata.outputs.version }}\n\n unset DPKG_ARCH\n case ${{ matrix.job.target }} in\n aarch64-*-linux-*) DPKG_ARCH=arm64 ;;\n arm-*-linux-*hf) DPKG_ARCH=armhf ;;\n i686-*-linux-*) DPKG_ARCH=i686 ;;\n x86_64-*-linux-*) DPKG_ARCH=amd64 ;;\n *) DPKG_ARCH=notset ;;\n esac;\n\n DPKG_NAME=\"${DPKG_BASENAME}_${DPKG_VERSION}_${DPKG_ARCH}.deb\"\n echo \"DPKG_NAME=${DPKG_NAME}\" >> $GITHUB_OUTPUT\n\n # Binary\n install -Dm755 \"${{ steps.bin.outputs.BIN_PATH }}\" \"${DPKG_DIR}/usr/bin/${{ steps.bin.outputs.BIN_NAME }}\"\n\n # Man page\n install -Dm644 'doc/${{ needs.crate_metadata.outputs.name }}.1' \"${DPKG_DIR}/usr/share/man/man1/${{ needs.crate_metadata.outputs.name }}.1\"\n gzip -n --best \"${DPKG_DIR}/usr/share/man/man1/${{ needs.crate_metadata.outputs.name }}.1\"\n\n # Autocompletion files\n install -Dm644 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'${{ needs.crate_metadata.outputs.name }}.bash' \"${DPKG_DIR}/usr/share/bash-completion/completions/${{ needs.crate_metadata.outputs.name }}\"\n install -Dm644 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'${{ needs.crate_metadata.outputs.name }}.fish' \"${DPKG_DIR}/usr/share/fish/vendor_completions.d/${{ needs.crate_metadata.outputs.name }}.fish\"\n install -Dm644 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'_${{ needs.crate_metadata.outputs.name }}' \"${DPKG_DIR}/usr/share/zsh/vendor-completions/_${{ needs.crate_metadata.outputs.name }}\"\n\n # README and LICENSE\n install -Dm644 \"README.md\" \"${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/README.md\"\n install -Dm644 \"LICENSE-MIT\" \"${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/LICENSE-MIT\"\n install -Dm644 \"LICENSE-APACHE\" \"${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/LICENSE-APACHE\"\n install -Dm644 \"CHANGELOG.md\" \"${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/changelog\"\n gzip -n --best \"${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/changelog\"\n\n cat > \"${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/copyright\" < \"${DPKG_DIR}/DEBIAN/control\" <> $GITHUB_OUTPUT\n\n # build dpkg\n fakeroot dpkg-deb --build \"${DPKG_DIR}\" \"${DPKG_PATH}\"\n\n - name: \"Artifact upload: tarball\"\n uses: actions/upload-artifact@master\n with:\n name: ${{ steps.package.outputs.PKG_NAME }}\n path: ${{ steps.package.outputs.PKG_PATH }}\n\n - name: \"Artifact upload: Debian package\"\n uses: actions/upload-artifact@master\n if: steps.debian-package.outputs.DPKG_NAME\n with:\n name: ${{ steps.debian-package.outputs.DPKG_NAME }}\n path: ${{ steps.debian-package.outputs.DPKG_PATH }}\n\n - name: Check for release\n id: is-release\n shell: bash\n run: |\n unset IS_RELEASE ; if [[ $GITHUB_REF =~ ^refs/tags/v[0-9].* ]]; then IS_RELEASE='true' ; fi\n echo \"IS_RELEASE=${IS_RELEASE}\" >> $GITHUB_OUTPUT\n\n - name: Publish archives and packages\n uses: softprops/action-gh-release@v2\n if: steps.is-release.outputs.IS_RELEASE\n with:\n files: |\n ${{ steps.package.outputs.PKG_PATH }}\n ${{ steps.debian-package.outputs.DPKG_PATH }}\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n\n winget:\n name: Publish to Winget\n runs-on: ubuntu-latest\n needs: build\n if: startsWith(github.ref, 'refs/tags/v')\n steps:\n - uses: vedantmgoyal2009/winget-releaser@v2\n with:\n identifier: sharkdp.hyperfine\n installers-regex: '-pc-windows-msvc\\.zip$'\n token: ${{ secrets.WINGET_TOKEN }}\n" }, { "path": ".gitignore", "content": "\n/target/\n**/*.rs.bk\n" }, { "path": "CHANGELOG.md", "content": "# v1.20.0\n\n## Features\n\n- Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally)\n- The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp)\n- Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp)\n- Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin)\n\n## Bugfixes\n\n- Fix bug where naming individual commands with parameter scan was not working correctly, see #794 (@teofr)\n\n## Other\n\n- Restrict `cat` tests to Unix environments, see #776 and #777 (@ritvikos)\n\n# v1.19.0\n\n## Features\n\n- Add a new `--reference ` option to specify a reference command for the relative speed comparison, see #579, #577 and #744 (@sharkdp)\n- Add `--conclude` argument (analog to `--prepare`), see #565 and #719 (@jackoconnordev)\n- Allow `--output=…` to appear once for each command, enabling use cases like `hyperfine --output=null my-cmd --output=./file.log my-cmd`, see #529 and #775 (@sharkdp)\n- The environment variable `$HYPERFINE_ITERATION` will now contain the current iteration number for each benchmarked command, see #775 (@sharkdp)\n- Add iteration information to failure error message, see #771 and #772 (@sharkdp)\n- Python scripts: \n - legend modification parameters and output DPI, see #758 (@Spreadcat)\n - Nicer whiskers plot, see #727 (@serpent7776)\n\n## Bugfixes\n\n- ETA not clearly visible on terminals with a block cursor, see #698 and #699 (@overclockworked64)\n- Fix zsh completions, see #717 (@xzfc)\n\n## Other\n\n- Build binaries for aarch64-apple-darwin, see #728 (@Phault)\n- Various cleanups (@hamirmahal, @one230six)\n\n# v1.18.0\n\n## Features\n\n- Add support for microseconds via `--time-unit microsecond`, see #684 (@sharkdp)\n\n## Bugfixes\n\n- Proper argument quoting on Windows CMD, see #296 and #678 (@PedroWitzel)\n\n\n# v1.17.0\n\n## Features\n\n- Add new `--sort` option to control the order in the rel. speed comparison and in markup export formats, see #601, #614, #655 (@sharkdp)\n- Parameters which are unused in the command line are now displayed in parentheses, see #600 and #644 (@sharkdp).\n- Added `--log-count` option for histogram plots, see `scripts/plot_histogram.py` (@sharkdp)\n\n## Changes\n\n- Updated hyperfine to use `windows-sys` instead of the unmaintained `winapi`, see #624, #639, #636, #641 (@clemenswasser)\n- Silenced deprecation warning in Python scripts, see #633 (@nicovank)\n- Major update of the man page, see 0ce6578, #647 (@sharkdp)\n\n## Bugfixes\n\n- Do not export intermediate results to stdout when using `-` as a file name, see #640 and #643 (@sharkdp)\n- Markup exporting does not fail if benchmark results are zero, see #642 (@sharkdp)\n\n\n# v1.16.1\n\n## Bugfixes\n\n- Fix line-wrapping of `--help` text (@sharkdp)\n- Fix `--input=null` (@sharkdp)\n\n\n# v1.16.0\n\n## Features\n\n- Added new `--input` option, see #541 and #563 (@snease)\n- Added possibility to specify `-` as the filename in the\n `--export-*` options, see #615 and #623 (@humblepenguinn)\n\n## Changes\n\n- Improve hints for outlier warnings if `--warmup` or `--prepare` are in use already,\n see #570 (@sharkdp)\n\n## Bugfixes\n\n- Fix uncolored output on Windows if `TERM` is not set, see #583 (@nabijaczleweli)\n- On Windows, only run `cmd.exe` with the `/C` option. Use `-c` for all other shells.\n See #568 and #582 (@FilipAndersson245)\n\n## Other\n\n- Thanks to @berombau for working on dependency upgrades, see #584\n- Fixed installationm on Windows, see #595 and #596 (@AntoniosBarotsis)\n\n\n# v1.15.0\n\n## Features\n\n- Disable colorized output in case of `TERM=dumb` or `NO_COLOR=1`, see #542 and #555 (@nabijaczleweli)\n- Add new (experimental) `--min-benchmarking-time ` option, see #527 (@sharkdp)\n\n## Bugfixes\n\n- Fix user and kernel times on Windows, see #368 and #538 (@clemenswasser)\n\n## Other\n\n- Improve `--help` texts of `--export-*` options, see #506 and #522 (@Engineer-of-Efficiency)\n\n\n# v1.14.0\n\n## Features\n\n- Add a new `--output={null,pipe,inherit,}` option to control\n where the output of the benchmarked program is redirected (if at all),\n see #377 and #509 (@tavianator, originally suggested by @BurntSushi)\n- Add Emacs org-mode as a new export format, see #491 (@ppaulweber)\n\n\n# v1.13.0\n\n## Features\n\n- Added a new `--shell=none`/`-N` option to disable the intermediate\n shell for executing the benchmarked commands. Hyperfine normally\n measures and subtracts the shell spawning time, but the intermediate\n shell always introduces a certain level of measurement noise. Using\n `--shell=none`/`-N` allows users to benchmark very fast commands\n (with a runtime on the order of a few milliseconds). See #336, #429,\n and #487 (@cipriancraciun and @sharkdp)\n- Added `--setup`/`-s` option that can be used to run `make all` or\n similar. It runs once per set of tests, like `--cleanup`/`-c` (@avar)\n- Added new `plot_progression.py` script to debug background interference\n effects.\n\n## Changes\n\n- Breaking change: the `-s` short option for `--style` is now used for\n the new `--setup` option.\n- The environment offset randomization is now also available on Windows,\n see #484\n\n## Other\n\n- Improved documentation and test coverage, cleaned up code base for\n future improvements.\n\n\n# v1.12.0\n\n## Features\n\n- `--command-name` can now take parameter names from `--parameter-*` options, see #351 and #391 (@silathdiir)\n- Exit codes (or signals) are now printed in cases of command failures, see #342 (@KaindlJulian)\n- Exit codes are now part of the JSON output, see #371 (@JordiChauzi)\n- Colorized output should now be enabled on Windows by default, see #427\n\n## Changes\n\n- When `--export-*` commands are used, result files are created before benchmark execution\n to fail early in case of, e.g., wrong permissions. See #306 (@s1ck).\n- When `--export-*` options are used, result files are written after each individual\n benchmark command instead of writing after all benchmarks have finished. See #306 (@s1ck).\n- Reduce number of shell startup time measurements from 200 to 50, generally speeding up benchmarks. See #378\n- User and system time are now in consistent time units, see #408 and #409 (@film42)\n\n\n\n# v1.11.0\n\n## Features\n\n- The `-L`/`--parameter-list` option can now be specified multiple times to\n evaluate all possible combinations of the listed parameters:\n\n ``` bash\n hyperfine -L number 1,2 -L letter a,b,c \\\n \"echo {number}{letter}\" \\\n \"printf '%s\\n' {number}{letter}\"\n # runs 12 benchmarks: 2 commands (echo and printf) times 6 combinations of\n # the \"letter\" and \"number\" parameters\n ```\n\n See: #253, #318 (@wchargin)\n\n- Add CLI option to identify a command with a custom name, see #326 (@scampi)\n\n## Changes\n\n- When parameters are used with `--parameter-list` or `--parameter-scan`, the JSON export format\n now contains a dictionary `parameters` instead of a single key `parameter`. See #253, #318.\n- The `plot_parametrized.py` script now infers the parameter name, and its `--parameter-name`\n argument has been deprecated. See #253, #318.\n\n## Bugfixes\n\n- Fix a bug in the outlier detection which would only detect \"slow outliers\" but not the fast\n ones (runs that are much faster than the rest of the benchmarking runs), see #329\n- Better error messages for very fast commands that would lead to inf/nan results in the relative\n speed comparison, see #319\n- Show error message if `--warmup` or `--*runs` arguments can not be parsed, see #337\n- Keep output colorized when the output is not interactive and `--style=full` or `--style=color` is used.\n\n\n# v1.10.0\n\n## Features\n\n- Hyperfine now comes with shell completion files for Bash, Zsh, Fish\n and PowerShell, see #290 (@four0000four).\n- Hyperfine now comes with a basic man page, see #257 (@cadeef)\n- During execution of benchmarks, hyperfine will now set a `HYPERFINE_RANDOMIZED_ENVIRONMENT_OFFSET` environment variable in order to randomize the memory layout. See #235 and #241 for references and details.\n- A few enhancements for the histogram plotting scripts and the\n advanced statistics script\n- Updates for the `plot_whisker.py` script, see #275 (@ghaiklor)\n\n## Bugfixes\n\n- Fix Spin Icon on Windows, see #229\n- A few typos have been fixed, see #292 (@McMartin)\n\n## Packaging\n\n- `hyperfine` is now available on MacPorts for macOS, see #281 (@herbygillot)\n- `hyperfine` is now available on OpenBSD, see #289 (@minusf)\n\nPackage authors: note that Hyperfine now comes with a set of shell completion files and a man page (see above)\n\n# v1.9.0\n\n## Features\n\n- The new `--parameter-list ` option can be used to run\n a parametrized benchmark on a user-specified list of values.\n This is similar to `--parameter-scan `, but doesn't\n necessarily required numeric arguments.\n\n ``` bash\n hyperfine --parameter-list compiler \"gcc,clang\" \\\n \"{compiler} -O2 main.cpp\"\n ```\n\n See: #227, #234 (@JuanPotato)\n\n- Added `none` as a possible choice for the `--style` option to\n run `hyperfine` without any output, see #193 (@knidarkness)\n\n- Added a few new scripts for plotting various types of benchmark\n results (https://github.com/sharkdp/hyperfine/tree/master/scripts)\n\n## Changes\n\n- The `--prepare` command is now also run during the warmup\n phase, see #182 (@sseemayer)\n\n- Better estimation of the remaining benchmark time due to an update\n of the `indicatif` crate.\n\n## Other\n\n- `hyperfine` is now available on NixOS, see #240 (@tuxinaut)\n\n# v1.8.0\n\n## Features\n\n- The `--prepare ` option can now be specified multiple times to\n run specific preparation commands for each of the benchmarked programs:\n\n ``` bash\n hyperfine --prepare \"make clean; git checkout master\" \"make\" \\\n --prepare \"make clean; git checkout feature\" \"make\"\n ```\n\n See: #216, #218 (@iamsauravsharma)\n\n- Added a new [`welch_ttest.py`](https://github.com/sharkdp/hyperfine/blob/master/scripts/welch_ttest.py) script to test whether or not the two benchmark\n results are the same, see #222 (@uetchy)\n\n- The Markdown export has been improved. The relative speed is now exported\n with a higher precision (see #208) and includes the standard deviation\n (see #225).\n\n## Other\n\n- Improved documentation for [`scripts`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder (@matthieusb)\n\n# v1.7.0\n\n## Features\n\n- Added a new `-D`,`--parameter-step-size` option that can be used to control\n the step size for `--parameter-scan` benchmarks. In addition, decimal numbers\n are now allowed for parameter scans. For example, the following command runs\n `sleep 0.3`, `sleep 0.5` and `sleep 0.7`:\n ``` bash\n hyperfine --parameter-scan delay 0.3 0.7 -D 0.2 'sleep {delay}'\n ```\n For more details, see #184 (@piyushrungta25)\n\n## Other\n\n- hyperfine is now in the official Alpine repositories, see #177 (@maxice8, @5paceToast)\n- hyperfine is now in the official Fedora repositories, see #196 (@ignatenkobrain)\n- hyperfine is now in the official Arch Linux repositories\n- hyperfine can be installed on FreeBSD, see #204 (@0mp)\n- Enabled LTO for slightly smaller binary sizes, see #179 (@Calinou)\n- Various small improvements all over the code base, see #194 (@phimuemue)\n\n# v1.6.0\n\n## Features\n\n- Added a `-c, --cleanup ` option to execute `CMD` after the completion of all benchmarking runs for a given command. This is useful if the commands to be benchmarked produce artifacts that need to be cleaned up. See #91 (@RalfJung and @colinwahl)\n- Add parameter values (for `--parameter-scan` benchmarks) to exported CSV and JSON files. See #131 (@bbannier)\n- Added AsciiDoc export option, see #137 (@5paceToast)\n- The relative speed is now part of the Markdown export, see #127 (@mathiasrw and @sharkdp).\n- The *median* run time is now exported via CSV and JSON, see #171 (@hosewiejacke and @sharkdp).\n\n## Other\n\n- Hyperfine has been updated to Rust 2018 (@AnderEnder). The minimum supported Rust version is now 1.31.\n\n# v1.5.0\n\n## Features\n\n- Show the number of runs in `hyperfine`s output (@tcmal)\n- Added two Python scripts to post-process exported benchmark results (see [`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder)\n\n## Other\n\n- Refined `--help` text for the `--export-*` flags (@psteinb)\n- Added Snapcraft file (@popey)\n- Small improvements in the progress bar \"experience\".\n\n# v1.4.0\n\n## Features\n\n- Added `-S`/`--shell` option to override the default shell, see #61 (@mqudsi and @jasonpeacock)\n- Added `-u`/`--time-unit` option to change the unit of time (`second` or `millisecond`), see #80 (@jasonpeacock)\n- Markdown export auto-selects time unit, see #71 (@jasonpeacock)\n\n# v1.3.0\n\n## Feature\n\n- Compute and print standard deviation of the speed ratio, see #83 (@Shnatsel)\n- More compact output format, see #70 (@jasonpeacock)\n- Added `--style=color`, see #70 (@jasonpeacock)\n- Added options to specify the max/exact numbers of runs, see #77 (@orium)\n\n## Bugfixes\n\n- Change Windows `cmd` interpreter to `cmd.exe` to prevent accidentally calling other programs, see #74 (@tathanhdinh)\n\n## Other\n\n- Binary releases for Windows are now available, see #87\n\n# v1.2.0\n\n- Support parameters in preparation commands, see #68 (@siiptuo)\n- Updated dependencies, see #69. The minimum required Rust version is now 1.24.\n\n# v1.1.0\n\n* Added `--show-output` option (@chrisduerr and @sevagh)\n* Refactoring work (@stevepentland)\n\n# v1.0.0\n\n## Features\n\n* Support for various export-formats like CSV, JSON and Markdown - see #38, #44, #49, #42 (@stevepentland)\n* Summary output that compares the different benchmarks, see #6 (@stevepentland)\n* Parameterized benchmarks via `-P`, `--parameter-scan `, see #19\n\n## Thanks\n\nI'd like to say a big THANK YOU to @stevepentland for implementing new features,\nfor reviewing pull requests and for giving very valuable feedback.\n\n# v0.5.0\n\n* Proper Windows support (@stevepentland)\n* Added `--style auto/basic/nocolor/full` option (@stevepentland)\n* Correctly estimate the full execution time, see #27 (@rleungx)\n* Added Void Linux install instructions (@wpbirney)\n\n# v0.4.0\n\n- New `--style` option to disable output coloring and interactive CLI features, see #24 (@stevepentland)\n- Statistical outlier detection, see #23 #18\n\n# v0.3.0\n\n## Features\n\n- In addition to 'real' (wall clock) time, Hyperfine can now also measure 'user' and 'system' time (see #5).\n- Added `--prepare` option that can be used to clear up disk caches before timing runs, for example (see #8).\n\n## Other\n\n- [Arch Linux package](https://aur.archlinux.org/packages/hyperfine) for Hyperfine (@jD91mZM2).\n- Ubuntu/Debian packages are now are available.\n\n# v0.2.0\n\nInitial public release\n" }, { "path": "CITATION.cff", "content": "cff-version: 1.2.0\ntitle: hyperfine\nmessage: >-\n If you use this software in scientific\n publications, please consider citing it using the\n metadata from this file.\ntype: software\nauthors:\n - given-names: David\n family-names: Peter\n email: mail@david-peter.de\n orcid: 'https://orcid.org/0000-0001-7950-9915'\nrepository-code: 'https://github.com/sharkdp/hyperfine'\nabstract: A command-line benchmarking tool.\nlicense: MIT\nversion: 1.16.1\ndate-released: '2023-03-21'\n" }, { "path": "Cargo.toml", "content": "[package]\nauthors = [\"David Peter \"]\ncategories = [\"command-line-utilities\"]\ndescription = \"A command-line benchmarking tool\"\nhomepage = \"https://github.com/sharkdp/hyperfine\"\nlicense = \"MIT OR Apache-2.0\"\nname = \"hyperfine\"\nreadme = \"README.md\"\nrepository = \"https://github.com/sharkdp/hyperfine\"\nversion = \"1.20.0\"\nedition = \"2018\"\nbuild = \"build.rs\"\nrust-version = \"1.88.0\"\n\n[features]\n# Use the nightly feature windows_process_extensions_main_thread_handle\nwindows_process_extensions_main_thread_handle = []\n\n[dependencies]\ncolored = \"2.1\"\nindicatif = \"=0.17.4\"\nstatistical = \"1.0\"\ncsv = \"1.3\"\nserde = { version = \"1.0\", features = [\"derive\"] }\nserde_json = \"1.0\"\nrust_decimal = \"1.36\"\nrand = \"0.8\"\nshell-words = \"1.0\"\nthiserror = \"2.0\"\nanyhow = \"1.0\"\n\n[target.'cfg(not(windows))'.dependencies]\nlibc = \"0.2\"\n\n[target.'cfg(windows)'.dependencies]\nwindows-sys = { version = \"0.59\", features = [\n \"Win32_Foundation\",\n \"Win32_Security\",\n \"Win32_System_JobObjects\",\n \"Win32_System_LibraryLoader\",\n \"Win32_System_Threading\",\n] }\n\n[target.'cfg(all(windows, not(windows_process_extensions_main_thread_handle)))'.dependencies]\nonce_cell = \"1.19\"\n\n[target.'cfg(target_os=\"linux\")'.dependencies]\nnix = { version = \"0.29\", features = [\"zerocopy\"] }\n\n[dependencies.clap]\nversion = \"4\"\ndefault-features = false\nfeatures = [\n \"suggestions\",\n \"color\",\n \"wrap_help\",\n \"cargo\",\n \"help\",\n \"usage\",\n \"error-context\",\n]\n\n[dev-dependencies]\napprox = \"0.5\"\nassert_cmd = \"2.0\"\ninsta = { version = \"1.41.1\", features = [\"yaml\"] }\npredicates = \"3.1\"\ntempfile = \"3.23\"\n\n[profile.dev.package]\ninsta.opt-level = 3\nsimilar.opt-level = 3\n\n[build-dependencies]\nclap = \"4.5.48\"\nclap_complete = \"4.2.1\"\n\n[profile.release]\nlto = true\nstrip = true\ncodegen-units = 1\n" }, { "path": "LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "LICENSE-MIT", "content": "MIT License\n\nCopyright (c) 2018-2022 David Peter, and all hyperfine contributors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "# hyperfine\n[![CICD](https://github.com/sharkdp/hyperfine/actions/workflows/CICD.yml/badge.svg)](https://github.com/sharkdp/hyperfine/actions/workflows/CICD.yml)\n[![Version info](https://img.shields.io/crates/v/hyperfine.svg)](https://crates.io/crates/hyperfine)\n[中文](https://github.com/chinanf-boy/hyperfine-zh)\n\nA command-line benchmarking tool.\n\n**Demo**: Benchmarking [`fd`](https://github.com/sharkdp/fd) and\n[`find`](https://www.gnu.org/software/findutils/):\n\n![hyperfine](https://i.imgur.com/z19OYxE.gif)\n\n### Sponsors\n\nA special *thank you* goes to our biggest sponsor:
\n\n\n \"Warp\"\n
\n Warp, the intelligent terminal\n
\n Available on MacOS, Linux, Windows\n\n\n## Features\n\n* Statistical analysis across multiple runs.\n* Support for arbitrary shell commands.\n* Constant feedback about the benchmark progress and current estimates.\n* Warmup runs can be executed before the actual benchmark.\n* Cache-clearing commands can be set up before each timing run.\n* Statistical outlier detection to detect interference from other programs and caching effects.\n* Export results to various formats: CSV, JSON, Markdown, AsciiDoc.\n* Parameterized benchmarks (e.g. vary the number of threads).\n* Cross-platform\n\n## Usage\n\n### Basic benchmarks\n\nTo run a benchmark, you can simply call `hyperfine ...`. The argument(s) can be any\nshell command. For example:\n```sh\nhyperfine 'sleep 0.3'\n```\n\nHyperfine will automatically determine the number of runs to perform for each command. By default,\nit will perform *at least* 10 benchmarking runs and measure for at least 3 seconds. To change this,\nyou can use the `-r`/`--runs` option:\n```sh\nhyperfine --runs 5 'sleep 0.3'\n```\n\nIf you want to compare the runtimes of different programs, you can pass multiple commands:\n```sh\nhyperfine 'hexdump file' 'xxd file'\n```\n\n### Warmup runs and preparation commands\n\nFor programs that perform a lot of disk I/O, the benchmarking results can be heavily influenced\nby disk caches and whether they are cold or warm.\n\nIf you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to\nperform a certain number of program executions before the actual benchmark:\n```sh\nhyperfine --warmup 3 'grep -R TODO *'\n```\n\nConversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare`\noption to run a special command before *each* timing run. For example, to clear harddisk caches\non Linux, you can run\n```sh\nsync; echo 3 | sudo tee /proc/sys/vm/drop_caches\n```\nTo use this specific command with hyperfine, call `sudo -v` to temporarily gain sudo permissions\nand then call:\n```sh\nhyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R TODO *'\n```\n\n### Parameterized benchmarks\n\nIf you want to run a series of benchmarks where a single parameter is varied (say, the number of\nthreads), you can use the `-P`/`--parameter-scan` option and call:\n```sh\nhyperfine --prepare 'make clean' --parameter-scan num_threads 1 12 'make -j {num_threads}'\n```\nThis also works with decimal numbers. The `-D`/`--parameter-step-size` option can be used\nto control the step size:\n```sh\nhyperfine --parameter-scan delay 0.3 0.7 -D 0.2 'sleep {delay}'\n```\nThis runs `sleep 0.3`, `sleep 0.5` and `sleep 0.7`.\n\nFor non-numeric parameters, you can also supply a list of values with the `-L`/`--parameter-list`\noption:\n```\nhyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'\n```\n\n### Intermediate shell\n\nBy default, commands are executed using a predefined shell (`/bin/sh` on Unix, `cmd.exe` on Windows).\nIf you want to use a different shell, you can use the `-S, --shell ` option:\n```sh\nhyperfine --shell zsh 'for i in {1..10000}; do echo test; done'\n```\n\nNote that hyperfine always *corrects for the shell spawning time*. To do this, it performs a calibration\nprocedure where it runs the shell with an empty command (multiple times), to measure the startup time\nof the shell. It will then subtract this time from the total to show the actual time used by the command\nin question.\n\nIf you want to run a benchmark *without an intermediate shell*, you can use the `-N` or `--shell=none`\noption. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would\nproduce a significant amount of noise. Note that you cannot use shell syntax like `*` or `~` in this case.\n```\nhyperfine -N 'grep TODO /home/user'\n```\n\n\n### Shell functions and aliases\n\nIf you are using bash, you can export shell functions to directly benchmark them with hyperfine:\n\n```bash\nmy_function() { sleep 1; }\nexport -f my_function\nhyperfine --shell=bash my_function\n```\n\nOtherwise, inline them into or source them from the benchmarked program:\n\n```sh\nhyperfine 'my_function() { sleep 1; }; my_function'\n\necho 'alias my_alias=\"sleep 1\"' > /tmp/my_alias.sh\nhyperfine '. /tmp/my_alias.sh; my_alias'\n```\n\n### Exporting results\n\nHyperfine has multiple options for exporting benchmark results to CSV, JSON, Markdown and other\nformats (see `--help` text for details).\n\n#### Markdown\n\nYou can use the `--export-markdown ` option to create tables like the following:\n\n| Command | Mean [s] | Min [s] | Max [s] | Relative |\n|:---|---:|---:|---:|---:|\n| `find . -iregex '.*[0-9]\\.jpg$'` | 2.275 ± 0.046 | 2.243 | 2.397 | 9.79 ± 0.22 |\n| `find . -iname '*[0-9].jpg'` | 1.427 ± 0.026 | 1.405 | 1.468 | 6.14 ± 0.13 |\n| `fd -HI '.*[0-9]\\.jpg$'` | 0.232 ± 0.002 | 0.230 | 0.236 | 1.00 |\n\n#### JSON\n\nThe JSON output is useful if you want to analyze the benchmark results in more detail. The\n[`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder includes a lot\nof helpful Python programs to further analyze benchmark results and create helpful\nvisualizations, like a histogram of runtimes or a whisker plot to compare\nmultiple benchmarks:\n\n| ![](doc/histogram.png) | ![](doc/whisker.png) |\n|---:|---:|\n\n\n### Detailed benchmark flowchart\n\nThe following chart explains the execution order of various timing runs when using options\nlike `--warmup`, `--prepare `, `--setup ` or `--cleanup `:\n\n![](doc/execution-order.png)\n\n## Installation\n\n[![Packaging status](https://repology.org/badge/vertical-allrepos/hyperfine.svg?columns=3&exclude_unsupported=1)](https://repology.org/project/hyperfine/versions)\n\n### On Ubuntu\n\nOn Ubuntu, hyperfine can be installed [from the official repositories](https://launchpad.net/ubuntu/+source/rust-hyperfine):\n```\napt install hyperfine\n```\n\nAlternatively, for the latest version, you can download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases) and install it via `dpkg`:\n```\nwget https://github.com/sharkdp/hyperfine/releases/download/v1.20.0/hyperfine_1.20.0_amd64.deb\nsudo dpkg -i hyperfine_1.20.0_amd64.deb\n```\n\n### On Fedora\n\nOn Fedora, hyperfine can be installed from the official repositories:\n\n```sh\ndnf install hyperfine\n```\n\n### On Alpine Linux\n\nOn Alpine Linux, hyperfine can be installed [from the official repositories](https://pkgs.alpinelinux.org/packages?name=hyperfine):\n```\napk add hyperfine\n```\n\n### On Arch Linux\n\nOn Arch Linux, hyperfine can be installed [from the official repositories](https://archlinux.org/packages/extra/x86_64/hyperfine/):\n```\npacman -S hyperfine\n```\n\n### On Debian Linux\n\nOn Debian Linux, hyperfine can be installed [from the official repositories](https://packages.debian.org/hyperfine):\n```\napt install hyperfine\n```\n\n### On Exherbo Linux\n\nOn Exherbo Linux, hyperfine can be installed [from the rust repositories](https://gitlab.exherbo.org/exherbo/rust/-/tree/master/packages/sys-apps/hyperfine):\n```\ncave resolve -x repository/rust\ncave resolve -x hyperfine\n```\n\n### On Funtoo Linux\n\nOn Funtoo Linux, hyperfine can be installed [from core-kit](https://github.com/funtoo/core-kit/tree/1.4-release/app-benchmarks/hyperfine):\n```\nemerge app-benchmarks/hyperfine\n```\n\n### On NixOS\n\nOn NixOS, hyperfine can be installed [from the official repositories](https://nixos.org/nixos/packages.html?query=hyperfine):\n```\nnix-env -i hyperfine\n```\n\n### On Flox\n\nOn Flox, hyperfine can be installed as follows.\n```\nflox install hyperfine\n```\nHyperfine's version in Flox follows that of Nix.\n\n### On openSUSE\n\nOn openSUSE, hyperfine can be installed [from the official repositories](https://software.opensuse.org/package/hyperfine):\n```\nzypper install hyperfine\n```\n\n### On Void Linux\n\nHyperfine can be installed via xbps\n\n```\nxbps-install -S hyperfine\n```\n\n### On macOS\n\nHyperfine can be installed via [Homebrew](https://brew.sh):\n```\nbrew install hyperfine\n```\n\nOr you can install using [MacPorts](https://www.macports.org):\n```\nsudo port selfupdate\nsudo port install hyperfine\n```\n\n### On FreeBSD\n\nHyperfine can be installed via pkg:\n```\npkg install hyperfine\n```\n\n### On OpenBSD\n\n```\ndoas pkg_add hyperfine\n```\n\n### On Windows\n\nHyperfine can be installed via [Chocolatey](https://community.chocolatey.org/packages/hyperfine), [Scoop](https://scoop.sh/#/apps?q=hyperfine&s=0&d=1&o=true&id=8f7c10f75ecf5f9e42a862c615257328e2f70f61), or [Winget](https://github.com/microsoft/winget-pkgs/tree/master/manifests/s/sharkdp/hyperfine):\n```\nchoco install hyperfine\n```\n```\nscoop install hyperfine\n```\n```\nwinget install hyperfine\n```\n\n### With conda\n\nHyperfine can be installed via [`conda`](https://conda.io/en/latest/) from the [`conda-forge`](https://anaconda.org/conda-forge/hyperfine) channel:\n```\nconda install -c conda-forge hyperfine\n```\n\n### With cargo (Linux, macOS, Windows)\n\nHyperfine can be installed from source via [cargo](https://doc.rust-lang.org/cargo/):\n```\ncargo install --locked hyperfine\n```\n\nMake sure that you use Rust 1.76 or newer.\n\n### From binaries (Linux, macOS, Windows)\n\nDownload the corresponding archive from the [Release page](https://github.com/sharkdp/hyperfine/releases).\n\n## Alternative tools\n\nHyperfine is inspired by [bench](https://github.com/Gabriella439/bench).\n\n## Integration with other tools\n\n[Chronologer](https://github.com/dandavison/chronologer) is a tool that uses `hyperfine` to\nvisualize changes in benchmark timings across your Git history.\n\n[Bencher](https://github.com/bencherdev/bencher) is a continuous benchmarking tool that supports `hyperfine` to\ntrack benchmarks and catch performance regressions in CI.\n\nDrop hyperfine JSON outputs onto the [Venz](https://try.venz.dev) chart to visualize the results,\nand manage hyperfine configurations.\n\nMake sure to check out the [`scripts` folder](https://github.com/sharkdp/hyperfine/tree/master/scripts)\nin this repository for a set of tools to work with `hyperfine` benchmark results.\n\n## Origin of the name\n\nThe name *hyperfine* was chosen in reference to the hyperfine levels of caesium 133 which play a crucial role in the\n[definition of our base unit of time](https://en.wikipedia.org/wiki/Second#History_of_definition)\n— the second.\n\n## Citing hyperfine\n\nThank you for considering to cite hyperfine in your research work. Please see the information\nin the sidebar on how to properly cite hyperfine.\n\n## License\n\n`hyperfine` is dual-licensed under the terms of the MIT License and the Apache License 2.0.\n\nSee the [LICENSE-APACHE](LICENSE-APACHE) and [LICENSE-MIT](LICENSE-MIT) files for details.\n" }, { "path": "build.rs", "content": "use std::fs;\n\nuse clap_complete::{generate_to, Shell};\n\ninclude!(\"src/cli.rs\");\n\nfn main() {\n let var = std::env::var_os(\"SHELL_COMPLETIONS_DIR\").or_else(|| std::env::var_os(\"OUT_DIR\"));\n let outdir = match var {\n None => return,\n Some(outdir) => outdir,\n };\n fs::create_dir_all(&outdir).unwrap();\n\n let mut command = build_command();\n for shell in [\n Shell::Bash,\n Shell::Fish,\n Shell::Zsh,\n Shell::PowerShell,\n Shell::Elvish,\n ] {\n generate_to(shell, &mut command, \"hyperfine\", &outdir).unwrap();\n }\n}\n" }, { "path": "doc/hyperfine.1", "content": ".TH HYPERFINE 1\n.SH NAME\nhyperfine \\- command\\-line benchmarking tool\n.SH SYNOPSIS\n.B hyperfine\n.RB [ \\-ihVN ]\n.RB [ \\-\\-warmup\n.IR NUM ]\n.RB [ \\-\\-min\\-runs\n.IR NUM ]\n.RB [ \\-\\-max\\-runs\n.IR NUM ]\n.RB [ \\-\\-runs\n.IR NUM ]\n.RB [ \\-\\-setup\n.IR CMD ]\n.RB [ \\-\\-prepare\n.IR CMD ]\n.RB [ \\-\\-conclude\n.IR CMD ]\n.RB [ \\-\\-cleanup\n.IR CMD ]\n.RB [ \\-\\-parameter\\-scan\n.IR VAR\n.IR MIN\n.IR MAX ]\n.RB [ \\-\\-parameter\\-step\\-size\n.IR DELTA ]\n.RB [ \\-\\-parameter\\-list\n.IR VAR\n.IR VALUES ]\n.RB [ \\-\\-shell\n.IR SHELL ]\n.RB [ \\-\\-style\n.IR TYPE ]\n.RB [ \\-\\-sort\n.IR METHOD ]\n.RB [ \\-\\-time-unit\n.IR UNIT ]\n.RB [ \\-\\-export\\-asciidoc\n.IR FILE ]\n.RB [ \\-\\-export\\-csv\n.IR FILE ]\n.RB [ \\-\\-export\\-json\n.IR FILE ]\n.RB [ \\-\\-export\\-markdown\n.IR FILE ]\n.RB [ \\-\\-export\\-orgmode\n.IR FILE ]\n.RB [ \\-\\-output\n.IR WHERE ]\n.RB [ \\-\\-input\n.IR WHERE ]\n.RB [ \\-\\-command\\-name\n.IR NAME ]\n.RI [ COMMAND... ]\n.SH DESCRIPTION\nA command\\-line benchmarking tool which includes:\n.LP\n.RS\n* Statistical analysis across multiple runs\n.RE\n.RS\n* Support for arbitrary shell commands\n.RE\n.RS\n* Constant feedback about the benchmark progress and current estimates\n.RE\n.RS\n* Warmup runs can be executed before the actual benchmark\n.RE\n.RS\n* Cache-clearing commands can be set up before each timing run\n.RE\n.RS\n* Statistical outlier detection to detect interference from other programs and caching effects\n.RE\n.RS\n* Export results to various formats: CSV, JSON, Markdown, AsciiDoc\n.RE\n.RS\n* Parameterized benchmarks (e.g. vary the number of threads)\n.RE\n.SH OPTIONS\n.HP\n\\fB\\-w\\fR, \\fB\\-\\-warmup\\fR \\fINUM\\fP\n.IP\nPerform \\fINUM\\fP warmup runs before the actual benchmark. This can be used\nto fill (disk) caches for I/O\\-heavy programs.\n.HP\n\\fB\\-m\\fR, \\fB\\-\\-min\\-runs\\fR \\fINUM\\fP\n.IP\nPerform at least \\fINUM\\fP runs for each command. Default: 10.\n.HP\n\\fB\\-M\\fR, \\fB\\-\\-max\\-runs\\fR \\fINUM\\fP\n.IP\nPerform at most \\fINUM\\fP runs for each command. By default, there is no\nlimit.\n.HP\n\\fB\\-r\\fR, \\fB\\-\\-runs\\fR \\fINUM\\fP\n.IP\nPerform exactly \\fINUM\\fP runs for each command. If this option is not specified,\n\\fBhyperfine\\fR automatically determines the number of runs.\n.HP\n\\fB\\-s\\fR, \\fB\\-\\-setup\\fR \\fICMD...\\fP\n.IP\nExecute \\fICMD\\fP once before each set of timing runs. This is useful\nfor compiling your software or with the provided parameters, or to do any\nother work that should happen once before a series of benchmark runs,\nnot every time as would happen with the \\fB\\-\\-prepare\\fR option.\n.HP\n\\fB\\-p\\fR, \\fB\\-\\-prepare\\fR \\fICMD...\\fP\n.IP\nExecute \\fICMD\\fP before each timing run. This is useful for clearing disk caches,\nfor example.\nThe \\fB\\-\\-prepare\\fR option can be specified once for all commands or multiple times,\nonce for each command. In the latter case, each preparation command will be\nrun prior to the corresponding benchmark command.\n.HP\n\\fB\\-\\-conclude\\fR \\fICMD...\\fP\n.IP\nExecute \\fICMD\\fP after each timing run. This is useful for clearing disk caches,\nfor example.\nThe \\fB\\-\\-conclude\\fR option can be specified once for all commands or multiple times,\nonce for each command. In the latter case, each conclusion command will be\nrun after the corresponding benchmark command.\n.HP\n\\fB\\-c\\fR, \\fB\\-\\-cleanup\\fR \\fICMD...\\fP\n.IP\nExecute \\fICMD\\fP after the completion of all benchmarking runs for each individual\ncommand to be benchmarked. This is useful if the commands to be benchmarked\nproduce artifacts that need to be cleaned up. It only runs once a series of\nbenchmark runs, as opposed to \\fB\\-\\-conclude\\fR option which runs after\nevery run.\n.HP\n\\fB\\-P\\fR, \\fB\\-\\-parameter\\-scan\\fR \\fIVAR\\fP \\fIMIN\\fP \\fIMAX\\fP\n.IP\nPerform benchmark runs for each value in the range \\fIMIN..MAX\\fP. Replaces the\nstring '{\\fIVAR\\fP}' in each command by the current parameter value.\n.IP\n.RS\nExample:\n.RS\n\\fBhyperfine\\fR \\fB\\-P\\fR threads 1 8 'make \\-j {threads}'\n.RE\n.RE\n.IP\nThis performs benchmarks for 'make \\-j 1', 'make \\-j 2', ..., 'make \\-j 8'.\n.IP\nTo have the value increase following different patterns, use shell\narithmetics.\n.IP\n.RS\nExample:\n.RS\n\\fBhyperfine\\fR \\fB\\-P\\fR size 0 3 'sleep $((2**{size}))'\n.RE\n.RE\n.IP\nThis performs benchmarks with power of 2 increases: 'sleep 1', 'sleep\n2', 'sleep 4', ...\n.IP\nThe exact syntax may vary depending on your shell and OS.\n.HP\n\\fB\\-D\\fR, \\fB\\-\\-parameter\\-step\\-size\\fR \\fIDELTA\\fP\n.IP\nThis argument requires \\fB\\-\\-parameter\\-scan\\fR to be specified as well. Traverse the\nrange \\fIMIN..MAX\\fP in steps of \\fIDELTA\\fP.\n.IP\n.RS\nExample:\n.RS\n\\fBhyperfine\\fR \\fB\\-P\\fR delay 0.3 0.7 \\fB\\-D\\fR 0.2 'sleep {delay}'\n.RE\n.RE\n.IP\nThis performs benchmarks for 'sleep 0.3', 'sleep 0.5' and 'sleep 0.7'.\n.HP\n\\fB\\-L\\fR, \\fB\\-\\-parameter\\-list\\fR \\fIVAR\\fP \\fIVALUES\\fP\n.IP\nPerform benchmark runs for each value in the comma\\-separated list of \\fIVALUES\\fP.\nReplaces the string '{\\fIVAR\\fP}' in each command by the current parameter value.\n.IP\n.RS\nExample:\n.RS\n\\fBhyperfine\\fR \\fB\\-L\\fR compiler gcc,clang '{compiler} \\-O2 main.cpp'\n.RE\n.RE\n.IP\nThis performs benchmarks for 'gcc \\-O2 main.cpp' and 'clang \\-O2 main.cpp'.\n.IP\nThe option can be specified multiple times to run benchmarks for all\npossible parameter combinations.\n.HP\n\\fB\\-S\\fR, \\fB\\-\\-shell\\fR \\fISHELL\\fP\n.IP\nSet the shell to use for executing benchmarked commands. This can be\nthe name or the path to the shell executable, or a full command line\nlike \"bash \\fB\\-\\-norc\\fR\". It can also be set to \"default\" to explicitly\nselect the default shell on this platform. Finally, this can also be\nset to \"none\" to disable the shell. In this case, commands will be\nexecuted directly. They can still have arguments, but more complex\nthings like \"sleep 0.1; sleep 0.2\" are not possible without a shell.\n.HP\n\\fB\\-N\\fR\n.IP\nAn alias for '\\-\\-shell=none'.\n.HP\n\\fB\\-i\\fR, \\fB\\-\\-ignore\\-failure\\fR\n.IP\nIgnore non\\-zero exit codes of the benchmarked programs.\n.HP\n\\fB\\-\\-style\\fR \\fITYPE\\fP\n.IP\nSet output style \\fITYPE\\fP (default: auto). Set this to 'basic' to disable output\ncoloring and interactive elements. Set it to 'full' to enable all effects even\nif no interactive terminal was detected. Set this to 'nocolor' to keep the\ninteractive output without any colors. Set this to 'color' to keep the colors\nwithout any interactive output. Set this to 'none' to disable all the output\nof the tool.\n.HP\n\\fB\\-\\-sort\\fR \\fIMETHOD\\fP\n.IP\nSpecify the sort order of the speed comparison summary and the\nexported tables for markup formats (Markdown, AsciiDoc, org\\-mode):\n.RS\n.IP \"auto (default)\"\nthe speed comparison will be ordered by time and\nthe markup tables will be ordered by command (input order).\n.IP \"command\"\norder benchmarks in the way they were specified\n.IP \"mean\\-time\"\norder benchmarks by mean runtime\n.RE\n.HP\n\\fB\\-u\\fR, \\fB\\-\\-time\\-unit\\fR \\fIUNIT\\fP\n.IP\nSet the time unit to be used. Possible values: microsecond, millisecond, second. If\nthe option is not given, the time unit is determined automatically.\nThis option affects the standard output as well as all export formats\nexcept for CSV and JSON.\n.HP\n\\fB\\-\\-export\\-asciidoc\\fR \\fIFILE\\fP \n.IP\nExport the timing summary statistics as an AsciiDoc table to the given \\fIFILE\\fP.\nThe output time unit can be changed using the \\fB\\-\\-time\\-unit\\fR option.\n.HP\n\\fB\\-\\-export\\-csv\\fR \\fIFILE\\fP\n.IP\nExport the timing summary statistics as CSV to the given \\fIFILE\\fP. If you need the\ntiming results for each individual run, use the JSON export format.\nThe output time unit is always seconds.\n.HP\n\\fB\\-\\-export\\-json\\fR \\fIFILE\\fP\n.IP\nExport the timing summary statistics and timings of individual runs as JSON to\nthe given \\fIFILE\\fP. The output time unit is always seconds.\n.HP\n\\fB\\-\\-export\\-markdown\\fR \\fIFILE\\fP\n.IP\nExport the timing summary statistics as a Markdown table to the given \\fIFILE\\fP.\nThe output time unit can be changed using the \\fB\\-\\-time\\-unit\\fR option.\n.HP\n\\fB\\-\\-export\\-orgmode\\fR \\fIFILE\\fP\n.IP\nExport the timing summary statistics as an Emacs org\\-mode table to the\ngiven \\fIFILE\\fP. The output time unit can be changed using the \\fB\\-\\-time\\-unit\\fR option.\n.HP\n\\fB\\-\\-show\\-output\\fR\n.IP\nPrint the stdout and stderr of the benchmark instead of suppressing it. This\nwill increase the time it takes for benchmarks to run, so it should only be\nused for debugging purposes or when trying to benchmark output speed.\n.HP\n\\fB\\-\\-output\\fR \\fIWHERE\\fP\n.IP\nControl where the output of the benchmark is redirected. Note that\nsome programs like 'grep' detect when standard output is \\fI\\,/dev/null\\/\\fP and\napply certain optimizations. To avoid that, consider using\n\\-\\-output=pipe.\n.IP\n\\fIWHERE\\fP can be:\n.RS\n.IP null\nRedirect output to \\fI\\,/dev/null\\/\\fP (the default).\n.IP pipe\nFeed the output through a pipe before discarding it.\n.IP inherit\nDon't redirect the output at all (same as \\&'\\-\\-show\\-output').\n.IP \"\"\nWrite the output to the given file.\n.RE\n.IP\nThis option can be specified once for all commands or multiple times,\nonce for each command. Note: If you want to log the output of each and\nevery iteration, you can use a shell redirection and the $HYPERFINE_ITERATION\nenvironment variable: 'my-command > output-${HYPERFINE_ITERATION}.log'\n.HP\n\\fB\\-\\-input\\fR \\fIWHERE\\fP\n.IP\nControl where the input of the benchmark comes from.\n.IP\n\\fIWHERE\\fP can be:\n.RS\n.IP null\nRead from \\fI\\,/dev/null\\/\\fP (the default).\n.IP \"\"\nRead the input from the given file.\n.RE\n.HP\n\\fB\\-n\\fR, \\fB\\-\\-command\\-name\\fR \\fiNAME\\fP\n.IP\nGive a meaningful \\fiNAME\\fP to a command. This can be specified multiple times\nif several commands are benchmarked.\n.HP\n\\fB\\-h\\fR, \\fB\\-\\-help\\fR\n.IP\nPrint help\n.HP\n\\fB\\-V\\fR, \\fB\\-\\-version\\fR\n.IP\nPrint version\n.SH EXAMPLES\n.LP\nBasic benchmark of 'find . -name todo.txt':\n.RS\n.nf\n\\fBhyperfine\\fR 'find . -name todo.txt'\n.fi\n.RE\n.LP\nPerform benchmarks for 'sleep 0.2' and 'sleep 3.2' with a minimum 5 runs each:\n.RS\n.nf\n\\fBhyperfine\\fR \\fB\\-\\-min\\-runs\\fR 5 'sleep 0.2' 'sleep 3.2'\n.fi\n.RE\n.LP\nPerform a benchmark of 'grep' with a warm disk cache by executing 3 runs up front that are not part\nof the measurement: \n.RS\n.nf\n\\fBhyperfine\\fR \\fB\\-\\-warmup\\fR 3 'grep -R TODO *'\n.fi\n.RE\n.LP\nExport the results of a parameter scan benchmark to a markdown table: \n.RS\n.nf\n\\fBhyperfine\\fR \\fB\\-\\-export\\-markdown\\fR output.md \\fB\\-\\-parameter-scan\\fR time 1 5 'sleep {time}'\n.fi\n.RE\n.LP\nDemonstrate when each of \\fB\\-\\-setup\\fR, \\fB\\-\\-prepare\\fR, \\fB\\-\\-conclude\\fR, \\fIcmd\\fP and \\fB\\-\\-cleanup\\fR will run:\n.RS\n.nf\n\\fBhyperfine\\fR \\fB\\-L\\fR n 1,2 \\fB\\-r\\fR 2 \\fB\\-\\-show-output\\fR \\\\\n\t\\fB\\-\\-setup\\fR 'echo setup n={n}' \\\\\n\t\\fB\\-\\-prepare\\fR 'echo prepare={n}' \\\\\n\t\\fB\\-\\-conclude\\fR 'echo conclude={n}' \\\\\n\t\\fB\\-\\-cleanup\\fR 'echo cleanup n={n}' \\\\\n\t'echo command n={n}'\n.fi\n.RE\n.RE\n.SH AUTHOR\n.LP\nDavid Peter \n.LP\nSource, bug tracker, and additional information can be found on GitHub:\n.I https://github.com/sharkdp/hyperfine\n" }, { "path": "doc/sponsors.md", "content": "## Sponsors\n\n`hyperfine` development is sponsored by many individuals and companies. Thank you very much!\n\nPlease note, that being sponsored does not affect the individuality of the `hyperfine`\nproject or affect the maintainers' actions in any way.\nWe remain impartial and continue to assess pull requests solely on merit - the\nfeatures added, bugs solved, and effect on the overall complexity of the code.\nNo issue will have a different priority based on sponsorship status of the\nreporter.\n\nContributions from anybody are most welcomed.\n\nIf you want to see our biggest sponsors, check the top of [`README.md`](../README.md#sponsors).\n" }, { "path": "scripts/README.md", "content": "This folder contains scripts that can be used in combination with hyperfines `--export-json` option.\n\n### Example:\n\n```bash\nhyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json\n./plot_whisker.py sleep.json\n```\n\n### Pre-requisites\n\nTo make these scripts work, you will need `numpy`, `matplotlib` and `scipy`.\n\nIf you have a Python package manager that understands [PEP-723](https://peps.python.org/pep-0723/)\ninline script requirements like [`uv`](https://github.com/astral-sh/uv) or [`pipx`](https://github.com/pypa/pipx),\nyou can directly run the scripts using\n\n```bash\nuv run plot_whisker.py sleep.json\n```\n\nOtherwise, install the dependencies via your system package manager or using `pip`:\n\n```bash\npip install numpy matplotlib scipy # pip3, if you are using python3\n```\n" }, { "path": "scripts/advanced_statistics.py", "content": "#!/usr/bin/env python\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"numpy\",\n# ]\n# ///\n\nimport argparse\nimport json\nfrom enum import Enum\n\nimport numpy as np\n\n\nclass Unit(Enum):\n SECOND = 1\n MILLISECOND = 2\n\n def factor(self):\n match self:\n case Unit.SECOND:\n return 1\n case Unit.MILLISECOND:\n return 1e3\n\n def __str__(self):\n match self:\n case Unit.SECOND:\n return \"s\"\n case Unit.MILLISECOND:\n return \"ms\"\n\n\nparser = argparse.ArgumentParser()\nparser.add_argument(\"file\", help=\"JSON file with benchmark results\")\nparser.add_argument(\n \"--time-unit\",\n help=\"The unit of time.\",\n default=\"second\",\n action=\"store\",\n choices=[\"second\", \"millisecond\"],\n dest=\"unit\",\n)\nargs = parser.parse_args()\n\nunit = Unit.MILLISECOND if args.unit == \"millisecond\" else Unit.SECOND\nunit_str = str(unit)\n\nwith open(args.file) as f:\n results = json.load(f)[\"results\"]\n\ncommands = [b[\"command\"] for b in results]\ntimes = [b[\"times\"] for b in results]\n\nfor command, ts in zip(commands, times):\n ts = [t * unit.factor() for t in ts]\n\n p05 = np.percentile(ts, 5)\n p25 = np.percentile(ts, 25)\n p75 = np.percentile(ts, 75)\n p95 = np.percentile(ts, 95)\n\n iqr = p75 - p25\n\n print(f\"Command '{command}'\")\n print(f\" runs: {len(ts):8d}\")\n print(f\" mean: {np.mean(ts):8.3f} {unit_str}\")\n print(f\" stddev: {np.std(ts, ddof=1):8.3f} {unit_str}\")\n print(f\" median: {np.median(ts):8.3f} {unit_str}\")\n print(f\" min: {np.min(ts):8.3f} {unit_str}\")\n print(f\" max: {np.max(ts):8.3f} {unit_str}\")\n print()\n print(\" percentiles:\")\n print(f\" P_05 .. P_95: {p05:.3f} {unit_str} .. {p95:.3f} {unit_str}\")\n print(\n f\" P_25 .. P_75: {p25:.3f} {unit_str} .. {p75:.3f} {unit_str} (IQR = {iqr:.3f} {unit_str})\"\n )\n print()\n" }, { "path": "scripts/plot_benchmark_comparison.py", "content": "#!/usr/bin/env python\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"matplotlib\",\n# \"pyqt6\",\n# \"numpy\",\n# ]\n# ///\n\n\"\"\"\nThis script shows `hyperfine` benchmark results as a bar plot grouped by command.\nNote all the input files must contain results for all commands.\n\"\"\"\n\nimport argparse\nimport json\nimport pathlib\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nparser = argparse.ArgumentParser(description=__doc__)\nparser.add_argument(\n \"files\", nargs=\"+\", type=pathlib.Path, help=\"JSON files with benchmark results\"\n)\nparser.add_argument(\"--title\", help=\"Plot Title\")\nparser.add_argument(\n \"--benchmark-names\", nargs=\"+\", help=\"Names of the benchmark groups\"\n)\nparser.add_argument(\"-o\", \"--output\", help=\"Save image to the given filename\")\n\nargs = parser.parse_args()\n\ncommands = None\ndata = []\ninputs = []\n\nif args.benchmark_names:\n assert len(args.files) == len(\n args.benchmark_names\n ), \"Number of benchmark names must match the number of input files.\"\n\nfor i, filename in enumerate(args.files):\n with open(filename) as f:\n results = json.load(f)[\"results\"]\n benchmark_commands = [b[\"command\"] for b in results]\n if commands is None:\n commands = benchmark_commands\n else:\n assert (\n commands == benchmark_commands\n ), f\"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}\"\n data.append([round(b[\"mean\"], 2) for b in results])\n if args.benchmark_names:\n inputs.append(args.benchmark_names[i])\n else:\n inputs.append(filename.stem)\n\ndata = np.transpose(data)\nx = np.arange(len(inputs)) # the label locations\nwidth = 0.25 # the width of the bars\n\nfig, ax = plt.subplots(layout=\"constrained\")\nfig.set_figheight(5)\nfig.set_figwidth(10)\nfor i, command in enumerate(commands):\n offset = width * (i + 1)\n rects = ax.bar(x + offset, data[i], width, label=command)\n\nax.set_xticks(x + 0.5, inputs)\nax.grid(visible=True, axis=\"y\")\n\nif args.title:\n plt.title(args.title)\nplt.xlabel(\"Benchmark\")\nplt.ylabel(\"Time [s]\")\nplt.legend(title=\"Command\")\n\nif args.output:\n plt.savefig(args.output)\nelse:\n plt.show()\n" }, { "path": "scripts/plot_histogram.py", "content": "#!/usr/bin/env python\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"matplotlib\",\n# \"pyqt6\",\n# \"numpy\",\n# ]\n# ///\n\n\"\"\"This program shows `hyperfine` benchmark results as a histogram.\"\"\"\n\nimport argparse\nimport json\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nparser = argparse.ArgumentParser(description=__doc__)\nparser.add_argument(\"file\", help=\"JSON file with benchmark results\")\nparser.add_argument(\"--title\", help=\"Plot title\")\nparser.add_argument(\n \"--labels\", help=\"Comma-separated list of entries for the plot legend\"\n)\nparser.add_argument(\"--bins\", help=\"Number of bins (default: auto)\")\nparser.add_argument(\n \"--legend-location\",\n help=\"Location of the legend on plot (default: upper center)\",\n choices=[\n \"upper center\",\n \"lower center\",\n \"right\",\n \"left\",\n \"best\",\n \"upper left\",\n \"upper right\",\n \"lower left\",\n \"lower right\",\n \"center left\",\n \"center right\",\n \"center\",\n ],\n default=\"upper center\",\n)\nparser.add_argument(\n \"--type\", help=\"Type of histogram (*bar*, barstacked, step, stepfilled)\"\n)\nparser.add_argument(\"-o\", \"--output\", help=\"Save image to the given filename.\")\nparser.add_argument(\n \"--t-min\", metavar=\"T\", help=\"Minimum time to be displayed (seconds)\"\n)\nparser.add_argument(\n \"--t-max\", metavar=\"T\", help=\"Maximum time to be displayed (seconds)\"\n)\nparser.add_argument(\n \"--log-count\",\n help=\"Use a logarithmic y-axis for the event count\",\n action=\"store_true\",\n)\n\nargs = parser.parse_args()\n\nwith open(args.file) as f:\n results = json.load(f)[\"results\"]\n\nif args.labels:\n labels = args.labels.split(\",\")\nelse:\n labels = [b[\"command\"] for b in results]\nall_times = [b[\"times\"] for b in results]\n\nt_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times)))\nt_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times)))\n\nbins = int(args.bins) if args.bins else \"auto\"\nhisttype = args.type if args.type else \"bar\"\n\nplt.figure(figsize=(10, 5))\nplt.hist(\n all_times,\n label=labels,\n bins=bins,\n histtype=histtype,\n range=(t_min, t_max),\n)\nplt.legend(\n loc=args.legend_location,\n fancybox=True,\n shadow=True,\n prop={\"size\": 10, \"family\": [\"Source Code Pro\", \"Fira Mono\", \"Courier New\"]},\n)\n\nplt.xlabel(\"Time [s]\")\nif args.title:\n plt.title(args.title)\n\nif args.log_count:\n plt.yscale(\"log\")\nelse:\n plt.ylim(0, None)\n\nif args.output:\n plt.savefig(args.output, dpi=600)\nelse:\n plt.show()\n" }, { "path": "scripts/plot_parametrized.py", "content": "#!/usr/bin/env python\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"matplotlib\",\n# \"pyqt6\",\n# ]\n# ///\n\n\"\"\"This program shows parametrized `hyperfine` benchmark results as an\nerrorbar plot.\"\"\"\n\nimport argparse\nimport json\nimport sys\n\nimport matplotlib.pyplot as plt\n\nparser = argparse.ArgumentParser(description=__doc__)\nparser.add_argument(\"file\", help=\"JSON file with benchmark results\", nargs=\"+\")\nparser.add_argument(\n \"--parameter-name\",\n metavar=\"name\",\n type=str,\n help=\"Deprecated; parameter names are now inferred from benchmark files\",\n)\nparser.add_argument(\n \"--log-x\", help=\"Use a logarithmic x (parameter) axis\", action=\"store_true\"\n)\nparser.add_argument(\n \"--log-time\", help=\"Use a logarithmic time axis\", action=\"store_true\"\n)\nparser.add_argument(\n \"--titles\", help=\"Comma-separated list of titles for the plot legend\"\n)\nparser.add_argument(\"-o\", \"--output\", help=\"Save image to the given filename.\")\n\nargs = parser.parse_args()\nif args.parameter_name is not None:\n sys.stderr.write(\n \"warning: --parameter-name is deprecated; names are inferred from \"\n \"benchmark results\\n\"\n )\n\n\ndef die(msg):\n sys.stderr.write(f\"fatal: {msg}\\n\")\n sys.exit(1)\n\n\ndef extract_parameters(results):\n \"\"\"Return `(parameter_name: str, parameter_values: List[float])`.\"\"\"\n if not results:\n die(\"no benchmark data to plot\")\n (names, values) = zip(*(unique_parameter(b) for b in results))\n names = frozenset(names)\n if len(names) != 1:\n die(\n f\"benchmarks must all have the same parameter name, but found: {sorted(names)}\"\n )\n return (next(iter(names)), list(values))\n\n\ndef unique_parameter(benchmark):\n \"\"\"Return the unique parameter `(name: str, value: float)`, or die.\"\"\"\n params_dict = benchmark.get(\"parameters\", {})\n if not params_dict:\n die(\"benchmarks must have exactly one parameter, but found none\")\n if len(params_dict) > 1:\n die(\n f\"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}\"\n )\n [(name, value)] = params_dict.items()\n return (name, float(value))\n\n\nparameter_name = None\n\nfor filename in args.file:\n with open(filename) as f:\n results = json.load(f)[\"results\"]\n\n (this_parameter_name, parameter_values) = extract_parameters(results)\n if parameter_name is not None and this_parameter_name != parameter_name:\n die(\n f\"files must all have the same parameter name, but found {parameter_name!r} vs. {this_parameter_name!r}\"\n )\n parameter_name = this_parameter_name\n\n times_mean = [b[\"mean\"] for b in results]\n times_stddev = [b[\"stddev\"] for b in results]\n\n plt.errorbar(x=parameter_values, y=times_mean, yerr=times_stddev, capsize=2)\n\nplt.xlabel(parameter_name)\nplt.ylabel(\"Time [s]\")\n\nif args.log_time:\n plt.yscale(\"log\")\nelse:\n plt.ylim(0, None)\n\nif args.log_x:\n plt.xscale(\"log\")\n\nif args.titles:\n plt.legend(args.titles.split(\",\"))\n\nif args.output:\n plt.savefig(args.output)\nelse:\n plt.show()\n" }, { "path": "scripts/plot_progression.py", "content": "#!/usr/bin/env python\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"pyqt6\",\n# \"matplotlib\",\n# \"numpy\",\n# ]\n# ///\n\n\"\"\"This program shows `hyperfine` benchmark results in a sequential way\nin order to debug possible background interference, caching effects,\nthermal throttling and similar effects.\n\"\"\"\n\nimport argparse\nimport json\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n\ndef moving_average(times, num_runs):\n times_padded = np.pad(\n times, (num_runs // 2, num_runs - 1 - num_runs // 2), mode=\"edge\"\n )\n kernel = np.ones(num_runs) / num_runs\n return np.convolve(times_padded, kernel, mode=\"valid\")\n\n\nparser = argparse.ArgumentParser(description=__doc__)\nparser.add_argument(\"file\", help=\"JSON file with benchmark results\")\nparser.add_argument(\"--title\", help=\"Plot Title\")\nparser.add_argument(\"-o\", \"--output\", help=\"Save image to the given filename.\")\nparser.add_argument(\n \"-w\",\n \"--moving-average-width\",\n type=int,\n metavar=\"num_runs\",\n help=\"Width of the moving-average window (default: N/5)\",\n)\nparser.add_argument(\n \"--no-moving-average\",\n action=\"store_true\",\n help=\"Do not show moving average curve\",\n)\n\n\nargs = parser.parse_args()\n\nwith open(args.file) as f:\n results = json.load(f)[\"results\"]\n\nfor result in results:\n label = result[\"command\"]\n times = result[\"times\"]\n num = len(times)\n nums = range(num)\n\n plt.scatter(x=nums, y=times, marker=\".\")\n plt.ylim([0, None])\n plt.xlim([-1, num])\n\n if not args.no_moving_average:\n moving_average_width = (\n num // 5 if args.moving_average_width is None else args.moving_average_width\n )\n\n average = moving_average(times, moving_average_width)\n plt.plot(nums, average, \"-\")\n\nif args.title:\n plt.title(args.title)\n\nlegend = []\nfor result in results:\n legend.append(result[\"command\"])\n if not args.no_moving_average:\n legend.append(\"moving average\")\nplt.legend(legend)\n\nplt.ylabel(\"Time [s]\")\n\nif args.output:\n plt.savefig(args.output)\nelse:\n plt.show()\n" }, { "path": "scripts/plot_whisker.py", "content": "#!/usr/bin/env python\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"matplotlib\",\n# \"pyqt6\",\n# ]\n# ///\n\n\"\"\"This program shows `hyperfine` benchmark results as a box and whisker plot.\n\nQuoting from the matplotlib documentation:\n The box extends from the lower to upper quartile values of the data, with\n a line at the median. The whiskers extend from the box to show the range\n of the data. Flier points are those past the end of the whiskers.\n\"\"\"\n\nimport argparse\nimport json\n\nimport matplotlib.pyplot as plt\n\nparser = argparse.ArgumentParser(description=__doc__)\nparser.add_argument(\"file\", help=\"JSON file with benchmark results\")\nparser.add_argument(\"--title\", help=\"Plot Title\")\nparser.add_argument(\"--sort-by\", choices=[\"median\"], help=\"Sort method\")\nparser.add_argument(\n \"--labels\", help=\"Comma-separated list of entries for the plot legend\"\n)\nparser.add_argument(\"-o\", \"--output\", help=\"Save image to the given filename.\")\n\nargs = parser.parse_args()\n\nwith open(args.file, encoding=\"utf-8\") as f:\n results = json.load(f)[\"results\"]\n\nif args.labels:\n labels = args.labels.split(\",\")\nelse:\n labels = [b[\"command\"] for b in results]\ntimes = [b[\"times\"] for b in results]\n\nif args.sort_by == \"median\":\n medians = [b[\"median\"] for b in results]\n indices = sorted(range(len(labels)), key=lambda k: medians[k])\n labels = [labels[i] for i in indices]\n times = [times[i] for i in indices]\n\nplt.figure(figsize=(10, 6), constrained_layout=True)\nboxplot = plt.boxplot(times, vert=True, patch_artist=True)\ncmap = plt.get_cmap(\"rainbow\")\ncolors = [cmap(val / len(times)) for val in range(len(times))]\n\nfor patch, color in zip(boxplot[\"boxes\"], colors):\n patch.set_facecolor(color)\n\nif args.title:\n plt.title(args.title)\nplt.legend(handles=boxplot[\"boxes\"], labels=labels, loc=\"best\", fontsize=\"medium\")\nplt.ylabel(\"Time [s]\")\nplt.ylim(0, None)\nplt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45)\nif args.output:\n plt.savefig(args.output)\nelse:\n plt.show()\n" }, { "path": "scripts/ruff.toml", "content": "target-version = \"py310\"\n\n[lint]\nextend-select = [\"I\", \"UP\", \"RUF\"]\n" }, { "path": "scripts/welch_ttest.py", "content": "#!/usr/bin/env python\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"scipy\",\n# ]\n# ///\n\n\"\"\"This script performs Welch's t-test on a JSON export file with two\nbenchmark results to test whether or not the two distributions are\nthe same.\"\"\"\n\nimport argparse\nimport json\nimport sys\n\nfrom scipy import stats\n\nparser = argparse.ArgumentParser(description=__doc__)\nparser.add_argument(\"file\", help=\"JSON file with two benchmark results\")\nargs = parser.parse_args()\n\nwith open(args.file) as f:\n results = json.load(f)[\"results\"]\n\nif len(results) != 2:\n print(\"The input file has to contain exactly two benchmarks\")\n sys.exit(1)\n\na, b = (x[\"command\"] for x in results[:2])\nX, Y = (x[\"times\"] for x in results[:2])\n\nprint(f\"Command 1: {a}\")\nprint(f\"Command 2: {b}\\n\")\n\nt, p = stats.ttest_ind(X, Y, equal_var=False)\nth = 0.05\ndispose = p < th\nprint(f\"t = {t:.3}, p = {p:.3}\")\nprint()\n\nif dispose:\n print(f\"There is a difference between the two benchmarks (p < {th}).\")\nelse:\n print(f\"The two benchmarks are almost the same (p >= {th}).\")\n" }, { "path": "src/benchmark/benchmark_result.rs", "content": "use std::collections::BTreeMap;\n\nuse serde::Serialize;\n\nuse crate::util::units::Second;\n\n/// Set of values that will be exported.\n// NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the\n// `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.\n#[derive(Debug, Default, Clone, Serialize, PartialEq)]\npub struct BenchmarkResult {\n /// The full command line of the program that is being benchmarked\n pub command: String,\n\n /// The full command line of the program that is being benchmarked, possibly including a list of\n /// parameters that were not used in the command line template.\n #[serde(skip_serializing)]\n pub command_with_unused_parameters: String,\n\n /// The average run time\n pub mean: Second,\n\n /// The standard deviation of all run times. Not available if only one run has been performed\n pub stddev: Option,\n\n /// The median run time\n pub median: Second,\n\n /// Time spent in user mode\n pub user: Second,\n\n /// Time spent in kernel mode\n pub system: Second,\n\n /// Minimum of all measured times\n pub min: Second,\n\n /// Maximum of all measured times\n pub max: Second,\n\n /// All run time measurements\n #[serde(skip_serializing_if = \"Option::is_none\")]\n pub times: Option>,\n\n /// Maximum memory usage of the process, in bytes\n #[serde(skip_serializing_if = \"Option::is_none\")]\n pub memory_usage_byte: Option>,\n\n /// Exit codes of all command invocations\n pub exit_codes: Vec>,\n\n /// Parameter values for this benchmark\n #[serde(skip_serializing_if = \"BTreeMap::is_empty\")]\n pub parameters: BTreeMap,\n}\n" }, { "path": "src/benchmark/executor.rs", "content": "#[cfg(windows)]\nuse std::os::windows::process::CommandExt;\nuse std::process::ExitStatus;\n\nuse crate::command::Command;\nuse crate::options::{\n CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,\n};\nuse crate::output::progress_bar::get_progress_bar;\nuse crate::timer::{execute_and_measure, TimerResult};\nuse crate::util::randomized_environment_offset;\nuse crate::util::units::Second;\n\nuse super::timing_result::TimingResult;\n\nuse anyhow::{bail, Context, Result};\nuse statistical::mean;\n\npub enum BenchmarkIteration {\n NonBenchmarkRun,\n Warmup(u64),\n Benchmark(u64),\n}\n\nimpl BenchmarkIteration {\n pub fn to_env_var_value(&self) -> Option {\n match self {\n BenchmarkIteration::NonBenchmarkRun => None,\n BenchmarkIteration::Warmup(i) => Some(format!(\"warmup-{}\", i)),\n BenchmarkIteration::Benchmark(i) => Some(format!(\"{}\", i)),\n }\n }\n}\n\npub trait Executor {\n /// Run the given command and measure the execution time\n fn run_command_and_measure(\n &self,\n command: &Command<'_>,\n iteration: BenchmarkIteration,\n command_failure_action: Option,\n output_policy: &CommandOutputPolicy,\n ) -> Result<(TimingResult, ExitStatus)>;\n\n /// Perform a calibration of this executor. For example,\n /// when running commands through a shell, we need to\n /// measure the shell spawning time separately in order\n /// to subtract it from the full runtime later.\n fn calibrate(&mut self) -> Result<()>;\n\n /// Return the time overhead for this executor when\n /// performing a measurement. This should return the time\n /// that is being used in addition to the actual runtime\n /// of the command.\n fn time_overhead(&self) -> Second;\n}\n\nfn run_command_and_measure_common(\n mut command: std::process::Command,\n iteration: BenchmarkIteration,\n command_failure_action: CmdFailureAction,\n command_input_policy: &CommandInputPolicy,\n command_output_policy: &CommandOutputPolicy,\n command_name: &str,\n) -> Result {\n let stdin = command_input_policy.get_stdin()?;\n let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;\n command.stdin(stdin).stdout(stdout).stderr(stderr);\n\n command.env(\n \"HYPERFINE_RANDOMIZED_ENVIRONMENT_OFFSET\",\n randomized_environment_offset::value(),\n );\n\n if let Some(value) = iteration.to_env_var_value() {\n command.env(\"HYPERFINE_ITERATION\", value);\n }\n\n let result = execute_and_measure(command)\n .with_context(|| format!(\"Failed to run command '{command_name}'\"))?;\n\n if !result.status.success() {\n use crate::util::exit_code::extract_exit_code;\n\n let should_fail = match command_failure_action {\n CmdFailureAction::RaiseError => true,\n CmdFailureAction::IgnoreAllFailures => false,\n CmdFailureAction::IgnoreSpecificFailures(ref codes) => {\n // Only fail if the exit code is not in the list of codes to ignore\n if let Some(exit_code) = extract_exit_code(result.status) {\n !codes.contains(&exit_code)\n } else {\n // If we can't extract an exit code, treat it as a failure\n true\n }\n }\n };\n\n if should_fail {\n let when = match iteration {\n BenchmarkIteration::NonBenchmarkRun => \"a non-benchmark run\".to_string(),\n BenchmarkIteration::Warmup(0) => \"the first warmup run\".to_string(),\n BenchmarkIteration::Warmup(i) => format!(\"warmup iteration {i}\"),\n BenchmarkIteration::Benchmark(0) => \"the first benchmark run\".to_string(),\n BenchmarkIteration::Benchmark(i) => format!(\"benchmark iteration {i}\"),\n };\n bail!(\n \"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \\\n Alternatively, use the '--show-output' option to debug what went wrong.\",\n cause=result.status.code().map_or(\n \"The process has been terminated by a signal\".into(),\n |c| format!(\"Command terminated with non-zero exit code {c}\")\n\n ),\n );\n }\n }\n\n Ok(result)\n}\n\npub struct RawExecutor<'a> {\n options: &'a Options,\n}\n\nimpl<'a> RawExecutor<'a> {\n pub fn new(options: &'a Options) -> Self {\n RawExecutor { options }\n }\n}\n\nimpl Executor for RawExecutor<'_> {\n fn run_command_and_measure(\n &self,\n command: &Command<'_>,\n iteration: BenchmarkIteration,\n command_failure_action: Option,\n output_policy: &CommandOutputPolicy,\n ) -> Result<(TimingResult, ExitStatus)> {\n let result = run_command_and_measure_common(\n command.get_command()?,\n iteration,\n command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),\n &self.options.command_input_policy,\n output_policy,\n &command.get_command_line(),\n )?;\n\n Ok((\n TimingResult {\n time_real: result.time_real,\n time_user: result.time_user,\n time_system: result.time_system,\n memory_usage_byte: result.memory_usage_byte,\n },\n result.status,\n ))\n }\n\n fn calibrate(&mut self) -> Result<()> {\n Ok(())\n }\n\n fn time_overhead(&self) -> Second {\n 0.0\n }\n}\n\npub struct ShellExecutor<'a> {\n options: &'a Options,\n shell: &'a Shell,\n shell_spawning_time: Option,\n}\n\nimpl<'a> ShellExecutor<'a> {\n pub fn new(shell: &'a Shell, options: &'a Options) -> Self {\n ShellExecutor {\n shell,\n options,\n shell_spawning_time: None,\n }\n }\n}\n\nimpl Executor for ShellExecutor<'_> {\n fn run_command_and_measure(\n &self,\n command: &Command<'_>,\n iteration: BenchmarkIteration,\n command_failure_action: Option,\n output_policy: &CommandOutputPolicy,\n ) -> Result<(TimingResult, ExitStatus)> {\n let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default(\"cmd.exe\");\n let mut command_builder = self.shell.command();\n command_builder.arg(if on_windows_cmd { \"/C\" } else { \"-c\" });\n\n // Windows needs special treatment for its behavior on parsing cmd arguments\n if on_windows_cmd {\n #[cfg(windows)]\n command_builder.raw_arg(command.get_command_line());\n } else {\n command_builder.arg(command.get_command_line());\n }\n\n let mut result = run_command_and_measure_common(\n command_builder,\n iteration,\n command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),\n &self.options.command_input_policy,\n output_policy,\n &command.get_command_line(),\n )?;\n\n // Subtract shell spawning time\n if let Some(spawning_time) = self.shell_spawning_time {\n result.time_real = (result.time_real - spawning_time.time_real).max(0.0);\n result.time_user = (result.time_user - spawning_time.time_user).max(0.0);\n result.time_system = (result.time_system - spawning_time.time_system).max(0.0);\n }\n\n Ok((\n TimingResult {\n time_real: result.time_real,\n time_user: result.time_user,\n time_system: result.time_system,\n memory_usage_byte: result.memory_usage_byte,\n },\n result.status,\n ))\n }\n\n /// Measure the average shell spawning time\n fn calibrate(&mut self) -> Result<()> {\n const COUNT: u64 = 50;\n let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {\n Some(get_progress_bar(\n COUNT,\n \"Measuring shell spawning time\",\n self.options.output_style,\n ))\n } else {\n None\n };\n\n let mut times_real: Vec = vec![];\n let mut times_user: Vec = vec![];\n let mut times_system: Vec = vec![];\n\n for _ in 0..COUNT {\n // Just run the shell without any command\n let res = self.run_command_and_measure(\n &Command::new(None, \"\"),\n BenchmarkIteration::NonBenchmarkRun,\n None,\n &CommandOutputPolicy::Null,\n );\n\n match res {\n Err(_) => {\n let shell_cmd = if cfg!(windows) {\n format!(\"{} /C \\\"\\\"\", self.shell)\n } else {\n format!(\"{} -c \\\"\\\"\", self.shell)\n };\n\n bail!(\n \"Could not measure shell execution time. Make sure you can run '{}'.\",\n shell_cmd\n );\n }\n Ok((r, _)) => {\n times_real.push(r.time_real);\n times_user.push(r.time_user);\n times_system.push(r.time_system);\n }\n }\n\n if let Some(bar) = progress_bar.as_ref() {\n bar.inc(1)\n }\n }\n\n if let Some(bar) = progress_bar.as_ref() {\n bar.finish_and_clear()\n }\n\n self.shell_spawning_time = Some(TimingResult {\n time_real: mean(×_real),\n time_user: mean(×_user),\n time_system: mean(×_system),\n memory_usage_byte: 0,\n });\n\n Ok(())\n }\n\n fn time_overhead(&self) -> Second {\n self.shell_spawning_time.unwrap().time_real\n }\n}\n\n#[derive(Clone)]\npub struct MockExecutor {\n shell: Option,\n}\n\nimpl MockExecutor {\n pub fn new(shell: Option) -> Self {\n MockExecutor { shell }\n }\n\n fn extract_time>(sleep_command: S) -> Second {\n assert!(sleep_command.as_ref().starts_with(\"sleep \"));\n sleep_command\n .as_ref()\n .trim_start_matches(\"sleep \")\n .parse::()\n .unwrap()\n }\n}\n\nimpl Executor for MockExecutor {\n fn run_command_and_measure(\n &self,\n command: &Command<'_>,\n _iteration: BenchmarkIteration,\n _command_failure_action: Option,\n _output_policy: &CommandOutputPolicy,\n ) -> Result<(TimingResult, ExitStatus)> {\n #[cfg(unix)]\n let status = {\n use std::os::unix::process::ExitStatusExt;\n ExitStatus::from_raw(0)\n };\n\n #[cfg(windows)]\n let status = {\n use std::os::windows::process::ExitStatusExt;\n ExitStatus::from_raw(0)\n };\n\n Ok((\n TimingResult {\n time_real: Self::extract_time(command.get_command_line()),\n time_user: 0.0,\n time_system: 0.0,\n memory_usage_byte: 0,\n },\n status,\n ))\n }\n\n fn calibrate(&mut self) -> Result<()> {\n Ok(())\n }\n\n fn time_overhead(&self) -> Second {\n match &self.shell {\n None => 0.0,\n Some(shell) => Self::extract_time(shell),\n }\n }\n}\n\n#[test]\nfn test_mock_executor_extract_time() {\n assert_eq!(MockExecutor::extract_time(\"sleep 0.1\"), 0.1);\n}\n" }, { "path": "src/benchmark/mod.rs", "content": "pub mod benchmark_result;\npub mod executor;\npub mod relative_speed;\npub mod scheduler;\npub mod timing_result;\n\nuse std::cmp;\n\nuse crate::benchmark::executor::BenchmarkIteration;\nuse crate::command::Command;\nuse crate::options::{\n CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,\n};\nuse crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};\nuse crate::output::format::{format_duration, format_duration_unit};\nuse crate::output::progress_bar::get_progress_bar;\nuse crate::output::warnings::{OutlierWarningOptions, Warnings};\nuse crate::parameter::ParameterNameAndValue;\nuse crate::util::exit_code::extract_exit_code;\nuse crate::util::min_max::{max, min};\nuse crate::util::units::Second;\nuse benchmark_result::BenchmarkResult;\nuse timing_result::TimingResult;\n\nuse anyhow::{anyhow, Result};\nuse colored::*;\nuse statistical::{mean, median, standard_deviation};\n\nuse self::executor::Executor;\n\n/// Threshold for warning about fast execution time\npub const MIN_EXECUTION_TIME: Second = 5e-3;\n\npub struct Benchmark<'a> {\n number: usize,\n command: &'a Command<'a>,\n options: &'a Options,\n executor: &'a dyn Executor,\n}\n\nimpl<'a> Benchmark<'a> {\n pub fn new(\n number: usize,\n command: &'a Command<'a>,\n options: &'a Options,\n executor: &'a dyn Executor,\n ) -> Self {\n Benchmark {\n number,\n command,\n options,\n executor,\n }\n }\n\n /// Run setup, cleanup, or preparation commands\n fn run_intermediate_command(\n &self,\n command: &Command<'_>,\n error_output: &'static str,\n output_policy: &CommandOutputPolicy,\n ) -> Result {\n self.executor\n .run_command_and_measure(\n command,\n executor::BenchmarkIteration::NonBenchmarkRun,\n Some(CmdFailureAction::RaiseError),\n output_policy,\n )\n .map(|r| r.0)\n .map_err(|_| anyhow!(error_output))\n }\n\n /// Run the command specified by `--setup`.\n fn run_setup_command(\n &self,\n parameters: impl IntoIterator>,\n output_policy: &CommandOutputPolicy,\n ) -> Result {\n let command = self\n .options\n .setup_command\n .as_ref()\n .map(|setup_command| Command::new_parametrized(None, setup_command, parameters));\n\n let error_output = \"The setup command terminated with a non-zero exit code. \\\n Append ' || true' to the command if you are sure that this can be ignored.\";\n\n Ok(command\n .map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy))\n .transpose()?\n .unwrap_or_default())\n }\n\n /// Run the command specified by `--cleanup`.\n fn run_cleanup_command(\n &self,\n parameters: impl IntoIterator>,\n output_policy: &CommandOutputPolicy,\n ) -> Result {\n let command = self\n .options\n .cleanup_command\n .as_ref()\n .map(|cleanup_command| Command::new_parametrized(None, cleanup_command, parameters));\n\n let error_output = \"The cleanup command terminated with a non-zero exit code. \\\n Append ' || true' to the command if you are sure that this can be ignored.\";\n\n Ok(command\n .map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy))\n .transpose()?\n .unwrap_or_default())\n }\n\n /// Run the command specified by `--prepare`.\n fn run_preparation_command(\n &self,\n command: &Command<'_>,\n output_policy: &CommandOutputPolicy,\n ) -> Result {\n let error_output = \"The preparation command terminated with a non-zero exit code. \\\n Append ' || true' to the command if you are sure that this can be ignored.\";\n\n self.run_intermediate_command(command, error_output, output_policy)\n }\n\n /// Run the command specified by `--conclude`.\n fn run_conclusion_command(\n &self,\n command: &Command<'_>,\n output_policy: &CommandOutputPolicy,\n ) -> Result {\n let error_output = \"The conclusion command terminated with a non-zero exit code. \\\n Append ' || true' to the command if you are sure that this can be ignored.\";\n\n self.run_intermediate_command(command, error_output, output_policy)\n }\n\n /// Run the benchmark for a single command\n pub fn run(&self) -> Result {\n if self.options.output_style != OutputStyleOption::Disabled {\n println!(\n \"{}{}: {}\",\n \"Benchmark \".bold(),\n (self.number + 1).to_string().bold(),\n self.command.get_name_with_unused_parameters(),\n );\n }\n\n let mut times_real: Vec = vec![];\n let mut times_user: Vec = vec![];\n let mut times_system: Vec = vec![];\n let mut memory_usage_byte: Vec = vec![];\n let mut exit_codes: Vec> = vec![];\n let mut all_succeeded = true;\n\n let output_policy = &self.options.command_output_policies[self.number];\n\n let preparation_command = self.options.preparation_command.as_ref().map(|values| {\n let preparation_command = if values.len() == 1 {\n &values[0]\n } else {\n &values[self.number]\n };\n Command::new_parametrized(\n None,\n preparation_command,\n self.command.get_parameters().iter().cloned(),\n )\n });\n\n let run_preparation_command = || {\n preparation_command\n .as_ref()\n .map(|cmd| self.run_preparation_command(cmd, output_policy))\n .transpose()\n };\n\n let conclusion_command = self.options.conclusion_command.as_ref().map(|values| {\n let conclusion_command = if values.len() == 1 {\n &values[0]\n } else {\n &values[self.number]\n };\n Command::new_parametrized(\n None,\n conclusion_command,\n self.command.get_parameters().iter().cloned(),\n )\n });\n let run_conclusion_command = || {\n conclusion_command\n .as_ref()\n .map(|cmd| self.run_conclusion_command(cmd, output_policy))\n .transpose()\n };\n\n self.run_setup_command(self.command.get_parameters().iter().cloned(), output_policy)?;\n\n // Warmup phase\n if self.options.warmup_count > 0 {\n let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {\n Some(get_progress_bar(\n self.options.warmup_count,\n \"Performing warmup runs\",\n self.options.output_style,\n ))\n } else {\n None\n };\n\n for i in 0..self.options.warmup_count {\n let _ = run_preparation_command()?;\n let _ = self.executor.run_command_and_measure(\n self.command,\n BenchmarkIteration::Warmup(i),\n None,\n output_policy,\n )?;\n let _ = run_conclusion_command()?;\n if let Some(bar) = progress_bar.as_ref() {\n bar.inc(1)\n }\n }\n if let Some(bar) = progress_bar.as_ref() {\n bar.finish_and_clear()\n }\n }\n\n // Set up progress bar (and spinner for initial measurement)\n let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {\n Some(get_progress_bar(\n self.options.run_bounds.min,\n \"Initial time measurement\",\n self.options.output_style,\n ))\n } else {\n None\n };\n\n let preparation_result = run_preparation_command()?;\n let preparation_overhead =\n preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());\n\n // Initial timing run\n let (res, status) = self.executor.run_command_and_measure(\n self.command,\n BenchmarkIteration::Benchmark(0),\n None,\n output_policy,\n )?;\n let success = status.success();\n\n let conclusion_result = run_conclusion_command()?;\n let conclusion_overhead =\n conclusion_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());\n\n // Determine number of benchmark runs\n let runs_in_min_time = (self.options.min_benchmarking_time\n / (res.time_real\n + self.executor.time_overhead()\n + preparation_overhead\n + conclusion_overhead)) as u64;\n\n let count = {\n let min = cmp::max(runs_in_min_time, self.options.run_bounds.min);\n\n self.options\n .run_bounds\n .max\n .as_ref()\n .map(|max| cmp::min(min, *max))\n .unwrap_or(min)\n };\n\n let count_remaining = count - 1;\n\n // Save the first result\n times_real.push(res.time_real);\n times_user.push(res.time_user);\n times_system.push(res.time_system);\n memory_usage_byte.push(res.memory_usage_byte);\n exit_codes.push(extract_exit_code(status));\n\n all_succeeded = all_succeeded && success;\n\n // Re-configure the progress bar\n if let Some(bar) = progress_bar.as_ref() {\n bar.set_length(count)\n }\n if let Some(bar) = progress_bar.as_ref() {\n bar.inc(1)\n }\n\n // Gather statistics (perform the actual benchmark)\n for i in 0..count_remaining {\n run_preparation_command()?;\n\n let msg = {\n let mean = format_duration(mean(×_real), self.options.time_unit);\n format!(\"Current estimate: {}\", mean.to_string().green())\n };\n\n if let Some(bar) = progress_bar.as_ref() {\n bar.set_message(msg.to_owned())\n }\n\n let (res, status) = self.executor.run_command_and_measure(\n self.command,\n BenchmarkIteration::Benchmark(i + 1),\n None,\n output_policy,\n )?;\n let success = status.success();\n\n times_real.push(res.time_real);\n times_user.push(res.time_user);\n times_system.push(res.time_system);\n memory_usage_byte.push(res.memory_usage_byte);\n exit_codes.push(extract_exit_code(status));\n\n all_succeeded = all_succeeded && success;\n\n if let Some(bar) = progress_bar.as_ref() {\n bar.inc(1)\n }\n\n run_conclusion_command()?;\n }\n\n if let Some(bar) = progress_bar.as_ref() {\n bar.finish_and_clear()\n }\n\n // Compute statistical quantities\n let t_num = times_real.len();\n let t_mean = mean(×_real);\n let t_stddev = if times_real.len() > 1 {\n Some(standard_deviation(×_real, Some(t_mean)))\n } else {\n None\n };\n let t_median = median(×_real);\n let t_min = min(×_real);\n let t_max = max(×_real);\n\n let user_mean = mean(×_user);\n let system_mean = mean(×_system);\n\n // Formatting and console output\n let (mean_str, time_unit) = format_duration_unit(t_mean, self.options.time_unit);\n let min_str = format_duration(t_min, Some(time_unit));\n let max_str = format_duration(t_max, Some(time_unit));\n let num_str = format!(\"{t_num} runs\");\n\n let user_str = format_duration(user_mean, Some(time_unit));\n let system_str = format_duration(system_mean, Some(time_unit));\n\n if self.options.output_style != OutputStyleOption::Disabled {\n if times_real.len() == 1 {\n println!(\n \" Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]\",\n \"abs\".green().bold(),\n mean_str.green().bold(),\n \" \", // alignment\n user_str.blue(),\n system_str.blue()\n );\n } else {\n let stddev_str = format_duration(t_stddev.unwrap(), Some(time_unit));\n\n println!(\n \" Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]\",\n \"mean\".green().bold(),\n \"σ\".green(),\n mean_str.green().bold(),\n stddev_str.green(),\n user_str.blue(),\n system_str.blue()\n );\n\n println!(\n \" Range ({} … {}): {:>8} … {:>8} {}\",\n \"min\".cyan(),\n \"max\".purple(),\n min_str.cyan(),\n max_str.purple(),\n num_str.dimmed()\n );\n }\n }\n\n // Warnings\n let mut warnings = vec![];\n\n // Check execution time\n if matches!(self.options.executor_kind, ExecutorKind::Shell(_))\n && times_real.iter().any(|&t| t < MIN_EXECUTION_TIME)\n {\n warnings.push(Warnings::FastExecutionTime);\n }\n\n // Check program exit codes\n if !all_succeeded {\n warnings.push(Warnings::NonZeroExitCode);\n }\n\n // Run outlier detection\n let scores = modified_zscores(×_real);\n\n let outlier_warning_options = OutlierWarningOptions {\n warmup_in_use: self.options.warmup_count > 0,\n prepare_in_use: self\n .options\n .preparation_command\n .as_ref()\n .map(|v| v.len())\n .unwrap_or(0)\n > 0,\n };\n\n if scores[0] > OUTLIER_THRESHOLD {\n warnings.push(Warnings::SlowInitialRun(\n times_real[0],\n outlier_warning_options,\n ));\n } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) {\n warnings.push(Warnings::OutliersDetected(outlier_warning_options));\n }\n\n if !warnings.is_empty() {\n eprintln!(\" \");\n\n for warning in &warnings {\n eprintln!(\" {}: {}\", \"Warning\".yellow(), warning);\n }\n }\n\n if self.options.output_style != OutputStyleOption::Disabled {\n println!(\" \");\n }\n\n self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?;\n\n Ok(BenchmarkResult {\n command: self.command.get_name(),\n command_with_unused_parameters: self.command.get_name_with_unused_parameters(),\n mean: t_mean,\n stddev: t_stddev,\n median: t_median,\n user: user_mean,\n system: system_mean,\n min: t_min,\n max: t_max,\n times: Some(times_real),\n memory_usage_byte: Some(memory_usage_byte),\n exit_codes,\n parameters: self\n .command\n .get_parameters()\n .iter()\n .map(|(name, value)| (name.to_string(), value.to_string()))\n .collect(),\n })\n }\n}\n" }, { "path": "src/benchmark/relative_speed.rs", "content": "use std::cmp::Ordering;\n\nuse super::benchmark_result::BenchmarkResult;\nuse crate::{options::SortOrder, util::units::Scalar};\n\n#[derive(Debug)]\npub struct BenchmarkResultWithRelativeSpeed<'a> {\n pub result: &'a BenchmarkResult,\n pub relative_speed: Scalar,\n pub relative_speed_stddev: Option,\n pub is_reference: bool,\n // Less means faster\n pub relative_ordering: Ordering,\n}\n\npub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {\n l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)\n}\n\npub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {\n results\n .iter()\n .min_by(|&l, &r| compare_mean_time(l, r))\n .expect(\"at least one benchmark result\")\n}\n\nfn compute_relative_speeds<'a>(\n results: &'a [BenchmarkResult],\n reference: &'a BenchmarkResult,\n sort_order: SortOrder,\n) -> Vec> {\n let mut results: Vec<_> = results\n .iter()\n .map(|result| {\n let is_reference = result == reference;\n let relative_ordering = compare_mean_time(result, reference);\n\n if result.mean == 0.0 {\n return BenchmarkResultWithRelativeSpeed {\n result,\n relative_speed: if is_reference { 1.0 } else { f64::INFINITY },\n relative_speed_stddev: None,\n is_reference,\n relative_ordering,\n };\n }\n\n let ratio = match relative_ordering {\n Ordering::Less => reference.mean / result.mean,\n Ordering::Equal => 1.0,\n Ordering::Greater => result.mean / reference.mean,\n };\n\n // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas\n // Covariance asssumed to be 0, i.e. variables are assumed to be independent\n let ratio_stddev = match (result.stddev, reference.stddev) {\n (Some(result_stddev), Some(fastest_stddev)) => Some(\n ratio\n * ((result_stddev / result.mean).powi(2)\n + (fastest_stddev / reference.mean).powi(2))\n .sqrt(),\n ),\n _ => None,\n };\n\n BenchmarkResultWithRelativeSpeed {\n result,\n relative_speed: ratio,\n relative_speed_stddev: ratio_stddev,\n is_reference,\n relative_ordering,\n }\n })\n .collect();\n\n match sort_order {\n SortOrder::Command => {}\n SortOrder::MeanTime => {\n results.sort_unstable_by(|r1, r2| compare_mean_time(r1.result, r2.result));\n }\n }\n\n results\n}\n\npub fn compute_with_check_from_reference<'a>(\n results: &'a [BenchmarkResult],\n reference: &'a BenchmarkResult,\n sort_order: SortOrder,\n) -> Option>> {\n if fastest_of(results).mean == 0.0 || reference.mean == 0.0 {\n return None;\n }\n\n Some(compute_relative_speeds(results, reference, sort_order))\n}\n\npub fn compute_with_check(\n results: &[BenchmarkResult],\n sort_order: SortOrder,\n) -> Option>> {\n let fastest = fastest_of(results);\n\n if fastest.mean == 0.0 {\n return None;\n }\n\n Some(compute_relative_speeds(results, fastest, sort_order))\n}\n\n/// Same as compute_with_check, potentially resulting in relative speeds of infinity\npub fn compute(\n results: &[BenchmarkResult],\n sort_order: SortOrder,\n) -> Vec> {\n let fastest = fastest_of(results);\n\n compute_relative_speeds(results, fastest, sort_order)\n}\n\n#[cfg(test)]\nfn create_result(name: &str, mean: Scalar) -> BenchmarkResult {\n use std::collections::BTreeMap;\n\n BenchmarkResult {\n command: name.into(),\n command_with_unused_parameters: name.into(),\n mean,\n stddev: Some(1.0),\n median: mean,\n user: mean,\n system: 0.0,\n min: mean,\n max: mean,\n times: None,\n memory_usage_byte: None,\n exit_codes: Vec::new(),\n parameters: BTreeMap::new(),\n }\n}\n\n#[test]\nfn test_compute_relative_speed() {\n use approx::assert_relative_eq;\n\n let results = vec![\n create_result(\"cmd1\", 3.0),\n create_result(\"cmd2\", 2.0),\n create_result(\"cmd3\", 5.0),\n ];\n\n let annotated_results = compute_with_check(&results, SortOrder::Command).unwrap();\n\n assert_relative_eq!(1.5, annotated_results[0].relative_speed);\n assert_relative_eq!(1.0, annotated_results[1].relative_speed);\n assert_relative_eq!(2.5, annotated_results[2].relative_speed);\n}\n\n#[test]\nfn test_compute_relative_speed_with_reference() {\n use approx::assert_relative_eq;\n\n let results = vec![create_result(\"cmd2\", 2.0), create_result(\"cmd3\", 5.0)];\n let reference = create_result(\"cmd2\", 4.0);\n\n let annotated_results =\n compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();\n\n assert_relative_eq!(2.0, annotated_results[0].relative_speed);\n assert_relative_eq!(1.25, annotated_results[1].relative_speed);\n}\n\n#[test]\nfn test_compute_relative_speed_for_zero_times() {\n let results = vec![create_result(\"cmd1\", 1.0), create_result(\"cmd2\", 0.0)];\n\n let annotated_results = compute_with_check(&results, SortOrder::Command);\n\n assert!(annotated_results.is_none());\n}\n" }, { "path": "src/benchmark/scheduler.rs", "content": "use super::benchmark_result::BenchmarkResult;\nuse super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};\nuse super::{relative_speed, Benchmark};\nuse colored::*;\nuse std::cmp::Ordering;\n\nuse crate::command::{Command, Commands};\nuse crate::export::ExportManager;\nuse crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};\n\nuse anyhow::Result;\n\npub struct Scheduler<'a> {\n commands: &'a Commands<'a>,\n options: &'a Options,\n export_manager: &'a ExportManager,\n results: Vec,\n}\n\nimpl<'a> Scheduler<'a> {\n pub fn new(\n commands: &'a Commands,\n options: &'a Options,\n export_manager: &'a ExportManager,\n ) -> Self {\n Self {\n commands,\n options,\n export_manager,\n results: vec![],\n }\n }\n\n pub fn run_benchmarks(&mut self) -> Result<()> {\n let mut executor: Box = match self.options.executor_kind {\n ExecutorKind::Raw => Box::new(RawExecutor::new(self.options)),\n ExecutorKind::Mock(ref shell) => Box::new(MockExecutor::new(shell.clone())),\n ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),\n };\n\n let reference = self\n .options\n .reference_command\n .as_ref()\n .map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd));\n\n executor.calibrate()?;\n\n for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {\n self.results\n .push(Benchmark::new(number, cmd, self.options, &*executor).run()?);\n\n // We export results after each individual benchmark, because\n // we would risk losing them if a later benchmark fails.\n self.export_manager.write_results(&self.results, true)?;\n }\n\n Ok(())\n }\n\n pub fn print_relative_speed_comparison(&self) {\n if self.options.output_style == OutputStyleOption::Disabled {\n return;\n }\n\n if self.results.len() < 2 {\n return;\n }\n\n let reference = self\n .options\n .reference_command\n .as_ref()\n .map(|_| &self.results[0])\n .unwrap_or_else(|| relative_speed::fastest_of(&self.results));\n\n if let Some(annotated_results) = relative_speed::compute_with_check_from_reference(\n &self.results,\n reference,\n self.options.sort_order_speed_comparison,\n ) {\n match self.options.sort_order_speed_comparison {\n SortOrder::MeanTime => {\n println!(\"{}\", \"Summary\".bold());\n\n let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();\n let others = annotated_results.iter().filter(|r| !r.is_reference);\n\n println!(\n \" {} ran\",\n reference.result.command_with_unused_parameters.cyan()\n );\n\n for item in others {\n let stddev = if let Some(stddev) = item.relative_speed_stddev {\n format!(\" ± {}\", format!(\"{:.2}\", stddev).green())\n } else {\n \"\".into()\n };\n let comparator = match item.relative_ordering {\n Ordering::Less => format!(\n \"{}{} times slower than\",\n format!(\"{:8.2}\", item.relative_speed).bold().green(),\n stddev\n ),\n Ordering::Greater => format!(\n \"{}{} times faster than\",\n format!(\"{:8.2}\", item.relative_speed).bold().green(),\n stddev\n ),\n Ordering::Equal => format!(\n \" As fast ({}{}) as\",\n format!(\"{:.2}\", item.relative_speed).bold().green(),\n stddev\n ),\n };\n println!(\n \"{} {}\",\n comparator,\n &item.result.command_with_unused_parameters.magenta()\n );\n }\n }\n SortOrder::Command => {\n println!(\"{}\", \"Relative speed comparison\".bold());\n\n for item in annotated_results {\n println!(\n \" {}{} {}\",\n format!(\"{:10.2}\", item.relative_speed).bold().green(),\n if item.is_reference {\n \" \".into()\n } else if let Some(stddev) = item.relative_speed_stddev {\n format!(\" ± {}\", format!(\"{stddev:5.2}\").green())\n } else {\n \" \".into()\n },\n &item.result.command_with_unused_parameters,\n );\n }\n }\n }\n } else {\n eprintln!(\n \"{}: The benchmark comparison could not be computed as some benchmark times are zero. \\\n This could be caused by background interference during the initial calibration phase \\\n of hyperfine, in combination with very fast commands (faster than a few milliseconds). \\\n Try to re-run the benchmark on a quiet system. If you did not do so already, try the \\\n --shell=none/-N option. If it does not help either, you command is most likely too fast \\\n to be accurately benchmarked by hyperfine.\",\n \"Note\".bold().red()\n );\n }\n }\n\n pub fn final_export(&self) -> Result<()> {\n self.export_manager.write_results(&self.results, false)\n }\n}\n\n#[cfg(test)]\nfn generate_results(args: &[&'static str]) -> Result> {\n use crate::cli::get_cli_arguments;\n\n let args = [\"hyperfine\", \"--debug-mode\", \"--style=none\"]\n .iter()\n .chain(args);\n let cli_arguments = get_cli_arguments(args);\n let mut options = Options::from_cli_arguments(&cli_arguments)?;\n\n assert_eq!(options.executor_kind, ExecutorKind::Mock(None));\n\n let commands = Commands::from_cli_arguments(&cli_arguments)?;\n let export_manager = ExportManager::from_cli_arguments(\n &cli_arguments,\n options.time_unit,\n options.sort_order_exports,\n )?;\n\n options.validate_against_command_list(&commands)?;\n\n let mut scheduler = Scheduler::new(&commands, &options, &export_manager);\n\n scheduler.run_benchmarks()?;\n Ok(scheduler.results)\n}\n\n#[test]\nfn scheduler_basic() -> Result<()> {\n insta::assert_yaml_snapshot!(generate_results(&[\"--runs=2\", \"sleep 0.123\", \"sleep 0.456\"])?, @r#\"\n - command: sleep 0.123\n mean: 0.123\n stddev: 0\n median: 0.123\n user: 0\n system: 0\n min: 0.123\n max: 0.123\n times:\n - 0.123\n - 0.123\n memory_usage_byte:\n - 0\n - 0\n exit_codes:\n - 0\n - 0\n - command: sleep 0.456\n mean: 0.456\n stddev: 0\n median: 0.456\n user: 0\n system: 0\n min: 0.456\n max: 0.456\n times:\n - 0.456\n - 0.456\n memory_usage_byte:\n - 0\n - 0\n exit_codes:\n - 0\n - 0\n \"#);\n\n Ok(())\n}\n" }, { "path": "src/benchmark/timing_result.rs", "content": "use crate::util::units::Second;\n\n/// Results from timing a single command\n#[derive(Debug, Default, Copy, Clone)]\npub struct TimingResult {\n /// Wall clock time\n pub time_real: Second,\n\n /// Time spent in user mode\n pub time_user: Second,\n\n /// Time spent in kernel mode\n pub time_system: Second,\n\n /// Maximum amount of memory used, in bytes\n pub memory_usage_byte: u64,\n}\n" }, { "path": "src/cli.rs", "content": "use std::ffi::OsString;\n\nuse clap::{\n builder::NonEmptyStringValueParser, crate_version, Arg, ArgAction, ArgMatches, Command,\n ValueHint,\n};\n\npub fn get_cli_arguments<'a, I, T>(args: I) -> ArgMatches\nwhere\n I: IntoIterator,\n T: Into + Clone + 'a,\n{\n let command = build_command();\n command.get_matches_from(args)\n}\n\n/// Build the clap command for parsing command line arguments\nfn build_command() -> Command {\n Command::new(\"hyperfine\")\n .version(crate_version!())\n .next_line_help(true)\n .hide_possible_values(true)\n .about(\"A command-line benchmarking tool.\")\n .help_expected(true)\n .max_term_width(80)\n .arg(\n Arg::new(\"command\")\n .help(\"The command to benchmark. This can be the name of an executable, a command \\\n line like \\\"grep -i todo\\\" or a shell command like \\\"sleep 0.5 && echo test\\\". \\\n The latter is only available if the shell is not explicitly disabled via \\\n '--shell=none'. If multiple commands are given, hyperfine will show a \\\n comparison of the respective runtimes.\")\n .required(true)\n .action(ArgAction::Append)\n .value_hint(ValueHint::CommandString)\n .value_parser(NonEmptyStringValueParser::new()),\n )\n .arg(\n Arg::new(\"warmup\")\n .long(\"warmup\")\n .short('w')\n .value_name(\"NUM\")\n .action(ArgAction::Set)\n .help(\n \"Perform NUM warmup runs before the actual benchmark. This can be used \\\n to fill (disk) caches for I/O-heavy programs.\",\n ),\n )\n .arg(\n Arg::new(\"min-runs\")\n .long(\"min-runs\")\n .short('m')\n .action(ArgAction::Set)\n .value_name(\"NUM\")\n .help(\"Perform at least NUM runs for each command (default: 10).\"),\n )\n .arg(\n Arg::new(\"max-runs\")\n .long(\"max-runs\")\n .short('M')\n .action(ArgAction::Set)\n .value_name(\"NUM\")\n .help(\"Perform at most NUM runs for each command. By default, there is no limit.\"),\n )\n .arg(\n Arg::new(\"runs\")\n .long(\"runs\")\n .conflicts_with_all([\"max-runs\", \"min-runs\"])\n .short('r')\n .action(ArgAction::Set)\n .value_name(\"NUM\")\n .help(\"Perform exactly NUM runs for each command. If this option is not specified, \\\n hyperfine automatically determines the number of runs.\"),\n )\n .arg(\n Arg::new(\"setup\")\n .long(\"setup\")\n .short('s')\n .action(ArgAction::Set)\n .value_name(\"CMD\")\n .value_hint(ValueHint::CommandString)\n .help(\n \"Execute CMD before each set of timing runs. This is useful for \\\n compiling your software with the provided parameters, or to do any \\\n other work that should happen once before a series of benchmark runs, \\\n not every time as would happen with the --prepare option.\"\n ),\n )\n .arg(\n Arg::new(\"reference\")\n .long(\"reference\")\n .action(ArgAction::Set)\n .value_name(\"CMD\")\n .help(\n \"The reference command for the relative comparison of results. \\\n If this is unset, results are compared with the fastest command as reference.\"\n )\n )\n .arg(\n Arg::new(\"reference-name\")\n .long(\"reference-name\")\n .action(ArgAction::Set)\n .value_name(\"CMD\")\n .help(\"Give a meaningful name to the reference command.\")\n .requires(\"reference\")\n )\n .arg(\n Arg::new(\"prepare\")\n .long(\"prepare\")\n .short('p')\n .action(ArgAction::Append)\n .num_args(1)\n .value_name(\"CMD\")\n .value_hint(ValueHint::CommandString)\n .help(\n \"Execute CMD before each timing run. This is useful for \\\n clearing disk caches, for example.\\nThe --prepare option can \\\n be specified once for all commands or multiple times, once for \\\n each command. In the latter case, each preparation command will \\\n be run prior to the corresponding benchmark command.\",\n ),\n )\n .arg(\n Arg::new(\"conclude\")\n .long(\"conclude\")\n .short('C')\n .action(ArgAction::Append)\n .num_args(1)\n .value_name(\"CMD\")\n .value_hint(ValueHint::CommandString)\n .help(\n \"Execute CMD after each timing run. This is useful for killing \\\n long-running processes started (e.g. a web server started in --prepare), \\\n for example.\\nThe --conclude option can be specified once for all \\\n commands or multiple times, once for each command. In the latter case, \\\n each conclude command will be run after the corresponding benchmark \\\n command.\",\n ),\n )\n .arg(\n Arg::new(\"cleanup\")\n .long(\"cleanup\")\n .short('c')\n .action(ArgAction::Set)\n .value_name(\"CMD\")\n .value_hint(ValueHint::CommandString)\n .help(\n \"Execute CMD after the completion of all benchmarking \\\n runs for each individual command to be benchmarked. \\\n This is useful if the commands to be benchmarked produce \\\n artifacts that need to be cleaned up.\"\n ),\n )\n .arg(\n Arg::new(\"parameter-scan\")\n .long(\"parameter-scan\")\n .short('P')\n .action(ArgAction::Set)\n .allow_hyphen_values(true)\n .value_names([\"VAR\", \"MIN\", \"MAX\"])\n .help(\n \"Perform benchmark runs for each value in the range MIN..MAX. Replaces the \\\n string '{VAR}' in each command by the current parameter value.\\n\\n \\\n Example: hyperfine -P threads 1 8 'make -j {threads}'\\n\\n\\\n This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\\n\\n\\\n To have the value increase following different patterns, use shell arithmetics.\\n\\n \\\n Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\\n\\n\\\n This performs benchmarks with power of 2 increases: 'sleep 1', 'sleep 2', 'sleep 4', …\\n\\\n The exact syntax may vary depending on your shell and OS.\"\n ),\n )\n .arg(\n Arg::new(\"parameter-step-size\")\n .long(\"parameter-step-size\")\n .short('D')\n .action(ArgAction::Set)\n .value_names([\"DELTA\"])\n .requires(\"parameter-scan\")\n .help(\n \"This argument requires --parameter-scan to be specified as well. \\\n Traverse the range MIN..MAX in steps of DELTA.\\n\\n \\\n Example: hyperfine -P delay 0.3 0.7 -D 0.2 'sleep {delay}'\\n\\n\\\n This performs benchmarks for 'sleep 0.3', 'sleep 0.5' and 'sleep 0.7'.\",\n ),\n )\n .arg(\n Arg::new(\"parameter-list\")\n .long(\"parameter-list\")\n .short('L')\n .action(ArgAction::Append)\n .allow_hyphen_values(true)\n .value_names([\"VAR\", \"VALUES\"])\n .conflicts_with_all([\"parameter-scan\", \"parameter-step-size\"])\n .help(\n \"Perform benchmark runs for each value in the comma-separated list VALUES. \\\n Replaces the string '{VAR}' in each command by the current parameter value\\\n .\\n\\nExample: hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'\\n\\n\\\n This performs benchmarks for 'gcc -O2 main.cpp' and 'clang -O2 main.cpp'.\\n\\n\\\n The option can be specified multiple times to run benchmarks for all \\\n possible parameter combinations.\\n\"\n ),\n )\n .arg(\n Arg::new(\"shell\")\n .long(\"shell\")\n .short('S')\n .action(ArgAction::Set)\n .value_name(\"SHELL\")\n .overrides_with(\"shell\")\n .value_hint(ValueHint::CommandString)\n .help(\"Set the shell to use for executing benchmarked commands. This can be the \\\n name or the path to the shell executable, or a full command line \\\n like \\\"bash --norc\\\". It can also be set to \\\"default\\\" to explicitly select \\\n the default shell on this platform. Finally, this can also be set to \\\n \\\"none\\\" to disable the shell. In this case, commands will be executed \\\n directly. They can still have arguments, but more complex things like \\\n \\\"sleep 0.1; sleep 0.2\\\" are not possible without a shell.\")\n )\n .arg(\n Arg::new(\"no-shell\")\n .short('N')\n .action(ArgAction::SetTrue)\n .conflicts_with_all([\"shell\", \"debug-mode\"])\n .help(\"An alias for '--shell=none'.\")\n )\n .arg(\n Arg::new(\"ignore-failure\")\n .long(\"ignore-failure\")\n .action(ArgAction::Set)\n .value_name(\"MODE\")\n .num_args(0..=1)\n .default_missing_value(\"all-non-zero\")\n .require_equals(true)\n .short('i')\n .help(\"Ignore failures of the benchmarked programs. Without a value or with \\\n 'all-non-zero', all non-zero exit codes are ignored. You can also provide \\\n a comma-separated list of exit codes to ignore (e.g., --ignore-failure=1,2).\"),\n )\n .arg(\n Arg::new(\"style\")\n .long(\"style\")\n .action(ArgAction::Set)\n .value_name(\"TYPE\")\n .value_parser([\"auto\", \"basic\", \"full\", \"nocolor\", \"color\", \"none\"])\n .help(\n \"Set output style type (default: auto). Set this to 'basic' to disable output \\\n coloring and interactive elements. Set it to 'full' to enable all effects \\\n even if no interactive terminal was detected. Set this to 'nocolor' to \\\n keep the interactive output without any colors. Set this to 'color' to keep \\\n the colors without any interactive output. Set this to 'none' to disable all \\\n the output of the tool.\",\n ),\n )\n .arg(\n Arg::new(\"sort\")\n .long(\"sort\")\n .action(ArgAction::Set)\n .value_name(\"METHOD\")\n .value_parser([\"auto\", \"command\", \"mean-time\"])\n .default_value(\"auto\")\n .hide_default_value(true)\n .help(\n \"Specify the sort order of the speed comparison summary and the exported tables for \\\n markup formats (Markdown, AsciiDoc, org-mode):\\n \\\n * 'auto' (default): the speed comparison will be ordered by time and\\n \\\n the markup tables will be ordered by command (input order).\\n \\\n * 'command': order benchmarks in the way they were specified\\n \\\n * 'mean-time': order benchmarks by mean runtime\\n\"\n ),\n )\n .arg(\n Arg::new(\"time-unit\")\n .long(\"time-unit\")\n .short('u')\n .action(ArgAction::Set)\n .value_name(\"UNIT\")\n .value_parser([\"microsecond\", \"millisecond\", \"second\"])\n .help(\"Set the time unit to be used. Possible values: microsecond, millisecond, second. \\\n If the option is not given, the time unit is determined automatically. \\\n This option affects the standard output as well as all export formats except for CSV and JSON.\"),\n )\n .arg(\n Arg::new(\"export-asciidoc\")\n .long(\"export-asciidoc\")\n .action(ArgAction::Set)\n .value_name(\"FILE\")\n .value_hint(ValueHint::FilePath)\n .help(\"Export the timing summary statistics as an AsciiDoc table to the given FILE. \\\n The output time unit can be changed using the --time-unit option.\"),\n )\n .arg(\n Arg::new(\"export-csv\")\n .long(\"export-csv\")\n .action(ArgAction::Set)\n .value_name(\"FILE\")\n .value_hint(ValueHint::FilePath)\n .help(\"Export the timing summary statistics as CSV to the given FILE. If you need \\\n the timing results for each individual run, use the JSON export format. \\\n The output time unit is always seconds.\"),\n )\n .arg(\n Arg::new(\"export-json\")\n .long(\"export-json\")\n .action(ArgAction::Set)\n .value_name(\"FILE\")\n .value_hint(ValueHint::FilePath)\n .help(\"Export the timing summary statistics and timings of individual runs as JSON to the given FILE. \\\n The output time unit is always seconds\"),\n )\n .arg(\n Arg::new(\"export-markdown\")\n .long(\"export-markdown\")\n .action(ArgAction::Set)\n .value_name(\"FILE\")\n .value_hint(ValueHint::FilePath)\n .help(\"Export the timing summary statistics as a Markdown table to the given FILE. \\\n The output time unit can be changed using the --time-unit option.\"),\n )\n .arg(\n Arg::new(\"export-orgmode\")\n .long(\"export-orgmode\")\n .action(ArgAction::Set)\n .value_name(\"FILE\")\n .value_hint(ValueHint::FilePath)\n .help(\"Export the timing summary statistics as an Emacs org-mode table to the given FILE. \\\n The output time unit can be changed using the --time-unit option.\"),\n )\n .arg(\n Arg::new(\"show-output\")\n .long(\"show-output\")\n .action(ArgAction::SetTrue)\n .conflicts_with(\"style\")\n .help(\n \"Print the stdout and stderr of the benchmark instead of suppressing it. \\\n This will increase the time it takes for benchmarks to run, \\\n so it should only be used for debugging purposes or \\\n when trying to benchmark output speed.\",\n ),\n )\n .arg(\n Arg::new(\"output\")\n .long(\"output\")\n .conflicts_with(\"show-output\")\n .action(ArgAction::Append)\n .value_name(\"WHERE\")\n .help(\n \"Control where the output of the benchmark is redirected. Note \\\n that some programs like 'grep' detect when standard output is \\\n /dev/null and apply certain optimizations. To avoid that, consider \\\n using '--output=pipe'.\\n\\\n \\n\\\n can be:\\n\\\n \\n \\\n null: Redirect output to /dev/null (the default).\\n\\\n \\n \\\n pipe: Feed the output through a pipe before discarding it.\\n\\\n \\n \\\n inherit: Don't redirect the output at all (same as '--show-output').\\n\\\n \\n \\\n : Write the output to the given file.\\n\\n\\\n This option can be specified once for all commands or multiple times, once for \\\n each command. Note: If you want to log the output of each and every iteration, \\\n you can use a shell redirection and the '$HYPERFINE_ITERATION' environment variable:\\n \\\n hyperfine 'my-command > output-${HYPERFINE_ITERATION}.log'\\n\\n\",\n ),\n )\n .arg(\n Arg::new(\"input\")\n .long(\"input\")\n .action(ArgAction::Set)\n .num_args(1)\n .value_name(\"WHERE\")\n .help(\"Control where the input of the benchmark comes from.\\n\\\n \\n\\\n can be:\\n\\\n \\n \\\n null: Read from /dev/null (the default).\\n\\\n \\n \\\n : Read the input from the given file.\"),\n )\n .arg(\n Arg::new(\"command-name\")\n .long(\"command-name\")\n .short('n')\n .action(ArgAction::Append)\n .num_args(1)\n .value_name(\"NAME\")\n .help(\"Give a meaningful name to a command. This can be specified multiple times \\\n if several commands are benchmarked.\"),\n )\n // This option is hidden for now, as it is not yet clear yet if we want to 'stabilize' this,\n // see discussion in https://github.com/sharkdp/hyperfine/issues/527\n .arg(\n Arg::new(\"min-benchmarking-time\")\n .long(\"min-benchmarking-time\")\n .action(ArgAction::Set)\n .hide(true)\n .help(\"Set the minimum time (in seconds) to run benchmarks. Note that the number of \\\n benchmark runs is additionally influenced by the `--min-runs`, `--max-runs`, and \\\n `--runs` option.\")\n )\n .arg(\n Arg::new(\"debug-mode\")\n .long(\"debug-mode\")\n .action(ArgAction::SetTrue)\n .hide(true)\n .help(\"Enable debug mode which does not actually run commands, but returns fake times when the command is 'sleep