Repository: sharkdp/hyperfine Branch: master Commit: 327d5f4d9107 Files: 62 Total size: 274.9 KB Directory structure: gitextract_zf1w7bdv/ ├── .github/ │ ├── dependabot.yml │ └── workflows/ │ └── CICD.yml ├── .gitignore ├── CHANGELOG.md ├── CITATION.cff ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── build.rs ├── doc/ │ ├── hyperfine.1 │ └── sponsors.md ├── scripts/ │ ├── README.md │ ├── advanced_statistics.py │ ├── plot_benchmark_comparison.py │ ├── plot_histogram.py │ ├── plot_parametrized.py │ ├── plot_progression.py │ ├── plot_whisker.py │ ├── ruff.toml │ └── welch_ttest.py ├── src/ │ ├── benchmark/ │ │ ├── benchmark_result.rs │ │ ├── executor.rs │ │ ├── mod.rs │ │ ├── relative_speed.rs │ │ ├── scheduler.rs │ │ └── timing_result.rs │ ├── cli.rs │ ├── command.rs │ ├── error.rs │ ├── export/ │ │ ├── asciidoc.rs │ │ ├── csv.rs │ │ ├── json.rs │ │ ├── markdown.rs │ │ ├── markup.rs │ │ ├── mod.rs │ │ ├── orgmode.rs │ │ └── tests.rs │ ├── main.rs │ ├── options.rs │ ├── outlier_detection.rs │ ├── output/ │ │ ├── format.rs │ │ ├── mod.rs │ │ ├── progress_bar.rs │ │ └── warnings.rs │ ├── parameter/ │ │ ├── mod.rs │ │ ├── range_step.rs │ │ └── tokenize.rs │ ├── timer/ │ │ ├── mod.rs │ │ ├── unix_timer.rs │ │ ├── wall_clock_timer.rs │ │ └── windows_timer.rs │ └── util/ │ ├── exit_code.rs │ ├── min_max.rs │ ├── mod.rs │ ├── number.rs │ ├── randomized_environment_offset.rs │ └── units.rs └── tests/ ├── common.rs ├── example_input_file.txt ├── execution_order_tests.rs └── integration_tests.rs ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: cargo directory: "/" schedule: interval: monthly time: "04:00" timezone: Europe/Berlin open-pull-requests-limit: 2 - package-ecosystem: "github-actions" directory: "/" schedule: interval: "daily" ================================================ FILE: .github/workflows/CICD.yml ================================================ name: CICD env: CICD_INTERMEDIATES_DIR: "_cicd-intermediates" MSRV_FEATURES: "" on: workflow_dispatch: pull_request: push: branches: - master tags: - '*' jobs: crate_metadata: name: Extract crate metadata runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - name: Extract crate information id: crate_metadata run: | cargo metadata --no-deps --format-version 1 | jq -r ' .packages[0] | [ "name=" + .name, "version=" + .version, "maintainer=" + (.authors[0] // ""), "homepage=" + (.homepage // ""), "msrv=" + (.rust_version // ""), "bin-name=" + ( (.targets[] | select(.kind[0] == "bin") | .name) // .name ) ] | join("\n") ' | tee -a $GITHUB_OUTPUT outputs: name: ${{ steps.crate_metadata.outputs.name }} version: ${{ steps.crate_metadata.outputs.version }} maintainer: ${{ steps.crate_metadata.outputs.maintainer }} homepage: ${{ steps.crate_metadata.outputs.homepage }} msrv: ${{ steps.crate_metadata.outputs.msrv }} bin-name: ${{ steps.crate_metadata.outputs.bin-name }} ensure_cargo_fmt: name: Ensure 'cargo fmt' has been run runs-on: ubuntu-24.04 steps: - uses: dtolnay/rust-toolchain@stable with: components: rustfmt - uses: actions/checkout@v5 - run: cargo fmt -- --check min_version: name: Minimum supported rust version runs-on: ubuntu-24.04 needs: crate_metadata steps: - name: Checkout source code uses: actions/checkout@v5 - name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }}) uses: dtolnay/rust-toolchain@master with: toolchain: ${{ needs.crate_metadata.outputs.msrv }} components: clippy - name: Run clippy (on minimum supported rust version to prevent warnings we can't fix) run: cargo clippy --locked --all-targets ${{ env.MSRV_FEATURES }} - name: Run tests run: cargo test --locked ${{ env.MSRV_FEATURES }} build: name: ${{ matrix.job.target }} (${{ matrix.job.os }}) runs-on: ${{ matrix.job.os }} needs: crate_metadata strategy: fail-fast: false matrix: job: - { target: aarch64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true } - { target: arm-unknown-linux-gnueabihf , os: ubuntu-24.04, use-cross: true } - { target: arm-unknown-linux-musleabihf, os: ubuntu-24.04, use-cross: true } - { target: i686-pc-windows-msvc , os: windows-2022 } - { target: i686-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true } - { target: i686-unknown-linux-musl , os: ubuntu-24.04, use-cross: true } - { target: x86_64-apple-darwin , os: macos-15 } - { target: aarch64-apple-darwin , os: macos-15 } # - { target: x86_64-pc-windows-gnu , os: windows-2022 } - { target: x86_64-pc-windows-msvc , os: windows-2022 } - { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true } - { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true } env: BUILD_CMD: cargo steps: - name: Checkout source code uses: actions/checkout@v5 - name: Install prerequisites shell: bash run: | case ${{ matrix.job.target }} in arm-unknown-linux-*) sudo apt-get -y update ; sudo apt-get -y install gcc-arm-linux-gnueabihf ;; aarch64-unknown-linux-gnu) sudo apt-get -y update ; sudo apt-get -y install gcc-aarch64-linux-gnu ;; esac - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: targets: ${{ matrix.job.target }} - name: Install cross if: matrix.job.use-cross uses: taiki-e/install-action@v2 with: tool: cross - name: Overwrite build command env variable if: matrix.job.use-cross shell: bash run: echo "BUILD_CMD=cross" >> $GITHUB_ENV - name: Show version information (Rust, cargo, GCC) shell: bash run: | set -x gcc --version || true rustup -V rustup toolchain list rustup default cargo -V rustc -V - name: Build shell: bash run: $BUILD_CMD build --locked --release --target=${{ matrix.job.target }} - name: Set binary name & path id: bin shell: bash run: | # Figure out suffix of binary EXE_suffix="" case ${{ matrix.job.target }} in *-pc-windows-*) EXE_suffix=".exe" ;; esac; # Setup paths BIN_NAME="${{ needs.crate_metadata.outputs.bin-name }}${EXE_suffix}" BIN_PATH="target/${{ matrix.job.target }}/release/${BIN_NAME}" # Let subsequent steps know where to find the binary echo "BIN_PATH=${BIN_PATH}" >> $GITHUB_OUTPUT echo "BIN_NAME=${BIN_NAME}" >> $GITHUB_OUTPUT - name: Set testing options id: test-options shell: bash run: | # test only library unit tests and binary for arm-type targets unset CARGO_TEST_OPTIONS unset CARGO_TEST_OPTIONS ; case ${{ matrix.job.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS="--bin ${{ steps.bin.outputs.BIN_NAME }}" ;; esac; echo "CARGO_TEST_OPTIONS=${CARGO_TEST_OPTIONS}" >> $GITHUB_OUTPUT - name: Run tests shell: bash run: $BUILD_CMD test --locked --target=${{ matrix.job.target }} ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS}} - name: Create tarball id: package shell: bash run: | PKG_suffix=".tar.gz" ; case ${{ matrix.job.target }} in *-pc-windows-*) PKG_suffix=".zip" ;; esac; PKG_BASENAME=${{ needs.crate_metadata.outputs.name }}-v${{ needs.crate_metadata.outputs.version }}-${{ matrix.job.target }} PKG_NAME=${PKG_BASENAME}${PKG_suffix} echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_OUTPUT PKG_STAGING="${{ env.CICD_INTERMEDIATES_DIR }}/package" ARCHIVE_DIR="${PKG_STAGING}/${PKG_BASENAME}/" mkdir -p "${ARCHIVE_DIR}" mkdir -p "${ARCHIVE_DIR}/autocomplete" # Binary cp "${{ steps.bin.outputs.BIN_PATH }}" "$ARCHIVE_DIR" # README, LICENSE and CHANGELOG files cp "README.md" "LICENSE-MIT" "LICENSE-APACHE" "CHANGELOG.md" "$ARCHIVE_DIR" # Man page cp 'doc/${{ needs.crate_metadata.outputs.name }}.1' "$ARCHIVE_DIR" # Autocompletion files cp 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'${{ needs.crate_metadata.outputs.name }}.bash' "$ARCHIVE_DIR/autocomplete/" cp 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'${{ needs.crate_metadata.outputs.name }}.fish' "$ARCHIVE_DIR/autocomplete/" cp 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'_${{ needs.crate_metadata.outputs.name }}.ps1' "$ARCHIVE_DIR/autocomplete/" cp 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'_${{ needs.crate_metadata.outputs.name }}' "$ARCHIVE_DIR/autocomplete/" # base compressed package pushd "${PKG_STAGING}/" >/dev/null case ${{ matrix.job.target }} in *-pc-windows-*) 7z -y a "${PKG_NAME}" "${PKG_BASENAME}"/* | tail -2 ;; *) tar czf "${PKG_NAME}" "${PKG_BASENAME}"/* ;; esac; popd >/dev/null # Let subsequent steps know where to find the compressed package echo "PKG_PATH=${PKG_STAGING}/${PKG_NAME}" >> $GITHUB_OUTPUT - name: Create Debian package id: debian-package shell: bash if: startsWith(matrix.job.os, 'ubuntu') run: | COPYRIGHT_YEARS="2018 - "$(date "+%Y") DPKG_STAGING="${{ env.CICD_INTERMEDIATES_DIR }}/debian-package" DPKG_DIR="${DPKG_STAGING}/dpkg" mkdir -p "${DPKG_DIR}" DPKG_BASENAME=${{ needs.crate_metadata.outputs.name }} DPKG_CONFLICTS=${{ needs.crate_metadata.outputs.name }}-musl case ${{ matrix.job.target }} in *-musl*) DPKG_BASENAME=${{ needs.crate_metadata.outputs.name }}-musl ; DPKG_CONFLICTS=${{ needs.crate_metadata.outputs.name }} ;; esac; DPKG_VERSION=${{ needs.crate_metadata.outputs.version }} unset DPKG_ARCH case ${{ matrix.job.target }} in aarch64-*-linux-*) DPKG_ARCH=arm64 ;; arm-*-linux-*hf) DPKG_ARCH=armhf ;; i686-*-linux-*) DPKG_ARCH=i686 ;; x86_64-*-linux-*) DPKG_ARCH=amd64 ;; *) DPKG_ARCH=notset ;; esac; DPKG_NAME="${DPKG_BASENAME}_${DPKG_VERSION}_${DPKG_ARCH}.deb" echo "DPKG_NAME=${DPKG_NAME}" >> $GITHUB_OUTPUT # Binary install -Dm755 "${{ steps.bin.outputs.BIN_PATH }}" "${DPKG_DIR}/usr/bin/${{ steps.bin.outputs.BIN_NAME }}" # Man page install -Dm644 'doc/${{ needs.crate_metadata.outputs.name }}.1' "${DPKG_DIR}/usr/share/man/man1/${{ needs.crate_metadata.outputs.name }}.1" gzip -n --best "${DPKG_DIR}/usr/share/man/man1/${{ needs.crate_metadata.outputs.name }}.1" # Autocompletion files install -Dm644 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'${{ needs.crate_metadata.outputs.name }}.bash' "${DPKG_DIR}/usr/share/bash-completion/completions/${{ needs.crate_metadata.outputs.name }}" install -Dm644 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'${{ needs.crate_metadata.outputs.name }}.fish' "${DPKG_DIR}/usr/share/fish/vendor_completions.d/${{ needs.crate_metadata.outputs.name }}.fish" install -Dm644 'target/${{ matrix.job.target }}/release/build/${{ needs.crate_metadata.outputs.name }}'*/out/'_${{ needs.crate_metadata.outputs.name }}' "${DPKG_DIR}/usr/share/zsh/vendor-completions/_${{ needs.crate_metadata.outputs.name }}" # README and LICENSE install -Dm644 "README.md" "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/README.md" install -Dm644 "LICENSE-MIT" "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/LICENSE-MIT" install -Dm644 "LICENSE-APACHE" "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/LICENSE-APACHE" install -Dm644 "CHANGELOG.md" "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/changelog" gzip -n --best "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/changelog" cat > "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/copyright" < "${DPKG_DIR}/DEBIAN/control" <> $GITHUB_OUTPUT # build dpkg fakeroot dpkg-deb --build "${DPKG_DIR}" "${DPKG_PATH}" - name: "Artifact upload: tarball" uses: actions/upload-artifact@master with: name: ${{ steps.package.outputs.PKG_NAME }} path: ${{ steps.package.outputs.PKG_PATH }} - name: "Artifact upload: Debian package" uses: actions/upload-artifact@master if: steps.debian-package.outputs.DPKG_NAME with: name: ${{ steps.debian-package.outputs.DPKG_NAME }} path: ${{ steps.debian-package.outputs.DPKG_PATH }} - name: Check for release id: is-release shell: bash run: | unset IS_RELEASE ; if [[ $GITHUB_REF =~ ^refs/tags/v[0-9].* ]]; then IS_RELEASE='true' ; fi echo "IS_RELEASE=${IS_RELEASE}" >> $GITHUB_OUTPUT - name: Publish archives and packages uses: softprops/action-gh-release@v2 if: steps.is-release.outputs.IS_RELEASE with: files: | ${{ steps.package.outputs.PKG_PATH }} ${{ steps.debian-package.outputs.DPKG_PATH }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} winget: name: Publish to Winget runs-on: ubuntu-latest needs: build if: startsWith(github.ref, 'refs/tags/v') steps: - uses: vedantmgoyal2009/winget-releaser@v2 with: identifier: sharkdp.hyperfine installers-regex: '-pc-windows-msvc\.zip$' token: ${{ secrets.WINGET_TOKEN }} ================================================ FILE: .gitignore ================================================ /target/ **/*.rs.bk ================================================ FILE: CHANGELOG.md ================================================ # v1.20.0 ## Features - Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally) - The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp) - Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp) - Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin) ## Bugfixes - Fix bug where naming individual commands with parameter scan was not working correctly, see #794 (@teofr) ## Other - Restrict `cat` tests to Unix environments, see #776 and #777 (@ritvikos) # v1.19.0 ## Features - Add a new `--reference ` option to specify a reference command for the relative speed comparison, see #579, #577 and #744 (@sharkdp) - Add `--conclude` argument (analog to `--prepare`), see #565 and #719 (@jackoconnordev) - Allow `--output=…` to appear once for each command, enabling use cases like `hyperfine --output=null my-cmd --output=./file.log my-cmd`, see #529 and #775 (@sharkdp) - The environment variable `$HYPERFINE_ITERATION` will now contain the current iteration number for each benchmarked command, see #775 (@sharkdp) - Add iteration information to failure error message, see #771 and #772 (@sharkdp) - Python scripts: - legend modification parameters and output DPI, see #758 (@Spreadcat) - Nicer whiskers plot, see #727 (@serpent7776) ## Bugfixes - ETA not clearly visible on terminals with a block cursor, see #698 and #699 (@overclockworked64) - Fix zsh completions, see #717 (@xzfc) ## Other - Build binaries for aarch64-apple-darwin, see #728 (@Phault) - Various cleanups (@hamirmahal, @one230six) # v1.18.0 ## Features - Add support for microseconds via `--time-unit microsecond`, see #684 (@sharkdp) ## Bugfixes - Proper argument quoting on Windows CMD, see #296 and #678 (@PedroWitzel) # v1.17.0 ## Features - Add new `--sort` option to control the order in the rel. speed comparison and in markup export formats, see #601, #614, #655 (@sharkdp) - Parameters which are unused in the command line are now displayed in parentheses, see #600 and #644 (@sharkdp). - Added `--log-count` option for histogram plots, see `scripts/plot_histogram.py` (@sharkdp) ## Changes - Updated hyperfine to use `windows-sys` instead of the unmaintained `winapi`, see #624, #639, #636, #641 (@clemenswasser) - Silenced deprecation warning in Python scripts, see #633 (@nicovank) - Major update of the man page, see 0ce6578, #647 (@sharkdp) ## Bugfixes - Do not export intermediate results to stdout when using `-` as a file name, see #640 and #643 (@sharkdp) - Markup exporting does not fail if benchmark results are zero, see #642 (@sharkdp) # v1.16.1 ## Bugfixes - Fix line-wrapping of `--help` text (@sharkdp) - Fix `--input=null` (@sharkdp) # v1.16.0 ## Features - Added new `--input` option, see #541 and #563 (@snease) - Added possibility to specify `-` as the filename in the `--export-*` options, see #615 and #623 (@humblepenguinn) ## Changes - Improve hints for outlier warnings if `--warmup` or `--prepare` are in use already, see #570 (@sharkdp) ## Bugfixes - Fix uncolored output on Windows if `TERM` is not set, see #583 (@nabijaczleweli) - On Windows, only run `cmd.exe` with the `/C` option. Use `-c` for all other shells. See #568 and #582 (@FilipAndersson245) ## Other - Thanks to @berombau for working on dependency upgrades, see #584 - Fixed installationm on Windows, see #595 and #596 (@AntoniosBarotsis) # v1.15.0 ## Features - Disable colorized output in case of `TERM=dumb` or `NO_COLOR=1`, see #542 and #555 (@nabijaczleweli) - Add new (experimental) `--min-benchmarking-time ` option, see #527 (@sharkdp) ## Bugfixes - Fix user and kernel times on Windows, see #368 and #538 (@clemenswasser) ## Other - Improve `--help` texts of `--export-*` options, see #506 and #522 (@Engineer-of-Efficiency) # v1.14.0 ## Features - Add a new `--output={null,pipe,inherit,}` option to control where the output of the benchmarked program is redirected (if at all), see #377 and #509 (@tavianator, originally suggested by @BurntSushi) - Add Emacs org-mode as a new export format, see #491 (@ppaulweber) # v1.13.0 ## Features - Added a new `--shell=none`/`-N` option to disable the intermediate shell for executing the benchmarked commands. Hyperfine normally measures and subtracts the shell spawning time, but the intermediate shell always introduces a certain level of measurement noise. Using `--shell=none`/`-N` allows users to benchmark very fast commands (with a runtime on the order of a few milliseconds). See #336, #429, and #487 (@cipriancraciun and @sharkdp) - Added `--setup`/`-s` option that can be used to run `make all` or similar. It runs once per set of tests, like `--cleanup`/`-c` (@avar) - Added new `plot_progression.py` script to debug background interference effects. ## Changes - Breaking change: the `-s` short option for `--style` is now used for the new `--setup` option. - The environment offset randomization is now also available on Windows, see #484 ## Other - Improved documentation and test coverage, cleaned up code base for future improvements. # v1.12.0 ## Features - `--command-name` can now take parameter names from `--parameter-*` options, see #351 and #391 (@silathdiir) - Exit codes (or signals) are now printed in cases of command failures, see #342 (@KaindlJulian) - Exit codes are now part of the JSON output, see #371 (@JordiChauzi) - Colorized output should now be enabled on Windows by default, see #427 ## Changes - When `--export-*` commands are used, result files are created before benchmark execution to fail early in case of, e.g., wrong permissions. See #306 (@s1ck). - When `--export-*` options are used, result files are written after each individual benchmark command instead of writing after all benchmarks have finished. See #306 (@s1ck). - Reduce number of shell startup time measurements from 200 to 50, generally speeding up benchmarks. See #378 - User and system time are now in consistent time units, see #408 and #409 (@film42) # v1.11.0 ## Features - The `-L`/`--parameter-list` option can now be specified multiple times to evaluate all possible combinations of the listed parameters: ``` bash hyperfine -L number 1,2 -L letter a,b,c \ "echo {number}{letter}" \ "printf '%s\n' {number}{letter}" # runs 12 benchmarks: 2 commands (echo and printf) times 6 combinations of # the "letter" and "number" parameters ``` See: #253, #318 (@wchargin) - Add CLI option to identify a command with a custom name, see #326 (@scampi) ## Changes - When parameters are used with `--parameter-list` or `--parameter-scan`, the JSON export format now contains a dictionary `parameters` instead of a single key `parameter`. See #253, #318. - The `plot_parametrized.py` script now infers the parameter name, and its `--parameter-name` argument has been deprecated. See #253, #318. ## Bugfixes - Fix a bug in the outlier detection which would only detect "slow outliers" but not the fast ones (runs that are much faster than the rest of the benchmarking runs), see #329 - Better error messages for very fast commands that would lead to inf/nan results in the relative speed comparison, see #319 - Show error message if `--warmup` or `--*runs` arguments can not be parsed, see #337 - Keep output colorized when the output is not interactive and `--style=full` or `--style=color` is used. # v1.10.0 ## Features - Hyperfine now comes with shell completion files for Bash, Zsh, Fish and PowerShell, see #290 (@four0000four). - Hyperfine now comes with a basic man page, see #257 (@cadeef) - During execution of benchmarks, hyperfine will now set a `HYPERFINE_RANDOMIZED_ENVIRONMENT_OFFSET` environment variable in order to randomize the memory layout. See #235 and #241 for references and details. - A few enhancements for the histogram plotting scripts and the advanced statistics script - Updates for the `plot_whisker.py` script, see #275 (@ghaiklor) ## Bugfixes - Fix Spin Icon on Windows, see #229 - A few typos have been fixed, see #292 (@McMartin) ## Packaging - `hyperfine` is now available on MacPorts for macOS, see #281 (@herbygillot) - `hyperfine` is now available on OpenBSD, see #289 (@minusf) Package authors: note that Hyperfine now comes with a set of shell completion files and a man page (see above) # v1.9.0 ## Features - The new `--parameter-list ` option can be used to run a parametrized benchmark on a user-specified list of values. This is similar to `--parameter-scan `, but doesn't necessarily required numeric arguments. ``` bash hyperfine --parameter-list compiler "gcc,clang" \ "{compiler} -O2 main.cpp" ``` See: #227, #234 (@JuanPotato) - Added `none` as a possible choice for the `--style` option to run `hyperfine` without any output, see #193 (@knidarkness) - Added a few new scripts for plotting various types of benchmark results (https://github.com/sharkdp/hyperfine/tree/master/scripts) ## Changes - The `--prepare` command is now also run during the warmup phase, see #182 (@sseemayer) - Better estimation of the remaining benchmark time due to an update of the `indicatif` crate. ## Other - `hyperfine` is now available on NixOS, see #240 (@tuxinaut) # v1.8.0 ## Features - The `--prepare ` option can now be specified multiple times to run specific preparation commands for each of the benchmarked programs: ``` bash hyperfine --prepare "make clean; git checkout master" "make" \ --prepare "make clean; git checkout feature" "make" ``` See: #216, #218 (@iamsauravsharma) - Added a new [`welch_ttest.py`](https://github.com/sharkdp/hyperfine/blob/master/scripts/welch_ttest.py) script to test whether or not the two benchmark results are the same, see #222 (@uetchy) - The Markdown export has been improved. The relative speed is now exported with a higher precision (see #208) and includes the standard deviation (see #225). ## Other - Improved documentation for [`scripts`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder (@matthieusb) # v1.7.0 ## Features - Added a new `-D`,`--parameter-step-size` option that can be used to control the step size for `--parameter-scan` benchmarks. In addition, decimal numbers are now allowed for parameter scans. For example, the following command runs `sleep 0.3`, `sleep 0.5` and `sleep 0.7`: ``` bash hyperfine --parameter-scan delay 0.3 0.7 -D 0.2 'sleep {delay}' ``` For more details, see #184 (@piyushrungta25) ## Other - hyperfine is now in the official Alpine repositories, see #177 (@maxice8, @5paceToast) - hyperfine is now in the official Fedora repositories, see #196 (@ignatenkobrain) - hyperfine is now in the official Arch Linux repositories - hyperfine can be installed on FreeBSD, see #204 (@0mp) - Enabled LTO for slightly smaller binary sizes, see #179 (@Calinou) - Various small improvements all over the code base, see #194 (@phimuemue) # v1.6.0 ## Features - Added a `-c, --cleanup ` option to execute `CMD` after the completion of all benchmarking runs for a given command. This is useful if the commands to be benchmarked produce artifacts that need to be cleaned up. See #91 (@RalfJung and @colinwahl) - Add parameter values (for `--parameter-scan` benchmarks) to exported CSV and JSON files. See #131 (@bbannier) - Added AsciiDoc export option, see #137 (@5paceToast) - The relative speed is now part of the Markdown export, see #127 (@mathiasrw and @sharkdp). - The *median* run time is now exported via CSV and JSON, see #171 (@hosewiejacke and @sharkdp). ## Other - Hyperfine has been updated to Rust 2018 (@AnderEnder). The minimum supported Rust version is now 1.31. # v1.5.0 ## Features - Show the number of runs in `hyperfine`s output (@tcmal) - Added two Python scripts to post-process exported benchmark results (see [`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder) ## Other - Refined `--help` text for the `--export-*` flags (@psteinb) - Added Snapcraft file (@popey) - Small improvements in the progress bar "experience". # v1.4.0 ## Features - Added `-S`/`--shell` option to override the default shell, see #61 (@mqudsi and @jasonpeacock) - Added `-u`/`--time-unit` option to change the unit of time (`second` or `millisecond`), see #80 (@jasonpeacock) - Markdown export auto-selects time unit, see #71 (@jasonpeacock) # v1.3.0 ## Feature - Compute and print standard deviation of the speed ratio, see #83 (@Shnatsel) - More compact output format, see #70 (@jasonpeacock) - Added `--style=color`, see #70 (@jasonpeacock) - Added options to specify the max/exact numbers of runs, see #77 (@orium) ## Bugfixes - Change Windows `cmd` interpreter to `cmd.exe` to prevent accidentally calling other programs, see #74 (@tathanhdinh) ## Other - Binary releases for Windows are now available, see #87 # v1.2.0 - Support parameters in preparation commands, see #68 (@siiptuo) - Updated dependencies, see #69. The minimum required Rust version is now 1.24. # v1.1.0 * Added `--show-output` option (@chrisduerr and @sevagh) * Refactoring work (@stevepentland) # v1.0.0 ## Features * Support for various export-formats like CSV, JSON and Markdown - see #38, #44, #49, #42 (@stevepentland) * Summary output that compares the different benchmarks, see #6 (@stevepentland) * Parameterized benchmarks via `-P`, `--parameter-scan `, see #19 ## Thanks I'd like to say a big THANK YOU to @stevepentland for implementing new features, for reviewing pull requests and for giving very valuable feedback. # v0.5.0 * Proper Windows support (@stevepentland) * Added `--style auto/basic/nocolor/full` option (@stevepentland) * Correctly estimate the full execution time, see #27 (@rleungx) * Added Void Linux install instructions (@wpbirney) # v0.4.0 - New `--style` option to disable output coloring and interactive CLI features, see #24 (@stevepentland) - Statistical outlier detection, see #23 #18 # v0.3.0 ## Features - In addition to 'real' (wall clock) time, Hyperfine can now also measure 'user' and 'system' time (see #5). - Added `--prepare` option that can be used to clear up disk caches before timing runs, for example (see #8). ## Other - [Arch Linux package](https://aur.archlinux.org/packages/hyperfine) for Hyperfine (@jD91mZM2). - Ubuntu/Debian packages are now are available. # v0.2.0 Initial public release ================================================ FILE: CITATION.cff ================================================ cff-version: 1.2.0 title: hyperfine message: >- If you use this software in scientific publications, please consider citing it using the metadata from this file. type: software authors: - given-names: David family-names: Peter email: mail@david-peter.de orcid: 'https://orcid.org/0000-0001-7950-9915' repository-code: 'https://github.com/sharkdp/hyperfine' abstract: A command-line benchmarking tool. license: MIT version: 1.16.1 date-released: '2023-03-21' ================================================ FILE: Cargo.toml ================================================ [package] authors = ["David Peter "] categories = ["command-line-utilities"] description = "A command-line benchmarking tool" homepage = "https://github.com/sharkdp/hyperfine" license = "MIT OR Apache-2.0" name = "hyperfine" readme = "README.md" repository = "https://github.com/sharkdp/hyperfine" version = "1.20.0" edition = "2018" build = "build.rs" rust-version = "1.88.0" [features] # Use the nightly feature windows_process_extensions_main_thread_handle windows_process_extensions_main_thread_handle = [] [dependencies] colored = "2.1" indicatif = "=0.17.4" statistical = "1.0" csv = "1.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" rust_decimal = "1.36" rand = "0.8" shell-words = "1.0" thiserror = "2.0" anyhow = "1.0" [target.'cfg(not(windows))'.dependencies] libc = "0.2" [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.59", features = [ "Win32_Foundation", "Win32_Security", "Win32_System_JobObjects", "Win32_System_LibraryLoader", "Win32_System_Threading", ] } [target.'cfg(all(windows, not(windows_process_extensions_main_thread_handle)))'.dependencies] once_cell = "1.19" [target.'cfg(target_os="linux")'.dependencies] nix = { version = "0.29", features = ["zerocopy"] } [dependencies.clap] version = "4" default-features = false features = [ "suggestions", "color", "wrap_help", "cargo", "help", "usage", "error-context", ] [dev-dependencies] approx = "0.5" assert_cmd = "2.0" insta = { version = "1.41.1", features = ["yaml"] } predicates = "3.1" tempfile = "3.23" [profile.dev.package] insta.opt-level = 3 similar.opt-level = 3 [build-dependencies] clap = "4.5.48" clap_complete = "4.2.1" [profile.release] lto = true strip = true codegen-units = 1 ================================================ FILE: LICENSE-APACHE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: LICENSE-MIT ================================================ MIT License Copyright (c) 2018-2022 David Peter, and all hyperfine contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # hyperfine [![CICD](https://github.com/sharkdp/hyperfine/actions/workflows/CICD.yml/badge.svg)](https://github.com/sharkdp/hyperfine/actions/workflows/CICD.yml) [![Version info](https://img.shields.io/crates/v/hyperfine.svg)](https://crates.io/crates/hyperfine) [中文](https://github.com/chinanf-boy/hyperfine-zh) A command-line benchmarking tool. **Demo**: Benchmarking [`fd`](https://github.com/sharkdp/fd) and [`find`](https://www.gnu.org/software/findutils/): ![hyperfine](https://i.imgur.com/z19OYxE.gif) ### Sponsors A special *thank you* goes to our biggest sponsor:
Warp
Warp, the intelligent terminal
Available on MacOS, Linux, Windows ## Features * Statistical analysis across multiple runs. * Support for arbitrary shell commands. * Constant feedback about the benchmark progress and current estimates. * Warmup runs can be executed before the actual benchmark. * Cache-clearing commands can be set up before each timing run. * Statistical outlier detection to detect interference from other programs and caching effects. * Export results to various formats: CSV, JSON, Markdown, AsciiDoc. * Parameterized benchmarks (e.g. vary the number of threads). * Cross-platform ## Usage ### Basic benchmarks To run a benchmark, you can simply call `hyperfine ...`. The argument(s) can be any shell command. For example: ```sh hyperfine 'sleep 0.3' ``` Hyperfine will automatically determine the number of runs to perform for each command. By default, it will perform *at least* 10 benchmarking runs and measure for at least 3 seconds. To change this, you can use the `-r`/`--runs` option: ```sh hyperfine --runs 5 'sleep 0.3' ``` If you want to compare the runtimes of different programs, you can pass multiple commands: ```sh hyperfine 'hexdump file' 'xxd file' ``` ### Warmup runs and preparation commands For programs that perform a lot of disk I/O, the benchmarking results can be heavily influenced by disk caches and whether they are cold or warm. If you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to perform a certain number of program executions before the actual benchmark: ```sh hyperfine --warmup 3 'grep -R TODO *' ``` Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare` option to run a special command before *each* timing run. For example, to clear harddisk caches on Linux, you can run ```sh sync; echo 3 | sudo tee /proc/sys/vm/drop_caches ``` To use this specific command with hyperfine, call `sudo -v` to temporarily gain sudo permissions and then call: ```sh hyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R TODO *' ``` ### Parameterized benchmarks If you want to run a series of benchmarks where a single parameter is varied (say, the number of threads), you can use the `-P`/`--parameter-scan` option and call: ```sh hyperfine --prepare 'make clean' --parameter-scan num_threads 1 12 'make -j {num_threads}' ``` This also works with decimal numbers. The `-D`/`--parameter-step-size` option can be used to control the step size: ```sh hyperfine --parameter-scan delay 0.3 0.7 -D 0.2 'sleep {delay}' ``` This runs `sleep 0.3`, `sleep 0.5` and `sleep 0.7`. For non-numeric parameters, you can also supply a list of values with the `-L`/`--parameter-list` option: ``` hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp' ``` ### Intermediate shell By default, commands are executed using a predefined shell (`/bin/sh` on Unix, `cmd.exe` on Windows). If you want to use a different shell, you can use the `-S, --shell ` option: ```sh hyperfine --shell zsh 'for i in {1..10000}; do echo test; done' ``` Note that hyperfine always *corrects for the shell spawning time*. To do this, it performs a calibration procedure where it runs the shell with an empty command (multiple times), to measure the startup time of the shell. It will then subtract this time from the total to show the actual time used by the command in question. If you want to run a benchmark *without an intermediate shell*, you can use the `-N` or `--shell=none` option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would produce a significant amount of noise. Note that you cannot use shell syntax like `*` or `~` in this case. ``` hyperfine -N 'grep TODO /home/user' ``` ### Shell functions and aliases If you are using bash, you can export shell functions to directly benchmark them with hyperfine: ```bash my_function() { sleep 1; } export -f my_function hyperfine --shell=bash my_function ``` Otherwise, inline them into or source them from the benchmarked program: ```sh hyperfine 'my_function() { sleep 1; }; my_function' echo 'alias my_alias="sleep 1"' > /tmp/my_alias.sh hyperfine '. /tmp/my_alias.sh; my_alias' ``` ### Exporting results Hyperfine has multiple options for exporting benchmark results to CSV, JSON, Markdown and other formats (see `--help` text for details). #### Markdown You can use the `--export-markdown ` option to create tables like the following: | Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| | `find . -iregex '.*[0-9]\.jpg$'` | 2.275 ± 0.046 | 2.243 | 2.397 | 9.79 ± 0.22 | | `find . -iname '*[0-9].jpg'` | 1.427 ± 0.026 | 1.405 | 1.468 | 6.14 ± 0.13 | | `fd -HI '.*[0-9]\.jpg$'` | 0.232 ± 0.002 | 0.230 | 0.236 | 1.00 | #### JSON The JSON output is useful if you want to analyze the benchmark results in more detail. The [`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder includes a lot of helpful Python programs to further analyze benchmark results and create helpful visualizations, like a histogram of runtimes or a whisker plot to compare multiple benchmarks: | ![](doc/histogram.png) | ![](doc/whisker.png) | |---:|---:| ### Detailed benchmark flowchart The following chart explains the execution order of various timing runs when using options like `--warmup`, `--prepare `, `--setup ` or `--cleanup `: ![](doc/execution-order.png) ## Installation [![Packaging status](https://repology.org/badge/vertical-allrepos/hyperfine.svg?columns=3&exclude_unsupported=1)](https://repology.org/project/hyperfine/versions) ### On Ubuntu On Ubuntu, hyperfine can be installed [from the official repositories](https://launchpad.net/ubuntu/+source/rust-hyperfine): ``` apt install hyperfine ``` Alternatively, for the latest version, you can download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases) and install it via `dpkg`: ``` wget https://github.com/sharkdp/hyperfine/releases/download/v1.20.0/hyperfine_1.20.0_amd64.deb sudo dpkg -i hyperfine_1.20.0_amd64.deb ``` ### On Fedora On Fedora, hyperfine can be installed from the official repositories: ```sh dnf install hyperfine ``` ### On Alpine Linux On Alpine Linux, hyperfine can be installed [from the official repositories](https://pkgs.alpinelinux.org/packages?name=hyperfine): ``` apk add hyperfine ``` ### On Arch Linux On Arch Linux, hyperfine can be installed [from the official repositories](https://archlinux.org/packages/extra/x86_64/hyperfine/): ``` pacman -S hyperfine ``` ### On Debian Linux On Debian Linux, hyperfine can be installed [from the official repositories](https://packages.debian.org/hyperfine): ``` apt install hyperfine ``` ### On Exherbo Linux On Exherbo Linux, hyperfine can be installed [from the rust repositories](https://gitlab.exherbo.org/exherbo/rust/-/tree/master/packages/sys-apps/hyperfine): ``` cave resolve -x repository/rust cave resolve -x hyperfine ``` ### On Funtoo Linux On Funtoo Linux, hyperfine can be installed [from core-kit](https://github.com/funtoo/core-kit/tree/1.4-release/app-benchmarks/hyperfine): ``` emerge app-benchmarks/hyperfine ``` ### On NixOS On NixOS, hyperfine can be installed [from the official repositories](https://nixos.org/nixos/packages.html?query=hyperfine): ``` nix-env -i hyperfine ``` ### On Flox On Flox, hyperfine can be installed as follows. ``` flox install hyperfine ``` Hyperfine's version in Flox follows that of Nix. ### On openSUSE On openSUSE, hyperfine can be installed [from the official repositories](https://software.opensuse.org/package/hyperfine): ``` zypper install hyperfine ``` ### On Void Linux Hyperfine can be installed via xbps ``` xbps-install -S hyperfine ``` ### On macOS Hyperfine can be installed via [Homebrew](https://brew.sh): ``` brew install hyperfine ``` Or you can install using [MacPorts](https://www.macports.org): ``` sudo port selfupdate sudo port install hyperfine ``` ### On FreeBSD Hyperfine can be installed via pkg: ``` pkg install hyperfine ``` ### On OpenBSD ``` doas pkg_add hyperfine ``` ### On Windows Hyperfine can be installed via [Chocolatey](https://community.chocolatey.org/packages/hyperfine), [Scoop](https://scoop.sh/#/apps?q=hyperfine&s=0&d=1&o=true&id=8f7c10f75ecf5f9e42a862c615257328e2f70f61), or [Winget](https://github.com/microsoft/winget-pkgs/tree/master/manifests/s/sharkdp/hyperfine): ``` choco install hyperfine ``` ``` scoop install hyperfine ``` ``` winget install hyperfine ``` ### With conda Hyperfine can be installed via [`conda`](https://conda.io/en/latest/) from the [`conda-forge`](https://anaconda.org/conda-forge/hyperfine) channel: ``` conda install -c conda-forge hyperfine ``` ### With cargo (Linux, macOS, Windows) Hyperfine can be installed from source via [cargo](https://doc.rust-lang.org/cargo/): ``` cargo install --locked hyperfine ``` Make sure that you use Rust 1.76 or newer. ### From binaries (Linux, macOS, Windows) Download the corresponding archive from the [Release page](https://github.com/sharkdp/hyperfine/releases). ## Alternative tools Hyperfine is inspired by [bench](https://github.com/Gabriella439/bench). ## Integration with other tools [Chronologer](https://github.com/dandavison/chronologer) is a tool that uses `hyperfine` to visualize changes in benchmark timings across your Git history. [Bencher](https://github.com/bencherdev/bencher) is a continuous benchmarking tool that supports `hyperfine` to track benchmarks and catch performance regressions in CI. Drop hyperfine JSON outputs onto the [Venz](https://try.venz.dev) chart to visualize the results, and manage hyperfine configurations. Make sure to check out the [`scripts` folder](https://github.com/sharkdp/hyperfine/tree/master/scripts) in this repository for a set of tools to work with `hyperfine` benchmark results. ## Origin of the name The name *hyperfine* was chosen in reference to the hyperfine levels of caesium 133 which play a crucial role in the [definition of our base unit of time](https://en.wikipedia.org/wiki/Second#History_of_definition) — the second. ## Citing hyperfine Thank you for considering to cite hyperfine in your research work. Please see the information in the sidebar on how to properly cite hyperfine. ## License `hyperfine` is dual-licensed under the terms of the MIT License and the Apache License 2.0. See the [LICENSE-APACHE](LICENSE-APACHE) and [LICENSE-MIT](LICENSE-MIT) files for details. ================================================ FILE: build.rs ================================================ use std::fs; use clap_complete::{generate_to, Shell}; include!("src/cli.rs"); fn main() { let var = std::env::var_os("SHELL_COMPLETIONS_DIR").or_else(|| std::env::var_os("OUT_DIR")); let outdir = match var { None => return, Some(outdir) => outdir, }; fs::create_dir_all(&outdir).unwrap(); let mut command = build_command(); for shell in [ Shell::Bash, Shell::Fish, Shell::Zsh, Shell::PowerShell, Shell::Elvish, ] { generate_to(shell, &mut command, "hyperfine", &outdir).unwrap(); } } ================================================ FILE: doc/hyperfine.1 ================================================ .TH HYPERFINE 1 .SH NAME hyperfine \- command\-line benchmarking tool .SH SYNOPSIS .B hyperfine .RB [ \-ihVN ] .RB [ \-\-warmup .IR NUM ] .RB [ \-\-min\-runs .IR NUM ] .RB [ \-\-max\-runs .IR NUM ] .RB [ \-\-runs .IR NUM ] .RB [ \-\-setup .IR CMD ] .RB [ \-\-prepare .IR CMD ] .RB [ \-\-conclude .IR CMD ] .RB [ \-\-cleanup .IR CMD ] .RB [ \-\-parameter\-scan .IR VAR .IR MIN .IR MAX ] .RB [ \-\-parameter\-step\-size .IR DELTA ] .RB [ \-\-parameter\-list .IR VAR .IR VALUES ] .RB [ \-\-shell .IR SHELL ] .RB [ \-\-style .IR TYPE ] .RB [ \-\-sort .IR METHOD ] .RB [ \-\-time-unit .IR UNIT ] .RB [ \-\-export\-asciidoc .IR FILE ] .RB [ \-\-export\-csv .IR FILE ] .RB [ \-\-export\-json .IR FILE ] .RB [ \-\-export\-markdown .IR FILE ] .RB [ \-\-export\-orgmode .IR FILE ] .RB [ \-\-output .IR WHERE ] .RB [ \-\-input .IR WHERE ] .RB [ \-\-command\-name .IR NAME ] .RI [ COMMAND... ] .SH DESCRIPTION A command\-line benchmarking tool which includes: .LP .RS * Statistical analysis across multiple runs .RE .RS * Support for arbitrary shell commands .RE .RS * Constant feedback about the benchmark progress and current estimates .RE .RS * Warmup runs can be executed before the actual benchmark .RE .RS * Cache-clearing commands can be set up before each timing run .RE .RS * Statistical outlier detection to detect interference from other programs and caching effects .RE .RS * Export results to various formats: CSV, JSON, Markdown, AsciiDoc .RE .RS * Parameterized benchmarks (e.g. vary the number of threads) .RE .SH OPTIONS .HP \fB\-w\fR, \fB\-\-warmup\fR \fINUM\fP .IP Perform \fINUM\fP warmup runs before the actual benchmark. This can be used to fill (disk) caches for I/O\-heavy programs. .HP \fB\-m\fR, \fB\-\-min\-runs\fR \fINUM\fP .IP Perform at least \fINUM\fP runs for each command. Default: 10. .HP \fB\-M\fR, \fB\-\-max\-runs\fR \fINUM\fP .IP Perform at most \fINUM\fP runs for each command. By default, there is no limit. .HP \fB\-r\fR, \fB\-\-runs\fR \fINUM\fP .IP Perform exactly \fINUM\fP runs for each command. If this option is not specified, \fBhyperfine\fR automatically determines the number of runs. .HP \fB\-s\fR, \fB\-\-setup\fR \fICMD...\fP .IP Execute \fICMD\fP once before each set of timing runs. This is useful for compiling your software or with the provided parameters, or to do any other work that should happen once before a series of benchmark runs, not every time as would happen with the \fB\-\-prepare\fR option. .HP \fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP .IP Execute \fICMD\fP before each timing run. This is useful for clearing disk caches, for example. The \fB\-\-prepare\fR option can be specified once for all commands or multiple times, once for each command. In the latter case, each preparation command will be run prior to the corresponding benchmark command. .HP \fB\-\-conclude\fR \fICMD...\fP .IP Execute \fICMD\fP after each timing run. This is useful for clearing disk caches, for example. The \fB\-\-conclude\fR option can be specified once for all commands or multiple times, once for each command. In the latter case, each conclusion command will be run after the corresponding benchmark command. .HP \fB\-c\fR, \fB\-\-cleanup\fR \fICMD...\fP .IP Execute \fICMD\fP after the completion of all benchmarking runs for each individual command to be benchmarked. This is useful if the commands to be benchmarked produce artifacts that need to be cleaned up. It only runs once a series of benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after every run. .HP \fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP .IP Perform benchmark runs for each value in the range \fIMIN..MAX\fP. Replaces the string '{\fIVAR\fP}' in each command by the current parameter value. .IP .RS Example: .RS \fBhyperfine\fR \fB\-P\fR threads 1 8 'make \-j {threads}' .RE .RE .IP This performs benchmarks for 'make \-j 1', 'make \-j 2', ..., 'make \-j 8'. .IP To have the value increase following different patterns, use shell arithmetics. .IP .RS Example: .RS \fBhyperfine\fR \fB\-P\fR size 0 3 'sleep $((2**{size}))' .RE .RE .IP This performs benchmarks with power of 2 increases: 'sleep 1', 'sleep 2', 'sleep 4', ... .IP The exact syntax may vary depending on your shell and OS. .HP \fB\-D\fR, \fB\-\-parameter\-step\-size\fR \fIDELTA\fP .IP This argument requires \fB\-\-parameter\-scan\fR to be specified as well. Traverse the range \fIMIN..MAX\fP in steps of \fIDELTA\fP. .IP .RS Example: .RS \fBhyperfine\fR \fB\-P\fR delay 0.3 0.7 \fB\-D\fR 0.2 'sleep {delay}' .RE .RE .IP This performs benchmarks for 'sleep 0.3', 'sleep 0.5' and 'sleep 0.7'. .HP \fB\-L\fR, \fB\-\-parameter\-list\fR \fIVAR\fP \fIVALUES\fP .IP Perform benchmark runs for each value in the comma\-separated list of \fIVALUES\fP. Replaces the string '{\fIVAR\fP}' in each command by the current parameter value. .IP .RS Example: .RS \fBhyperfine\fR \fB\-L\fR compiler gcc,clang '{compiler} \-O2 main.cpp' .RE .RE .IP This performs benchmarks for 'gcc \-O2 main.cpp' and 'clang \-O2 main.cpp'. .IP The option can be specified multiple times to run benchmarks for all possible parameter combinations. .HP \fB\-S\fR, \fB\-\-shell\fR \fISHELL\fP .IP Set the shell to use for executing benchmarked commands. This can be the name or the path to the shell executable, or a full command line like "bash \fB\-\-norc\fR". It can also be set to "default" to explicitly select the default shell on this platform. Finally, this can also be set to "none" to disable the shell. In this case, commands will be executed directly. They can still have arguments, but more complex things like "sleep 0.1; sleep 0.2" are not possible without a shell. .HP \fB\-N\fR .IP An alias for '\-\-shell=none'. .HP \fB\-i\fR, \fB\-\-ignore\-failure\fR .IP Ignore non\-zero exit codes of the benchmarked programs. .HP \fB\-\-style\fR \fITYPE\fP .IP Set output style \fITYPE\fP (default: auto). Set this to 'basic' to disable output coloring and interactive elements. Set it to 'full' to enable all effects even if no interactive terminal was detected. Set this to 'nocolor' to keep the interactive output without any colors. Set this to 'color' to keep the colors without any interactive output. Set this to 'none' to disable all the output of the tool. .HP \fB\-\-sort\fR \fIMETHOD\fP .IP Specify the sort order of the speed comparison summary and the exported tables for markup formats (Markdown, AsciiDoc, org\-mode): .RS .IP "auto (default)" the speed comparison will be ordered by time and the markup tables will be ordered by command (input order). .IP "command" order benchmarks in the way they were specified .IP "mean\-time" order benchmarks by mean runtime .RE .HP \fB\-u\fR, \fB\-\-time\-unit\fR \fIUNIT\fP .IP Set the time unit to be used. Possible values: microsecond, millisecond, second. If the option is not given, the time unit is determined automatically. This option affects the standard output as well as all export formats except for CSV and JSON. .HP \fB\-\-export\-asciidoc\fR \fIFILE\fP .IP Export the timing summary statistics as an AsciiDoc table to the given \fIFILE\fP. The output time unit can be changed using the \fB\-\-time\-unit\fR option. .HP \fB\-\-export\-csv\fR \fIFILE\fP .IP Export the timing summary statistics as CSV to the given \fIFILE\fP. If you need the timing results for each individual run, use the JSON export format. The output time unit is always seconds. .HP \fB\-\-export\-json\fR \fIFILE\fP .IP Export the timing summary statistics and timings of individual runs as JSON to the given \fIFILE\fP. The output time unit is always seconds. .HP \fB\-\-export\-markdown\fR \fIFILE\fP .IP Export the timing summary statistics as a Markdown table to the given \fIFILE\fP. The output time unit can be changed using the \fB\-\-time\-unit\fR option. .HP \fB\-\-export\-orgmode\fR \fIFILE\fP .IP Export the timing summary statistics as an Emacs org\-mode table to the given \fIFILE\fP. The output time unit can be changed using the \fB\-\-time\-unit\fR option. .HP \fB\-\-show\-output\fR .IP Print the stdout and stderr of the benchmark instead of suppressing it. This will increase the time it takes for benchmarks to run, so it should only be used for debugging purposes or when trying to benchmark output speed. .HP \fB\-\-output\fR \fIWHERE\fP .IP Control where the output of the benchmark is redirected. Note that some programs like 'grep' detect when standard output is \fI\,/dev/null\/\fP and apply certain optimizations. To avoid that, consider using \-\-output=pipe. .IP \fIWHERE\fP can be: .RS .IP null Redirect output to \fI\,/dev/null\/\fP (the default). .IP pipe Feed the output through a pipe before discarding it. .IP inherit Don't redirect the output at all (same as \&'\-\-show\-output'). .IP "" Write the output to the given file. .RE .IP This option can be specified once for all commands or multiple times, once for each command. Note: If you want to log the output of each and every iteration, you can use a shell redirection and the $HYPERFINE_ITERATION environment variable: 'my-command > output-${HYPERFINE_ITERATION}.log' .HP \fB\-\-input\fR \fIWHERE\fP .IP Control where the input of the benchmark comes from. .IP \fIWHERE\fP can be: .RS .IP null Read from \fI\,/dev/null\/\fP (the default). .IP "" Read the input from the given file. .RE .HP \fB\-n\fR, \fB\-\-command\-name\fR \fiNAME\fP .IP Give a meaningful \fiNAME\fP to a command. This can be specified multiple times if several commands are benchmarked. .HP \fB\-h\fR, \fB\-\-help\fR .IP Print help .HP \fB\-V\fR, \fB\-\-version\fR .IP Print version .SH EXAMPLES .LP Basic benchmark of 'find . -name todo.txt': .RS .nf \fBhyperfine\fR 'find . -name todo.txt' .fi .RE .LP Perform benchmarks for 'sleep 0.2' and 'sleep 3.2' with a minimum 5 runs each: .RS .nf \fBhyperfine\fR \fB\-\-min\-runs\fR 5 'sleep 0.2' 'sleep 3.2' .fi .RE .LP Perform a benchmark of 'grep' with a warm disk cache by executing 3 runs up front that are not part of the measurement: .RS .nf \fBhyperfine\fR \fB\-\-warmup\fR 3 'grep -R TODO *' .fi .RE .LP Export the results of a parameter scan benchmark to a markdown table: .RS .nf \fBhyperfine\fR \fB\-\-export\-markdown\fR output.md \fB\-\-parameter-scan\fR time 1 5 'sleep {time}' .fi .RE .LP Demonstrate when each of \fB\-\-setup\fR, \fB\-\-prepare\fR, \fB\-\-conclude\fR, \fIcmd\fP and \fB\-\-cleanup\fR will run: .RS .nf \fBhyperfine\fR \fB\-L\fR n 1,2 \fB\-r\fR 2 \fB\-\-show-output\fR \\ \fB\-\-setup\fR 'echo setup n={n}' \\ \fB\-\-prepare\fR 'echo prepare={n}' \\ \fB\-\-conclude\fR 'echo conclude={n}' \\ \fB\-\-cleanup\fR 'echo cleanup n={n}' \\ 'echo command n={n}' .fi .RE .RE .SH AUTHOR .LP David Peter .LP Source, bug tracker, and additional information can be found on GitHub: .I https://github.com/sharkdp/hyperfine ================================================ FILE: doc/sponsors.md ================================================ ## Sponsors `hyperfine` development is sponsored by many individuals and companies. Thank you very much! Please note, that being sponsored does not affect the individuality of the `hyperfine` project or affect the maintainers' actions in any way. We remain impartial and continue to assess pull requests solely on merit - the features added, bugs solved, and effect on the overall complexity of the code. No issue will have a different priority based on sponsorship status of the reporter. Contributions from anybody are most welcomed. If you want to see our biggest sponsors, check the top of [`README.md`](../README.md#sponsors). ================================================ FILE: scripts/README.md ================================================ This folder contains scripts that can be used in combination with hyperfines `--export-json` option. ### Example: ```bash hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json ./plot_whisker.py sleep.json ``` ### Pre-requisites To make these scripts work, you will need `numpy`, `matplotlib` and `scipy`. If you have a Python package manager that understands [PEP-723](https://peps.python.org/pep-0723/) inline script requirements like [`uv`](https://github.com/astral-sh/uv) or [`pipx`](https://github.com/pypa/pipx), you can directly run the scripts using ```bash uv run plot_whisker.py sleep.json ``` Otherwise, install the dependencies via your system package manager or using `pip`: ```bash pip install numpy matplotlib scipy # pip3, if you are using python3 ``` ================================================ FILE: scripts/advanced_statistics.py ================================================ #!/usr/bin/env python # /// script # requires-python = ">=3.10" # dependencies = [ # "numpy", # ] # /// import argparse import json from enum import Enum import numpy as np class Unit(Enum): SECOND = 1 MILLISECOND = 2 def factor(self): match self: case Unit.SECOND: return 1 case Unit.MILLISECOND: return 1e3 def __str__(self): match self: case Unit.SECOND: return "s" case Unit.MILLISECOND: return "ms" parser = argparse.ArgumentParser() parser.add_argument("file", help="JSON file with benchmark results") parser.add_argument( "--time-unit", help="The unit of time.", default="second", action="store", choices=["second", "millisecond"], dest="unit", ) args = parser.parse_args() unit = Unit.MILLISECOND if args.unit == "millisecond" else Unit.SECOND unit_str = str(unit) with open(args.file) as f: results = json.load(f)["results"] commands = [b["command"] for b in results] times = [b["times"] for b in results] for command, ts in zip(commands, times): ts = [t * unit.factor() for t in ts] p05 = np.percentile(ts, 5) p25 = np.percentile(ts, 25) p75 = np.percentile(ts, 75) p95 = np.percentile(ts, 95) iqr = p75 - p25 print(f"Command '{command}'") print(f" runs: {len(ts):8d}") print(f" mean: {np.mean(ts):8.3f} {unit_str}") print(f" stddev: {np.std(ts, ddof=1):8.3f} {unit_str}") print(f" median: {np.median(ts):8.3f} {unit_str}") print(f" min: {np.min(ts):8.3f} {unit_str}") print(f" max: {np.max(ts):8.3f} {unit_str}") print() print(" percentiles:") print(f" P_05 .. P_95: {p05:.3f} {unit_str} .. {p95:.3f} {unit_str}") print( f" P_25 .. P_75: {p25:.3f} {unit_str} .. {p75:.3f} {unit_str} (IQR = {iqr:.3f} {unit_str})" ) print() ================================================ FILE: scripts/plot_benchmark_comparison.py ================================================ #!/usr/bin/env python # /// script # requires-python = ">=3.10" # dependencies = [ # "matplotlib", # "pyqt6", # "numpy", # ] # /// """ This script shows `hyperfine` benchmark results as a bar plot grouped by command. Note all the input files must contain results for all commands. """ import argparse import json import pathlib import matplotlib.pyplot as plt import numpy as np parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results" ) parser.add_argument("--title", help="Plot Title") parser.add_argument( "--benchmark-names", nargs="+", help="Names of the benchmark groups" ) parser.add_argument("-o", "--output", help="Save image to the given filename") args = parser.parse_args() commands = None data = [] inputs = [] if args.benchmark_names: assert len(args.files) == len( args.benchmark_names ), "Number of benchmark names must match the number of input files." for i, filename in enumerate(args.files): with open(filename) as f: results = json.load(f)["results"] benchmark_commands = [b["command"] for b in results] if commands is None: commands = benchmark_commands else: assert ( commands == benchmark_commands ), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}" data.append([round(b["mean"], 2) for b in results]) if args.benchmark_names: inputs.append(args.benchmark_names[i]) else: inputs.append(filename.stem) data = np.transpose(data) x = np.arange(len(inputs)) # the label locations width = 0.25 # the width of the bars fig, ax = plt.subplots(layout="constrained") fig.set_figheight(5) fig.set_figwidth(10) for i, command in enumerate(commands): offset = width * (i + 1) rects = ax.bar(x + offset, data[i], width, label=command) ax.set_xticks(x + 0.5, inputs) ax.grid(visible=True, axis="y") if args.title: plt.title(args.title) plt.xlabel("Benchmark") plt.ylabel("Time [s]") plt.legend(title="Command") if args.output: plt.savefig(args.output) else: plt.show() ================================================ FILE: scripts/plot_histogram.py ================================================ #!/usr/bin/env python # /// script # requires-python = ">=3.10" # dependencies = [ # "matplotlib", # "pyqt6", # "numpy", # ] # /// """This program shows `hyperfine` benchmark results as a histogram.""" import argparse import json import matplotlib.pyplot as plt import numpy as np parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("file", help="JSON file with benchmark results") parser.add_argument("--title", help="Plot title") parser.add_argument( "--labels", help="Comma-separated list of entries for the plot legend" ) parser.add_argument("--bins", help="Number of bins (default: auto)") parser.add_argument( "--legend-location", help="Location of the legend on plot (default: upper center)", choices=[ "upper center", "lower center", "right", "left", "best", "upper left", "upper right", "lower left", "lower right", "center left", "center right", "center", ], default="upper center", ) parser.add_argument( "--type", help="Type of histogram (*bar*, barstacked, step, stepfilled)" ) parser.add_argument("-o", "--output", help="Save image to the given filename.") parser.add_argument( "--t-min", metavar="T", help="Minimum time to be displayed (seconds)" ) parser.add_argument( "--t-max", metavar="T", help="Maximum time to be displayed (seconds)" ) parser.add_argument( "--log-count", help="Use a logarithmic y-axis for the event count", action="store_true", ) args = parser.parse_args() with open(args.file) as f: results = json.load(f)["results"] if args.labels: labels = args.labels.split(",") else: labels = [b["command"] for b in results] all_times = [b["times"] for b in results] t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times))) t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times))) bins = int(args.bins) if args.bins else "auto" histtype = args.type if args.type else "bar" plt.figure(figsize=(10, 5)) plt.hist( all_times, label=labels, bins=bins, histtype=histtype, range=(t_min, t_max), ) plt.legend( loc=args.legend_location, fancybox=True, shadow=True, prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]}, ) plt.xlabel("Time [s]") if args.title: plt.title(args.title) if args.log_count: plt.yscale("log") else: plt.ylim(0, None) if args.output: plt.savefig(args.output, dpi=600) else: plt.show() ================================================ FILE: scripts/plot_parametrized.py ================================================ #!/usr/bin/env python # /// script # requires-python = ">=3.10" # dependencies = [ # "matplotlib", # "pyqt6", # ] # /// """This program shows parametrized `hyperfine` benchmark results as an errorbar plot.""" import argparse import json import sys import matplotlib.pyplot as plt parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("file", help="JSON file with benchmark results", nargs="+") parser.add_argument( "--parameter-name", metavar="name", type=str, help="Deprecated; parameter names are now inferred from benchmark files", ) parser.add_argument( "--log-x", help="Use a logarithmic x (parameter) axis", action="store_true" ) parser.add_argument( "--log-time", help="Use a logarithmic time axis", action="store_true" ) parser.add_argument( "--titles", help="Comma-separated list of titles for the plot legend" ) parser.add_argument("-o", "--output", help="Save image to the given filename.") args = parser.parse_args() if args.parameter_name is not None: sys.stderr.write( "warning: --parameter-name is deprecated; names are inferred from " "benchmark results\n" ) def die(msg): sys.stderr.write(f"fatal: {msg}\n") sys.exit(1) def extract_parameters(results): """Return `(parameter_name: str, parameter_values: List[float])`.""" if not results: die("no benchmark data to plot") (names, values) = zip(*(unique_parameter(b) for b in results)) names = frozenset(names) if len(names) != 1: die( f"benchmarks must all have the same parameter name, but found: {sorted(names)}" ) return (next(iter(names)), list(values)) def unique_parameter(benchmark): """Return the unique parameter `(name: str, value: float)`, or die.""" params_dict = benchmark.get("parameters", {}) if not params_dict: die("benchmarks must have exactly one parameter, but found none") if len(params_dict) > 1: die( f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}" ) [(name, value)] = params_dict.items() return (name, float(value)) parameter_name = None for filename in args.file: with open(filename) as f: results = json.load(f)["results"] (this_parameter_name, parameter_values) = extract_parameters(results) if parameter_name is not None and this_parameter_name != parameter_name: die( f"files must all have the same parameter name, but found {parameter_name!r} vs. {this_parameter_name!r}" ) parameter_name = this_parameter_name times_mean = [b["mean"] for b in results] times_stddev = [b["stddev"] for b in results] plt.errorbar(x=parameter_values, y=times_mean, yerr=times_stddev, capsize=2) plt.xlabel(parameter_name) plt.ylabel("Time [s]") if args.log_time: plt.yscale("log") else: plt.ylim(0, None) if args.log_x: plt.xscale("log") if args.titles: plt.legend(args.titles.split(",")) if args.output: plt.savefig(args.output) else: plt.show() ================================================ FILE: scripts/plot_progression.py ================================================ #!/usr/bin/env python # /// script # requires-python = ">=3.10" # dependencies = [ # "pyqt6", # "matplotlib", # "numpy", # ] # /// """This program shows `hyperfine` benchmark results in a sequential way in order to debug possible background interference, caching effects, thermal throttling and similar effects. """ import argparse import json import matplotlib.pyplot as plt import numpy as np def moving_average(times, num_runs): times_padded = np.pad( times, (num_runs // 2, num_runs - 1 - num_runs // 2), mode="edge" ) kernel = np.ones(num_runs) / num_runs return np.convolve(times_padded, kernel, mode="valid") parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("file", help="JSON file with benchmark results") parser.add_argument("--title", help="Plot Title") parser.add_argument("-o", "--output", help="Save image to the given filename.") parser.add_argument( "-w", "--moving-average-width", type=int, metavar="num_runs", help="Width of the moving-average window (default: N/5)", ) parser.add_argument( "--no-moving-average", action="store_true", help="Do not show moving average curve", ) args = parser.parse_args() with open(args.file) as f: results = json.load(f)["results"] for result in results: label = result["command"] times = result["times"] num = len(times) nums = range(num) plt.scatter(x=nums, y=times, marker=".") plt.ylim([0, None]) plt.xlim([-1, num]) if not args.no_moving_average: moving_average_width = ( num // 5 if args.moving_average_width is None else args.moving_average_width ) average = moving_average(times, moving_average_width) plt.plot(nums, average, "-") if args.title: plt.title(args.title) legend = [] for result in results: legend.append(result["command"]) if not args.no_moving_average: legend.append("moving average") plt.legend(legend) plt.ylabel("Time [s]") if args.output: plt.savefig(args.output) else: plt.show() ================================================ FILE: scripts/plot_whisker.py ================================================ #!/usr/bin/env python # /// script # requires-python = ">=3.10" # dependencies = [ # "matplotlib", # "pyqt6", # ] # /// """This program shows `hyperfine` benchmark results as a box and whisker plot. Quoting from the matplotlib documentation: The box extends from the lower to upper quartile values of the data, with a line at the median. The whiskers extend from the box to show the range of the data. Flier points are those past the end of the whiskers. """ import argparse import json import matplotlib.pyplot as plt parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("file", help="JSON file with benchmark results") parser.add_argument("--title", help="Plot Title") parser.add_argument("--sort-by", choices=["median"], help="Sort method") parser.add_argument( "--labels", help="Comma-separated list of entries for the plot legend" ) parser.add_argument("-o", "--output", help="Save image to the given filename.") args = parser.parse_args() with open(args.file, encoding="utf-8") as f: results = json.load(f)["results"] if args.labels: labels = args.labels.split(",") else: labels = [b["command"] for b in results] times = [b["times"] for b in results] if args.sort_by == "median": medians = [b["median"] for b in results] indices = sorted(range(len(labels)), key=lambda k: medians[k]) labels = [labels[i] for i in indices] times = [times[i] for i in indices] plt.figure(figsize=(10, 6), constrained_layout=True) boxplot = plt.boxplot(times, vert=True, patch_artist=True) cmap = plt.get_cmap("rainbow") colors = [cmap(val / len(times)) for val in range(len(times))] for patch, color in zip(boxplot["boxes"], colors): patch.set_facecolor(color) if args.title: plt.title(args.title) plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium") plt.ylabel("Time [s]") plt.ylim(0, None) plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45) if args.output: plt.savefig(args.output) else: plt.show() ================================================ FILE: scripts/ruff.toml ================================================ target-version = "py310" [lint] extend-select = ["I", "UP", "RUF"] ================================================ FILE: scripts/welch_ttest.py ================================================ #!/usr/bin/env python # /// script # requires-python = ">=3.10" # dependencies = [ # "scipy", # ] # /// """This script performs Welch's t-test on a JSON export file with two benchmark results to test whether or not the two distributions are the same.""" import argparse import json import sys from scipy import stats parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("file", help="JSON file with two benchmark results") args = parser.parse_args() with open(args.file) as f: results = json.load(f)["results"] if len(results) != 2: print("The input file has to contain exactly two benchmarks") sys.exit(1) a, b = (x["command"] for x in results[:2]) X, Y = (x["times"] for x in results[:2]) print(f"Command 1: {a}") print(f"Command 2: {b}\n") t, p = stats.ttest_ind(X, Y, equal_var=False) th = 0.05 dispose = p < th print(f"t = {t:.3}, p = {p:.3}") print() if dispose: print(f"There is a difference between the two benchmarks (p < {th}).") else: print(f"The two benchmarks are almost the same (p >= {th}).") ================================================ FILE: src/benchmark/benchmark_result.rs ================================================ use std::collections::BTreeMap; use serde::Serialize; use crate::util::units::Second; /// Set of values that will be exported. // NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the // `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate. #[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct BenchmarkResult { /// The full command line of the program that is being benchmarked pub command: String, /// The full command line of the program that is being benchmarked, possibly including a list of /// parameters that were not used in the command line template. #[serde(skip_serializing)] pub command_with_unused_parameters: String, /// The average run time pub mean: Second, /// The standard deviation of all run times. Not available if only one run has been performed pub stddev: Option, /// The median run time pub median: Second, /// Time spent in user mode pub user: Second, /// Time spent in kernel mode pub system: Second, /// Minimum of all measured times pub min: Second, /// Maximum of all measured times pub max: Second, /// All run time measurements #[serde(skip_serializing_if = "Option::is_none")] pub times: Option>, /// Maximum memory usage of the process, in bytes #[serde(skip_serializing_if = "Option::is_none")] pub memory_usage_byte: Option>, /// Exit codes of all command invocations pub exit_codes: Vec>, /// Parameter values for this benchmark #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub parameters: BTreeMap, } ================================================ FILE: src/benchmark/executor.rs ================================================ #[cfg(windows)] use std::os::windows::process::CommandExt; use std::process::ExitStatus; use crate::command::Command; use crate::options::{ CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell, }; use crate::output::progress_bar::get_progress_bar; use crate::timer::{execute_and_measure, TimerResult}; use crate::util::randomized_environment_offset; use crate::util::units::Second; use super::timing_result::TimingResult; use anyhow::{bail, Context, Result}; use statistical::mean; pub enum BenchmarkIteration { NonBenchmarkRun, Warmup(u64), Benchmark(u64), } impl BenchmarkIteration { pub fn to_env_var_value(&self) -> Option { match self { BenchmarkIteration::NonBenchmarkRun => None, BenchmarkIteration::Warmup(i) => Some(format!("warmup-{}", i)), BenchmarkIteration::Benchmark(i) => Some(format!("{}", i)), } } } pub trait Executor { /// Run the given command and measure the execution time fn run_command_and_measure( &self, command: &Command<'_>, iteration: BenchmarkIteration, command_failure_action: Option, output_policy: &CommandOutputPolicy, ) -> Result<(TimingResult, ExitStatus)>; /// Perform a calibration of this executor. For example, /// when running commands through a shell, we need to /// measure the shell spawning time separately in order /// to subtract it from the full runtime later. fn calibrate(&mut self) -> Result<()>; /// Return the time overhead for this executor when /// performing a measurement. This should return the time /// that is being used in addition to the actual runtime /// of the command. fn time_overhead(&self) -> Second; } fn run_command_and_measure_common( mut command: std::process::Command, iteration: BenchmarkIteration, command_failure_action: CmdFailureAction, command_input_policy: &CommandInputPolicy, command_output_policy: &CommandOutputPolicy, command_name: &str, ) -> Result { let stdin = command_input_policy.get_stdin()?; let (stdout, stderr) = command_output_policy.get_stdout_stderr()?; command.stdin(stdin).stdout(stdout).stderr(stderr); command.env( "HYPERFINE_RANDOMIZED_ENVIRONMENT_OFFSET", randomized_environment_offset::value(), ); if let Some(value) = iteration.to_env_var_value() { command.env("HYPERFINE_ITERATION", value); } let result = execute_and_measure(command) .with_context(|| format!("Failed to run command '{command_name}'"))?; if !result.status.success() { use crate::util::exit_code::extract_exit_code; let should_fail = match command_failure_action { CmdFailureAction::RaiseError => true, CmdFailureAction::IgnoreAllFailures => false, CmdFailureAction::IgnoreSpecificFailures(ref codes) => { // Only fail if the exit code is not in the list of codes to ignore if let Some(exit_code) = extract_exit_code(result.status) { !codes.contains(&exit_code) } else { // If we can't extract an exit code, treat it as a failure true } } }; if should_fail { let when = match iteration { BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(), BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(), BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"), BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(), BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"), }; bail!( "{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \ Alternatively, use the '--show-output' option to debug what went wrong.", cause=result.status.code().map_or( "The process has been terminated by a signal".into(), |c| format!("Command terminated with non-zero exit code {c}") ), ); } } Ok(result) } pub struct RawExecutor<'a> { options: &'a Options, } impl<'a> RawExecutor<'a> { pub fn new(options: &'a Options) -> Self { RawExecutor { options } } } impl Executor for RawExecutor<'_> { fn run_command_and_measure( &self, command: &Command<'_>, iteration: BenchmarkIteration, command_failure_action: Option, output_policy: &CommandOutputPolicy, ) -> Result<(TimingResult, ExitStatus)> { let result = run_command_and_measure_common( command.get_command()?, iteration, command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()), &self.options.command_input_policy, output_policy, &command.get_command_line(), )?; Ok(( TimingResult { time_real: result.time_real, time_user: result.time_user, time_system: result.time_system, memory_usage_byte: result.memory_usage_byte, }, result.status, )) } fn calibrate(&mut self) -> Result<()> { Ok(()) } fn time_overhead(&self) -> Second { 0.0 } } pub struct ShellExecutor<'a> { options: &'a Options, shell: &'a Shell, shell_spawning_time: Option, } impl<'a> ShellExecutor<'a> { pub fn new(shell: &'a Shell, options: &'a Options) -> Self { ShellExecutor { shell, options, shell_spawning_time: None, } } } impl Executor for ShellExecutor<'_> { fn run_command_and_measure( &self, command: &Command<'_>, iteration: BenchmarkIteration, command_failure_action: Option, output_policy: &CommandOutputPolicy, ) -> Result<(TimingResult, ExitStatus)> { let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe"); let mut command_builder = self.shell.command(); command_builder.arg(if on_windows_cmd { "/C" } else { "-c" }); // Windows needs special treatment for its behavior on parsing cmd arguments if on_windows_cmd { #[cfg(windows)] command_builder.raw_arg(command.get_command_line()); } else { command_builder.arg(command.get_command_line()); } let mut result = run_command_and_measure_common( command_builder, iteration, command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()), &self.options.command_input_policy, output_policy, &command.get_command_line(), )?; // Subtract shell spawning time if let Some(spawning_time) = self.shell_spawning_time { result.time_real = (result.time_real - spawning_time.time_real).max(0.0); result.time_user = (result.time_user - spawning_time.time_user).max(0.0); result.time_system = (result.time_system - spawning_time.time_system).max(0.0); } Ok(( TimingResult { time_real: result.time_real, time_user: result.time_user, time_system: result.time_system, memory_usage_byte: result.memory_usage_byte, }, result.status, )) } /// Measure the average shell spawning time fn calibrate(&mut self) -> Result<()> { const COUNT: u64 = 50; let progress_bar = if self.options.output_style != OutputStyleOption::Disabled { Some(get_progress_bar( COUNT, "Measuring shell spawning time", self.options.output_style, )) } else { None }; let mut times_real: Vec = vec![]; let mut times_user: Vec = vec![]; let mut times_system: Vec = vec![]; for _ in 0..COUNT { // Just run the shell without any command let res = self.run_command_and_measure( &Command::new(None, ""), BenchmarkIteration::NonBenchmarkRun, None, &CommandOutputPolicy::Null, ); match res { Err(_) => { let shell_cmd = if cfg!(windows) { format!("{} /C \"\"", self.shell) } else { format!("{} -c \"\"", self.shell) }; bail!( "Could not measure shell execution time. Make sure you can run '{}'.", shell_cmd ); } Ok((r, _)) => { times_real.push(r.time_real); times_user.push(r.time_user); times_system.push(r.time_system); } } if let Some(bar) = progress_bar.as_ref() { bar.inc(1) } } if let Some(bar) = progress_bar.as_ref() { bar.finish_and_clear() } self.shell_spawning_time = Some(TimingResult { time_real: mean(×_real), time_user: mean(×_user), time_system: mean(×_system), memory_usage_byte: 0, }); Ok(()) } fn time_overhead(&self) -> Second { self.shell_spawning_time.unwrap().time_real } } #[derive(Clone)] pub struct MockExecutor { shell: Option, } impl MockExecutor { pub fn new(shell: Option) -> Self { MockExecutor { shell } } fn extract_time>(sleep_command: S) -> Second { assert!(sleep_command.as_ref().starts_with("sleep ")); sleep_command .as_ref() .trim_start_matches("sleep ") .parse::() .unwrap() } } impl Executor for MockExecutor { fn run_command_and_measure( &self, command: &Command<'_>, _iteration: BenchmarkIteration, _command_failure_action: Option, _output_policy: &CommandOutputPolicy, ) -> Result<(TimingResult, ExitStatus)> { #[cfg(unix)] let status = { use std::os::unix::process::ExitStatusExt; ExitStatus::from_raw(0) }; #[cfg(windows)] let status = { use std::os::windows::process::ExitStatusExt; ExitStatus::from_raw(0) }; Ok(( TimingResult { time_real: Self::extract_time(command.get_command_line()), time_user: 0.0, time_system: 0.0, memory_usage_byte: 0, }, status, )) } fn calibrate(&mut self) -> Result<()> { Ok(()) } fn time_overhead(&self) -> Second { match &self.shell { None => 0.0, Some(shell) => Self::extract_time(shell), } } } #[test] fn test_mock_executor_extract_time() { assert_eq!(MockExecutor::extract_time("sleep 0.1"), 0.1); } ================================================ FILE: src/benchmark/mod.rs ================================================ pub mod benchmark_result; pub mod executor; pub mod relative_speed; pub mod scheduler; pub mod timing_result; use std::cmp; use crate::benchmark::executor::BenchmarkIteration; use crate::command::Command; use crate::options::{ CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption, }; use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD}; use crate::output::format::{format_duration, format_duration_unit}; use crate::output::progress_bar::get_progress_bar; use crate::output::warnings::{OutlierWarningOptions, Warnings}; use crate::parameter::ParameterNameAndValue; use crate::util::exit_code::extract_exit_code; use crate::util::min_max::{max, min}; use crate::util::units::Second; use benchmark_result::BenchmarkResult; use timing_result::TimingResult; use anyhow::{anyhow, Result}; use colored::*; use statistical::{mean, median, standard_deviation}; use self::executor::Executor; /// Threshold for warning about fast execution time pub const MIN_EXECUTION_TIME: Second = 5e-3; pub struct Benchmark<'a> { number: usize, command: &'a Command<'a>, options: &'a Options, executor: &'a dyn Executor, } impl<'a> Benchmark<'a> { pub fn new( number: usize, command: &'a Command<'a>, options: &'a Options, executor: &'a dyn Executor, ) -> Self { Benchmark { number, command, options, executor, } } /// Run setup, cleanup, or preparation commands fn run_intermediate_command( &self, command: &Command<'_>, error_output: &'static str, output_policy: &CommandOutputPolicy, ) -> Result { self.executor .run_command_and_measure( command, executor::BenchmarkIteration::NonBenchmarkRun, Some(CmdFailureAction::RaiseError), output_policy, ) .map(|r| r.0) .map_err(|_| anyhow!(error_output)) } /// Run the command specified by `--setup`. fn run_setup_command( &self, parameters: impl IntoIterator>, output_policy: &CommandOutputPolicy, ) -> Result { let command = self .options .setup_command .as_ref() .map(|setup_command| Command::new_parametrized(None, setup_command, parameters)); let error_output = "The setup command terminated with a non-zero exit code. \ Append ' || true' to the command if you are sure that this can be ignored."; Ok(command .map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy)) .transpose()? .unwrap_or_default()) } /// Run the command specified by `--cleanup`. fn run_cleanup_command( &self, parameters: impl IntoIterator>, output_policy: &CommandOutputPolicy, ) -> Result { let command = self .options .cleanup_command .as_ref() .map(|cleanup_command| Command::new_parametrized(None, cleanup_command, parameters)); let error_output = "The cleanup command terminated with a non-zero exit code. \ Append ' || true' to the command if you are sure that this can be ignored."; Ok(command .map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy)) .transpose()? .unwrap_or_default()) } /// Run the command specified by `--prepare`. fn run_preparation_command( &self, command: &Command<'_>, output_policy: &CommandOutputPolicy, ) -> Result { let error_output = "The preparation command terminated with a non-zero exit code. \ Append ' || true' to the command if you are sure that this can be ignored."; self.run_intermediate_command(command, error_output, output_policy) } /// Run the command specified by `--conclude`. fn run_conclusion_command( &self, command: &Command<'_>, output_policy: &CommandOutputPolicy, ) -> Result { let error_output = "The conclusion command terminated with a non-zero exit code. \ Append ' || true' to the command if you are sure that this can be ignored."; self.run_intermediate_command(command, error_output, output_policy) } /// Run the benchmark for a single command pub fn run(&self) -> Result { if self.options.output_style != OutputStyleOption::Disabled { println!( "{}{}: {}", "Benchmark ".bold(), (self.number + 1).to_string().bold(), self.command.get_name_with_unused_parameters(), ); } let mut times_real: Vec = vec![]; let mut times_user: Vec = vec![]; let mut times_system: Vec = vec![]; let mut memory_usage_byte: Vec = vec![]; let mut exit_codes: Vec> = vec![]; let mut all_succeeded = true; let output_policy = &self.options.command_output_policies[self.number]; let preparation_command = self.options.preparation_command.as_ref().map(|values| { let preparation_command = if values.len() == 1 { &values[0] } else { &values[self.number] }; Command::new_parametrized( None, preparation_command, self.command.get_parameters().iter().cloned(), ) }); let run_preparation_command = || { preparation_command .as_ref() .map(|cmd| self.run_preparation_command(cmd, output_policy)) .transpose() }; let conclusion_command = self.options.conclusion_command.as_ref().map(|values| { let conclusion_command = if values.len() == 1 { &values[0] } else { &values[self.number] }; Command::new_parametrized( None, conclusion_command, self.command.get_parameters().iter().cloned(), ) }); let run_conclusion_command = || { conclusion_command .as_ref() .map(|cmd| self.run_conclusion_command(cmd, output_policy)) .transpose() }; self.run_setup_command(self.command.get_parameters().iter().cloned(), output_policy)?; // Warmup phase if self.options.warmup_count > 0 { let progress_bar = if self.options.output_style != OutputStyleOption::Disabled { Some(get_progress_bar( self.options.warmup_count, "Performing warmup runs", self.options.output_style, )) } else { None }; for i in 0..self.options.warmup_count { let _ = run_preparation_command()?; let _ = self.executor.run_command_and_measure( self.command, BenchmarkIteration::Warmup(i), None, output_policy, )?; let _ = run_conclusion_command()?; if let Some(bar) = progress_bar.as_ref() { bar.inc(1) } } if let Some(bar) = progress_bar.as_ref() { bar.finish_and_clear() } } // Set up progress bar (and spinner for initial measurement) let progress_bar = if self.options.output_style != OutputStyleOption::Disabled { Some(get_progress_bar( self.options.run_bounds.min, "Initial time measurement", self.options.output_style, )) } else { None }; let preparation_result = run_preparation_command()?; let preparation_overhead = preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead()); // Initial timing run let (res, status) = self.executor.run_command_and_measure( self.command, BenchmarkIteration::Benchmark(0), None, output_policy, )?; let success = status.success(); let conclusion_result = run_conclusion_command()?; let conclusion_overhead = conclusion_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead()); // Determine number of benchmark runs let runs_in_min_time = (self.options.min_benchmarking_time / (res.time_real + self.executor.time_overhead() + preparation_overhead + conclusion_overhead)) as u64; let count = { let min = cmp::max(runs_in_min_time, self.options.run_bounds.min); self.options .run_bounds .max .as_ref() .map(|max| cmp::min(min, *max)) .unwrap_or(min) }; let count_remaining = count - 1; // Save the first result times_real.push(res.time_real); times_user.push(res.time_user); times_system.push(res.time_system); memory_usage_byte.push(res.memory_usage_byte); exit_codes.push(extract_exit_code(status)); all_succeeded = all_succeeded && success; // Re-configure the progress bar if let Some(bar) = progress_bar.as_ref() { bar.set_length(count) } if let Some(bar) = progress_bar.as_ref() { bar.inc(1) } // Gather statistics (perform the actual benchmark) for i in 0..count_remaining { run_preparation_command()?; let msg = { let mean = format_duration(mean(×_real), self.options.time_unit); format!("Current estimate: {}", mean.to_string().green()) }; if let Some(bar) = progress_bar.as_ref() { bar.set_message(msg.to_owned()) } let (res, status) = self.executor.run_command_and_measure( self.command, BenchmarkIteration::Benchmark(i + 1), None, output_policy, )?; let success = status.success(); times_real.push(res.time_real); times_user.push(res.time_user); times_system.push(res.time_system); memory_usage_byte.push(res.memory_usage_byte); exit_codes.push(extract_exit_code(status)); all_succeeded = all_succeeded && success; if let Some(bar) = progress_bar.as_ref() { bar.inc(1) } run_conclusion_command()?; } if let Some(bar) = progress_bar.as_ref() { bar.finish_and_clear() } // Compute statistical quantities let t_num = times_real.len(); let t_mean = mean(×_real); let t_stddev = if times_real.len() > 1 { Some(standard_deviation(×_real, Some(t_mean))) } else { None }; let t_median = median(×_real); let t_min = min(×_real); let t_max = max(×_real); let user_mean = mean(×_user); let system_mean = mean(×_system); // Formatting and console output let (mean_str, time_unit) = format_duration_unit(t_mean, self.options.time_unit); let min_str = format_duration(t_min, Some(time_unit)); let max_str = format_duration(t_max, Some(time_unit)); let num_str = format!("{t_num} runs"); let user_str = format_duration(user_mean, Some(time_unit)); let system_str = format_duration(system_mean, Some(time_unit)); if self.options.output_style != OutputStyleOption::Disabled { if times_real.len() == 1 { println!( " Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]", "abs".green().bold(), mean_str.green().bold(), " ", // alignment user_str.blue(), system_str.blue() ); } else { let stddev_str = format_duration(t_stddev.unwrap(), Some(time_unit)); println!( " Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]", "mean".green().bold(), "σ".green(), mean_str.green().bold(), stddev_str.green(), user_str.blue(), system_str.blue() ); println!( " Range ({} … {}): {:>8} … {:>8} {}", "min".cyan(), "max".purple(), min_str.cyan(), max_str.purple(), num_str.dimmed() ); } } // Warnings let mut warnings = vec![]; // Check execution time if matches!(self.options.executor_kind, ExecutorKind::Shell(_)) && times_real.iter().any(|&t| t < MIN_EXECUTION_TIME) { warnings.push(Warnings::FastExecutionTime); } // Check program exit codes if !all_succeeded { warnings.push(Warnings::NonZeroExitCode); } // Run outlier detection let scores = modified_zscores(×_real); let outlier_warning_options = OutlierWarningOptions { warmup_in_use: self.options.warmup_count > 0, prepare_in_use: self .options .preparation_command .as_ref() .map(|v| v.len()) .unwrap_or(0) > 0, }; if scores[0] > OUTLIER_THRESHOLD { warnings.push(Warnings::SlowInitialRun( times_real[0], outlier_warning_options, )); } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) { warnings.push(Warnings::OutliersDetected(outlier_warning_options)); } if !warnings.is_empty() { eprintln!(" "); for warning in &warnings { eprintln!(" {}: {}", "Warning".yellow(), warning); } } if self.options.output_style != OutputStyleOption::Disabled { println!(" "); } self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?; Ok(BenchmarkResult { command: self.command.get_name(), command_with_unused_parameters: self.command.get_name_with_unused_parameters(), mean: t_mean, stddev: t_stddev, median: t_median, user: user_mean, system: system_mean, min: t_min, max: t_max, times: Some(times_real), memory_usage_byte: Some(memory_usage_byte), exit_codes, parameters: self .command .get_parameters() .iter() .map(|(name, value)| (name.to_string(), value.to_string())) .collect(), }) } } ================================================ FILE: src/benchmark/relative_speed.rs ================================================ use std::cmp::Ordering; use super::benchmark_result::BenchmarkResult; use crate::{options::SortOrder, util::units::Scalar}; #[derive(Debug)] pub struct BenchmarkResultWithRelativeSpeed<'a> { pub result: &'a BenchmarkResult, pub relative_speed: Scalar, pub relative_speed_stddev: Option, pub is_reference: bool, // Less means faster pub relative_ordering: Ordering, } pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal) } pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { results .iter() .min_by(|&l, &r| compare_mean_time(l, r)) .expect("at least one benchmark result") } fn compute_relative_speeds<'a>( results: &'a [BenchmarkResult], reference: &'a BenchmarkResult, sort_order: SortOrder, ) -> Vec> { let mut results: Vec<_> = results .iter() .map(|result| { let is_reference = result == reference; let relative_ordering = compare_mean_time(result, reference); if result.mean == 0.0 { return BenchmarkResultWithRelativeSpeed { result, relative_speed: if is_reference { 1.0 } else { f64::INFINITY }, relative_speed_stddev: None, is_reference, relative_ordering, }; } let ratio = match relative_ordering { Ordering::Less => reference.mean / result.mean, Ordering::Equal => 1.0, Ordering::Greater => result.mean / reference.mean, }; // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas // Covariance asssumed to be 0, i.e. variables are assumed to be independent let ratio_stddev = match (result.stddev, reference.stddev) { (Some(result_stddev), Some(fastest_stddev)) => Some( ratio * ((result_stddev / result.mean).powi(2) + (fastest_stddev / reference.mean).powi(2)) .sqrt(), ), _ => None, }; BenchmarkResultWithRelativeSpeed { result, relative_speed: ratio, relative_speed_stddev: ratio_stddev, is_reference, relative_ordering, } }) .collect(); match sort_order { SortOrder::Command => {} SortOrder::MeanTime => { results.sort_unstable_by(|r1, r2| compare_mean_time(r1.result, r2.result)); } } results } pub fn compute_with_check_from_reference<'a>( results: &'a [BenchmarkResult], reference: &'a BenchmarkResult, sort_order: SortOrder, ) -> Option>> { if fastest_of(results).mean == 0.0 || reference.mean == 0.0 { return None; } Some(compute_relative_speeds(results, reference, sort_order)) } pub fn compute_with_check( results: &[BenchmarkResult], sort_order: SortOrder, ) -> Option>> { let fastest = fastest_of(results); if fastest.mean == 0.0 { return None; } Some(compute_relative_speeds(results, fastest, sort_order)) } /// Same as compute_with_check, potentially resulting in relative speeds of infinity pub fn compute( results: &[BenchmarkResult], sort_order: SortOrder, ) -> Vec> { let fastest = fastest_of(results); compute_relative_speeds(results, fastest, sort_order) } #[cfg(test)] fn create_result(name: &str, mean: Scalar) -> BenchmarkResult { use std::collections::BTreeMap; BenchmarkResult { command: name.into(), command_with_unused_parameters: name.into(), mean, stddev: Some(1.0), median: mean, user: mean, system: 0.0, min: mean, max: mean, times: None, memory_usage_byte: None, exit_codes: Vec::new(), parameters: BTreeMap::new(), } } #[test] fn test_compute_relative_speed() { use approx::assert_relative_eq; let results = vec![ create_result("cmd1", 3.0), create_result("cmd2", 2.0), create_result("cmd3", 5.0), ]; let annotated_results = compute_with_check(&results, SortOrder::Command).unwrap(); assert_relative_eq!(1.5, annotated_results[0].relative_speed); assert_relative_eq!(1.0, annotated_results[1].relative_speed); assert_relative_eq!(2.5, annotated_results[2].relative_speed); } #[test] fn test_compute_relative_speed_with_reference() { use approx::assert_relative_eq; let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)]; let reference = create_result("cmd2", 4.0); let annotated_results = compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap(); assert_relative_eq!(2.0, annotated_results[0].relative_speed); assert_relative_eq!(1.25, annotated_results[1].relative_speed); } #[test] fn test_compute_relative_speed_for_zero_times() { let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)]; let annotated_results = compute_with_check(&results, SortOrder::Command); assert!(annotated_results.is_none()); } ================================================ FILE: src/benchmark/scheduler.rs ================================================ use super::benchmark_result::BenchmarkResult; use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor}; use super::{relative_speed, Benchmark}; use colored::*; use std::cmp::Ordering; use crate::command::{Command, Commands}; use crate::export::ExportManager; use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder}; use anyhow::Result; pub struct Scheduler<'a> { commands: &'a Commands<'a>, options: &'a Options, export_manager: &'a ExportManager, results: Vec, } impl<'a> Scheduler<'a> { pub fn new( commands: &'a Commands, options: &'a Options, export_manager: &'a ExportManager, ) -> Self { Self { commands, options, export_manager, results: vec![], } } pub fn run_benchmarks(&mut self) -> Result<()> { let mut executor: Box = match self.options.executor_kind { ExecutorKind::Raw => Box::new(RawExecutor::new(self.options)), ExecutorKind::Mock(ref shell) => Box::new(MockExecutor::new(shell.clone())), ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)), }; let reference = self .options .reference_command .as_ref() .map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd)); executor.calibrate()?; for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() { self.results .push(Benchmark::new(number, cmd, self.options, &*executor).run()?); // We export results after each individual benchmark, because // we would risk losing them if a later benchmark fails. self.export_manager.write_results(&self.results, true)?; } Ok(()) } pub fn print_relative_speed_comparison(&self) { if self.options.output_style == OutputStyleOption::Disabled { return; } if self.results.len() < 2 { return; } let reference = self .options .reference_command .as_ref() .map(|_| &self.results[0]) .unwrap_or_else(|| relative_speed::fastest_of(&self.results)); if let Some(annotated_results) = relative_speed::compute_with_check_from_reference( &self.results, reference, self.options.sort_order_speed_comparison, ) { match self.options.sort_order_speed_comparison { SortOrder::MeanTime => { println!("{}", "Summary".bold()); let reference = annotated_results.iter().find(|r| r.is_reference).unwrap(); let others = annotated_results.iter().filter(|r| !r.is_reference); println!( " {} ran", reference.result.command_with_unused_parameters.cyan() ); for item in others { let stddev = if let Some(stddev) = item.relative_speed_stddev { format!(" ± {}", format!("{:.2}", stddev).green()) } else { "".into() }; let comparator = match item.relative_ordering { Ordering::Less => format!( "{}{} times slower than", format!("{:8.2}", item.relative_speed).bold().green(), stddev ), Ordering::Greater => format!( "{}{} times faster than", format!("{:8.2}", item.relative_speed).bold().green(), stddev ), Ordering::Equal => format!( " As fast ({}{}) as", format!("{:.2}", item.relative_speed).bold().green(), stddev ), }; println!( "{} {}", comparator, &item.result.command_with_unused_parameters.magenta() ); } } SortOrder::Command => { println!("{}", "Relative speed comparison".bold()); for item in annotated_results { println!( " {}{} {}", format!("{:10.2}", item.relative_speed).bold().green(), if item.is_reference { " ".into() } else if let Some(stddev) = item.relative_speed_stddev { format!(" ± {}", format!("{stddev:5.2}").green()) } else { " ".into() }, &item.result.command_with_unused_parameters, ); } } } } else { eprintln!( "{}: The benchmark comparison could not be computed as some benchmark times are zero. \ This could be caused by background interference during the initial calibration phase \ of hyperfine, in combination with very fast commands (faster than a few milliseconds). \ Try to re-run the benchmark on a quiet system. If you did not do so already, try the \ --shell=none/-N option. If it does not help either, you command is most likely too fast \ to be accurately benchmarked by hyperfine.", "Note".bold().red() ); } } pub fn final_export(&self) -> Result<()> { self.export_manager.write_results(&self.results, false) } } #[cfg(test)] fn generate_results(args: &[&'static str]) -> Result> { use crate::cli::get_cli_arguments; let args = ["hyperfine", "--debug-mode", "--style=none"] .iter() .chain(args); let cli_arguments = get_cli_arguments(args); let mut options = Options::from_cli_arguments(&cli_arguments)?; assert_eq!(options.executor_kind, ExecutorKind::Mock(None)); let commands = Commands::from_cli_arguments(&cli_arguments)?; let export_manager = ExportManager::from_cli_arguments( &cli_arguments, options.time_unit, options.sort_order_exports, )?; options.validate_against_command_list(&commands)?; let mut scheduler = Scheduler::new(&commands, &options, &export_manager); scheduler.run_benchmarks()?; Ok(scheduler.results) } #[test] fn scheduler_basic() -> Result<()> { insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @r#" - command: sleep 0.123 mean: 0.123 stddev: 0 median: 0.123 user: 0 system: 0 min: 0.123 max: 0.123 times: - 0.123 - 0.123 memory_usage_byte: - 0 - 0 exit_codes: - 0 - 0 - command: sleep 0.456 mean: 0.456 stddev: 0 median: 0.456 user: 0 system: 0 min: 0.456 max: 0.456 times: - 0.456 - 0.456 memory_usage_byte: - 0 - 0 exit_codes: - 0 - 0 "#); Ok(()) } ================================================ FILE: src/benchmark/timing_result.rs ================================================ use crate::util::units::Second; /// Results from timing a single command #[derive(Debug, Default, Copy, Clone)] pub struct TimingResult { /// Wall clock time pub time_real: Second, /// Time spent in user mode pub time_user: Second, /// Time spent in kernel mode pub time_system: Second, /// Maximum amount of memory used, in bytes pub memory_usage_byte: u64, } ================================================ FILE: src/cli.rs ================================================ use std::ffi::OsString; use clap::{ builder::NonEmptyStringValueParser, crate_version, Arg, ArgAction, ArgMatches, Command, ValueHint, }; pub fn get_cli_arguments<'a, I, T>(args: I) -> ArgMatches where I: IntoIterator, T: Into + Clone + 'a, { let command = build_command(); command.get_matches_from(args) } /// Build the clap command for parsing command line arguments fn build_command() -> Command { Command::new("hyperfine") .version(crate_version!()) .next_line_help(true) .hide_possible_values(true) .about("A command-line benchmarking tool.") .help_expected(true) .max_term_width(80) .arg( Arg::new("command") .help("The command to benchmark. This can be the name of an executable, a command \ line like \"grep -i todo\" or a shell command like \"sleep 0.5 && echo test\". \ The latter is only available if the shell is not explicitly disabled via \ '--shell=none'. If multiple commands are given, hyperfine will show a \ comparison of the respective runtimes.") .required(true) .action(ArgAction::Append) .value_hint(ValueHint::CommandString) .value_parser(NonEmptyStringValueParser::new()), ) .arg( Arg::new("warmup") .long("warmup") .short('w') .value_name("NUM") .action(ArgAction::Set) .help( "Perform NUM warmup runs before the actual benchmark. This can be used \ to fill (disk) caches for I/O-heavy programs.", ), ) .arg( Arg::new("min-runs") .long("min-runs") .short('m') .action(ArgAction::Set) .value_name("NUM") .help("Perform at least NUM runs for each command (default: 10)."), ) .arg( Arg::new("max-runs") .long("max-runs") .short('M') .action(ArgAction::Set) .value_name("NUM") .help("Perform at most NUM runs for each command. By default, there is no limit."), ) .arg( Arg::new("runs") .long("runs") .conflicts_with_all(["max-runs", "min-runs"]) .short('r') .action(ArgAction::Set) .value_name("NUM") .help("Perform exactly NUM runs for each command. If this option is not specified, \ hyperfine automatically determines the number of runs."), ) .arg( Arg::new("setup") .long("setup") .short('s') .action(ArgAction::Set) .value_name("CMD") .value_hint(ValueHint::CommandString) .help( "Execute CMD before each set of timing runs. This is useful for \ compiling your software with the provided parameters, or to do any \ other work that should happen once before a series of benchmark runs, \ not every time as would happen with the --prepare option." ), ) .arg( Arg::new("reference") .long("reference") .action(ArgAction::Set) .value_name("CMD") .help( "The reference command for the relative comparison of results. \ If this is unset, results are compared with the fastest command as reference." ) ) .arg( Arg::new("reference-name") .long("reference-name") .action(ArgAction::Set) .value_name("CMD") .help("Give a meaningful name to the reference command.") .requires("reference") ) .arg( Arg::new("prepare") .long("prepare") .short('p') .action(ArgAction::Append) .num_args(1) .value_name("CMD") .value_hint(ValueHint::CommandString) .help( "Execute CMD before each timing run. This is useful for \ clearing disk caches, for example.\nThe --prepare option can \ be specified once for all commands or multiple times, once for \ each command. In the latter case, each preparation command will \ be run prior to the corresponding benchmark command.", ), ) .arg( Arg::new("conclude") .long("conclude") .short('C') .action(ArgAction::Append) .num_args(1) .value_name("CMD") .value_hint(ValueHint::CommandString) .help( "Execute CMD after each timing run. This is useful for killing \ long-running processes started (e.g. a web server started in --prepare), \ for example.\nThe --conclude option can be specified once for all \ commands or multiple times, once for each command. In the latter case, \ each conclude command will be run after the corresponding benchmark \ command.", ), ) .arg( Arg::new("cleanup") .long("cleanup") .short('c') .action(ArgAction::Set) .value_name("CMD") .value_hint(ValueHint::CommandString) .help( "Execute CMD after the completion of all benchmarking \ runs for each individual command to be benchmarked. \ This is useful if the commands to be benchmarked produce \ artifacts that need to be cleaned up." ), ) .arg( Arg::new("parameter-scan") .long("parameter-scan") .short('P') .action(ArgAction::Set) .allow_hyphen_values(true) .value_names(["VAR", "MIN", "MAX"]) .help( "Perform benchmark runs for each value in the range MIN..MAX. Replaces the \ string '{VAR}' in each command by the current parameter value.\n\n \ Example: hyperfine -P threads 1 8 'make -j {threads}'\n\n\ This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\ To have the value increase following different patterns, use shell arithmetics.\n\n \ Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\ This performs benchmarks with power of 2 increases: 'sleep 1', 'sleep 2', 'sleep 4', …\n\ The exact syntax may vary depending on your shell and OS." ), ) .arg( Arg::new("parameter-step-size") .long("parameter-step-size") .short('D') .action(ArgAction::Set) .value_names(["DELTA"]) .requires("parameter-scan") .help( "This argument requires --parameter-scan to be specified as well. \ Traverse the range MIN..MAX in steps of DELTA.\n\n \ Example: hyperfine -P delay 0.3 0.7 -D 0.2 'sleep {delay}'\n\n\ This performs benchmarks for 'sleep 0.3', 'sleep 0.5' and 'sleep 0.7'.", ), ) .arg( Arg::new("parameter-list") .long("parameter-list") .short('L') .action(ArgAction::Append) .allow_hyphen_values(true) .value_names(["VAR", "VALUES"]) .conflicts_with_all(["parameter-scan", "parameter-step-size"]) .help( "Perform benchmark runs for each value in the comma-separated list VALUES. \ Replaces the string '{VAR}' in each command by the current parameter value\ .\n\nExample: hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'\n\n\ This performs benchmarks for 'gcc -O2 main.cpp' and 'clang -O2 main.cpp'.\n\n\ The option can be specified multiple times to run benchmarks for all \ possible parameter combinations.\n" ), ) .arg( Arg::new("shell") .long("shell") .short('S') .action(ArgAction::Set) .value_name("SHELL") .overrides_with("shell") .value_hint(ValueHint::CommandString) .help("Set the shell to use for executing benchmarked commands. This can be the \ name or the path to the shell executable, or a full command line \ like \"bash --norc\". It can also be set to \"default\" to explicitly select \ the default shell on this platform. Finally, this can also be set to \ \"none\" to disable the shell. In this case, commands will be executed \ directly. They can still have arguments, but more complex things like \ \"sleep 0.1; sleep 0.2\" are not possible without a shell.") ) .arg( Arg::new("no-shell") .short('N') .action(ArgAction::SetTrue) .conflicts_with_all(["shell", "debug-mode"]) .help("An alias for '--shell=none'.") ) .arg( Arg::new("ignore-failure") .long("ignore-failure") .action(ArgAction::Set) .value_name("MODE") .num_args(0..=1) .default_missing_value("all-non-zero") .require_equals(true) .short('i') .help("Ignore failures of the benchmarked programs. Without a value or with \ 'all-non-zero', all non-zero exit codes are ignored. You can also provide \ a comma-separated list of exit codes to ignore (e.g., --ignore-failure=1,2)."), ) .arg( Arg::new("style") .long("style") .action(ArgAction::Set) .value_name("TYPE") .value_parser(["auto", "basic", "full", "nocolor", "color", "none"]) .help( "Set output style type (default: auto). Set this to 'basic' to disable output \ coloring and interactive elements. Set it to 'full' to enable all effects \ even if no interactive terminal was detected. Set this to 'nocolor' to \ keep the interactive output without any colors. Set this to 'color' to keep \ the colors without any interactive output. Set this to 'none' to disable all \ the output of the tool.", ), ) .arg( Arg::new("sort") .long("sort") .action(ArgAction::Set) .value_name("METHOD") .value_parser(["auto", "command", "mean-time"]) .default_value("auto") .hide_default_value(true) .help( "Specify the sort order of the speed comparison summary and the exported tables for \ markup formats (Markdown, AsciiDoc, org-mode):\n \ * 'auto' (default): the speed comparison will be ordered by time and\n \ the markup tables will be ordered by command (input order).\n \ * 'command': order benchmarks in the way they were specified\n \ * 'mean-time': order benchmarks by mean runtime\n" ), ) .arg( Arg::new("time-unit") .long("time-unit") .short('u') .action(ArgAction::Set) .value_name("UNIT") .value_parser(["microsecond", "millisecond", "second"]) .help("Set the time unit to be used. Possible values: microsecond, millisecond, second. \ If the option is not given, the time unit is determined automatically. \ This option affects the standard output as well as all export formats except for CSV and JSON."), ) .arg( Arg::new("export-asciidoc") .long("export-asciidoc") .action(ArgAction::Set) .value_name("FILE") .value_hint(ValueHint::FilePath) .help("Export the timing summary statistics as an AsciiDoc table to the given FILE. \ The output time unit can be changed using the --time-unit option."), ) .arg( Arg::new("export-csv") .long("export-csv") .action(ArgAction::Set) .value_name("FILE") .value_hint(ValueHint::FilePath) .help("Export the timing summary statistics as CSV to the given FILE. If you need \ the timing results for each individual run, use the JSON export format. \ The output time unit is always seconds."), ) .arg( Arg::new("export-json") .long("export-json") .action(ArgAction::Set) .value_name("FILE") .value_hint(ValueHint::FilePath) .help("Export the timing summary statistics and timings of individual runs as JSON to the given FILE. \ The output time unit is always seconds"), ) .arg( Arg::new("export-markdown") .long("export-markdown") .action(ArgAction::Set) .value_name("FILE") .value_hint(ValueHint::FilePath) .help("Export the timing summary statistics as a Markdown table to the given FILE. \ The output time unit can be changed using the --time-unit option."), ) .arg( Arg::new("export-orgmode") .long("export-orgmode") .action(ArgAction::Set) .value_name("FILE") .value_hint(ValueHint::FilePath) .help("Export the timing summary statistics as an Emacs org-mode table to the given FILE. \ The output time unit can be changed using the --time-unit option."), ) .arg( Arg::new("show-output") .long("show-output") .action(ArgAction::SetTrue) .conflicts_with("style") .help( "Print the stdout and stderr of the benchmark instead of suppressing it. \ This will increase the time it takes for benchmarks to run, \ so it should only be used for debugging purposes or \ when trying to benchmark output speed.", ), ) .arg( Arg::new("output") .long("output") .conflicts_with("show-output") .action(ArgAction::Append) .value_name("WHERE") .help( "Control where the output of the benchmark is redirected. Note \ that some programs like 'grep' detect when standard output is \ /dev/null and apply certain optimizations. To avoid that, consider \ using '--output=pipe'.\n\ \n\ can be:\n\ \n \ null: Redirect output to /dev/null (the default).\n\ \n \ pipe: Feed the output through a pipe before discarding it.\n\ \n \ inherit: Don't redirect the output at all (same as '--show-output').\n\ \n \ : Write the output to the given file.\n\n\ This option can be specified once for all commands or multiple times, once for \ each command. Note: If you want to log the output of each and every iteration, \ you can use a shell redirection and the '$HYPERFINE_ITERATION' environment variable:\n \ hyperfine 'my-command > output-${HYPERFINE_ITERATION}.log'\n\n", ), ) .arg( Arg::new("input") .long("input") .action(ArgAction::Set) .num_args(1) .value_name("WHERE") .help("Control where the input of the benchmark comes from.\n\ \n\ can be:\n\ \n \ null: Read from /dev/null (the default).\n\ \n \ : Read the input from the given file."), ) .arg( Arg::new("command-name") .long("command-name") .short('n') .action(ArgAction::Append) .num_args(1) .value_name("NAME") .help("Give a meaningful name to a command. This can be specified multiple times \ if several commands are benchmarked."), ) // This option is hidden for now, as it is not yet clear yet if we want to 'stabilize' this, // see discussion in https://github.com/sharkdp/hyperfine/issues/527 .arg( Arg::new("min-benchmarking-time") .long("min-benchmarking-time") .action(ArgAction::Set) .hide(true) .help("Set the minimum time (in seconds) to run benchmarks. Note that the number of \ benchmark runs is additionally influenced by the `--min-runs`, `--max-runs`, and \ `--runs` option.") ) .arg( Arg::new("debug-mode") .long("debug-mode") .action(ArgAction::SetTrue) .hide(true) .help("Enable debug mode which does not actually run commands, but returns fake times when the command is 'sleep