[ { "path": ".clippy.toml", "content": "# Clippy configuration\n# https://doc.rust-lang.org/nightly/clippy/lint_configuration.html\n\nallow-private-module-inception = true\navoid-breaking-exported-api = false\ndisallowed-names = []\ndisallowed-macros = [\n { path = \"std::dbg\", reason = \"it is okay to use during development, but please do not include it in master branch\" },\n]\ndisallowed-methods = [\n # Use helper for safer allocation instead.\n { path = \"alloc::alloc::alloc\", replacement = \"crate::alloc_helper::Global::allocate\" },\n { path = \"alloc::alloc::alloc_zeroed\", replacement = \"crate::alloc_helper::Global::allocate_zeroed\" },\n { path = \"alloc::alloc::dealloc\", replacement = \"crate::alloc_helper::Global::deallocate\" },\n]\ndisallowed-types = [\n]\n" }, { "path": ".editorconfig", "content": "# EditorConfig configuration\n# https://editorconfig.org\n\nroot = true\n\n[*]\ncharset = utf-8\nend_of_line = lf\nindent_size = 4\nindent_style = space\ninsert_final_newline = true\ntrim_trailing_whitespace = true\n\n[*.{css,html,json,md,rb,sh,yml,yaml}]\nindent_size = 2\n\n[*.{js,yml,yaml}]\nquote_type = single\n\n[*.sh]\n# https://google.github.io/styleguide/shellguide.html#s5.3-pipelines\nbinary_next_line = true\n# https://google.github.io/styleguide/shellguide.html#s5.5-case-statement\nswitch_case_indent = true\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: cargo\n directory: /\n schedule:\n interval: weekly\n commit-message:\n prefix: ''\n labels: []\n - package-ecosystem: github-actions\n directory: /\n schedule:\n interval: weekly\n commit-message:\n prefix: ''\n labels: []\n" }, { "path": ".github/workflows/ci.yml", "content": "name: CI\n\npermissions:\n contents: read\n\non:\n pull_request:\n push:\n branches:\n - master\n schedule:\n - cron: '0 2 * * *'\n workflow_dispatch:\n\nenv:\n CARGO_INCREMENTAL: 0\n CARGO_NET_GIT_FETCH_WITH_CLI: true\n CARGO_NET_RETRY: 10\n CARGO_TERM_COLOR: always\n RUST_BACKTRACE: 1\n RUSTDOCFLAGS: -D warnings\n RUSTFLAGS: -D warnings\n RUSTUP_MAX_RETRIES: 10\n\ndefaults:\n run:\n shell: bash --noprofile --norc -CeEuxo pipefail {0}\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}\n cancel-in-progress: true\n\njobs:\n check-external-types:\n uses: taiki-e/github-actions/.github/workflows/check-external-types.yml@main\n clippy:\n uses: taiki-e/github-actions/.github/workflows/clippy.yml@main\n with:\n fuzzy_provenance_casts: false # TODO\n docs:\n uses: taiki-e/github-actions/.github/workflows/docs.yml@main\n\n # Test crates on their minimum Rust versions and nightly Rust.\n test:\n env:\n RUST_VERSION: ${{ matrix.rust }}\n RUST_TARGET: ${{ matrix.target }}\n strategy:\n fail-fast: false\n matrix:\n include:\n - rust: '1.74'\n os: ubuntu-latest\n - rust: '1.74'\n os: ubuntu-24.04-arm\n - rust: '1.74'\n os: windows-latest\n - rust: stable\n os: ubuntu-latest\n - rust: stable\n os: ubuntu-24.04-arm\n - rust: stable\n os: windows-latest\n - rust: nightly\n os: ubuntu-latest\n - rust: nightly\n os: ubuntu-24.04-arm\n - rust: nightly\n os: macos-latest\n - rust: nightly\n os: windows-latest\n - rust: nightly\n os: ubuntu-latest\n target: i686-unknown-linux-gnu\n - rust: nightly\n os: ubuntu-24.04-arm\n target: armv7-unknown-linux-gnueabihf\n - rust: nightly\n os: ubuntu-latest\n target: powerpc64le-unknown-linux-gnu\n - rust: nightly\n os: ubuntu-latest\n target: s390x-unknown-linux-gnu\n # Test 32-bit target that does not have AtomicU64/AtomicI64.\n - rust: nightly\n os: ubuntu-latest\n target: armv5te-unknown-linux-gnueabi\n # Test target without stable inline asm support.\n - rust: stable\n os: ubuntu-latest\n target: sparc64-unknown-linux-gnu\n runs-on: ${{ matrix.os }}\n timeout-minutes: 60\n steps:\n - uses: taiki-e/checkout-action@v1\n - name: Install Rust\n uses: taiki-e/github-actions/install-rust@main\n with:\n toolchain: ${{ matrix.rust }}\n - uses: taiki-e/setup-cross-toolchain-action@v1\n with:\n target: ${{ matrix.target }}\n if: matrix.target != ''\n - run: |\n echo \"RUSTFLAGS=${RUSTFLAGS} --cfg gha_macos_runner\" >>\"${GITHUB_ENV}\"\n echo \"RUSTDOCFLAGS=${RUSTDOCFLAGS} --cfg gha_macos_runner\" >>\"${GITHUB_ENV}\"\n if: startsWith(matrix.os, 'macos')\n - name: Test\n run: ci/test.sh\n\n # Check all feature combinations works properly.\n features:\n env:\n RUST_VERSION: ${{ matrix.rust }}\n strategy:\n fail-fast: false\n matrix:\n rust:\n - msrv\n - nightly\n runs-on: ubuntu-latest\n timeout-minutes: 60\n steps:\n - uses: taiki-e/checkout-action@v1\n - name: Install Rust\n uses: taiki-e/github-actions/install-rust@main\n with:\n toolchain: ${{ matrix.rust }}\n if: matrix.rust != 'msrv'\n - name: Install cargo-hack\n uses: taiki-e/install-action@cargo-hack\n - name: Check features\n run: ci/check-features.sh\n\n # Check for duplicate dependencies.\n dependencies:\n runs-on: ubuntu-latest\n timeout-minutes: 60\n steps:\n - uses: taiki-e/checkout-action@v1\n - name: Install Rust\n uses: taiki-e/github-actions/install-rust@nightly\n - name: Install cargo-hack\n uses: taiki-e/install-action@cargo-hack\n - name: Install cargo-minimal-versions\n uses: taiki-e/install-action@cargo-minimal-versions\n - name: dependency tree check\n run: ci/dependencies.sh\n\n # When this job failed, run ci/no_atomic.sh and commit result changes.\n codegen:\n runs-on: ubuntu-latest\n permissions:\n contents: write\n pull-requests: write\n timeout-minutes: 60\n steps:\n - uses: taiki-e/checkout-action@v1\n - name: Install Rust\n uses: taiki-e/github-actions/install-rust@stable\n - run: ci/no_atomic.sh\n - run: git add -N . && git diff --exit-code\n if: github.repository_owner != 'crossbeam-rs' || github.event_name != 'schedule'\n - id: diff\n run: |\n git config user.name \"Taiki Endo\"\n git config user.email \"te316e89@gmail.com\"\n git add -N .\n if ! git diff --exit-code; then\n git add .\n git commit -m \"Update no_atomic.rs\"\n echo 'success=false' >>\"${GITHUB_OUTPUT}\"\n fi\n if: github.repository_owner == 'crossbeam-rs' && github.event_name == 'schedule'\n - uses: peter-evans/create-pull-request@v8\n with:\n title: Update no_atomic.rs\n body: |\n Auto-generated by [create-pull-request][1]\n [Please close and immediately reopen this pull request to run CI.][2]\n\n [1]: https://github.com/peter-evans/create-pull-request\n [2]: https://github.com/peter-evans/create-pull-request/blob/HEAD/docs/concepts-guidelines.md#workarounds-to-trigger-further-workflow-runs\n branch: update-no-atomic-rs\n if: github.repository_owner == 'crossbeam-rs' && github.event_name == 'schedule' && steps.diff.outputs.success == 'false'\n\n # Check formatting.\n fmt:\n runs-on: ubuntu-latest\n timeout-minutes: 60\n steps:\n - uses: taiki-e/checkout-action@v1\n - name: Install Rust\n uses: taiki-e/github-actions/install-rust@nightly\n with:\n component: rustfmt\n - name: Install shfmt\n uses: taiki-e/install-action@shfmt\n # `cargo fmt` cannot recognize files not included in the current workspace and modules\n # defined inside macros, so run rustfmt directly.\n # We need to use nightly rustfmt because we use the unstable formatting options of rustfmt.\n - run: rustfmt --check $(git ls-files '*.rs')\n - run: shfmt -d $(git ls-files '*.sh')\n - run: RUST_LOG=warn npx -y @taplo/cli fmt --check $(git ls-files '*.toml')\n\n # Run miri.\n miri:\n strategy:\n fail-fast: false\n matrix:\n group:\n - channel\n - others\n miriflags:\n - ''\n - '-Zmiri-tree-borrows'\n runs-on: ubuntu-latest\n timeout-minutes: 60\n steps:\n - uses: taiki-e/checkout-action@v1\n - name: Install Rust\n uses: taiki-e/github-actions/install-rust@nightly\n with:\n component: miri\n - name: miri\n run: ci/miri.sh ${{ matrix.group }}\n env:\n MIRIFLAGS: ${{ matrix.miriflags }}\n\n # Run cargo-careful.\n careful:\n runs-on: ubuntu-latest\n timeout-minutes: 60\n steps:\n - uses: taiki-e/checkout-action@v1\n - name: Install Rust\n uses: taiki-e/github-actions/install-rust@nightly\n with:\n component: rust-src\n - uses: taiki-e/install-action@cargo-careful\n - name: Run cargo-careful\n run: ci/careful.sh\n\n # Run sanitizers.\n san:\n runs-on: ubuntu-latest\n timeout-minutes: 60\n steps:\n - uses: taiki-e/checkout-action@v1\n - name: Install Rust\n uses: taiki-e/github-actions/install-rust@nightly\n with:\n component: rust-src\n target: x86_64-unknown-linux-gnuasan\n - name: Run sanitizers\n run: ci/san.sh\n\n # Run loom tests.\n loom:\n runs-on: ubuntu-latest\n timeout-minutes: 60\n steps:\n - uses: taiki-e/checkout-action@v1\n - name: Install Rust\n uses: taiki-e/github-actions/install-rust@stable\n - name: loom\n run: ci/crossbeam-epoch-loom.sh\n\n shellcheck:\n runs-on: ubuntu-latest\n timeout-minutes: 60\n steps:\n - uses: taiki-e/checkout-action@v1\n - name: Install shellcheck\n uses: taiki-e/install-action@shellcheck\n - run: shellcheck $(git ls-files '*.sh')\n" }, { "path": ".github/workflows/release.yml", "content": "name: Release\n\npermissions:\n contents: read\n\non:\n push:\n tags:\n - crossbeam-[0-9]+.*\n - crossbeam-[a-z]+-[0-9]+.*\n\ndefaults:\n run:\n shell: bash --noprofile --norc -CeEuxo pipefail {0}\n\njobs:\n create-release:\n if: github.repository_owner == 'crossbeam-rs'\n runs-on: ubuntu-latest\n timeout-minutes: 60\n permissions:\n contents: write # for taiki-e/create-gh-release-action\n steps:\n - uses: taiki-e/checkout-action@v1\n - uses: taiki-e/create-gh-release-action@v1\n with:\n prefix: crossbeam(-[a-z]+)?\n changelog: $prefix/CHANGELOG.md\n title: $prefix $version\n branch: master\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n if: \"!startsWith(github.ref_name, 'crossbeam-0')\"\n # Since the crossbeam crate is located at the root, we need to handle it differently.\n # TODO: Consider moving crossbeam crate to a subdirectory.\n - uses: taiki-e/create-gh-release-action@v1\n with:\n prefix: crossbeam(-[a-z]+)?\n changelog: CHANGELOG.md\n title: crossbeam $version\n branch: master\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n if: startsWith(github.ref_name, 'crossbeam-0')\n" }, { "path": ".gitignore", "content": "/crossbeam-channel/benchmarks/*.txt\n/crossbeam-channel/benchmarks/*.png\ntarget/\nCargo.lock\n" }, { "path": ".rustfmt.toml", "content": "# Rustfmt configuration\n# https://github.com/rust-lang/rustfmt/blob/HEAD/Configurations.md\n\n# Override the default formatting style.\n# This is unstable (tracking issue: https://github.com/rust-lang/rustfmt/issues/4991).\nimports_granularity = \"Crate\"\n# This is unstable (tracking issue: https://github.com/rust-lang/rustfmt/issues/5083).\ngroup_imports = \"StdExternalCrate\"\n\n# Set the default settings again to always apply the proper formatting without\n# being affected by the editor settings.\nedition = \"2021\"\nstyle_edition = \"2024\"\nhard_tabs = false\nnewline_style = \"Unix\"\ntab_spaces = 4\n" }, { "path": ".shellcheckrc", "content": "# ShellCheck configuration\n# https://github.com/koalaman/shellcheck/blob/HEAD/shellcheck.1.md#rc-files\n\n# See also:\n# https://github.com/koalaman/shellcheck/wiki/Optional\n# https://google.github.io/styleguide/shellguide.html\n\n# https://github.com/koalaman/shellcheck/wiki/Directive#external-sources\nexternal-sources=true\n\n# https://github.com/koalaman/shellcheck/wiki/SC2249\n# enable=add-default-case\n\n# https://github.com/koalaman/shellcheck/wiki/SC2244\nenable=avoid-nullary-conditions\n\n# https://github.com/koalaman/shellcheck/wiki/SC2312\n# enable=check-extra-masked-returns\n\n# https://github.com/koalaman/shellcheck/wiki/SC2310\n# https://github.com/koalaman/shellcheck/wiki/SC2311\n# enable=check-set-e-suppressed\n\n# enable=check-unassigned-uppercase\n\n# https://github.com/koalaman/shellcheck/wiki/SC2230\nenable=deprecate-which\n\n# https://github.com/koalaman/shellcheck/wiki/SC2248\nenable=quote-safe-variables\n\n# https://github.com/koalaman/shellcheck/wiki/SC2292\n# https://google.github.io/styleguide/shellguide.html#s6.3-tests\nenable=require-double-brackets\n\n# https://github.com/koalaman/shellcheck/wiki/SC2250\n# https://google.github.io/styleguide/shellguide.html#s5.6-variable-expansion\nenable=require-variable-braces\n" }, { "path": ".taplo.toml", "content": "# Taplo configuration\n# https://taplo.tamasfe.dev/configuration/formatter-options.html\n\n[formatting]\nalign_comments = false\nallowed_blank_lines = 1\narray_auto_collapse = false\narray_auto_expand = false\nindent_string = \" \"\n" }, { "path": "CHANGELOG.md", "content": "# Version 0.8.4\n\n- Remove dependency on `cfg-if`. (#1072)\n\n# Version 0.8.3\n\n- Bump the minimum supported Rust version to 1.61. (#1037)\n\n# Version 0.8.2\n\n- Bump the minimum supported Rust version to 1.38. (#877)\n\n# Version 0.8.1\n\n- Support targets that do not have atomic CAS on stable Rust (#698)\n\n# Version 0.8.0\n\n- Bump the minimum supported Rust version to 1.36.\n- Bump `crossbeam-channel` to `0.5`.\n- Bump `crossbeam-deque` to `0.8`.\n- Bump `crossbeam-epoch` to `0.9`.\n- Bump `crossbeam-queue` to `0.3`.\n- Bump `crossbeam-utils` to `0.8`.\n\n# Version 0.7.3\n\n- Fix breakage with nightly feature due to rust-lang/rust#65214.\n- Bump `crossbeam-channel` to `0.4`.\n- Bump `crossbeam-epoch` to `0.8`.\n- Bump `crossbeam-queue` to `0.2`.\n- Bump `crossbeam-utils` to `0.7`.\n\n# Version 0.7.2\n\n- Bump `crossbeam-channel` to `0.3.9`.\n- Bump `crossbeam-epoch` to `0.7.2`.\n- Bump `crossbeam-utils` to `0.6.6`.\n\n# Version 0.7.1\n\n- Bump `crossbeam-utils` to `0.6.5`.\n\n# Version 0.7.0\n\n- Remove `ArcCell`, `MsQueue`, and `TreiberStack`.\n- Change the interface of `ShardedLock` to match `RwLock`.\n- Add `SegQueue::len()`.\n- Rename `SegQueue::try_pop()` to `SegQueue::pop()`.\n- Change the return type of `SegQueue::pop()` to `Result`.\n- Introduce `ArrayQueue`.\n- Update dependencies.\n\n# Version 0.6.0\n\n- Update dependencies.\n\n# Version 0.5.0\n\n- Update `crossbeam-channel` to 0.3.\n- Update `crossbeam-utils` to 0.6.\n- Add `AtomicCell`, `SharedLock`, and `WaitGroup`.\n\n# Version 0.4.1\n\n- Fix a double-free bug in `MsQueue` and `SegQueue`.\n\n# Version 0.4\n\n- Switch to the new implementation of epoch-based reclamation in\n [`crossbeam-epoch`](https://github.com/crossbeam-rs/crossbeam-epoch), fixing numerous bugs in the\n old implementation. Its API is changed in a backward-incompatible way.\n- Switch to the new implementation of `CachePadded` and scoped thread in\n [`crossbeam-utils`](https://github.com/crossbeam-rs/crossbeam-utils). The scoped thread API is\n changed in a backward-incompatible way.\n- Switch to the new implementation of Chase-Lev deque in\n [`crossbeam-deque`](https://github.com/crossbeam-rs/crossbeam-deque). Its API is changed in a\n backward-incompatible way.\n- Export channel implemented in\n [`crossbeam-channel`](https://github.com/crossbeam-rs/crossbeam-channel).\n- Remove `AtomicOption`.\n- Implement `Default` and `From` traits.\n\n# Version 0.3\n\n- Introduced `ScopedThreadBuilder` with the ability to name threads and set stack size\n- `Worker` methods in the Chase-Lev deque don't require mutable access anymore\n- Fixed a bug when unblocking `pop()` in `MsQueue`\n- Implemented `Drop` for `MsQueue`, `SegQueue`, and `TreiberStack`\n- Implemented `Default` for `TreiberStack`\n- Added `is_empty` to `SegQueue`\n- Renamed `mem::epoch` to `epoch`\n- Other bug fixes\n\n# Version 0.2\n\n- Changed existing non-blocking `pop` methods to `try_pop`\n- Added blocking `pop` support to Michael-Scott queue\n- Added Chase-Lev work-stealing deque\n\n# Version 0.1\n\n- Added [epoch-based memory management](http://aturon.github.io/blog/2015/08/27/epoch/)\n- Added Michael-Scott queue\n- Added Segmented array queue\n" }, { "path": "Cargo.toml", "content": "[package]\nname = \"crossbeam\"\n# When publishing a new version:\n# - Update CHANGELOG.md\n# - Update README.md (when increasing major or minor version)\n# - Run './tools/publish.sh crossbeam '\nversion = \"0.8.4\"\nedition = \"2021\"\n# NB: Sync with msrv badge and \"Compatibility\" section in README.md\nrust-version = \"1.74\"\nlicense = \"MIT OR Apache-2.0\"\nrepository = \"https://github.com/crossbeam-rs/crossbeam\"\nhomepage = \"https://github.com/crossbeam-rs/crossbeam\"\ndescription = \"Tools for concurrent programming\"\nkeywords = [\"atomic\", \"garbage\", \"non-blocking\", \"lock-free\", \"rcu\"]\ncategories = [\"concurrency\", \"memory-management\", \"data-structures\", \"no-std\"]\nexclude = [\"/.*\", \"/ci\", \"/tools\"]\n\n[package.metadata.docs.rs]\ntargets = [\"x86_64-unknown-linux-gnu\"]\n\n[package.metadata.cargo_check_external_types]\n# The following are external types that are allowed to be exposed in our public API.\nallowed_external_types = [\n \"crossbeam_channel\",\n \"crossbeam_channel::*\",\n \"crossbeam_deque\",\n \"crossbeam_epoch\",\n \"crossbeam_queue\",\n \"crossbeam_utils::*\",\n]\n\n[features]\ndefault = [\"std\"]\n\n# Enable to use APIs that require `std`.\n# This is enabled by default.\nstd = [\n \"alloc\",\n \"crossbeam-channel/std\",\n \"crossbeam-deque/std\",\n \"crossbeam-epoch/std\",\n \"crossbeam-queue/std\",\n \"crossbeam-utils/std\",\n]\n\n# Enable to use APIs that require `alloc`.\n# This is enabled by default and also enabled if the `std` feature is enabled.\nalloc = [\"crossbeam-epoch/alloc\", \"crossbeam-queue/alloc\"]\n\n[dependencies]\ncrossbeam-channel = { version = \"0.5.10\", path = \"crossbeam-channel\", default-features = false, optional = true }\ncrossbeam-deque = { version = \"0.8.4\", path = \"crossbeam-deque\", default-features = false, optional = true }\ncrossbeam-epoch = { version = \"0.9.17\", path = \"crossbeam-epoch\", default-features = false, optional = true }\ncrossbeam-queue = { version = \"0.3.10\", path = \"crossbeam-queue\", default-features = false, optional = true }\ncrossbeam-utils = { version = \"0.8.18\", path = \"crossbeam-utils\", default-features = false, features = [\"atomic\"] }\n\n[dev-dependencies]\n\n[lints]\nworkspace = true\n\n[workspace]\nresolver = \"2\"\nmembers = [\n \".\",\n \"crossbeam-channel\",\n \"crossbeam-channel/benchmarks\",\n \"crossbeam-deque\",\n \"crossbeam-epoch\",\n \"crossbeam-queue\",\n \"crossbeam-skiplist\",\n \"crossbeam-utils\",\n]\n\n[workspace.lints.rust]\nmissing_debug_implementations = \"warn\"\nrust_2018_idioms = \"warn\"\nsingle_use_lifetimes = \"warn\"\nunexpected_cfgs = { level = \"warn\", check-cfg = [\n 'cfg(crossbeam_loom)',\n 'cfg(crossbeam_sanitize)',\n 'cfg(gha_macos_runner)',\n] }\nunreachable_pub = \"warn\"\n# unsafe_op_in_unsafe_fn = \"warn\" # Set at crate-level instead since https://github.com/rust-lang/rust/pull/100081 is not available on MSRV\n[workspace.lints.clippy]\n# Suppress buggy or noisy clippy lints\nbool_assert_comparison = { level = \"allow\", priority = 1 }\ndeclare_interior_mutable_const = { level = \"allow\", priority = 1 } # https://github.com/rust-lang/rust-clippy/issues/7665\nincompatible_msrv = { level = \"allow\", priority = 1 } # buggy: doesn't consider cfg, https://github.com/rust-lang/rust-clippy/issues/12280, https://github.com/rust-lang/rust-clippy/issues/12257#issuecomment-2093667187\nlint_groups_priority = { level = \"allow\", priority = 1 } # https://github.com/rust-lang/rust-clippy/issues/12920\nmanual_map = { level = \"allow\", priority = 1 }\nmanual_range_contains = { level = \"allow\", priority = 1 } # https://github.com/rust-lang/rust-clippy/issues/6455#issuecomment-1225966395\ntype_complexity = { level = \"allow\", priority = 1 }\n" }, { "path": "LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright 2019 The Crossbeam Project Developers\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "LICENSE-MIT", "content": "The MIT License (MIT)\n\nCopyright (c) 2019 The Crossbeam Project Developers\n\nPermission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "README.md", "content": "# Crossbeam\n\n[![Build Status](https://github.com/crossbeam-rs/crossbeam/workflows/CI/badge.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/actions)\n[![License](https://img.shields.io/badge/license-MIT_OR_Apache--2.0-blue.svg)](\nhttps://github.com/crossbeam-rs/crossbeam#license)\n[![Cargo](https://img.shields.io/crates/v/crossbeam.svg)](\nhttps://crates.io/crates/crossbeam)\n[![Documentation](https://docs.rs/crossbeam/badge.svg)](\nhttps://docs.rs/crossbeam)\n[![Rust 1.74+](https://img.shields.io/badge/rust-1.74+-lightgray.svg)](\nhttps://www.rust-lang.org)\n[![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ)\n\nThis crate provides a set of tools for concurrent programming:\n\n#### Atomics\n\n* [`AtomicCell`], a thread-safe mutable memory location.(no_std)\n* [`AtomicConsume`], for reading from primitive atomic types with \"consume\" ordering.(no_std)\n\n#### Data structures\n\n* [`deque`], work-stealing deques for building task schedulers.\n* [`ArrayQueue`], a bounded MPMC queue that allocates a fixed-capacity buffer on construction.(alloc)\n* [`SegQueue`], an unbounded MPMC queue that allocates small buffers, segments, on demand.(alloc)\n\n#### Memory management\n\n* [`epoch`], an epoch-based garbage collector.(alloc)\n\n#### Thread synchronization\n\n* [`channel`], multi-producer multi-consumer channels for message passing.\n* [`Parker`], a thread parking primitive.\n* [`ShardedLock`], a sharded reader-writer lock with fast concurrent reads.\n* [`WaitGroup`], for synchronizing the beginning or end of some computation.\n\n#### Utilities\n\n* [`Backoff`], for exponential backoff in spin loops.(no_std)\n* [`CachePadded`], for padding and aligning a value to the length of a cache line.(no_std)\n* [`scope`], for spawning threads that borrow local variables from the stack.\n\n*Features marked with (no_std) can be used in `no_std` environments.*
\n*Features marked with (alloc) can be used in `no_std` environments, but only if `alloc`\nfeature is enabled.*\n\n[`AtomicCell`]: https://docs.rs/crossbeam/latest/crossbeam/atomic/struct.AtomicCell.html\n[`AtomicConsume`]: https://docs.rs/crossbeam/latest/crossbeam/atomic/trait.AtomicConsume.html\n[`deque`]: https://docs.rs/crossbeam/latest/crossbeam/deque/index.html\n[`ArrayQueue`]: https://docs.rs/crossbeam/latest/crossbeam/queue/struct.ArrayQueue.html\n[`SegQueue`]: https://docs.rs/crossbeam/latest/crossbeam/queue/struct.SegQueue.html\n[`channel`]: https://docs.rs/crossbeam/latest/crossbeam/channel/index.html\n[`Parker`]: https://docs.rs/crossbeam/latest/crossbeam/sync/struct.Parker.html\n[`ShardedLock`]: https://docs.rs/crossbeam/latest/crossbeam/sync/struct.ShardedLock.html\n[`WaitGroup`]: https://docs.rs/crossbeam/latest/crossbeam/sync/struct.WaitGroup.html\n[`epoch`]: https://docs.rs/crossbeam/latest/crossbeam/epoch/index.html\n[`Backoff`]: https://docs.rs/crossbeam/latest/crossbeam/utils/struct.Backoff.html\n[`CachePadded`]: https://docs.rs/crossbeam/latest/crossbeam/utils/struct.CachePadded.html\n[`scope`]: https://docs.rs/crossbeam/latest/crossbeam/fn.scope.html\n\n## Crates\n\nThe main `crossbeam` crate just [re-exports](src/lib.rs) tools from\nsmaller subcrates:\n\n* [`crossbeam-channel`](crossbeam-channel)\n provides multi-producer multi-consumer channels for message passing.\n* [`crossbeam-deque`](crossbeam-deque)\n provides work-stealing deques, which are primarily intended for building task schedulers.\n* [`crossbeam-epoch`](crossbeam-epoch)\n provides epoch-based garbage collection for building concurrent data structures.\n* [`crossbeam-queue`](crossbeam-queue)\n provides concurrent queues that can be shared among threads.\n* [`crossbeam-utils`](crossbeam-utils)\n provides atomics, synchronization primitives, scoped threads, and other utilities.\n\nThere is one more experimental subcrate that is not yet included in `crossbeam`:\n\n* [`crossbeam-skiplist`](crossbeam-skiplist)\n provides concurrent maps and sets based on lock-free skip lists.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\ncrossbeam = \"0.8\"\n```\n\n## Compatibility\n\nCrossbeam supports stable Rust releases going back at least six months,\nand every time the minimum supported Rust version is increased, a new minor\nversion is released. Currently, the minimum supported Rust version is 1.74.\n\n## Contributing\n\nCrossbeam welcomes contribution from everyone in the form of suggestions, bug reports,\npull requests, and feedback. 💛\n\nIf you need ideas for contribution, there are several ways to get started:\n\n* Found a bug or have a feature request?\n [Submit an issue](https://github.com/crossbeam-rs/crossbeam/issues/new)!\n* Issues and PRs labeled with\n [feedback wanted](https://github.com/crossbeam-rs/crossbeam/issues?utf8=%E2%9C%93&q=is%3Aopen+sort%3Aupdated-desc+label%3A%22feedback+wanted%22+)\n need feedback from users and contributors.\n* Issues labeled with\n [good first issue](https://github.com/crossbeam-rs/crossbeam/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22)\n are relatively easy starter issues.\n\n#### RFCs\n\nWe also have the [RFCs](https://github.com/crossbeam-rs/rfcs) repository for more\nhigh-level discussion, which is the place where we brainstorm ideas and propose\nsubstantial changes to Crossbeam.\n\nYou are welcome to participate in any open\n[issues](https://github.com/crossbeam-rs/rfcs/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc)\nor\n[pull requests](https://github.com/crossbeam-rs/rfcs/pulls?q=is%3Apr+is%3Aopen+sort%3Aupdated-desc).\n\n#### Learning resources\n\nIf you'd like to learn more about concurrency and non-blocking data structures, there's a\nlist of learning resources in our [wiki](https://github.com/crossbeam-rs/rfcs/wiki),\nwhich includes relevant blog posts, papers, videos, and other similar projects.\n\nAnother good place to visit is [merged RFCs](https://github.com/crossbeam-rs/rfcs/tree/master/text).\nThey contain elaborate descriptions and rationale for features we've introduced to\nCrossbeam, but keep in mind that some of the written information is now out of date.\n\n#### Conduct\n\nThe Crossbeam project adheres to the\n[Rust Code of Conduct](https://www.rust-lang.org/policies/code-of-conduct).\nThis describes the minimum behavior expected from all contributors.\n\n## License\n\nLicensed under either of\n\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\nSome Crossbeam subcrates have additional licensing notices.\nTake a look at other readme files in this repository for more information.\n\n#### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "path": "build-common.rs", "content": "// The target triplets have the form of 'arch-vendor-system'.\n//\n// When building for Linux (e.g. the 'system' part is\n// 'linux-something'), replace the vendor with 'unknown'\n// so that mapping to rust standard targets happens correctly.\nfn convert_custom_linux_target(target: String) -> String {\n let mut parts: Vec<&str> = target.split('-').collect();\n let system = parts.get(2);\n if system == Some(&\"linux\") {\n parts[1] = \"unknown\";\n };\n parts.join(\"-\")\n}\n" }, { "path": "ci/careful.sh", "content": "#!/bin/bash\nset -euxo pipefail\nIFS=$'\\n\\t'\ncd \"$(dirname \"$0\")\"/..\n\nexport RUSTFLAGS=\"${RUSTFLAGS:-} -Z randomize-layout\"\n\ncargo careful test --all --all-features --exclude benchmarks -- --test-threads=1\n" }, { "path": "ci/check-features.sh", "content": "#!/bin/bash\nset -euxo pipefail\nIFS=$'\\n\\t'\ncd \"$(dirname \"$0\")\"/..\n\n# * `--feature-powerset` - run for the feature powerset which includes --no-default-features and default features of package\n# * `--no-dev-deps` - build without dev-dependencies to avoid https://github.com/rust-lang/cargo/issues/4866\n# * `--exclude benchmarks` - benchmarks doesn't published.\nif [[ \"${RUST_VERSION}\" == \"msrv\" ]]; then\n cargo hack build --all --feature-powerset --no-dev-deps --exclude crossbeam-utils --exclude benchmarks --rust-version\n # atomic feature requires Rust 1.74.\n cargo hack build -p crossbeam-utils --feature-powerset --no-dev-deps --rust-version --exclude-features atomic\n cargo +1.74 hack build -p crossbeam-utils --feature-powerset --no-dev-deps\nelse\n cargo hack build --all --feature-powerset --no-dev-deps --exclude benchmarks\nfi\n\nif [[ \"${RUST_VERSION}\" == \"nightly\"* ]]; then\n # Build for no_std environment.\n # thumbv7m-none-eabi supports atomic CAS.\n # thumbv6m-none-eabi supports atomic, but not atomic CAS.\n # riscv32i-unknown-none-elf does not support atomic at all.\n rustup target add thumbv7m-none-eabi\n rustup target add thumbv6m-none-eabi\n rustup target add riscv32i-unknown-none-elf\n cargo hack build --all --feature-powerset --no-dev-deps --exclude benchmarks --target thumbv7m-none-eabi --skip std,default\n cargo hack build --all --feature-powerset --no-dev-deps --exclude benchmarks --target thumbv6m-none-eabi --skip std,default\n cargo hack build --all --feature-powerset --no-dev-deps --exclude benchmarks --target riscv32i-unknown-none-elf --skip std,default\nfi\n" }, { "path": "ci/crossbeam-epoch-loom.sh", "content": "#!/bin/bash\nset -euxo pipefail\nIFS=$'\\n\\t'\ncd \"$(dirname \"$0\")\"/../crossbeam-epoch\n\nexport RUSTFLAGS=\"${RUSTFLAGS:-} --cfg crossbeam_loom --cfg crossbeam_sanitize\"\n\n# With MAX_PREEMPTIONS=2 the loom tests (currently) take around 11m.\n# If we were to run with =3, they would take several times that,\n# which is probably too costly for CI.\nenv LOOM_MAX_PREEMPTIONS=2 cargo test --test loom --release --features loom -- --nocapture\n" }, { "path": "ci/dependencies.sh", "content": "#!/bin/bash\nset -euxo pipefail\nIFS=$'\\n\\t'\ncd \"$(dirname \"$0\")\"/..\n\ncargo tree\ncargo tree --duplicate\n\n# Check minimal versions.\ncargo minimal-versions build --workspace --all-features --exclude benchmarks\n" }, { "path": "ci/miri.sh", "content": "#!/bin/bash\nset -euxo pipefail\nIFS=$'\\n\\t'\ncd \"$(dirname \"$0\")\"/..\n\ngroup=$1\n\n# We need 'ts' for the per-line timing\nsudo apt-get -y install moreutils\necho\n\nexport RUSTFLAGS=\"${RUSTFLAGS:-} -Z randomize-layout\"\nexport RUSTDOCFLAGS=\"${RUSTDOCFLAGS:-} -Z randomize-layout\"\nexport MIRIFLAGS=\"${MIRIFLAGS:-} -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation\"\n\ncase \"${group}\" in\n channel)\n MIRI_LEAK_CHECK='1' \\\n cargo miri test --all-features \\\n -p crossbeam-channel 2>&1 | ts -i '%.s '\n # -Zmiri-ignore-leaks is needed because we use detached threads in tests in tests/golang.rs: https://github.com/rust-lang/miri/issues/1371\n MIRIFLAGS=\"${MIRIFLAGS} -Zmiri-ignore-leaks\" \\\n cargo miri test --all-features \\\n -p crossbeam-channel --test golang 2>&1 | ts -i '%.s '\n ;;\n others)\n cargo miri test --all-features \\\n -p crossbeam-queue \\\n -p crossbeam-epoch \\\n -p crossbeam-utils \\\n -p crossbeam-skiplist \\\n -p crossbeam 2>&1 | ts -i '%.s '\n # -Zmiri-preemption-rate=0 is needed because this code technically has UB and Miri catches that.\n MIRI_FALLIBLE_WEAK_CAS='1' \\\n MIRIFLAGS=\"${MIRIFLAGS} -Zmiri-preemption-rate=0\" \\\n cargo miri test --all-features \\\n -p crossbeam-deque 2>&1 | ts -i '%.s '\n # -Zmiri-compare-exchange-weak-failure-rate=0.0 is needed because some sequential tests (e.g.,\n # doctest of Injector::steal) incorrectly assume that sequential weak CAS will never fail.\n # -Zmiri-preemption-rate=0 is needed because this code technically has UB and Miri catches that.\n MIRIFLAGS=\"${MIRIFLAGS} -Zmiri-compare-exchange-weak-failure-rate=0.0 -Zmiri-preemption-rate=0\" \\\n cargo miri test --all-features \\\n -p crossbeam-deque --test injector 2>&1 | ts -i '%.s '\n MIRIFLAGS=\"${MIRIFLAGS} -Zmiri-compare-exchange-weak-failure-rate=0.0 -Zmiri-preemption-rate=0\" \\\n cargo miri test --all-features \\\n -p crossbeam-deque --doc 2>&1 | ts -i '%.s '\n ;;\n *)\n echo \"unknown crate group '${group}'\"\n exit 1\n ;;\nesac\n" }, { "path": "ci/no_atomic.sh", "content": "#!/bin/bash\nset -euo pipefail\nIFS=$'\\n\\t'\ncd \"$(dirname \"$0\")\"/..\n\n# Update the list of targets that do not support atomic/CAS operations.\n#\n# Usage:\n# ./ci/no_atomic.sh\n\nfile=no_atomic.rs\n\n# `\"max-atomic-width\" == 0` means that atomic is not supported at all.\n# We do not have a cfg for targets with {8,16}-bit atomic only, so\n# for now we treat them the same as targets that do not support atomic.\n# It is not clear exactly what `\"max-atomic-width\" == null` means, but they\n# actually seem to have the same max-atomic-width as the target-pointer-width.\n# The targets currently included in this group are \"mipsel-sony-psp\",\n# \"thumbv4t-none-eabi\", \"thumbv6m-none-eabi\", all of which are\n# `\"target-pointer-width\" == \"32\"`, so assuming them `\"max-atomic-width\" == 32`\n# for now.\nno_atomic=$(RUSTC_BOOTSTRAP=1 rustc +stable -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | select((.value.\"max-atomic-width\" == 0) or (.value.\"min-atomic-width\" and .value.\"min-atomic-width\" != 8)) | \" \\\"\" + .key + \"\\\",\"')\n\ncat >\"${file}\" <>()` smaller.\n- Several minor optimizations.\n- Add more tests.\n\n# Version 0.3.6\n\n- Fix a bug in initialization of unbounded channels.\n\n# Version 0.3.5\n\n- New implementation for unbounded channels.\n- A number of small performance improvements.\n- Remove `crossbeam-epoch` dependency.\n\n# Version 0.3.4\n\n- Bump `crossbeam-epoch` to `0.7`.\n- Improve documentation.\n\n# Version 0.3.3\n\n- Relax the lifetime in `SelectedOperation<'_>`.\n- Add `Select::try_ready()`, `Select::ready()`, and `Select::ready_timeout()`.\n- Update licensing notices.\n- Improve documentation.\n- Add methods `is_disconnected()`, `is_timeout()`, `is_empty()`, and `is_full()` on error types.\n\n# Version 0.3.2\n\n- More elaborate licensing notices.\n\n# Version 0.3.1\n\n- Update `crossbeam-utils` to `0.6`.\n\n# Version 0.3.0\n\n- Add a special `never` channel type.\n- Dropping all receivers now closes the channel.\n- The interface of sending and receiving methods is now very similar to those in v0.1.\n- The syntax for `send` in `select!` is now `send(sender, msg) -> res => body`.\n- The syntax for `recv` in `select!` is now `recv(receiver) -> res => body`.\n- New, more efficient interface for `Select` without callbacks.\n- Timeouts can be specified in `select!`.\n\n# Version 0.2.6\n\n- `Select` struct that can add cases dynamically.\n- More documentation (in particular, the FAQ section).\n- Optimize contended sends/receives in unbounded channels.\n\n# Version 0.2.5\n\n- Use `LocalKey::try_with` instead of `LocalKey::with`.\n- Remove helper macros `__crossbeam_channel*`.\n\n# Version 0.2.4\n\n- Make `select!` linearizable with other channel operations.\n- Update `crossbeam-utils` to `0.5.0`.\n- Update `parking_lot` to `0.6.3`.\n- Remove Mac OS X tests.\n\n# Version 0.2.3\n\n- Add Mac OS X tests.\n- Lower some memory orderings.\n- Eliminate calls to `mem::unitialized`, which caused bugs with ZST.\n\n# Version 0.2.2\n\n- Add more tests.\n- Update `crossbeam-epoch` to 0.5.0\n- Initialize the RNG seed to a random value.\n- Replace `libc::abort` with `std::process::abort`.\n- Ignore clippy warnings in `select!`.\n- Better interaction of `select!` with the NLL borrow checker.\n\n# Version 0.2.1\n\n- Fix compilation errors when using `select!` with `#[deny(unsafe_code)]`.\n\n# Version 0.2.0\n\n- Implement `IntoIterator` for `Receiver`.\n- Add a new `select!` macro.\n- Add special channels `after` and `tick`.\n- Dropping receivers doesn't close the channel anymore.\n- Change the signature of `recv`, `send`, and `try_recv`.\n- Remove `Sender::is_closed` and `Receiver::is_closed`.\n- Remove `Sender::close` and `Receiver::close`.\n- Remove `Sender::send_timeout` and `Receiver::recv_timeout`.\n- Remove `Sender::try_send`.\n- Remove `Select` and `select_loop!`.\n- Remove all error types.\n- Remove `Iter`, `TryIter`, and `IntoIter`.\n- Remove the `nightly` feature.\n- Remove ordering operators for `Sender` and `Receiver`.\n\n# Version 0.1.3\n\n- Add `Sender::disconnect` and `Receiver::disconnect`.\n- Implement comparison operators for `Sender` and `Receiver`.\n- Allow arbitrary patterns in place of `msg` in `recv(r, msg)`.\n- Add a few conversion impls between error types.\n- Add benchmarks for `atomicring` and `mpmc`.\n- Add benchmarks for different message sizes.\n- Documentation improvements.\n- Update `crossbeam-epoch` to 0.4.0\n- Update `crossbeam-utils` to 0.3.0\n- Update `parking_lot` to 0.5\n- Update `rand` to 0.4\n\n# Version 0.1.2\n\n- Allow conditional cases in `select_loop!` macro.\n- Fix typos in documentation.\n- Fix deadlock in selection when all channels are disconnected and a timeout is specified.\n\n# Version 0.1.1\n\n- Implement `Debug` for `Sender`, `Receiver`, `Iter`, `TryIter`, `IntoIter`, and `Select`.\n- Implement `Default` for `Select`.\n\n# Version 0.1.0\n\n- First implementation of the channels.\n- Add `select_loop!` macro by @TimNN.\n" }, { "path": "crossbeam-channel/Cargo.toml", "content": "[package]\nname = \"crossbeam-channel\"\n# When publishing a new version:\n# - Update CHANGELOG.md\n# - Update README.md (when increasing major or minor version)\n# - Run './tools/publish.sh crossbeam-channel '\nversion = \"0.5.15\"\nedition = \"2021\"\n# NB: Sync with msrv badge and \"Compatibility\" section in README.md\nrust-version = \"1.74\"\nlicense = \"MIT OR Apache-2.0\"\nrepository = \"https://github.com/crossbeam-rs/crossbeam\"\nhomepage = \"https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-channel\"\ndescription = \"Multi-producer multi-consumer channels for message passing\"\nkeywords = [\"channel\", \"mpmc\", \"select\", \"golang\", \"message\"]\ncategories = [\"algorithms\", \"concurrency\", \"data-structures\"]\n\n[package.metadata.docs.rs]\ntargets = [\"x86_64-unknown-linux-gnu\"]\n\n[package.metadata.cargo_check_external_types]\n# The following are external types that are allowed to be exposed in our public API.\nallowed_external_types = [\n]\n\n[features]\ndefault = [\"std\"]\n\n# Enable to use APIs that require `std`.\n# This is enabled by default.\n#\n# NOTE: Disabling `std` feature is not supported yet.\nstd = [\"crossbeam-utils/std\"]\n\n[dependencies]\ncrossbeam-utils = { version = \"0.8.18\", path = \"../crossbeam-utils\", default-features = false, features = [\"atomic\"] }\n\n[dev-dependencies]\nfastrand = \"2\"\nrustversion = \"1\"\nsignal-hook = \"0.4\"\n\n[lints]\nworkspace = true\n" }, { "path": "crossbeam-channel/LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "crossbeam-channel/LICENSE-MIT", "content": "The MIT License (MIT)\n\nCopyright (c) 2019 The Crossbeam Project Developers\n\nPermission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "crossbeam-channel/LICENSE-THIRD-PARTY", "content": "===============================================================================\n\nmatching.go\nhttps://creativecommons.org/licenses/by/3.0/legalcode\n\nCreative Commons Legal Code\n\nAttribution 3.0 Unported\n\n CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE\n LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN\n ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS\n INFORMATION ON AN \"AS-IS\" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES\n REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR\n DAMAGES RESULTING FROM ITS USE.\n\nLicense\n\nTHE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE\nCOMMONS PUBLIC LICENSE (\"CCPL\" OR \"LICENSE\"). THE WORK IS PROTECTED BY\nCOPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS\nAUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.\n\nBY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE\nTO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY\nBE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS\nCONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND\nCONDITIONS.\n\n1. Definitions\n\n a. \"Adaptation\" means a work based upon the Work, or upon the Work and\n other pre-existing works, such as a translation, adaptation,\n derivative work, arrangement of music or other alterations of a\n literary or artistic work, or phonogram or performance and includes\n cinematographic adaptations or any other form in which the Work may be\n recast, transformed, or adapted including in any form recognizably\n derived from the original, except that a work that constitutes a\n Collection will not be considered an Adaptation for the purpose of\n this License. For the avoidance of doubt, where the Work is a musical\n work, performance or phonogram, the synchronization of the Work in\n timed-relation with a moving image (\"synching\") will be considered an\n Adaptation for the purpose of this License.\n b. \"Collection\" means a collection of literary or artistic works, such as\n encyclopedias and anthologies, or performances, phonograms or\n broadcasts, or other works or subject matter other than works listed\n in Section 1(f) below, which, by reason of the selection and\n arrangement of their contents, constitute intellectual creations, in\n which the Work is included in its entirety in unmodified form along\n with one or more other contributions, each constituting separate and\n independent works in themselves, which together are assembled into a\n collective whole. A work that constitutes a Collection will not be\n considered an Adaptation (as defined above) for the purposes of this\n License.\n c. \"Distribute\" means to make available to the public the original and\n copies of the Work or Adaptation, as appropriate, through sale or\n other transfer of ownership.\n d. \"Licensor\" means the individual, individuals, entity or entities that\n offer(s) the Work under the terms of this License.\n e. \"Original Author\" means, in the case of a literary or artistic work,\n the individual, individuals, entity or entities who created the Work\n or if no individual or entity can be identified, the publisher; and in\n addition (i) in the case of a performance the actors, singers,\n musicians, dancers, and other persons who act, sing, deliver, declaim,\n play in, interpret or otherwise perform literary or artistic works or\n expressions of folklore; (ii) in the case of a phonogram the producer\n being the person or legal entity who first fixes the sounds of a\n performance or other sounds; and, (iii) in the case of broadcasts, the\n organization that transmits the broadcast.\n f. \"Work\" means the literary and/or artistic work offered under the terms\n of this License including without limitation any production in the\n literary, scientific and artistic domain, whatever may be the mode or\n form of its expression including digital form, such as a book,\n pamphlet and other writing; a lecture, address, sermon or other work\n of the same nature; a dramatic or dramatico-musical work; a\n choreographic work or entertainment in dumb show; a musical\n composition with or without words; a cinematographic work to which are\n assimilated works expressed by a process analogous to cinematography;\n a work of drawing, painting, architecture, sculpture, engraving or\n lithography; a photographic work to which are assimilated works\n expressed by a process analogous to photography; a work of applied\n art; an illustration, map, plan, sketch or three-dimensional work\n relative to geography, topography, architecture or science; a\n performance; a broadcast; a phonogram; a compilation of data to the\n extent it is protected as a copyrightable work; or a work performed by\n a variety or circus performer to the extent it is not otherwise\n considered a literary or artistic work.\n g. \"You\" means an individual or entity exercising rights under this\n License who has not previously violated the terms of this License with\n respect to the Work, or who has received express permission from the\n Licensor to exercise rights under this License despite a previous\n violation.\n h. \"Publicly Perform\" means to perform public recitations of the Work and\n to communicate to the public those public recitations, by any means or\n process, including by wire or wireless means or public digital\n performances; to make available to the public Works in such a way that\n members of the public may access these Works from a place and at a\n place individually chosen by them; to perform the Work to the public\n by any means or process and the communication to the public of the\n performances of the Work, including by public digital performance; to\n broadcast and rebroadcast the Work by any means including signs,\n sounds or images.\n i. \"Reproduce\" means to make copies of the Work by any means including\n without limitation by sound or visual recordings and the right of\n fixation and reproducing fixations of the Work, including storage of a\n protected performance or phonogram in digital form or other electronic\n medium.\n\n2. Fair Dealing Rights. Nothing in this License is intended to reduce,\nlimit, or restrict any uses free from copyright or rights arising from\nlimitations or exceptions that are provided for in connection with the\ncopyright protection under copyright law or other applicable laws.\n\n3. License Grant. Subject to the terms and conditions of this License,\nLicensor hereby grants You a worldwide, royalty-free, non-exclusive,\nperpetual (for the duration of the applicable copyright) license to\nexercise the rights in the Work as stated below:\n\n a. to Reproduce the Work, to incorporate the Work into one or more\n Collections, and to Reproduce the Work as incorporated in the\n Collections;\n b. to create and Reproduce Adaptations provided that any such Adaptation,\n including any translation in any medium, takes reasonable steps to\n clearly label, demarcate or otherwise identify that changes were made\n to the original Work. For example, a translation could be marked \"The\n original work was translated from English to Spanish,\" or a\n modification could indicate \"The original work has been modified.\";\n c. to Distribute and Publicly Perform the Work including as incorporated\n in Collections; and,\n d. to Distribute and Publicly Perform Adaptations.\n e. For the avoidance of doubt:\n\n i. Non-waivable Compulsory License Schemes. In those jurisdictions in\n which the right to collect royalties through any statutory or\n compulsory licensing scheme cannot be waived, the Licensor\n reserves the exclusive right to collect such royalties for any\n exercise by You of the rights granted under this License;\n ii. Waivable Compulsory License Schemes. In those jurisdictions in\n which the right to collect royalties through any statutory or\n compulsory licensing scheme can be waived, the Licensor waives the\n exclusive right to collect such royalties for any exercise by You\n of the rights granted under this License; and,\n iii. Voluntary License Schemes. The Licensor waives the right to\n collect royalties, whether individually or, in the event that the\n Licensor is a member of a collecting society that administers\n voluntary licensing schemes, via that society, from any exercise\n by You of the rights granted under this License.\n\nThe above rights may be exercised in all media and formats whether now\nknown or hereafter devised. The above rights include the right to make\nsuch modifications as are technically necessary to exercise the rights in\nother media and formats. Subject to Section 8(f), all rights not expressly\ngranted by Licensor are hereby reserved.\n\n4. Restrictions. The license granted in Section 3 above is expressly made\nsubject to and limited by the following restrictions:\n\n a. You may Distribute or Publicly Perform the Work only under the terms\n of this License. You must include a copy of, or the Uniform Resource\n Identifier (URI) for, this License with every copy of the Work You\n Distribute or Publicly Perform. You may not offer or impose any terms\n on the Work that restrict the terms of this License or the ability of\n the recipient of the Work to exercise the rights granted to that\n recipient under the terms of the License. You may not sublicense the\n Work. You must keep intact all notices that refer to this License and\n to the disclaimer of warranties with every copy of the Work You\n Distribute or Publicly Perform. When You Distribute or Publicly\n Perform the Work, You may not impose any effective technological\n measures on the Work that restrict the ability of a recipient of the\n Work from You to exercise the rights granted to that recipient under\n the terms of the License. This Section 4(a) applies to the Work as\n incorporated in a Collection, but this does not require the Collection\n apart from the Work itself to be made subject to the terms of this\n License. If You create a Collection, upon notice from any Licensor You\n must, to the extent practicable, remove from the Collection any credit\n as required by Section 4(b), as requested. If You create an\n Adaptation, upon notice from any Licensor You must, to the extent\n practicable, remove from the Adaptation any credit as required by\n Section 4(b), as requested.\n b. If You Distribute, or Publicly Perform the Work or any Adaptations or\n Collections, You must, unless a request has been made pursuant to\n Section 4(a), keep intact all copyright notices for the Work and\n provide, reasonable to the medium or means You are utilizing: (i) the\n name of the Original Author (or pseudonym, if applicable) if supplied,\n and/or if the Original Author and/or Licensor designate another party\n or parties (e.g., a sponsor institute, publishing entity, journal) for\n attribution (\"Attribution Parties\") in Licensor's copyright notice,\n terms of service or by other reasonable means, the name of such party\n or parties; (ii) the title of the Work if supplied; (iii) to the\n extent reasonably practicable, the URI, if any, that Licensor\n specifies to be associated with the Work, unless such URI does not\n refer to the copyright notice or licensing information for the Work;\n and (iv) , consistent with Section 3(b), in the case of an Adaptation,\n a credit identifying the use of the Work in the Adaptation (e.g.,\n \"French translation of the Work by Original Author,\" or \"Screenplay\n based on original Work by Original Author\"). The credit required by\n this Section 4 (b) may be implemented in any reasonable manner;\n provided, however, that in the case of a Adaptation or Collection, at\n a minimum such credit will appear, if a credit for all contributing\n authors of the Adaptation or Collection appears, then as part of these\n credits and in a manner at least as prominent as the credits for the\n other contributing authors. For the avoidance of doubt, You may only\n use the credit required by this Section for the purpose of attribution\n in the manner set out above and, by exercising Your rights under this\n License, You may not implicitly or explicitly assert or imply any\n connection with, sponsorship or endorsement by the Original Author,\n Licensor and/or Attribution Parties, as appropriate, of You or Your\n use of the Work, without the separate, express prior written\n permission of the Original Author, Licensor and/or Attribution\n Parties.\n c. Except as otherwise agreed in writing by the Licensor or as may be\n otherwise permitted by applicable law, if You Reproduce, Distribute or\n Publicly Perform the Work either by itself or as part of any\n Adaptations or Collections, You must not distort, mutilate, modify or\n take other derogatory action in relation to the Work which would be\n prejudicial to the Original Author's honor or reputation. Licensor\n agrees that in those jurisdictions (e.g. Japan), in which any exercise\n of the right granted in Section 3(b) of this License (the right to\n make Adaptations) would be deemed to be a distortion, mutilation,\n modification or other derogatory action prejudicial to the Original\n Author's honor and reputation, the Licensor will waive or not assert,\n as appropriate, this Section, to the fullest extent permitted by the\n applicable national law, to enable You to reasonably exercise Your\n right under Section 3(b) of this License (right to make Adaptations)\n but not otherwise.\n\n5. Representations, Warranties and Disclaimer\n\nUNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR\nOFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY\nKIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE,\nINCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY,\nFITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF\nLATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS,\nWHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION\nOF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.\n\n6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE\nLAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR\nANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES\nARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS\nBEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.\n\n7. Termination\n\n a. This License and the rights granted hereunder will terminate\n automatically upon any breach by You of the terms of this License.\n Individuals or entities who have received Adaptations or Collections\n from You under this License, however, will not have their licenses\n terminated provided such individuals or entities remain in full\n compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will\n survive any termination of this License.\n b. Subject to the above terms and conditions, the license granted here is\n perpetual (for the duration of the applicable copyright in the Work).\n Notwithstanding the above, Licensor reserves the right to release the\n Work under different license terms or to stop distributing the Work at\n any time; provided, however that any such election will not serve to\n withdraw this License (or any other license that has been, or is\n required to be, granted under the terms of this License), and this\n License will continue in full force and effect unless terminated as\n stated above.\n\n8. Miscellaneous\n\n a. Each time You Distribute or Publicly Perform the Work or a Collection,\n the Licensor offers to the recipient a license to the Work on the same\n terms and conditions as the license granted to You under this License.\n b. Each time You Distribute or Publicly Perform an Adaptation, Licensor\n offers to the recipient a license to the original Work on the same\n terms and conditions as the license granted to You under this License.\n c. If any provision of this License is invalid or unenforceable under\n applicable law, it shall not affect the validity or enforceability of\n the remainder of the terms of this License, and without further action\n by the parties to this agreement, such provision shall be reformed to\n the minimum extent necessary to make such provision valid and\n enforceable.\n d. No term or provision of this License shall be deemed waived and no\n breach consented to unless such waiver or consent shall be in writing\n and signed by the party to be charged with such waiver or consent.\n e. This License constitutes the entire agreement between the parties with\n respect to the Work licensed here. There are no understandings,\n agreements or representations with respect to the Work not specified\n here. Licensor shall not be bound by any additional provisions that\n may appear in any communication from You. This License may not be\n modified without the mutual written agreement of the Licensor and You.\n f. The rights granted under, and the subject matter referenced, in this\n License were drafted utilizing the terminology of the Berne Convention\n for the Protection of Literary and Artistic Works (as amended on\n September 28, 1979), the Rome Convention of 1961, the WIPO Copyright\n Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996\n and the Universal Copyright Convention (as revised on July 24, 1971).\n These rights and subject matter take effect in the relevant\n jurisdiction in which the License terms are sought to be enforced\n according to the corresponding provisions of the implementation of\n those treaty provisions in the applicable national law. If the\n standard suite of rights granted under applicable copyright law\n includes additional rights not granted under this License, such\n additional rights are deemed to be included in the License; this\n License is not intended to restrict the license of any rights under\n applicable law.\n\n\nCreative Commons Notice\n\n Creative Commons is not a party to this License, and makes no warranty\n whatsoever in connection with the Work. Creative Commons will not be\n liable to You or any party on any legal theory for any damages\n whatsoever, including without limitation any general, special,\n incidental or consequential damages arising in connection to this\n license. Notwithstanding the foregoing two (2) sentences, if Creative\n Commons has expressly identified itself as the Licensor hereunder, it\n shall have all rights and obligations of Licensor.\n\n Except for the limited purpose of indicating to the public that the\n Work is licensed under the CCPL, Creative Commons does not authorize\n the use by either party of the trademark \"Creative Commons\" or any\n related trademark or logo of Creative Commons without the prior\n written consent of Creative Commons. Any permitted use will be in\n compliance with Creative Commons' then-current trademark usage\n guidelines, as may be published on its website or otherwise made\n available upon request from time to time. For the avoidance of doubt,\n this trademark restriction does not form part of this License.\n\n Creative Commons may be contacted at https://creativecommons.org/.\n\n===============================================================================\n\nThe Go Programming Language\nhttps://golang.org/LICENSE\n\nCopyright (c) 2009 The Go Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n===============================================================================\n\nThe Rust Programming Language\nhttps://github.com/rust-lang/rust/blob/master/LICENSE-MIT\n\nPermission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n\n===============================================================================\n\nThe Rust Programming Language\nhttps://github.com/rust-lang/rust/blob/master/LICENSE-APACHE\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "crossbeam-channel/README.md", "content": "# Crossbeam Channel\n\n[![Build Status](https://github.com/crossbeam-rs/crossbeam/workflows/CI/badge.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/actions)\n[![License](https://img.shields.io/badge/license-MIT_OR_Apache--2.0-blue.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-channel#license)\n[![Cargo](https://img.shields.io/crates/v/crossbeam-channel.svg)](\nhttps://crates.io/crates/crossbeam-channel)\n[![Documentation](https://docs.rs/crossbeam-channel/badge.svg)](\nhttps://docs.rs/crossbeam-channel)\n[![Rust 1.74+](https://img.shields.io/badge/rust-1.74+-lightgray.svg)](\nhttps://www.rust-lang.org)\n[![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ)\n\nThis crate provides multi-producer multi-consumer channels for message passing.\nIt is an alternative to [`std::sync::mpsc`] with more features and better performance.\n\nSome highlights:\n\n* [`Sender`]s and [`Receiver`]s can be cloned and shared among threads.\n* Two main kinds of channels are [`bounded`] and [`unbounded`].\n* Convenient extra channels like [`after`], [`never`], and [`tick`].\n* The [`select!`] macro can block on multiple channel operations.\n* [`Select`] can select over a dynamically built list of channel operations.\n* Channels use locks very sparingly for maximum [performance](benchmarks).\n\n[`std::sync::mpsc`]: https://doc.rust-lang.org/std/sync/mpsc/index.html\n[`Sender`]: https://docs.rs/crossbeam-channel/latest/crossbeam_channel/struct.Sender.html\n[`Receiver`]: https://docs.rs/crossbeam-channel/latest/crossbeam_channel/struct.Receiver.html\n[`bounded`]: https://docs.rs/crossbeam-channel/latest/crossbeam_channel/fn.bounded.html\n[`unbounded`]: https://docs.rs/crossbeam-channel/latest/crossbeam_channel/fn.unbounded.html\n[`after`]: https://docs.rs/crossbeam-channel/latest/crossbeam_channel/fn.after.html\n[`never`]: https://docs.rs/crossbeam-channel/latest/crossbeam_channel/fn.never.html\n[`tick`]: https://docs.rs/crossbeam-channel/latest/crossbeam_channel/fn.tick.html\n[`select!`]: https://docs.rs/crossbeam-channel/latest/crossbeam_channel/macro.select.html\n[`Select`]: https://docs.rs/crossbeam-channel/latest/crossbeam_channel/struct.Select.html\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\ncrossbeam-channel = \"0.5\"\n```\n\n## Compatibility\n\nCrossbeam Channel supports stable Rust releases going back at least six months,\nand every time the minimum supported Rust version is increased, a new minor\nversion is released. Currently, the minimum supported Rust version is 1.74.\n\n## License\n\nLicensed under either of\n\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n#### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n\n#### Third party software\n\nThis product includes copies and modifications of software developed by third parties:\n\n* [examples/matching.rs](examples/matching.rs) includes\n [matching.go](http://www.nada.kth.se/~snilsson/concurrency/src/matching.go) by Stefan Nilsson,\n licensed under Creative Commons Attribution 3.0 Unported License.\n\n* [tests/mpsc.rs](tests/mpsc.rs) includes modifications of code from The Rust Programming Language,\n licensed under the MIT License and the Apache License, Version 2.0.\n\n* [tests/golang.rs](tests/golang.rs) is based on code from The Go Programming Language, licensed\n under the 3-Clause BSD License.\n\nSee the source code files for more details.\n\nCopies of third party licenses can be found in [LICENSE-THIRD-PARTY](LICENSE-THIRD-PARTY).\n" }, { "path": "crossbeam-channel/benches/crossbeam.rs", "content": "#![feature(test)]\n\nextern crate test;\n\nuse std::num::NonZeroUsize;\n\nuse crossbeam_channel::{bounded, unbounded};\nuse crossbeam_utils::thread::scope;\nuse test::Bencher;\n\nconst TOTAL_STEPS: usize = 40_000;\n\nfn num_cpus() -> usize {\n std::thread::available_parallelism()\n .unwrap_or(NonZeroUsize::MIN)\n .into()\n}\n\nmod unbounded {\n use super::*;\n\n #[bench]\n fn create(b: &mut Bencher) {\n b.iter(unbounded::);\n }\n\n #[bench]\n fn oneshot(b: &mut Bencher) {\n b.iter(|| {\n let (s, r) = unbounded::();\n s.send(0).unwrap();\n r.recv().unwrap();\n });\n }\n\n #[bench]\n fn inout(b: &mut Bencher) {\n let (s, r) = unbounded::();\n b.iter(|| {\n s.send(0).unwrap();\n r.recv().unwrap();\n });\n }\n\n #[bench]\n fn par_inout(b: &mut Bencher) {\n let threads = num_cpus();\n let steps = TOTAL_STEPS / threads;\n let (s, r) = unbounded::();\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n r.recv().unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn spsc(b: &mut Bencher) {\n let steps = TOTAL_STEPS;\n let (s, r) = unbounded::();\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n\n b.iter(|| {\n s1.send(()).unwrap();\n for _ in 0..steps {\n r.recv().unwrap();\n }\n r2.recv().unwrap();\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn spmc(b: &mut Bencher) {\n let threads = num_cpus() - 1;\n let steps = TOTAL_STEPS / threads;\n let (s, r) = unbounded::();\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for i in 0..steps * threads {\n s.send(i as i32).unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn mpsc(b: &mut Bencher) {\n let threads = num_cpus() - 1;\n let steps = TOTAL_STEPS / threads;\n let (s, r) = unbounded::();\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for _ in 0..steps * threads {\n r.recv().unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn mpmc(b: &mut Bencher) {\n let threads = num_cpus();\n let steps = TOTAL_STEPS / threads;\n let (s, r) = unbounded::();\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads / 2 {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n for _ in 0..threads / 2 {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n}\n\nmod bounded_n {\n use super::*;\n\n #[bench]\n fn spsc(b: &mut Bencher) {\n let steps = TOTAL_STEPS;\n let (s, r) = bounded::(steps);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n\n b.iter(|| {\n s1.send(()).unwrap();\n for _ in 0..steps {\n r.recv().unwrap();\n }\n r2.recv().unwrap();\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn spmc(b: &mut Bencher) {\n let threads = num_cpus() - 1;\n let steps = TOTAL_STEPS / threads;\n let (s, r) = bounded::(steps * threads);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for i in 0..steps * threads {\n s.send(i as i32).unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn mpsc(b: &mut Bencher) {\n let threads = num_cpus() - 1;\n let steps = TOTAL_STEPS / threads;\n let (s, r) = bounded::(steps * threads);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for _ in 0..steps * threads {\n r.recv().unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn par_inout(b: &mut Bencher) {\n let threads = num_cpus();\n let steps = TOTAL_STEPS / threads;\n let (s, r) = bounded::(threads);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n r.recv().unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn mpmc(b: &mut Bencher) {\n let threads = num_cpus();\n assert_eq!(threads % 2, 0);\n let steps = TOTAL_STEPS / threads;\n let (s, r) = bounded::(steps * threads);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads / 2 {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n for _ in 0..threads / 2 {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n}\n\nmod bounded_1 {\n use super::*;\n\n #[bench]\n fn create(b: &mut Bencher) {\n b.iter(|| bounded::(1));\n }\n\n #[bench]\n fn oneshot(b: &mut Bencher) {\n b.iter(|| {\n let (s, r) = bounded::(1);\n s.send(0).unwrap();\n r.recv().unwrap();\n });\n }\n\n #[bench]\n fn spsc(b: &mut Bencher) {\n let steps = TOTAL_STEPS;\n let (s, r) = bounded::(1);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n\n b.iter(|| {\n s1.send(()).unwrap();\n for _ in 0..steps {\n r.recv().unwrap();\n }\n r2.recv().unwrap();\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn spmc(b: &mut Bencher) {\n let threads = num_cpus() - 1;\n let steps = TOTAL_STEPS / threads;\n let (s, r) = bounded::(1);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for i in 0..steps * threads {\n s.send(i as i32).unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn mpsc(b: &mut Bencher) {\n let threads = num_cpus() - 1;\n let steps = TOTAL_STEPS / threads;\n let (s, r) = bounded::(1);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for _ in 0..steps * threads {\n r.recv().unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn mpmc(b: &mut Bencher) {\n let threads = num_cpus();\n let steps = TOTAL_STEPS / threads;\n let (s, r) = bounded::(1);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads / 2 {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n for _ in 0..threads / 2 {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n}\n\nmod bounded_0 {\n use super::*;\n\n #[bench]\n fn create(b: &mut Bencher) {\n b.iter(|| bounded::(0));\n }\n\n #[bench]\n fn spsc(b: &mut Bencher) {\n let steps = TOTAL_STEPS;\n let (s, r) = bounded::(0);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n\n b.iter(|| {\n s1.send(()).unwrap();\n for _ in 0..steps {\n r.recv().unwrap();\n }\n r2.recv().unwrap();\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn spmc(b: &mut Bencher) {\n let threads = num_cpus() - 1;\n let steps = TOTAL_STEPS / threads;\n let (s, r) = bounded::(0);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for i in 0..steps * threads {\n s.send(i as i32).unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn mpsc(b: &mut Bencher) {\n let threads = num_cpus() - 1;\n let steps = TOTAL_STEPS / threads;\n let (s, r) = bounded::(0);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for _ in 0..steps * threads {\n r.recv().unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n\n #[bench]\n fn mpmc(b: &mut Bencher) {\n let threads = num_cpus();\n let steps = TOTAL_STEPS / threads;\n let (s, r) = bounded::(0);\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n scope(|scope| {\n for _ in 0..threads / 2 {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for i in 0..steps {\n s.send(i as i32).unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n for _ in 0..threads / 2 {\n scope.spawn(|_| {\n while r1.recv().is_ok() {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n s2.send(()).unwrap();\n }\n });\n }\n\n b.iter(|| {\n for _ in 0..threads {\n s1.send(()).unwrap();\n }\n for _ in 0..threads {\n r2.recv().unwrap();\n }\n });\n drop(s1);\n })\n .unwrap();\n }\n}\n" }, { "path": "crossbeam-channel/benchmarks/Cargo.toml", "content": "[package]\nname = \"benchmarks\"\nversion = \"0.0.0\"\nedition = \"2021\"\npublish = false\n\n[dependencies]\natomicring = \"1.2.9\"\nbus = \"2.3.0\"\ncrossbeam = { path = \"../..\" }\ncrossbeam-channel = { path = \"..\" }\ncrossbeam-deque = { path = \"../../crossbeam-deque\" }\nflume = \"0.12\"\nfutures = { version = \"0.3\", features = [\"thread-pool\"] }\nlockfree = \"0.5.1\"\nmpmc = \"0.1.6\"\n\n[[bin]]\nname = \"atomicring\"\npath = \"atomicring.rs\"\ndoc = false\n\n[[bin]]\nname = \"atomicringqueue\"\npath = \"atomicringqueue.rs\"\ndoc = false\n\n[[bin]]\nname = \"bus\"\npath = \"bus.rs\"\ndoc = false\n\n[[bin]]\nname = \"crossbeam-channel\"\npath = \"crossbeam-channel.rs\"\ndoc = false\n\n[[bin]]\nname = \"crossbeam-deque\"\npath = \"crossbeam-deque.rs\"\ndoc = false\n\n[[bin]]\nname = \"flume\"\npath = \"flume.rs\"\ndoc = false\n\n[[bin]]\nname = \"futures-channel\"\npath = \"futures-channel.rs\"\ndoc = false\n\n[[bin]]\nname = \"lockfree\"\npath = \"lockfree.rs\"\ndoc = false\n\n[[bin]]\nname = \"mpsc\"\npath = \"mpsc.rs\"\ndoc = false\n\n[[bin]]\nname = \"segqueue\"\npath = \"segqueue.rs\"\ndoc = false\n\n[[bin]]\nname = \"mpmc\"\npath = \"mpmc.rs\"\ndoc = false\n\n[lints]\nworkspace = true\n" }, { "path": "crossbeam-channel/benchmarks/README.md", "content": "# Benchmarks\n\n### Tests\n\n* `seq`: A single thread sends `N` messages. Then it receives `N` messages.\n* `spsc`: One thread sends `N` messages. Another thread receives `N` messages.\n* `mpsc`: `T` threads send `N / T` messages each. One thread receives `N` messages.\n* `mpmc`: `T` threads send `N / T` messages each. `T` other threads receive `N / T` messages each.\n* `select_rx`: `T` threads send `N / T` messages each into a separate channel. Another thread receives `N` messages by selecting over the `T` channels.\n* `select_both`: `T` threads send `N / T` messages each by selecting over `T` channels. `T` other threads receive `N / T` messages each by selecting over the `T` channels.\n\nDefault configuration:\n\n- `N = 5000000`\n- `T = 4`\n\n### Running\n\nRuns benchmarks, stores results into `*.txt` files, and generates `plot.png`:\n\n```\n./run.sh\n```\n\nDependencies:\n\n- Rust \n- Go\n- Bash\n- Python \n- Matplotlib\n\n### Results\n\nMachine: Intel(R) Core(TM) i7-5500U (2 physical cores, 4 logical cores)\n\nRust: `rustc 1.63.0 (4b91a6ea7 2022-08-08)`\n\nGo: `go version go1.19 linux/amd64`\n\nCommit: `7070018` (2022-08-24)\n\n![Benchmark results](plot.png)\n" }, { "path": "crossbeam-channel/benchmarks/atomicring.rs", "content": "use std::thread;\n\nuse atomicring::AtomicRingBuffer;\n\nmod message;\n\nconst MESSAGES: usize = 5_000_000;\nconst THREADS: usize = 4;\n\nfn seq(cap: usize) {\n let q = AtomicRingBuffer::with_capacity(cap);\n\n for i in 0..MESSAGES {\n loop {\n if q.try_push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n\n for _ in 0..MESSAGES {\n q.try_pop().unwrap();\n }\n}\n\nfn spsc(cap: usize) {\n let q = AtomicRingBuffer::with_capacity(cap);\n\n crossbeam::scope(|scope| {\n scope.spawn(|_| {\n for i in 0..MESSAGES {\n loop {\n if q.try_push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n });\n\n for _ in 0..MESSAGES {\n loop {\n if q.try_pop().is_none() {\n thread::yield_now();\n } else {\n break;\n }\n }\n }\n })\n .unwrap();\n}\n\nfn mpsc(cap: usize) {\n let q = AtomicRingBuffer::with_capacity(cap);\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n loop {\n if q.try_push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n });\n }\n\n for _ in 0..MESSAGES {\n loop {\n if q.try_pop().is_none() {\n thread::yield_now();\n } else {\n break;\n }\n }\n }\n })\n .unwrap();\n}\n\nfn mpmc(cap: usize) {\n let q = AtomicRingBuffer::with_capacity(cap);\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n loop {\n if q.try_push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n });\n }\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..MESSAGES / THREADS {\n loop {\n if q.try_pop().is_none() {\n thread::yield_now();\n } else {\n break;\n }\n }\n }\n });\n }\n })\n .unwrap();\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust atomicring\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n run!(\"bounded_mpmc\", mpmc(MESSAGES));\n run!(\"bounded_mpsc\", mpsc(MESSAGES));\n run!(\"bounded_seq\", seq(MESSAGES));\n run!(\"bounded_spsc\", spsc(MESSAGES));\n}\n" }, { "path": "crossbeam-channel/benchmarks/atomicringqueue.rs", "content": "use std::thread;\n\nuse atomicring::AtomicRingQueue;\n\nmod message;\n\nconst MESSAGES: usize = 5_000_000;\nconst THREADS: usize = 4;\n\nfn seq(cap: usize) {\n let q = AtomicRingQueue::with_capacity(cap);\n\n for i in 0..MESSAGES {\n loop {\n if q.try_push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n\n for _ in 0..MESSAGES {\n q.pop();\n }\n}\n\nfn spsc(cap: usize) {\n let q = AtomicRingQueue::with_capacity(cap);\n\n crossbeam::scope(|scope| {\n scope.spawn(|_| {\n for i in 0..MESSAGES {\n loop {\n if q.try_push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n });\n\n for _ in 0..MESSAGES {\n q.pop();\n }\n })\n .unwrap();\n}\n\nfn mpsc(cap: usize) {\n let q = AtomicRingQueue::with_capacity(cap);\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n loop {\n if q.try_push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n });\n }\n\n for _ in 0..MESSAGES {\n q.pop();\n }\n })\n .unwrap();\n}\n\nfn mpmc(cap: usize) {\n let q = AtomicRingQueue::with_capacity(cap);\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n loop {\n if q.try_push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n });\n }\n\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..MESSAGES / THREADS {\n q.pop();\n }\n });\n }\n })\n .unwrap();\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust atomicringqueue\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n run!(\"bounded_mpmc\", mpmc(MESSAGES));\n run!(\"bounded_mpsc\", mpsc(MESSAGES));\n run!(\"bounded_seq\", seq(MESSAGES));\n run!(\"bounded_spsc\", spsc(MESSAGES));\n}\n" }, { "path": "crossbeam-channel/benchmarks/bus.rs", "content": "use bus::Bus;\n\nmod message;\n\nconst MESSAGES: usize = 5_000_000;\n\nfn seq(cap: usize) {\n let mut tx = Bus::new(cap);\n let mut rx = tx.add_rx();\n\n for i in 0..MESSAGES {\n tx.broadcast(message::new(i));\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n}\n\nfn spsc(cap: usize) {\n let mut tx = Bus::new(cap);\n let mut rx = tx.add_rx();\n\n crossbeam::scope(|scope| {\n scope.spawn(|_| {\n for i in 0..MESSAGES {\n tx.broadcast(message::new(i));\n }\n });\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust bus\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n run!(\"bounded1_spsc\", spsc(1));\n\n run!(\"bounded_seq\", seq(MESSAGES));\n run!(\"bounded_spsc\", spsc(MESSAGES));\n}\n" }, { "path": "crossbeam-channel/benchmarks/crossbeam-channel.rs", "content": "use crossbeam_channel::{Receiver, Select, Sender, bounded, unbounded};\n\nmod message;\n\nconst MESSAGES: usize = 5_000_000;\nconst THREADS: usize = 4;\n\nfn new(cap: Option) -> (Sender, Receiver) {\n match cap {\n None => unbounded(),\n Some(cap) => bounded(cap),\n }\n}\n\nfn seq(cap: Option) {\n let (tx, rx) = new(cap);\n\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n}\n\nfn spsc(cap: Option) {\n let (tx, rx) = new(cap);\n\n crossbeam::scope(|scope| {\n scope.spawn(|_| {\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n });\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn mpsc(cap: Option) {\n let (tx, rx) = new(cap);\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n tx.send(message::new(i)).unwrap();\n }\n });\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn mpmc(cap: Option) {\n let (tx, rx) = new(cap);\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n tx.send(message::new(i)).unwrap();\n }\n });\n }\n\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..MESSAGES / THREADS {\n rx.recv().unwrap();\n }\n });\n }\n })\n .unwrap();\n}\n\nfn select_rx(cap: Option) {\n let chans = (0..THREADS).map(|_| new(cap)).collect::>();\n\n crossbeam::scope(|scope| {\n for (tx, _) in &chans {\n let tx = tx.clone();\n scope.spawn(move |_| {\n for i in 0..MESSAGES / THREADS {\n tx.send(message::new(i)).unwrap();\n }\n });\n }\n\n for _ in 0..MESSAGES {\n let mut sel = Select::new();\n for (_, rx) in &chans {\n sel.recv(rx);\n }\n let case = sel.select();\n let index = case.index();\n case.recv(&chans[index].1).unwrap();\n }\n })\n .unwrap();\n}\n\nfn select_both(cap: Option) {\n let chans = (0..THREADS).map(|_| new(cap)).collect::>();\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n let mut sel = Select::new();\n for (tx, _) in &chans {\n sel.send(tx);\n }\n let case = sel.select();\n let index = case.index();\n case.send(&chans[index].0, message::new(i)).unwrap();\n }\n });\n }\n\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..MESSAGES / THREADS {\n let mut sel = Select::new();\n for (_, rx) in &chans {\n sel.recv(rx);\n }\n let case = sel.select();\n let index = case.index();\n case.recv(&chans[index].1).unwrap();\n }\n });\n }\n })\n .unwrap();\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust crossbeam-channel\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n run!(\"bounded0_mpmc\", mpmc(Some(0)));\n run!(\"bounded0_mpsc\", mpsc(Some(0)));\n run!(\"bounded0_select_both\", select_both(Some(0)));\n run!(\"bounded0_select_rx\", select_rx(Some(0)));\n run!(\"bounded0_spsc\", spsc(Some(0)));\n\n run!(\"bounded1_mpmc\", mpmc(Some(1)));\n run!(\"bounded1_mpsc\", mpsc(Some(1)));\n run!(\"bounded1_select_both\", select_both(Some(1)));\n run!(\"bounded1_select_rx\", select_rx(Some(1)));\n run!(\"bounded1_spsc\", spsc(Some(1)));\n\n run!(\"bounded_mpmc\", mpmc(Some(MESSAGES)));\n run!(\"bounded_mpsc\", mpsc(Some(MESSAGES)));\n run!(\"bounded_select_both\", select_both(Some(MESSAGES)));\n run!(\"bounded_select_rx\", select_rx(Some(MESSAGES)));\n run!(\"bounded_seq\", seq(Some(MESSAGES)));\n run!(\"bounded_spsc\", spsc(Some(MESSAGES)));\n\n run!(\"unbounded_mpmc\", mpmc(None));\n run!(\"unbounded_mpsc\", mpsc(None));\n run!(\"unbounded_select_both\", select_both(None));\n run!(\"unbounded_select_rx\", select_rx(None));\n run!(\"unbounded_seq\", seq(None));\n run!(\"unbounded_spsc\", spsc(None));\n}\n" }, { "path": "crossbeam-channel/benchmarks/crossbeam-deque.rs", "content": "use std::thread;\n\nuse crossbeam_deque::{Steal, Worker};\n\nmod message;\n\nconst MESSAGES: usize = 5_000_000;\n\nfn seq() {\n let tx = Worker::new_lifo();\n let rx = tx.stealer();\n\n for i in 0..MESSAGES {\n tx.push(message::new(i));\n }\n\n for _ in 0..MESSAGES {\n match rx.steal() {\n Steal::Success(_) => {}\n Steal::Retry => panic!(),\n Steal::Empty => panic!(),\n }\n }\n}\n\nfn spsc() {\n let tx = Worker::new_lifo();\n let rx = tx.stealer();\n\n crossbeam::scope(|scope| {\n scope.spawn(move |_| {\n for i in 0..MESSAGES {\n tx.push(message::new(i));\n }\n });\n\n scope.spawn(move |_| {\n for _ in 0..MESSAGES {\n loop {\n match rx.steal() {\n Steal::Success(_) => break,\n Steal::Retry | Steal::Empty => thread::yield_now(),\n }\n }\n }\n });\n })\n .unwrap();\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust crossbeam-deque\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n run!(\"unbounded_seq\", seq());\n run!(\"unbounded_spsc\", spsc());\n}\n" }, { "path": "crossbeam-channel/benchmarks/flume.rs", "content": "mod message;\n\nconst MESSAGES: usize = 5_000_000;\nconst THREADS: usize = 4;\n\npub fn shuffle(v: &mut [T]) {\n use std::{cell::Cell, num::Wrapping};\n\n let len = v.len();\n if len <= 1 {\n return;\n }\n\n thread_local! {\n static RNG: Cell> = const { Cell::new(Wrapping(1)) };\n }\n\n RNG.with(|rng| {\n for i in 1..len {\n // This is the 32-bit variant of Xorshift.\n // https://en.wikipedia.org/wiki/Xorshift\n let mut x = rng.get();\n x ^= x << 13;\n x ^= x >> 17;\n x ^= x << 5;\n rng.set(x);\n\n let x = x.0;\n let n = i + 1;\n\n // This is a fast alternative to `let j = x % n`.\n // https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/\n let j = ((x as u64).wrapping_mul(n as u64) >> 32) as u32 as usize;\n\n v.swap(i, j);\n }\n });\n}\n\nfn seq_unbounded() {\n let (tx, rx) = flume::unbounded();\n\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n}\n\nfn seq_bounded(cap: usize) {\n let (tx, rx) = flume::bounded(cap);\n\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n}\n\nfn spsc_unbounded() {\n let (tx, rx) = flume::unbounded();\n\n crossbeam::scope(|scope| {\n scope.spawn(move |_| {\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n });\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn spsc_bounded(cap: usize) {\n let (tx, rx) = flume::bounded(cap);\n\n crossbeam::scope(|scope| {\n scope.spawn(move |_| {\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n });\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn mpsc_unbounded() {\n let (tx, rx) = flume::unbounded();\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n let tx = tx.clone();\n scope.spawn(move |_| {\n for i in 0..MESSAGES / THREADS {\n tx.send(message::new(i)).unwrap();\n }\n });\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn mpsc_bounded(cap: usize) {\n let (tx, rx) = flume::bounded(cap);\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n let tx = tx.clone();\n scope.spawn(move |_| {\n for i in 0..MESSAGES / THREADS {\n tx.send(message::new(i)).unwrap();\n }\n });\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust flume\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n // run!(\"bounded0_mpsc\", mpsc_bounded(0));\n // run!(\"bounded0_spsc\", spsc_bounded(0));\n\n run!(\"bounded1_mpsc\", mpsc_bounded(1));\n run!(\"bounded1_spsc\", spsc_bounded(1));\n\n run!(\"bounded_mpsc\", mpsc_bounded(MESSAGES));\n run!(\"bounded_seq\", seq_bounded(MESSAGES));\n run!(\"bounded_spsc\", spsc_bounded(MESSAGES));\n\n run!(\"unbounded_mpsc\", mpsc_unbounded());\n run!(\"unbounded_seq\", seq_unbounded());\n run!(\"unbounded_spsc\", spsc_unbounded());\n}\n" }, { "path": "crossbeam-channel/benchmarks/futures-channel.rs", "content": "use futures::{\n SinkExt, StreamExt,\n channel::mpsc,\n executor::{ThreadPool, block_on},\n future, stream,\n};\n\nmod message;\n\nconst MESSAGES: usize = 5_000_000;\nconst THREADS: usize = 4;\n\nfn seq_unbounded() {\n block_on(async {\n let (tx, rx) = mpsc::unbounded();\n for i in 0..MESSAGES {\n tx.unbounded_send(message::new(i)).unwrap();\n }\n drop(tx);\n\n rx.for_each(|_| future::ready(())).await\n });\n}\n\nfn seq_bounded(cap: usize) {\n let (mut tx, rx) = mpsc::channel(cap);\n block_on(async {\n for i in 0..MESSAGES {\n tx.try_send(message::new(i)).unwrap();\n }\n drop(tx);\n\n rx.for_each(|_| future::ready(())).await\n });\n}\n\nfn spsc_unbounded() {\n let pool = ThreadPool::new().unwrap();\n block_on(async {\n let (mut tx, rx) = mpsc::unbounded();\n\n pool.spawn_ok(async move {\n tx.send_all(&mut stream::iter((0..MESSAGES).map(message::new).map(Ok)))\n .await\n .unwrap()\n });\n\n rx.for_each(|_| future::ready(())).await\n });\n}\n\nfn spsc_bounded(cap: usize) {\n let pool = ThreadPool::new().unwrap();\n block_on(async {\n let (mut tx, rx) = mpsc::channel(cap);\n\n pool.spawn_ok(async move {\n tx.send_all(&mut stream::iter((0..MESSAGES).map(message::new).map(Ok)))\n .await\n .unwrap()\n });\n\n rx.for_each(|_| future::ready(())).await\n });\n}\n\nfn mpsc_unbounded() {\n let pool = ThreadPool::new().unwrap();\n block_on(async {\n let (tx, rx) = mpsc::unbounded();\n\n for _ in 0..THREADS {\n let mut tx = tx.clone();\n pool.spawn_ok(async move {\n tx.send_all(&mut stream::iter(\n (0..MESSAGES / THREADS).map(message::new).map(Ok),\n ))\n .await\n .unwrap()\n });\n }\n drop(tx);\n\n rx.for_each(|_| future::ready(())).await\n });\n}\n\nfn mpsc_bounded(cap: usize) {\n let pool = ThreadPool::new().unwrap();\n block_on(async {\n let (tx, rx) = mpsc::channel(cap);\n\n for _ in 0..THREADS {\n let mut tx = tx.clone();\n pool.spawn_ok(async move {\n tx.send_all(&mut stream::iter(\n (0..MESSAGES / THREADS).map(message::new).map(Ok),\n ))\n .await\n .unwrap()\n });\n }\n drop(tx);\n\n rx.for_each(|_| future::ready(())).await\n });\n}\n\nfn select_rx_unbounded() {\n let pool = ThreadPool::new().unwrap();\n block_on(async {\n let chans = (0..THREADS).map(|_| mpsc::unbounded()).collect::>();\n\n for (tx, _) in &chans {\n let tx = tx.clone();\n pool.spawn_ok(async move {\n for i in 0..MESSAGES / THREADS {\n tx.unbounded_send(message::new(i)).unwrap();\n }\n });\n }\n\n stream::select_all(chans.into_iter().map(|(_, rx)| rx))\n .for_each(|_| future::ready(()))\n .await\n });\n}\n\nfn select_rx_bounded(cap: usize) {\n let pool = ThreadPool::new().unwrap();\n block_on(async {\n let chans = (0..THREADS).map(|_| mpsc::channel(cap)).collect::>();\n\n for (tx, _) in &chans {\n let mut tx = tx.clone();\n pool.spawn_ok(async move {\n tx.send_all(&mut stream::iter(\n (0..MESSAGES / THREADS).map(message::new).map(Ok),\n ))\n .await\n .unwrap()\n });\n }\n\n stream::select_all(chans.into_iter().map(|(_, rx)| rx))\n .for_each(|_| future::ready(()))\n .await\n });\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust futures-channel\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n run!(\"bounded0_mpsc\", mpsc_bounded(0));\n run!(\"bounded0_select_rx\", select_rx_bounded(0));\n run!(\"bounded0_spsc\", spsc_bounded(0));\n\n run!(\"bounded1_mpsc\", mpsc_bounded(1));\n run!(\"bounded1_select_rx\", select_rx_bounded(1));\n run!(\"bounded1_spsc\", spsc_bounded(1));\n\n run!(\"bounded_mpsc\", mpsc_bounded(MESSAGES));\n run!(\"bounded_select_rx\", select_rx_bounded(MESSAGES));\n run!(\"bounded_seq\", seq_bounded(MESSAGES));\n run!(\"bounded_spsc\", spsc_bounded(MESSAGES));\n\n run!(\"unbounded_mpsc\", mpsc_unbounded());\n run!(\"unbounded_select_rx\", select_rx_unbounded());\n run!(\"unbounded_seq\", seq_unbounded());\n run!(\"unbounded_spsc\", spsc_unbounded());\n}\n" }, { "path": "crossbeam-channel/benchmarks/go.go", "content": "package main\n\nimport \"fmt\"\nimport \"time\"\n\nconst MESSAGES = 5 * 1000 * 1000\nconst THREADS = 4\n\ntype Message = uintptr\n\nfunc seq(cap int) {\n var c = make(chan Message, cap)\n\n for i := 0; i < MESSAGES; i++ {\n c <- Message(i)\n }\n\n for i := 0; i < MESSAGES; i++ {\n <-c\n }\n}\n\nfunc spsc(cap int) {\n var c = make(chan Message, cap)\n var done = make(chan bool)\n\n go func() {\n for i := 0; i < MESSAGES; i++ {\n c <- Message(i)\n }\n done <- true\n }()\n\n for i := 0; i < MESSAGES; i++ {\n <-c\n }\n\n <-done\n}\n\nfunc mpsc(cap int) {\n var c = make(chan Message, cap)\n var done = make(chan bool)\n\n for t := 0; t < THREADS; t++ {\n go func() {\n for i := 0; i < MESSAGES / THREADS; i++ {\n c <- Message(i)\n }\n done <- true\n }()\n }\n\n for i := 0; i < MESSAGES; i++ {\n <-c\n }\n\n for t := 0; t < THREADS; t++ {\n <-done\n }\n}\n\nfunc mpmc(cap int) {\n var c = make(chan Message, cap)\n var done = make(chan bool)\n\n for t := 0; t < THREADS; t++ {\n go func() {\n for i := 0; i < MESSAGES / THREADS; i++ {\n c <- Message(i)\n }\n done <- true\n }()\n\n }\n\n for t := 0; t < THREADS; t++ {\n go func() {\n for i := 0; i < MESSAGES / THREADS; i++ {\n <-c\n }\n done <- true\n }()\n }\n\n for t := 0; t < THREADS; t++ {\n <-done\n <-done\n }\n}\n\nfunc select_rx(cap int) {\n if THREADS != 4 {\n panic(\"assumed there are 4 threads\")\n }\n\n var c0 = make(chan Message, cap)\n var c1 = make(chan Message, cap)\n var c2 = make(chan Message, cap)\n var c3 = make(chan Message, cap)\n var done = make(chan bool)\n\n var producer = func(c chan Message) {\n for i := 0; i < MESSAGES / THREADS; i++ {\n c <- Message(i)\n }\n done <- true\n }\n go producer(c0)\n go producer(c1)\n go producer(c2)\n go producer(c3)\n\n for i := 0; i < MESSAGES; i++ {\n select {\n case <-c0:\n case <-c1:\n case <-c2:\n case <-c3:\n }\n }\n\n for t := 0; t < THREADS; t++ {\n <-done\n }\n}\n\nfunc select_both(cap int) {\n if THREADS != 4 {\n panic(\"assumed there are 4 threads\")\n }\n\n var c0 = make(chan Message, cap)\n var c1 = make(chan Message, cap)\n var c2 = make(chan Message, cap)\n var c3 = make(chan Message, cap)\n var done = make(chan bool)\n\n var producer = func(c0 chan Message, c1 chan Message, c2 chan Message, c3 chan Message) {\n for i := 0; i < MESSAGES / THREADS; i++ {\n select {\n case c0 <- Message(i):\n case c1 <- Message(i):\n case c2 <- Message(i):\n case c3 <- Message(i):\n }\n }\n done <- true\n }\n go producer(c0,c1,c2,c3)\n go producer(c0,c1,c2,c3)\n go producer(c0,c1,c2,c3)\n go producer(c0,c1,c2,c3)\n\n for t := 0; t < THREADS; t++ {\n go func() {\n for i := 0; i < MESSAGES / THREADS; i++ {\n select {\n case <-c0:\n case <-c1:\n case <-c2:\n case <-c3:\n }\n }\n done <- true\n }()\n }\n\n for t := 0; t < THREADS; t++ {\n <-done\n <-done\n }\n}\n\nfunc run(name string, f func(int), cap int) {\n var now = time.Now()\n f(cap)\n var elapsed = time.Now().Sub(now)\n fmt.Printf(\"%-25v %-15v %7.3f sec\\n\", name, \"Go chan\", float64(elapsed) / float64(time.Second))\n}\n\nfunc main() {\n run(\"bounded0_mpmc\", mpmc, 0)\n run(\"bounded0_mpsc\", mpsc, 0)\n run(\"bounded0_select_both\", select_both, 0)\n run(\"bounded0_select_rx\", select_rx, 0)\n run(\"bounded0_spsc\", spsc, 0)\n\n run(\"bounded1_mpmc\", mpmc, 1)\n run(\"bounded1_mpsc\", mpsc, 1)\n run(\"bounded1_select_both\", select_both, 1)\n run(\"bounded1_select_rx\", select_rx, 1)\n run(\"bounded1_spsc\", spsc, 1)\n\n run(\"bounded_mpmc\", mpmc, MESSAGES)\n run(\"bounded_mpsc\", mpsc, MESSAGES)\n run(\"bounded_select_both\", select_both, MESSAGES)\n run(\"bounded_select_rx\", select_rx, MESSAGES)\n run(\"bounded_seq\", seq, MESSAGES)\n run(\"bounded_spsc\", spsc, MESSAGES)\n}\n" }, { "path": "crossbeam-channel/benchmarks/lockfree.rs", "content": "use lockfree::channel;\n\nmod message;\n\nconst MESSAGES: usize = 5_000_000;\nconst THREADS: usize = 4;\n\nuse std::thread;\n\nfn seq() {\n let (mut tx, mut rx) = channel::spsc::create();\n\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n\n for _ in 0..MESSAGES {\n while rx.recv().is_err() {\n thread::yield_now();\n }\n }\n}\n\nfn spsc() {\n let (mut tx, mut rx) = channel::spsc::create();\n\n crossbeam::scope(|scope| {\n scope.spawn(|_| {\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n });\n\n for _ in 0..MESSAGES {\n while rx.recv().is_err() {\n thread::yield_now();\n }\n }\n })\n .unwrap();\n}\n\nfn mpsc() {\n let (tx, mut rx) = channel::mpsc::create();\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n tx.send(message::new(i)).unwrap();\n }\n });\n }\n\n for _ in 0..MESSAGES {\n while rx.recv().is_err() {\n thread::yield_now();\n }\n }\n })\n .unwrap();\n}\n\nfn mpmc() {\n let (tx, rx) = channel::mpmc::create();\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n tx.send(message::new(i)).unwrap();\n }\n });\n }\n\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..MESSAGES / THREADS {\n while rx.recv().is_err() {\n thread::yield_now();\n }\n }\n });\n }\n })\n .unwrap();\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust lockfree\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n run!(\"unbounded_mpmc\", mpmc());\n run!(\"unbounded_mpsc\", mpsc());\n run!(\"unbounded_seq\", seq());\n run!(\"unbounded_spsc\", spsc());\n}\n" }, { "path": "crossbeam-channel/benchmarks/message.rs", "content": "use std::fmt;\n\nconst LEN: usize = 1;\n\n#[derive(Clone, Copy)]\npub(crate) struct Message(#[allow(dead_code)] [usize; LEN]);\n\nimpl fmt::Debug for Message {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Message\")\n }\n}\n\n#[inline]\npub(crate) fn new(num: usize) -> Message {\n Message([num; LEN])\n}\n" }, { "path": "crossbeam-channel/benchmarks/mpmc.rs", "content": "use std::thread;\n\nmod message;\n\nconst MESSAGES: usize = 5_000_000;\nconst THREADS: usize = 4;\n\nfn seq(cap: usize) {\n let q = mpmc::Queue::with_capacity(cap);\n\n for i in 0..MESSAGES {\n loop {\n if q.push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n\n for _ in 0..MESSAGES {\n q.pop().unwrap();\n }\n}\n\nfn spsc(cap: usize) {\n let q = mpmc::Queue::with_capacity(cap);\n\n crossbeam::scope(|scope| {\n scope.spawn(|_| {\n for i in 0..MESSAGES {\n loop {\n if q.push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n });\n\n for _ in 0..MESSAGES {\n loop {\n if q.pop().is_none() {\n thread::yield_now();\n } else {\n break;\n }\n }\n }\n })\n .unwrap();\n}\n\nfn mpsc(cap: usize) {\n let q = mpmc::Queue::with_capacity(cap);\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n loop {\n if q.push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n });\n }\n\n for _ in 0..MESSAGES {\n loop {\n if q.pop().is_none() {\n thread::yield_now();\n } else {\n break;\n }\n }\n }\n })\n .unwrap();\n}\n\nfn mpmc(cap: usize) {\n let q = mpmc::Queue::with_capacity(cap);\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n loop {\n if q.push(message::new(i)).is_ok() {\n break;\n } else {\n thread::yield_now();\n }\n }\n }\n });\n }\n\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..MESSAGES / THREADS {\n loop {\n if q.pop().is_none() {\n thread::yield_now();\n } else {\n break;\n }\n }\n }\n });\n }\n })\n .unwrap();\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust mpmc\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n run!(\"bounded_mpmc\", mpmc(MESSAGES));\n run!(\"bounded_mpsc\", mpsc(MESSAGES));\n run!(\"bounded_seq\", seq(MESSAGES));\n run!(\"bounded_spsc\", spsc(MESSAGES));\n}\n" }, { "path": "crossbeam-channel/benchmarks/mpsc.rs", "content": "use std::sync::mpsc;\n\nmod message;\n\nconst MESSAGES: usize = 5_000_000;\nconst THREADS: usize = 4;\n\npub fn shuffle(v: &mut [T]) {\n use std::{cell::Cell, num::Wrapping};\n\n let len = v.len();\n if len <= 1 {\n return;\n }\n\n thread_local! {\n static RNG: Cell> = const { Cell::new(Wrapping(1)) };\n }\n\n RNG.with(|rng| {\n for i in 1..len {\n // This is the 32-bit variant of Xorshift.\n // https://en.wikipedia.org/wiki/Xorshift\n let mut x = rng.get();\n x ^= x << 13;\n x ^= x >> 17;\n x ^= x << 5;\n rng.set(x);\n\n let x = x.0;\n let n = i + 1;\n\n // This is a fast alternative to `let j = x % n`.\n // https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/\n let j = ((x as u64).wrapping_mul(n as u64) >> 32) as u32 as usize;\n\n v.swap(i, j);\n }\n });\n}\n\nfn seq_async() {\n let (tx, rx) = mpsc::channel();\n\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n}\n\nfn seq_sync(cap: usize) {\n let (tx, rx) = mpsc::sync_channel(cap);\n\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n}\n\nfn spsc_async() {\n let (tx, rx) = mpsc::channel();\n\n crossbeam::scope(|scope| {\n scope.spawn(move |_| {\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n });\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn spsc_sync(cap: usize) {\n let (tx, rx) = mpsc::sync_channel(cap);\n\n crossbeam::scope(|scope| {\n scope.spawn(move |_| {\n for i in 0..MESSAGES {\n tx.send(message::new(i)).unwrap();\n }\n });\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn mpsc_async() {\n let (tx, rx) = mpsc::channel();\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n let tx = tx.clone();\n scope.spawn(move |_| {\n for i in 0..MESSAGES / THREADS {\n tx.send(message::new(i)).unwrap();\n }\n });\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn mpsc_sync(cap: usize) {\n let (tx, rx) = mpsc::sync_channel(cap);\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n let tx = tx.clone();\n scope.spawn(move |_| {\n for i in 0..MESSAGES / THREADS {\n tx.send(message::new(i)).unwrap();\n }\n });\n }\n\n for _ in 0..MESSAGES {\n rx.recv().unwrap();\n }\n })\n .unwrap();\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust mpsc\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n run!(\"bounded0_mpsc\", mpsc_sync(0));\n run!(\"bounded0_spsc\", spsc_sync(0));\n\n run!(\"bounded1_mpsc\", mpsc_sync(1));\n run!(\"bounded1_spsc\", spsc_sync(1));\n\n run!(\"bounded_mpsc\", mpsc_sync(MESSAGES));\n run!(\"bounded_seq\", seq_sync(MESSAGES));\n run!(\"bounded_spsc\", spsc_sync(MESSAGES));\n\n run!(\"unbounded_mpsc\", mpsc_async());\n run!(\"unbounded_seq\", seq_async());\n run!(\"unbounded_spsc\", spsc_async());\n}\n" }, { "path": "crossbeam-channel/benchmarks/plot.py", "content": "#!/usr/bin/env python3\nimport random\nimport sys\nimport matplotlib.pyplot as plt\n\n\ndef read_data(files):\n results = []\n for f in files:\n with open(f) as f:\n for line in f.readlines():\n test, lang, impl, secs, _ = line.split()\n splt = test.split('_')\n results.append((splt[0], '_'.join(splt[1:]), lang, impl, float(secs)))\n return results\n\n\ndef get_runs(results, prefix):\n runs = set()\n for pre, test, lang, impl, secs in results:\n if pre == prefix:\n runs.add(test)\n result = list(runs)\n result.sort()\n return result\n\n\ndef find(s, x):\n for i in range(len(s)):\n if s[i] == x:\n return i\n return None\n\n\ncolor_set = {\n 'aqua': '#00ffff',\n 'azure': '#f0ffff',\n 'beige': '#f5f5dc',\n 'black': '#000000',\n 'blue': '#0000ff',\n 'brown': '#a52a2a',\n 'cyan': '#00ffff',\n 'darkblue': '#00008b',\n 'darkcyan': '#008b8b',\n 'darkgrey': '#a9a9a9',\n 'darkgreen': '#006400',\n 'darkkhaki': '#bdb76b',\n 'darkmagenta': '#8b008b',\n 'darkolivegreen': '#556b2f',\n 'darkorange': '#ff8c00',\n 'darkorchid': '#9932cc',\n 'darkred': '#8b0000',\n 'darksalmon': '#e9967a',\n 'darkviolet': '#9400d3',\n 'fuchsia': '#ff00ff',\n 'gold': '#ffd700',\n 'green': '#008000',\n 'indigo': '#4b0082',\n 'khaki': '#f0e68c',\n 'lightblue': '#add8e6',\n 'lightcyan': '#e0ffff',\n 'lightgreen': '#90ee90',\n 'lightgrey': '#d3d3d3',\n 'lightpink': '#ffb6c1',\n 'lightyellow': '#ffffe0',\n 'lime': '#00ff00',\n 'magenta': '#ff00ff',\n 'maroon': '#800000',\n 'navy': '#000080',\n 'olive': '#808000',\n 'orange': '#ffa500',\n 'pink': '#ffc0cb',\n 'purple': '#800080',\n 'red': '#ff0000',\n}\nsaved_color = {}\n\n\ndef get_color(name):\n if name not in saved_color:\n color = color_set.popitem()\n saved_color[name] = color\n return saved_color[name][1]\n\n\ndef plot(results, fig, subplot, title, prefix):\n runs = get_runs(results, prefix)\n\n ys = [len(runs) * (i + 1) for i in range(len(runs))]\n\n ax = fig.add_subplot(subplot)\n ax.set_title(title)\n ax.set_yticks(ys)\n ax.set_yticklabels(runs)\n ax.tick_params(which='major', length=0)\n ax.set_xlabel('seconds')\n\n scores = {}\n\n for pre, test, lang, impl, secs in results:\n if pre == prefix:\n name = impl if lang == 'Rust' else impl + f' ({lang})'\n if name not in scores:\n scores[name] = [0] * len(runs)\n scores[name][find(runs, test)] = secs\n\n opts = dict(height=0.8, align='center')\n x_max = max(max(scores.values(), key=lambda x: max(x)))\n for i, (name, score) in enumerate(scores.items()):\n yy = [y + i - len(runs) // 2 + 0.2 for y in ys]\n ax.barh(yy, score, color=get_color(name), **opts)\n for xxx, yyy in zip(score, yy):\n if xxx:\n ax.text(min(x_max - len(name) * 0.018 * x_max, xxx), yyy - 0.25, name, fontsize=9)\n\n\ndef plot_all(results, descriptions, labels):\n fig = plt.figure(figsize=(10, 10))\n # TODO support more subplots\n subplot = [221, 222, 223, 224]\n for p, d, l in zip(subplot, descriptions, labels):\n plot(results, fig, p, d, l)\n plt.subplots_adjust(\n top=0.95,\n bottom=0.05,\n left=0.1,\n right=0.95,\n wspace=0.3,\n hspace=0.2,\n )\n plt.savefig('plot.png')\n # plt.show()\n\n\ndef main():\n results = read_data(sys.argv[1:])\n descriptions = [\n 'Bounded channel of capacity 0',\n 'Bounded channel of capacity 1',\n 'Bounded channel of capacity N',\n 'Unbounded channel',\n ]\n labels = ['bounded0', 'bounded1', 'bounded', 'unbounded']\n plot_all(results, descriptions, labels)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "crossbeam-channel/benchmarks/run.sh", "content": "#!/bin/bash\nset -euxo pipefail\nIFS=$'\\n\\t'\ncd \"$(dirname \"$0\")\"\n\ncargo run --release --bin crossbeam-channel | tee crossbeam-channel.txt\ncargo run --release --bin futures-channel | tee futures-channel.txt\ncargo run --release --bin mpsc | tee mpsc.txt\ncargo run --release --bin flume | tee flume.txt\ngo run go.go | tee go.txt\n\n# These can also be run, but too many plot bars mess\n# up the plot (they start to overlap). So only 5 contenders\n# with the most tests are included by default.\n\n# cargo run --release --bin atomicringqueue | tee atomicringqueue.txt\n# cargo run --release --bin atomicring | tee atomicring.txt\n# cargo run --release --bin bus | tee bus.txt\n# cargo run --release --bin crossbeam-deque | tee crossbeam-deque.txt\n# cargo run --release --bin lockfree | tee lockfree.txt\n# cargo run --release --bin segqueue | tee segqueue.txt\n# cargo run --release --bin mpmc | tee mpmc.txt\n\n./plot.py ./*.txt\n" }, { "path": "crossbeam-channel/benchmarks/segqueue.rs", "content": "use std::thread;\n\nuse crossbeam::queue::SegQueue;\n\nmod message;\n\nconst MESSAGES: usize = 5_000_000;\nconst THREADS: usize = 4;\n\nfn seq() {\n let q = SegQueue::new();\n\n for i in 0..MESSAGES {\n q.push(message::new(i));\n }\n\n for _ in 0..MESSAGES {\n q.pop().unwrap();\n }\n}\n\nfn spsc() {\n let q = SegQueue::new();\n\n crossbeam::scope(|scope| {\n scope.spawn(|_| {\n for i in 0..MESSAGES {\n q.push(message::new(i));\n }\n });\n\n for _ in 0..MESSAGES {\n loop {\n if q.pop().is_none() {\n thread::yield_now();\n } else {\n break;\n }\n }\n }\n })\n .unwrap();\n}\n\nfn mpsc() {\n let q = SegQueue::new();\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n q.push(message::new(i));\n }\n });\n }\n\n for _ in 0..MESSAGES {\n loop {\n if q.pop().is_none() {\n thread::yield_now();\n } else {\n break;\n }\n }\n }\n })\n .unwrap();\n}\n\nfn mpmc() {\n let q = SegQueue::new();\n\n crossbeam::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..MESSAGES / THREADS {\n q.push(message::new(i));\n }\n });\n }\n\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..MESSAGES / THREADS {\n loop {\n if q.pop().is_none() {\n thread::yield_now();\n } else {\n break;\n }\n }\n }\n });\n }\n })\n .unwrap();\n}\n\nfn main() {\n macro_rules! run {\n ($name:expr, $f:expr) => {\n let now = ::std::time::Instant::now();\n $f;\n let elapsed = now.elapsed();\n println!(\n \"{:25} {:15} {:7.3} sec\",\n $name,\n \"Rust segqueue\",\n elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9\n );\n };\n }\n\n run!(\"unbounded_mpmc\", mpmc());\n run!(\"unbounded_mpsc\", mpsc());\n run!(\"unbounded_seq\", seq());\n run!(\"unbounded_spsc\", spsc());\n}\n" }, { "path": "crossbeam-channel/examples/fibonacci.rs", "content": "//! An asynchronous fibonacci sequence generator.\n\nuse std::thread;\n\nuse crossbeam_channel::{Sender, bounded};\n\n// Sends the Fibonacci sequence into the channel until it becomes disconnected.\nfn fibonacci(sender: Sender) {\n let (mut x, mut y) = (0, 1);\n while sender.send(x).is_ok() {\n let tmp = x;\n x = y;\n y += tmp;\n }\n}\n\nfn main() {\n let (s, r) = bounded(0);\n thread::spawn(|| fibonacci(s));\n\n // Print the first 20 Fibonacci numbers.\n for num in r.iter().take(20) {\n println!(\"{}\", num);\n }\n}\n" }, { "path": "crossbeam-channel/examples/matching.rs", "content": "//! Using `select!` to send and receive on the same channel at the same time.\n//!\n//! This example is based on the following program in Go.\n//!\n//! Source:\n//! - https://web.archive.org/web/20171209034309/https://www.nada.kth.se/~snilsson/concurrency\n//! - http://www.nada.kth.se/~snilsson/concurrency/src/matching.go\n//!\n//! Copyright & License:\n//! - Stefan Nilsson\n//! - Creative Commons Attribution 3.0 Unported License\n//! - https://creativecommons.org/licenses/by/3.0/\n//!\n//! ```go\n//! func main() {\n//! people := []string{\"Anna\", \"Bob\", \"Cody\", \"Dave\", \"Eva\"}\n//! match := make(chan string, 1) // Make room for one unmatched send.\n//! wg := new(sync.WaitGroup)\n//! for _, name := range people {\n//! wg.Add(1)\n//! go Seek(name, match, wg)\n//! }\n//! wg.Wait()\n//! select {\n//! case name := <-match:\n//! fmt.Printf(\"No one received %s’s message.\\n\", name)\n//! default:\n//! // There was no pending send operation.\n//! }\n//! }\n//!\n//! // Seek either sends or receives, whichever possible, a name on the match\n//! // channel and notifies the wait group when done.\n//! func Seek(name string, match chan string, wg *sync.WaitGroup) {\n//! select {\n//! case peer := <-match:\n//! fmt.Printf(\"%s received a message from %s.\\n\", name, peer)\n//! case match <- name:\n//! // Wait for someone to receive my message.\n//! }\n//! wg.Done()\n//! }\n//! ```\n\nuse crossbeam_channel::{bounded, select};\nuse crossbeam_utils::thread;\n\nfn main() {\n let people = vec![\"Anna\", \"Bob\", \"Cody\", \"Dave\", \"Eva\"];\n let (s, r) = bounded(1); // Make room for one unmatched send.\n\n // Either send my name into the channel or receive someone else's, whatever happens first.\n let seek = |name, s, r| {\n select! {\n recv(r) -> peer => println!(\"{} received a message from {}.\", name, peer.unwrap()),\n send(s, name) -> _ => {}, // Wait for someone to receive my message.\n }\n };\n\n thread::scope(|scope| {\n for name in people {\n let (s, r) = (s.clone(), r.clone());\n scope.spawn(move |_| seek(name, s, r));\n }\n })\n .unwrap();\n\n // Check if there is a pending send operation.\n if let Ok(name) = r.try_recv() {\n println!(\"No one received {}’s message.\", name);\n }\n}\n" }, { "path": "crossbeam-channel/examples/stopwatch.rs", "content": "//! Prints the elapsed time every 1 second and quits on Ctrl+C.\n\n#[cfg(windows)] // signal_hook::iterator does not work on windows\nfn main() {\n println!(\"This example does not work on Windows\");\n}\n\n#[cfg(not(windows))]\nfn main() {\n use std::{\n io, thread,\n time::{Duration, Instant},\n };\n\n use crossbeam_channel::{Receiver, bounded, select, tick};\n use signal_hook::{consts::SIGINT, iterator::Signals};\n\n // Creates a channel that gets a message every time `SIGINT` is signalled.\n fn sigint_notifier() -> io::Result> {\n let (s, r) = bounded(100);\n let mut signals = Signals::new([SIGINT])?;\n\n thread::spawn(move || {\n for _ in signals.forever() {\n if s.send(()).is_err() {\n break;\n }\n }\n });\n\n Ok(r)\n }\n\n // Prints the elapsed time.\n fn show(dur: Duration) {\n println!(\"Elapsed: {}.{:03} sec\", dur.as_secs(), dur.subsec_millis());\n }\n\n let start = Instant::now();\n let update = tick(Duration::from_secs(1));\n let ctrl_c = sigint_notifier().unwrap();\n\n loop {\n select! {\n recv(update) -> _ => {\n show(start.elapsed());\n }\n recv(ctrl_c) -> _ => {\n println!();\n println!(\"Goodbye!\");\n show(start.elapsed());\n break;\n }\n }\n }\n}\n" }, { "path": "crossbeam-channel/src/channel.rs", "content": "//! The channel interface.\n\nuse alloc::sync::Arc;\nuse core::{\n fmt,\n iter::FusedIterator,\n mem,\n panic::{RefUnwindSafe, UnwindSafe},\n time::Duration,\n};\nuse std::time::Instant;\n\nuse crate::{\n context::Context,\n counter,\n err::{RecvError, RecvTimeoutError, SendError, SendTimeoutError, TryRecvError, TrySendError},\n flavors,\n select::{Operation, SelectHandle, Token},\n};\n\n/// Creates a multi-producer multi-consumer channel of unbounded capacity.\n///\n/// This channel has a growable buffer that can hold any number of messages at a time.\n///\n/// For more info on how to use the channel see [module level documentation](index.html).\n///\n/// # Examples\n///\n/// ```\n/// use std::thread;\n/// use crossbeam_channel::unbounded;\n///\n/// let (s, r) = unbounded();\n///\n/// // Computes the n-th Fibonacci number.\n/// fn fib(n: i32) -> i32 {\n/// if n <= 1 {\n/// n\n/// } else {\n/// fib(n - 1) + fib(n - 2)\n/// }\n/// }\n///\n/// // Spawn an asynchronous computation.\n/// thread::spawn(move || s.send(fib(20)).unwrap());\n///\n/// // Print the result of the computation.\n/// println!(\"{}\", r.recv().unwrap());\n/// ```\npub fn unbounded() -> (Sender, Receiver) {\n let (s, r) = counter::new(flavors::list::Channel::new());\n let s = Sender {\n flavor: SenderFlavor::List(s),\n };\n let r = Receiver {\n flavor: ReceiverFlavor::List(r),\n };\n (s, r)\n}\n\n/// Creates a multi-producer multi-consumer channel of bounded capacity.\n///\n/// This channel has a buffer that can hold at most `cap` messages at a time.\n///\n/// A special case is zero-capacity channel, which cannot hold any messages. Instead, send and\n/// receive operations must appear at the same time in order to pair up and pass the message over.\n///\n/// For more info on how to use the channel see [module level documentation](index.html).\n///\n/// # Examples\n///\n/// A channel of capacity 1:\n///\n/// ```\n/// use std::thread;\n/// use std::time::Duration;\n/// use crossbeam_channel::bounded;\n///\n/// let (s, r) = bounded(1);\n///\n/// // This call returns immediately because there is enough space in the channel.\n/// s.send(1).unwrap();\n///\n/// thread::spawn(move || {\n/// // This call blocks the current thread because the channel is full.\n/// // It will be able to complete only after the first message is received.\n/// s.send(2).unwrap();\n/// });\n///\n/// thread::sleep(Duration::from_secs(1));\n/// assert_eq!(r.recv(), Ok(1));\n/// assert_eq!(r.recv(), Ok(2));\n/// ```\n///\n/// A zero-capacity channel:\n///\n/// ```\n/// use std::thread;\n/// use std::time::Duration;\n/// use crossbeam_channel::bounded;\n///\n/// let (s, r) = bounded(0);\n///\n/// thread::spawn(move || {\n/// // This call blocks the current thread until a receive operation appears\n/// // on the other side of the channel.\n/// s.send(1).unwrap();\n/// });\n///\n/// thread::sleep(Duration::from_secs(1));\n/// assert_eq!(r.recv(), Ok(1));\n/// ```\npub fn bounded(cap: usize) -> (Sender, Receiver) {\n if cap == 0 {\n let (s, r) = counter::new(flavors::zero::Channel::new());\n let s = Sender {\n flavor: SenderFlavor::Zero(s),\n };\n let r = Receiver {\n flavor: ReceiverFlavor::Zero(r),\n };\n (s, r)\n } else {\n let (s, r) = counter::new(flavors::array::Channel::with_capacity(cap));\n let s = Sender {\n flavor: SenderFlavor::Array(s),\n };\n let r = Receiver {\n flavor: ReceiverFlavor::Array(r),\n };\n (s, r)\n }\n}\n\n/// Creates a receiver that delivers a message after a certain duration of time.\n///\n/// The channel is bounded with capacity of 1 and never gets disconnected. Exactly one message will\n/// be sent into the channel after `duration` elapses. The message is the instant at which it is\n/// sent.\n///\n/// # Examples\n///\n/// Using an `after` channel for timeouts:\n///\n/// ```\n/// use std::time::Duration;\n/// use crossbeam_channel::{after, select, unbounded};\n///\n/// let (s, r) = unbounded::();\n/// let timeout = Duration::from_millis(100);\n///\n/// select! {\n/// recv(r) -> msg => println!(\"received {:?}\", msg),\n/// recv(after(timeout)) -> _ => println!(\"timed out\"),\n/// }\n/// ```\n///\n/// When the message gets sent:\n///\n/// ```\n/// # if cfg!(gha_macos_runner) { return; } // GitHub-hosted macOS runner is slow\n/// use std::thread;\n/// use std::time::{Duration, Instant};\n/// use crossbeam_channel::after;\n///\n/// // Converts a number of milliseconds into a `Duration`.\n/// let ms = |ms| Duration::from_millis(ms);\n///\n/// // Returns `true` if `a` and `b` are very close `Instant`s.\n/// let eq = |a, b| a + ms(60) > b && b + ms(60) > a;\n///\n/// let start = Instant::now();\n/// let r = after(ms(100));\n///\n/// thread::sleep(ms(500));\n///\n/// // This message was sent 100 ms from the start and received 500 ms from the start.\n/// assert!(eq(r.recv().unwrap(), start + ms(100)));\n/// assert!(eq(Instant::now(), start + ms(500)));\n/// ```\npub fn after(duration: Duration) -> Receiver {\n match Instant::now().checked_add(duration) {\n Some(deadline) => Receiver {\n flavor: ReceiverFlavor::At(Arc::new(flavors::at::Channel::new_deadline(deadline))),\n },\n None => never(),\n }\n}\n\n/// Creates a receiver that delivers a message at a certain instant in time.\n///\n/// The channel is bounded with capacity of 1 and never gets disconnected. Exactly one message will\n/// be sent into the channel at the moment in time `when`. The message is the instant at which it\n/// is sent, which is the same as `when`. If `when` is in the past, the message will be delivered\n/// instantly to the receiver.\n///\n/// # Examples\n///\n/// Using an `at` channel for timeouts:\n///\n/// ```\n/// use std::time::{Instant, Duration};\n/// use crossbeam_channel::{at, select, unbounded};\n///\n/// let (s, r) = unbounded::();\n/// let deadline = Instant::now() + Duration::from_millis(500);\n///\n/// select! {\n/// recv(r) -> msg => println!(\"received {:?}\", msg),\n/// recv(at(deadline)) -> _ => println!(\"timed out\"),\n/// }\n/// ```\n///\n/// When the message gets sent:\n///\n/// ```\n/// use std::time::{Duration, Instant};\n/// use crossbeam_channel::at;\n///\n/// // Converts a number of milliseconds into a `Duration`.\n/// let ms = |ms| Duration::from_millis(ms);\n///\n/// let start = Instant::now();\n/// let end = start + ms(100);\n///\n/// let r = at(end);\n///\n/// // This message was sent 100 ms from the start\n/// assert_eq!(r.recv().unwrap(), end);\n/// assert!(Instant::now() > start + ms(100));\n/// ```\npub fn at(when: Instant) -> Receiver {\n Receiver {\n flavor: ReceiverFlavor::At(Arc::new(flavors::at::Channel::new_deadline(when))),\n }\n}\n\n/// Creates a receiver that never delivers messages.\n///\n/// The channel is bounded with capacity of 0 and never gets disconnected.\n///\n/// # Examples\n///\n/// Using a `never` channel to optionally add a timeout to [`select!`]:\n///\n/// [`select!`]: crate::select!\n///\n/// ```\n/// use std::thread;\n/// use std::time::Duration;\n/// use crossbeam_channel::{after, select, never, unbounded};\n///\n/// let (s, r) = unbounded();\n///\n/// # let t =\n/// thread::spawn(move || {\n/// thread::sleep(Duration::from_secs(1));\n/// s.send(1).unwrap();\n/// });\n///\n/// // Suppose this duration can be a `Some` or a `None`.\n/// let duration = Some(Duration::from_millis(100));\n///\n/// // Create a channel that times out after the specified duration.\n/// let timeout = duration\n/// .map(after)\n/// .unwrap_or_else(never);\n///\n/// select! {\n/// recv(r) -> msg => assert_eq!(msg, Ok(1)),\n/// recv(timeout) -> _ => println!(\"timed out\"),\n/// }\n/// # t.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n/// ```\npub fn never() -> Receiver {\n Receiver {\n flavor: ReceiverFlavor::Never(flavors::never::Channel::new()),\n }\n}\n\n/// Creates a receiver that delivers messages periodically.\n///\n/// The channel is bounded with capacity of 1 and never gets disconnected. Messages will be\n/// sent into the channel in intervals of `duration`. Each message is the instant at which it is\n/// sent.\n///\n/// # Examples\n///\n/// Using a `tick` channel to periodically print elapsed time:\n///\n/// ```\n/// use std::time::{Duration, Instant};\n/// use crossbeam_channel::tick;\n///\n/// let start = Instant::now();\n/// let ticker = tick(Duration::from_millis(100));\n///\n/// for _ in 0..5 {\n/// ticker.recv().unwrap();\n/// println!(\"elapsed: {:?}\", start.elapsed());\n/// }\n/// ```\n///\n/// When messages get sent:\n///\n/// ```\n/// # if cfg!(gha_macos_runner) { return; } // GitHub-hosted macOS runner is slow\n/// use std::thread;\n/// use std::time::{Duration, Instant};\n/// use crossbeam_channel::tick;\n///\n/// // Converts a number of milliseconds into a `Duration`.\n/// let ms = |ms| Duration::from_millis(ms);\n///\n/// // Returns `true` if `a` and `b` are very close `Instant`s.\n/// let eq = |a, b| a + ms(65) > b && b + ms(65) > a;\n///\n/// let start = Instant::now();\n/// let r = tick(ms(100));\n///\n/// // This message was sent 100 ms from the start and received 100 ms from the start.\n/// assert!(eq(r.recv().unwrap(), start + ms(100)));\n/// assert!(eq(Instant::now(), start + ms(100)));\n///\n/// thread::sleep(ms(500));\n///\n/// // This message was sent 200 ms from the start and received 600 ms from the start.\n/// assert!(eq(r.recv().unwrap(), start + ms(200)));\n/// assert!(eq(Instant::now(), start + ms(600)));\n///\n/// // This message was sent 700 ms from the start and received 700 ms from the start.\n/// assert!(eq(r.recv().unwrap(), start + ms(700)));\n/// assert!(eq(Instant::now(), start + ms(700)));\n/// ```\npub fn tick(duration: Duration) -> Receiver {\n match Instant::now().checked_add(duration) {\n Some(delivery_time) => Receiver {\n flavor: ReceiverFlavor::Tick(Arc::new(flavors::tick::Channel::new(\n delivery_time,\n duration,\n ))),\n },\n None => never(),\n }\n}\n\n/// The sending side of a channel.\n///\n/// # Examples\n///\n/// ```\n/// use std::thread;\n/// use crossbeam_channel::unbounded;\n///\n/// let (s1, r) = unbounded();\n/// let s2 = s1.clone();\n///\n/// thread::spawn(move || s1.send(1).unwrap());\n/// thread::spawn(move || s2.send(2).unwrap());\n///\n/// let msg1 = r.recv().unwrap();\n/// let msg2 = r.recv().unwrap();\n///\n/// assert_eq!(msg1 + msg2, 3);\n/// ```\npub struct Sender {\n flavor: SenderFlavor,\n}\n\n/// Sender flavors.\nenum SenderFlavor {\n /// Bounded channel based on a preallocated array.\n Array(counter::Sender>),\n\n /// Unbounded channel implemented as a linked list.\n List(counter::Sender>),\n\n /// Zero-capacity channel.\n Zero(counter::Sender>),\n}\n\nunsafe impl Send for Sender {}\nunsafe impl Sync for Sender {}\n\nimpl UnwindSafe for Sender {}\nimpl RefUnwindSafe for Sender {}\n\nimpl Sender {\n /// Attempts to send a message into the channel without blocking.\n ///\n /// This method will either send a message into the channel immediately or return an error if\n /// the channel is full or disconnected. The returned error contains the original message.\n ///\n /// If called on a zero-capacity channel, this method will send the message only if there\n /// happens to be a receive operation on the other side of the channel at the same time.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{bounded, TrySendError};\n ///\n /// let (s, r) = bounded(1);\n ///\n /// assert_eq!(s.try_send(1), Ok(()));\n /// assert_eq!(s.try_send(2), Err(TrySendError::Full(2)));\n ///\n /// drop(r);\n /// assert_eq!(s.try_send(3), Err(TrySendError::Disconnected(3)));\n /// ```\n pub fn try_send(&self, msg: T) -> Result<(), TrySendError> {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.try_send(msg),\n SenderFlavor::List(chan) => chan.try_send(msg),\n SenderFlavor::Zero(chan) => chan.try_send(msg),\n }\n }\n\n /// Blocks the current thread until a message is sent or the channel is disconnected.\n ///\n /// If the channel is full and not disconnected, this call will block until the send operation\n /// can proceed. If the channel becomes disconnected, this call will wake up and return an\n /// error. The returned error contains the original message.\n ///\n /// If called on a zero-capacity channel, this method will wait for a receive operation to\n /// appear on the other side of the channel.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::Duration;\n /// use crossbeam_channel::{bounded, SendError};\n ///\n /// let (s, r) = bounded(1);\n /// assert_eq!(s.send(1), Ok(()));\n ///\n /// thread::spawn(move || {\n /// assert_eq!(r.recv(), Ok(1));\n /// thread::sleep(Duration::from_secs(1));\n /// drop(r);\n /// });\n ///\n /// assert_eq!(s.send(2), Ok(()));\n /// assert_eq!(s.send(3), Err(SendError(3)));\n /// ```\n pub fn send(&self, msg: T) -> Result<(), SendError> {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.send(msg, None),\n SenderFlavor::List(chan) => chan.send(msg, None),\n SenderFlavor::Zero(chan) => chan.send(msg, None),\n }\n .map_err(|err| match err {\n SendTimeoutError::Disconnected(msg) => SendError(msg),\n SendTimeoutError::Timeout(_) => unreachable!(),\n })\n }\n\n /// Waits for a message to be sent into the channel, but only for a limited time.\n ///\n /// If the channel is full and not disconnected, this call will block until the send operation\n /// can proceed or the operation times out. If the channel becomes disconnected, this call will\n /// wake up and return an error. The returned error contains the original message.\n ///\n /// If called on a zero-capacity channel, this method will wait for a receive operation to\n /// appear on the other side of the channel.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::Duration;\n /// use crossbeam_channel::{bounded, SendTimeoutError};\n ///\n /// let (s, r) = bounded(0);\n ///\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// assert_eq!(r.recv(), Ok(2));\n /// drop(r);\n /// });\n ///\n /// assert_eq!(\n /// s.send_timeout(1, Duration::from_millis(500)),\n /// Err(SendTimeoutError::Timeout(1)),\n /// );\n /// assert_eq!(\n /// s.send_timeout(2, Duration::from_secs(1)),\n /// Ok(()),\n /// );\n /// assert_eq!(\n /// s.send_timeout(3, Duration::from_millis(500)),\n /// Err(SendTimeoutError::Disconnected(3)),\n /// );\n /// ```\n pub fn send_timeout(&self, msg: T, timeout: Duration) -> Result<(), SendTimeoutError> {\n match Instant::now().checked_add(timeout) {\n Some(deadline) => self.send_deadline(msg, deadline),\n None => self.send(msg).map_err(SendTimeoutError::from),\n }\n }\n\n /// Waits for a message to be sent into the channel, but only until a given deadline.\n ///\n /// If the channel is full and not disconnected, this call will block until the send operation\n /// can proceed or the operation times out. If the channel becomes disconnected, this call will\n /// wake up and return an error. The returned error contains the original message.\n ///\n /// If called on a zero-capacity channel, this method will wait for a receive operation to\n /// appear on the other side of the channel.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::{Duration, Instant};\n /// use crossbeam_channel::{bounded, SendTimeoutError};\n ///\n /// let (s, r) = bounded(0);\n ///\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// assert_eq!(r.recv(), Ok(2));\n /// drop(r);\n /// });\n ///\n /// let now = Instant::now();\n ///\n /// assert_eq!(\n /// s.send_deadline(1, now + Duration::from_millis(500)),\n /// Err(SendTimeoutError::Timeout(1)),\n /// );\n /// assert_eq!(\n /// s.send_deadline(2, now + Duration::from_millis(1500)),\n /// Ok(()),\n /// );\n /// assert_eq!(\n /// s.send_deadline(3, now + Duration::from_millis(2000)),\n /// Err(SendTimeoutError::Disconnected(3)),\n /// );\n /// ```\n pub fn send_deadline(&self, msg: T, deadline: Instant) -> Result<(), SendTimeoutError> {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.send(msg, Some(deadline)),\n SenderFlavor::List(chan) => chan.send(msg, Some(deadline)),\n SenderFlavor::Zero(chan) => chan.send(msg, Some(deadline)),\n }\n }\n\n /// Returns `true` if the channel is empty.\n ///\n /// Note: Zero-capacity channels are always empty.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::unbounded;\n ///\n /// let (s, r) = unbounded();\n /// assert!(s.is_empty());\n ///\n /// s.send(0).unwrap();\n /// assert!(!s.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.is_empty(),\n SenderFlavor::List(chan) => chan.is_empty(),\n SenderFlavor::Zero(chan) => chan.is_empty(),\n }\n }\n\n /// Returns `true` if the channel is full.\n ///\n /// Note: Zero-capacity channels are always full.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::bounded;\n ///\n /// let (s, r) = bounded(1);\n ///\n /// assert!(!s.is_full());\n /// s.send(0).unwrap();\n /// assert!(s.is_full());\n /// ```\n pub fn is_full(&self) -> bool {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.is_full(),\n SenderFlavor::List(chan) => chan.is_full(),\n SenderFlavor::Zero(chan) => chan.is_full(),\n }\n }\n\n /// Returns the number of messages in the channel.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::unbounded;\n ///\n /// let (s, r) = unbounded();\n /// assert_eq!(s.len(), 0);\n ///\n /// s.send(1).unwrap();\n /// s.send(2).unwrap();\n /// assert_eq!(s.len(), 2);\n /// ```\n pub fn len(&self) -> usize {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.len(),\n SenderFlavor::List(chan) => chan.len(),\n SenderFlavor::Zero(chan) => chan.len(),\n }\n }\n\n /// If the channel is bounded, returns its capacity.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{bounded, unbounded};\n ///\n /// let (s, _) = unbounded::();\n /// assert_eq!(s.capacity(), None);\n ///\n /// let (s, _) = bounded::(5);\n /// assert_eq!(s.capacity(), Some(5));\n ///\n /// let (s, _) = bounded::(0);\n /// assert_eq!(s.capacity(), Some(0));\n /// ```\n pub fn capacity(&self) -> Option {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.capacity(),\n SenderFlavor::List(chan) => chan.capacity(),\n SenderFlavor::Zero(chan) => chan.capacity(),\n }\n }\n\n /// Returns `true` if senders belong to the same channel.\n ///\n /// # Examples\n ///\n /// ```rust\n /// use crossbeam_channel::unbounded;\n ///\n /// let (s, _) = unbounded::();\n ///\n /// let s2 = s.clone();\n /// assert!(s.same_channel(&s2));\n ///\n /// let (s3, _) = unbounded();\n /// assert!(!s.same_channel(&s3));\n /// ```\n pub fn same_channel(&self, other: &Self) -> bool {\n match (&self.flavor, &other.flavor) {\n (SenderFlavor::Array(ref a), SenderFlavor::Array(ref b)) => a == b,\n (SenderFlavor::List(ref a), SenderFlavor::List(ref b)) => a == b,\n (SenderFlavor::Zero(ref a), SenderFlavor::Zero(ref b)) => a == b,\n _ => false,\n }\n }\n\n pub(crate) fn addr(&self) -> usize {\n match &self.flavor {\n SenderFlavor::Array(a) => a.addr(),\n SenderFlavor::List(a) => a.addr(),\n SenderFlavor::Zero(a) => a.addr(),\n }\n }\n}\n\nimpl Drop for Sender {\n fn drop(&mut self) {\n unsafe {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.release(|c| c.disconnect()),\n SenderFlavor::List(chan) => chan.release(|c| c.disconnect_senders()),\n SenderFlavor::Zero(chan) => chan.release(|c| c.disconnect()),\n }\n }\n }\n}\n\nimpl Clone for Sender {\n fn clone(&self) -> Self {\n let flavor = match &self.flavor {\n SenderFlavor::Array(chan) => SenderFlavor::Array(chan.acquire()),\n SenderFlavor::List(chan) => SenderFlavor::List(chan.acquire()),\n SenderFlavor::Zero(chan) => SenderFlavor::Zero(chan.acquire()),\n };\n\n Self { flavor }\n }\n}\n\nimpl fmt::Debug for Sender {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Sender { .. }\")\n }\n}\n\n/// The receiving side of a channel.\n///\n/// # Examples\n///\n/// ```\n/// use std::thread;\n/// use std::time::Duration;\n/// use crossbeam_channel::unbounded;\n///\n/// let (s, r) = unbounded();\n///\n/// thread::spawn(move || {\n/// let _ = s.send(1);\n/// thread::sleep(Duration::from_secs(1));\n/// let _ = s.send(2);\n/// });\n///\n/// assert_eq!(r.recv(), Ok(1)); // Received immediately.\n/// assert_eq!(r.recv(), Ok(2)); // Received after 1 second.\n/// ```\npub struct Receiver {\n flavor: ReceiverFlavor,\n}\n\n/// Receiver flavors.\nenum ReceiverFlavor {\n /// Bounded channel based on a preallocated array.\n Array(counter::Receiver>),\n\n /// Unbounded channel implemented as a linked list.\n List(counter::Receiver>),\n\n /// Zero-capacity channel.\n Zero(counter::Receiver>),\n\n /// The after flavor.\n At(Arc),\n\n /// The tick flavor.\n Tick(Arc),\n\n /// The never flavor.\n Never(flavors::never::Channel),\n}\n\nunsafe impl Send for Receiver {}\nunsafe impl Sync for Receiver {}\n\nimpl UnwindSafe for Receiver {}\nimpl RefUnwindSafe for Receiver {}\n\nimpl Receiver {\n /// Attempts to receive a message from the channel without blocking.\n ///\n /// This method will either receive a message from the channel immediately or return an error\n /// if the channel is empty.\n ///\n /// If called on a zero-capacity channel, this method will receive a message only if there\n /// happens to be a send operation on the other side of the channel at the same time.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{unbounded, TryRecvError};\n ///\n /// let (s, r) = unbounded();\n /// assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n ///\n /// s.send(5).unwrap();\n /// drop(s);\n ///\n /// assert_eq!(r.try_recv(), Ok(5));\n /// assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected));\n /// ```\n pub fn try_recv(&self) -> Result {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.try_recv(),\n ReceiverFlavor::List(chan) => chan.try_recv(),\n ReceiverFlavor::Zero(chan) => chan.try_recv(),\n ReceiverFlavor::At(chan) => {\n let msg = chan.try_recv();\n unsafe {\n mem::transmute_copy::, Result>(\n &msg,\n )\n }\n }\n ReceiverFlavor::Tick(chan) => {\n let msg = chan.try_recv();\n unsafe {\n mem::transmute_copy::, Result>(\n &msg,\n )\n }\n }\n ReceiverFlavor::Never(chan) => chan.try_recv(),\n }\n }\n\n /// Blocks the current thread until a message is received or the channel is empty and\n /// disconnected.\n ///\n /// If the channel is empty and not disconnected, this call will block until the receive\n /// operation can proceed. If the channel is empty and becomes disconnected, this call will\n /// wake up and return an error.\n ///\n /// If called on a zero-capacity channel, this method will wait for a send operation to appear\n /// on the other side of the channel.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::Duration;\n /// use crossbeam_channel::{unbounded, RecvError};\n ///\n /// let (s, r) = unbounded();\n ///\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// s.send(5).unwrap();\n /// drop(s);\n /// });\n ///\n /// assert_eq!(r.recv(), Ok(5));\n /// assert_eq!(r.recv(), Err(RecvError));\n /// ```\n pub fn recv(&self) -> Result {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.recv(None),\n ReceiverFlavor::List(chan) => chan.recv(None),\n ReceiverFlavor::Zero(chan) => chan.recv(None),\n ReceiverFlavor::At(chan) => {\n let msg = chan.recv(None);\n unsafe {\n mem::transmute_copy::<\n Result,\n Result,\n >(&msg)\n }\n }\n ReceiverFlavor::Tick(chan) => {\n let msg = chan.recv(None);\n unsafe {\n mem::transmute_copy::<\n Result,\n Result,\n >(&msg)\n }\n }\n ReceiverFlavor::Never(chan) => chan.recv(None),\n }\n .map_err(|_| RecvError)\n }\n\n /// Waits for a message to be received from the channel, but only for a limited time.\n ///\n /// If the channel is empty and not disconnected, this call will block until the receive\n /// operation can proceed or the operation times out. If the channel is empty and becomes\n /// disconnected, this call will wake up and return an error.\n ///\n /// If called on a zero-capacity channel, this method will wait for a send operation to appear\n /// on the other side of the channel.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::Duration;\n /// use crossbeam_channel::{unbounded, RecvTimeoutError};\n ///\n /// let (s, r) = unbounded();\n ///\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// s.send(5).unwrap();\n /// drop(s);\n /// });\n ///\n /// assert_eq!(\n /// r.recv_timeout(Duration::from_millis(500)),\n /// Err(RecvTimeoutError::Timeout),\n /// );\n /// assert_eq!(\n /// r.recv_timeout(Duration::from_secs(1)),\n /// Ok(5),\n /// );\n /// assert_eq!(\n /// r.recv_timeout(Duration::from_secs(1)),\n /// Err(RecvTimeoutError::Disconnected),\n /// );\n /// ```\n pub fn recv_timeout(&self, timeout: Duration) -> Result {\n match Instant::now().checked_add(timeout) {\n Some(deadline) => self.recv_deadline(deadline),\n None => self.recv().map_err(RecvTimeoutError::from),\n }\n }\n\n /// Waits for a message to be received from the channel, but only before a given deadline.\n ///\n /// If the channel is empty and not disconnected, this call will block until the receive\n /// operation can proceed or the operation times out. If the channel is empty and becomes\n /// disconnected, this call will wake up and return an error.\n ///\n /// If called on a zero-capacity channel, this method will wait for a send operation to appear\n /// on the other side of the channel.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::{Instant, Duration};\n /// use crossbeam_channel::{unbounded, RecvTimeoutError};\n ///\n /// let (s, r) = unbounded();\n ///\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// s.send(5).unwrap();\n /// drop(s);\n /// });\n ///\n /// let now = Instant::now();\n ///\n /// assert_eq!(\n /// r.recv_deadline(now + Duration::from_millis(500)),\n /// Err(RecvTimeoutError::Timeout),\n /// );\n /// assert_eq!(\n /// r.recv_deadline(now + Duration::from_millis(1500)),\n /// Ok(5),\n /// );\n /// assert_eq!(\n /// r.recv_deadline(now + Duration::from_secs(5)),\n /// Err(RecvTimeoutError::Disconnected),\n /// );\n /// ```\n pub fn recv_deadline(&self, deadline: Instant) -> Result {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.recv(Some(deadline)),\n ReceiverFlavor::List(chan) => chan.recv(Some(deadline)),\n ReceiverFlavor::Zero(chan) => chan.recv(Some(deadline)),\n ReceiverFlavor::At(chan) => {\n let msg = chan.recv(Some(deadline));\n unsafe {\n mem::transmute_copy::<\n Result,\n Result,\n >(&msg)\n }\n }\n ReceiverFlavor::Tick(chan) => {\n let msg = chan.recv(Some(deadline));\n unsafe {\n mem::transmute_copy::<\n Result,\n Result,\n >(&msg)\n }\n }\n ReceiverFlavor::Never(chan) => chan.recv(Some(deadline)),\n }\n }\n\n /// Returns `true` if the channel is empty.\n ///\n /// Note: Zero-capacity channels are always empty.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::unbounded;\n ///\n /// let (s, r) = unbounded();\n ///\n /// assert!(r.is_empty());\n /// s.send(0).unwrap();\n /// assert!(!r.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.is_empty(),\n ReceiverFlavor::List(chan) => chan.is_empty(),\n ReceiverFlavor::Zero(chan) => chan.is_empty(),\n ReceiverFlavor::At(chan) => chan.is_empty(),\n ReceiverFlavor::Tick(chan) => chan.is_empty(),\n ReceiverFlavor::Never(chan) => chan.is_empty(),\n }\n }\n\n /// Returns `true` if the channel is full.\n ///\n /// Note: Zero-capacity channels are always full.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::bounded;\n ///\n /// let (s, r) = bounded(1);\n ///\n /// assert!(!r.is_full());\n /// s.send(0).unwrap();\n /// assert!(r.is_full());\n /// ```\n pub fn is_full(&self) -> bool {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.is_full(),\n ReceiverFlavor::List(chan) => chan.is_full(),\n ReceiverFlavor::Zero(chan) => chan.is_full(),\n ReceiverFlavor::At(chan) => chan.is_full(),\n ReceiverFlavor::Tick(chan) => chan.is_full(),\n ReceiverFlavor::Never(chan) => chan.is_full(),\n }\n }\n\n /// Returns the number of messages in the channel.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::unbounded;\n ///\n /// let (s, r) = unbounded();\n /// assert_eq!(r.len(), 0);\n ///\n /// s.send(1).unwrap();\n /// s.send(2).unwrap();\n /// assert_eq!(r.len(), 2);\n /// ```\n pub fn len(&self) -> usize {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.len(),\n ReceiverFlavor::List(chan) => chan.len(),\n ReceiverFlavor::Zero(chan) => chan.len(),\n ReceiverFlavor::At(chan) => chan.len(),\n ReceiverFlavor::Tick(chan) => chan.len(),\n ReceiverFlavor::Never(chan) => chan.len(),\n }\n }\n\n /// If the channel is bounded, returns its capacity.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{bounded, unbounded};\n ///\n /// let (_, r) = unbounded::();\n /// assert_eq!(r.capacity(), None);\n ///\n /// let (_, r) = bounded::(5);\n /// assert_eq!(r.capacity(), Some(5));\n ///\n /// let (_, r) = bounded::(0);\n /// assert_eq!(r.capacity(), Some(0));\n /// ```\n pub fn capacity(&self) -> Option {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.capacity(),\n ReceiverFlavor::List(chan) => chan.capacity(),\n ReceiverFlavor::Zero(chan) => chan.capacity(),\n ReceiverFlavor::At(chan) => chan.capacity(),\n ReceiverFlavor::Tick(chan) => chan.capacity(),\n ReceiverFlavor::Never(chan) => chan.capacity(),\n }\n }\n\n /// A blocking iterator over messages in the channel.\n ///\n /// Each call to [`next`] blocks waiting for the next message and then returns it. However, if\n /// the channel becomes empty and disconnected, it returns [`None`] without blocking.\n ///\n /// [`next`]: Iterator::next\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use crossbeam_channel::unbounded;\n ///\n /// let (s, r) = unbounded();\n ///\n /// thread::spawn(move || {\n /// s.send(1).unwrap();\n /// s.send(2).unwrap();\n /// s.send(3).unwrap();\n /// drop(s); // Disconnect the channel.\n /// });\n ///\n /// // Collect all messages from the channel.\n /// // Note that the call to `collect` blocks until the sender is dropped.\n /// let v: Vec<_> = r.iter().collect();\n ///\n /// assert_eq!(v, [1, 2, 3]);\n /// ```\n pub fn iter(&self) -> Iter<'_, T> {\n Iter { receiver: self }\n }\n\n /// A non-blocking iterator over messages in the channel.\n ///\n /// Each call to [`next`] returns a message if there is one ready to be received. The iterator\n /// never blocks waiting for the next message.\n ///\n /// [`next`]: Iterator::next\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::Duration;\n /// use crossbeam_channel::unbounded;\n ///\n /// let (s, r) = unbounded::();\n ///\n /// # let t =\n /// thread::spawn(move || {\n /// s.send(1).unwrap();\n /// thread::sleep(Duration::from_secs(1));\n /// s.send(2).unwrap();\n /// thread::sleep(Duration::from_secs(2));\n /// s.send(3).unwrap();\n /// });\n ///\n /// thread::sleep(Duration::from_secs(2));\n ///\n /// // Collect all messages from the channel without blocking.\n /// // The third message hasn't been sent yet so we'll collect only the first two.\n /// let v: Vec<_> = r.try_iter().collect();\n ///\n /// assert_eq!(v, [1, 2]);\n /// # t.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n pub fn try_iter(&self) -> TryIter<'_, T> {\n TryIter { receiver: self }\n }\n\n /// Returns `true` if receivers belong to the same channel.\n ///\n /// # Examples\n ///\n /// ```rust\n /// use crossbeam_channel::unbounded;\n ///\n /// let (_, r) = unbounded::();\n ///\n /// let r2 = r.clone();\n /// assert!(r.same_channel(&r2));\n ///\n /// let (_, r3) = unbounded();\n /// assert!(!r.same_channel(&r3));\n /// ```\n pub fn same_channel(&self, other: &Self) -> bool {\n match (&self.flavor, &other.flavor) {\n (ReceiverFlavor::Array(a), ReceiverFlavor::Array(b)) => a == b,\n (ReceiverFlavor::List(a), ReceiverFlavor::List(b)) => a == b,\n (ReceiverFlavor::Zero(a), ReceiverFlavor::Zero(b)) => a == b,\n (ReceiverFlavor::At(a), ReceiverFlavor::At(b)) => Arc::ptr_eq(a, b),\n (ReceiverFlavor::Tick(a), ReceiverFlavor::Tick(b)) => Arc::ptr_eq(a, b),\n (ReceiverFlavor::Never(_), ReceiverFlavor::Never(_)) => true,\n _ => false,\n }\n }\n\n pub(crate) fn addr(&self) -> usize {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.addr(),\n ReceiverFlavor::List(chan) => chan.addr(),\n ReceiverFlavor::Zero(chan) => chan.addr(),\n ReceiverFlavor::At(chan) => Arc::as_ptr(chan) as usize,\n ReceiverFlavor::Tick(chan) => Arc::as_ptr(chan) as usize,\n ReceiverFlavor::Never(_chan) => 0,\n }\n }\n}\n\nimpl Drop for Receiver {\n fn drop(&mut self) {\n unsafe {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.release(|c| c.disconnect()),\n ReceiverFlavor::List(chan) => chan.release(|c| c.disconnect_receivers()),\n ReceiverFlavor::Zero(chan) => chan.release(|c| c.disconnect()),\n ReceiverFlavor::At(_) => {}\n ReceiverFlavor::Tick(_) => {}\n ReceiverFlavor::Never(_) => {}\n }\n }\n }\n}\n\nimpl Clone for Receiver {\n fn clone(&self) -> Self {\n let flavor = match &self.flavor {\n ReceiverFlavor::Array(chan) => ReceiverFlavor::Array(chan.acquire()),\n ReceiverFlavor::List(chan) => ReceiverFlavor::List(chan.acquire()),\n ReceiverFlavor::Zero(chan) => ReceiverFlavor::Zero(chan.acquire()),\n ReceiverFlavor::At(chan) => ReceiverFlavor::At(chan.clone()),\n ReceiverFlavor::Tick(chan) => ReceiverFlavor::Tick(chan.clone()),\n ReceiverFlavor::Never(_) => ReceiverFlavor::Never(flavors::never::Channel::new()),\n };\n\n Self { flavor }\n }\n}\n\nimpl fmt::Debug for Receiver {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Receiver { .. }\")\n }\n}\n\nimpl<'a, T> IntoIterator for &'a Receiver {\n type Item = T;\n type IntoIter = Iter<'a, T>;\n\n fn into_iter(self) -> Self::IntoIter {\n self.iter()\n }\n}\n\nimpl IntoIterator for Receiver {\n type Item = T;\n type IntoIter = IntoIter;\n\n fn into_iter(self) -> Self::IntoIter {\n IntoIter { receiver: self }\n }\n}\n\n/// A blocking iterator over messages in a channel.\n///\n/// Each call to [`next`] blocks waiting for the next message and then returns it. However, if the\n/// channel becomes empty and disconnected, it returns [`None`] without blocking.\n///\n/// [`next`]: Iterator::next\n///\n/// # Examples\n///\n/// ```\n/// use std::thread;\n/// use crossbeam_channel::unbounded;\n///\n/// let (s, r) = unbounded();\n///\n/// thread::spawn(move || {\n/// s.send(1).unwrap();\n/// s.send(2).unwrap();\n/// s.send(3).unwrap();\n/// drop(s); // Disconnect the channel.\n/// });\n///\n/// // Collect all messages from the channel.\n/// // Note that the call to `collect` blocks until the sender is dropped.\n/// let v: Vec<_> = r.iter().collect();\n///\n/// assert_eq!(v, [1, 2, 3]);\n/// ```\npub struct Iter<'a, T> {\n receiver: &'a Receiver,\n}\n\nimpl FusedIterator for Iter<'_, T> {}\n\nimpl Iterator for Iter<'_, T> {\n type Item = T;\n\n fn next(&mut self) -> Option {\n self.receiver.recv().ok()\n }\n}\n\nimpl fmt::Debug for Iter<'_, T> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Iter { .. }\")\n }\n}\n\n/// A non-blocking iterator over messages in a channel.\n///\n/// Each call to [`next`] returns a message if there is one ready to be received. The iterator\n/// never blocks waiting for the next message.\n///\n/// [`next`]: Iterator::next\n///\n/// # Examples\n///\n/// ```\n/// use std::thread;\n/// use std::time::Duration;\n/// use crossbeam_channel::unbounded;\n///\n/// let (s, r) = unbounded::();\n///\n/// # let t =\n/// thread::spawn(move || {\n/// s.send(1).unwrap();\n/// thread::sleep(Duration::from_secs(1));\n/// s.send(2).unwrap();\n/// thread::sleep(Duration::from_secs(2));\n/// s.send(3).unwrap();\n/// });\n///\n/// thread::sleep(Duration::from_secs(2));\n///\n/// // Collect all messages from the channel without blocking.\n/// // The third message hasn't been sent yet so we'll collect only the first two.\n/// let v: Vec<_> = r.try_iter().collect();\n///\n/// assert_eq!(v, [1, 2]);\n/// # t.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n/// ```\npub struct TryIter<'a, T> {\n receiver: &'a Receiver,\n}\n\nimpl Iterator for TryIter<'_, T> {\n type Item = T;\n\n fn next(&mut self) -> Option {\n self.receiver.try_recv().ok()\n }\n}\n\nimpl fmt::Debug for TryIter<'_, T> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"TryIter { .. }\")\n }\n}\n\n/// A blocking iterator over messages in a channel.\n///\n/// Each call to [`next`] blocks waiting for the next message and then returns it. However, if the\n/// channel becomes empty and disconnected, it returns [`None`] without blocking.\n///\n/// [`next`]: Iterator::next\n///\n/// # Examples\n///\n/// ```\n/// use std::thread;\n/// use crossbeam_channel::unbounded;\n///\n/// let (s, r) = unbounded();\n///\n/// thread::spawn(move || {\n/// s.send(1).unwrap();\n/// s.send(2).unwrap();\n/// s.send(3).unwrap();\n/// drop(s); // Disconnect the channel.\n/// });\n///\n/// // Collect all messages from the channel.\n/// // Note that the call to `collect` blocks until the sender is dropped.\n/// let v: Vec<_> = r.into_iter().collect();\n///\n/// assert_eq!(v, [1, 2, 3]);\n/// ```\npub struct IntoIter {\n receiver: Receiver,\n}\n\nimpl FusedIterator for IntoIter {}\n\nimpl Iterator for IntoIter {\n type Item = T;\n\n fn next(&mut self) -> Option {\n self.receiver.recv().ok()\n }\n}\n\nimpl fmt::Debug for IntoIter {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"IntoIter { .. }\")\n }\n}\n\nimpl SelectHandle for Sender {\n fn try_select(&self, token: &mut Token) -> bool {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.sender().try_select(token),\n SenderFlavor::List(chan) => chan.sender().try_select(token),\n SenderFlavor::Zero(chan) => chan.sender().try_select(token),\n }\n }\n\n fn deadline(&self) -> Option {\n None\n }\n\n fn register(&self, oper: Operation, cx: &Context) -> bool {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.sender().register(oper, cx),\n SenderFlavor::List(chan) => chan.sender().register(oper, cx),\n SenderFlavor::Zero(chan) => chan.sender().register(oper, cx),\n }\n }\n\n fn unregister(&self, oper: Operation) {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.sender().unregister(oper),\n SenderFlavor::List(chan) => chan.sender().unregister(oper),\n SenderFlavor::Zero(chan) => chan.sender().unregister(oper),\n }\n }\n\n fn accept(&self, token: &mut Token, cx: &Context) -> bool {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.sender().accept(token, cx),\n SenderFlavor::List(chan) => chan.sender().accept(token, cx),\n SenderFlavor::Zero(chan) => chan.sender().accept(token, cx),\n }\n }\n\n fn is_ready(&self) -> bool {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.sender().is_ready(),\n SenderFlavor::List(chan) => chan.sender().is_ready(),\n SenderFlavor::Zero(chan) => chan.sender().is_ready(),\n }\n }\n\n fn watch(&self, oper: Operation, cx: &Context) -> bool {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.sender().watch(oper, cx),\n SenderFlavor::List(chan) => chan.sender().watch(oper, cx),\n SenderFlavor::Zero(chan) => chan.sender().watch(oper, cx),\n }\n }\n\n fn unwatch(&self, oper: Operation) {\n match &self.flavor {\n SenderFlavor::Array(chan) => chan.sender().unwatch(oper),\n SenderFlavor::List(chan) => chan.sender().unwatch(oper),\n SenderFlavor::Zero(chan) => chan.sender().unwatch(oper),\n }\n }\n}\n\nimpl SelectHandle for Receiver {\n fn try_select(&self, token: &mut Token) -> bool {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.receiver().try_select(token),\n ReceiverFlavor::List(chan) => chan.receiver().try_select(token),\n ReceiverFlavor::Zero(chan) => chan.receiver().try_select(token),\n ReceiverFlavor::At(chan) => chan.try_select(token),\n ReceiverFlavor::Tick(chan) => chan.try_select(token),\n ReceiverFlavor::Never(chan) => chan.try_select(token),\n }\n }\n\n fn deadline(&self) -> Option {\n match &self.flavor {\n ReceiverFlavor::Array(_) => None,\n ReceiverFlavor::List(_) => None,\n ReceiverFlavor::Zero(_) => None,\n ReceiverFlavor::At(chan) => chan.deadline(),\n ReceiverFlavor::Tick(chan) => chan.deadline(),\n ReceiverFlavor::Never(chan) => chan.deadline(),\n }\n }\n\n fn register(&self, oper: Operation, cx: &Context) -> bool {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.receiver().register(oper, cx),\n ReceiverFlavor::List(chan) => chan.receiver().register(oper, cx),\n ReceiverFlavor::Zero(chan) => chan.receiver().register(oper, cx),\n ReceiverFlavor::At(chan) => chan.register(oper, cx),\n ReceiverFlavor::Tick(chan) => chan.register(oper, cx),\n ReceiverFlavor::Never(chan) => chan.register(oper, cx),\n }\n }\n\n fn unregister(&self, oper: Operation) {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.receiver().unregister(oper),\n ReceiverFlavor::List(chan) => chan.receiver().unregister(oper),\n ReceiverFlavor::Zero(chan) => chan.receiver().unregister(oper),\n ReceiverFlavor::At(chan) => chan.unregister(oper),\n ReceiverFlavor::Tick(chan) => chan.unregister(oper),\n ReceiverFlavor::Never(chan) => chan.unregister(oper),\n }\n }\n\n fn accept(&self, token: &mut Token, cx: &Context) -> bool {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.receiver().accept(token, cx),\n ReceiverFlavor::List(chan) => chan.receiver().accept(token, cx),\n ReceiverFlavor::Zero(chan) => chan.receiver().accept(token, cx),\n ReceiverFlavor::At(chan) => chan.accept(token, cx),\n ReceiverFlavor::Tick(chan) => chan.accept(token, cx),\n ReceiverFlavor::Never(chan) => chan.accept(token, cx),\n }\n }\n\n fn is_ready(&self) -> bool {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.receiver().is_ready(),\n ReceiverFlavor::List(chan) => chan.receiver().is_ready(),\n ReceiverFlavor::Zero(chan) => chan.receiver().is_ready(),\n ReceiverFlavor::At(chan) => chan.is_ready(),\n ReceiverFlavor::Tick(chan) => chan.is_ready(),\n ReceiverFlavor::Never(chan) => chan.is_ready(),\n }\n }\n\n fn watch(&self, oper: Operation, cx: &Context) -> bool {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.receiver().watch(oper, cx),\n ReceiverFlavor::List(chan) => chan.receiver().watch(oper, cx),\n ReceiverFlavor::Zero(chan) => chan.receiver().watch(oper, cx),\n ReceiverFlavor::At(chan) => chan.watch(oper, cx),\n ReceiverFlavor::Tick(chan) => chan.watch(oper, cx),\n ReceiverFlavor::Never(chan) => chan.watch(oper, cx),\n }\n }\n\n fn unwatch(&self, oper: Operation) {\n match &self.flavor {\n ReceiverFlavor::Array(chan) => chan.receiver().unwatch(oper),\n ReceiverFlavor::List(chan) => chan.receiver().unwatch(oper),\n ReceiverFlavor::Zero(chan) => chan.receiver().unwatch(oper),\n ReceiverFlavor::At(chan) => chan.unwatch(oper),\n ReceiverFlavor::Tick(chan) => chan.unwatch(oper),\n ReceiverFlavor::Never(chan) => chan.unwatch(oper),\n }\n }\n}\n\n/// Writes a message into the channel.\npub(crate) unsafe fn write(s: &Sender, token: &mut Token, msg: T) -> Result<(), T> {\n unsafe {\n match &s.flavor {\n SenderFlavor::Array(chan) => chan.write(token, msg),\n SenderFlavor::List(chan) => chan.write(token, msg),\n SenderFlavor::Zero(chan) => chan.write(token, msg),\n }\n }\n}\n\n/// Reads a message from the channel.\npub(crate) unsafe fn read(r: &Receiver, token: &mut Token) -> Result {\n unsafe {\n match &r.flavor {\n ReceiverFlavor::Array(chan) => chan.read(token),\n ReceiverFlavor::List(chan) => chan.read(token),\n ReceiverFlavor::Zero(chan) => chan.read(token),\n ReceiverFlavor::At(chan) => {\n mem::transmute_copy::, Result>(&chan.read(token))\n }\n ReceiverFlavor::Tick(chan) => {\n mem::transmute_copy::, Result>(&chan.read(token))\n }\n ReceiverFlavor::Never(chan) => chan.read(token),\n }\n }\n}\n" }, { "path": "crossbeam-channel/src/context.rs", "content": "//! Thread-local context used in select.\n\nuse alloc::sync::Arc;\nuse core::{\n cell::Cell,\n ptr,\n sync::atomic::{AtomicPtr, AtomicUsize, Ordering},\n};\nuse std::{\n thread::{self, Thread, ThreadId},\n time::Instant,\n};\n\nuse crossbeam_utils::Backoff;\n\nuse crate::select::Selected;\n\n/// Thread-local context used in select.\n// This is a private API that is used by the select macro.\n#[derive(Debug, Clone)]\npub struct Context {\n inner: Arc,\n}\n\n/// Inner representation of `Context`.\n#[derive(Debug)]\nstruct Inner {\n /// Selected operation.\n select: AtomicUsize,\n\n /// A slot into which another thread may store a pointer to its `Packet`.\n packet: AtomicPtr<()>,\n\n /// Thread handle.\n thread: Thread,\n\n /// Thread id.\n thread_id: ThreadId,\n}\n\nimpl Context {\n /// Creates a new context for the duration of the closure.\n #[inline]\n pub fn with(f: F) -> R\n where\n F: FnOnce(&Self) -> R,\n {\n std::thread_local! {\n /// Cached thread-local context.\n static CONTEXT: Cell> = Cell::new(Some(Context::new()));\n }\n\n let mut f = Some(f);\n let mut f = |cx: &Self| -> R {\n let f = f.take().unwrap();\n f(cx)\n };\n\n CONTEXT\n .try_with(|cell| match cell.take() {\n None => f(&Self::new()),\n Some(cx) => {\n cx.reset();\n let res = f(&cx);\n cell.set(Some(cx));\n res\n }\n })\n .unwrap_or_else(|_| f(&Self::new()))\n }\n\n /// Creates a new `Context`.\n #[cold]\n fn new() -> Self {\n Self {\n inner: Arc::new(Inner {\n select: AtomicUsize::new(Selected::Waiting.into()),\n packet: AtomicPtr::new(ptr::null_mut()),\n thread: thread::current(),\n thread_id: thread::current().id(),\n }),\n }\n }\n\n /// Resets `select` and `packet`.\n #[inline]\n fn reset(&self) {\n self.inner\n .select\n .store(Selected::Waiting.into(), Ordering::Release);\n self.inner.packet.store(ptr::null_mut(), Ordering::Release);\n }\n\n /// Attempts to select an operation.\n ///\n /// On failure, the previously selected operation is returned.\n #[inline]\n pub fn try_select(&self, select: Selected) -> Result<(), Selected> {\n self.inner\n .select\n .compare_exchange(\n Selected::Waiting.into(),\n select.into(),\n Ordering::AcqRel,\n Ordering::Acquire,\n )\n .map(|_| ())\n .map_err(|e| e.into())\n }\n\n /// Returns the selected operation.\n #[inline]\n pub fn selected(&self) -> Selected {\n Selected::from(self.inner.select.load(Ordering::Acquire))\n }\n\n /// Stores a packet.\n ///\n /// This method must be called after `try_select` succeeds and there is a packet to provide.\n #[inline]\n pub fn store_packet(&self, packet: *mut ()) {\n if !packet.is_null() {\n self.inner.packet.store(packet, Ordering::Release);\n }\n }\n\n /// Waits until a packet is provided and returns it.\n #[inline]\n pub fn wait_packet(&self) -> *mut () {\n let backoff = Backoff::new();\n loop {\n let packet = self.inner.packet.load(Ordering::Acquire);\n if !packet.is_null() {\n return packet;\n }\n backoff.snooze();\n }\n }\n\n /// Waits until an operation is selected and returns it.\n ///\n /// If the deadline is reached, `Selected::Aborted` will be selected.\n #[inline]\n pub fn wait_until(&self, deadline: Option) -> Selected {\n loop {\n // Check whether an operation has been selected.\n let sel = Selected::from(self.inner.select.load(Ordering::Acquire));\n if sel != Selected::Waiting {\n return sel;\n }\n\n // If there's a deadline, park the current thread until the deadline is reached.\n if let Some(end) = deadline {\n let now = Instant::now();\n\n if now < end {\n thread::park_timeout(end - now);\n } else {\n // The deadline has been reached. Try aborting select.\n return match self.try_select(Selected::Aborted) {\n Ok(()) => Selected::Aborted,\n Err(s) => s,\n };\n }\n } else {\n thread::park();\n }\n }\n }\n\n /// Unparks the thread this context belongs to.\n #[inline]\n pub fn unpark(&self) {\n self.inner.thread.unpark();\n }\n\n /// Returns the id of the thread this context belongs to.\n #[inline]\n pub fn thread_id(&self) -> ThreadId {\n self.inner.thread_id\n }\n}\n" }, { "path": "crossbeam-channel/src/counter.rs", "content": "//! Reference counter for channels.\n\nuse alloc::boxed::Box;\nuse core::{\n ops,\n ptr::NonNull,\n sync::atomic::{AtomicBool, AtomicUsize, Ordering},\n};\nuse std::process;\n\n/// Reference counter internals.\nstruct Counter {\n /// The number of senders associated with the channel.\n senders: AtomicUsize,\n\n /// The number of receivers associated with the channel.\n receivers: AtomicUsize,\n\n /// Set to `true` if the last sender or the last receiver reference deallocates the channel.\n destroy: AtomicBool,\n\n /// The internal channel.\n chan: C,\n}\n\n/// Wraps a channel into the reference counter.\npub(crate) fn new(chan: C) -> (Sender, Receiver) {\n let counter = NonNull::from(Box::leak(Box::new(Counter {\n senders: AtomicUsize::new(1),\n receivers: AtomicUsize::new(1),\n destroy: AtomicBool::new(false),\n chan,\n })));\n let s = Sender { counter };\n let r = Receiver { counter };\n (s, r)\n}\n\n/// The sending side.\npub(crate) struct Sender {\n counter: NonNull>,\n}\n\nimpl Sender {\n /// Returns the internal `Counter`.\n fn counter(&self) -> &Counter {\n unsafe { self.counter.as_ref() }\n }\n\n /// Acquires another sender reference.\n pub(crate) fn acquire(&self) -> Self {\n let count = self.counter().senders.fetch_add(1, Ordering::Relaxed);\n\n // Cloning senders and calling `mem::forget` on the clones could potentially overflow the\n // counter. It's very difficult to recover sensibly from such degenerate scenarios so we\n // just abort when the count becomes very large.\n if count > isize::MAX as usize {\n process::abort();\n }\n\n Self {\n counter: self.counter,\n }\n }\n\n /// Releases the sender reference.\n ///\n /// Function `disconnect` will be called if this is the last sender reference.\n pub(crate) unsafe fn release bool>(&self, disconnect: F) {\n if self.counter().senders.fetch_sub(1, Ordering::AcqRel) == 1 {\n disconnect(&self.counter().chan);\n\n if self.counter().destroy.swap(true, Ordering::AcqRel) {\n drop(unsafe { Box::from_raw(self.counter.as_ptr()) });\n }\n }\n }\n\n pub(crate) fn addr(&self) -> usize {\n self.counter.as_ptr() as usize\n }\n}\n\nimpl ops::Deref for Sender {\n type Target = C;\n\n fn deref(&self) -> &C {\n &self.counter().chan\n }\n}\n\nimpl PartialEq for Sender {\n fn eq(&self, other: &Self) -> bool {\n self.counter == other.counter\n }\n}\n\n/// The receiving side.\npub(crate) struct Receiver {\n counter: NonNull>,\n}\n\nimpl Receiver {\n /// Returns the internal `Counter`.\n fn counter(&self) -> &Counter {\n unsafe { self.counter.as_ref() }\n }\n\n /// Acquires another receiver reference.\n pub(crate) fn acquire(&self) -> Self {\n let count = self.counter().receivers.fetch_add(1, Ordering::Relaxed);\n\n // Cloning receivers and calling `mem::forget` on the clones could potentially overflow the\n // counter. It's very difficult to recover sensibly from such degenerate scenarios so we\n // just abort when the count becomes very large.\n if count > isize::MAX as usize {\n process::abort();\n }\n\n Self {\n counter: self.counter,\n }\n }\n\n /// Releases the receiver reference.\n ///\n /// Function `disconnect` will be called if this is the last receiver reference.\n pub(crate) unsafe fn release bool>(&self, disconnect: F) {\n if self.counter().receivers.fetch_sub(1, Ordering::AcqRel) == 1 {\n disconnect(&self.counter().chan);\n\n if self.counter().destroy.swap(true, Ordering::AcqRel) {\n drop(unsafe { Box::from_raw(self.counter.as_ptr()) });\n }\n }\n }\n\n pub(crate) fn addr(&self) -> usize {\n self.counter.as_ptr() as usize\n }\n}\n\nimpl ops::Deref for Receiver {\n type Target = C;\n\n fn deref(&self) -> &C {\n &self.counter().chan\n }\n}\n\nimpl PartialEq for Receiver {\n fn eq(&self, other: &Self) -> bool {\n self.counter == other.counter\n }\n}\n" }, { "path": "crossbeam-channel/src/err.rs", "content": "use core::fmt;\nuse std::error;\n\n/// An error returned from the [`send`] method.\n///\n/// The message could not be sent because the channel is disconnected.\n///\n/// The error contains the message so it can be recovered.\n///\n/// [`send`]: super::Sender::send\n#[derive(PartialEq, Eq, Clone, Copy)]\npub struct SendError(pub T);\n\n/// An error returned from the [`try_send`] method.\n///\n/// The error contains the message being sent so it can be recovered.\n///\n/// [`try_send`]: super::Sender::try_send\n#[derive(PartialEq, Eq, Clone, Copy)]\npub enum TrySendError {\n /// The message could not be sent because the channel is full.\n ///\n /// If this is a zero-capacity channel, then the error indicates that there was no receiver\n /// available to receive the message at the time.\n Full(T),\n\n /// The message could not be sent because the channel is disconnected.\n Disconnected(T),\n}\n\n/// An error returned from the [`send_timeout`] method.\n///\n/// The error contains the message being sent so it can be recovered.\n///\n/// [`send_timeout`]: super::Sender::send_timeout\n#[derive(PartialEq, Eq, Clone, Copy)]\npub enum SendTimeoutError {\n /// The message could not be sent because the channel is full and the operation timed out.\n ///\n /// If this is a zero-capacity channel, then the error indicates that there was no receiver\n /// available to receive the message and the operation timed out.\n Timeout(T),\n\n /// The message could not be sent because the channel is disconnected.\n Disconnected(T),\n}\n\n/// An error returned from the [`recv`] method.\n///\n/// A message could not be received because the channel is empty and disconnected.\n///\n/// [`recv`]: super::Receiver::recv\n#[derive(PartialEq, Eq, Clone, Copy, Debug)]\npub struct RecvError;\n\n/// An error returned from the [`try_recv`] method.\n///\n/// [`try_recv`]: super::Receiver::try_recv\n#[derive(PartialEq, Eq, Clone, Copy, Debug)]\npub enum TryRecvError {\n /// A message could not be received because the channel is empty.\n ///\n /// If this is a zero-capacity channel, then the error indicates that there was no sender\n /// available to send a message at the time.\n Empty,\n\n /// The message could not be received because the channel is empty and disconnected.\n Disconnected,\n}\n\n/// An error returned from the [`recv_timeout`] method.\n///\n/// [`recv_timeout`]: super::Receiver::recv_timeout\n#[derive(PartialEq, Eq, Clone, Copy, Debug)]\npub enum RecvTimeoutError {\n /// A message could not be received because the channel is empty and the operation timed out.\n ///\n /// If this is a zero-capacity channel, then the error indicates that there was no sender\n /// available to send a message and the operation timed out.\n Timeout,\n\n /// The message could not be received because the channel is empty and disconnected.\n Disconnected,\n}\n\n/// An error returned from the [`try_select`] method.\n///\n/// Failed because none of the channel operations were ready.\n///\n/// [`try_select`]: super::Select::try_select\n#[derive(PartialEq, Eq, Clone, Copy, Debug)]\npub struct TrySelectError;\n\n/// An error returned from the [`select_timeout`] method.\n///\n/// Failed because none of the channel operations became ready before the timeout.\n///\n/// [`select_timeout`]: super::Select::select_timeout\n#[derive(PartialEq, Eq, Clone, Copy, Debug)]\npub struct SelectTimeoutError;\n\n/// An error returned from the [`try_ready`] method.\n///\n/// Failed because none of the channel operations were ready.\n///\n/// [`try_ready`]: super::Select::try_ready\n#[derive(PartialEq, Eq, Clone, Copy, Debug)]\npub struct TryReadyError;\n\n/// An error returned from the [`ready_timeout`] method.\n///\n/// Failed because none of the channel operations became ready before the timeout.\n///\n/// [`ready_timeout`]: super::Select::ready_timeout\n#[derive(PartialEq, Eq, Clone, Copy, Debug)]\npub struct ReadyTimeoutError;\n\nimpl fmt::Debug for SendError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n \"SendError(..)\".fmt(f)\n }\n}\n\nimpl fmt::Display for SendError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n \"sending on a disconnected channel\".fmt(f)\n }\n}\n\nimpl error::Error for SendError {}\n\nimpl SendError {\n /// Unwraps the message.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::unbounded;\n ///\n /// let (s, r) = unbounded();\n /// drop(r);\n ///\n /// if let Err(err) = s.send(\"foo\") {\n /// assert_eq!(err.into_inner(), \"foo\");\n /// }\n /// ```\n pub fn into_inner(self) -> T {\n self.0\n }\n}\n\nimpl fmt::Debug for TrySendError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match *self {\n Self::Full(..) => \"Full(..)\".fmt(f),\n Self::Disconnected(..) => \"Disconnected(..)\".fmt(f),\n }\n }\n}\n\nimpl fmt::Display for TrySendError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match *self {\n Self::Full(..) => \"sending on a full channel\".fmt(f),\n Self::Disconnected(..) => \"sending on a disconnected channel\".fmt(f),\n }\n }\n}\n\nimpl error::Error for TrySendError {}\n\nimpl From> for TrySendError {\n fn from(err: SendError) -> Self {\n match err {\n SendError(t) => Self::Disconnected(t),\n }\n }\n}\n\nimpl TrySendError {\n /// Unwraps the message.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::bounded;\n ///\n /// let (s, r) = bounded(0);\n ///\n /// if let Err(err) = s.try_send(\"foo\") {\n /// assert_eq!(err.into_inner(), \"foo\");\n /// }\n /// ```\n pub fn into_inner(self) -> T {\n match self {\n Self::Full(v) => v,\n Self::Disconnected(v) => v,\n }\n }\n\n /// Returns `true` if the send operation failed because the channel is full.\n pub fn is_full(&self) -> bool {\n matches!(self, Self::Full(_))\n }\n\n /// Returns `true` if the send operation failed because the channel is disconnected.\n pub fn is_disconnected(&self) -> bool {\n matches!(self, Self::Disconnected(_))\n }\n}\n\nimpl fmt::Debug for SendTimeoutError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n \"SendTimeoutError(..)\".fmt(f)\n }\n}\n\nimpl fmt::Display for SendTimeoutError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match *self {\n Self::Timeout(..) => \"timed out waiting on send operation\".fmt(f),\n Self::Disconnected(..) => \"sending on a disconnected channel\".fmt(f),\n }\n }\n}\n\nimpl error::Error for SendTimeoutError {}\n\nimpl From> for SendTimeoutError {\n fn from(err: SendError) -> Self {\n match err {\n SendError(e) => Self::Disconnected(e),\n }\n }\n}\n\nimpl SendTimeoutError {\n /// Unwraps the message.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::time::Duration;\n /// use crossbeam_channel::unbounded;\n ///\n /// let (s, r) = unbounded();\n ///\n /// if let Err(err) = s.send_timeout(\"foo\", Duration::from_secs(1)) {\n /// assert_eq!(err.into_inner(), \"foo\");\n /// }\n /// ```\n pub fn into_inner(self) -> T {\n match self {\n Self::Timeout(v) => v,\n Self::Disconnected(v) => v,\n }\n }\n\n /// Returns `true` if the send operation timed out.\n pub fn is_timeout(&self) -> bool {\n matches!(self, Self::Timeout(_))\n }\n\n /// Returns `true` if the send operation failed because the channel is disconnected.\n pub fn is_disconnected(&self) -> bool {\n matches!(self, Self::Disconnected(_))\n }\n}\n\nimpl fmt::Display for RecvError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n \"receiving on an empty and disconnected channel\".fmt(f)\n }\n}\n\nimpl error::Error for RecvError {}\n\nimpl fmt::Display for TryRecvError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match *self {\n Self::Empty => \"receiving on an empty channel\".fmt(f),\n Self::Disconnected => \"receiving on an empty and disconnected channel\".fmt(f),\n }\n }\n}\n\nimpl error::Error for TryRecvError {}\n\nimpl From for TryRecvError {\n fn from(err: RecvError) -> Self {\n match err {\n RecvError => Self::Disconnected,\n }\n }\n}\n\nimpl TryRecvError {\n /// Returns `true` if the receive operation failed because the channel is empty.\n pub fn is_empty(&self) -> bool {\n matches!(self, Self::Empty)\n }\n\n /// Returns `true` if the receive operation failed because the channel is disconnected.\n pub fn is_disconnected(&self) -> bool {\n matches!(self, Self::Disconnected)\n }\n}\n\nimpl fmt::Display for RecvTimeoutError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match *self {\n Self::Timeout => \"timed out waiting on receive operation\".fmt(f),\n Self::Disconnected => \"channel is empty and disconnected\".fmt(f),\n }\n }\n}\n\nimpl error::Error for RecvTimeoutError {}\n\nimpl From for RecvTimeoutError {\n fn from(err: RecvError) -> Self {\n match err {\n RecvError => Self::Disconnected,\n }\n }\n}\n\nimpl RecvTimeoutError {\n /// Returns `true` if the receive operation timed out.\n pub fn is_timeout(&self) -> bool {\n matches!(self, Self::Timeout)\n }\n\n /// Returns `true` if the receive operation failed because the channel is disconnected.\n pub fn is_disconnected(&self) -> bool {\n matches!(self, Self::Disconnected)\n }\n}\n\nimpl fmt::Display for TrySelectError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n \"all operations in select would block\".fmt(f)\n }\n}\n\nimpl error::Error for TrySelectError {}\n\nimpl fmt::Display for SelectTimeoutError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n \"timed out waiting on select\".fmt(f)\n }\n}\n\nimpl error::Error for SelectTimeoutError {}\n" }, { "path": "crossbeam-channel/src/flavors/array.rs", "content": "//! Bounded channel based on a preallocated array.\n//!\n//! This flavor has a fixed, positive capacity.\n//!\n//! The implementation is based on Dmitry Vyukov's bounded MPMC queue.\n//!\n//! Source:\n//! - \n//! - \n\nuse alloc::boxed::Box;\nuse core::{\n cell::UnsafeCell,\n mem::{self, MaybeUninit},\n ptr,\n sync::atomic::{self, AtomicUsize, Ordering},\n};\nuse std::time::Instant;\n\nuse crossbeam_utils::{Backoff, CachePadded};\n\nuse crate::{\n context::Context,\n err::{RecvTimeoutError, SendTimeoutError, TryRecvError, TrySendError},\n select::{Operation, SelectHandle, Selected, Token},\n waker::SyncWaker,\n};\n\n/// A slot in a channel.\nstruct Slot {\n /// The current stamp.\n stamp: AtomicUsize,\n\n /// The message in this slot.\n msg: UnsafeCell>,\n}\n\n/// The token type for the array flavor.\n#[derive(Debug)]\npub(crate) struct ArrayToken {\n /// Slot to read from or write to.\n slot: *const u8,\n\n /// Stamp to store into the slot after reading or writing.\n stamp: usize,\n}\n\nimpl Default for ArrayToken {\n #[inline]\n fn default() -> Self {\n Self {\n slot: ptr::null(),\n stamp: 0,\n }\n }\n}\n\n/// Bounded channel based on a preallocated array.\npub(crate) struct Channel {\n /// The head of the channel.\n ///\n /// This value is a \"stamp\" consisting of an index into the buffer, a mark bit, and a lap, but\n /// packed into a single `usize`. The lower bits represent the index, while the upper bits\n /// represent the lap. The mark bit in the head is always zero.\n ///\n /// Messages are popped from the head of the channel.\n head: CachePadded,\n\n /// The tail of the channel.\n ///\n /// This value is a \"stamp\" consisting of an index into the buffer, a mark bit, and a lap, but\n /// packed into a single `usize`. The lower bits represent the index, while the upper bits\n /// represent the lap. The mark bit indicates that the channel is disconnected.\n ///\n /// Messages are pushed into the tail of the channel.\n tail: CachePadded,\n\n /// The buffer holding slots.\n buffer: Box<[Slot]>,\n\n /// A stamp with the value of `{ lap: 1, mark: 0, index: 0 }`.\n one_lap: usize,\n\n /// If this bit is set in the tail, that means the channel is disconnected.\n mark_bit: usize,\n\n /// Senders waiting while the channel is full.\n senders: SyncWaker,\n\n /// Receivers waiting while the channel is empty and not disconnected.\n receivers: SyncWaker,\n}\n\nimpl Channel {\n /// Creates a bounded channel of capacity `cap`.\n pub(crate) fn with_capacity(cap: usize) -> Self {\n assert!(cap > 0, \"capacity must be positive\");\n\n // Compute constants `mark_bit` and `one_lap`.\n let mark_bit = (cap + 1).next_power_of_two();\n let one_lap = mark_bit * 2;\n\n // Head is initialized to `{ lap: 0, mark: 0, index: 0 }`.\n let head = 0;\n // Tail is initialized to `{ lap: 0, mark: 0, index: 0 }`.\n let tail = 0;\n\n // Allocate a buffer of `cap` slots initialized\n // with stamps.\n let buffer: Box<[Slot]> = (0..cap)\n .map(|i| {\n // Set the stamp to `{ lap: 0, mark: 0, index: i }`.\n Slot {\n stamp: AtomicUsize::new(i),\n msg: UnsafeCell::new(MaybeUninit::uninit()),\n }\n })\n .collect();\n\n Self {\n buffer,\n one_lap,\n mark_bit,\n head: CachePadded::new(AtomicUsize::new(head)),\n tail: CachePadded::new(AtomicUsize::new(tail)),\n senders: SyncWaker::new(),\n receivers: SyncWaker::new(),\n }\n }\n\n /// Returns a receiver handle to the channel.\n pub(crate) fn receiver(&self) -> Receiver<'_, T> {\n Receiver(self)\n }\n\n /// Returns a sender handle to the channel.\n pub(crate) fn sender(&self) -> Sender<'_, T> {\n Sender(self)\n }\n\n /// Attempts to reserve a slot for sending a message.\n fn start_send(&self, token: &mut Token) -> bool {\n let backoff = Backoff::new();\n let mut tail = self.tail.load(Ordering::Relaxed);\n\n loop {\n // Check if the channel is disconnected.\n if tail & self.mark_bit != 0 {\n token.array.slot = ptr::null();\n token.array.stamp = 0;\n return true;\n }\n\n // Deconstruct the tail.\n let index = tail & (self.mark_bit - 1);\n let lap = tail & !(self.one_lap - 1);\n\n // Inspect the corresponding slot.\n debug_assert!(index < self.buffer.len());\n let slot = unsafe { self.buffer.get_unchecked(index) };\n let stamp = slot.stamp.load(Ordering::Acquire);\n\n // If the tail and the stamp match, we may attempt to push.\n if tail == stamp {\n let new_tail = if index + 1 < self.cap() {\n // Same lap, incremented index.\n // Set to `{ lap: lap, mark: 0, index: index + 1 }`.\n tail + 1\n } else {\n // One lap forward, index wraps around to zero.\n // Set to `{ lap: lap.wrapping_add(1), mark: 0, index: 0 }`.\n lap.wrapping_add(self.one_lap)\n };\n\n // Try moving the tail.\n match self.tail.compare_exchange_weak(\n tail,\n new_tail,\n Ordering::SeqCst,\n Ordering::Relaxed,\n ) {\n Ok(_) => {\n // Prepare the token for the follow-up call to `write`.\n token.array.slot = slot as *const Slot as *const u8;\n token.array.stamp = tail + 1;\n return true;\n }\n Err(t) => {\n tail = t;\n backoff.spin();\n }\n }\n } else if stamp.wrapping_add(self.one_lap) == tail + 1 {\n atomic::fence(Ordering::SeqCst);\n let head = self.head.load(Ordering::Relaxed);\n\n // If the head lags one lap behind the tail as well...\n if head.wrapping_add(self.one_lap) == tail {\n // ...then the channel is full.\n return false;\n }\n\n backoff.spin();\n tail = self.tail.load(Ordering::Relaxed);\n } else {\n // Snooze because we need to wait for the stamp to get updated.\n backoff.snooze();\n tail = self.tail.load(Ordering::Relaxed);\n }\n }\n }\n\n /// Writes a message into the channel.\n pub(crate) unsafe fn write(&self, token: &mut Token, msg: T) -> Result<(), T> {\n // If there is no slot, the channel is disconnected.\n if token.array.slot.is_null() {\n return Err(msg);\n }\n\n let slot: &Slot = unsafe { &*token.array.slot.cast::>() };\n\n // Write the message into the slot and update the stamp.\n unsafe { slot.msg.get().write(MaybeUninit::new(msg)) }\n slot.stamp.store(token.array.stamp, Ordering::Release);\n\n // Wake a sleeping receiver.\n self.receivers.notify();\n Ok(())\n }\n\n /// Attempts to reserve a slot for receiving a message.\n fn start_recv(&self, token: &mut Token) -> bool {\n let backoff = Backoff::new();\n let mut head = self.head.load(Ordering::Relaxed);\n\n loop {\n // Deconstruct the head.\n let index = head & (self.mark_bit - 1);\n let lap = head & !(self.one_lap - 1);\n\n // Inspect the corresponding slot.\n debug_assert!(index < self.buffer.len());\n let slot = unsafe { self.buffer.get_unchecked(index) };\n let stamp = slot.stamp.load(Ordering::Acquire);\n\n // If the stamp is ahead of the head by 1, we may attempt to pop.\n if head + 1 == stamp {\n let new = if index + 1 < self.cap() {\n // Same lap, incremented index.\n // Set to `{ lap: lap, mark: 0, index: index + 1 }`.\n head + 1\n } else {\n // One lap forward, index wraps around to zero.\n // Set to `{ lap: lap.wrapping_add(1), mark: 0, index: 0 }`.\n lap.wrapping_add(self.one_lap)\n };\n\n // Try moving the head.\n match self.head.compare_exchange_weak(\n head,\n new,\n Ordering::SeqCst,\n Ordering::Relaxed,\n ) {\n Ok(_) => {\n // Prepare the token for the follow-up call to `read`.\n token.array.slot = slot as *const Slot as *const u8;\n token.array.stamp = head.wrapping_add(self.one_lap);\n return true;\n }\n Err(h) => {\n head = h;\n backoff.spin();\n }\n }\n } else if stamp == head {\n atomic::fence(Ordering::SeqCst);\n let tail = self.tail.load(Ordering::Relaxed);\n\n // If the tail equals the head, that means the channel is empty.\n if (tail & !self.mark_bit) == head {\n // If the channel is disconnected...\n if tail & self.mark_bit != 0 {\n // ...then receive an error.\n token.array.slot = ptr::null();\n token.array.stamp = 0;\n return true;\n } else {\n // Otherwise, the receive operation is not ready.\n return false;\n }\n }\n\n backoff.spin();\n head = self.head.load(Ordering::Relaxed);\n } else {\n // Snooze because we need to wait for the stamp to get updated.\n backoff.snooze();\n head = self.head.load(Ordering::Relaxed);\n }\n }\n }\n\n /// Reads a message from the channel.\n pub(crate) unsafe fn read(&self, token: &mut Token) -> Result {\n if token.array.slot.is_null() {\n // The channel is disconnected.\n return Err(());\n }\n\n let slot: &Slot = unsafe { &*token.array.slot.cast::>() };\n\n // Read the message from the slot and update the stamp.\n let msg = unsafe { slot.msg.get().read().assume_init() };\n slot.stamp.store(token.array.stamp, Ordering::Release);\n\n // Wake a sleeping sender.\n self.senders.notify();\n Ok(msg)\n }\n\n /// Attempts to send a message into the channel.\n pub(crate) fn try_send(&self, msg: T) -> Result<(), TrySendError> {\n let token = &mut Token::default();\n if self.start_send(token) {\n unsafe { self.write(token, msg).map_err(TrySendError::Disconnected) }\n } else {\n Err(TrySendError::Full(msg))\n }\n }\n\n /// Sends a message into the channel.\n pub(crate) fn send(\n &self,\n msg: T,\n deadline: Option,\n ) -> Result<(), SendTimeoutError> {\n let token = &mut Token::default();\n loop {\n // Try sending a message several times.\n let backoff = Backoff::new();\n loop {\n if self.start_send(token) {\n let res = unsafe { self.write(token, msg) };\n return res.map_err(SendTimeoutError::Disconnected);\n }\n\n if backoff.is_completed() {\n break;\n } else {\n backoff.snooze();\n }\n }\n\n if let Some(d) = deadline {\n if Instant::now() >= d {\n return Err(SendTimeoutError::Timeout(msg));\n }\n }\n\n Context::with(|cx| {\n // Prepare for blocking until a receiver wakes us up.\n let oper = Operation::hook(token);\n self.senders.register(oper, cx);\n\n // Has the channel become ready just now?\n if !self.is_full() || self.is_disconnected() {\n let _ = cx.try_select(Selected::Aborted);\n }\n\n // Block the current thread.\n let sel = cx.wait_until(deadline);\n\n match sel {\n Selected::Waiting => unreachable!(),\n Selected::Aborted | Selected::Disconnected => {\n self.senders.unregister(oper).unwrap();\n }\n Selected::Operation(_) => {}\n }\n });\n }\n }\n\n /// Attempts to receive a message without blocking.\n pub(crate) fn try_recv(&self) -> Result {\n let token = &mut Token::default();\n\n if self.start_recv(token) {\n unsafe { self.read(token).map_err(|_| TryRecvError::Disconnected) }\n } else {\n Err(TryRecvError::Empty)\n }\n }\n\n /// Receives a message from the channel.\n pub(crate) fn recv(&self, deadline: Option) -> Result {\n let token = &mut Token::default();\n loop {\n // Try receiving a message several times.\n let backoff = Backoff::new();\n loop {\n if self.start_recv(token) {\n let res = unsafe { self.read(token) };\n return res.map_err(|_| RecvTimeoutError::Disconnected);\n }\n\n if backoff.is_completed() {\n break;\n } else {\n backoff.snooze();\n }\n }\n\n if let Some(d) = deadline {\n if Instant::now() >= d {\n return Err(RecvTimeoutError::Timeout);\n }\n }\n\n Context::with(|cx| {\n // Prepare for blocking until a sender wakes us up.\n let oper = Operation::hook(token);\n self.receivers.register(oper, cx);\n\n // Has the channel become ready just now?\n if !self.is_empty() || self.is_disconnected() {\n let _ = cx.try_select(Selected::Aborted);\n }\n\n // Block the current thread.\n let sel = cx.wait_until(deadline);\n\n match sel {\n Selected::Waiting => unreachable!(),\n Selected::Aborted | Selected::Disconnected => {\n self.receivers.unregister(oper).unwrap();\n // If the channel was disconnected, we still have to check for remaining\n // messages.\n }\n Selected::Operation(_) => {}\n }\n });\n }\n }\n\n /// Returns the current number of messages inside the channel.\n pub(crate) fn len(&self) -> usize {\n loop {\n // Load the tail, then load the head.\n let tail = self.tail.load(Ordering::SeqCst);\n let head = self.head.load(Ordering::SeqCst);\n\n // If the tail didn't change, we've got consistent values to work with.\n if self.tail.load(Ordering::SeqCst) == tail {\n let hix = head & (self.mark_bit - 1);\n let tix = tail & (self.mark_bit - 1);\n\n return if hix < tix {\n tix - hix\n } else if hix > tix {\n self.cap() - hix + tix\n } else if (tail & !self.mark_bit) == head {\n 0\n } else {\n self.cap()\n };\n }\n }\n }\n\n /// Returns the capacity of the channel.\n #[inline]\n fn cap(&self) -> usize {\n self.buffer.len()\n }\n\n /// Returns the capacity of the channel.\n pub(crate) fn capacity(&self) -> Option {\n Some(self.cap())\n }\n\n /// Disconnects the channel and wakes up all blocked senders and receivers.\n ///\n /// Returns `true` if this call disconnected the channel.\n pub(crate) fn disconnect(&self) -> bool {\n let tail = self.tail.fetch_or(self.mark_bit, Ordering::SeqCst);\n\n if tail & self.mark_bit == 0 {\n self.senders.disconnect();\n self.receivers.disconnect();\n true\n } else {\n false\n }\n }\n\n /// Returns `true` if the channel is disconnected.\n pub(crate) fn is_disconnected(&self) -> bool {\n self.tail.load(Ordering::SeqCst) & self.mark_bit != 0\n }\n\n /// Returns `true` if the channel is empty.\n pub(crate) fn is_empty(&self) -> bool {\n let head = self.head.load(Ordering::SeqCst);\n let tail = self.tail.load(Ordering::SeqCst);\n\n // Is the tail equal to the head?\n //\n // Note: If the head changes just before we load the tail, that means there was a moment\n // when the channel was not empty, so it is safe to just return `false`.\n (tail & !self.mark_bit) == head\n }\n\n /// Returns `true` if the channel is full.\n pub(crate) fn is_full(&self) -> bool {\n let tail = self.tail.load(Ordering::SeqCst);\n let head = self.head.load(Ordering::SeqCst);\n\n // Is the head lagging one lap behind tail?\n //\n // Note: If the tail changes just before we load the head, that means there was a moment\n // when the channel was not full, so it is safe to just return `false`.\n head.wrapping_add(self.one_lap) == tail & !self.mark_bit\n }\n}\n\nimpl Drop for Channel {\n fn drop(&mut self) {\n if mem::needs_drop::() {\n // Get the index of the head.\n let head = *self.head.get_mut();\n let tail = *self.tail.get_mut();\n\n let hix = head & (self.mark_bit - 1);\n let tix = tail & (self.mark_bit - 1);\n\n let len = if hix < tix {\n tix - hix\n } else if hix > tix {\n self.cap() - hix + tix\n } else if (tail & !self.mark_bit) == head {\n 0\n } else {\n self.cap()\n };\n\n // Loop over all slots that hold a message and drop them.\n for i in 0..len {\n // Compute the index of the next slot holding a message.\n let index = if hix + i < self.cap() {\n hix + i\n } else {\n hix + i - self.cap()\n };\n\n unsafe {\n debug_assert!(index < self.buffer.len());\n let slot = self.buffer.get_unchecked_mut(index);\n (*slot.msg.get()).assume_init_drop();\n }\n }\n }\n }\n}\n\n/// Receiver handle to a channel.\npub(crate) struct Receiver<'a, T>(&'a Channel);\n\n/// Sender handle to a channel.\npub(crate) struct Sender<'a, T>(&'a Channel);\n\nimpl SelectHandle for Receiver<'_, T> {\n fn try_select(&self, token: &mut Token) -> bool {\n self.0.start_recv(token)\n }\n\n fn deadline(&self) -> Option {\n None\n }\n\n fn register(&self, oper: Operation, cx: &Context) -> bool {\n self.0.receivers.register(oper, cx);\n self.is_ready()\n }\n\n fn unregister(&self, oper: Operation) {\n self.0.receivers.unregister(oper);\n }\n\n fn accept(&self, token: &mut Token, _cx: &Context) -> bool {\n self.try_select(token)\n }\n\n fn is_ready(&self) -> bool {\n !self.0.is_empty() || self.0.is_disconnected()\n }\n\n fn watch(&self, oper: Operation, cx: &Context) -> bool {\n self.0.receivers.watch(oper, cx);\n self.is_ready()\n }\n\n fn unwatch(&self, oper: Operation) {\n self.0.receivers.unwatch(oper);\n }\n}\n\nimpl SelectHandle for Sender<'_, T> {\n fn try_select(&self, token: &mut Token) -> bool {\n self.0.start_send(token)\n }\n\n fn deadline(&self) -> Option {\n None\n }\n\n fn register(&self, oper: Operation, cx: &Context) -> bool {\n self.0.senders.register(oper, cx);\n self.is_ready()\n }\n\n fn unregister(&self, oper: Operation) {\n self.0.senders.unregister(oper);\n }\n\n fn accept(&self, token: &mut Token, _cx: &Context) -> bool {\n self.try_select(token)\n }\n\n fn is_ready(&self) -> bool {\n !self.0.is_full() || self.0.is_disconnected()\n }\n\n fn watch(&self, oper: Operation, cx: &Context) -> bool {\n self.0.senders.watch(oper, cx);\n self.is_ready()\n }\n\n fn unwatch(&self, oper: Operation) {\n self.0.senders.unwatch(oper);\n }\n}\n" }, { "path": "crossbeam-channel/src/flavors/at.rs", "content": "//! Channel that delivers a message at a certain moment in time.\n//!\n//! Messages cannot be sent into this kind of channel; they are materialized on demand.\n\nuse core::sync::atomic::{AtomicBool, Ordering};\nuse std::{thread, time::Instant};\n\nuse crate::{\n context::Context,\n err::{RecvTimeoutError, TryRecvError},\n select::{Operation, SelectHandle, Token},\n utils,\n};\n\n/// Result of a receive operation.\npub(crate) type AtToken = Option;\n\n/// Channel that delivers a message at a certain moment in time\npub(crate) struct Channel {\n /// The instant at which the message will be delivered.\n delivery_time: Instant,\n\n /// `true` if the message has been received.\n received: AtomicBool,\n}\n\nimpl Channel {\n /// Creates a channel that delivers a message at a certain instant in time.\n #[inline]\n pub(crate) fn new_deadline(when: Instant) -> Self {\n Self {\n delivery_time: when,\n received: AtomicBool::new(false),\n }\n }\n\n /// Attempts to receive a message without blocking.\n #[inline]\n pub(crate) fn try_recv(&self) -> Result {\n // We use relaxed ordering because this is just an optional optimistic check.\n if self.received.load(Ordering::Relaxed) {\n // The message has already been received.\n return Err(TryRecvError::Empty);\n }\n\n if Instant::now() < self.delivery_time {\n // The message was not delivered yet.\n return Err(TryRecvError::Empty);\n }\n\n // Try receiving the message if it is still available.\n if !self.received.swap(true, Ordering::SeqCst) {\n // Success! Return delivery time as the message.\n Ok(self.delivery_time)\n } else {\n // The message was already received.\n Err(TryRecvError::Empty)\n }\n }\n\n /// Receives a message from the channel.\n #[inline]\n pub(crate) fn recv(&self, deadline: Option) -> Result {\n // We use relaxed ordering because this is just an optional optimistic check.\n if self.received.load(Ordering::Relaxed) {\n // The message has already been received.\n utils::sleep_until(deadline);\n return Err(RecvTimeoutError::Timeout);\n }\n\n // Wait until the message is received or the deadline is reached.\n loop {\n let now = Instant::now();\n\n let deadline = match deadline {\n // Check if we can receive the next message.\n _ if now >= self.delivery_time => break,\n // Check if the timeout deadline has been reached.\n Some(d) if now >= d => return Err(RecvTimeoutError::Timeout),\n\n // Sleep until one of the above happens\n Some(d) if d < self.delivery_time => d,\n _ => self.delivery_time,\n };\n\n thread::sleep(deadline - now);\n }\n\n // Try receiving the message if it is still available.\n if !self.received.swap(true, Ordering::SeqCst) {\n // Success! Return the message, which is the instant at which it was delivered.\n Ok(self.delivery_time)\n } else {\n // The message was already received. Block forever.\n utils::sleep_until(None);\n unreachable!()\n }\n }\n\n /// Reads a message from the channel.\n #[inline]\n pub(crate) unsafe fn read(&self, token: &mut Token) -> Result {\n token.at.ok_or(())\n }\n\n /// Returns `true` if the channel is empty.\n #[inline]\n pub(crate) fn is_empty(&self) -> bool {\n // We use relaxed ordering because this is just an optional optimistic check.\n if self.received.load(Ordering::Relaxed) {\n return true;\n }\n\n // If the delivery time hasn't been reached yet, the channel is empty.\n if Instant::now() < self.delivery_time {\n return true;\n }\n\n // The delivery time has been reached. The channel is empty only if the message has already\n // been received.\n self.received.load(Ordering::SeqCst)\n }\n\n /// Returns `true` if the channel is full.\n #[inline]\n pub(crate) fn is_full(&self) -> bool {\n !self.is_empty()\n }\n\n /// Returns the number of messages in the channel.\n #[inline]\n pub(crate) fn len(&self) -> usize {\n usize::from(!self.is_empty())\n }\n\n /// Returns the capacity of the channel.\n #[inline]\n pub(crate) fn capacity(&self) -> Option {\n Some(1)\n }\n}\n\nimpl SelectHandle for Channel {\n #[inline]\n fn try_select(&self, token: &mut Token) -> bool {\n match self.try_recv() {\n Ok(msg) => {\n token.at = Some(msg);\n true\n }\n Err(TryRecvError::Disconnected) => {\n token.at = None;\n true\n }\n Err(TryRecvError::Empty) => false,\n }\n }\n\n #[inline]\n fn deadline(&self) -> Option {\n // We use relaxed ordering because this is just an optional optimistic check.\n if self.received.load(Ordering::Relaxed) {\n None\n } else {\n Some(self.delivery_time)\n }\n }\n\n #[inline]\n fn register(&self, _oper: Operation, _cx: &Context) -> bool {\n self.is_ready()\n }\n\n #[inline]\n fn unregister(&self, _oper: Operation) {}\n\n #[inline]\n fn accept(&self, token: &mut Token, _cx: &Context) -> bool {\n self.try_select(token)\n }\n\n #[inline]\n fn is_ready(&self) -> bool {\n !self.is_empty()\n }\n\n #[inline]\n fn watch(&self, _oper: Operation, _cx: &Context) -> bool {\n self.is_ready()\n }\n\n #[inline]\n fn unwatch(&self, _oper: Operation) {}\n}\n" }, { "path": "crossbeam-channel/src/flavors/list.rs", "content": "//! Unbounded channel implemented as a linked list.\n\nuse alloc::{alloc::handle_alloc_error, boxed::Box};\nuse core::{\n alloc::Layout,\n cell::UnsafeCell,\n marker::PhantomData,\n mem::MaybeUninit,\n ptr,\n sync::atomic::{self, AtomicPtr, AtomicUsize, Ordering},\n};\nuse std::time::Instant;\n\nuse crossbeam_utils::{Backoff, CachePadded};\n\nuse crate::{\n alloc_helper::Global,\n context::Context,\n err::{RecvTimeoutError, SendTimeoutError, TryRecvError, TrySendError},\n select::{Operation, SelectHandle, Selected, Token},\n waker::SyncWaker,\n};\n\n// TODO(stjepang): Once we bump the minimum required Rust version to 1.28 or newer, re-apply the\n// following changes by @kleimkuhler:\n//\n// 1. https://github.com/crossbeam-rs/crossbeam-channel/pull/100\n// 2. https://github.com/crossbeam-rs/crossbeam-channel/pull/101\n\n// Bits indicating the state of a slot:\n// * If a message has been written into the slot, `WRITE` is set.\n// * If a message has been read from the slot, `READ` is set.\n// * If the block is being destroyed, `DESTROY` is set.\nconst WRITE: usize = 1;\nconst READ: usize = 2;\nconst DESTROY: usize = 4;\n\n// Each block covers one \"lap\" of indices.\nconst LAP: usize = 32;\n// The maximum number of messages a block can hold.\nconst BLOCK_CAP: usize = LAP - 1;\n// How many lower bits are reserved for metadata.\nconst SHIFT: usize = 1;\n// Has two different purposes:\n// * If set in head, indicates that the block is not the last one.\n// * If set in tail, indicates that the channel is disconnected.\nconst MARK_BIT: usize = 1;\n\n/// A slot in a block.\nstruct Slot {\n /// The message.\n msg: UnsafeCell>,\n\n /// The state of the slot.\n state: AtomicUsize,\n}\n\nimpl Slot {\n /// Waits until a message is written into the slot.\n fn wait_write(&self) {\n let backoff = Backoff::new();\n while self.state.load(Ordering::Acquire) & WRITE == 0 {\n backoff.snooze();\n }\n }\n}\n\n/// A block in a linked list.\n///\n/// Each block in the list can hold up to `BLOCK_CAP` messages.\nstruct Block {\n /// The next block in the linked list.\n next: AtomicPtr>,\n\n /// Slots for messages.\n slots: [Slot; BLOCK_CAP],\n}\n\nimpl Block {\n const LAYOUT: Layout = {\n let layout = Layout::new::();\n assert!(\n layout.size() != 0,\n \"Block should never be zero-sized, as it has an AtomicPtr field\"\n );\n layout\n };\n\n /// Creates an empty block.\n fn new() -> Box {\n // unsafe { Box::new_zeroed().assume_init() } requires Rust 1.92\n match Global.allocate_zeroed(Self::LAYOUT) {\n Some(ptr) => {\n // SAFETY: This is safe because:\n // [1] `Block::next` (AtomicPtr) may be safely zero initialized.\n // [2] `Block::slots` (Array) may be safely zero initialized because of [3, 4].\n // [3] `Slot::msg` (UnsafeCell) may be safely zero initialized because it\n // holds a MaybeUninit.\n // [4] `Slot::state` (AtomicUsize) may be safely zero initialized.\n unsafe { Box::from_raw(ptr.as_ptr().cast()) }\n }\n // Handle allocation failure\n None => handle_alloc_error(Self::LAYOUT),\n }\n }\n\n /// Waits until the next pointer is set.\n fn wait_next(&self) -> *mut Self {\n let backoff = Backoff::new();\n loop {\n let next = self.next.load(Ordering::Acquire);\n if !next.is_null() {\n return next;\n }\n backoff.snooze();\n }\n }\n\n /// Sets the `DESTROY` bit in slots starting from `start` and destroys the block.\n unsafe fn destroy(this: *mut Self, start: usize) {\n // It is not necessary to set the `DESTROY` bit in the last slot because that slot has\n // begun destruction of the block.\n for i in start..BLOCK_CAP - 1 {\n let slot = unsafe { (*this).slots.get_unchecked(i) };\n\n // Mark the `DESTROY` bit if a thread is still using the slot.\n if slot.state.load(Ordering::Acquire) & READ == 0\n && slot.state.fetch_or(DESTROY, Ordering::AcqRel) & READ == 0\n {\n // If a thread is still using the slot, it will continue destruction of the block.\n return;\n }\n }\n\n // No thread is using the block, now it is safe to destroy it.\n drop(unsafe { Box::from_raw(this) });\n }\n}\n\n/// A position in a channel.\n#[derive(Debug)]\nstruct Position {\n /// The index in the channel.\n index: AtomicUsize,\n\n /// The block in the linked list.\n block: AtomicPtr>,\n}\n\n/// The token type for the list flavor.\n#[derive(Debug)]\npub(crate) struct ListToken {\n /// The block of slots.\n block: *const u8,\n\n /// The offset into the block.\n offset: usize,\n}\n\nimpl Default for ListToken {\n #[inline]\n fn default() -> Self {\n Self {\n block: ptr::null(),\n offset: 0,\n }\n }\n}\n\n/// Unbounded channel implemented as a linked list.\n///\n/// Each message sent into the channel is assigned a sequence number, i.e. an index. Indices are\n/// represented as numbers of type `usize` and wrap on overflow.\n///\n/// Consecutive messages are grouped into blocks in order to put less pressure on the allocator and\n/// improve cache efficiency.\npub(crate) struct Channel {\n /// The head of the channel.\n head: CachePadded>,\n\n /// The tail of the channel.\n tail: CachePadded>,\n\n /// Receivers waiting while the channel is empty and not disconnected.\n receivers: SyncWaker,\n\n /// Indicates that dropping a `Channel` may drop messages of type `T`.\n _marker: PhantomData,\n}\n\nimpl Channel {\n /// Creates a new unbounded channel.\n pub(crate) fn new() -> Self {\n Self {\n head: CachePadded::new(Position {\n block: AtomicPtr::new(ptr::null_mut()),\n index: AtomicUsize::new(0),\n }),\n tail: CachePadded::new(Position {\n block: AtomicPtr::new(ptr::null_mut()),\n index: AtomicUsize::new(0),\n }),\n receivers: SyncWaker::new(),\n _marker: PhantomData,\n }\n }\n\n /// Returns a receiver handle to the channel.\n pub(crate) fn receiver(&self) -> Receiver<'_, T> {\n Receiver(self)\n }\n\n /// Returns a sender handle to the channel.\n pub(crate) fn sender(&self) -> Sender<'_, T> {\n Sender(self)\n }\n\n /// Attempts to reserve a slot for sending a message.\n fn start_send(&self, token: &mut Token) -> bool {\n let backoff = Backoff::new();\n let mut tail = self.tail.index.load(Ordering::Acquire);\n let mut block = self.tail.block.load(Ordering::Acquire);\n let mut next_block = None;\n\n loop {\n // Check if the channel is disconnected.\n if tail & MARK_BIT != 0 {\n token.list.block = ptr::null();\n return true;\n }\n\n // Calculate the offset of the index into the block.\n let offset = (tail >> SHIFT) % LAP;\n\n // If we reached the end of the block, wait until the next one is installed.\n if offset == BLOCK_CAP {\n backoff.snooze();\n tail = self.tail.index.load(Ordering::Acquire);\n block = self.tail.block.load(Ordering::Acquire);\n continue;\n }\n\n // If we're going to have to install the next block, allocate it in advance in order to\n // make the wait for other threads as short as possible.\n if offset + 1 == BLOCK_CAP && next_block.is_none() {\n next_block = Some(Block::::new());\n }\n\n // If this is the first message to be sent into the channel, we need to allocate the\n // first block and install it.\n if block.is_null() {\n let new = Box::into_raw(Block::::new());\n\n if self\n .tail\n .block\n .compare_exchange(block, new, Ordering::Release, Ordering::Relaxed)\n .is_ok()\n {\n self.head.block.store(new, Ordering::Release);\n block = new;\n } else {\n next_block = unsafe { Some(Box::from_raw(new)) };\n tail = self.tail.index.load(Ordering::Acquire);\n block = self.tail.block.load(Ordering::Acquire);\n continue;\n }\n }\n\n let new_tail = tail + (1 << SHIFT);\n\n // Try advancing the tail forward.\n match self.tail.index.compare_exchange_weak(\n tail,\n new_tail,\n Ordering::SeqCst,\n Ordering::Acquire,\n ) {\n Ok(_) => unsafe {\n // If we've reached the end of the block, install the next one.\n if offset + 1 == BLOCK_CAP {\n let next_block = Box::into_raw(next_block.unwrap());\n self.tail.block.store(next_block, Ordering::Release);\n self.tail.index.fetch_add(1 << SHIFT, Ordering::Release);\n (*block).next.store(next_block, Ordering::Release);\n }\n\n token.list.block = block as *const u8;\n token.list.offset = offset;\n return true;\n },\n Err(t) => {\n tail = t;\n block = self.tail.block.load(Ordering::Acquire);\n backoff.spin();\n }\n }\n }\n }\n\n /// Writes a message into the channel.\n pub(crate) unsafe fn write(&self, token: &mut Token, msg: T) -> Result<(), T> {\n // If there is no slot, the channel is disconnected.\n if token.list.block.is_null() {\n return Err(msg);\n }\n\n // Write the message into the slot.\n let block = token.list.block.cast::>();\n let offset = token.list.offset;\n let slot = unsafe { (*block).slots.get_unchecked(offset) };\n unsafe { slot.msg.get().write(MaybeUninit::new(msg)) }\n slot.state.fetch_or(WRITE, Ordering::Release);\n\n // Wake a sleeping receiver.\n self.receivers.notify();\n Ok(())\n }\n\n /// Attempts to reserve a slot for receiving a message.\n fn start_recv(&self, token: &mut Token) -> bool {\n let backoff = Backoff::new();\n let mut head = self.head.index.load(Ordering::Acquire);\n let mut block = self.head.block.load(Ordering::Acquire);\n\n loop {\n // Calculate the offset of the index into the block.\n let offset = (head >> SHIFT) % LAP;\n\n // If we reached the end of the block, wait until the next one is installed.\n if offset == BLOCK_CAP {\n backoff.snooze();\n head = self.head.index.load(Ordering::Acquire);\n block = self.head.block.load(Ordering::Acquire);\n continue;\n }\n\n let mut new_head = head + (1 << SHIFT);\n\n if new_head & MARK_BIT == 0 {\n atomic::fence(Ordering::SeqCst);\n let tail = self.tail.index.load(Ordering::Relaxed);\n\n // If the tail equals the head, that means the channel is empty.\n if head >> SHIFT == tail >> SHIFT {\n // If the channel is disconnected...\n if tail & MARK_BIT != 0 {\n // ...then receive an error.\n token.list.block = ptr::null();\n return true;\n } else {\n // Otherwise, the receive operation is not ready.\n return false;\n }\n }\n\n // If head and tail are not in the same block, set `MARK_BIT` in head.\n if (head >> SHIFT) / LAP != (tail >> SHIFT) / LAP {\n new_head |= MARK_BIT;\n }\n }\n\n // The block can be null here only if the first message is being sent into the channel.\n // In that case, just wait until it gets initialized.\n if block.is_null() {\n backoff.snooze();\n head = self.head.index.load(Ordering::Acquire);\n block = self.head.block.load(Ordering::Acquire);\n continue;\n }\n\n // Try moving the head index forward.\n match self.head.index.compare_exchange_weak(\n head,\n new_head,\n Ordering::SeqCst,\n Ordering::Acquire,\n ) {\n Ok(_) => unsafe {\n // If we've reached the end of the block, move to the next one.\n if offset + 1 == BLOCK_CAP {\n let next = (*block).wait_next();\n let mut next_index = (new_head & !MARK_BIT).wrapping_add(1 << SHIFT);\n if !(*next).next.load(Ordering::Relaxed).is_null() {\n next_index |= MARK_BIT;\n }\n\n self.head.block.store(next, Ordering::Release);\n self.head.index.store(next_index, Ordering::Release);\n }\n\n token.list.block = block as *const u8;\n token.list.offset = offset;\n return true;\n },\n Err(h) => {\n head = h;\n block = self.head.block.load(Ordering::Acquire);\n backoff.spin();\n }\n }\n }\n }\n\n /// Reads a message from the channel.\n pub(crate) unsafe fn read(&self, token: &mut Token) -> Result {\n if token.list.block.is_null() {\n // The channel is disconnected.\n return Err(());\n }\n\n // Read the message.\n let block = token.list.block as *mut Block;\n let offset = token.list.offset;\n let slot = unsafe { (*block).slots.get_unchecked(offset) };\n slot.wait_write();\n let msg = unsafe { slot.msg.get().read().assume_init() };\n\n // Destroy the block if we've reached the end, or if another thread wanted to destroy but\n // couldn't because we were busy reading from the slot.\n unsafe {\n if offset + 1 == BLOCK_CAP {\n Block::destroy(block, 0);\n } else if slot.state.fetch_or(READ, Ordering::AcqRel) & DESTROY != 0 {\n Block::destroy(block, offset + 1);\n }\n }\n\n Ok(msg)\n }\n\n /// Attempts to send a message into the channel.\n pub(crate) fn try_send(&self, msg: T) -> Result<(), TrySendError> {\n self.send(msg, None).map_err(|err| match err {\n SendTimeoutError::Disconnected(msg) => TrySendError::Disconnected(msg),\n SendTimeoutError::Timeout(_) => unreachable!(),\n })\n }\n\n /// Sends a message into the channel.\n pub(crate) fn send(\n &self,\n msg: T,\n _deadline: Option,\n ) -> Result<(), SendTimeoutError> {\n let token = &mut Token::default();\n assert!(self.start_send(token));\n unsafe {\n self.write(token, msg)\n .map_err(SendTimeoutError::Disconnected)\n }\n }\n\n /// Attempts to receive a message without blocking.\n pub(crate) fn try_recv(&self) -> Result {\n let token = &mut Token::default();\n\n if self.start_recv(token) {\n unsafe { self.read(token).map_err(|_| TryRecvError::Disconnected) }\n } else {\n Err(TryRecvError::Empty)\n }\n }\n\n /// Receives a message from the channel.\n pub(crate) fn recv(&self, deadline: Option) -> Result {\n let token = &mut Token::default();\n loop {\n // Try receiving a message several times.\n let backoff = Backoff::new();\n loop {\n if self.start_recv(token) {\n unsafe {\n return self.read(token).map_err(|_| RecvTimeoutError::Disconnected);\n }\n }\n\n if backoff.is_completed() {\n break;\n } else {\n backoff.snooze();\n }\n }\n\n if let Some(d) = deadline {\n if Instant::now() >= d {\n return Err(RecvTimeoutError::Timeout);\n }\n }\n\n // Prepare for blocking until a sender wakes us up.\n Context::with(|cx| {\n let oper = Operation::hook(token);\n self.receivers.register(oper, cx);\n\n // Has the channel become ready just now?\n if !self.is_empty() || self.is_disconnected() {\n let _ = cx.try_select(Selected::Aborted);\n }\n\n // Block the current thread.\n let sel = cx.wait_until(deadline);\n\n match sel {\n Selected::Waiting => unreachable!(),\n Selected::Aborted | Selected::Disconnected => {\n self.receivers.unregister(oper).unwrap();\n // If the channel was disconnected, we still have to check for remaining\n // messages.\n }\n Selected::Operation(_) => {}\n }\n });\n }\n }\n\n /// Returns the current number of messages inside the channel.\n pub(crate) fn len(&self) -> usize {\n loop {\n // Load the tail index, then load the head index.\n let mut tail = self.tail.index.load(Ordering::SeqCst);\n let mut head = self.head.index.load(Ordering::SeqCst);\n\n // If the tail index didn't change, we've got consistent indices to work with.\n if self.tail.index.load(Ordering::SeqCst) == tail {\n // Erase the lower bits.\n tail &= !((1 << SHIFT) - 1);\n head &= !((1 << SHIFT) - 1);\n\n // Fix up indices if they fall onto block ends.\n if (tail >> SHIFT) & (LAP - 1) == LAP - 1 {\n tail = tail.wrapping_add(1 << SHIFT);\n }\n if (head >> SHIFT) & (LAP - 1) == LAP - 1 {\n head = head.wrapping_add(1 << SHIFT);\n }\n\n // Rotate indices so that head falls into the first block.\n let lap = (head >> SHIFT) / LAP;\n tail = tail.wrapping_sub((lap * LAP) << SHIFT);\n head = head.wrapping_sub((lap * LAP) << SHIFT);\n\n // Remove the lower bits.\n tail >>= SHIFT;\n head >>= SHIFT;\n\n // Return the difference minus the number of blocks between tail and head.\n return tail - head - tail / LAP;\n }\n }\n }\n\n /// Returns the capacity of the channel.\n pub(crate) fn capacity(&self) -> Option {\n None\n }\n\n /// Disconnects senders and wakes up all blocked receivers.\n ///\n /// Returns `true` if this call disconnected the channel.\n pub(crate) fn disconnect_senders(&self) -> bool {\n let tail = self.tail.index.fetch_or(MARK_BIT, Ordering::SeqCst);\n\n if tail & MARK_BIT == 0 {\n self.receivers.disconnect();\n true\n } else {\n false\n }\n }\n\n /// Disconnects receivers.\n ///\n /// Returns `true` if this call disconnected the channel.\n pub(crate) fn disconnect_receivers(&self) -> bool {\n let tail = self.tail.index.fetch_or(MARK_BIT, Ordering::SeqCst);\n\n if tail & MARK_BIT == 0 {\n // If receivers are dropped first, discard all messages to free\n // memory eagerly.\n self.discard_all_messages();\n true\n } else {\n false\n }\n }\n\n /// Discards all messages.\n ///\n /// This method should only be called when all receivers are dropped.\n fn discard_all_messages(&self) {\n let backoff = Backoff::new();\n let mut tail = self.tail.index.load(Ordering::Acquire);\n loop {\n let offset = (tail >> SHIFT) % LAP;\n if offset != BLOCK_CAP {\n break;\n }\n\n // New updates to tail will be rejected by MARK_BIT and aborted unless it's\n // at boundary. We need to wait for the updates take affect otherwise there\n // can be memory leaks.\n backoff.snooze();\n tail = self.tail.index.load(Ordering::Acquire);\n }\n\n let mut head = self.head.index.load(Ordering::Acquire);\n // The channel may be uninitialized, so we have to swap to avoid overwriting any sender's attempts\n // to initialize the first block before noticing that the receivers disconnected. Late allocations\n // will be deallocated by the sender in Drop\n let mut block = self.head.block.swap(ptr::null_mut(), Ordering::AcqRel);\n\n // If we're going to be dropping messages we need to synchronize with initialization\n if head >> SHIFT != tail >> SHIFT {\n // The block can be null here only if a sender is in the process of initializing the\n // channel while another sender managed to send a message by inserting it into the\n // semi-initialized channel and advanced the tail.\n // In that case, just wait until it gets initialized.\n while block.is_null() {\n backoff.snooze();\n block = self.head.block.swap(ptr::null_mut(), Ordering::AcqRel);\n }\n }\n\n unsafe {\n // Drop all messages between head and tail and deallocate the heap-allocated blocks.\n while head >> SHIFT != tail >> SHIFT {\n let offset = (head >> SHIFT) % LAP;\n\n if offset < BLOCK_CAP {\n // Drop the message in the slot.\n let slot = (*block).slots.get_unchecked(offset);\n slot.wait_write();\n (*slot.msg.get()).assume_init_drop();\n } else {\n (*block).wait_next();\n // Deallocate the block and move to the next one.\n let next = (*block).next.load(Ordering::Acquire);\n drop(Box::from_raw(block));\n block = next;\n }\n\n head = head.wrapping_add(1 << SHIFT);\n }\n\n // Deallocate the last remaining block.\n if !block.is_null() {\n drop(Box::from_raw(block));\n }\n }\n head &= !MARK_BIT;\n self.head.index.store(head, Ordering::Release);\n }\n\n /// Returns `true` if the channel is disconnected.\n pub(crate) fn is_disconnected(&self) -> bool {\n self.tail.index.load(Ordering::SeqCst) & MARK_BIT != 0\n }\n\n /// Returns `true` if the channel is empty.\n pub(crate) fn is_empty(&self) -> bool {\n let head = self.head.index.load(Ordering::SeqCst);\n let tail = self.tail.index.load(Ordering::SeqCst);\n head >> SHIFT == tail >> SHIFT\n }\n\n /// Returns `true` if the channel is full.\n pub(crate) fn is_full(&self) -> bool {\n false\n }\n}\n\nimpl Drop for Channel {\n fn drop(&mut self) {\n let mut head = *self.head.index.get_mut();\n let mut tail = *self.tail.index.get_mut();\n let mut block = *self.head.block.get_mut();\n\n // Erase the lower bits.\n head &= !((1 << SHIFT) - 1);\n tail &= !((1 << SHIFT) - 1);\n\n unsafe {\n // Drop all messages between head and tail and deallocate the heap-allocated blocks.\n while head != tail {\n let offset = (head >> SHIFT) % LAP;\n\n if offset < BLOCK_CAP {\n // Drop the message in the slot.\n let slot = (*block).slots.get_unchecked(offset);\n (*slot.msg.get()).assume_init_drop();\n } else {\n // Deallocate the block and move to the next one.\n let next = *(*block).next.get_mut();\n drop(Box::from_raw(block));\n block = next;\n }\n\n head = head.wrapping_add(1 << SHIFT);\n }\n\n // Deallocate the last remaining block.\n if !block.is_null() {\n drop(Box::from_raw(block));\n }\n }\n }\n}\n\n/// Receiver handle to a channel.\npub(crate) struct Receiver<'a, T>(&'a Channel);\n\n/// Sender handle to a channel.\npub(crate) struct Sender<'a, T>(&'a Channel);\n\nimpl SelectHandle for Receiver<'_, T> {\n fn try_select(&self, token: &mut Token) -> bool {\n self.0.start_recv(token)\n }\n\n fn deadline(&self) -> Option {\n None\n }\n\n fn register(&self, oper: Operation, cx: &Context) -> bool {\n self.0.receivers.register(oper, cx);\n self.is_ready()\n }\n\n fn unregister(&self, oper: Operation) {\n self.0.receivers.unregister(oper);\n }\n\n fn accept(&self, token: &mut Token, _cx: &Context) -> bool {\n self.try_select(token)\n }\n\n fn is_ready(&self) -> bool {\n !self.0.is_empty() || self.0.is_disconnected()\n }\n\n fn watch(&self, oper: Operation, cx: &Context) -> bool {\n self.0.receivers.watch(oper, cx);\n self.is_ready()\n }\n\n fn unwatch(&self, oper: Operation) {\n self.0.receivers.unwatch(oper);\n }\n}\n\nimpl SelectHandle for Sender<'_, T> {\n fn try_select(&self, token: &mut Token) -> bool {\n self.0.start_send(token)\n }\n\n fn deadline(&self) -> Option {\n None\n }\n\n fn register(&self, _oper: Operation, _cx: &Context) -> bool {\n self.is_ready()\n }\n\n fn unregister(&self, _oper: Operation) {}\n\n fn accept(&self, token: &mut Token, _cx: &Context) -> bool {\n self.try_select(token)\n }\n\n fn is_ready(&self) -> bool {\n true\n }\n\n fn watch(&self, _oper: Operation, _cx: &Context) -> bool {\n self.is_ready()\n }\n\n fn unwatch(&self, _oper: Operation) {}\n}\n" }, { "path": "crossbeam-channel/src/flavors/mod.rs", "content": "//! Channel flavors.\n//!\n//! There are six flavors:\n//!\n//! 1. `at` - Channel that delivers a message after a certain amount of time.\n//! 2. `array` - Bounded channel based on a preallocated array.\n//! 3. `list` - Unbounded channel implemented as a linked list.\n//! 4. `never` - Channel that never delivers messages.\n//! 5. `tick` - Channel that delivers messages periodically.\n//! 6. `zero` - Zero-capacity channel.\n\npub(crate) mod array;\npub(crate) mod at;\npub(crate) mod list;\npub(crate) mod never;\npub(crate) mod tick;\npub(crate) mod zero;\n" }, { "path": "crossbeam-channel/src/flavors/never.rs", "content": "//! Channel that never delivers messages.\n//!\n//! Messages cannot be sent into this kind of channel.\n\nuse core::marker::PhantomData;\nuse std::time::Instant;\n\nuse crate::{\n context::Context,\n err::{RecvTimeoutError, TryRecvError},\n select::{Operation, SelectHandle, Token},\n utils,\n};\n\n/// This flavor doesn't need a token.\npub(crate) type NeverToken = ();\n\n/// Channel that never delivers messages.\npub(crate) struct Channel {\n _marker: PhantomData,\n}\n\nimpl Channel {\n /// Creates a channel that never delivers messages.\n #[inline]\n pub(crate) fn new() -> Self {\n Self {\n _marker: PhantomData,\n }\n }\n\n /// Attempts to receive a message without blocking.\n #[inline]\n pub(crate) fn try_recv(&self) -> Result {\n Err(TryRecvError::Empty)\n }\n\n /// Receives a message from the channel.\n #[inline]\n pub(crate) fn recv(&self, deadline: Option) -> Result {\n utils::sleep_until(deadline);\n Err(RecvTimeoutError::Timeout)\n }\n\n /// Reads a message from the channel.\n #[inline]\n pub(crate) unsafe fn read(&self, _token: &mut Token) -> Result {\n Err(())\n }\n\n /// Returns `true` if the channel is empty.\n #[inline]\n pub(crate) fn is_empty(&self) -> bool {\n true\n }\n\n /// Returns `true` if the channel is full.\n #[inline]\n pub(crate) fn is_full(&self) -> bool {\n true\n }\n\n /// Returns the number of messages in the channel.\n #[inline]\n pub(crate) fn len(&self) -> usize {\n 0\n }\n\n /// Returns the capacity of the channel.\n #[inline]\n pub(crate) fn capacity(&self) -> Option {\n Some(0)\n }\n}\n\nimpl SelectHandle for Channel {\n #[inline]\n fn try_select(&self, _token: &mut Token) -> bool {\n false\n }\n\n #[inline]\n fn deadline(&self) -> Option {\n None\n }\n\n #[inline]\n fn register(&self, _oper: Operation, _cx: &Context) -> bool {\n self.is_ready()\n }\n\n #[inline]\n fn unregister(&self, _oper: Operation) {}\n\n #[inline]\n fn accept(&self, token: &mut Token, _cx: &Context) -> bool {\n self.try_select(token)\n }\n\n #[inline]\n fn is_ready(&self) -> bool {\n false\n }\n\n #[inline]\n fn watch(&self, _oper: Operation, _cx: &Context) -> bool {\n self.is_ready()\n }\n\n #[inline]\n fn unwatch(&self, _oper: Operation) {}\n}\n" }, { "path": "crossbeam-channel/src/flavors/tick.rs", "content": "//! Channel that delivers messages periodically.\n//!\n//! Messages cannot be sent into this kind of channel; they are materialized on demand.\n\nuse core::time::Duration;\nuse std::{thread, time::Instant};\n\nuse crossbeam_utils::atomic::AtomicCell;\n\nuse crate::{\n context::Context,\n err::{RecvTimeoutError, TryRecvError},\n select::{Operation, SelectHandle, Token},\n};\n\n/// Result of a receive operation.\npub(crate) type TickToken = Option;\n\n#[derive(Clone, Copy, PartialEq, Eq)]\n#[repr(C)]\n#[cfg_attr(\n any(\n // List of target_arch that can have 128-bit atomics: https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/atomic128/README.md\n target_arch = \"x86_64\",\n target_arch = \"aarch64\",\n target_arch = \"arm64ec\",\n target_arch = \"riscv64\",\n target_arch = \"powerpc64\",\n target_arch = \"s390x\",\n target_arch = \"loongarch64\",\n target_arch = \"mips64r6\",\n target_arch = \"nvptx64\",\n ),\n repr(align(16))\n)]\nstruct Align(T);\n\n#[test]\nfn is_lock_free() {\n assert_eq!(\n core::mem::size_of::(),\n core::mem::size_of::>()\n );\n if cfg!(any(\n all(target_arch = \"x86_64\", target_vendor = \"apple\"),\n target_arch = \"aarch64\",\n )) || rustversion::cfg!(since(1.78)) && cfg!(all(target_arch = \"x86_64\", windows))\n || rustversion::cfg!(since(1.84))\n && cfg!(any(target_arch = \"arm64ec\", target_arch = \"s390x\"))\n || rustversion::cfg!(since(1.95))\n && cfg!(all(target_arch = \"powerpc64\", target_endian = \"little\"))\n {\n assert_eq!(\n AtomicCell::>::is_lock_free(),\n cfg!(not(any(miri, crossbeam_loom, crossbeam_sanitize)))\n );\n }\n}\n\n/// Channel that delivers messages periodically.\npub(crate) struct Channel {\n /// The instant at which the next message will be delivered.\n delivery_time: AtomicCell>,\n\n /// The time interval in which messages get delivered.\n duration: Duration,\n}\n\nimpl Channel {\n /// Creates a channel that delivers messages periodically.\n #[inline]\n pub(crate) fn new(delivery_time: Instant, dur: Duration) -> Self {\n Self {\n delivery_time: AtomicCell::new(Align(delivery_time)),\n duration: dur,\n }\n }\n\n /// Attempts to receive a message without blocking.\n #[inline]\n pub(crate) fn try_recv(&self) -> Result {\n loop {\n let now = Instant::now();\n let delivery_time = self.delivery_time.load();\n\n if now < delivery_time.0 {\n return Err(TryRecvError::Empty);\n }\n\n if self\n .delivery_time\n .compare_exchange(delivery_time, Align(now + self.duration))\n .is_ok()\n {\n return Ok(delivery_time.0);\n }\n }\n }\n\n /// Receives a message from the channel.\n #[inline]\n pub(crate) fn recv(&self, deadline: Option) -> Result {\n loop {\n let delivery_time = self.delivery_time.load();\n let now = Instant::now();\n\n if let Some(d) = deadline {\n if d < delivery_time.0 {\n if now < d {\n thread::sleep(d - now);\n }\n return Err(RecvTimeoutError::Timeout);\n }\n }\n\n if self\n .delivery_time\n .compare_exchange(\n delivery_time,\n Align(delivery_time.0.max(now) + self.duration),\n )\n .is_ok()\n {\n if now < delivery_time.0 {\n thread::sleep(delivery_time.0 - now);\n }\n return Ok(delivery_time.0);\n }\n }\n }\n\n /// Reads a message from the channel.\n #[inline]\n pub(crate) unsafe fn read(&self, token: &mut Token) -> Result {\n token.tick.ok_or(())\n }\n\n /// Returns `true` if the channel is empty.\n #[inline]\n pub(crate) fn is_empty(&self) -> bool {\n Instant::now() < self.delivery_time.load().0\n }\n\n /// Returns `true` if the channel is full.\n #[inline]\n pub(crate) fn is_full(&self) -> bool {\n !self.is_empty()\n }\n\n /// Returns the number of messages in the channel.\n #[inline]\n pub(crate) fn len(&self) -> usize {\n usize::from(!self.is_empty())\n }\n\n /// Returns the capacity of the channel.\n #[inline]\n pub(crate) fn capacity(&self) -> Option {\n Some(1)\n }\n}\n\nimpl SelectHandle for Channel {\n #[inline]\n fn try_select(&self, token: &mut Token) -> bool {\n match self.try_recv() {\n Ok(msg) => {\n token.tick = Some(msg);\n true\n }\n Err(TryRecvError::Disconnected) => {\n token.tick = None;\n true\n }\n Err(TryRecvError::Empty) => false,\n }\n }\n\n #[inline]\n fn deadline(&self) -> Option {\n Some(self.delivery_time.load().0)\n }\n\n #[inline]\n fn register(&self, _oper: Operation, _cx: &Context) -> bool {\n self.is_ready()\n }\n\n #[inline]\n fn unregister(&self, _oper: Operation) {}\n\n #[inline]\n fn accept(&self, token: &mut Token, _cx: &Context) -> bool {\n self.try_select(token)\n }\n\n #[inline]\n fn is_ready(&self) -> bool {\n !self.is_empty()\n }\n\n #[inline]\n fn watch(&self, _oper: Operation, _cx: &Context) -> bool {\n self.is_ready()\n }\n\n #[inline]\n fn unwatch(&self, _oper: Operation) {}\n}\n" }, { "path": "crossbeam-channel/src/flavors/zero.rs", "content": "//! Zero-capacity channel.\n//!\n//! This kind of channel is also known as *rendezvous* channel.\n\nuse alloc::boxed::Box;\nuse core::{\n cell::UnsafeCell,\n fmt,\n marker::PhantomData,\n ptr,\n sync::atomic::{AtomicBool, Ordering},\n};\nuse std::{sync::Mutex, time::Instant};\n\nuse crossbeam_utils::Backoff;\n\nuse crate::{\n context::Context,\n err::{RecvTimeoutError, SendTimeoutError, TryRecvError, TrySendError},\n select::{Operation, SelectHandle, Selected, Token},\n waker::Waker,\n};\n\n/// A pointer to a packet.\npub(crate) struct ZeroToken(*mut ());\n\nimpl Default for ZeroToken {\n fn default() -> Self {\n Self(ptr::null_mut())\n }\n}\n\nimpl fmt::Debug for ZeroToken {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n fmt::Debug::fmt(&(self.0 as usize), f)\n }\n}\n\n/// A slot for passing one message from a sender to a receiver.\nstruct Packet {\n /// Equals `true` if the packet is allocated on the stack.\n on_stack: bool,\n\n /// Equals `true` once the packet is ready for reading or writing.\n ready: AtomicBool,\n\n /// The message.\n msg: UnsafeCell>,\n}\n\nimpl Packet {\n /// Creates an empty packet on the stack.\n fn empty_on_stack() -> Self {\n Self {\n on_stack: true,\n ready: AtomicBool::new(false),\n msg: UnsafeCell::new(None),\n }\n }\n\n /// Creates an empty packet on the heap.\n fn empty_on_heap() -> Box {\n Box::new(Self {\n on_stack: false,\n ready: AtomicBool::new(false),\n msg: UnsafeCell::new(None),\n })\n }\n\n /// Creates a packet on the stack, containing a message.\n fn message_on_stack(msg: T) -> Self {\n Self {\n on_stack: true,\n ready: AtomicBool::new(false),\n msg: UnsafeCell::new(Some(msg)),\n }\n }\n\n /// Waits until the packet becomes ready for reading or writing.\n fn wait_ready(&self) {\n let backoff = Backoff::new();\n while !self.ready.load(Ordering::Acquire) {\n backoff.snooze();\n }\n }\n}\n\n/// Inner representation of a zero-capacity channel.\nstruct Inner {\n /// Senders waiting to pair up with a receive operation.\n senders: Waker,\n\n /// Receivers waiting to pair up with a send operation.\n receivers: Waker,\n\n /// Equals `true` when the channel is disconnected.\n is_disconnected: bool,\n}\n\n/// Zero-capacity channel.\npub(crate) struct Channel {\n /// Inner representation of the channel.\n inner: Mutex,\n\n /// Indicates that dropping a `Channel` may drop values of type `T`.\n _marker: PhantomData,\n}\n\nimpl Channel {\n /// Constructs a new zero-capacity channel.\n pub(crate) fn new() -> Self {\n Self {\n inner: Mutex::new(Inner {\n senders: Waker::new(),\n receivers: Waker::new(),\n is_disconnected: false,\n }),\n _marker: PhantomData,\n }\n }\n\n /// Returns a receiver handle to the channel.\n pub(crate) fn receiver(&self) -> Receiver<'_, T> {\n Receiver(self)\n }\n\n /// Returns a sender handle to the channel.\n pub(crate) fn sender(&self) -> Sender<'_, T> {\n Sender(self)\n }\n\n /// Attempts to reserve a slot for sending a message.\n fn start_send(&self, token: &mut Token) -> bool {\n let mut inner = self.inner.lock().unwrap();\n\n // If there's a waiting receiver, pair up with it.\n if let Some(operation) = inner.receivers.try_select() {\n token.zero.0 = operation.packet;\n true\n } else if inner.is_disconnected {\n token.zero.0 = ptr::null_mut();\n true\n } else {\n false\n }\n }\n\n /// Writes a message into the packet.\n pub(crate) unsafe fn write(&self, token: &mut Token, msg: T) -> Result<(), T> {\n // If there is no packet, the channel is disconnected.\n if token.zero.0.is_null() {\n return Err(msg);\n }\n\n let packet = unsafe { &*(token.zero.0 as *const Packet) };\n unsafe { packet.msg.get().write(Some(msg)) }\n packet.ready.store(true, Ordering::Release);\n Ok(())\n }\n\n /// Attempts to pair up with a sender.\n fn start_recv(&self, token: &mut Token) -> bool {\n let mut inner = self.inner.lock().unwrap();\n\n // If there's a waiting sender, pair up with it.\n if let Some(operation) = inner.senders.try_select() {\n token.zero.0 = operation.packet;\n true\n } else if inner.is_disconnected {\n token.zero.0 = ptr::null_mut();\n true\n } else {\n false\n }\n }\n\n /// Reads a message from the packet.\n pub(crate) unsafe fn read(&self, token: &mut Token) -> Result {\n // If there is no packet, the channel is disconnected.\n if token.zero.0.is_null() {\n return Err(());\n }\n\n let packet = unsafe { &*(token.zero.0 as *const Packet) };\n\n if packet.on_stack {\n // The message has been in the packet from the beginning, so there is no need to wait\n // for it. However, after reading the message, we need to set `ready` to `true` in\n // order to signal that the packet can be destroyed.\n let msg = unsafe { packet.msg.get().replace(None).unwrap() };\n packet.ready.store(true, Ordering::Release);\n Ok(msg)\n } else {\n // Wait until the message becomes available, then read it and destroy the\n // heap-allocated packet.\n packet.wait_ready();\n let msg = unsafe { packet.msg.get().replace(None).unwrap() };\n drop(unsafe { Box::from_raw(token.zero.0.cast::>()) });\n Ok(msg)\n }\n }\n\n /// Attempts to send a message into the channel.\n pub(crate) fn try_send(&self, msg: T) -> Result<(), TrySendError> {\n let token = &mut Token::default();\n let mut inner = self.inner.lock().unwrap();\n\n // If there's a waiting receiver, pair up with it.\n if let Some(operation) = inner.receivers.try_select() {\n token.zero.0 = operation.packet;\n drop(inner);\n unsafe {\n self.write(token, msg).ok().unwrap();\n }\n Ok(())\n } else if inner.is_disconnected {\n Err(TrySendError::Disconnected(msg))\n } else {\n Err(TrySendError::Full(msg))\n }\n }\n\n /// Sends a message into the channel.\n pub(crate) fn send(\n &self,\n msg: T,\n deadline: Option,\n ) -> Result<(), SendTimeoutError> {\n let token = &mut Token::default();\n let mut inner = self.inner.lock().unwrap();\n\n // If there's a waiting receiver, pair up with it.\n if let Some(operation) = inner.receivers.try_select() {\n token.zero.0 = operation.packet;\n drop(inner);\n unsafe {\n self.write(token, msg).ok().unwrap();\n }\n return Ok(());\n }\n\n if inner.is_disconnected {\n return Err(SendTimeoutError::Disconnected(msg));\n }\n\n Context::with(|cx| {\n // Prepare for blocking until a receiver wakes us up.\n let oper = Operation::hook(token);\n let mut packet = Packet::::message_on_stack(msg);\n inner\n .senders\n .register_with_packet(oper, &mut packet as *mut Packet as *mut (), cx);\n inner.receivers.notify();\n drop(inner);\n\n // Block the current thread.\n let sel = cx.wait_until(deadline);\n\n match sel {\n Selected::Waiting => unreachable!(),\n Selected::Aborted => {\n self.inner.lock().unwrap().senders.unregister(oper).unwrap();\n let msg = unsafe { packet.msg.get().replace(None).unwrap() };\n Err(SendTimeoutError::Timeout(msg))\n }\n Selected::Disconnected => {\n self.inner.lock().unwrap().senders.unregister(oper).unwrap();\n let msg = unsafe { packet.msg.get().replace(None).unwrap() };\n Err(SendTimeoutError::Disconnected(msg))\n }\n Selected::Operation(_) => {\n // Wait until the message is read, then drop the packet.\n packet.wait_ready();\n Ok(())\n }\n }\n })\n }\n\n /// Attempts to receive a message without blocking.\n pub(crate) fn try_recv(&self) -> Result {\n let token = &mut Token::default();\n let mut inner = self.inner.lock().unwrap();\n\n // If there's a waiting sender, pair up with it.\n if let Some(operation) = inner.senders.try_select() {\n token.zero.0 = operation.packet;\n drop(inner);\n unsafe { self.read(token).map_err(|_| TryRecvError::Disconnected) }\n } else if inner.is_disconnected {\n Err(TryRecvError::Disconnected)\n } else {\n Err(TryRecvError::Empty)\n }\n }\n\n /// Receives a message from the channel.\n pub(crate) fn recv(&self, deadline: Option) -> Result {\n let token = &mut Token::default();\n let mut inner = self.inner.lock().unwrap();\n\n // If there's a waiting sender, pair up with it.\n if let Some(operation) = inner.senders.try_select() {\n token.zero.0 = operation.packet;\n drop(inner);\n unsafe {\n return self.read(token).map_err(|_| RecvTimeoutError::Disconnected);\n }\n }\n\n if inner.is_disconnected {\n return Err(RecvTimeoutError::Disconnected);\n }\n\n Context::with(|cx| {\n // Prepare for blocking until a sender wakes us up.\n let oper = Operation::hook(token);\n let mut packet = Packet::::empty_on_stack();\n inner.receivers.register_with_packet(\n oper,\n &mut packet as *mut Packet as *mut (),\n cx,\n );\n inner.senders.notify();\n drop(inner);\n\n // Block the current thread.\n let sel = cx.wait_until(deadline);\n\n match sel {\n Selected::Waiting => unreachable!(),\n Selected::Aborted => {\n self.inner\n .lock()\n .unwrap()\n .receivers\n .unregister(oper)\n .unwrap();\n Err(RecvTimeoutError::Timeout)\n }\n Selected::Disconnected => {\n self.inner\n .lock()\n .unwrap()\n .receivers\n .unregister(oper)\n .unwrap();\n Err(RecvTimeoutError::Disconnected)\n }\n Selected::Operation(_) => {\n // Wait until the message is provided, then read it.\n packet.wait_ready();\n unsafe { Ok(packet.msg.get().replace(None).unwrap()) }\n }\n }\n })\n }\n\n /// Disconnects the channel and wakes up all blocked senders and receivers.\n ///\n /// Returns `true` if this call disconnected the channel.\n pub(crate) fn disconnect(&self) -> bool {\n let mut inner = self.inner.lock().unwrap();\n\n if !inner.is_disconnected {\n inner.is_disconnected = true;\n inner.senders.disconnect();\n inner.receivers.disconnect();\n true\n } else {\n false\n }\n }\n\n /// Returns the current number of messages inside the channel.\n pub(crate) fn len(&self) -> usize {\n 0\n }\n\n /// Returns the capacity of the channel.\n pub(crate) fn capacity(&self) -> Option {\n Some(0)\n }\n\n /// Returns `true` if the channel is empty.\n pub(crate) fn is_empty(&self) -> bool {\n true\n }\n\n /// Returns `true` if the channel is full.\n pub(crate) fn is_full(&self) -> bool {\n true\n }\n}\n\n/// Receiver handle to a channel.\npub(crate) struct Receiver<'a, T>(&'a Channel);\n\n/// Sender handle to a channel.\npub(crate) struct Sender<'a, T>(&'a Channel);\n\nimpl SelectHandle for Receiver<'_, T> {\n fn try_select(&self, token: &mut Token) -> bool {\n self.0.start_recv(token)\n }\n\n fn deadline(&self) -> Option {\n None\n }\n\n fn register(&self, oper: Operation, cx: &Context) -> bool {\n let packet = Box::into_raw(Packet::::empty_on_heap());\n\n let mut inner = self.0.inner.lock().unwrap();\n inner\n .receivers\n .register_with_packet(oper, packet.cast::<()>(), cx);\n inner.senders.notify();\n inner.senders.can_select() || inner.is_disconnected\n }\n\n fn unregister(&self, oper: Operation) {\n if let Some(operation) = self.0.inner.lock().unwrap().receivers.unregister(oper) {\n unsafe {\n drop(Box::from_raw(operation.packet.cast::>()));\n }\n }\n }\n\n fn accept(&self, token: &mut Token, cx: &Context) -> bool {\n token.zero.0 = cx.wait_packet();\n true\n }\n\n fn is_ready(&self) -> bool {\n let inner = self.0.inner.lock().unwrap();\n inner.senders.can_select() || inner.is_disconnected\n }\n\n fn watch(&self, oper: Operation, cx: &Context) -> bool {\n let mut inner = self.0.inner.lock().unwrap();\n inner.receivers.watch(oper, cx);\n inner.senders.can_select() || inner.is_disconnected\n }\n\n fn unwatch(&self, oper: Operation) {\n let mut inner = self.0.inner.lock().unwrap();\n inner.receivers.unwatch(oper);\n }\n}\n\nimpl SelectHandle for Sender<'_, T> {\n fn try_select(&self, token: &mut Token) -> bool {\n self.0.start_send(token)\n }\n\n fn deadline(&self) -> Option {\n None\n }\n\n fn register(&self, oper: Operation, cx: &Context) -> bool {\n let packet = Box::into_raw(Packet::::empty_on_heap());\n\n let mut inner = self.0.inner.lock().unwrap();\n inner\n .senders\n .register_with_packet(oper, packet.cast::<()>(), cx);\n inner.receivers.notify();\n inner.receivers.can_select() || inner.is_disconnected\n }\n\n fn unregister(&self, oper: Operation) {\n if let Some(operation) = self.0.inner.lock().unwrap().senders.unregister(oper) {\n unsafe {\n drop(Box::from_raw(operation.packet.cast::>()));\n }\n }\n }\n\n fn accept(&self, token: &mut Token, cx: &Context) -> bool {\n token.zero.0 = cx.wait_packet();\n true\n }\n\n fn is_ready(&self) -> bool {\n let inner = self.0.inner.lock().unwrap();\n inner.receivers.can_select() || inner.is_disconnected\n }\n\n fn watch(&self, oper: Operation, cx: &Context) -> bool {\n let mut inner = self.0.inner.lock().unwrap();\n inner.senders.watch(oper, cx);\n inner.receivers.can_select() || inner.is_disconnected\n }\n\n fn unwatch(&self, oper: Operation) {\n let mut inner = self.0.inner.lock().unwrap();\n inner.senders.unwatch(oper);\n }\n}\n" }, { "path": "crossbeam-channel/src/lib.rs", "content": "//! Multi-producer multi-consumer channels for message passing.\n//!\n//! This crate is an alternative to [`std::sync::mpsc`] with more features and better performance.\n//!\n//! # Hello, world!\n//!\n//! ```\n//! use crossbeam_channel::unbounded;\n//!\n//! // Create a channel of unbounded capacity.\n//! let (s, r) = unbounded();\n//!\n//! // Send a message into the channel.\n//! s.send(\"Hello, world!\").unwrap();\n//!\n//! // Receive the message from the channel.\n//! assert_eq!(r.recv(), Ok(\"Hello, world!\"));\n//! ```\n//!\n//! # Channel types\n//!\n//! Channels can be created using two functions:\n//!\n//! * [`bounded`] creates a channel of bounded capacity, i.e. there is a limit to how many messages\n//! it can hold at a time.\n//!\n//! * [`unbounded`] creates a channel of unbounded capacity, i.e. it can hold any number of\n//! messages at a time.\n//!\n//! Both functions return a [`Sender`] and a [`Receiver`], which represent the two opposite sides\n//! of a channel.\n//!\n//! Creating a bounded channel:\n//!\n//! ```\n//! use crossbeam_channel::bounded;\n//!\n//! // Create a channel that can hold at most 5 messages at a time.\n//! let (s, r) = bounded(5);\n//!\n//! // Can send only 5 messages without blocking.\n//! for i in 0..5 {\n//! s.send(i).unwrap();\n//! }\n//!\n//! // Another call to `send` would block because the channel is full.\n//! // s.send(5).unwrap();\n//! ```\n//!\n//! Creating an unbounded channel:\n//!\n//! ```\n//! use crossbeam_channel::unbounded;\n//!\n//! // Create an unbounded channel.\n//! let (s, r) = unbounded();\n//!\n//! // Can send any number of messages into the channel without blocking.\n//! for i in 0..1000 {\n//! s.send(i).unwrap();\n//! }\n//! ```\n//!\n//! A special case is zero-capacity channel, which cannot hold any messages. Instead, send and\n//! receive operations must appear at the same time in order to pair up and pass the message over:\n//!\n//! ```\n//! use std::thread;\n//! use crossbeam_channel::bounded;\n//!\n//! // Create a zero-capacity channel.\n//! let (s, r) = bounded(0);\n//!\n//! // Sending blocks until a receive operation appears on the other side.\n//! thread::spawn(move || s.send(\"Hi!\").unwrap());\n//!\n//! // Receiving blocks until a send operation appears on the other side.\n//! assert_eq!(r.recv(), Ok(\"Hi!\"));\n//! ```\n//!\n//! # Sharing channels\n//!\n//! Senders and receivers can be cloned and sent to other threads:\n//!\n//! ```\n//! use std::thread;\n//! use crossbeam_channel::bounded;\n//!\n//! let (s1, r1) = bounded(0);\n//! let (s2, r2) = (s1.clone(), r1.clone());\n//!\n//! // Spawn a thread that receives a message and then sends one.\n//! thread::spawn(move || {\n//! r2.recv().unwrap();\n//! s2.send(2).unwrap();\n//! });\n//!\n//! // Send a message and then receive one.\n//! s1.send(1).unwrap();\n//! r1.recv().unwrap();\n//! ```\n//!\n//! Note that cloning only creates a new handle to the same sending or receiving side. It does not\n//! create a separate stream of messages in any way:\n//!\n//! ```\n//! use crossbeam_channel::unbounded;\n//!\n//! let (s1, r1) = unbounded();\n//! let (s2, r2) = (s1.clone(), r1.clone());\n//! let (s3, r3) = (s2.clone(), r2.clone());\n//!\n//! s1.send(10).unwrap();\n//! s2.send(20).unwrap();\n//! s3.send(30).unwrap();\n//!\n//! assert_eq!(r3.recv(), Ok(10));\n//! assert_eq!(r1.recv(), Ok(20));\n//! assert_eq!(r2.recv(), Ok(30));\n//! ```\n//!\n//! It's also possible to share senders and receivers by reference:\n//!\n//! ```\n//! use crossbeam_channel::bounded;\n//! use crossbeam_utils::thread::scope;\n//!\n//! let (s, r) = bounded(0);\n//!\n//! scope(|scope| {\n//! // Spawn a thread that receives a message and then sends one.\n//! scope.spawn(|_| {\n//! r.recv().unwrap();\n//! s.send(2).unwrap();\n//! });\n//!\n//! // Send a message and then receive one.\n//! s.send(1).unwrap();\n//! r.recv().unwrap();\n//! }).unwrap();\n//! ```\n//!\n//! # Disconnection\n//!\n//! When all senders or all receivers associated with a channel get dropped, the channel becomes\n//! disconnected. No more messages can be sent, but any remaining messages can still be received.\n//! Send and receive operations on a disconnected channel never block.\n//!\n//! ```\n//! use crossbeam_channel::{unbounded, RecvError};\n//!\n//! let (s, r) = unbounded();\n//! s.send(1).unwrap();\n//! s.send(2).unwrap();\n//! s.send(3).unwrap();\n//!\n//! // The only sender is dropped, disconnecting the channel.\n//! drop(s);\n//!\n//! // The remaining messages can be received.\n//! assert_eq!(r.recv(), Ok(1));\n//! assert_eq!(r.recv(), Ok(2));\n//! assert_eq!(r.recv(), Ok(3));\n//!\n//! // There are no more messages in the channel.\n//! assert!(r.is_empty());\n//!\n//! // Note that calling `r.recv()` does not block.\n//! // Instead, `Err(RecvError)` is returned immediately.\n//! assert_eq!(r.recv(), Err(RecvError));\n//! ```\n//!\n//! # Blocking operations\n//!\n//! Send and receive operations come in three flavors:\n//!\n//! * Non-blocking (returns immediately with success or failure).\n//! * Blocking (waits until the operation succeeds or the channel becomes disconnected).\n//! * Blocking with a timeout (blocks only for a certain duration of time).\n//!\n//! A simple example showing the difference between non-blocking and blocking operations:\n//!\n//! ```\n//! use crossbeam_channel::{bounded, RecvError, TryRecvError};\n//!\n//! let (s, r) = bounded(1);\n//!\n//! // Send a message into the channel.\n//! s.send(\"foo\").unwrap();\n//!\n//! // This call would block because the channel is full.\n//! // s.send(\"bar\").unwrap();\n//!\n//! // Receive the message.\n//! assert_eq!(r.recv(), Ok(\"foo\"));\n//!\n//! // This call would block because the channel is empty.\n//! // r.recv();\n//!\n//! // Try receiving a message without blocking.\n//! assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n//!\n//! // Disconnect the channel.\n//! drop(s);\n//!\n//! // This call doesn't block because the channel is now disconnected.\n//! assert_eq!(r.recv(), Err(RecvError));\n//! ```\n//!\n//! # Iteration\n//!\n//! Receivers can be used as iterators. For example, method [`iter`] creates an iterator that\n//! receives messages until the channel becomes empty and disconnected. Note that iteration may\n//! block waiting for next message to arrive.\n//!\n//! ```\n//! use std::thread;\n//! use crossbeam_channel::unbounded;\n//!\n//! let (s, r) = unbounded();\n//!\n//! thread::spawn(move || {\n//! s.send(1).unwrap();\n//! s.send(2).unwrap();\n//! s.send(3).unwrap();\n//! drop(s); // Disconnect the channel.\n//! });\n//!\n//! // Collect all messages from the channel.\n//! // Note that the call to `collect` blocks until the sender is dropped.\n//! let v: Vec<_> = r.iter().collect();\n//!\n//! assert_eq!(v, [1, 2, 3]);\n//! ```\n//!\n//! A non-blocking iterator can be created using [`try_iter`], which receives all available\n//! messages without blocking:\n//!\n//! ```\n//! use crossbeam_channel::unbounded;\n//!\n//! let (s, r) = unbounded();\n//! s.send(1).unwrap();\n//! s.send(2).unwrap();\n//! s.send(3).unwrap();\n//! // No need to drop the sender.\n//!\n//! // Receive all messages currently in the channel.\n//! let v: Vec<_> = r.try_iter().collect();\n//!\n//! assert_eq!(v, [1, 2, 3]);\n//! ```\n//!\n//! # Selection\n//!\n//! The [`select!`] macro allows you to define a set of channel operations, wait until any one of\n//! them becomes ready, and finally execute it. If multiple operations are ready at the same time,\n//! a random one among them is selected.\n//!\n//! It is also possible to define a `default` case that gets executed if none of the operations are\n//! ready, either right away or for a certain duration of time.\n//!\n//! An operation is considered to be ready if it doesn't have to block. Note that it is ready even\n//! when it will simply return an error because the channel is disconnected.\n//!\n//! An example of receiving a message from two channels:\n//!\n//! ```\n//! use std::thread;\n//! use std::time::Duration;\n//! use crossbeam_channel::{select, unbounded};\n//!\n//! let (s1, r1) = unbounded();\n//! let (s2, r2) = unbounded();\n//!\n//! # let t1 =\n//! thread::spawn(move || s1.send(10).unwrap());\n//! # let t2 =\n//! thread::spawn(move || s2.send(20).unwrap());\n//!\n//! // At most one of these two receive operations will be executed.\n//! select! {\n//! recv(r1) -> msg => assert_eq!(msg, Ok(10)),\n//! recv(r2) -> msg => assert_eq!(msg, Ok(20)),\n//! default(Duration::from_secs(1)) => println!(\"timed out\"),\n//! }\n//! # t1.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n//! # t2.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n//! ```\n//!\n//! If you need to select over a dynamically created list of channel operations, use [`Select`]\n//! instead. The [`select!`] macro is just a convenience wrapper around [`Select`].\n//!\n//! # Extra channels\n//!\n//! Three functions can create special kinds of channels, all of which return just a [`Receiver`]\n//! handle:\n//!\n//! * [`after`] creates a channel that delivers a single message after a certain duration of time.\n//! * [`tick`] creates a channel that delivers messages periodically.\n//! * [`never`](never()) creates a channel that never delivers messages.\n//!\n//! These channels are very efficient because messages get lazily generated on receive operations.\n//!\n//! An example that prints elapsed time every 50 milliseconds for the duration of 1 second:\n//!\n//! ```\n//! use std::time::{Duration, Instant};\n//! use crossbeam_channel::{after, select, tick};\n//!\n//! let start = Instant::now();\n//! let ticker = tick(Duration::from_millis(50));\n//! let timeout = after(Duration::from_secs(1));\n//!\n//! loop {\n//! select! {\n//! recv(ticker) -> _ => println!(\"elapsed: {:?}\", start.elapsed()),\n//! recv(timeout) -> _ => break,\n//! }\n//! }\n//! ```\n//!\n//! [`send`]: Sender::send\n//! [`recv`]: Receiver::recv\n//! [`iter`]: Receiver::iter\n//! [`try_iter`]: Receiver::try_iter\n\n#![no_std]\n#![doc(test(\n no_crate_inject,\n attr(allow(dead_code, unused_assignments, unused_variables))\n))]\n#![warn(\n missing_docs,\n unsafe_op_in_unsafe_fn,\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\n\n#[cfg(feature = \"std\")]\nextern crate alloc;\n#[cfg(feature = \"std\")]\nextern crate std;\n\n#[cfg(feature = \"std\")]\nmod alloc_helper;\n\n#[cfg(feature = \"std\")]\nmod channel;\n#[cfg(feature = \"std\")]\nmod context;\n#[cfg(feature = \"std\")]\nmod counter;\n#[cfg(feature = \"std\")]\nmod err;\n#[cfg(feature = \"std\")]\nmod flavors;\n#[cfg(feature = \"std\")]\nmod select;\n#[cfg(feature = \"std\")]\nmod select_macro;\n#[cfg(feature = \"std\")]\nmod utils;\n#[cfg(feature = \"std\")]\nmod waker;\n\n/// Crate internals used by the `select!` macro.\n#[doc(hidden)]\n#[cfg(feature = \"std\")]\npub mod internal {\n pub use crate::select::{\n SelectHandle, receiver_addr, select, select_timeout, sender_addr, try_select,\n };\n}\n\n#[cfg(feature = \"std\")]\npub use crate::{\n channel::{\n IntoIter, Iter, Receiver, Sender, TryIter, after, at, bounded, never, tick, unbounded,\n },\n err::{\n ReadyTimeoutError, RecvError, RecvTimeoutError, SelectTimeoutError, SendError,\n SendTimeoutError, TryReadyError, TryRecvError, TrySelectError, TrySendError,\n },\n select::{Select, SelectedOperation},\n};\n" }, { "path": "crossbeam-channel/src/select.rs", "content": "//! Interface to the select mechanism.\n\nuse alloc::vec::Vec;\nuse core::{fmt, marker::PhantomData, mem, time::Duration};\nuse std::time::Instant;\n\nuse crossbeam_utils::Backoff;\n\nuse crate::{\n channel::{self, Receiver, Sender},\n context::Context,\n err::{\n ReadyTimeoutError, RecvError, SelectTimeoutError, SendError, TryReadyError, TrySelectError,\n },\n flavors, utils,\n};\n\n/// Temporary data that gets initialized during select or a blocking operation, and is consumed by\n/// `read` or `write`.\n///\n/// Each field contains data associated with a specific channel flavor.\n// This is a private API that is used by the select macro.\n#[derive(Debug, Default)]\npub struct Token {\n pub(crate) at: flavors::at::AtToken,\n pub(crate) array: flavors::array::ArrayToken,\n pub(crate) list: flavors::list::ListToken,\n #[allow(dead_code)]\n pub(crate) never: flavors::never::NeverToken,\n pub(crate) tick: flavors::tick::TickToken,\n pub(crate) zero: flavors::zero::ZeroToken,\n}\n\n/// Identifier associated with an operation by a specific thread on a specific channel.\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub struct Operation(usize);\n\nimpl Operation {\n /// Creates an operation identifier from a mutable reference.\n ///\n /// This function essentially just turns the address of the reference into a number. The\n /// reference should point to a variable that is specific to the thread and the operation,\n /// and is alive for the entire duration of select or blocking operation.\n #[inline]\n pub fn hook(r: &mut T) -> Self {\n let val = r as *mut T as usize;\n // Make sure that the pointer address doesn't equal the numerical representation of\n // `Selected::{Waiting, Aborted, Disconnected}`.\n assert!(val > 2);\n Self(val)\n }\n}\n\n/// Current state of a select or a blocking operation.\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub enum Selected {\n /// Still waiting for an operation.\n Waiting,\n\n /// The attempt to block the current thread has been aborted.\n Aborted,\n\n /// An operation became ready because a channel is disconnected.\n Disconnected,\n\n /// An operation became ready because a message can be sent or received.\n Operation(Operation),\n}\n\nimpl From for Selected {\n #[inline]\n fn from(val: usize) -> Self {\n match val {\n 0 => Self::Waiting,\n 1 => Self::Aborted,\n 2 => Self::Disconnected,\n oper => Self::Operation(Operation(oper)),\n }\n }\n}\n\nimpl From for usize {\n #[inline]\n fn from(val: Selected) -> Self {\n match val {\n Selected::Waiting => 0,\n Selected::Aborted => 1,\n Selected::Disconnected => 2,\n Selected::Operation(Operation(val)) => val,\n }\n }\n}\n\n/// A receiver or a sender that can participate in select.\n///\n/// This is a handle that assists select in executing an operation, registration, deciding on the\n/// appropriate deadline for blocking, etc.\n// This is a private API (exposed inside crossbeam_channel::internal module) that is used by the select macro.\npub trait SelectHandle {\n /// Attempts to select an operation and returns `true` on success.\n fn try_select(&self, token: &mut Token) -> bool;\n\n /// Returns a deadline for an operation, if there is one.\n fn deadline(&self) -> Option;\n\n /// Registers an operation for execution and returns `true` if it is now ready.\n fn register(&self, oper: Operation, cx: &Context) -> bool;\n\n /// Unregisters an operation for execution.\n fn unregister(&self, oper: Operation);\n\n /// Attempts to select an operation the thread got woken up for and returns `true` on success.\n fn accept(&self, token: &mut Token, cx: &Context) -> bool;\n\n /// Returns `true` if an operation can be executed without blocking.\n fn is_ready(&self) -> bool;\n\n /// Registers an operation for readiness notification and returns `true` if it is now ready.\n fn watch(&self, oper: Operation, cx: &Context) -> bool;\n\n /// Unregisters an operation for readiness notification.\n fn unwatch(&self, oper: Operation);\n}\n\nimpl SelectHandle for &T {\n fn try_select(&self, token: &mut Token) -> bool {\n (**self).try_select(token)\n }\n\n fn deadline(&self) -> Option {\n (**self).deadline()\n }\n\n fn register(&self, oper: Operation, cx: &Context) -> bool {\n (**self).register(oper, cx)\n }\n\n fn unregister(&self, oper: Operation) {\n (**self).unregister(oper);\n }\n\n fn accept(&self, token: &mut Token, cx: &Context) -> bool {\n (**self).accept(token, cx)\n }\n\n fn is_ready(&self) -> bool {\n (**self).is_ready()\n }\n\n fn watch(&self, oper: Operation, cx: &Context) -> bool {\n (**self).watch(oper, cx)\n }\n\n fn unwatch(&self, oper: Operation) {\n (**self).unwatch(oper)\n }\n}\n\n/// Determines when a select operation should time out.\n#[derive(Clone, Copy, Eq, PartialEq)]\nenum Timeout {\n /// No blocking.\n Now,\n\n /// Block forever.\n Never,\n\n /// Time out after the time instant.\n At(Instant),\n}\n\n/// Runs until one of the operations is selected, potentially blocking the current thread.\n///\n/// Successful receive operations will have to be followed up by `channel::read()` and successful\n/// send operations by `channel::write()`.\nfn run_select(\n handles: &mut [(&dyn SelectHandle, usize, usize)],\n timeout: Timeout,\n is_biased: bool,\n) -> Option<(Token, usize, usize)> {\n if handles.is_empty() {\n // Wait until the timeout and return.\n match timeout {\n Timeout::Now => return None,\n Timeout::Never => {\n utils::sleep_until(None);\n unreachable!();\n }\n Timeout::At(when) => {\n utils::sleep_until(Some(when));\n return None;\n }\n }\n }\n\n if !is_biased {\n // Shuffle the operations for fairness.\n utils::shuffle(handles);\n }\n\n // Create a token, which serves as a temporary variable that gets initialized in this function\n // and is later used by a call to `channel::read()` or `channel::write()` that completes the\n // selected operation.\n let mut token = Token::default();\n\n // Try selecting one of the operations without blocking.\n for &(handle, i, addr) in handles.iter() {\n if handle.try_select(&mut token) {\n return Some((token, i, addr));\n }\n }\n\n loop {\n // Prepare for blocking.\n let res = Context::with(|cx| {\n let mut sel = Selected::Waiting;\n let mut registered_count = 0;\n let mut index_ready = None;\n\n if let Timeout::Now = timeout {\n cx.try_select(Selected::Aborted).unwrap();\n }\n\n // Register all operations.\n for (handle, i, _) in handles.iter_mut() {\n registered_count += 1;\n\n // If registration returns `false`, that means the operation has just become ready.\n if handle.register(Operation::hook::<&dyn SelectHandle>(handle), cx) {\n // Try aborting select.\n sel = match cx.try_select(Selected::Aborted) {\n Ok(()) => {\n index_ready = Some(*i);\n Selected::Aborted\n }\n Err(s) => s,\n };\n break;\n }\n\n // If another thread has already selected one of the operations, stop registration.\n sel = cx.selected();\n if sel != Selected::Waiting {\n break;\n }\n }\n\n if sel == Selected::Waiting {\n // Check with each operation for how long we're allowed to block, and compute the\n // earliest deadline.\n let mut deadline: Option = match timeout {\n Timeout::Now => return None,\n Timeout::Never => None,\n Timeout::At(when) => Some(when),\n };\n for &(handle, _, _) in handles.iter() {\n if let Some(x) = handle.deadline() {\n deadline = deadline.map(|y| x.min(y)).or(Some(x));\n }\n }\n\n // Block the current thread.\n sel = cx.wait_until(deadline);\n }\n\n // Unregister all registered operations.\n for (handle, _, _) in handles.iter_mut().take(registered_count) {\n handle.unregister(Operation::hook::<&dyn SelectHandle>(handle));\n }\n\n match sel {\n Selected::Waiting => unreachable!(),\n Selected::Aborted => {\n // If an operation became ready during registration, try selecting it.\n if let Some(index_ready) = index_ready {\n for &(handle, i, addr) in handles.iter() {\n if i == index_ready && handle.try_select(&mut token) {\n return Some((i, addr));\n }\n }\n }\n }\n Selected::Disconnected => {}\n Selected::Operation(_) => {\n // Find the selected operation.\n for (handle, i, addr) in handles.iter_mut() {\n // Is this the selected operation?\n if sel == Selected::Operation(Operation::hook::<&dyn SelectHandle>(handle))\n {\n // Try selecting this operation.\n if handle.accept(&mut token, cx) {\n return Some((*i, *addr));\n }\n }\n }\n }\n }\n\n None\n });\n\n // Return if an operation was selected.\n if let Some((i, addr)) = res {\n return Some((token, i, addr));\n }\n\n // Try selecting one of the operations without blocking.\n for &(handle, i, addr) in handles.iter() {\n if handle.try_select(&mut token) {\n return Some((token, i, addr));\n }\n }\n\n match timeout {\n Timeout::Now => return None,\n Timeout::Never => {}\n Timeout::At(when) => {\n if Instant::now() >= when {\n return None;\n }\n }\n }\n }\n}\n\n/// Runs until one of the operations becomes ready, potentially blocking the current thread.\nfn run_ready(\n handles: &mut [(&dyn SelectHandle, usize, usize)],\n timeout: Timeout,\n is_biased: bool,\n) -> Option {\n if handles.is_empty() {\n // Wait until the timeout and return.\n match timeout {\n Timeout::Now => return None,\n Timeout::Never => {\n utils::sleep_until(None);\n unreachable!();\n }\n Timeout::At(when) => {\n utils::sleep_until(Some(when));\n return None;\n }\n }\n }\n\n if !is_biased {\n // Shuffle the operations for fairness.\n utils::shuffle(handles);\n }\n\n loop {\n let backoff = Backoff::new();\n loop {\n // Check operations for readiness.\n for &(handle, i, _) in handles.iter() {\n if handle.is_ready() {\n return Some(i);\n }\n }\n\n if backoff.is_completed() {\n break;\n } else {\n backoff.snooze();\n }\n }\n\n // Check for timeout.\n match timeout {\n Timeout::Now => return None,\n Timeout::Never => {}\n Timeout::At(when) => {\n if Instant::now() >= when {\n return None;\n }\n }\n }\n\n // Prepare for blocking.\n let res = Context::with(|cx| {\n let mut sel = Selected::Waiting;\n let mut registered_count = 0;\n\n // Begin watching all operations.\n for (handle, _, _) in handles.iter_mut() {\n registered_count += 1;\n let oper = Operation::hook::<&dyn SelectHandle>(handle);\n\n // If registration returns `false`, that means the operation has just become ready.\n if handle.watch(oper, cx) {\n sel = match cx.try_select(Selected::Operation(oper)) {\n Ok(()) => Selected::Operation(oper),\n Err(s) => s,\n };\n break;\n }\n\n // If another thread has already chosen one of the operations, stop registration.\n sel = cx.selected();\n if sel != Selected::Waiting {\n break;\n }\n }\n\n if sel == Selected::Waiting {\n // Check with each operation for how long we're allowed to block, and compute the\n // earliest deadline.\n let mut deadline: Option = match timeout {\n Timeout::Now => unreachable!(),\n Timeout::Never => None,\n Timeout::At(when) => Some(when),\n };\n for &(handle, _, _) in handles.iter() {\n if let Some(x) = handle.deadline() {\n deadline = deadline.map(|y| x.min(y)).or(Some(x));\n }\n }\n\n // Block the current thread.\n sel = cx.wait_until(deadline);\n }\n\n // Unwatch all operations.\n for (handle, _, _) in handles.iter_mut().take(registered_count) {\n handle.unwatch(Operation::hook::<&dyn SelectHandle>(handle));\n }\n\n match sel {\n Selected::Waiting => unreachable!(),\n Selected::Aborted => {}\n Selected::Disconnected => {}\n Selected::Operation(_) => {\n for (handle, i, _) in handles.iter_mut() {\n let oper = Operation::hook::<&dyn SelectHandle>(handle);\n if sel == Selected::Operation(oper) {\n return Some(*i);\n }\n }\n }\n }\n\n None\n });\n\n // Return if an operation became ready.\n if res.is_some() {\n return res;\n }\n }\n}\n\n/// Attempts to select one of the operations without blocking.\n// This is a private API (exposed inside crossbeam_channel::internal module) that is used by the select macro.\n#[inline]\npub fn try_select<'a>(\n handles: &mut [(&'a dyn SelectHandle, usize, usize)],\n is_biased: bool,\n) -> Result, TrySelectError> {\n match run_select(handles, Timeout::Now, is_biased) {\n None => Err(TrySelectError),\n Some((token, index, addr)) => Ok(SelectedOperation {\n token,\n index,\n addr,\n _marker: PhantomData,\n }),\n }\n}\n\n/// Blocks until one of the operations becomes ready and selects it.\n// This is a private API (exposed inside crossbeam_channel::internal module) that is used by the select macro.\n#[inline]\npub fn select<'a>(\n handles: &mut [(&'a dyn SelectHandle, usize, usize)],\n is_biased: bool,\n) -> SelectedOperation<'a> {\n if handles.is_empty() {\n panic!(\"no operations have been added to `Select`\");\n }\n\n let (token, index, addr) = run_select(handles, Timeout::Never, is_biased).unwrap();\n SelectedOperation {\n token,\n index,\n addr,\n _marker: PhantomData,\n }\n}\n\n/// Blocks for a limited time until one of the operations becomes ready and selects it.\n// This is a private API (exposed inside crossbeam_channel::internal module) that is used by the select macro.\n#[inline]\npub fn select_timeout<'a>(\n handles: &mut [(&'a dyn SelectHandle, usize, usize)],\n timeout: Duration,\n is_biased: bool,\n) -> Result, SelectTimeoutError> {\n match Instant::now().checked_add(timeout) {\n Some(deadline) => select_deadline(handles, deadline, is_biased),\n None => Ok(select(handles, is_biased)),\n }\n}\n\n/// Blocks until a given deadline, or until one of the operations becomes ready and selects it.\n#[inline]\npub(crate) fn select_deadline<'a>(\n handles: &mut [(&'a dyn SelectHandle, usize, usize)],\n deadline: Instant,\n is_biased: bool,\n) -> Result, SelectTimeoutError> {\n match run_select(handles, Timeout::At(deadline), is_biased) {\n None => Err(SelectTimeoutError),\n Some((token, index, addr)) => Ok(SelectedOperation {\n token,\n index,\n addr,\n _marker: PhantomData,\n }),\n }\n}\n\n#[doc(hidden)]\npub fn sender_addr(s: &Sender) -> usize {\n s.addr()\n}\n#[doc(hidden)]\npub fn receiver_addr(s: &Receiver) -> usize {\n s.addr()\n}\n\n/// Selects from a set of channel operations.\n///\n/// `Select` allows you to define a set of channel operations, wait until any one of them becomes\n/// ready, and finally execute it. If multiple operations are ready at the same time, a random one\n/// among them is selected.\n///\n/// An operation is considered to be ready if it doesn't have to block. Note that it is ready even\n/// when it will simply return an error because the channel is disconnected.\n///\n/// The [`select!`] macro is a convenience wrapper around `Select`. However, it cannot select over a\n/// dynamically created list of channel operations.\n///\n/// [`select!`]: crate::select!\n///\n/// Once a list of operations has been built with `Select`, there are two different ways of\n/// proceeding:\n///\n/// * Select an operation with [`try_select`], [`select`], or [`select_timeout`]. If successful,\n/// the returned selected operation has already begun and **must** be completed. If we don't\n/// complete it, a panic will occur.\n///\n/// * Wait for an operation to become ready with [`try_ready`], [`ready`], or [`ready_timeout`]. If\n/// successful, we may attempt to execute the operation, but are not obliged to. In fact, it's\n/// possible for another thread to make the operation not ready just before we try executing it,\n/// so it's wise to use a retry loop. However, note that these methods might return with success\n/// spuriously, so it's a good idea to always double check if the operation is really ready.\n///\n/// # Examples\n///\n/// Use [`select`] to receive a message from a list of receivers:\n///\n/// ```\n/// use crossbeam_channel::{Receiver, RecvError, Select};\n///\n/// fn recv_multiple(rs: &[Receiver]) -> Result {\n/// // Build a list of operations.\n/// let mut sel = Select::new();\n/// for r in rs {\n/// sel.recv(r);\n/// }\n///\n/// // Complete the selected operation.\n/// let oper = sel.select();\n/// let index = oper.index();\n/// oper.recv(&rs[index])\n/// }\n/// ```\n///\n/// Use [`ready`] to receive a message from a list of receivers:\n///\n/// ```\n/// use crossbeam_channel::{Receiver, RecvError, Select};\n///\n/// fn recv_multiple(rs: &[Receiver]) -> Result {\n/// // Build a list of operations.\n/// let mut sel = Select::new();\n/// for r in rs {\n/// sel.recv(r);\n/// }\n///\n/// loop {\n/// // Wait until a receive operation becomes ready and try executing it.\n/// let index = sel.ready();\n/// let res = rs[index].try_recv();\n///\n/// // If the operation turns out not to be ready, retry.\n/// if let Err(e) = res {\n/// if e.is_empty() {\n/// continue;\n/// }\n/// }\n///\n/// // Success!\n/// return res.map_err(|_| RecvError);\n/// }\n/// }\n/// ```\n///\n/// [`try_select`]: Select::try_select\n/// [`select`]: Select::select\n/// [`select_timeout`]: Select::select_timeout\n/// [`try_ready`]: Select::try_ready\n/// [`ready`]: Select::ready\n/// [`ready_timeout`]: Select::ready_timeout\npub struct Select<'a> {\n /// A list of senders and receivers participating in selection.\n handles: Vec<(&'a dyn SelectHandle, usize, usize)>,\n\n /// The next index to assign to an operation.\n next_index: usize,\n\n /// Whether to use the index of handles as bias for selecting ready operations.\n biased: bool,\n}\n\nunsafe impl Send for Select<'_> {}\nunsafe impl Sync for Select<'_> {}\n\nimpl<'a> Select<'a> {\n /// Creates an empty list of channel operations for selection.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::Select;\n ///\n /// let mut sel = Select::new();\n ///\n /// // The list of operations is empty, which means no operation can be selected.\n /// assert!(sel.try_select().is_err());\n /// ```\n pub fn new() -> Self {\n Self {\n handles: Vec::with_capacity(4),\n next_index: 0,\n biased: false,\n }\n }\n\n /// Creates an empty list of channel operations with biased selection.\n ///\n /// When multiple handles are ready, this will select the operation with the lowest index.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::Select;\n ///\n /// let mut sel = Select::new_biased();\n ///\n /// // The list of operations is empty, which means no operation can be selected.\n /// assert!(sel.try_select().is_err());\n /// ```\n pub fn new_biased() -> Self {\n Self {\n biased: true,\n ..Default::default()\n }\n }\n\n /// Adds a send operation.\n ///\n /// Returns the index of the added operation.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let (s, r) = unbounded::();\n ///\n /// let mut sel = Select::new();\n /// let index = sel.send(&s);\n /// ```\n pub fn send(&mut self, s: &'a Sender) -> usize {\n let i = self.next_index;\n let addr = s.addr();\n self.handles.push((s, i, addr));\n self.next_index += 1;\n i\n }\n\n /// Adds a receive operation.\n ///\n /// Returns the index of the added operation.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let (s, r) = unbounded::();\n ///\n /// let mut sel = Select::new();\n /// let index = sel.recv(&r);\n /// ```\n pub fn recv(&mut self, r: &'a Receiver) -> usize {\n let i = self.next_index;\n let addr = r.addr();\n self.handles.push((r, i, addr));\n self.next_index += 1;\n i\n }\n\n /// Removes a previously added operation.\n ///\n /// This is useful when an operation is selected because the channel got disconnected and we\n /// want to try again to select a different operation instead.\n ///\n /// If new operations are added after removing some, the indices of removed operations will not\n /// be reused.\n ///\n /// # Panics\n ///\n /// An attempt to remove a non-existing or already removed operation will panic.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let (s1, r1) = unbounded::();\n /// let (_, r2) = unbounded::();\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.recv(&r1);\n /// let oper2 = sel.recv(&r2);\n ///\n /// // Both operations are initially ready, so a random one will be executed.\n /// let oper = sel.select();\n /// assert_eq!(oper.index(), oper2);\n /// assert!(oper.recv(&r2).is_err());\n /// sel.remove(oper2);\n ///\n /// s1.send(10).unwrap();\n ///\n /// let oper = sel.select();\n /// assert_eq!(oper.index(), oper1);\n /// assert_eq!(oper.recv(&r1), Ok(10));\n /// ```\n pub fn remove(&mut self, index: usize) {\n assert!(\n index < self.next_index,\n \"index out of bounds; {} >= {}\",\n index,\n self.next_index,\n );\n\n let i = self\n .handles\n .iter()\n .enumerate()\n .find(|(_, (_, i, _))| *i == index)\n .expect(\"no operation with this index\")\n .0;\n\n self.handles.swap_remove(i);\n }\n\n /// Attempts to select one of the operations without blocking.\n ///\n /// If an operation is ready, it is selected and returned. If multiple operations are ready at\n /// the same time, a random one among them is selected. If none of the operations are ready, an\n /// error is returned.\n ///\n /// An operation is considered to be ready if it doesn't have to block. Note that it is ready\n /// even when it will simply return an error because the channel is disconnected.\n ///\n /// The selected operation must be completed with [`SelectedOperation::send`]\n /// or [`SelectedOperation::recv`].\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let (s1, r1) = unbounded();\n /// let (s2, r2) = unbounded();\n ///\n /// s1.send(10).unwrap();\n /// s2.send(20).unwrap();\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.recv(&r1);\n /// let oper2 = sel.recv(&r2);\n ///\n /// // Both operations are initially ready, so a random one will be executed.\n /// let oper = sel.try_select();\n /// match oper {\n /// Err(_) => panic!(\"both operations should be ready\"),\n /// Ok(oper) => match oper.index() {\n /// i if i == oper1 => assert_eq!(oper.recv(&r1), Ok(10)),\n /// i if i == oper2 => assert_eq!(oper.recv(&r2), Ok(20)),\n /// _ => unreachable!(),\n /// }\n /// }\n /// ```\n pub fn try_select(&mut self) -> Result, TrySelectError> {\n try_select(&mut self.handles, self.biased)\n }\n\n /// Blocks until one of the operations becomes ready and selects it.\n ///\n /// Once an operation becomes ready, it is selected and returned. If multiple operations are\n /// ready at the same time, a random one among them is selected.\n ///\n /// An operation is considered to be ready if it doesn't have to block. Note that it is ready\n /// even when it will simply return an error because the channel is disconnected.\n ///\n /// The selected operation must be completed with [`SelectedOperation::send`]\n /// or [`SelectedOperation::recv`].\n ///\n /// # Panics\n ///\n /// Panics if no operations have been added to `Select`.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::Duration;\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let (s1, r1) = unbounded();\n /// let (s2, r2) = unbounded();\n ///\n /// # let t1 =\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// s1.send(10).unwrap();\n /// });\n /// # let t2 =\n /// thread::spawn(move || s2.send(20).unwrap());\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.recv(&r1);\n /// let oper2 = sel.recv(&r2);\n ///\n /// // The second operation will be selected because it becomes ready first.\n /// let oper = sel.select();\n /// match oper.index() {\n /// i if i == oper1 => assert_eq!(oper.recv(&r1), Ok(10)),\n /// i if i == oper2 => assert_eq!(oper.recv(&r2), Ok(20)),\n /// _ => unreachable!(),\n /// }\n /// # t1.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// # t2.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n pub fn select(&mut self) -> SelectedOperation<'a> {\n select(&mut self.handles, self.biased)\n }\n\n /// Blocks for a limited time until one of the operations becomes ready and selects it.\n ///\n /// If an operation becomes ready, it is selected and returned. If multiple operations are\n /// ready at the same time, a random one among them is selected. If none of the operations\n /// become ready for the specified duration, an error is returned.\n ///\n /// An operation is considered to be ready if it doesn't have to block. Note that it is ready\n /// even when it will simply return an error because the channel is disconnected.\n ///\n /// The selected operation must be completed with [`SelectedOperation::send`]\n /// or [`SelectedOperation::recv`].\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::Duration;\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let (s1, r1) = unbounded();\n /// let (s2, r2) = unbounded();\n ///\n /// # let t1 =\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// s1.send(10).unwrap();\n /// });\n /// # let t2 =\n /// thread::spawn(move || s2.send(20).unwrap());\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.recv(&r1);\n /// let oper2 = sel.recv(&r2);\n ///\n /// // The second operation will be selected because it becomes ready first.\n /// let oper = sel.select_timeout(Duration::from_millis(500));\n /// match oper {\n /// Err(_) => panic!(\"should not have timed out\"),\n /// Ok(oper) => match oper.index() {\n /// i if i == oper1 => assert_eq!(oper.recv(&r1), Ok(10)),\n /// i if i == oper2 => assert_eq!(oper.recv(&r2), Ok(20)),\n /// _ => unreachable!(),\n /// }\n /// }\n /// # t1.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// # t2.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n pub fn select_timeout(\n &mut self,\n timeout: Duration,\n ) -> Result, SelectTimeoutError> {\n select_timeout(&mut self.handles, timeout, self.biased)\n }\n\n /// Blocks until a given deadline, or until one of the operations becomes ready and selects it.\n ///\n /// If an operation becomes ready, it is selected and returned. If multiple operations are\n /// ready at the same time, a random one among them is selected. If none of the operations\n /// become ready before the given deadline, an error is returned.\n ///\n /// An operation is considered to be ready if it doesn't have to block. Note that it is ready\n /// even when it will simply return an error because the channel is disconnected.\n ///\n /// The selected operation must be completed with [`SelectedOperation::send`]\n /// or [`SelectedOperation::recv`].\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::{Instant, Duration};\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let (s1, r1) = unbounded();\n /// let (s2, r2) = unbounded();\n ///\n /// # let t1 =\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// s1.send(10).unwrap();\n /// });\n /// # let t2 =\n /// thread::spawn(move || s2.send(20).unwrap());\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.recv(&r1);\n /// let oper2 = sel.recv(&r2);\n ///\n /// let deadline = Instant::now() + Duration::from_millis(500);\n ///\n /// // The second operation will be selected because it becomes ready first.\n /// let oper = sel.select_deadline(deadline);\n /// match oper {\n /// Err(_) => panic!(\"should not have timed out\"),\n /// Ok(oper) => match oper.index() {\n /// i if i == oper1 => assert_eq!(oper.recv(&r1), Ok(10)),\n /// i if i == oper2 => assert_eq!(oper.recv(&r2), Ok(20)),\n /// _ => unreachable!(),\n /// }\n /// }\n /// # t1.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// # t2.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n pub fn select_deadline(\n &mut self,\n deadline: Instant,\n ) -> Result, SelectTimeoutError> {\n select_deadline(&mut self.handles, deadline, self.biased)\n }\n\n /// Attempts to find a ready operation without blocking.\n ///\n /// If an operation is ready, its index is returned. If multiple operations are ready at the\n /// same time, a random one among them is chosen. If none of the operations are ready, an error\n /// is returned.\n ///\n /// An operation is considered to be ready if it doesn't have to block. Note that it is ready\n /// even when it will simply return an error because the channel is disconnected.\n ///\n /// Note that this method might return with success spuriously, so it's a good idea to always\n /// double check if the operation is really ready.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let (s1, r1) = unbounded();\n /// let (s2, r2) = unbounded();\n ///\n /// s1.send(10).unwrap();\n /// s2.send(20).unwrap();\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.recv(&r1);\n /// let oper2 = sel.recv(&r2);\n ///\n /// // Both operations are initially ready, so a random one will be chosen.\n /// match sel.try_ready() {\n /// Err(_) => panic!(\"both operations should be ready\"),\n /// Ok(i) if i == oper1 => assert_eq!(r1.try_recv(), Ok(10)),\n /// Ok(i) if i == oper2 => assert_eq!(r2.try_recv(), Ok(20)),\n /// Ok(_) => unreachable!(),\n /// }\n /// ```\n pub fn try_ready(&mut self) -> Result {\n match run_ready(&mut self.handles, Timeout::Now, self.biased) {\n None => Err(TryReadyError),\n Some(index) => Ok(index),\n }\n }\n\n /// Blocks until one of the operations becomes ready.\n ///\n /// Once an operation becomes ready, its index is returned. If multiple operations are ready at\n /// the same time, a random one among them is chosen.\n ///\n /// An operation is considered to be ready if it doesn't have to block. Note that it is ready\n /// even when it will simply return an error because the channel is disconnected.\n ///\n /// Note that this method might return with success spuriously, so it's a good idea to always\n /// double check if the operation is really ready.\n ///\n /// # Panics\n ///\n /// Panics if no operations have been added to `Select`.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::Duration;\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let (s1, r1) = unbounded();\n /// let (s2, r2) = unbounded();\n ///\n /// # let t1 =\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// s1.send(10).unwrap();\n /// });\n /// # let t2 =\n /// thread::spawn(move || s2.send(20).unwrap());\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.recv(&r1);\n /// let oper2 = sel.recv(&r2);\n ///\n /// // The second operation will be selected because it becomes ready first.\n /// match sel.ready() {\n /// i if i == oper1 => assert_eq!(r1.try_recv(), Ok(10)),\n /// i if i == oper2 => assert_eq!(r2.try_recv(), Ok(20)),\n /// _ => unreachable!(),\n /// }\n /// # t1.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// # t2.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n pub fn ready(&mut self) -> usize {\n if self.handles.is_empty() {\n panic!(\"no operations have been added to `Select`\");\n }\n\n run_ready(&mut self.handles, Timeout::Never, self.biased).unwrap()\n }\n\n /// Blocks for a limited time until one of the operations becomes ready.\n ///\n /// If an operation becomes ready, its index is returned. If multiple operations are ready at\n /// the same time, a random one among them is chosen. If none of the operations become ready\n /// for the specified duration, an error is returned.\n ///\n /// An operation is considered to be ready if it doesn't have to block. Note that it is ready\n /// even when it will simply return an error because the channel is disconnected.\n ///\n /// Note that this method might return with success spuriously, so it's a good idea to double\n /// check if the operation is really ready.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::Duration;\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let (s1, r1) = unbounded();\n /// let (s2, r2) = unbounded();\n ///\n /// # let t1 =\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// s1.send(10).unwrap();\n /// });\n /// # let t2 =\n /// thread::spawn(move || s2.send(20).unwrap());\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.recv(&r1);\n /// let oper2 = sel.recv(&r2);\n ///\n /// // The second operation will be selected because it becomes ready first.\n /// match sel.ready_timeout(Duration::from_millis(500)) {\n /// Err(_) => panic!(\"should not have timed out\"),\n /// Ok(i) if i == oper1 => assert_eq!(r1.try_recv(), Ok(10)),\n /// Ok(i) if i == oper2 => assert_eq!(r2.try_recv(), Ok(20)),\n /// Ok(_) => unreachable!(),\n /// }\n /// # t1.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// # t2.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n pub fn ready_timeout(&mut self, timeout: Duration) -> Result {\n match Instant::now().checked_add(timeout) {\n Some(deadline) => self.ready_deadline(deadline),\n None => Ok(self.ready()),\n }\n }\n\n /// Blocks until a given deadline, or until one of the operations becomes ready.\n ///\n /// If an operation becomes ready, its index is returned. If multiple operations are ready at\n /// the same time, a random one among them is chosen. If none of the operations become ready\n /// before the deadline, an error is returned.\n ///\n /// An operation is considered to be ready if it doesn't have to block. Note that it is ready\n /// even when it will simply return an error because the channel is disconnected.\n ///\n /// Note that this method might return with success spuriously, so it's a good idea to double\n /// check if the operation is really ready.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::{Duration, Instant};\n /// use crossbeam_channel::{unbounded, Select};\n ///\n /// let deadline = Instant::now() + Duration::from_millis(500);\n ///\n /// let (s1, r1) = unbounded();\n /// let (s2, r2) = unbounded();\n ///\n /// # let t1 =\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_secs(1));\n /// s1.send(10).unwrap();\n /// });\n /// # let t2 =\n /// thread::spawn(move || s2.send(20).unwrap());\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.recv(&r1);\n /// let oper2 = sel.recv(&r2);\n ///\n /// // The second operation will be selected because it becomes ready first.\n /// match sel.ready_deadline(deadline) {\n /// Err(_) => panic!(\"should not have timed out\"),\n /// Ok(i) if i == oper1 => assert_eq!(r1.try_recv(), Ok(10)),\n /// Ok(i) if i == oper2 => assert_eq!(r2.try_recv(), Ok(20)),\n /// Ok(_) => unreachable!(),\n /// }\n /// # t1.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// # t2.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n pub fn ready_deadline(&mut self, deadline: Instant) -> Result {\n match run_ready(&mut self.handles, Timeout::At(deadline), self.biased) {\n None => Err(ReadyTimeoutError),\n Some(index) => Ok(index),\n }\n }\n}\n\nimpl Clone for Select<'_> {\n fn clone(&self) -> Self {\n Self {\n handles: self.handles.clone(),\n next_index: self.next_index,\n biased: self.biased,\n }\n }\n}\n\nimpl Default for Select<'_> {\n fn default() -> Self {\n Self::new()\n }\n}\n\nimpl fmt::Debug for Select<'_> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Select { .. }\")\n }\n}\n\n/// A selected operation that needs to be completed.\n///\n/// To complete the operation, call [`send`] or [`recv`].\n///\n/// # Panics\n///\n/// Forgetting to complete the operation is an error and might lead to deadlocks. If a\n/// `SelectedOperation` is dropped without completion, a panic occurs.\n///\n/// [`send`]: SelectedOperation::send\n/// [`recv`]: SelectedOperation::recv\n#[must_use]\npub struct SelectedOperation<'a> {\n /// Token needed to complete the operation.\n token: Token,\n\n /// The index of the selected operation.\n index: usize,\n\n /// The address of the selected `Sender` or `Receiver`.\n addr: usize,\n\n /// Indicates that `Sender`s and `Receiver`s are borrowed.\n _marker: PhantomData<&'a ()>,\n}\n\nimpl SelectedOperation<'_> {\n /// Returns the index of the selected operation.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{bounded, Select};\n ///\n /// let (s1, r1) = bounded::<()>(0);\n /// let (s2, r2) = bounded::<()>(0);\n /// let (s3, r3) = bounded::<()>(1);\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.send(&s1);\n /// let oper2 = sel.recv(&r2);\n /// let oper3 = sel.send(&s3);\n ///\n /// // Only the last operation is ready.\n /// let oper = sel.select();\n /// assert_eq!(oper.index(), 2);\n /// assert_eq!(oper.index(), oper3);\n ///\n /// // Complete the operation.\n /// oper.send(&s3, ()).unwrap();\n /// ```\n pub fn index(&self) -> usize {\n self.index\n }\n\n /// Completes the send operation.\n ///\n /// The passed [`Sender`] reference must be the same one that was used in [`Select::send`]\n /// when the operation was added.\n ///\n /// # Panics\n ///\n /// Panics if an incorrect [`Sender`] reference is passed.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{bounded, Select, SendError};\n ///\n /// let (s, r) = bounded::(0);\n /// drop(r);\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.send(&s);\n ///\n /// let oper = sel.select();\n /// assert_eq!(oper.index(), oper1);\n /// assert_eq!(oper.send(&s, 10), Err(SendError(10)));\n /// ```\n pub fn send(mut self, s: &Sender, msg: T) -> Result<(), SendError> {\n assert!(\n s.addr() == self.addr,\n \"passed a sender that wasn't selected\",\n );\n let res = unsafe { channel::write(s, &mut self.token, msg) };\n mem::forget(self);\n res.map_err(SendError)\n }\n\n /// Completes the receive operation.\n ///\n /// The passed [`Receiver`] reference must be the same one that was used in [`Select::recv`]\n /// when the operation was added.\n ///\n /// # Panics\n ///\n /// Panics if an incorrect [`Receiver`] reference is passed.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_channel::{bounded, Select, RecvError};\n ///\n /// let (s, r) = bounded::(0);\n /// drop(s);\n ///\n /// let mut sel = Select::new();\n /// let oper1 = sel.recv(&r);\n ///\n /// let oper = sel.select();\n /// assert_eq!(oper.index(), oper1);\n /// assert_eq!(oper.recv(&r), Err(RecvError));\n /// ```\n pub fn recv(mut self, r: &Receiver) -> Result {\n assert!(\n r.addr() == self.addr,\n \"passed a receiver that wasn't selected\",\n );\n let res = unsafe { channel::read(r, &mut self.token) };\n mem::forget(self);\n res.map_err(|_| RecvError)\n }\n}\n\nimpl fmt::Debug for SelectedOperation<'_> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"SelectedOperation { .. }\")\n }\n}\n\nimpl Drop for SelectedOperation<'_> {\n fn drop(&mut self) {\n panic!(\"dropped `SelectedOperation` without completing the operation\");\n }\n}\n" }, { "path": "crossbeam-channel/src/select_macro.rs", "content": "//! The `select!` macro.\n\n/// A helper macro for `select!` to hide the long list of macro patterns from the documentation.\n///\n/// The macro consists of two stages:\n/// 1. Parsing\n/// 2. Code generation\n///\n/// The parsing stage consists of these subparts:\n/// 1. `@list`: Turns a list of tokens into a list of cases.\n/// 2. `@list_errorN`: Diagnoses the syntax error.\n/// 3. `@case`: Parses a single case and verifies its argument list.\n///\n/// The codegen stage consists of these subparts:\n/// 1. `@init`: Attempts to optimize `select!` away and initializes the list of handles.\n/// 1. `@count`: Counts the listed cases.\n/// 3. `@add`: Adds send/receive operations to the list of handles and starts selection.\n/// 4. `@complete`: Completes the selected send/receive operation.\n///\n/// If the parsing stage encounters a syntax error or the codegen stage ends up with too many\n/// cases to process, the macro fails with a compile-time error.\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! crossbeam_channel_internal {\n // The list is empty. Now check the arguments of each processed case.\n (@list\n ()\n ($($head:tt)*)\n ) => {\n $crate::crossbeam_channel_internal!(\n @case\n ($($head)*)\n ()\n ()\n )\n };\n // If necessary, insert an empty argument list after `default`.\n (@list\n (default => $($tail:tt)*)\n ($($head:tt)*)\n ) => {\n $crate::crossbeam_channel_internal!(\n @list\n (default() => $($tail)*)\n ($($head)*)\n )\n };\n // But print an error if `default` is followed by a `->`.\n (@list\n (default -> $($tail:tt)*)\n ($($head:tt)*)\n ) => {\n compile_error!(\n \"expected `=>` after `default` case, found `->`\"\n )\n };\n // Print an error if there's an `->` after the argument list in the default case.\n (@list\n (default $args:tt -> $($tail:tt)*)\n ($($head:tt)*)\n ) => {\n compile_error!(\n \"expected `=>` after `default` case, found `->`\"\n )\n };\n // Print an error if there is a missing result in a recv case.\n (@list\n (recv($($args:tt)*) => $($tail:tt)*)\n ($($head:tt)*)\n ) => {\n compile_error!(\n \"expected `->` after `recv` case, found `=>`\"\n )\n };\n // Print an error if there is a missing result in a send case.\n (@list\n (send($($args:tt)*) => $($tail:tt)*)\n ($($head:tt)*)\n ) => {\n compile_error!(\n \"expected `->` after `send` operation, found `=>`\"\n )\n };\n // Make sure the arrow and the result are not repeated.\n (@list\n ($case:ident $args:tt -> $res:tt -> $($tail:tt)*)\n ($($head:tt)*)\n ) => {\n compile_error!(\"expected `=>`, found `->`\")\n };\n // Print an error if there is a semicolon after the block.\n (@list\n ($case:ident $args:tt $(-> $res:pat)* => { $($body:tt)* }; $($tail:tt)*)\n ($($head:tt)*)\n ) => {\n compile_error!(\n \"did you mean to put a comma instead of the semicolon after `}`?\"\n )\n };\n // The first case is separated by a comma.\n (@list\n ($case:ident ($($args:tt)*) $(-> $res:pat)* => { $($body:tt)* }, $($tail:tt)*)\n ($($head:tt)*)\n ) => {\n $crate::crossbeam_channel_internal!(\n @list\n ($($tail)*)\n ($($head)* $case ($($args)*) $(-> $res)* => { $($body)* },)\n )\n };\n (@list\n ($case:ident ($($args:tt)*) $(-> $res:pat)* => $body:expr, $($tail:tt)*)\n ($($head:tt)*)\n ) => {\n $crate::crossbeam_channel_internal!(\n @list\n ($($tail)*)\n ($($head)* $case ($($args)*) $(-> $res)* => { $body },)\n )\n };\n // Don't require a comma after the case if it has a proper block.\n (@list\n ($case:ident ($($args:tt)*) $(-> $res:pat)* => { $($body:tt)* } $($tail:tt)*)\n ($($head:tt)*)\n ) => {\n $crate::crossbeam_channel_internal!(\n @list\n ($($tail)*)\n ($($head)* $case ($($args)*) $(-> $res)* => { $($body)* },)\n )\n };\n // Only one case remains.\n (@list\n ($case:ident ($($args:tt)*) $(-> $res:pat)* => $body:expr $(,)?)\n ($($head:tt)*)\n ) => {\n $crate::crossbeam_channel_internal!(\n @list\n ()\n ($($head)* $case ($($args)*) $(-> $res)* => { $body },)\n )\n };\n // Diagnose and print an error.\n (@list\n ($($tail:tt)*)\n ($($head:tt)*)\n ) => {\n $crate::crossbeam_channel_internal!(@list_error1 $($tail)*)\n };\n // Stage 1: check the case type.\n (@list_error1 recv $($tail:tt)*) => {\n $crate::crossbeam_channel_internal!(@list_error2 recv $($tail)*)\n };\n (@list_error1 send $($tail:tt)*) => {\n $crate::crossbeam_channel_internal!(@list_error2 send $($tail)*)\n };\n (@list_error1 default $($tail:tt)*) => {\n $crate::crossbeam_channel_internal!(@list_error2 default $($tail)*)\n };\n (@list_error1 $t:tt $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"expected one of `recv`, `send`, or `default`, found `\",\n stringify!($t),\n \"`\",\n )\n )\n };\n (@list_error1 $($tail:tt)*) => {\n $crate::crossbeam_channel_internal!(@list_error2 $($tail)*);\n };\n // Stage 2: check the argument list.\n (@list_error2 $case:ident) => {\n compile_error!(\n concat!(\n \"missing argument list after `\",\n stringify!($case),\n \"`\",\n )\n )\n };\n (@list_error2 $case:ident => $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"missing argument list after `\",\n stringify!($case),\n \"`\",\n )\n )\n };\n (@list_error2 $($tail:tt)*) => {\n $crate::crossbeam_channel_internal!(@list_error3 $($tail)*)\n };\n // Stage 3: check the `=>` and what comes after it.\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)*) => {\n compile_error!(\n concat!(\n \"missing `=>` after `\",\n stringify!($case),\n \"` case\",\n )\n )\n };\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)* =>) => {\n compile_error!(\n \"expected expression after `=>`\"\n )\n };\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)* => $body:expr; $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"did you mean to put a comma instead of the semicolon after `\",\n stringify!($body),\n \"`?\",\n )\n )\n };\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)* => recv($($a:tt)*) $($tail:tt)*) => {\n compile_error!(\n \"expected an expression after `=>`\"\n )\n };\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)* => send($($a:tt)*) $($tail:tt)*) => {\n compile_error!(\n \"expected an expression after `=>`\"\n )\n };\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)* => default($($a:tt)*) $($tail:tt)*) => {\n compile_error!(\n \"expected an expression after `=>`\"\n )\n };\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)* => $f:ident($($a:tt)*) $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"did you mean to put a comma after `\",\n stringify!($f),\n \"(\",\n stringify!($($a)*),\n \")`?\",\n )\n )\n };\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)* => $f:ident!($($a:tt)*) $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"did you mean to put a comma after `\",\n stringify!($f),\n \"!(\",\n stringify!($($a)*),\n \")`?\",\n )\n )\n };\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)* => $f:ident![$($a:tt)*] $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"did you mean to put a comma after `\",\n stringify!($f),\n \"![\",\n stringify!($($a)*),\n \"]`?\",\n )\n )\n };\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)* => $f:ident!{$($a:tt)*} $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"did you mean to put a comma after `\",\n stringify!($f),\n \"!{\",\n stringify!($($a)*),\n \"}`?\",\n )\n )\n };\n (@list_error3 $case:ident($($args:tt)*) $(-> $r:pat)* => $body:tt $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"did you mean to put a comma after `\",\n stringify!($body),\n \"`?\",\n )\n )\n };\n (@list_error3 $case:ident($($args:tt)*) -> => $($tail:tt)*) => {\n compile_error!(\"missing pattern after `->`\")\n };\n (@list_error3 $case:ident($($args:tt)*) $t:tt $(-> $r:pat)* => $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"expected `->`, found `\",\n stringify!($t),\n \"`\",\n )\n )\n };\n (@list_error3 $case:ident($($args:tt)*) -> $t:tt $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"expected a pattern, found `\",\n stringify!($t),\n \"`\",\n )\n )\n };\n (@list_error3 recv($($args:tt)*) $t:tt $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"expected `->`, found `\",\n stringify!($t),\n \"`\",\n )\n )\n };\n (@list_error3 send($($args:tt)*) $t:tt $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"expected `->`, found `\",\n stringify!($t),\n \"`\",\n )\n )\n };\n (@list_error3 recv $args:tt $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"expected an argument list after `recv`, found `\",\n stringify!($args),\n \"`\",\n )\n )\n };\n (@list_error3 send $args:tt $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"expected an argument list after `send`, found `\",\n stringify!($args),\n \"`\",\n )\n )\n };\n (@list_error3 default $args:tt $($tail:tt)*) => {\n compile_error!(\n concat!(\n \"expected an argument list or `=>` after `default`, found `\",\n stringify!($args),\n \"`\",\n )\n )\n };\n (@list_error3 $($tail:tt)*) => {\n $crate::crossbeam_channel_internal!(@list_error4 $($tail)*)\n };\n // Stage 4: fail with a generic error message.\n (@list_error4 $($tail:tt)*) => {\n compile_error!(\"invalid syntax\")\n };\n\n // Success! All cases were parsed.\n (@case\n ()\n $cases:tt\n $default:tt\n ) => {\n $crate::crossbeam_channel_internal!(\n @init\n $cases\n $default\n )\n };\n\n // Check the format of a recv case.\n (@case\n (recv($r:expr $(,)?) -> $res:pat => $body:tt, $($tail:tt)*)\n ($($cases:tt)*)\n $default:tt\n ) => {\n $crate::crossbeam_channel_internal!(\n @case\n ($($tail)*)\n ($($cases)* recv($r) -> $res => $body,)\n $default\n )\n };\n // Print an error if the argument list is invalid.\n (@case\n (recv($($args:tt)*) -> $res:pat => $body:tt, $($tail:tt)*)\n ($($cases:tt)*)\n $default:tt\n ) => {\n compile_error!(\n concat!(\n \"invalid argument list in `recv(\",\n stringify!($($args)*),\n \")`\",\n )\n )\n };\n // Print an error if there is no argument list.\n (@case\n (recv $t:tt $($tail:tt)*)\n ($($cases:tt)*)\n $default:tt\n ) => {\n compile_error!(\n concat!(\n \"expected an argument list after `recv`, found `\",\n stringify!($t),\n \"`\",\n )\n )\n };\n\n // Check the format of a send case.\n (@case\n (send($s:expr, $m:expr $(,)?) -> $res:pat => $body:tt, $($tail:tt)*)\n ($($cases:tt)*)\n $default:tt\n ) => {\n $crate::crossbeam_channel_internal!(\n @case\n ($($tail)*)\n ($($cases)* send($s, $m) -> $res => $body,)\n $default\n )\n };\n // Print an error if the argument list is invalid.\n (@case\n (send($($args:tt)*) -> $res:pat => $body:tt, $($tail:tt)*)\n ($($cases:tt)*)\n $default:tt\n ) => {\n compile_error!(\n concat!(\n \"invalid argument list in `send(\",\n stringify!($($args)*),\n \")`\",\n )\n )\n };\n // Print an error if there is no argument list.\n (@case\n (send $t:tt $($tail:tt)*)\n ($($cases:tt)*)\n $default:tt\n ) => {\n compile_error!(\n concat!(\n \"expected an argument list after `send`, found `\",\n stringify!($t),\n \"`\",\n )\n )\n };\n\n // Check the format of a default case.\n (@case\n (default() => $body:tt, $($tail:tt)*)\n $cases:tt\n ()\n ) => {\n $crate::crossbeam_channel_internal!(\n @case\n ($($tail)*)\n $cases\n (default() => $body,)\n )\n };\n // Check the format of a default case with timeout.\n (@case\n (default($timeout:expr $(,)?) => $body:tt, $($tail:tt)*)\n $cases:tt\n ()\n ) => {\n $crate::crossbeam_channel_internal!(\n @case\n ($($tail)*)\n $cases\n (default($timeout) => $body,)\n )\n };\n // Check for duplicate default cases...\n (@case\n (default $($tail:tt)*)\n $cases:tt\n ($($def:tt)+)\n ) => {\n compile_error!(\n \"there can be only one `default` case in a `select!` block\"\n )\n };\n // Print an error if the argument list is invalid.\n (@case\n (default($($args:tt)*) => $body:tt, $($tail:tt)*)\n $cases:tt\n $default:tt\n ) => {\n compile_error!(\n concat!(\n \"invalid argument list in `default(\",\n stringify!($($args)*),\n \")`\",\n )\n )\n };\n // Print an error if there is an unexpected token after `default`.\n (@case\n (default $t:tt $($tail:tt)*)\n $cases:tt\n $default:tt\n ) => {\n compile_error!(\n concat!(\n \"expected an argument list or `=>` after `default`, found `\",\n stringify!($t),\n \"`\",\n )\n )\n };\n\n // The case was not consumed, therefore it must be invalid.\n (@case\n ($case:ident $($tail:tt)*)\n $cases:tt\n $default:tt\n ) => {\n compile_error!(\n concat!(\n \"expected one of `recv`, `send`, or `default`, found `\",\n stringify!($case),\n \"`\",\n )\n )\n };\n\n // Optimize `select!` into `try_recv()`.\n (@init\n (recv($r:expr) -> $res:pat => $recv_body:tt,)\n (default() => $default_body:tt,)\n ) => {{\n match $r {\n ref _r => {\n let _r: &$crate::Receiver<_> = _r;\n match _r.try_recv() {\n ::std::result::Result::Err($crate::TryRecvError::Empty) => {\n $default_body\n }\n _res => {\n let _res = _res.map_err(|_| $crate::RecvError);\n let $res = _res;\n $recv_body\n }\n }\n }\n }\n }};\n // Optimize `select!` into `recv()`.\n (@init\n (recv($r:expr) -> $res:pat => $body:tt,)\n ()\n ) => {{\n match $r {\n ref _r => {\n let _r: &$crate::Receiver<_> = _r;\n let _res = _r.recv();\n let $res = _res;\n $body\n }\n }\n }};\n // Optimize `select!` into `recv_timeout()`.\n (@init\n (recv($r:expr) -> $res:pat => $recv_body:tt,)\n (default($timeout:expr) => $default_body:tt,)\n ) => {{\n match $r {\n ref _r => {\n let _r: &$crate::Receiver<_> = _r;\n match _r.recv_timeout($timeout) {\n ::std::result::Result::Err($crate::RecvTimeoutError::Timeout) => {\n $default_body\n }\n _res => {\n let _res = _res.map_err(|_| $crate::RecvError);\n let $res = _res;\n $recv_body\n }\n }\n }\n }\n }};\n\n // // Optimize the non-blocking case with two receive operations.\n // (@init\n // (recv($r1:expr) -> $res1:pat => $recv_body1:tt,)\n // (recv($r2:expr) -> $res2:pat => $recv_body2:tt,)\n // (default() => $default_body:tt,)\n // ) => {{\n // match $r1 {\n // ref _r1 => {\n // let _r1: &$crate::Receiver<_> = _r1;\n //\n // match $r2 {\n // ref _r2 => {\n // let _r2: &$crate::Receiver<_> = _r2;\n //\n // // TODO(stjepang): Implement this optimization.\n // }\n // }\n // }\n // }\n // }};\n // // Optimize the blocking case with two receive operations.\n // (@init\n // (recv($r1:expr) -> $res1:pat => $body1:tt,)\n // (recv($r2:expr) -> $res2:pat => $body2:tt,)\n // ()\n // ) => {{\n // match $r1 {\n // ref _r1 => {\n // let _r1: &$crate::Receiver<_> = _r1;\n //\n // match $r2 {\n // ref _r2 => {\n // let _r2: &$crate::Receiver<_> = _r2;\n //\n // // TODO(stjepang): Implement this optimization.\n // }\n // }\n // }\n // }\n // }};\n // // Optimize the case with two receive operations and a timeout.\n // (@init\n // (recv($r1:expr) -> $res1:pat => $recv_body1:tt,)\n // (recv($r2:expr) -> $res2:pat => $recv_body2:tt,)\n // (default($timeout:expr) => $default_body:tt,)\n // ) => {{\n // match $r1 {\n // ref _r1 => {\n // let _r1: &$crate::Receiver<_> = _r1;\n //\n // match $r2 {\n // ref _r2 => {\n // let _r2: &$crate::Receiver<_> = _r2;\n //\n // // TODO(stjepang): Implement this optimization.\n // }\n // }\n // }\n // }\n // }};\n\n // // Optimize `select!` into `try_send()`.\n // (@init\n // (send($s:expr, $m:expr) -> $res:pat => $send_body:tt,)\n // (default() => $default_body:tt,)\n // ) => {{\n // match $s {\n // ref _s => {\n // let _s: &$crate::Sender<_> = _s;\n // // TODO(stjepang): Implement this optimization.\n // }\n // }\n // }};\n // // Optimize `select!` into `send()`.\n // (@init\n // (send($s:expr, $m:expr) -> $res:pat => $body:tt,)\n // ()\n // ) => {{\n // match $s {\n // ref _s => {\n // let _s: &$crate::Sender<_> = _s;\n // // TODO(stjepang): Implement this optimization.\n // }\n // }\n // }};\n // // Optimize `select!` into `send_timeout()`.\n // (@init\n // (send($s:expr, $m:expr) -> $res:pat => $body:tt,)\n // (default($timeout:expr) => $body:tt,)\n // ) => {{\n // match $s {\n // ref _s => {\n // let _s: &$crate::Sender<_> = _s;\n // // TODO(stjepang): Implement this optimization.\n // }\n // }\n // }};\n\n // Create the list of handles and add operations to it.\n (@init\n ($($cases:tt)*)\n $default:tt\n ) => {{\n const _LEN: usize = $crate::crossbeam_channel_internal!(@count ($($cases)*));\n let _handle: &dyn $crate::internal::SelectHandle = &$crate::never::<()>();\n\n #[allow(unused_mut, clippy::zero_repeat_side_effects)]\n let mut _sel = [(_handle, 0, 0); _LEN];\n\n $crate::crossbeam_channel_internal!(\n @add\n _sel\n ($($cases)*)\n $default\n (\n (0usize _oper0)\n (1usize _oper1)\n (2usize _oper2)\n (3usize _oper3)\n (4usize _oper4)\n (5usize _oper5)\n (6usize _oper6)\n (7usize _oper7)\n (8usize _oper8)\n (9usize _oper9)\n (10usize _oper10)\n (11usize _oper11)\n (12usize _oper12)\n (13usize _oper13)\n (14usize _oper14)\n (15usize _oper15)\n (16usize _oper16)\n (17usize _oper17)\n (18usize _oper18)\n (19usize _oper19)\n (20usize _oper20)\n (21usize _oper21)\n (22usize _oper22)\n (23usize _oper23)\n (24usize _oper24)\n (25usize _oper25)\n (26usize _oper26)\n (27usize _oper27)\n (28usize _oper28)\n (29usize _oper29)\n (30usize _oper30)\n (31usize _oper31)\n )\n ()\n )\n }};\n\n // Count the listed cases.\n (@count ()) => {\n 0\n };\n (@count ($oper:ident $args:tt -> $res:pat => $body:tt, $($cases:tt)*)) => {\n 1 + $crate::crossbeam_channel_internal!(@count ($($cases)*))\n };\n\n // Run blocking selection.\n (@add\n $sel:ident\n ()\n ()\n $labels:tt\n $cases:tt\n ) => {{\n let _oper: $crate::SelectedOperation<'_> = {\n let _oper = $crate::internal::select(&mut $sel, _IS_BIASED);\n\n // Erase the lifetime so that `sel` can be dropped early even without NLL.\n unsafe { ::std::mem::transmute(_oper) }\n };\n\n $crate::crossbeam_channel_internal! {\n @complete\n $sel\n _oper\n $cases\n }\n }};\n // Run non-blocking selection.\n (@add\n $sel:ident\n ()\n (default() => $body:tt,)\n $labels:tt\n $cases:tt\n ) => {{\n let _oper: ::std::option::Option<$crate::SelectedOperation<'_>> = {\n let _oper = $crate::internal::try_select(&mut $sel, _IS_BIASED);\n\n // Erase the lifetime so that `sel` can be dropped early even without NLL.\n unsafe { ::std::mem::transmute(_oper) }\n };\n\n match _oper {\n None => {\n { $sel };\n $body\n }\n Some(_oper) => {\n $crate::crossbeam_channel_internal! {\n @complete\n $sel\n _oper\n $cases\n }\n }\n }\n }};\n // Run selection with a timeout.\n (@add\n $sel:ident\n ()\n (default($timeout:expr) => $body:tt,)\n $labels:tt\n $cases:tt\n ) => {{\n let _oper: ::std::option::Option<$crate::SelectedOperation<'_>> = {\n let _oper = $crate::internal::select_timeout(&mut $sel, $timeout, _IS_BIASED);\n\n // Erase the lifetime so that `sel` can be dropped early even without NLL.\n unsafe { ::std::mem::transmute(_oper) }\n };\n\n match _oper {\n ::std::option::Option::None => {\n { $sel };\n $body\n }\n ::std::option::Option::Some(_oper) => {\n $crate::crossbeam_channel_internal! {\n @complete\n $sel\n _oper\n $cases\n }\n }\n }\n }};\n // Have we used up all labels?\n (@add\n $sel:ident\n $input:tt\n $default:tt\n ()\n $cases:tt\n ) => {\n compile_error!(\"too many operations in a `select!` block\")\n };\n // Add a receive operation to `sel`.\n (@add\n $sel:ident\n (recv($r:expr) -> $res:pat => $body:tt, $($tail:tt)*)\n $default:tt\n (($i:tt $var:ident) $($labels:tt)*)\n ($($cases:tt)*)\n ) => {{\n match $r {\n ref _r => {\n let $var: &$crate::Receiver<_> = unsafe {\n let _r: &$crate::Receiver<_> = _r;\n\n // Erase the lifetime so that `sel` can be dropped early even without NLL.\n unsafe fn unbind<'a, T>(x: &T) -> &'a T {\n ::std::mem::transmute(x)\n }\n unbind(_r)\n };\n $sel[$i] = ($var, $i, $crate::internal::receiver_addr($var));\n\n $crate::crossbeam_channel_internal!(\n @add\n $sel\n ($($tail)*)\n $default\n ($($labels)*)\n ($($cases)* [$i] recv($var) -> $res => $body,)\n )\n }\n }\n }};\n // Add a send operation to `sel`.\n (@add\n $sel:ident\n (send($s:expr, $m:expr) -> $res:pat => $body:tt, $($tail:tt)*)\n $default:tt\n (($i:tt $var:ident) $($labels:tt)*)\n ($($cases:tt)*)\n ) => {{\n match $s {\n ref _s => {\n let $var: &$crate::Sender<_> = unsafe {\n let _s: &$crate::Sender<_> = _s;\n\n // Erase the lifetime so that `sel` can be dropped early even without NLL.\n unsafe fn unbind<'a, T>(x: &T) -> &'a T {\n ::std::mem::transmute(x)\n }\n unbind(_s)\n };\n $sel[$i] = ($var, $i, $crate::internal::sender_addr($var));\n\n $crate::crossbeam_channel_internal!(\n @add\n $sel\n ($($tail)*)\n $default\n ($($labels)*)\n ($($cases)* [$i] send($var, $m) -> $res => $body,)\n )\n }\n }\n }};\n\n // Complete a receive operation.\n (@complete\n $sel:ident\n $oper:ident\n ([$i:tt] recv($r:ident) -> $res:pat => $body:tt, $($tail:tt)*)\n ) => {{\n if $oper.index() == $i {\n let _res = $oper.recv($r);\n { $sel };\n\n let $res = _res;\n $body\n } else {\n $crate::crossbeam_channel_internal! {\n @complete\n $sel\n $oper\n ($($tail)*)\n }\n }\n }};\n // Complete a send operation.\n (@complete\n $sel:ident\n $oper:ident\n ([$i:tt] send($s:ident, $m:expr) -> $res:pat => $body:tt, $($tail:tt)*)\n ) => {{\n if $oper.index() == $i {\n let _res = $oper.send($s, $m);\n { $sel };\n\n let $res = _res;\n $body\n } else {\n $crate::crossbeam_channel_internal! {\n @complete\n $sel\n $oper\n ($($tail)*)\n }\n }\n }};\n // Panic if we don't identify the selected case, but this should never happen.\n (@complete\n $sel:ident\n $oper:ident\n ()\n ) => {{\n unreachable!(\n \"internal error in crossbeam-channel: invalid case\"\n )\n }};\n\n // Catches a bug within this macro (should not happen).\n (@$($tokens:tt)*) => {\n compile_error!(\n concat!(\n \"internal error in crossbeam-channel: \",\n stringify!(@$($tokens)*),\n )\n )\n };\n\n // The entry points.\n () => {\n compile_error!(\"empty `select!` block\")\n };\n ($($case:ident $(($($args:tt)*))* => $body:expr $(,)*)*) => {\n $crate::crossbeam_channel_internal!(\n @list\n ($($case $(($($args)*))* => { $body },)*)\n ()\n )\n };\n ($($tokens:tt)*) => {\n $crate::crossbeam_channel_internal!(\n @list\n ($($tokens)*)\n ()\n )\n };\n}\n\n/// Selects from a set of channel operations.\n///\n/// This macro allows you to define a set of channel operations, wait until any one of them becomes\n/// ready, and finally execute it. If multiple operations are ready at the same time, a random one\n/// among them is selected (i.e. the unbiased selection). Use [`select_biased!`] for the biased\n/// selection.\n///\n/// It is also possible to define a `default` case that gets executed if none of the operations are\n/// ready, either right away or for a certain duration of time.\n///\n/// An operation is considered to be ready if it doesn't have to block. Note that it is ready even\n/// when it will simply return an error because the channel is disconnected.\n///\n/// The `select!` macro is a convenience wrapper around [`Select`]. However, it cannot select over a\n/// dynamically created list of channel operations.\n///\n/// [`Select`]: super::Select\n/// [`select_biased!`]: super::select_biased\n///\n/// # Examples\n///\n/// Block until a send or a receive operation is selected:\n///\n/// ```\n/// use crossbeam_channel::{select, unbounded};\n///\n/// let (s1, r1) = unbounded();\n/// let (s2, r2) = unbounded();\n/// s1.send(10).unwrap();\n///\n/// // Since both operations are initially ready, a random one will be executed.\n/// select! {\n/// recv(r1) -> msg => assert_eq!(msg, Ok(10)),\n/// send(s2, 20) -> res => {\n/// assert_eq!(res, Ok(()));\n/// assert_eq!(r2.recv(), Ok(20));\n/// }\n/// }\n/// ```\n///\n/// Select from a set of operations without blocking:\n///\n/// ```\n/// use std::thread;\n/// use std::time::Duration;\n/// use crossbeam_channel::{select, unbounded};\n///\n/// let (s1, r1) = unbounded();\n/// let (s2, r2) = unbounded();\n///\n/// # let t1 =\n/// thread::spawn(move || {\n/// thread::sleep(Duration::from_secs(1));\n/// s1.send(10).unwrap();\n/// });\n/// # let t2 =\n/// thread::spawn(move || {\n/// thread::sleep(Duration::from_millis(500));\n/// s2.send(20).unwrap();\n/// });\n///\n/// // None of the operations are initially ready.\n/// select! {\n/// recv(r1) -> msg => panic!(),\n/// recv(r2) -> msg => panic!(),\n/// default => println!(\"not ready\"),\n/// }\n/// # t1.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n/// # t2.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n/// ```\n///\n/// Select over a set of operations with a timeout:\n///\n/// ```\n/// use std::thread;\n/// use std::time::Duration;\n/// use crossbeam_channel::{select, unbounded};\n///\n/// let (s1, r1) = unbounded();\n/// let (s2, r2) = unbounded();\n///\n/// # let t1 =\n/// thread::spawn(move || {\n/// thread::sleep(Duration::from_secs(1));\n/// s1.send(10).unwrap();\n/// });\n/// # let t2 =\n/// thread::spawn(move || {\n/// thread::sleep(Duration::from_millis(500));\n/// s2.send(20).unwrap();\n/// });\n///\n/// // None of the two operations will become ready within 100 milliseconds.\n/// select! {\n/// recv(r1) -> msg => panic!(),\n/// recv(r2) -> msg => panic!(),\n/// default(Duration::from_millis(100)) => println!(\"timed out\"),\n/// }\n/// # t1.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n/// # t2.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n/// ```\n///\n/// Optionally add a receive operation to `select!` using [`never`]:\n///\n/// ```\n/// # #[allow(clippy::unnecessary_literal_unwrap)] {\n/// use std::thread;\n/// use std::time::Duration;\n/// use crossbeam_channel::{select, never, unbounded};\n///\n/// let (s1, r1) = unbounded();\n/// let (s2, r2) = unbounded();\n///\n/// # let t1 =\n/// thread::spawn(move || {\n/// thread::sleep(Duration::from_secs(1));\n/// s1.send(10).unwrap();\n/// });\n/// # let t2 =\n/// thread::spawn(move || {\n/// thread::sleep(Duration::from_millis(500));\n/// s2.send(20).unwrap();\n/// });\n///\n/// // This receiver can be a `Some` or a `None`.\n/// let r2 = Some(&r2);\n///\n/// // None of the two operations will become ready within 100 milliseconds.\n/// select! {\n/// recv(r1) -> msg => panic!(),\n/// recv(r2.unwrap_or(&never())) -> msg => assert_eq!(msg, Ok(20)),\n/// }\n/// # t1.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n/// # t2.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n/// # }\n/// ```\n///\n/// To optionally add a timeout to `select!`, see the [example] for [`never`].\n///\n/// [`never`]: super::never\n/// [example]: super::never#examples\n#[macro_export]\nmacro_rules! select {\n ($($tokens:tt)*) => {\n {\n const _IS_BIASED: bool = false;\n\n $crate::crossbeam_channel_internal!(\n $($tokens)*\n )\n }\n };\n}\n\n/// Selects from a set of channel operations.\n///\n/// This macro allows you to define a list of channel operations, wait until any one of them\n/// becomes ready, and finally execute it. If multiple operations are ready at the same time, the\n/// operation nearest to the front of the list is always selected (i.e. the biased selection). Use\n/// [`select!`] for the unbiased selection.\n///\n/// Otherwise, this macro's functionality is identical to [`select!`]. Refer to it for the syntax.\n#[macro_export]\nmacro_rules! select_biased {\n ($($tokens:tt)*) => {\n {\n const _IS_BIASED: bool = true;\n\n $crate::crossbeam_channel_internal!(\n $($tokens)*\n )\n }\n };\n}\n" }, { "path": "crossbeam-channel/src/utils.rs", "content": "//! Miscellaneous utilities.\n\nuse core::{cell::Cell, num::Wrapping, time::Duration};\nuse std::{thread, time::Instant};\n\n/// Randomly shuffles a slice.\npub(crate) fn shuffle(v: &mut [T]) {\n let len = v.len();\n if len <= 1 {\n return;\n }\n\n std::thread_local! {\n static RNG: Cell> = const { Cell::new(Wrapping(1_406_868_647)) };\n }\n\n let _ = RNG.try_with(|rng| {\n for i in 1..len {\n // This is the 32-bit variant of Xorshift.\n //\n // Source: https://en.wikipedia.org/wiki/Xorshift\n let mut x = rng.get();\n x ^= x << 13;\n x ^= x >> 17;\n x ^= x << 5;\n rng.set(x);\n\n let x = x.0;\n let n = i + 1;\n\n // This is a fast alternative to `let j = x % n`.\n //\n // Author: Daniel Lemire\n // Source: https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/\n let j = ((x as u64).wrapping_mul(n as u64) >> 32) as u32 as usize;\n\n v.swap(i, j);\n }\n });\n}\n\n/// Sleeps until the deadline, or forever if the deadline isn't specified.\npub(crate) fn sleep_until(deadline: Option) {\n loop {\n match deadline {\n None => thread::sleep(Duration::from_secs(1000)),\n Some(d) => {\n let now = Instant::now();\n if now >= d {\n break;\n }\n thread::sleep(d - now);\n }\n }\n }\n}\n" }, { "path": "crossbeam-channel/src/waker.rs", "content": "//! Waking mechanism for threads blocked on channel operations.\n\nuse alloc::vec::Vec;\nuse core::{\n ptr,\n sync::atomic::{AtomicBool, Ordering},\n};\nuse std::{\n sync::Mutex,\n thread::{self, ThreadId},\n};\n\nuse crate::{\n context::Context,\n select::{Operation, Selected},\n};\n\n/// Represents a thread blocked on a specific channel operation.\npub(crate) struct Entry {\n /// The operation.\n pub(crate) oper: Operation,\n\n /// Optional packet.\n pub(crate) packet: *mut (),\n\n /// Context associated with the thread owning this operation.\n pub(crate) cx: Context,\n}\n\n/// A queue of threads blocked on channel operations.\n///\n/// This data structure is used by threads to register blocking operations and get woken up once\n/// an operation becomes ready.\npub(crate) struct Waker {\n /// A list of select operations.\n selectors: Vec,\n\n /// A list of operations waiting to be ready.\n observers: Vec,\n}\n\nimpl Waker {\n /// Creates a new `Waker`.\n #[inline]\n pub(crate) fn new() -> Self {\n Self {\n selectors: Vec::new(),\n observers: Vec::new(),\n }\n }\n\n /// Registers a select operation.\n #[inline]\n pub(crate) fn register(&mut self, oper: Operation, cx: &Context) {\n self.register_with_packet(oper, ptr::null_mut(), cx);\n }\n\n /// Registers a select operation and a packet.\n #[inline]\n pub(crate) fn register_with_packet(&mut self, oper: Operation, packet: *mut (), cx: &Context) {\n self.selectors.push(Entry {\n oper,\n packet,\n cx: cx.clone(),\n });\n }\n\n /// Unregisters a select operation.\n #[inline]\n pub(crate) fn unregister(&mut self, oper: Operation) -> Option {\n if let Some((i, _)) = self\n .selectors\n .iter()\n .enumerate()\n .find(|&(_, entry)| entry.oper == oper)\n {\n let entry = self.selectors.remove(i);\n Some(entry)\n } else {\n None\n }\n }\n\n /// Attempts to find another thread's entry, select the operation, and wake it up.\n #[inline]\n pub(crate) fn try_select(&mut self) -> Option {\n if self.selectors.is_empty() {\n None\n } else {\n let thread_id = current_thread_id();\n\n self.selectors\n .iter()\n .position(|selector| {\n // Does the entry belong to a different thread?\n selector.cx.thread_id() != thread_id\n && selector // Try selecting this operation.\n .cx\n .try_select(Selected::Operation(selector.oper))\n .is_ok()\n && {\n // Provide the packet.\n selector.cx.store_packet(selector.packet);\n // Wake the thread up.\n selector.cx.unpark();\n true\n }\n })\n // Remove the entry from the queue to keep it clean and improve\n // performance.\n .map(|pos| self.selectors.remove(pos))\n }\n }\n\n /// Returns `true` if there is an entry which can be selected by the current thread.\n #[inline]\n pub(crate) fn can_select(&self) -> bool {\n if self.selectors.is_empty() {\n false\n } else {\n let thread_id = current_thread_id();\n\n self.selectors.iter().any(|entry| {\n entry.cx.thread_id() != thread_id && entry.cx.selected() == Selected::Waiting\n })\n }\n }\n\n /// Registers an operation waiting to be ready.\n #[inline]\n pub(crate) fn watch(&mut self, oper: Operation, cx: &Context) {\n self.observers.push(Entry {\n oper,\n packet: ptr::null_mut(),\n cx: cx.clone(),\n });\n }\n\n /// Unregisters an operation waiting to be ready.\n #[inline]\n pub(crate) fn unwatch(&mut self, oper: Operation) {\n self.observers.retain(|e| e.oper != oper);\n }\n\n /// Notifies all operations waiting to be ready.\n #[inline]\n pub(crate) fn notify(&mut self) {\n for entry in self.observers.drain(..) {\n if entry.cx.try_select(Selected::Operation(entry.oper)).is_ok() {\n entry.cx.unpark();\n }\n }\n }\n\n /// Notifies all registered operations that the channel is disconnected.\n #[inline]\n pub(crate) fn disconnect(&mut self) {\n for entry in self.selectors.iter() {\n if entry.cx.try_select(Selected::Disconnected).is_ok() {\n // Wake the thread up.\n //\n // Here we don't remove the entry from the queue. Registered threads must\n // unregister from the waker by themselves. They might also want to recover the\n // packet value and destroy it, if necessary.\n entry.cx.unpark();\n }\n }\n\n self.notify();\n }\n}\n\nimpl Drop for Waker {\n #[inline]\n fn drop(&mut self) {\n debug_assert_eq!(self.selectors.len(), 0);\n debug_assert_eq!(self.observers.len(), 0);\n }\n}\n\n/// A waker that can be shared among threads without locking.\n///\n/// This is a simple wrapper around `Waker` that internally uses a mutex for synchronization.\npub(crate) struct SyncWaker {\n /// The inner `Waker`.\n inner: Mutex,\n\n /// `true` if the waker is empty.\n is_empty: AtomicBool,\n}\n\nimpl SyncWaker {\n /// Creates a new `SyncWaker`.\n #[inline]\n pub(crate) fn new() -> Self {\n Self {\n inner: Mutex::new(Waker::new()),\n is_empty: AtomicBool::new(true),\n }\n }\n\n /// Registers the current thread with an operation.\n #[inline]\n pub(crate) fn register(&self, oper: Operation, cx: &Context) {\n let mut inner = self.inner.lock().unwrap();\n inner.register(oper, cx);\n self.is_empty.store(\n inner.selectors.is_empty() && inner.observers.is_empty(),\n Ordering::SeqCst,\n );\n }\n\n /// Unregisters an operation previously registered by the current thread.\n #[inline]\n pub(crate) fn unregister(&self, oper: Operation) -> Option {\n let mut inner = self.inner.lock().unwrap();\n let entry = inner.unregister(oper);\n self.is_empty.store(\n inner.selectors.is_empty() && inner.observers.is_empty(),\n Ordering::SeqCst,\n );\n entry\n }\n\n /// Attempts to find one thread (not the current one), select its operation, and wake it up.\n #[inline]\n pub(crate) fn notify(&self) {\n if !self.is_empty.load(Ordering::SeqCst) {\n let mut inner = self.inner.lock().unwrap();\n if !self.is_empty.load(Ordering::SeqCst) {\n inner.try_select();\n inner.notify();\n self.is_empty.store(\n inner.selectors.is_empty() && inner.observers.is_empty(),\n Ordering::SeqCst,\n );\n }\n }\n }\n\n /// Registers an operation waiting to be ready.\n #[inline]\n pub(crate) fn watch(&self, oper: Operation, cx: &Context) {\n let mut inner = self.inner.lock().unwrap();\n inner.watch(oper, cx);\n self.is_empty.store(\n inner.selectors.is_empty() && inner.observers.is_empty(),\n Ordering::SeqCst,\n );\n }\n\n /// Unregisters an operation waiting to be ready.\n #[inline]\n pub(crate) fn unwatch(&self, oper: Operation) {\n let mut inner = self.inner.lock().unwrap();\n inner.unwatch(oper);\n self.is_empty.store(\n inner.selectors.is_empty() && inner.observers.is_empty(),\n Ordering::SeqCst,\n );\n }\n\n /// Notifies all threads that the channel is disconnected.\n #[inline]\n pub(crate) fn disconnect(&self) {\n let mut inner = self.inner.lock().unwrap();\n inner.disconnect();\n self.is_empty.store(\n inner.selectors.is_empty() && inner.observers.is_empty(),\n Ordering::SeqCst,\n );\n }\n}\n\nimpl Drop for SyncWaker {\n #[inline]\n fn drop(&mut self) {\n debug_assert!(self.is_empty.load(Ordering::SeqCst));\n }\n}\n\n/// Returns the id of the current thread.\n#[inline]\nfn current_thread_id() -> ThreadId {\n std::thread_local! {\n /// Cached thread-local id.\n static THREAD_ID: ThreadId = thread::current().id();\n }\n\n THREAD_ID\n .try_with(|id| *id)\n .unwrap_or_else(|_| thread::current().id())\n}\n" }, { "path": "crossbeam-channel/tests/after.rs", "content": "//! Tests for the after channel flavor.\n\n#![cfg(not(miri))] // TODO: many assertions failed due to Miri is slow\n\nuse std::{\n sync::atomic::{AtomicUsize, Ordering},\n thread,\n time::{Duration, Instant},\n};\n\nuse crossbeam_channel::{Select, TryRecvError, after, select};\nuse crossbeam_utils::thread::scope;\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\nfn fire() {\n let start = Instant::now();\n let r = after(ms(50));\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n thread::sleep(ms(100));\n\n let fired = r.try_recv().unwrap();\n assert!(start < fired);\n assert!(fired - start >= ms(50));\n\n let now = Instant::now();\n assert!(fired < now);\n assert!(now - fired >= ms(50));\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n\n select! {\n recv(r) -> _ => panic!(),\n default => {}\n }\n\n select! {\n recv(r) -> _ => panic!(),\n recv(after(ms(200))) -> _ => {}\n }\n}\n\n#[test]\nfn capacity() {\n const COUNT: usize = 10;\n\n for i in 0..COUNT {\n let r = after(ms(i as u64));\n assert_eq!(r.capacity(), Some(1));\n }\n}\n\n#[test]\nfn len_empty_full() {\n let r = after(ms(50));\n\n assert_eq!(r.len(), 0);\n assert!(r.is_empty());\n assert!(!r.is_full());\n\n thread::sleep(ms(100));\n\n assert_eq!(r.len(), 1);\n assert!(!r.is_empty());\n assert!(r.is_full());\n\n r.try_recv().unwrap();\n\n assert_eq!(r.len(), 0);\n assert!(r.is_empty());\n assert!(!r.is_full());\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn try_recv() {\n let r = after(ms(200));\n assert!(r.try_recv().is_err());\n\n thread::sleep(ms(100));\n assert!(r.try_recv().is_err());\n\n thread::sleep(ms(200));\n assert!(r.try_recv().is_ok());\n assert!(r.try_recv().is_err());\n\n thread::sleep(ms(200));\n assert!(r.try_recv().is_err());\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn recv() {\n let start = Instant::now();\n let r = after(ms(50));\n\n let fired = r.recv().unwrap();\n assert!(start < fired);\n assert!(fired - start >= ms(50));\n\n let now = Instant::now();\n assert!(fired < now);\n assert!(now - fired < fired - start);\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn recv_timeout() {\n let start = Instant::now();\n let r = after(ms(200));\n\n assert!(r.recv_timeout(ms(100)).is_err());\n let now = Instant::now();\n assert!(now - start >= ms(100));\n assert!(now - start <= ms(150));\n\n let fired = r.recv_timeout(ms(200)).unwrap();\n assert!(fired - start >= ms(200));\n assert!(fired - start <= ms(250));\n\n assert!(r.recv_timeout(ms(200)).is_err());\n let now = Instant::now();\n assert!(now - start >= ms(400));\n assert!(now - start <= ms(450));\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\nfn recv_two() {\n let r1 = after(ms(50));\n let r2 = after(ms(50));\n\n scope(|scope| {\n scope.spawn(|_| {\n select! {\n recv(r1) -> _ => {}\n recv(r2) -> _ => {}\n }\n });\n scope.spawn(|_| {\n select! {\n recv(r1) -> _ => {}\n recv(r2) -> _ => {}\n }\n });\n })\n .unwrap();\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn recv_race() {\n select! {\n recv(after(ms(50))) -> _ => {}\n recv(after(ms(100))) -> _ => panic!(),\n }\n\n select! {\n recv(after(ms(100))) -> _ => panic!(),\n recv(after(ms(50))) -> _ => {}\n }\n}\n\n#[test]\nfn stress_default() {\n const COUNT: usize = 10;\n\n for _ in 0..COUNT {\n select! {\n recv(after(ms(0))) -> _ => {}\n default => panic!(),\n }\n }\n\n for _ in 0..COUNT {\n select! {\n recv(after(ms(100))) -> _ => panic!(),\n default => {}\n }\n }\n}\n\n#[test]\nfn select() {\n const THREADS: usize = 4;\n const COUNT: usize = 1000;\n const TIMEOUT_MS: u64 = 100;\n\n let v = (0..COUNT)\n .map(|i| after(ms(i as u64 / TIMEOUT_MS / 2)))\n .collect::>();\n let hits = AtomicUsize::new(0);\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n let v: Vec<&_> = v.iter().collect();\n\n loop {\n let timeout = after(ms(TIMEOUT_MS));\n let mut sel = Select::new();\n for r in &v {\n sel.recv(r);\n }\n let oper_timeout = sel.recv(&timeout);\n\n let oper = sel.select();\n match oper.index() {\n i if i == oper_timeout => {\n oper.recv(&timeout).unwrap();\n break;\n }\n i => {\n oper.recv(v[i]).unwrap();\n hits.fetch_add(1, Ordering::SeqCst);\n }\n }\n }\n });\n }\n })\n .unwrap();\n\n assert_eq!(hits.load(Ordering::SeqCst), COUNT);\n}\n\n#[test]\nfn ready() {\n const THREADS: usize = 4;\n const COUNT: usize = 1000;\n const TIMEOUT_MS: u64 = 100;\n\n let v = (0..COUNT)\n .map(|i| after(ms(i as u64 / TIMEOUT_MS / 2)))\n .collect::>();\n let hits = AtomicUsize::new(0);\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n let v: Vec<&_> = v.iter().collect();\n\n loop {\n let timeout = after(ms(TIMEOUT_MS));\n let mut sel = Select::new();\n for r in &v {\n sel.recv(r);\n }\n let oper_timeout = sel.recv(&timeout);\n\n loop {\n let i = sel.ready();\n if i == oper_timeout {\n timeout.try_recv().unwrap();\n return;\n } else if v[i].try_recv().is_ok() {\n hits.fetch_add(1, Ordering::SeqCst);\n break;\n }\n }\n }\n });\n }\n })\n .unwrap();\n\n assert_eq!(hits.load(Ordering::SeqCst), COUNT);\n}\n\n#[test]\nfn stress_clone() {\n const RUNS: usize = 1000;\n const THREADS: usize = 10;\n const COUNT: usize = 50;\n\n for i in 0..RUNS {\n let r = after(ms(i as u64));\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n let r = r.clone();\n let _ = r.try_recv();\n\n for _ in 0..COUNT {\n drop(r.clone());\n thread::yield_now();\n }\n });\n }\n })\n .unwrap();\n }\n}\n\n#[test]\nfn fairness() {\n const COUNT: usize = 1000;\n\n for &dur in &[0, 1] {\n let mut hits = [0usize; 2];\n\n for _ in 0..COUNT {\n select! {\n recv(after(ms(dur))) -> _ => hits[0] += 1,\n recv(after(ms(dur))) -> _ => hits[1] += 1,\n }\n }\n\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n }\n}\n\n#[test]\nfn fairness_duplicates() {\n const COUNT: usize = 1000;\n\n for &dur in &[0, 1] {\n let mut hits = [0usize; 5];\n\n for _ in 0..COUNT {\n let r = after(ms(dur));\n select! {\n recv(r) -> _ => hits[0] += 1,\n recv(r) -> _ => hits[1] += 1,\n recv(r) -> _ => hits[2] += 1,\n recv(r) -> _ => hits[3] += 1,\n recv(r) -> _ => hits[4] += 1,\n }\n }\n\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n }\n}\n" }, { "path": "crossbeam-channel/tests/array.rs", "content": "//! Tests for the array channel flavor.\n\nuse std::{\n any::Any,\n sync::atomic::{AtomicUsize, Ordering},\n thread,\n time::Duration,\n};\n\nuse crossbeam_channel::{\n Receiver, RecvError, RecvTimeoutError, SendError, SendTimeoutError, TryRecvError, TrySendError,\n bounded, select,\n};\nuse crossbeam_utils::thread::scope;\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\nfn smoke() {\n let (s, r) = bounded(1);\n s.send(7).unwrap();\n assert_eq!(r.try_recv(), Ok(7));\n\n s.send(8).unwrap();\n assert_eq!(r.recv(), Ok(8));\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n assert_eq!(r.recv_timeout(ms(1000)), Err(RecvTimeoutError::Timeout));\n}\n\n#[test]\nfn capacity() {\n for i in 1..10 {\n let (s, r) = bounded::<()>(i);\n assert_eq!(s.capacity(), Some(i));\n assert_eq!(r.capacity(), Some(i));\n }\n}\n\n#[test]\nfn len_empty_full() {\n let (s, r) = bounded(2);\n\n assert_eq!(s.len(), 0);\n assert!(s.is_empty());\n assert!(!s.is_full());\n assert_eq!(r.len(), 0);\n assert!(r.is_empty());\n assert!(!r.is_full());\n\n s.send(()).unwrap();\n\n assert_eq!(s.len(), 1);\n assert!(!s.is_empty());\n assert!(!s.is_full());\n assert_eq!(r.len(), 1);\n assert!(!r.is_empty());\n assert!(!r.is_full());\n\n s.send(()).unwrap();\n\n assert_eq!(s.len(), 2);\n assert!(!s.is_empty());\n assert!(s.is_full());\n assert_eq!(r.len(), 2);\n assert!(!r.is_empty());\n assert!(r.is_full());\n\n r.recv().unwrap();\n\n assert_eq!(s.len(), 1);\n assert!(!s.is_empty());\n assert!(!s.is_full());\n assert_eq!(r.len(), 1);\n assert!(!r.is_empty());\n assert!(!r.is_full());\n}\n\n#[test]\nfn try_recv() {\n let (s, r) = bounded(100);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n thread::sleep(ms(1500));\n assert_eq!(r.try_recv(), Ok(7));\n thread::sleep(ms(500));\n assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n s.send(7).unwrap();\n });\n })\n .unwrap();\n}\n\n#[test]\nfn recv() {\n let (s, r) = bounded(100);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.recv(), Ok(7));\n thread::sleep(ms(1000));\n assert_eq!(r.recv(), Ok(8));\n thread::sleep(ms(1000));\n assert_eq!(r.recv(), Ok(9));\n assert_eq!(r.recv(), Err(RecvError));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n s.send(7).unwrap();\n s.send(8).unwrap();\n s.send(9).unwrap();\n });\n })\n .unwrap();\n}\n\n#[test]\nfn recv_timeout() {\n let (s, r) = bounded::(100);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.recv_timeout(ms(1000)), Err(RecvTimeoutError::Timeout));\n assert_eq!(r.recv_timeout(ms(1000)), Ok(7));\n assert_eq!(\n r.recv_timeout(ms(1000)),\n Err(RecvTimeoutError::Disconnected)\n );\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n s.send(7).unwrap();\n });\n })\n .unwrap();\n}\n\n#[test]\nfn try_send() {\n let (s, r) = bounded(1);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(s.try_send(1), Ok(()));\n assert_eq!(s.try_send(2), Err(TrySendError::Full(2)));\n thread::sleep(ms(1500));\n assert_eq!(s.try_send(3), Ok(()));\n thread::sleep(ms(500));\n assert_eq!(s.try_send(4), Err(TrySendError::Disconnected(4)));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n assert_eq!(r.try_recv(), Ok(1));\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n assert_eq!(r.recv(), Ok(3));\n });\n })\n .unwrap();\n}\n\n#[test]\nfn send() {\n let (s, r) = bounded(1);\n\n scope(|scope| {\n scope.spawn(|_| {\n s.send(7).unwrap();\n thread::sleep(ms(1000));\n s.send(8).unwrap();\n thread::sleep(ms(1000));\n s.send(9).unwrap();\n thread::sleep(ms(1000));\n s.send(10).unwrap();\n });\n scope.spawn(|_| {\n thread::sleep(ms(1500));\n assert_eq!(r.recv(), Ok(7));\n assert_eq!(r.recv(), Ok(8));\n assert_eq!(r.recv(), Ok(9));\n });\n })\n .unwrap();\n}\n\n#[test]\nfn send_timeout() {\n let (s, r) = bounded(2);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(s.send_timeout(1, ms(1000)), Ok(()));\n assert_eq!(s.send_timeout(2, ms(1000)), Ok(()));\n assert_eq!(\n s.send_timeout(3, ms(500)),\n Err(SendTimeoutError::Timeout(3))\n );\n thread::sleep(ms(1000));\n assert_eq!(s.send_timeout(4, ms(1000)), Ok(()));\n thread::sleep(ms(1000));\n assert_eq!(s.send(5), Err(SendError(5)));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n assert_eq!(r.recv(), Ok(1));\n thread::sleep(ms(1000));\n assert_eq!(r.recv(), Ok(2));\n assert_eq!(r.recv(), Ok(4));\n });\n })\n .unwrap();\n}\n\n#[test]\nfn send_after_disconnect() {\n let (s, r) = bounded(100);\n\n s.send(1).unwrap();\n s.send(2).unwrap();\n s.send(3).unwrap();\n\n drop(r);\n\n assert_eq!(s.send(4), Err(SendError(4)));\n assert_eq!(s.try_send(5), Err(TrySendError::Disconnected(5)));\n assert_eq!(\n s.send_timeout(6, ms(500)),\n Err(SendTimeoutError::Disconnected(6))\n );\n}\n\n#[test]\nfn recv_after_disconnect() {\n let (s, r) = bounded(100);\n\n s.send(1).unwrap();\n s.send(2).unwrap();\n s.send(3).unwrap();\n\n drop(s);\n\n assert_eq!(r.recv(), Ok(1));\n assert_eq!(r.recv(), Ok(2));\n assert_eq!(r.recv(), Ok(3));\n assert_eq!(r.recv(), Err(RecvError));\n}\n\n#[test]\nfn len() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 25_000 };\n const CAP: usize = if cfg!(miri) { 50 } else { 1000 };\n\n let (s, r) = bounded(CAP);\n\n assert_eq!(s.len(), 0);\n assert_eq!(r.len(), 0);\n\n for _ in 0..CAP / 10 {\n for i in 0..50 {\n s.send(i).unwrap();\n assert_eq!(s.len(), i + 1);\n }\n\n for i in 0..50 {\n r.recv().unwrap();\n assert_eq!(r.len(), 50 - i - 1);\n }\n }\n\n assert_eq!(s.len(), 0);\n assert_eq!(r.len(), 0);\n\n for i in 0..CAP {\n s.send(i).unwrap();\n assert_eq!(s.len(), i + 1);\n }\n\n for _ in 0..CAP {\n r.recv().unwrap();\n }\n\n assert_eq!(s.len(), 0);\n assert_eq!(r.len(), 0);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n assert_eq!(r.recv(), Ok(i));\n let len = r.len();\n assert!(len <= CAP);\n }\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n s.send(i).unwrap();\n let len = s.len();\n assert!(len <= CAP);\n }\n });\n })\n .unwrap();\n\n assert_eq!(s.len(), 0);\n assert_eq!(r.len(), 0);\n}\n\n#[test]\nfn disconnect_wakes_sender() {\n let (s, r) = bounded(1);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(s.send(()), Ok(()));\n assert_eq!(s.send(()), Err(SendError(())));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n drop(r);\n });\n })\n .unwrap();\n}\n\n#[test]\nfn disconnect_wakes_receiver() {\n let (s, r) = bounded::<()>(1);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.recv(), Err(RecvError));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n drop(s);\n });\n })\n .unwrap();\n}\n\n#[test]\nfn spsc() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 100_000 };\n\n let (s, r) = bounded(3);\n\n scope(|scope| {\n scope.spawn(move |_| {\n for i in 0..COUNT {\n assert_eq!(r.recv(), Ok(i));\n }\n assert_eq!(r.recv(), Err(RecvError));\n });\n scope.spawn(move |_| {\n for i in 0..COUNT {\n s.send(i).unwrap();\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn mpmc() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 25_000 };\n const THREADS: usize = 4;\n\n let (s, r) = bounded::(3);\n let v = (0..COUNT).map(|_| AtomicUsize::new(0)).collect::>();\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n let n = r.recv().unwrap();\n v[n].fetch_add(1, Ordering::SeqCst);\n }\n });\n }\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..COUNT {\n s.send(i).unwrap();\n }\n });\n }\n })\n .unwrap();\n\n for c in v {\n assert_eq!(c.load(Ordering::SeqCst), THREADS);\n }\n}\n\n#[test]\nfn stress_oneshot() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n for _ in 0..COUNT {\n let (s, r) = bounded(1);\n\n scope(|scope| {\n scope.spawn(|_| r.recv().unwrap());\n scope.spawn(|_| s.send(0).unwrap());\n })\n .unwrap();\n }\n}\n\n#[test]\nfn stress_iter() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 100_000 };\n\n let (request_s, request_r) = bounded(1);\n let (response_s, response_r) = bounded(1);\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut count = 0;\n loop {\n for x in response_r.try_iter() {\n count += x;\n if count == COUNT {\n return;\n }\n }\n request_s.send(()).unwrap();\n }\n });\n\n for _ in request_r.iter() {\n if response_s.send(1).is_err() {\n break;\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_timeout_two_threads() {\n const COUNT: usize = 100;\n\n let (s, r) = bounded(2);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(50));\n }\n loop {\n if let Ok(()) = s.send_timeout(i, ms(10)) {\n break;\n }\n }\n }\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(50));\n }\n loop {\n if let Ok(x) = r.recv_timeout(ms(10)) {\n assert_eq!(x, i);\n break;\n }\n }\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn drops() {\n const RUNS: usize = if cfg!(miri) { 10 } else { 100 };\n const STEPS: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n static DROPS: AtomicUsize = AtomicUsize::new(0);\n\n #[derive(Debug, PartialEq)]\n struct DropCounter;\n\n impl Drop for DropCounter {\n fn drop(&mut self) {\n DROPS.fetch_add(1, Ordering::SeqCst);\n }\n }\n\n let mut rng = fastrand::Rng::new();\n\n for _ in 0..RUNS {\n let steps = rng.usize(0..STEPS);\n let additional = rng.usize(0..50);\n\n DROPS.store(0, Ordering::SeqCst);\n let (s, r) = bounded::(50);\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n });\n\n scope.spawn(|_| {\n for _ in 0..steps {\n s.send(DropCounter).unwrap();\n }\n });\n })\n .unwrap();\n\n for _ in 0..additional {\n s.send(DropCounter).unwrap();\n }\n\n assert_eq!(DROPS.load(Ordering::SeqCst), steps);\n drop(s);\n drop(r);\n assert_eq!(DROPS.load(Ordering::SeqCst), steps + additional);\n }\n}\n\n#[test]\nfn linearizable() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 25_000 };\n const THREADS: usize = 4;\n\n let (s, r) = bounded(THREADS);\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n s.send(0).unwrap();\n r.try_recv().unwrap();\n }\n });\n }\n })\n .unwrap();\n}\n\n#[test]\nfn fairness() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = bounded::<()>(COUNT);\n let (s2, r2) = bounded::<()>(COUNT);\n\n for _ in 0..COUNT {\n s1.send(()).unwrap();\n s2.send(()).unwrap();\n }\n\n let mut hits = [0usize; 2];\n for _ in 0..COUNT {\n select! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n}\n\n#[test]\nfn fairness_duplicates() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s, r) = bounded::<()>(COUNT);\n\n for _ in 0..COUNT {\n s.send(()).unwrap();\n }\n\n let mut hits = [0usize; 5];\n for _ in 0..COUNT {\n select! {\n recv(r) -> _ => hits[0] += 1,\n recv(r) -> _ => hits[1] += 1,\n recv(r) -> _ => hits[2] += 1,\n recv(r) -> _ => hits[3] += 1,\n recv(r) -> _ => hits[4] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n}\n\n#[test]\nfn recv_in_send() {\n let (s, _r) = bounded(1);\n s.send(()).unwrap();\n\n #[allow(unreachable_code, clippy::diverging_sub_expression)]\n {\n select! {\n send(s, panic!()) -> _ => panic!(),\n default => {}\n }\n }\n\n let (s, r) = bounded(2);\n s.send(()).unwrap();\n\n select! {\n send(s, assert_eq!(r.recv(), Ok(()))) -> _ => {}\n }\n}\n\n#[test]\nfn channel_through_channel() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 1000 };\n\n type T = Box;\n\n let (s, r) = bounded::(1);\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut s = s;\n\n for _ in 0..COUNT {\n let (new_s, new_r) = bounded(1);\n let new_r: T = Box::new(Some(new_r));\n\n s.send(new_r).unwrap();\n s = new_s;\n }\n });\n\n scope.spawn(move |_| {\n let mut r = r;\n\n for _ in 0..COUNT {\n r = r\n .recv()\n .unwrap()\n .downcast_mut::>>()\n .unwrap()\n .take()\n .unwrap()\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn panic_on_drop() {\n struct Msg1<'a>(&'a mut bool);\n impl Drop for Msg1<'_> {\n fn drop(&mut self) {\n if *self.0 && !std::thread::panicking() {\n panic!(\"double drop\");\n } else {\n *self.0 = true;\n }\n }\n }\n\n struct Msg2<'a>(&'a mut bool);\n impl Drop for Msg2<'_> {\n fn drop(&mut self) {\n if *self.0 {\n panic!(\"double drop\");\n } else {\n *self.0 = true;\n panic!(\"first drop\");\n }\n }\n }\n\n // normal\n let (s, r) = bounded(2);\n let (mut a, mut b) = (false, false);\n s.send(Msg1(&mut a)).unwrap();\n s.send(Msg1(&mut b)).unwrap();\n drop(s);\n drop(r);\n assert!(a);\n assert!(b);\n\n // panic on drop\n let (s, r) = bounded(2);\n let (mut a, mut b) = (false, false);\n s.send(Msg2(&mut a)).unwrap();\n s.send(Msg2(&mut b)).unwrap();\n drop(s);\n let res = std::panic::catch_unwind(move || {\n drop(r);\n });\n assert_eq!(\n *res.unwrap_err().downcast_ref::<&str>().unwrap(),\n \"first drop\"\n );\n assert!(a);\n // Elements after the panicked element will leak.\n assert!(!b);\n}\n" }, { "path": "crossbeam-channel/tests/golang.rs", "content": "//! Tests copied from Go and manually rewritten in Rust.\n//!\n//! Source:\n//! - https://github.com/golang/go\n//!\n//! Copyright & License:\n//! - Copyright (c) 2009 The Go Authors\n//! - https://golang.org/AUTHORS\n//! - https://golang.org/LICENSE\n//! - https://golang.org/PATENTS\n\n#![allow(clippy::redundant_clone)]\n\nuse std::{\n alloc::{GlobalAlloc, Layout, System},\n any::Any,\n cell::Cell,\n collections::HashMap,\n sync::{\n Arc, Condvar, Mutex,\n atomic::{AtomicI32, AtomicUsize, Ordering::SeqCst},\n },\n thread,\n time::Duration,\n};\n\nuse crossbeam_channel::{Receiver, Select, Sender, bounded, never, select, tick, unbounded};\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\nstruct Chan {\n inner: Arc>>,\n}\n\nstruct ChanInner {\n s: Option>,\n r: Option>,\n // Receiver to use when r is None (Go blocks on receiving from nil)\n nil_r: Receiver,\n // Sender to use when s is None (Go blocks on sending to nil)\n nil_s: Sender,\n // Hold this receiver to prevent nil sender channel from disconnection\n _nil_sr: Receiver,\n}\n\nimpl Clone for Chan {\n fn clone(&self) -> Self {\n Self {\n inner: self.inner.clone(),\n }\n }\n}\n\nimpl Chan {\n fn send(&self, msg: T) {\n let s = self\n .inner\n .lock()\n .unwrap()\n .s\n .as_ref()\n .expect(\"sending into closed channel\")\n .clone();\n let _ = s.send(msg);\n }\n\n fn try_recv(&self) -> Option {\n let r = self.inner.lock().unwrap().r.as_ref().unwrap().clone();\n r.try_recv().ok()\n }\n\n fn recv(&self) -> Option {\n let r = self.inner.lock().unwrap().r.as_ref().unwrap().clone();\n r.recv().ok()\n }\n\n fn close_s(&self) {\n self.inner\n .lock()\n .unwrap()\n .s\n .take()\n .expect(\"channel sender already closed\");\n }\n\n fn close_r(&self) {\n self.inner\n .lock()\n .unwrap()\n .r\n .take()\n .expect(\"channel receiver already closed\");\n }\n\n fn has_rx(&self) -> bool {\n self.inner.lock().unwrap().r.is_some()\n }\n\n fn has_tx(&self) -> bool {\n self.inner.lock().unwrap().s.is_some()\n }\n\n fn rx(&self) -> Receiver {\n let inner = self.inner.lock().unwrap();\n match inner.r.as_ref() {\n None => inner.nil_r.clone(),\n Some(r) => r.clone(),\n }\n }\n\n fn tx(&self) -> Sender {\n let inner = self.inner.lock().unwrap();\n match inner.s.as_ref() {\n None => inner.nil_s.clone(),\n Some(s) => s.clone(),\n }\n }\n}\n\nimpl Iterator for Chan {\n type Item = T;\n\n fn next(&mut self) -> Option {\n self.recv()\n }\n}\n\nimpl IntoIterator for &Chan {\n type Item = T;\n type IntoIter = Chan;\n\n fn into_iter(self) -> Self::IntoIter {\n self.clone()\n }\n}\n\nfn make(cap: usize) -> Chan {\n let (s, r) = bounded(cap);\n let (nil_s, _nil_sr) = bounded(0);\n Chan {\n inner: Arc::new(Mutex::new(ChanInner {\n s: Some(s),\n r: Some(r),\n nil_r: never(),\n nil_s,\n _nil_sr,\n })),\n }\n}\n\nfn make_unbounded() -> Chan {\n let (s, r) = unbounded();\n let (nil_s, _nil_sr) = bounded(0);\n Chan {\n inner: Arc::new(Mutex::new(ChanInner {\n s: Some(s),\n r: Some(r),\n nil_r: never(),\n nil_s,\n _nil_sr,\n })),\n }\n}\n\n#[derive(Clone)]\nstruct WaitGroup(Arc);\n\nstruct WaitGroupInner {\n cond: Condvar,\n count: Mutex,\n}\n\nimpl WaitGroup {\n fn new() -> Self {\n Self(Arc::new(WaitGroupInner {\n cond: Condvar::new(),\n count: Mutex::new(0),\n }))\n }\n\n fn add(&self, delta: i32) {\n let mut count = self.0.count.lock().unwrap();\n *count += delta;\n assert!(*count >= 0);\n self.0.cond.notify_all();\n }\n\n fn done(&self) {\n self.add(-1);\n }\n\n fn wait(&self) {\n let mut count = self.0.count.lock().unwrap();\n while *count > 0 {\n count = self.0.cond.wait(count).unwrap();\n }\n }\n}\n\nstruct Defer {\n f: Option>,\n}\n\nimpl Drop for Defer {\n fn drop(&mut self) {\n let f = self.f.take().unwrap();\n let mut f = Some(f);\n let mut f = move || f.take().unwrap()();\n f();\n }\n}\n\nstruct Counter;\n\nstatic ALLOCATED: AtomicUsize = AtomicUsize::new(0);\nunsafe impl GlobalAlloc for Counter {\n unsafe fn alloc(&self, layout: Layout) -> *mut u8 {\n let ret = System.alloc(layout);\n if !ret.is_null() {\n ALLOCATED.fetch_add(layout.size(), SeqCst);\n }\n ret\n }\n\n unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {\n System.dealloc(ptr, layout);\n ALLOCATED.fetch_sub(layout.size(), SeqCst);\n }\n}\n\n#[global_allocator]\nstatic A: Counter = Counter;\n\nmacro_rules! defer {\n ($body:expr) => {\n let _defer = Defer {\n f: Some(Box::new(|| $body)),\n };\n };\n}\n\nmacro_rules! go {\n (@parse $v:ident, $($tail:tt)*) => {{\n let $v = $v.clone();\n go!(@parse $($tail)*)\n }};\n (@parse $body:expr) => {\n ::std::thread::spawn(move || {\n let res = ::std::panic::catch_unwind(::std::panic::AssertUnwindSafe(|| {\n $body\n }));\n if res.is_err() {\n eprintln!(\"goroutine panicked: {:?}\", res);\n ::std::process::abort();\n }\n })\n };\n (@parse $($tail:tt)*) => {\n compile_error!(\"invalid `go!` syntax\")\n };\n ($($tail:tt)*) => {{\n go!(@parse $($tail)*)\n }};\n}\n\n// https://github.com/golang/go/blob/master/test/chan/doubleselect.go\nmod doubleselect {\n use super::*;\n\n const ITERATIONS: i32 = if cfg!(miri) { 100 } else { 10_000 };\n\n fn sender(n: i32, c1: Chan, c2: Chan, c3: Chan, c4: Chan) {\n defer! { c1.close_s() }\n defer! { c2.close_s() }\n defer! { c3.close_s() }\n defer! { c4.close_s() }\n\n for i in 0..n {\n select! {\n send(c1.tx(), i) -> _ => {}\n send(c2.tx(), i) -> _ => {}\n send(c3.tx(), i) -> _ => {}\n send(c4.tx(), i) -> _ => {}\n }\n }\n }\n\n fn mux(out: Chan, inp: Chan, done: Chan) {\n for v in inp {\n out.send(v);\n }\n done.send(true);\n }\n\n fn recver(inp: Chan) {\n let mut seen = HashMap::new();\n\n for v in &inp {\n if seen.contains_key(&v) {\n panic!(\"got duplicate value for {}\", v);\n }\n seen.insert(v, true);\n }\n }\n\n #[test]\n fn main() {\n let c1 = make::(0);\n let c2 = make::(0);\n let c3 = make::(0);\n let c4 = make::(0);\n let done = make::(0);\n let cmux = make::(0);\n\n go!(c1, c2, c3, c4, sender(ITERATIONS, c1, c2, c3, c4));\n go!(cmux, c1, done, mux(cmux, c1, done));\n go!(cmux, c2, done, mux(cmux, c2, done));\n go!(cmux, c3, done, mux(cmux, c3, done));\n go!(cmux, c4, done, mux(cmux, c4, done));\n go!(done, cmux, {\n done.recv();\n done.recv();\n done.recv();\n done.recv();\n cmux.close_s();\n });\n recver(cmux);\n }\n}\n\n// https://github.com/golang/go/blob/master/test/chan/fifo.go\nmod fifo {\n use super::*;\n\n const N: i32 = 10;\n\n #[test]\n fn asynch_fifo() {\n let ch = make::(N as usize);\n for i in 0..N {\n ch.send(i);\n }\n for i in 0..N {\n if ch.recv() != Some(i) {\n panic!(\"bad receive\");\n }\n }\n }\n\n fn chain(ch: Chan, val: i32, inp: Chan, out: Chan) {\n inp.recv();\n if ch.recv() != Some(val) {\n panic!(\"{}\", val);\n }\n out.send(1);\n }\n\n #[test]\n fn synch_fifo() {\n let ch = make::(0);\n let mut inp = make::(0);\n let start = inp.clone();\n\n for i in 0..N {\n let out = make::(0);\n go!(ch, i, inp, out, chain(ch, i, inp, out));\n inp = out;\n }\n\n start.send(0);\n for i in 0..N {\n ch.send(i);\n }\n inp.recv();\n }\n}\n\n// https://github.com/golang/go/blob/master/test/chan/goroutines.go\nmod goroutines {\n use super::*;\n\n fn f(left: Chan, right: Chan) {\n left.send(right.recv().unwrap());\n }\n\n #[test]\n fn main() {\n let n = 100i32;\n\n let leftmost = make::(0);\n let mut right = leftmost.clone();\n let mut left = leftmost.clone();\n\n for _ in 0..n {\n right = make::(0);\n go!(left, right, f(left, right));\n left = right.clone();\n }\n\n go!(right, right.send(1));\n leftmost.recv().unwrap();\n }\n}\n\n// https://github.com/golang/go/blob/master/test/chan/nonblock.go\nmod nonblock {\n use super::*;\n\n fn i32receiver(c: Chan, strobe: Chan) {\n if c.recv().unwrap() != 123 {\n panic!(\"i32 value\");\n }\n strobe.send(true);\n }\n\n fn i32sender(c: Chan, strobe: Chan) {\n c.send(234);\n strobe.send(true);\n }\n\n fn i64receiver(c: Chan, strobe: Chan) {\n if c.recv().unwrap() != 123456 {\n panic!(\"i64 value\");\n }\n strobe.send(true);\n }\n\n fn i64sender(c: Chan, strobe: Chan) {\n c.send(234567);\n strobe.send(true);\n }\n\n fn breceiver(c: Chan, strobe: Chan) {\n if !c.recv().unwrap() {\n panic!(\"b value\");\n }\n strobe.send(true);\n }\n\n fn bsender(c: Chan, strobe: Chan) {\n c.send(true);\n strobe.send(true);\n }\n\n fn sreceiver(c: Chan, strobe: Chan) {\n if c.recv().unwrap() != \"hello\" {\n panic!(\"x value\");\n }\n strobe.send(true);\n }\n\n fn ssender(c: Chan, strobe: Chan) {\n c.send(\"hello again\".to_string());\n strobe.send(true);\n }\n\n const MAX_TRIES: usize = 10000; // Up to 100ms per test.\n\n #[test]\n fn main() {\n let ticker = tick(Duration::new(0, 10_000)); // 10 us\n let sleep = || {\n ticker.recv().unwrap();\n ticker.recv().unwrap();\n thread::yield_now();\n thread::yield_now();\n thread::yield_now();\n };\n\n let sync = make::(0);\n\n for buffer in 0..2 {\n let c32 = make::(buffer);\n let c64 = make::(buffer);\n let cb = make::(buffer);\n let cs = make::(buffer);\n\n select! {\n recv(c32.rx()) -> _ => panic!(\"blocked i32sender\"),\n default => {}\n }\n\n select! {\n recv(c64.rx()) -> _ => panic!(\"blocked i64sender\"),\n default => {}\n }\n\n select! {\n recv(cb.rx()) -> _ => panic!(\"blocked bsender\"),\n default => {}\n }\n\n select! {\n recv(cs.rx()) -> _ => panic!(\"blocked ssender\"),\n default => {}\n }\n\n go!(c32, sync, i32receiver(c32, sync));\n let mut r#try = 0;\n loop {\n select! {\n send(c32.tx(), 123) -> _ => break,\n default => {\n r#try += 1;\n if r#try > MAX_TRIES {\n println!(\"i32receiver buffer={}\", buffer);\n panic!(\"fail\")\n }\n sleep();\n }\n }\n }\n sync.recv();\n go!(c32, sync, i32sender(c32, sync));\n if buffer > 0 {\n sync.recv();\n }\n let mut r#try = 0;\n loop {\n select! {\n recv(c32.rx()) -> v => {\n if v != Ok(234) {\n panic!(\"i32sender value\");\n }\n break;\n }\n default => {\n r#try += 1;\n if r#try > MAX_TRIES {\n println!(\"i32sender buffer={}\", buffer);\n panic!(\"fail\");\n }\n sleep();\n }\n }\n }\n if buffer == 0 {\n sync.recv();\n }\n\n go!(c64, sync, i64receiver(c64, sync));\n let mut r#try = 0;\n loop {\n select! {\n send(c64.tx(), 123456) -> _ => break,\n default => {\n r#try += 1;\n if r#try > MAX_TRIES {\n println!(\"i64receiver buffer={}\", buffer);\n panic!(\"fail\")\n }\n sleep();\n }\n }\n }\n sync.recv();\n go!(c64, sync, i64sender(c64, sync));\n if buffer > 0 {\n sync.recv();\n }\n let mut r#try = 0;\n loop {\n select! {\n recv(c64.rx()) -> v => {\n if v != Ok(234567) {\n panic!(\"i64sender value\");\n }\n break;\n }\n default => {\n r#try += 1;\n if r#try > MAX_TRIES {\n println!(\"i64sender buffer={}\", buffer);\n panic!(\"fail\");\n }\n sleep();\n }\n }\n }\n if buffer == 0 {\n sync.recv();\n }\n\n go!(cb, sync, breceiver(cb, sync));\n let mut r#try = 0;\n loop {\n select! {\n send(cb.tx(), true) -> _ => break,\n default => {\n r#try += 1;\n if r#try > MAX_TRIES {\n println!(\"breceiver buffer={}\", buffer);\n panic!(\"fail\")\n }\n sleep();\n }\n }\n }\n sync.recv();\n go!(cb, sync, bsender(cb, sync));\n if buffer > 0 {\n sync.recv();\n }\n let mut r#try = 0;\n loop {\n select! {\n recv(cb.rx()) -> v => {\n if v != Ok(true) {\n panic!(\"bsender value\");\n }\n break;\n }\n default => {\n r#try += 1;\n if r#try > MAX_TRIES {\n println!(\"bsender buffer={}\", buffer);\n panic!(\"fail\");\n }\n sleep();\n }\n }\n }\n if buffer == 0 {\n sync.recv();\n }\n\n go!(cs, sync, sreceiver(cs, sync));\n let mut r#try = 0;\n loop {\n select! {\n send(cs.tx(), \"hello\".to_string()) -> _ => break,\n default => {\n r#try += 1;\n if r#try > MAX_TRIES {\n println!(\"sreceiver buffer={}\", buffer);\n panic!(\"fail\")\n }\n sleep();\n }\n }\n }\n sync.recv();\n go!(cs, sync, ssender(cs, sync));\n if buffer > 0 {\n sync.recv();\n }\n let mut r#try = 0;\n loop {\n select! {\n recv(cs.rx()) -> v => {\n if v != Ok(\"hello again\".to_string()) {\n panic!(\"ssender value\");\n }\n break;\n }\n default => {\n r#try += 1;\n if r#try > MAX_TRIES {\n println!(\"ssender buffer={}\", buffer);\n panic!(\"fail\");\n }\n sleep();\n }\n }\n }\n if buffer == 0 {\n sync.recv();\n }\n }\n }\n}\n\n// https://github.com/golang/go/blob/master/test/chan/select.go\nmod select {\n use super::*;\n\n #[test]\n fn main() {\n let shift = Cell::new(0);\n let counter = Cell::new(0);\n\n let get_value = || {\n counter.set(counter.get() + 1);\n 1 << shift.get()\n };\n\n let send = |mut a: Option<&Chan>, mut b: Option<&Chan>| {\n let mut i = 0;\n let never = make::(0);\n loop {\n let nil1 = never.tx();\n let nil2 = never.tx();\n let v1 = get_value();\n let v2 = get_value();\n select! {\n send(a.map(|c| c.tx()).unwrap_or(nil1), v1) -> _ => {\n i += 1;\n a = None;\n }\n send(b.map(|c| c.tx()).unwrap_or(nil2), v2) -> _ => {\n i += 1;\n b = None;\n }\n default => break,\n }\n shift.set(shift.get() + 1);\n }\n i\n };\n\n let a = make::(1);\n let b = make::(1);\n\n assert_eq!(send(Some(&a), Some(&b)), 2);\n\n let av = a.recv().unwrap();\n let bv = b.recv().unwrap();\n assert_eq!(av | bv, 3);\n\n assert_eq!(send(Some(&a), None), 1);\n assert_eq!(counter.get(), 10);\n }\n}\n\n// https://github.com/golang/go/blob/master/test/chan/select2.go\nmod select2 {\n use super::*;\n\n const N: i32 = if cfg!(miri) { 200 } else { 100000 };\n\n #[test]\n fn main() {\n fn sender(c: &Chan, n: i32) {\n for _ in 0..n {\n c.send(1);\n }\n }\n\n fn receiver(c: &Chan, dummy: &Chan, n: i32) {\n for _ in 0..n {\n select! {\n recv(c.rx()) -> _ => {}\n recv(dummy.rx()) -> _ => {\n panic!(\"dummy\");\n }\n }\n }\n }\n\n let c = make_unbounded::();\n let dummy = make_unbounded::();\n\n go!(c, sender(&c, N));\n receiver(&c, &dummy, N);\n\n let alloc = ALLOCATED.load(SeqCst);\n\n go!(c, sender(&c, N));\n receiver(&c, &dummy, N);\n\n let final_alloc = ALLOCATED.load(SeqCst);\n\n assert!(!(final_alloc > alloc && final_alloc - alloc > N as usize + 10000));\n }\n}\n\n// https://github.com/golang/go/blob/master/test/chan/select3.go\nmod select3 {\n // TODO\n}\n\n// https://github.com/golang/go/blob/master/test/chan/select4.go\nmod select4 {\n use super::*;\n\n #[test]\n fn main() {\n let c = make::(1);\n let c1 = make::(0);\n c.send(42);\n select! {\n recv(c1.rx()) -> _ => panic!(\"BUG\"),\n recv(c.rx()) -> v => assert_eq!(v, Ok(42)),\n }\n }\n}\n\n// https://github.com/golang/go/blob/master/test/chan/select6.go\nmod select6 {\n use super::*;\n\n #[test]\n fn main() {\n let c1 = make::(0);\n let c2 = make::(0);\n let c3 = make::(0);\n\n go!(c1, c1.recv());\n go!(c1, c2, c3, {\n select! {\n recv(c1.rx()) -> _ => panic!(\"dummy\"),\n recv(c2.rx()) -> _ => c3.send(true),\n }\n c1.recv();\n });\n go!(c2, c2.send(true));\n\n c3.recv();\n c1.send(true);\n c1.send(true);\n }\n}\n\n// https://github.com/golang/go/blob/master/test/chan/select7.go\nmod select7 {\n use super::*;\n\n fn recv1(c: Chan) {\n c.recv().unwrap();\n }\n\n fn recv2(c: Chan) {\n select! {\n recv(c.rx()) -> _ => ()\n }\n }\n\n fn recv3(c: Chan) {\n let c2 = make::(1);\n select! {\n recv(c.rx()) -> _ => (),\n recv(c2.rx()) -> _ => ()\n }\n }\n\n fn send1(recv: fn(Chan)) {\n let c = make::(1);\n go!(c, recv(c));\n thread::yield_now();\n c.send(1);\n }\n\n fn send2(recv: fn(Chan)) {\n let c = make::(1);\n go!(c, recv(c));\n thread::yield_now();\n select! {\n send(c.tx(), 1) -> _ => ()\n }\n }\n\n fn send3(recv: fn(Chan)) {\n let c = make::(1);\n go!(c, recv(c));\n thread::yield_now();\n let c2 = make::(1);\n select! {\n send(c.tx(), 1) -> _ => (),\n send(c2.tx(), 1) -> _ => ()\n }\n }\n\n #[test]\n fn main() {\n // https://github.com/rust-lang/miri/issues/1371\n if option_env!(\"MIRI_LEAK_CHECK\").is_some() {\n return;\n }\n\n send1(recv1);\n send2(recv1);\n send3(recv1);\n send1(recv2);\n send2(recv2);\n send3(recv2);\n send1(recv3);\n send2(recv3);\n send3(recv3);\n }\n}\n\n// https://github.com/golang/go/blob/master/test/chan/sieve1.go\nmod sieve1 {\n use super::*;\n\n fn generate(ch: Chan) {\n let mut i = 2;\n loop {\n ch.send(i);\n i += 1;\n }\n }\n\n fn filter(in_ch: Chan, out_ch: Chan, prime: i32) {\n for i in in_ch {\n if i % prime != 0 {\n out_ch.send(i);\n }\n }\n }\n\n fn sieve(primes: Chan) {\n let mut ch = make::(1);\n go!(ch, generate(ch));\n loop {\n let prime = ch.recv().unwrap();\n primes.send(prime);\n\n let ch1 = make::(1);\n go!(ch, ch1, prime, filter(ch, ch1, prime));\n ch = ch1;\n }\n }\n\n #[test]\n fn main() {\n // https://github.com/rust-lang/miri/issues/1371\n if option_env!(\"MIRI_LEAK_CHECK\").is_some() {\n return;\n }\n\n let primes = make::(1);\n go!(primes, sieve(primes));\n\n let a = [\n 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83,\n 89, 97,\n ];\n #[cfg(miri)]\n let a = &a[..10];\n\n for item in a.iter() {\n let x = primes.recv().unwrap();\n if x != *item {\n println!(\"{} != {}\", x, item);\n panic!(\"fail\");\n }\n }\n }\n}\n\n// https://github.com/golang/go/blob/master/test/chan/zerosize.go\nmod zerosize {\n use super::*;\n\n #[test]\n fn zero_size_struct() {\n struct ZeroSize;\n let _ = make::(0);\n }\n\n #[test]\n fn zero_size_array() {\n let _ = make::<[u8; 0]>(0);\n }\n}\n\n// https://github.com/golang/go/blob/master/src/runtime/chan_test.go\nmod chan_test {\n use super::*;\n\n #[test]\n fn test_chan() {\n const N: i32 = if cfg!(miri) { 12 } else { 200 };\n\n const MESSAGES_COUNT: i32 = if cfg!(miri) { 20 } else { 100 };\n\n for cap in 0..N {\n {\n // Ensure that receive from empty chan blocks.\n let c = make::(cap as usize);\n\n let recv1 = Arc::new(Mutex::new(false));\n go!(c, recv1, {\n c.recv();\n *recv1.lock().unwrap() = true;\n });\n\n let recv2 = Arc::new(Mutex::new(false));\n go!(c, recv2, {\n c.recv();\n *recv2.lock().unwrap() = true;\n });\n\n thread::sleep(ms(1));\n\n if *recv1.lock().unwrap() || *recv2.lock().unwrap() {\n panic!();\n }\n\n // Ensure that non-blocking receive does not block.\n select! {\n recv(c.rx()) -> _ => panic!(),\n default => {}\n }\n select! {\n recv(c.rx()) -> _ => panic!(),\n default => {}\n }\n\n c.send(0);\n c.send(0);\n }\n\n {\n // Ensure that send to full chan blocks.\n let c = make::(cap as usize);\n for i in 0..cap {\n c.send(i);\n }\n\n let sent = Arc::new(Mutex::new(0));\n go!(sent, c, {\n c.send(0);\n *sent.lock().unwrap() = 1;\n });\n\n thread::sleep(ms(1));\n\n if *sent.lock().unwrap() != 0 {\n panic!();\n }\n\n // Ensure that non-blocking send does not block.\n select! {\n send(c.tx(), 0) -> _ => panic!(),\n default => {}\n }\n c.recv();\n }\n\n {\n // Ensure that we receive 0 from closed chan.\n let c = make::(cap as usize);\n for i in 0..cap {\n c.send(i);\n }\n c.close_s();\n\n for i in 0..cap {\n let v = c.recv();\n if v != Some(i) {\n panic!();\n }\n }\n\n if c.recv().is_some() {\n panic!();\n }\n if c.try_recv().is_some() {\n panic!();\n }\n }\n\n {\n // Ensure that close unblocks receive.\n let c = make::(cap as usize);\n let done = make::(0);\n\n go!(c, done, {\n let v = c.try_recv();\n done.send(v.is_none());\n });\n\n thread::sleep(ms(1));\n c.close_s();\n\n if !done.recv().unwrap() {\n panic!();\n }\n }\n\n {\n // Send many integers,\n // ensure that we receive them non-corrupted in FIFO order.\n let c = make::(cap as usize);\n go!(c, {\n for i in 0..MESSAGES_COUNT {\n c.send(i);\n }\n });\n for i in 0..MESSAGES_COUNT {\n if c.recv() != Some(i) {\n panic!();\n }\n }\n\n // Same, but using recv2.\n go!(c, {\n for i in 0..MESSAGES_COUNT {\n c.send(i);\n }\n });\n for i in 0..MESSAGES_COUNT {\n if c.recv() != Some(i) {\n panic!();\n }\n }\n }\n }\n }\n\n #[test]\n fn test_nonblock_recv_race() {\n const N: usize = if cfg!(miri) { 100 } else { 1000 };\n\n for _ in 0..N {\n let c = make::(1);\n c.send(1);\n\n let t = go!(c, {\n select! {\n recv(c.rx()) -> _ => {}\n default => panic!(\"chan is not ready\"),\n }\n });\n\n c.close_s();\n c.recv();\n t.join().unwrap();\n }\n }\n\n #[test]\n fn test_nonblock_select_race() {\n const N: usize = if cfg!(miri) { 100 } else { 1000 };\n\n let done = make::(1);\n for _ in 0..N {\n let c1 = make::(1);\n let c2 = make::(1);\n c1.send(1);\n\n go!(c1, c2, done, {\n select! {\n recv(c1.rx()) -> _ => {}\n recv(c2.rx()) -> _ => {}\n default => {\n done.send(false);\n return;\n }\n }\n done.send(true);\n });\n\n c2.send(1);\n select! {\n recv(c1.rx()) -> _ => {}\n default => {}\n }\n if !done.recv().unwrap() {\n panic!(\"no chan is ready\");\n }\n }\n }\n\n #[test]\n fn test_nonblock_select_race2() {\n const N: usize = if cfg!(miri) { 100 } else { 1000 };\n\n let done = make::(1);\n for _ in 0..N {\n let c1 = make::(1);\n let c2 = make::(0);\n c1.send(1);\n\n go!(c1, c2, done, {\n select! {\n recv(c1.rx()) -> _ => {}\n recv(c2.rx()) -> _ => {}\n default => {\n done.send(false);\n return;\n }\n }\n done.send(true);\n });\n\n c2.close_s();\n select! {\n recv(c1.rx()) -> _ => {}\n default => {}\n }\n if !done.recv().unwrap() {\n panic!(\"no chan is ready\");\n }\n }\n }\n\n #[test]\n fn test_self_select() {\n // Ensure that send/recv on the same chan in select\n // does not crash nor deadlock.\n\n const N: usize = if cfg!(miri) { 100 } else { 1000 };\n\n for &cap in &[0, 10] {\n let wg = WaitGroup::new();\n wg.add(2);\n let c = make::(cap);\n\n for p in 0..2 {\n go!(wg, p, c, {\n defer! { wg.done() }\n for i in 0..N {\n if p == 0 || i % 2 == 0 {\n select! {\n send(c.tx(), p) -> _ => {}\n recv(c.rx()) -> v => {\n if cap == 0 && v.ok() == Some(p) {\n panic!(\"self receive\");\n }\n }\n }\n } else {\n select! {\n recv(c.rx()) -> v => {\n if cap == 0 && v.ok() == Some(p) {\n panic!(\"self receive\");\n }\n }\n send(c.tx(), p) -> _ => {}\n }\n }\n }\n });\n }\n wg.wait();\n }\n }\n\n #[test]\n fn test_select_stress() {\n const N: usize = if cfg!(miri) { 100 } else { 10000 };\n\n let c = vec![\n make::(0),\n make::(0),\n make::(2),\n make::(3),\n ];\n\n // There are 4 goroutines that send N values on each of the chans,\n // + 4 goroutines that receive N values on each of the chans,\n // + 1 goroutine that sends N values on each of the chans in a single select,\n // + 1 goroutine that receives N values on each of the chans in a single select.\n // All these sends, receives and selects interact chaotically at runtime,\n // but we are careful that this whole construct does not deadlock.\n let wg = WaitGroup::new();\n wg.add(10);\n\n for k in 0..4 {\n go!(k, c, wg, {\n for _ in 0..N {\n c[k].send(0);\n }\n wg.done();\n });\n go!(k, c, wg, {\n for _ in 0..N {\n c[k].recv();\n }\n wg.done();\n });\n }\n\n go!(c, wg, {\n let mut n = [0; 4];\n let mut c1 = c.iter().map(|c| Some(c.rx().clone())).collect::>();\n\n for _ in 0..4 * N {\n let index = {\n let mut sel = Select::new();\n let mut opers = [!0; 4];\n for &i in &[3, 2, 0, 1] {\n if let Some(c) = &c1[i] {\n opers[i] = sel.recv(c);\n }\n }\n\n let oper = sel.select();\n let mut index = !0;\n for i in 0..4 {\n if opers[i] == oper.index() {\n index = i;\n let _ = oper.recv(c1[i].as_ref().unwrap());\n break;\n }\n }\n index\n };\n\n n[index] += 1;\n if n[index] == N {\n c1[index] = None;\n }\n }\n wg.done();\n });\n\n go!(c, wg, {\n let mut n = [0; 4];\n let mut c1 = c.iter().map(|c| Some(c.tx().clone())).collect::>();\n\n for _ in 0..4 * N {\n let index = {\n let mut sel = Select::new();\n let mut opers = [!0; 4];\n for &i in &[0, 1, 2, 3] {\n if let Some(c) = &c1[i] {\n opers[i] = sel.send(c);\n }\n }\n\n let oper = sel.select();\n let mut index = !0;\n for i in 0..4 {\n if opers[i] == oper.index() {\n index = i;\n let _ = oper.send(c1[i].as_ref().unwrap(), 0);\n break;\n }\n }\n index\n };\n\n n[index] += 1;\n if n[index] == N {\n c1[index] = None;\n }\n }\n wg.done();\n });\n\n wg.wait();\n }\n\n #[test]\n fn test_select_fairness() {\n const TRIALS: usize = if cfg!(miri) { 100 } else { 10000 };\n\n let c1 = make::(TRIALS + 1);\n let c2 = make::(TRIALS + 1);\n\n for _ in 0..TRIALS + 1 {\n c1.send(1);\n c2.send(2);\n }\n\n let c3 = make::(0);\n let c4 = make::(0);\n let out = make::(0);\n let done = make::(0);\n let wg = WaitGroup::new();\n\n wg.add(1);\n go!(wg, c1, c2, c3, c4, out, done, {\n defer! { wg.done() };\n loop {\n let b;\n select! {\n recv(c3.rx()) -> m => b = m.unwrap(),\n recv(c4.rx()) -> m => b = m.unwrap(),\n recv(c1.rx()) -> m => b = m.unwrap(),\n recv(c2.rx()) -> m => b = m.unwrap(),\n }\n select! {\n send(out.tx(), b) -> _ => {}\n recv(done.rx()) -> _ => return,\n }\n }\n });\n\n let (mut cnt1, mut cnt2) = (0, 0);\n for _ in 0..TRIALS {\n match out.recv() {\n Some(1) => cnt1 += 1,\n Some(2) => cnt2 += 1,\n b => panic!(\"unexpected value {:?} on channel\", b),\n }\n }\n\n // If the select in the goroutine is fair,\n // cnt1 and cnt2 should be about the same value.\n // With 10,000 trials, the expected margin of error at\n // a confidence level of five nines is 4.4172 / (2 * Sqrt(10000)).\n\n let r = cnt1 as f64 / TRIALS as f64;\n let e = (r - 0.5).abs();\n\n if e > 4.4172 / (2.0 * (TRIALS as f64).sqrt()) {\n panic!(\n \"unfair select: in {} trials, results were {}, {}\",\n TRIALS, cnt1, cnt2,\n );\n }\n\n done.close_s();\n wg.wait();\n }\n\n #[test]\n fn test_chan_send_interface() {\n struct Mt;\n\n let c = make::>(1);\n c.send(Box::new(Mt));\n\n select! {\n send(c.tx(), Box::new(Mt)) -> _ => {}\n default => {}\n }\n\n select! {\n send(c.tx(), Box::new(Mt)) -> _ => {}\n send(c.tx(), Box::new(Mt)) -> _ => {}\n default => {}\n }\n }\n\n #[test]\n fn test_pseudo_random_send() {\n const N: usize = if cfg!(miri) { 20 } else { 100 };\n\n for cap in 0..N {\n let c = make::(cap);\n let l = Arc::new(Mutex::new(vec![0i32; N]));\n let done = make::(0);\n\n go!(c, done, l, {\n let mut l = l.lock().unwrap();\n for i in 0..N {\n thread::yield_now();\n l[i] = c.recv().unwrap();\n }\n done.send(true);\n });\n\n for _ in 0..N {\n select! {\n send(c.tx(), 1) -> _ => {}\n send(c.tx(), 0) -> _ => {}\n }\n }\n done.recv();\n\n let mut n0 = 0;\n let mut n1 = 0;\n for &i in l.lock().unwrap().iter() {\n n0 += (i + 1) % 2;\n n1 += i;\n }\n\n if n0 <= N as i32 / 10 || n1 <= N as i32 / 10 {\n panic!(\n \"Want pseudorandom, got {} zeros and {} ones (chan cap {})\",\n n0, n1, cap,\n );\n }\n }\n }\n\n #[test]\n fn test_multi_consumer() {\n const NWORK: usize = 23;\n const NITER: usize = if cfg!(miri) { 50 } else { 271828 };\n\n let pn = [2, 3, 7, 11, 13, 17, 19, 23, 27, 31];\n\n let q = make::(NWORK * 3);\n let r = make::(NWORK * 3);\n\n let wg = WaitGroup::new();\n for i in 0..NWORK {\n wg.add(1);\n let w = i;\n go!(q, r, wg, pn, {\n for v in &q {\n if pn[w % pn.len()] == v {\n thread::yield_now();\n }\n r.send(v);\n }\n wg.done();\n });\n }\n\n let expect = Arc::new(Mutex::new(0));\n go!(q, r, expect, wg, pn, {\n for i in 0..NITER {\n let v = pn[i % pn.len()];\n *expect.lock().unwrap() += v;\n q.send(v);\n }\n q.close_s();\n wg.wait();\n r.close_s();\n });\n\n let mut n = 0;\n let mut s = 0;\n for v in &r {\n n += 1;\n s += v;\n }\n\n if n != NITER || s != *expect.lock().unwrap() {\n panic!();\n }\n }\n\n #[test]\n fn test_select_duplicate_channel() {\n // This test makes sure we can queue a G on\n // the same channel multiple times.\n let c = make::(0);\n let d = make::(0);\n let e = make::(0);\n\n go!(c, d, e, {\n select! {\n recv(c.rx()) -> _ => {}\n recv(d.rx()) -> _ => {}\n recv(e.rx()) -> _ => {}\n }\n e.send(9);\n });\n thread::sleep(ms(1));\n\n go!(c, c.recv());\n thread::sleep(ms(1));\n\n d.send(7);\n e.recv();\n c.send(8);\n }\n}\n\n// https://github.com/golang/go/blob/master/test/closedchan.go\nmod closedchan {\n // TODO\n}\n\n// https://github.com/golang/go/blob/master/src/runtime/chanbarrier_test.go\nmod chanbarrier_test {\n // TODO\n}\n\n// https://github.com/golang/go/blob/master/src/runtime/race/testdata/chan_test.go\nmod race_chan_test {\n // TODO\n}\n\n// https://github.com/golang/go/blob/master/test/ken/chan.go\nmod chan {\n use super::*;\n\n const MESSAGES_PER_CHANEL: u32 = 76;\n const MESSAGES_RANGE_LEN: u32 = 100;\n const END: i32 = 10000;\n\n struct ChanWithVals {\n chan: Chan,\n /// Next value to send\n sv: Arc,\n /// Next value to receive\n rv: Arc,\n }\n\n struct Totals {\n /// Total sent messages\n tots: u32,\n /// Total received messages\n totr: u32,\n }\n\n struct Context {\n nproc: Arc>,\n cval: Arc>,\n tot: Arc>,\n nc: ChanWithVals,\n randx: Arc>,\n }\n\n impl ChanWithVals {\n fn with_capacity(capacity: usize) -> Self {\n Self {\n chan: make(capacity),\n sv: Arc::new(AtomicI32::new(0)),\n rv: Arc::new(AtomicI32::new(0)),\n }\n }\n\n fn closed() -> Self {\n let ch = Self::with_capacity(0);\n ch.chan.close_r();\n ch.chan.close_s();\n ch\n }\n\n fn rv(&self) -> i32 {\n self.rv.load(SeqCst)\n }\n\n fn sv(&self) -> i32 {\n self.sv.load(SeqCst)\n }\n\n fn send(&mut self, tot: &Mutex) -> bool {\n {\n let mut tot = tot.lock().unwrap();\n tot.tots += 1\n }\n let esv = expect(self.sv(), self.sv());\n self.sv.store(esv, SeqCst);\n if self.sv() == END {\n self.chan.close_s();\n return true;\n }\n false\n }\n\n fn recv(&mut self, v: i32, tot: &Mutex) -> bool {\n {\n let mut tot = tot.lock().unwrap();\n tot.totr += 1\n }\n let erv = expect(self.rv(), v);\n self.rv.store(erv, SeqCst);\n if self.rv() == END {\n self.chan.close_r();\n return true;\n }\n false\n }\n }\n\n impl Clone for ChanWithVals {\n fn clone(&self) -> Self {\n Self {\n chan: self.chan.clone(),\n sv: self.sv.clone(),\n rv: self.rv.clone(),\n }\n }\n }\n\n impl Context {\n fn nproc(&self) -> &Mutex {\n self.nproc.as_ref()\n }\n\n fn cval(&self) -> &Mutex {\n self.cval.as_ref()\n }\n\n fn tot(&self) -> &Mutex {\n self.tot.as_ref()\n }\n\n fn randx(&self) -> &Mutex {\n self.randx.as_ref()\n }\n }\n\n impl Clone for Context {\n fn clone(&self) -> Self {\n Self {\n nproc: self.nproc.clone(),\n cval: self.cval.clone(),\n tot: self.tot.clone(),\n nc: self.nc.clone(),\n randx: self.randx.clone(),\n }\n }\n }\n\n fn nrand(n: i32, randx: &Mutex) -> i32 {\n let mut randx = randx.lock().unwrap();\n *randx += 10007;\n if *randx >= 1000000 {\n *randx -= 1000000\n }\n *randx % n\n }\n\n fn change_nproc(adjust: i32, nproc: &Mutex) -> i32 {\n let mut nproc = nproc.lock().unwrap();\n *nproc += adjust;\n *nproc\n }\n\n fn mkchan(c: usize, n: usize, cval: &Mutex) -> Vec {\n let mut ca = Vec::::with_capacity(n);\n let mut cval = cval.lock().unwrap();\n for _ in 0..n {\n *cval += MESSAGES_RANGE_LEN as i32;\n let chl = ChanWithVals::with_capacity(c);\n chl.sv.store(*cval, SeqCst);\n chl.rv.store(*cval, SeqCst);\n ca.push(chl);\n }\n ca\n }\n\n fn expect(v: i32, v0: i32) -> i32 {\n if v == v0 {\n return if v % MESSAGES_RANGE_LEN as i32 == MESSAGES_PER_CHANEL as i32 - 1 {\n END\n } else {\n v + 1\n };\n }\n panic!(\"got {}, expected {}\", v, v0 + 1);\n }\n\n fn send(mut c: ChanWithVals, ctx: Context) {\n loop {\n for _ in 0..=nrand(10, ctx.randx()) {\n thread::yield_now();\n }\n c.chan.tx().send(c.sv()).unwrap();\n if c.send(ctx.tot()) {\n break;\n }\n }\n change_nproc(-1, ctx.nproc());\n }\n\n fn recv(mut c: ChanWithVals, ctx: Context) {\n loop {\n for _ in (0..nrand(10, ctx.randx())).rev() {\n thread::yield_now();\n }\n let v = c.chan.rx().recv().unwrap();\n if c.recv(v, ctx.tot()) {\n break;\n }\n }\n change_nproc(-1, ctx.nproc());\n }\n\n #[allow(clippy::too_many_arguments)]\n fn sel(\n mut r0: ChanWithVals,\n mut r1: ChanWithVals,\n mut r2: ChanWithVals,\n mut r3: ChanWithVals,\n mut s0: ChanWithVals,\n mut s1: ChanWithVals,\n mut s2: ChanWithVals,\n mut s3: ChanWithVals,\n ctx: Context,\n ) {\n let mut a = 0; // local chans running\n\n if r0.chan.has_rx() {\n a += 1;\n }\n if r1.chan.has_rx() {\n a += 1;\n }\n if r2.chan.has_rx() {\n a += 1;\n }\n if r3.chan.has_rx() {\n a += 1;\n }\n if s0.chan.has_tx() {\n a += 1;\n }\n if s1.chan.has_tx() {\n a += 1;\n }\n if s2.chan.has_tx() {\n a += 1;\n }\n if s3.chan.has_tx() {\n a += 1;\n }\n\n loop {\n for _ in 0..=nrand(5, ctx.randx()) {\n thread::yield_now();\n }\n select! {\n recv(r0.chan.rx()) -> v => if r0.recv(v.unwrap(), ctx.tot()) { a -= 1 },\n recv(r1.chan.rx()) -> v => if r1.recv(v.unwrap(), ctx.tot()) { a -= 1 },\n recv(r2.chan.rx()) -> v => if r2.recv(v.unwrap(), ctx.tot()) { a -= 1 },\n recv(r3.chan.rx()) -> v => if r3.recv(v.unwrap(), ctx.tot()) { a -= 1 },\n send(s0.chan.tx(), s0.sv()) -> _ => if s0.send(ctx.tot()) { a -= 1 },\n send(s1.chan.tx(), s1.sv()) -> _ => if s1.send(ctx.tot()) { a -= 1 },\n send(s2.chan.tx(), s2.sv()) -> _ => if s2.send(ctx.tot()) { a -= 1 },\n send(s3.chan.tx(), s3.sv()) -> _ => if s3.send(ctx.tot()) { a -= 1 },\n }\n if a == 0 {\n break;\n }\n }\n change_nproc(-1, ctx.nproc());\n }\n\n fn get(vec: &[ChanWithVals], idx: usize) -> ChanWithVals {\n vec.get(idx).unwrap().clone()\n }\n\n /// Direct send to direct recv\n fn test1(c: ChanWithVals, ctx: &mut Context) {\n change_nproc(2, ctx.nproc());\n go!(c, ctx, send(c, ctx));\n go!(c, ctx, recv(c, ctx));\n }\n\n /// Direct send to select recv\n fn test2(c: usize, ctx: &mut Context) {\n let ca = mkchan(c, 4, ctx.cval());\n\n change_nproc(4, ctx.nproc());\n go!(ca, ctx, send(get(&ca, 0), ctx));\n go!(ca, ctx, send(get(&ca, 1), ctx));\n go!(ca, ctx, send(get(&ca, 2), ctx));\n go!(ca, ctx, send(get(&ca, 3), ctx));\n\n change_nproc(1, ctx.nproc());\n go!(\n ca,\n ctx,\n sel(\n get(&ca, 0),\n get(&ca, 1),\n get(&ca, 2),\n get(&ca, 3),\n ctx.nc.clone(),\n ctx.nc.clone(),\n ctx.nc.clone(),\n ctx.nc.clone(),\n ctx,\n )\n );\n }\n\n /// Select send to direct recv\n fn test3(c: usize, ctx: &mut Context) {\n let ca = mkchan(c, 4, ctx.cval());\n\n change_nproc(4, ctx.nproc());\n go!(ca, ctx, recv(get(&ca, 0), ctx));\n go!(ca, ctx, recv(get(&ca, 1), ctx));\n go!(ca, ctx, recv(get(&ca, 2), ctx));\n go!(ca, ctx, recv(get(&ca, 3), ctx));\n\n change_nproc(1, ctx.nproc());\n go!(\n ca,\n ctx,\n sel(\n ctx.nc.clone(),\n ctx.nc.clone(),\n ctx.nc.clone(),\n ctx.nc.clone(),\n get(&ca, 0),\n get(&ca, 1),\n get(&ca, 2),\n get(&ca, 3),\n ctx,\n )\n );\n }\n\n /// Select send to select recv, 4 channels\n fn test4(c: usize, ctx: &mut Context) {\n let ca = mkchan(c, 4, ctx.cval());\n\n change_nproc(2, ctx.nproc());\n go!(\n ca,\n ctx,\n sel(\n ctx.nc.clone(),\n ctx.nc.clone(),\n ctx.nc.clone(),\n ctx.nc.clone(),\n get(&ca, 0),\n get(&ca, 1),\n get(&ca, 2),\n get(&ca, 3),\n ctx,\n )\n );\n go!(\n ca,\n ctx,\n sel(\n get(&ca, 0),\n get(&ca, 1),\n get(&ca, 2),\n get(&ca, 3),\n ctx.nc.clone(),\n ctx.nc.clone(),\n ctx.nc.clone(),\n ctx.nc.clone(),\n ctx,\n )\n );\n }\n\n /// Select send to select recv, 8 channels\n fn test5(c: usize, ctx: &mut Context) {\n let ca = mkchan(c, 8, ctx.cval());\n\n change_nproc(2, ctx.nproc());\n go!(\n ca,\n ctx,\n sel(\n get(&ca, 4),\n get(&ca, 5),\n get(&ca, 6),\n get(&ca, 7),\n get(&ca, 0),\n get(&ca, 1),\n get(&ca, 2),\n get(&ca, 3),\n ctx,\n )\n );\n go!(\n ca,\n ctx,\n sel(\n get(&ca, 0),\n get(&ca, 1),\n get(&ca, 2),\n get(&ca, 3),\n get(&ca, 4),\n get(&ca, 5),\n get(&ca, 6),\n get(&ca, 7),\n ctx,\n )\n );\n }\n\n // Direct and select send to direct and select recv\n fn test6(c: usize, ctx: &mut Context) {\n let ca = mkchan(c, 12, ctx.cval());\n\n change_nproc(4, ctx.nproc());\n go!(ca, ctx, send(get(&ca, 4), ctx));\n go!(ca, ctx, send(get(&ca, 5), ctx));\n go!(ca, ctx, send(get(&ca, 6), ctx));\n go!(ca, ctx, send(get(&ca, 7), ctx));\n\n change_nproc(4, ctx.nproc());\n go!(ca, ctx, recv(get(&ca, 8), ctx));\n go!(ca, ctx, recv(get(&ca, 9), ctx));\n go!(ca, ctx, recv(get(&ca, 10), ctx));\n go!(ca, ctx, recv(get(&ca, 11), ctx));\n\n change_nproc(2, ctx.nproc());\n go!(\n ca,\n ctx,\n sel(\n get(&ca, 4),\n get(&ca, 5),\n get(&ca, 6),\n get(&ca, 7),\n get(&ca, 0),\n get(&ca, 1),\n get(&ca, 2),\n get(&ca, 3),\n ctx,\n )\n );\n go!(\n ca,\n ctx,\n sel(\n get(&ca, 0),\n get(&ca, 1),\n get(&ca, 2),\n get(&ca, 3),\n get(&ca, 8),\n get(&ca, 9),\n get(&ca, 10),\n get(&ca, 11),\n ctx,\n )\n );\n }\n\n fn wait(ctx: &mut Context) {\n thread::yield_now();\n while change_nproc(0, ctx.nproc()) != 0 {\n thread::yield_now();\n }\n }\n\n fn tests(c: usize, ctx: &mut Context) {\n let ca = mkchan(c, 4, ctx.cval());\n test1(get(&ca, 0), ctx);\n test1(get(&ca, 1), ctx);\n test1(get(&ca, 2), ctx);\n test1(get(&ca, 3), ctx);\n wait(ctx);\n\n test2(c, ctx);\n wait(ctx);\n\n test3(c, ctx);\n wait(ctx);\n\n test4(c, ctx);\n wait(ctx);\n\n test5(c, ctx);\n wait(ctx);\n\n test6(c, ctx);\n wait(ctx);\n }\n\n #[test]\n #[cfg_attr(miri, ignore)] // Miri is too slow\n fn main() {\n let mut ctx = Context {\n nproc: Arc::new(Mutex::new(0)),\n cval: Arc::new(Mutex::new(0)),\n tot: Arc::new(Mutex::new(Totals { tots: 0, totr: 0 })),\n nc: ChanWithVals::closed(),\n randx: Arc::new(Mutex::new(0)),\n };\n\n tests(0, &mut ctx);\n tests(1, &mut ctx);\n tests(10, &mut ctx);\n tests(100, &mut ctx);\n\n #[rustfmt::skip]\n let t = 4 * // buffer sizes\n (4*4 + // tests 1,2,3,4 channels\n 8 + // test 5 channels\n 12) * // test 6 channels\n MESSAGES_PER_CHANEL; // sends/recvs on a channel\n\n let tot = ctx.tot.lock().unwrap();\n if tot.tots != t || tot.totr != t {\n panic!(\"tots={} totr={} sb={}\", tot.tots, tot.totr, t);\n }\n }\n}\n\n// https://github.com/golang/go/blob/master/test/ken/chan1.go\nmod chan1 {\n use super::*;\n\n // sent messages\n const N: usize = if cfg!(miri) { 20 } else { 1000 };\n // receiving \"goroutines\"\n const M: usize = 10;\n // channel buffering\n const W: usize = 2;\n\n fn r(c: Chan, m: usize, h: Arc>) {\n loop {\n select! {\n recv(c.rx()) -> rr => {\n let r = rr.unwrap();\n let mut data = h.lock().unwrap();\n if data[r] != 1 {\n println!(\"r\\nm={}\\nr={}\\nh={}\\n\", m, r, data[r]);\n panic!(\"fail\")\n }\n data[r] = 2;\n }\n }\n }\n }\n\n fn s(c: Chan, h: Arc>) {\n for n in 0..N {\n let r = n;\n let mut data = h.lock().unwrap();\n if data[r] != 0 {\n println!(\"s\");\n panic!(\"fail\");\n }\n data[r] = 1;\n // https://github.com/crossbeam-rs/crossbeam/pull/615#discussion_r550281094\n drop(data);\n c.send(r);\n }\n }\n\n #[test]\n fn main() {\n // https://github.com/rust-lang/miri/issues/1371\n if option_env!(\"MIRI_LEAK_CHECK\").is_some() {\n return;\n }\n\n let h = Arc::new(Mutex::new([0usize; N]));\n let c = make::(W);\n for m in 0..M {\n go!(c, h, {\n r(c, m, h);\n });\n thread::yield_now();\n }\n thread::yield_now();\n thread::yield_now();\n s(c, h);\n }\n}\n" }, { "path": "crossbeam-channel/tests/iter.rs", "content": "//! Tests for iteration over receivers.\n\nuse crossbeam_channel::unbounded;\nuse crossbeam_utils::thread::scope;\n\n#[test]\nfn nested_recv_iter() {\n let (s, r) = unbounded::();\n let (total_s, total_r) = unbounded::();\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut acc = 0;\n for x in r.iter() {\n acc += x;\n }\n total_s.send(acc).unwrap();\n });\n\n s.send(3).unwrap();\n s.send(1).unwrap();\n s.send(2).unwrap();\n drop(s);\n assert_eq!(total_r.recv().unwrap(), 6);\n })\n .unwrap();\n}\n\n#[test]\nfn recv_iter_break() {\n let (s, r) = unbounded::();\n let (count_s, count_r) = unbounded();\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut count = 0;\n for x in r.iter() {\n if count >= 3 {\n break;\n } else {\n count += x;\n }\n }\n count_s.send(count).unwrap();\n });\n\n s.send(2).unwrap();\n s.send(2).unwrap();\n s.send(2).unwrap();\n let _ = s.send(2);\n drop(s);\n assert_eq!(count_r.recv().unwrap(), 4);\n })\n .unwrap();\n}\n\n#[test]\nfn recv_try_iter() {\n let (request_s, request_r) = unbounded();\n let (response_s, response_r) = unbounded();\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut count = 0;\n loop {\n for x in response_r.try_iter() {\n count += x;\n if count == 6 {\n return;\n }\n }\n request_s.send(()).unwrap();\n }\n });\n\n for _ in request_r.iter() {\n if response_s.send(2).is_err() {\n break;\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn recv_into_iter_owned() {\n let mut iter = {\n let (s, r) = unbounded::();\n s.send(1).unwrap();\n s.send(2).unwrap();\n r.into_iter()\n };\n\n assert_eq!(iter.next().unwrap(), 1);\n assert_eq!(iter.next().unwrap(), 2);\n assert!(iter.next().is_none());\n}\n\n#[test]\nfn recv_into_iter_borrowed() {\n let (s, r) = unbounded::();\n s.send(1).unwrap();\n s.send(2).unwrap();\n drop(s);\n\n let mut iter = (&r).into_iter();\n assert_eq!(iter.next().unwrap(), 1);\n assert_eq!(iter.next().unwrap(), 2);\n assert!(iter.next().is_none());\n}\n" }, { "path": "crossbeam-channel/tests/list.rs", "content": "//! Tests for the list channel flavor.\n\nuse std::{\n any::Any,\n sync::atomic::{AtomicUsize, Ordering},\n thread,\n time::Duration,\n};\n\nuse crossbeam_channel::{\n Receiver, RecvError, RecvTimeoutError, SendError, SendTimeoutError, TryRecvError, TrySendError,\n select, unbounded,\n};\nuse crossbeam_utils::thread::scope;\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\nfn smoke() {\n let (s, r) = unbounded();\n s.try_send(7).unwrap();\n assert_eq!(r.try_recv(), Ok(7));\n\n s.send(8).unwrap();\n assert_eq!(r.recv(), Ok(8));\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n assert_eq!(r.recv_timeout(ms(1000)), Err(RecvTimeoutError::Timeout));\n}\n\n#[test]\nfn capacity() {\n let (s, r) = unbounded::<()>();\n assert_eq!(s.capacity(), None);\n assert_eq!(r.capacity(), None);\n}\n\n#[test]\nfn len_empty_full() {\n let (s, r) = unbounded();\n\n assert_eq!(s.len(), 0);\n assert!(s.is_empty());\n assert!(!s.is_full());\n assert_eq!(r.len(), 0);\n assert!(r.is_empty());\n assert!(!r.is_full());\n\n s.send(()).unwrap();\n\n assert_eq!(s.len(), 1);\n assert!(!s.is_empty());\n assert!(!s.is_full());\n assert_eq!(r.len(), 1);\n assert!(!r.is_empty());\n assert!(!r.is_full());\n\n r.recv().unwrap();\n\n assert_eq!(s.len(), 0);\n assert!(s.is_empty());\n assert!(!s.is_full());\n assert_eq!(r.len(), 0);\n assert!(r.is_empty());\n assert!(!r.is_full());\n}\n\n#[test]\n#[cfg_attr(miri, ignore)] // this test makes timing assumptions, but Miri is so slow it violates them\nfn try_recv() {\n let (s, r) = unbounded();\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n thread::sleep(ms(1500));\n assert_eq!(r.try_recv(), Ok(7));\n thread::sleep(ms(500));\n assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n s.send(7).unwrap();\n });\n })\n .unwrap();\n}\n\n#[test]\nfn recv() {\n let (s, r) = unbounded();\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.recv(), Ok(7));\n thread::sleep(ms(1000));\n assert_eq!(r.recv(), Ok(8));\n thread::sleep(ms(1000));\n assert_eq!(r.recv(), Ok(9));\n assert_eq!(r.recv(), Err(RecvError));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n s.send(7).unwrap();\n s.send(8).unwrap();\n s.send(9).unwrap();\n });\n })\n .unwrap();\n}\n\n#[test]\nfn recv_timeout() {\n let (s, r) = unbounded::();\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.recv_timeout(ms(1000)), Err(RecvTimeoutError::Timeout));\n assert_eq!(r.recv_timeout(ms(1000)), Ok(7));\n assert_eq!(\n r.recv_timeout(ms(1000)),\n Err(RecvTimeoutError::Disconnected)\n );\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n s.send(7).unwrap();\n });\n })\n .unwrap();\n}\n\n#[test]\nfn try_send() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 1000 };\n\n let (s, r) = unbounded();\n for i in 0..COUNT {\n assert_eq!(s.try_send(i), Ok(()));\n }\n\n drop(r);\n assert_eq!(s.try_send(777), Err(TrySendError::Disconnected(777)));\n}\n\n#[test]\nfn send() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 1000 };\n\n let (s, r) = unbounded();\n for i in 0..COUNT {\n assert_eq!(s.send(i), Ok(()));\n }\n\n drop(r);\n assert_eq!(s.send(777), Err(SendError(777)));\n}\n\n#[test]\nfn send_timeout() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 1000 };\n\n let (s, r) = unbounded();\n for i in 0..COUNT {\n assert_eq!(s.send_timeout(i, ms(i as u64)), Ok(()));\n }\n\n drop(r);\n assert_eq!(\n s.send_timeout(777, ms(0)),\n Err(SendTimeoutError::Disconnected(777))\n );\n}\n\n#[test]\nfn send_after_disconnect() {\n let (s, r) = unbounded();\n\n s.send(1).unwrap();\n s.send(2).unwrap();\n s.send(3).unwrap();\n\n drop(r);\n\n assert_eq!(s.send(4), Err(SendError(4)));\n assert_eq!(s.try_send(5), Err(TrySendError::Disconnected(5)));\n assert_eq!(\n s.send_timeout(6, ms(0)),\n Err(SendTimeoutError::Disconnected(6))\n );\n}\n\n#[test]\nfn recv_after_disconnect() {\n let (s, r) = unbounded();\n\n s.send(1).unwrap();\n s.send(2).unwrap();\n s.send(3).unwrap();\n\n drop(s);\n\n assert_eq!(r.recv(), Ok(1));\n assert_eq!(r.recv(), Ok(2));\n assert_eq!(r.recv(), Ok(3));\n assert_eq!(r.recv(), Err(RecvError));\n}\n\n#[test]\nfn len() {\n let (s, r) = unbounded();\n\n assert_eq!(s.len(), 0);\n assert_eq!(r.len(), 0);\n\n for i in 0..50 {\n s.send(i).unwrap();\n assert_eq!(s.len(), i + 1);\n }\n\n for i in 0..50 {\n r.recv().unwrap();\n assert_eq!(r.len(), 50 - i - 1);\n }\n\n assert_eq!(s.len(), 0);\n assert_eq!(r.len(), 0);\n}\n\n#[test]\nfn disconnect_wakes_receiver() {\n let (s, r) = unbounded::<()>();\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.recv(), Err(RecvError));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n drop(s);\n });\n })\n .unwrap();\n}\n\n#[test]\nfn spsc() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 100_000 };\n\n let (s, r) = unbounded();\n\n scope(|scope| {\n scope.spawn(move |_| {\n for i in 0..COUNT {\n assert_eq!(r.recv(), Ok(i));\n }\n assert_eq!(r.recv(), Err(RecvError));\n });\n scope.spawn(move |_| {\n for i in 0..COUNT {\n s.send(i).unwrap();\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn mpmc() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 25_000 };\n const THREADS: usize = 4;\n\n let (s, r) = unbounded::();\n let v = (0..COUNT).map(|_| AtomicUsize::new(0)).collect::>();\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n let n = r.recv().unwrap();\n v[n].fetch_add(1, Ordering::SeqCst);\n }\n });\n }\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..COUNT {\n s.send(i).unwrap();\n }\n });\n }\n })\n .unwrap();\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n\n for c in v {\n assert_eq!(c.load(Ordering::SeqCst), THREADS);\n }\n}\n\n#[test]\nfn stress_oneshot() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n for _ in 0..COUNT {\n let (s, r) = unbounded();\n\n scope(|scope| {\n scope.spawn(|_| r.recv().unwrap());\n scope.spawn(|_| s.send(0).unwrap());\n })\n .unwrap();\n }\n}\n\n#[test]\nfn stress_iter() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 100_000 };\n\n let (request_s, request_r) = unbounded();\n let (response_s, response_r) = unbounded();\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut count = 0;\n loop {\n for x in response_r.try_iter() {\n count += x;\n if count == COUNT {\n return;\n }\n }\n request_s.send(()).unwrap();\n }\n });\n\n for _ in request_r.iter() {\n if response_s.send(1).is_err() {\n break;\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_timeout_two_threads() {\n const COUNT: usize = 100;\n\n let (s, r) = unbounded();\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(50));\n }\n s.send(i).unwrap();\n }\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(50));\n }\n loop {\n if let Ok(x) = r.recv_timeout(ms(10)) {\n assert_eq!(x, i);\n break;\n }\n }\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn drops() {\n const RUNS: usize = if cfg!(miri) { 20 } else { 100 };\n const STEPS: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n static DROPS: AtomicUsize = AtomicUsize::new(0);\n\n #[derive(Debug, PartialEq)]\n struct DropCounter;\n\n impl Drop for DropCounter {\n fn drop(&mut self) {\n DROPS.fetch_add(1, Ordering::SeqCst);\n }\n }\n\n let mut rng = fastrand::Rng::new();\n\n for _ in 0..RUNS {\n let steps = rng.usize(0..STEPS);\n let additional = rng.usize(0..STEPS / 10);\n\n DROPS.store(0, Ordering::SeqCst);\n let (s, r) = unbounded::();\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n });\n\n scope.spawn(|_| {\n for _ in 0..steps {\n s.send(DropCounter).unwrap();\n }\n });\n })\n .unwrap();\n\n for _ in 0..additional {\n s.try_send(DropCounter).unwrap();\n }\n\n assert_eq!(DROPS.load(Ordering::SeqCst), steps);\n drop(s);\n drop(r);\n assert_eq!(DROPS.load(Ordering::SeqCst), steps + additional);\n }\n}\n\n#[test]\nfn linearizable() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 25_000 };\n const THREADS: usize = 4;\n\n let (s, r) = unbounded();\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n s.send(0).unwrap();\n r.try_recv().unwrap();\n }\n });\n }\n })\n .unwrap();\n}\n\n#[test]\nfn fairness() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = unbounded::<()>();\n let (s2, r2) = unbounded::<()>();\n\n for _ in 0..COUNT {\n s1.send(()).unwrap();\n s2.send(()).unwrap();\n }\n\n let mut hits = [0usize; 2];\n for _ in 0..COUNT {\n select! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n}\n\n#[test]\nfn fairness_duplicates() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s, r) = unbounded();\n\n for _ in 0..COUNT {\n s.send(()).unwrap();\n }\n\n let mut hits = [0usize; 5];\n for _ in 0..COUNT {\n select! {\n recv(r) -> _ => hits[0] += 1,\n recv(r) -> _ => hits[1] += 1,\n recv(r) -> _ => hits[2] += 1,\n recv(r) -> _ => hits[3] += 1,\n recv(r) -> _ => hits[4] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n}\n\n#[test]\nfn recv_in_send() {\n let (s, r) = unbounded();\n s.send(()).unwrap();\n\n select! {\n send(s, assert_eq!(r.recv(), Ok(()))) -> _ => {}\n }\n}\n\n#[test]\nfn channel_through_channel() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 1000 };\n\n type T = Box;\n\n let (s, r) = unbounded::();\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut s = s;\n\n for _ in 0..COUNT {\n let (new_s, new_r) = unbounded();\n let new_r: T = Box::new(Some(new_r));\n\n s.send(new_r).unwrap();\n s = new_s;\n }\n });\n\n scope.spawn(move |_| {\n let mut r = r;\n\n for _ in 0..COUNT {\n r = r\n .recv()\n .unwrap()\n .downcast_mut::>>()\n .unwrap()\n .take()\n .unwrap()\n }\n });\n })\n .unwrap();\n}\n\n// If `Block` is created on the stack, the array of slots will multiply this `BigStruct` and\n// probably overflow the thread stack. It's now directly created on the heap to avoid this.\n#[test]\nfn stack_overflow() {\n const N: usize = 32_768;\n struct BigStruct {\n _data: [u8; N],\n }\n\n let (sender, receiver) = unbounded::();\n sender.send(BigStruct { _data: [0u8; N] }).unwrap();\n\n for _data in receiver.try_iter() {}\n}\n" }, { "path": "crossbeam-channel/tests/mpsc.rs", "content": "//! Tests copied from `std::sync::mpsc`.\n//!\n//! This is a copy of tests for the `std::sync::mpsc` channels from the standard library, but\n//! modified to work with `crossbeam-channel` instead.\n//!\n//! Minor tweaks were needed to make the tests compile:\n//!\n//! - Replace `box` syntax with `Box::new`.\n//! - Replace all uses of `Select` with `select!`.\n//! - Change the imports.\n//! - Join all spawned threads.\n//! - Removed assertion from oneshot_multi_thread_send_close_stress tests.\n//!\n//! Source:\n//! - https://github.com/rust-lang/rust/tree/master/src/libstd/sync/mpsc\n//!\n//! Copyright & License:\n//! - Copyright 2013-2014 The Rust Project Developers\n//! - Apache License, Version 2.0 or MIT license, at your option\n//! - https://github.com/rust-lang/rust/blob/master/COPYRIGHT\n//! - https://www.rust-lang.org/en-US/legal.html\n\n#![allow(clippy::match_single_binding, clippy::redundant_clone)]\n\nuse std::{\n sync::mpsc::{RecvError, RecvTimeoutError, SendError, TryRecvError, TrySendError},\n thread::JoinHandle,\n time::Duration,\n};\n\nuse crossbeam_channel as cc;\n\nstruct Sender {\n inner: cc::Sender,\n}\n\nimpl Sender {\n fn send(&self, t: T) -> Result<(), SendError> {\n self.inner.send(t).map_err(|cc::SendError(m)| SendError(m))\n }\n}\n\nimpl Clone for Sender {\n fn clone(&self) -> Self {\n Self {\n inner: self.inner.clone(),\n }\n }\n}\n\nstruct SyncSender {\n inner: cc::Sender,\n}\n\nimpl SyncSender {\n fn send(&self, t: T) -> Result<(), SendError> {\n self.inner.send(t).map_err(|cc::SendError(m)| SendError(m))\n }\n\n fn try_send(&self, t: T) -> Result<(), TrySendError> {\n self.inner.try_send(t).map_err(|err| match err {\n cc::TrySendError::Full(m) => TrySendError::Full(m),\n cc::TrySendError::Disconnected(m) => TrySendError::Disconnected(m),\n })\n }\n}\n\nimpl Clone for SyncSender {\n fn clone(&self) -> Self {\n Self {\n inner: self.inner.clone(),\n }\n }\n}\n\nstruct Receiver {\n inner: cc::Receiver,\n}\n\nimpl Receiver {\n fn try_recv(&self) -> Result {\n self.inner.try_recv().map_err(|err| match err {\n cc::TryRecvError::Empty => TryRecvError::Empty,\n cc::TryRecvError::Disconnected => TryRecvError::Disconnected,\n })\n }\n\n fn recv(&self) -> Result {\n self.inner.recv().map_err(|_| RecvError)\n }\n\n fn recv_timeout(&self, timeout: Duration) -> Result {\n self.inner.recv_timeout(timeout).map_err(|err| match err {\n cc::RecvTimeoutError::Timeout => RecvTimeoutError::Timeout,\n cc::RecvTimeoutError::Disconnected => RecvTimeoutError::Disconnected,\n })\n }\n\n fn iter(&self) -> Iter<'_, T> {\n Iter { inner: self }\n }\n\n fn try_iter(&self) -> TryIter<'_, T> {\n TryIter { inner: self }\n }\n}\n\nimpl<'a, T> IntoIterator for &'a Receiver {\n type Item = T;\n type IntoIter = Iter<'a, T>;\n\n fn into_iter(self) -> Self::IntoIter {\n self.iter()\n }\n}\n\nimpl IntoIterator for Receiver {\n type Item = T;\n type IntoIter = IntoIter;\n\n fn into_iter(self) -> Self::IntoIter {\n IntoIter { inner: self }\n }\n}\n\nstruct TryIter<'a, T> {\n inner: &'a Receiver,\n}\n\nimpl Iterator for TryIter<'_, T> {\n type Item = T;\n\n fn next(&mut self) -> Option {\n self.inner.try_recv().ok()\n }\n}\n\nstruct Iter<'a, T> {\n inner: &'a Receiver,\n}\n\nimpl Iterator for Iter<'_, T> {\n type Item = T;\n\n fn next(&mut self) -> Option {\n self.inner.recv().ok()\n }\n}\n\nstruct IntoIter {\n inner: Receiver,\n}\n\nimpl Iterator for IntoIter {\n type Item = T;\n\n fn next(&mut self) -> Option {\n self.inner.recv().ok()\n }\n}\n\nfn channel() -> (Sender, Receiver) {\n let (s, r) = cc::unbounded();\n let s = Sender { inner: s };\n let r = Receiver { inner: r };\n (s, r)\n}\n\nfn sync_channel(bound: usize) -> (SyncSender, Receiver) {\n let (s, r) = cc::bounded(bound);\n let s = SyncSender { inner: s };\n let r = Receiver { inner: r };\n (s, r)\n}\n\nmacro_rules! select {\n (\n $($name:pat = $rx:ident.$meth:ident() => $code:expr),+\n ) => ({\n const _IS_BIASED: bool = false;\n\n cc::crossbeam_channel_internal! {\n $(\n $meth(($rx).inner) -> res => {\n let $name = res.map_err(|_| ::std::sync::mpsc::RecvError);\n $code\n }\n )+\n }\n })\n}\n\n// Source: https://github.com/rust-lang/rust/blob/master/src/libstd/sync/mpsc/mod.rs\nmod channel_tests {\n use std::{env, thread, time::Instant};\n\n use super::*;\n\n fn stress_factor() -> usize {\n match env::var(\"RUST_TEST_STRESS\") {\n Ok(val) => val.parse().unwrap(),\n Err(..) => 1,\n }\n }\n\n #[test]\n fn smoke() {\n let (tx, rx) = channel::();\n tx.send(1).unwrap();\n assert_eq!(rx.recv().unwrap(), 1);\n }\n\n #[test]\n fn drop_full() {\n let (tx, _rx) = channel::>();\n tx.send(Box::new(1)).unwrap();\n }\n\n #[test]\n fn drop_full_shared() {\n let (tx, _rx) = channel::>();\n drop(tx.clone());\n drop(tx.clone());\n tx.send(Box::new(1)).unwrap();\n }\n\n #[test]\n fn smoke_shared() {\n let (tx, rx) = channel::();\n tx.send(1).unwrap();\n assert_eq!(rx.recv().unwrap(), 1);\n let tx = tx.clone();\n tx.send(1).unwrap();\n assert_eq!(rx.recv().unwrap(), 1);\n }\n\n #[test]\n fn smoke_threads() {\n let (tx, rx) = channel::();\n let t = thread::spawn(move || {\n tx.send(1).unwrap();\n });\n assert_eq!(rx.recv().unwrap(), 1);\n t.join().unwrap();\n }\n\n #[test]\n fn smoke_port_gone() {\n let (tx, rx) = channel::();\n drop(rx);\n assert!(tx.send(1).is_err());\n }\n\n #[test]\n fn smoke_shared_port_gone() {\n let (tx, rx) = channel::();\n drop(rx);\n assert!(tx.send(1).is_err())\n }\n\n #[test]\n fn smoke_shared_port_gone2() {\n let (tx, rx) = channel::();\n drop(rx);\n let tx2 = tx.clone();\n drop(tx);\n assert!(tx2.send(1).is_err());\n }\n\n #[test]\n fn port_gone_concurrent() {\n let (tx, rx) = channel::();\n let t = thread::spawn(move || {\n rx.recv().unwrap();\n });\n while tx.send(1).is_ok() {}\n t.join().unwrap();\n }\n\n #[test]\n fn port_gone_concurrent_shared() {\n let (tx, rx) = channel::();\n let tx2 = tx.clone();\n let t = thread::spawn(move || {\n rx.recv().unwrap();\n });\n while tx.send(1).is_ok() && tx2.send(1).is_ok() {}\n t.join().unwrap();\n }\n\n #[test]\n fn smoke_chan_gone() {\n let (tx, rx) = channel::();\n drop(tx);\n assert!(rx.recv().is_err());\n }\n\n #[test]\n fn smoke_chan_gone_shared() {\n let (tx, rx) = channel::<()>();\n let tx2 = tx.clone();\n drop(tx);\n drop(tx2);\n assert!(rx.recv().is_err());\n }\n\n #[test]\n fn chan_gone_concurrent() {\n let (tx, rx) = channel::();\n let t = thread::spawn(move || {\n tx.send(1).unwrap();\n tx.send(1).unwrap();\n });\n while rx.recv().is_ok() {}\n t.join().unwrap();\n }\n\n #[test]\n fn stress() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10000 };\n\n let (tx, rx) = channel::();\n let t = thread::spawn(move || {\n for _ in 0..COUNT {\n tx.send(1).unwrap();\n }\n });\n for _ in 0..COUNT {\n assert_eq!(rx.recv().unwrap(), 1);\n }\n t.join().unwrap();\n }\n\n #[test]\n fn stress_shared() {\n let amt: u32 = if cfg!(miri) { 100 } else { 10_000 };\n let nthreads: u32 = if cfg!(miri) { 4 } else { 8 };\n let (tx, rx) = channel::();\n\n let t = thread::spawn(move || {\n for _ in 0..amt * nthreads {\n assert_eq!(rx.recv().unwrap(), 1);\n }\n assert!(rx.try_recv().is_err());\n });\n\n let mut ts = Vec::with_capacity(nthreads as usize);\n for _ in 0..nthreads {\n let tx = tx.clone();\n let t = thread::spawn(move || {\n for _ in 0..amt {\n tx.send(1).unwrap();\n }\n });\n ts.push(t);\n }\n drop(tx);\n t.join().unwrap();\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn send_from_outside_runtime() {\n let (tx1, rx1) = channel::<()>();\n let (tx2, rx2) = channel::();\n let t1 = thread::spawn(move || {\n tx1.send(()).unwrap();\n for _ in 0..40 {\n assert_eq!(rx2.recv().unwrap(), 1);\n }\n });\n rx1.recv().unwrap();\n let t2 = thread::spawn(move || {\n for _ in 0..40 {\n tx2.send(1).unwrap();\n }\n });\n t1.join().unwrap();\n t2.join().unwrap();\n }\n\n #[test]\n fn recv_from_outside_runtime() {\n let (tx, rx) = channel::();\n let t = thread::spawn(move || {\n for _ in 0..40 {\n assert_eq!(rx.recv().unwrap(), 1);\n }\n });\n for _ in 0..40 {\n tx.send(1).unwrap();\n }\n t.join().unwrap();\n }\n\n #[test]\n fn no_runtime() {\n let (tx1, rx1) = channel::();\n let (tx2, rx2) = channel::();\n let t1 = thread::spawn(move || {\n assert_eq!(rx1.recv().unwrap(), 1);\n tx2.send(2).unwrap();\n });\n let t2 = thread::spawn(move || {\n tx1.send(1).unwrap();\n assert_eq!(rx2.recv().unwrap(), 2);\n });\n t1.join().unwrap();\n t2.join().unwrap();\n }\n\n #[test]\n fn oneshot_single_thread_close_port_first() {\n // Simple test of closing without sending\n let (_tx, rx) = channel::();\n drop(rx);\n }\n\n #[test]\n fn oneshot_single_thread_close_chan_first() {\n // Simple test of closing without sending\n let (tx, _rx) = channel::();\n drop(tx);\n }\n\n #[test]\n fn oneshot_single_thread_send_port_close() {\n // Testing that the sender cleans up the payload if receiver is closed\n let (tx, rx) = channel::>();\n drop(rx);\n assert!(tx.send(Box::new(0)).is_err());\n }\n\n #[test]\n fn oneshot_single_thread_recv_chan_close() {\n let (tx, rx) = channel::();\n drop(tx);\n assert_eq!(rx.recv(), Err(RecvError));\n }\n\n #[test]\n fn oneshot_single_thread_send_then_recv() {\n let (tx, rx) = channel::>();\n tx.send(Box::new(10)).unwrap();\n assert!(*rx.recv().unwrap() == 10);\n }\n\n #[test]\n fn oneshot_single_thread_try_send_open() {\n let (tx, rx) = channel::();\n assert!(tx.send(10).is_ok());\n assert!(rx.recv().unwrap() == 10);\n }\n\n #[test]\n fn oneshot_single_thread_try_send_closed() {\n let (tx, rx) = channel::();\n drop(rx);\n assert!(tx.send(10).is_err());\n }\n\n #[test]\n fn oneshot_single_thread_try_recv_open() {\n let (tx, rx) = channel::();\n tx.send(10).unwrap();\n assert!(rx.recv() == Ok(10));\n }\n\n #[test]\n fn oneshot_single_thread_try_recv_closed() {\n let (tx, rx) = channel::();\n drop(tx);\n assert!(rx.recv().is_err());\n }\n\n #[test]\n fn oneshot_single_thread_peek_data() {\n let (tx, rx) = channel::();\n assert_eq!(rx.try_recv(), Err(TryRecvError::Empty));\n tx.send(10).unwrap();\n assert_eq!(rx.try_recv(), Ok(10));\n }\n\n #[test]\n fn oneshot_single_thread_peek_close() {\n let (tx, rx) = channel::();\n drop(tx);\n assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected));\n assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected));\n }\n\n #[test]\n fn oneshot_single_thread_peek_open() {\n let (_tx, rx) = channel::();\n assert_eq!(rx.try_recv(), Err(TryRecvError::Empty));\n }\n\n #[test]\n fn oneshot_multi_task_recv_then_send() {\n let (tx, rx) = channel::>();\n let t = thread::spawn(move || {\n assert!(*rx.recv().unwrap() == 10);\n });\n\n tx.send(Box::new(10)).unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn oneshot_multi_task_recv_then_close() {\n let (tx, rx) = channel::>();\n let t = thread::spawn(move || {\n drop(tx);\n });\n thread::spawn(move || {\n assert_eq!(rx.recv(), Err(RecvError));\n })\n .join()\n .unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn oneshot_multi_thread_close_stress() {\n let stress_factor = stress_factor();\n let mut ts = Vec::with_capacity(stress_factor);\n for _ in 0..stress_factor {\n let (tx, rx) = channel::();\n let t = thread::spawn(move || {\n drop(rx);\n });\n ts.push(t);\n drop(tx);\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn oneshot_multi_thread_send_close_stress() {\n let stress_factor = stress_factor();\n let mut ts = Vec::with_capacity(2 * stress_factor);\n for _ in 0..stress_factor {\n let (tx, rx) = channel::();\n let t = thread::spawn(move || {\n drop(rx);\n });\n ts.push(t);\n thread::spawn(move || {\n let _ = tx.send(1);\n })\n .join()\n .unwrap();\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn oneshot_multi_thread_recv_close_stress() {\n let stress_factor = stress_factor();\n let mut ts = Vec::with_capacity(2 * stress_factor);\n for _ in 0..stress_factor {\n let (tx, rx) = channel::();\n let t = thread::spawn(move || {\n thread::spawn(move || {\n assert_eq!(rx.recv(), Err(RecvError));\n })\n .join()\n .unwrap();\n });\n ts.push(t);\n let t2 = thread::spawn(move || {\n let t = thread::spawn(move || {\n drop(tx);\n });\n t.join().unwrap();\n });\n ts.push(t2);\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn oneshot_multi_thread_send_recv_stress() {\n let stress_factor = stress_factor();\n let mut ts = Vec::with_capacity(stress_factor);\n for _ in 0..stress_factor {\n let (tx, rx) = channel::>();\n let t = thread::spawn(move || {\n tx.send(Box::new(10)).unwrap();\n });\n ts.push(t);\n assert!(*rx.recv().unwrap() == 10);\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn stream_send_recv_stress() {\n let stress_factor = stress_factor();\n let mut ts = Vec::with_capacity(2 * stress_factor);\n for _ in 0..stress_factor {\n let (tx, rx) = channel();\n\n if let Some(t) = send(tx, 0) {\n ts.push(t);\n }\n if let Some(t2) = recv(rx, 0) {\n ts.push(t2);\n }\n\n fn send(tx: Sender>, i: i32) -> Option> {\n if i == 10 {\n return None;\n }\n\n Some(thread::spawn(move || {\n tx.send(Box::new(i)).unwrap();\n send(tx, i + 1);\n }))\n }\n\n fn recv(rx: Receiver>, i: i32) -> Option> {\n if i == 10 {\n return None;\n }\n\n Some(thread::spawn(move || {\n assert!(*rx.recv().unwrap() == i);\n recv(rx, i + 1);\n }))\n }\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn oneshot_single_thread_recv_timeout() {\n let (tx, rx) = channel();\n tx.send(()).unwrap();\n assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(()));\n assert_eq!(\n rx.recv_timeout(Duration::from_millis(1)),\n Err(RecvTimeoutError::Timeout)\n );\n tx.send(()).unwrap();\n assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(()));\n }\n\n #[test]\n fn stress_recv_timeout_two_threads() {\n let (tx, rx) = channel();\n let stress = stress_factor() + 100;\n let timeout = Duration::from_millis(100);\n\n let t = thread::spawn(move || {\n for i in 0..stress {\n if i % 2 == 0 {\n thread::sleep(timeout * 2);\n }\n tx.send(1usize).unwrap();\n }\n });\n\n let mut recv_count = 0;\n loop {\n match rx.recv_timeout(timeout) {\n Ok(n) => {\n assert_eq!(n, 1usize);\n recv_count += 1;\n }\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n assert_eq!(recv_count, stress);\n t.join().unwrap()\n }\n\n #[test]\n fn recv_timeout_upgrade() {\n let (tx, rx) = channel::<()>();\n let timeout = Duration::from_millis(1);\n let _tx_clone = tx.clone();\n\n let start = Instant::now();\n assert_eq!(rx.recv_timeout(timeout), Err(RecvTimeoutError::Timeout));\n assert!(Instant::now() >= start + timeout);\n }\n\n #[test]\n fn stress_recv_timeout_shared() {\n let (tx, rx) = channel();\n let stress = stress_factor() + 100;\n\n let mut ts = Vec::with_capacity(stress);\n for i in 0..stress {\n let tx = tx.clone();\n let t = thread::spawn(move || {\n thread::sleep(Duration::from_millis(i as u64 * 10));\n tx.send(1usize).unwrap();\n });\n ts.push(t);\n }\n\n drop(tx);\n\n let mut recv_count = 0;\n loop {\n match rx.recv_timeout(Duration::from_millis(10)) {\n Ok(n) => {\n assert_eq!(n, 1usize);\n recv_count += 1;\n }\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n assert_eq!(recv_count, stress);\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn recv_a_lot() {\n const N: usize = if cfg!(miri) { 50 } else { 10000 };\n\n // Regression test that we don't run out of stack in scheduler context\n let (tx, rx) = channel();\n for _ in 0..N {\n tx.send(()).unwrap();\n }\n for _ in 0..N {\n rx.recv().unwrap();\n }\n }\n\n #[test]\n fn shared_recv_timeout() {\n let (tx, rx) = channel();\n let total = 5;\n let mut ts = Vec::with_capacity(total);\n for _ in 0..total {\n let tx = tx.clone();\n let t = thread::spawn(move || {\n tx.send(()).unwrap();\n });\n ts.push(t);\n }\n\n for _ in 0..total {\n rx.recv().unwrap();\n }\n\n assert_eq!(\n rx.recv_timeout(Duration::from_millis(1)),\n Err(RecvTimeoutError::Timeout)\n );\n tx.send(()).unwrap();\n assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(()));\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn shared_chan_stress() {\n let (tx, rx) = channel();\n let total = stress_factor() + 100;\n let mut ts = Vec::with_capacity(total);\n for _ in 0..total {\n let tx = tx.clone();\n let t = thread::spawn(move || {\n tx.send(()).unwrap();\n });\n ts.push(t);\n }\n\n for _ in 0..total {\n rx.recv().unwrap();\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn test_nested_recv_iter() {\n let (tx, rx) = channel::();\n let (total_tx, total_rx) = channel::();\n\n let t = thread::spawn(move || {\n let mut acc = 0;\n for x in rx.iter() {\n acc += x;\n }\n total_tx.send(acc).unwrap();\n });\n\n tx.send(3).unwrap();\n tx.send(1).unwrap();\n tx.send(2).unwrap();\n drop(tx);\n assert_eq!(total_rx.recv().unwrap(), 6);\n t.join().unwrap();\n }\n\n #[test]\n fn test_recv_iter_break() {\n let (tx, rx) = channel::();\n let (count_tx, count_rx) = channel();\n\n let t = thread::spawn(move || {\n let mut count = 0;\n for x in rx.iter() {\n if count >= 3 {\n break;\n } else {\n count += x;\n }\n }\n count_tx.send(count).unwrap();\n });\n\n tx.send(2).unwrap();\n tx.send(2).unwrap();\n tx.send(2).unwrap();\n let _ = tx.send(2);\n drop(tx);\n assert_eq!(count_rx.recv().unwrap(), 4);\n t.join().unwrap();\n }\n\n #[test]\n fn test_recv_try_iter() {\n let (request_tx, request_rx) = channel();\n let (response_tx, response_rx) = channel();\n\n // Request `x`s until we have `6`.\n let t = thread::spawn(move || {\n let mut count = 0;\n loop {\n for x in response_rx.try_iter() {\n count += x;\n if count == 6 {\n return count;\n }\n }\n request_tx.send(()).unwrap();\n }\n });\n\n for _ in request_rx.iter() {\n if response_tx.send(2).is_err() {\n break;\n }\n }\n\n assert_eq!(t.join().unwrap(), 6);\n }\n\n #[test]\n fn test_recv_into_iter_owned() {\n let mut iter = {\n let (tx, rx) = channel::();\n tx.send(1).unwrap();\n tx.send(2).unwrap();\n\n rx.into_iter()\n };\n assert_eq!(iter.next().unwrap(), 1);\n assert_eq!(iter.next().unwrap(), 2);\n assert!(iter.next().is_none());\n }\n\n #[test]\n fn test_recv_into_iter_borrowed() {\n let (tx, rx) = channel::();\n tx.send(1).unwrap();\n tx.send(2).unwrap();\n drop(tx);\n let mut iter = (&rx).into_iter();\n assert_eq!(iter.next().unwrap(), 1);\n assert_eq!(iter.next().unwrap(), 2);\n assert!(iter.next().is_none());\n }\n\n #[test]\n fn try_recv_states() {\n let (tx1, rx1) = channel::();\n let (tx2, rx2) = channel::<()>();\n let (tx3, rx3) = channel::<()>();\n let t = thread::spawn(move || {\n rx2.recv().unwrap();\n tx1.send(1).unwrap();\n tx3.send(()).unwrap();\n rx2.recv().unwrap();\n drop(tx1);\n tx3.send(()).unwrap();\n });\n\n assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty));\n tx2.send(()).unwrap();\n rx3.recv().unwrap();\n assert_eq!(rx1.try_recv(), Ok(1));\n assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty));\n tx2.send(()).unwrap();\n rx3.recv().unwrap();\n assert_eq!(rx1.try_recv(), Err(TryRecvError::Disconnected));\n t.join().unwrap();\n }\n\n // This bug used to end up in a livelock inside of the Receiver destructor\n // because the internal state of the Shared packet was corrupted\n #[test]\n fn destroy_upgraded_shared_port_when_sender_still_active() {\n let (tx, rx) = channel();\n let (tx2, rx2) = channel();\n let t = thread::spawn(move || {\n rx.recv().unwrap(); // wait on a oneshot\n drop(rx); // destroy a shared\n tx2.send(()).unwrap();\n });\n // make sure the other thread has gone to sleep\n for _ in 0..5000 {\n thread::yield_now();\n }\n\n // upgrade to a shared chan and send a message\n let tx2 = tx.clone();\n drop(tx);\n tx2.send(()).unwrap();\n\n // wait for the child thread to exit before we exit\n rx2.recv().unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn issue_32114() {\n let (tx, _) = channel();\n let _ = tx.send(123);\n assert_eq!(tx.send(123), Err(SendError(123)));\n }\n}\n\n// Source: https://github.com/rust-lang/rust/blob/master/src/libstd/sync/mpsc/mod.rs\nmod sync_channel_tests {\n use std::{env, thread};\n\n use super::*;\n\n fn stress_factor() -> usize {\n match env::var(\"RUST_TEST_STRESS\") {\n Ok(val) => val.parse().unwrap(),\n Err(..) => 1,\n }\n }\n\n #[test]\n fn smoke() {\n let (tx, rx) = sync_channel::(1);\n tx.send(1).unwrap();\n assert_eq!(rx.recv().unwrap(), 1);\n }\n\n #[test]\n fn drop_full() {\n let (tx, _rx) = sync_channel::>(1);\n tx.send(Box::new(1)).unwrap();\n }\n\n #[test]\n fn smoke_shared() {\n let (tx, rx) = sync_channel::(1);\n tx.send(1).unwrap();\n assert_eq!(rx.recv().unwrap(), 1);\n let tx = tx.clone();\n tx.send(1).unwrap();\n assert_eq!(rx.recv().unwrap(), 1);\n }\n\n #[test]\n fn recv_timeout() {\n let (tx, rx) = sync_channel::(1);\n assert_eq!(\n rx.recv_timeout(Duration::from_millis(1)),\n Err(RecvTimeoutError::Timeout)\n );\n tx.send(1).unwrap();\n assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(1));\n }\n\n #[test]\n fn smoke_threads() {\n let (tx, rx) = sync_channel::(0);\n let t = thread::spawn(move || {\n tx.send(1).unwrap();\n });\n assert_eq!(rx.recv().unwrap(), 1);\n t.join().unwrap();\n }\n\n #[test]\n fn smoke_port_gone() {\n let (tx, rx) = sync_channel::(0);\n drop(rx);\n assert!(tx.send(1).is_err());\n }\n\n #[test]\n fn smoke_shared_port_gone2() {\n let (tx, rx) = sync_channel::(0);\n drop(rx);\n let tx2 = tx.clone();\n drop(tx);\n assert!(tx2.send(1).is_err());\n }\n\n #[test]\n fn port_gone_concurrent() {\n let (tx, rx) = sync_channel::(0);\n let t = thread::spawn(move || {\n rx.recv().unwrap();\n });\n while tx.send(1).is_ok() {}\n t.join().unwrap();\n }\n\n #[test]\n fn port_gone_concurrent_shared() {\n let (tx, rx) = sync_channel::(0);\n let tx2 = tx.clone();\n let t = thread::spawn(move || {\n rx.recv().unwrap();\n });\n while tx.send(1).is_ok() && tx2.send(1).is_ok() {}\n t.join().unwrap();\n }\n\n #[test]\n fn smoke_chan_gone() {\n let (tx, rx) = sync_channel::(0);\n drop(tx);\n assert!(rx.recv().is_err());\n }\n\n #[test]\n fn smoke_chan_gone_shared() {\n let (tx, rx) = sync_channel::<()>(0);\n let tx2 = tx.clone();\n drop(tx);\n drop(tx2);\n assert!(rx.recv().is_err());\n }\n\n #[test]\n fn chan_gone_concurrent() {\n let (tx, rx) = sync_channel::(0);\n let t = thread::spawn(move || {\n tx.send(1).unwrap();\n tx.send(1).unwrap();\n });\n while rx.recv().is_ok() {}\n t.join().unwrap();\n }\n\n #[test]\n fn stress() {\n const N: usize = if cfg!(miri) { 100 } else { 10000 };\n\n let (tx, rx) = sync_channel::(0);\n let t = thread::spawn(move || {\n for _ in 0..N {\n tx.send(1).unwrap();\n }\n });\n for _ in 0..N {\n assert_eq!(rx.recv().unwrap(), 1);\n }\n t.join().unwrap();\n }\n\n #[test]\n fn stress_recv_timeout_two_threads() {\n const N: usize = if cfg!(miri) { 100 } else { 10000 };\n\n let (tx, rx) = sync_channel::(0);\n\n let t = thread::spawn(move || {\n for _ in 0..N {\n tx.send(1).unwrap();\n }\n });\n\n let mut recv_count = 0;\n loop {\n match rx.recv_timeout(Duration::from_millis(1)) {\n Ok(v) => {\n assert_eq!(v, 1);\n recv_count += 1;\n }\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n assert_eq!(recv_count, N);\n t.join().unwrap();\n }\n\n #[test]\n fn stress_recv_timeout_shared() {\n const AMT: u32 = if cfg!(miri) { 100 } else { 1000 };\n const NTHREADS: u32 = 8;\n let (tx, rx) = sync_channel::(0);\n let (dtx, drx) = sync_channel::<()>(0);\n\n let t = thread::spawn(move || {\n let mut recv_count = 0;\n loop {\n match rx.recv_timeout(Duration::from_millis(10)) {\n Ok(v) => {\n assert_eq!(v, 1);\n recv_count += 1;\n }\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n assert_eq!(recv_count, AMT * NTHREADS);\n assert!(rx.try_recv().is_err());\n\n dtx.send(()).unwrap();\n });\n\n let mut ts = Vec::with_capacity(NTHREADS as usize);\n for _ in 0..NTHREADS {\n let tx = tx.clone();\n let t = thread::spawn(move || {\n for _ in 0..AMT {\n tx.send(1).unwrap();\n }\n });\n ts.push(t);\n }\n\n drop(tx);\n\n drx.recv().unwrap();\n for t in ts {\n t.join().unwrap();\n }\n t.join().unwrap();\n }\n\n #[test]\n fn stress_shared() {\n const AMT: u32 = if cfg!(miri) { 100 } else { 1000 };\n const NTHREADS: u32 = 8;\n let (tx, rx) = sync_channel::(0);\n let (dtx, drx) = sync_channel::<()>(0);\n\n let t = thread::spawn(move || {\n for _ in 0..AMT * NTHREADS {\n assert_eq!(rx.recv().unwrap(), 1);\n }\n assert!(rx.try_recv().is_err());\n dtx.send(()).unwrap();\n });\n\n let mut ts = Vec::with_capacity(NTHREADS as usize);\n for _ in 0..NTHREADS {\n let tx = tx.clone();\n let t = thread::spawn(move || {\n for _ in 0..AMT {\n tx.send(1).unwrap();\n }\n });\n ts.push(t);\n }\n drop(tx);\n drx.recv().unwrap();\n for t in ts {\n t.join().unwrap();\n }\n t.join().unwrap();\n }\n\n #[test]\n fn oneshot_single_thread_close_port_first() {\n // Simple test of closing without sending\n let (_tx, rx) = sync_channel::(0);\n drop(rx);\n }\n\n #[test]\n fn oneshot_single_thread_close_chan_first() {\n // Simple test of closing without sending\n let (tx, _rx) = sync_channel::(0);\n drop(tx);\n }\n\n #[test]\n fn oneshot_single_thread_send_port_close() {\n // Testing that the sender cleans up the payload if receiver is closed\n let (tx, rx) = sync_channel::>(0);\n drop(rx);\n assert!(tx.send(Box::new(0)).is_err());\n }\n\n #[test]\n fn oneshot_single_thread_recv_chan_close() {\n let (tx, rx) = sync_channel::(0);\n drop(tx);\n assert_eq!(rx.recv(), Err(RecvError));\n }\n\n #[test]\n fn oneshot_single_thread_send_then_recv() {\n let (tx, rx) = sync_channel::>(1);\n tx.send(Box::new(10)).unwrap();\n assert!(*rx.recv().unwrap() == 10);\n }\n\n #[test]\n fn oneshot_single_thread_try_send_open() {\n let (tx, rx) = sync_channel::(1);\n assert_eq!(tx.try_send(10), Ok(()));\n assert!(rx.recv().unwrap() == 10);\n }\n\n #[test]\n fn oneshot_single_thread_try_send_closed() {\n let (tx, rx) = sync_channel::(0);\n drop(rx);\n assert_eq!(tx.try_send(10), Err(TrySendError::Disconnected(10)));\n }\n\n #[test]\n fn oneshot_single_thread_try_send_closed2() {\n let (tx, _rx) = sync_channel::(0);\n assert_eq!(tx.try_send(10), Err(TrySendError::Full(10)));\n }\n\n #[test]\n fn oneshot_single_thread_try_recv_open() {\n let (tx, rx) = sync_channel::(1);\n tx.send(10).unwrap();\n assert!(rx.recv() == Ok(10));\n }\n\n #[test]\n fn oneshot_single_thread_try_recv_closed() {\n let (tx, rx) = sync_channel::(0);\n drop(tx);\n assert!(rx.recv().is_err());\n }\n\n #[test]\n fn oneshot_single_thread_try_recv_closed_with_data() {\n let (tx, rx) = sync_channel::(1);\n tx.send(10).unwrap();\n drop(tx);\n assert_eq!(rx.try_recv(), Ok(10));\n assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected));\n }\n\n #[test]\n fn oneshot_single_thread_peek_data() {\n let (tx, rx) = sync_channel::(1);\n assert_eq!(rx.try_recv(), Err(TryRecvError::Empty));\n tx.send(10).unwrap();\n assert_eq!(rx.try_recv(), Ok(10));\n }\n\n #[test]\n fn oneshot_single_thread_peek_close() {\n let (tx, rx) = sync_channel::(0);\n drop(tx);\n assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected));\n assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected));\n }\n\n #[test]\n fn oneshot_single_thread_peek_open() {\n let (_tx, rx) = sync_channel::(0);\n assert_eq!(rx.try_recv(), Err(TryRecvError::Empty));\n }\n\n #[test]\n fn oneshot_multi_task_recv_then_send() {\n let (tx, rx) = sync_channel::>(0);\n let t = thread::spawn(move || {\n assert!(*rx.recv().unwrap() == 10);\n });\n\n tx.send(Box::new(10)).unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn oneshot_multi_task_recv_then_close() {\n let (tx, rx) = sync_channel::>(0);\n let t = thread::spawn(move || {\n drop(tx);\n });\n thread::spawn(move || {\n assert_eq!(rx.recv(), Err(RecvError));\n })\n .join()\n .unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn oneshot_multi_thread_close_stress() {\n let stress_factor = stress_factor();\n let mut ts = Vec::with_capacity(stress_factor);\n for _ in 0..stress_factor {\n let (tx, rx) = sync_channel::(0);\n let t = thread::spawn(move || {\n drop(rx);\n });\n ts.push(t);\n drop(tx);\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn oneshot_multi_thread_send_close_stress() {\n let stress_factor = stress_factor();\n let mut ts = Vec::with_capacity(stress_factor);\n for _ in 0..stress_factor {\n let (tx, rx) = sync_channel::(0);\n let t = thread::spawn(move || {\n drop(rx);\n });\n ts.push(t);\n thread::spawn(move || {\n let _ = tx.send(1);\n })\n .join()\n .unwrap();\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn oneshot_multi_thread_recv_close_stress() {\n let stress_factor = stress_factor();\n let mut ts = Vec::with_capacity(2 * stress_factor);\n for _ in 0..stress_factor {\n let (tx, rx) = sync_channel::(0);\n let t = thread::spawn(move || {\n thread::spawn(move || {\n assert_eq!(rx.recv(), Err(RecvError));\n })\n .join()\n .unwrap();\n });\n ts.push(t);\n let t2 = thread::spawn(move || {\n thread::spawn(move || {\n drop(tx);\n });\n });\n ts.push(t2);\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn oneshot_multi_thread_send_recv_stress() {\n let stress_factor = stress_factor();\n let mut ts = Vec::with_capacity(stress_factor);\n for _ in 0..stress_factor {\n let (tx, rx) = sync_channel::>(0);\n let t = thread::spawn(move || {\n tx.send(Box::new(10)).unwrap();\n });\n ts.push(t);\n assert!(*rx.recv().unwrap() == 10);\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn stream_send_recv_stress() {\n let stress_factor = stress_factor();\n let mut ts = Vec::with_capacity(2 * stress_factor);\n for _ in 0..stress_factor {\n let (tx, rx) = sync_channel::>(0);\n\n if let Some(t) = send(tx, 0) {\n ts.push(t);\n }\n if let Some(t) = recv(rx, 0) {\n ts.push(t);\n }\n\n fn send(tx: SyncSender>, i: i32) -> Option> {\n if i == 10 {\n return None;\n }\n\n Some(thread::spawn(move || {\n tx.send(Box::new(i)).unwrap();\n send(tx, i + 1);\n }))\n }\n\n fn recv(rx: Receiver>, i: i32) -> Option> {\n if i == 10 {\n return None;\n }\n\n Some(thread::spawn(move || {\n assert!(*rx.recv().unwrap() == i);\n recv(rx, i + 1);\n }))\n }\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn recv_a_lot() {\n const N: usize = if cfg!(miri) { 100 } else { 10000 };\n\n // Regression test that we don't run out of stack in scheduler context\n let (tx, rx) = sync_channel(N);\n for _ in 0..N {\n tx.send(()).unwrap();\n }\n for _ in 0..N {\n rx.recv().unwrap();\n }\n }\n\n #[test]\n fn shared_chan_stress() {\n let (tx, rx) = sync_channel(0);\n let total = stress_factor() + 100;\n let mut ts = Vec::with_capacity(total);\n for _ in 0..total {\n let tx = tx.clone();\n let t = thread::spawn(move || {\n tx.send(()).unwrap();\n });\n ts.push(t);\n }\n\n for _ in 0..total {\n rx.recv().unwrap();\n }\n for t in ts {\n t.join().unwrap();\n }\n }\n\n #[test]\n fn test_nested_recv_iter() {\n let (tx, rx) = sync_channel::(0);\n let (total_tx, total_rx) = sync_channel::(0);\n\n let t = thread::spawn(move || {\n let mut acc = 0;\n for x in rx.iter() {\n acc += x;\n }\n total_tx.send(acc).unwrap();\n });\n\n tx.send(3).unwrap();\n tx.send(1).unwrap();\n tx.send(2).unwrap();\n drop(tx);\n assert_eq!(total_rx.recv().unwrap(), 6);\n t.join().unwrap();\n }\n\n #[test]\n fn test_recv_iter_break() {\n let (tx, rx) = sync_channel::(0);\n let (count_tx, count_rx) = sync_channel(0);\n\n let t = thread::spawn(move || {\n let mut count = 0;\n for x in rx.iter() {\n if count >= 3 {\n break;\n } else {\n count += x;\n }\n }\n count_tx.send(count).unwrap();\n });\n\n tx.send(2).unwrap();\n tx.send(2).unwrap();\n tx.send(2).unwrap();\n let _ = tx.try_send(2);\n drop(tx);\n assert_eq!(count_rx.recv().unwrap(), 4);\n t.join().unwrap();\n }\n\n #[test]\n fn try_recv_states() {\n let (tx1, rx1) = sync_channel::(1);\n let (tx2, rx2) = sync_channel::<()>(1);\n let (tx3, rx3) = sync_channel::<()>(1);\n let t = thread::spawn(move || {\n rx2.recv().unwrap();\n tx1.send(1).unwrap();\n tx3.send(()).unwrap();\n rx2.recv().unwrap();\n drop(tx1);\n tx3.send(()).unwrap();\n });\n\n assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty));\n tx2.send(()).unwrap();\n rx3.recv().unwrap();\n assert_eq!(rx1.try_recv(), Ok(1));\n assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty));\n tx2.send(()).unwrap();\n rx3.recv().unwrap();\n assert_eq!(rx1.try_recv(), Err(TryRecvError::Disconnected));\n t.join().unwrap();\n }\n\n // This bug used to end up in a livelock inside of the Receiver destructor\n // because the internal state of the Shared packet was corrupted\n #[test]\n fn destroy_upgraded_shared_port_when_sender_still_active() {\n let (tx, rx) = sync_channel::<()>(0);\n let (tx2, rx2) = sync_channel::<()>(0);\n let t = thread::spawn(move || {\n rx.recv().unwrap(); // wait on a oneshot\n drop(rx); // destroy a shared\n tx2.send(()).unwrap();\n });\n // make sure the other thread has gone to sleep\n for _ in 0..5000 {\n thread::yield_now();\n }\n\n // upgrade to a shared chan and send a message\n let tx2 = tx.clone();\n drop(tx);\n tx2.send(()).unwrap();\n\n // wait for the child thread to exit before we exit\n rx2.recv().unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn send1() {\n let (tx, rx) = sync_channel::(0);\n let t = thread::spawn(move || {\n rx.recv().unwrap();\n });\n assert_eq!(tx.send(1), Ok(()));\n t.join().unwrap();\n }\n\n #[test]\n fn send2() {\n let (tx, rx) = sync_channel::(0);\n let t = thread::spawn(move || {\n drop(rx);\n });\n assert!(tx.send(1).is_err());\n t.join().unwrap();\n }\n\n #[test]\n fn send3() {\n let (tx, rx) = sync_channel::(1);\n assert_eq!(tx.send(1), Ok(()));\n let t = thread::spawn(move || {\n drop(rx);\n });\n assert!(tx.send(1).is_err());\n t.join().unwrap();\n }\n\n #[test]\n fn send4() {\n let (tx, rx) = sync_channel::(0);\n let tx2 = tx.clone();\n let (done, donerx) = channel();\n let done2 = done.clone();\n let t = thread::spawn(move || {\n assert!(tx.send(1).is_err());\n done.send(()).unwrap();\n });\n let t2 = thread::spawn(move || {\n assert!(tx2.send(2).is_err());\n done2.send(()).unwrap();\n });\n drop(rx);\n donerx.recv().unwrap();\n donerx.recv().unwrap();\n t.join().unwrap();\n t2.join().unwrap();\n }\n\n #[test]\n fn try_send1() {\n let (tx, _rx) = sync_channel::(0);\n assert_eq!(tx.try_send(1), Err(TrySendError::Full(1)));\n }\n\n #[test]\n fn try_send2() {\n let (tx, _rx) = sync_channel::(1);\n assert_eq!(tx.try_send(1), Ok(()));\n assert_eq!(tx.try_send(1), Err(TrySendError::Full(1)));\n }\n\n #[test]\n fn try_send3() {\n let (tx, rx) = sync_channel::(1);\n assert_eq!(tx.try_send(1), Ok(()));\n drop(rx);\n assert_eq!(tx.try_send(1), Err(TrySendError::Disconnected(1)));\n }\n\n #[test]\n fn issue_15761() {\n fn repro() {\n let (tx1, rx1) = sync_channel::<()>(3);\n let (tx2, rx2) = sync_channel::<()>(3);\n\n let _t = thread::spawn(move || {\n rx1.recv().unwrap();\n tx2.try_send(()).unwrap();\n });\n\n tx1.try_send(()).unwrap();\n rx2.recv().unwrap();\n }\n\n for _ in 0..100 {\n repro()\n }\n }\n}\n\n// Source: https://github.com/rust-lang/rust/blob/master/src/libstd/sync/mpsc/select.rs\nmod select_tests {\n use std::thread;\n\n use super::*;\n\n #[test]\n fn smoke() {\n let (tx1, rx1) = channel::();\n let (tx2, rx2) = channel::();\n tx1.send(1).unwrap();\n select! {\n foo = rx1.recv() => assert_eq!(foo.unwrap(), 1),\n _bar = rx2.recv() => panic!()\n }\n tx2.send(2).unwrap();\n select! {\n _foo = rx1.recv() => panic!(),\n bar = rx2.recv() => assert_eq!(bar.unwrap(), 2)\n }\n drop(tx1);\n select! {\n foo = rx1.recv() => assert!(foo.is_err()),\n _bar = rx2.recv() => panic!()\n }\n drop(tx2);\n select! {\n bar = rx2.recv() => assert!(bar.is_err())\n }\n }\n\n #[test]\n fn smoke2() {\n let (_tx1, rx1) = channel::();\n let (_tx2, rx2) = channel::();\n let (_tx3, rx3) = channel::();\n let (_tx4, rx4) = channel::();\n let (tx5, rx5) = channel::();\n tx5.send(4).unwrap();\n select! {\n _foo = rx1.recv() => panic!(\"1\"),\n _foo = rx2.recv() => panic!(\"2\"),\n _foo = rx3.recv() => panic!(\"3\"),\n _foo = rx4.recv() => panic!(\"4\"),\n foo = rx5.recv() => assert_eq!(foo.unwrap(), 4)\n }\n }\n\n #[test]\n fn closed() {\n let (_tx1, rx1) = channel::();\n let (tx2, rx2) = channel::();\n drop(tx2);\n\n select! {\n _a1 = rx1.recv() => panic!(),\n a2 = rx2.recv() => assert!(a2.is_err())\n }\n }\n\n #[test]\n fn unblocks() {\n let (tx1, rx1) = channel::();\n let (_tx2, rx2) = channel::();\n let (tx3, rx3) = channel::();\n\n let t = thread::spawn(move || {\n for _ in 0..20 {\n thread::yield_now();\n }\n tx1.send(1).unwrap();\n rx3.recv().unwrap();\n for _ in 0..20 {\n thread::yield_now();\n }\n });\n\n select! {\n a = rx1.recv() => assert_eq!(a.unwrap(), 1),\n _b = rx2.recv() => panic!()\n }\n tx3.send(1).unwrap();\n select! {\n a = rx1.recv() => assert!(a.is_err()),\n _b = rx2.recv() => panic!()\n }\n t.join().unwrap();\n }\n\n #[test]\n fn both_ready() {\n let (tx1, rx1) = channel::();\n let (tx2, rx2) = channel::();\n let (tx3, rx3) = channel::<()>();\n\n let t = thread::spawn(move || {\n for _ in 0..20 {\n thread::yield_now();\n }\n tx1.send(1).unwrap();\n tx2.send(2).unwrap();\n rx3.recv().unwrap();\n });\n\n select! {\n a = rx1.recv() => { assert_eq!(a.unwrap(), 1); },\n a = rx2.recv() => { assert_eq!(a.unwrap(), 2); }\n }\n select! {\n a = rx1.recv() => { assert_eq!(a.unwrap(), 1); },\n a = rx2.recv() => { assert_eq!(a.unwrap(), 2); }\n }\n assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty));\n assert_eq!(rx2.try_recv(), Err(TryRecvError::Empty));\n tx3.send(()).unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn stress() {\n const AMT: i32 = if cfg!(miri) { 100 } else { 10000 };\n\n let (tx1, rx1) = channel::();\n let (tx2, rx2) = channel::();\n let (tx3, rx3) = channel::<()>();\n\n let t = thread::spawn(move || {\n for i in 0..AMT {\n if i % 2 == 0 {\n tx1.send(i).unwrap();\n } else {\n tx2.send(i).unwrap();\n }\n rx3.recv().unwrap();\n }\n });\n\n for i in 0..AMT {\n select! {\n i1 = rx1.recv() => { assert!(i % 2 == 0 && i == i1.unwrap()); },\n i2 = rx2.recv() => { assert!(i % 2 == 1 && i == i2.unwrap()); }\n }\n tx3.send(()).unwrap();\n }\n t.join().unwrap();\n }\n\n #[allow(unused_must_use)]\n #[test]\n fn cloning() {\n let (tx1, rx1) = channel::();\n let (_tx2, rx2) = channel::();\n let (tx3, rx3) = channel::<()>();\n\n let t = thread::spawn(move || {\n rx3.recv().unwrap();\n tx1.clone();\n assert_eq!(rx3.try_recv(), Err(TryRecvError::Empty));\n tx1.send(2).unwrap();\n rx3.recv().unwrap();\n });\n\n tx3.send(()).unwrap();\n select! {\n _i1 = rx1.recv() => {},\n _i2 = rx2.recv() => panic!()\n }\n tx3.send(()).unwrap();\n t.join().unwrap();\n }\n\n #[allow(unused_must_use)]\n #[test]\n fn cloning2() {\n let (tx1, rx1) = channel::();\n let (_tx2, rx2) = channel::();\n let (tx3, rx3) = channel::<()>();\n\n let t = thread::spawn(move || {\n rx3.recv().unwrap();\n tx1.clone();\n assert_eq!(rx3.try_recv(), Err(TryRecvError::Empty));\n tx1.send(2).unwrap();\n rx3.recv().unwrap();\n });\n\n tx3.send(()).unwrap();\n select! {\n _i1 = rx1.recv() => {},\n _i2 = rx2.recv() => panic!()\n }\n tx3.send(()).unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn cloning3() {\n let (tx1, rx1) = channel::<()>();\n let (tx2, rx2) = channel::<()>();\n let (tx3, rx3) = channel::<()>();\n let t = thread::spawn(move || {\n select! {\n _ = rx1.recv() => panic!(),\n _ = rx2.recv() => {}\n }\n tx3.send(()).unwrap();\n });\n\n for _ in 0..1000 {\n thread::yield_now();\n }\n drop(tx1.clone());\n tx2.send(()).unwrap();\n rx3.recv().unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn preflight1() {\n let (tx, rx) = channel();\n tx.send(()).unwrap();\n select! {\n _n = rx.recv() => {}\n }\n }\n\n #[test]\n fn preflight2() {\n let (tx, rx) = channel();\n tx.send(()).unwrap();\n tx.send(()).unwrap();\n select! {\n _n = rx.recv() => {}\n }\n }\n\n #[test]\n fn preflight3() {\n let (tx, rx) = channel();\n drop(tx.clone());\n tx.send(()).unwrap();\n select! {\n _n = rx.recv() => {}\n }\n }\n\n #[test]\n fn preflight4() {\n let (tx, rx) = channel();\n tx.send(()).unwrap();\n select! {\n _ = rx.recv() => {}\n }\n }\n\n #[test]\n fn preflight5() {\n let (tx, rx) = channel();\n tx.send(()).unwrap();\n tx.send(()).unwrap();\n select! {\n _ = rx.recv() => {}\n }\n }\n\n #[test]\n fn preflight6() {\n let (tx, rx) = channel();\n drop(tx.clone());\n tx.send(()).unwrap();\n select! {\n _ = rx.recv() => {}\n }\n }\n\n #[test]\n fn preflight7() {\n let (tx, rx) = channel::<()>();\n drop(tx);\n select! {\n _ = rx.recv() => {}\n }\n }\n\n #[test]\n fn preflight8() {\n let (tx, rx) = channel();\n tx.send(()).unwrap();\n drop(tx);\n rx.recv().unwrap();\n select! {\n _ = rx.recv() => {}\n }\n }\n\n #[test]\n fn preflight9() {\n let (tx, rx) = channel();\n drop(tx.clone());\n tx.send(()).unwrap();\n drop(tx);\n rx.recv().unwrap();\n select! {\n _ = rx.recv() => {}\n }\n }\n\n #[test]\n fn oneshot_data_waiting() {\n let (tx1, rx1) = channel();\n let (tx2, rx2) = channel();\n let t = thread::spawn(move || {\n select! {\n _n = rx1.recv() => {}\n }\n tx2.send(()).unwrap();\n });\n\n for _ in 0..100 {\n thread::yield_now()\n }\n tx1.send(()).unwrap();\n rx2.recv().unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn stream_data_waiting() {\n let (tx1, rx1) = channel();\n let (tx2, rx2) = channel();\n tx1.send(()).unwrap();\n tx1.send(()).unwrap();\n rx1.recv().unwrap();\n rx1.recv().unwrap();\n let t = thread::spawn(move || {\n select! {\n _n = rx1.recv() => {}\n }\n tx2.send(()).unwrap();\n });\n\n for _ in 0..100 {\n thread::yield_now()\n }\n tx1.send(()).unwrap();\n rx2.recv().unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn shared_data_waiting() {\n let (tx1, rx1) = channel();\n let (tx2, rx2) = channel();\n drop(tx1.clone());\n tx1.send(()).unwrap();\n rx1.recv().unwrap();\n let t = thread::spawn(move || {\n select! {\n _n = rx1.recv() => {}\n }\n tx2.send(()).unwrap();\n });\n\n for _ in 0..100 {\n thread::yield_now()\n }\n tx1.send(()).unwrap();\n rx2.recv().unwrap();\n t.join().unwrap();\n }\n\n #[test]\n fn sync1() {\n let (tx, rx) = sync_channel::(1);\n tx.send(1).unwrap();\n select! {\n n = rx.recv() => { assert_eq!(n.unwrap(), 1); }\n }\n }\n\n #[test]\n fn sync2() {\n let (tx, rx) = sync_channel::(0);\n let t = thread::spawn(move || {\n for _ in 0..100 {\n thread::yield_now()\n }\n tx.send(1).unwrap();\n });\n select! {\n n = rx.recv() => { assert_eq!(n.unwrap(), 1); }\n }\n t.join().unwrap();\n }\n\n #[test]\n fn sync3() {\n let (tx1, rx1) = sync_channel::(0);\n let (tx2, rx2): (Sender, Receiver) = channel();\n let t = thread::spawn(move || {\n tx1.send(1).unwrap();\n });\n let t2 = thread::spawn(move || {\n tx2.send(2).unwrap();\n });\n select! {\n n = rx1.recv() => {\n let n = n.unwrap();\n assert_eq!(n, 1);\n assert_eq!(rx2.recv().unwrap(), 2);\n },\n n = rx2.recv() => {\n let n = n.unwrap();\n assert_eq!(n, 2);\n assert_eq!(rx1.recv().unwrap(), 1);\n }\n }\n t.join().unwrap();\n t2.join().unwrap();\n }\n}\n" }, { "path": "crossbeam-channel/tests/never.rs", "content": "//! Tests for the never channel flavor.\n\nuse std::{\n thread,\n time::{Duration, Instant},\n};\n\nuse crossbeam_channel::{never, select, tick, unbounded};\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\nfn smoke() {\n select! {\n recv(never::()) -> _ => panic!(),\n default => {}\n }\n}\n\n#[test]\nfn optional() {\n let (s, r) = unbounded::();\n s.send(1).unwrap();\n s.send(2).unwrap();\n\n let mut r = Some(&r);\n select! {\n recv(r.unwrap_or(&never())) -> _ => {}\n default => panic!(),\n }\n\n r = None;\n select! {\n recv(r.unwrap_or(&never())) -> _ => panic!(),\n default => {}\n }\n}\n\n#[test]\nfn tick_n() {\n let mut r = tick(ms(100));\n let mut step = 0;\n\n loop {\n select! {\n recv(r) -> _ => step += 1,\n default(ms(500)) => break,\n }\n\n if step == 10 {\n r = never();\n }\n }\n\n assert_eq!(step, 10);\n}\n\n#[test]\nfn capacity() {\n let r = never::();\n assert_eq!(r.capacity(), Some(0));\n}\n\n#[test]\nfn len_empty_full() {\n let r = never::();\n assert_eq!(r.len(), 0);\n assert!(r.is_empty());\n assert!(r.is_full());\n}\n\n#[test]\nfn try_recv() {\n let r = never::();\n assert!(r.try_recv().is_err());\n\n thread::sleep(ms(100));\n assert!(r.try_recv().is_err());\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn recv_timeout() {\n let start = Instant::now();\n let r = never::();\n\n assert!(r.recv_timeout(ms(100)).is_err());\n let now = Instant::now();\n assert!(now - start >= ms(100));\n assert!(now - start <= ms(150));\n\n assert!(r.recv_timeout(ms(100)).is_err());\n let now = Instant::now();\n assert!(now - start >= ms(200));\n assert!(now - start <= ms(250));\n}\n" }, { "path": "crossbeam-channel/tests/ready.rs", "content": "//! Tests for channel readiness using the `Select` struct.\n\nuse std::{\n any::Any,\n cell::Cell,\n thread,\n time::{Duration, Instant},\n};\n\nuse crossbeam_channel::{\n Receiver, Select, TryRecvError, TrySendError, after, bounded, tick, unbounded,\n};\nuse crossbeam_utils::thread::scope;\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\nfn smoke1() {\n let (s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n s1.send(1).unwrap();\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n assert_eq!(sel.ready(), 0);\n assert_eq!(r1.try_recv(), Ok(1));\n\n s2.send(2).unwrap();\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n assert_eq!(sel.ready(), 1);\n assert_eq!(r2.try_recv(), Ok(2));\n}\n\n#[test]\nfn smoke2() {\n let (_s1, r1) = unbounded::();\n let (_s2, r2) = unbounded::();\n let (_s3, r3) = unbounded::();\n let (_s4, r4) = unbounded::();\n let (s5, r5) = unbounded::();\n\n s5.send(5).unwrap();\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n sel.recv(&r3);\n sel.recv(&r4);\n sel.recv(&r5);\n assert_eq!(sel.ready(), 4);\n assert_eq!(r5.try_recv(), Ok(5));\n}\n\n#[test]\nfn disconnected() {\n let (s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n scope(|scope| {\n scope.spawn(|_| {\n drop(s1);\n thread::sleep(ms(500));\n s2.send(5).unwrap();\n });\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n match sel.ready_timeout(ms(1000)) {\n Ok(0) => assert_eq!(r1.try_recv(), Err(TryRecvError::Disconnected)),\n _ => panic!(),\n }\n\n r2.recv().unwrap();\n })\n .unwrap();\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n match sel.ready_timeout(ms(1000)) {\n Ok(0) => assert_eq!(r1.try_recv(), Err(TryRecvError::Disconnected)),\n _ => panic!(),\n }\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n drop(s2);\n });\n\n let mut sel = Select::new();\n sel.recv(&r2);\n match sel.ready_timeout(ms(1000)) {\n Ok(0) => assert_eq!(r2.try_recv(), Err(TryRecvError::Disconnected)),\n _ => panic!(),\n }\n })\n .unwrap();\n}\n\n#[test]\nfn default() {\n let (s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n assert!(sel.try_ready().is_err());\n\n drop(s1);\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n match sel.try_ready() {\n Ok(0) => assert!(r1.try_recv().is_err()),\n _ => panic!(),\n }\n\n s2.send(2).unwrap();\n\n let mut sel = Select::new();\n sel.recv(&r2);\n match sel.try_ready() {\n Ok(0) => assert_eq!(r2.try_recv(), Ok(2)),\n _ => panic!(),\n }\n\n let mut sel = Select::new();\n sel.recv(&r2);\n assert!(sel.try_ready().is_err());\n\n let mut sel = Select::new();\n assert!(sel.try_ready().is_err());\n}\n\n#[test]\nfn timeout() {\n let (_s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(1500));\n s2.send(2).unwrap();\n });\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n assert!(sel.ready_timeout(ms(1000)).is_err());\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n match sel.ready_timeout(ms(1000)) {\n Ok(1) => assert_eq!(r2.try_recv(), Ok(2)),\n _ => panic!(),\n }\n })\n .unwrap();\n\n scope(|scope| {\n let (s, r) = unbounded::();\n\n scope.spawn(move |_| {\n thread::sleep(ms(500));\n drop(s);\n });\n\n let mut sel = Select::new();\n assert!(sel.ready_timeout(ms(1000)).is_err());\n\n let mut sel = Select::new();\n sel.recv(&r);\n match sel.try_ready() {\n Ok(0) => assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected)),\n _ => panic!(),\n }\n })\n .unwrap();\n}\n\n#[test]\nfn default_when_disconnected() {\n let (_, r) = unbounded::();\n\n let mut sel = Select::new();\n sel.recv(&r);\n match sel.try_ready() {\n Ok(0) => assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected)),\n _ => panic!(),\n }\n\n let (_, r) = unbounded::();\n\n let mut sel = Select::new();\n sel.recv(&r);\n match sel.ready_timeout(ms(1000)) {\n Ok(0) => assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected)),\n _ => panic!(),\n }\n\n let (s, _) = bounded::(0);\n\n let mut sel = Select::new();\n sel.send(&s);\n match sel.try_ready() {\n Ok(0) => assert_eq!(s.try_send(0), Err(TrySendError::Disconnected(0))),\n _ => panic!(),\n }\n\n let (s, _) = bounded::(0);\n\n let mut sel = Select::new();\n sel.send(&s);\n match sel.ready_timeout(ms(1000)) {\n Ok(0) => assert_eq!(s.try_send(0), Err(TrySendError::Disconnected(0))),\n _ => panic!(),\n }\n}\n\n#[test]\n#[cfg_attr(miri, ignore)] // this test makes timing assumptions, but Miri is so slow it violates them\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn default_only() {\n let start = Instant::now();\n\n let mut sel = Select::new();\n assert!(sel.try_ready().is_err());\n let now = Instant::now();\n assert!(now - start <= ms(50));\n\n let start = Instant::now();\n let mut sel = Select::new();\n assert!(sel.ready_timeout(ms(500)).is_err());\n let now = Instant::now();\n assert!(now - start >= ms(450));\n assert!(now - start <= ms(550));\n}\n\n#[test]\nfn unblocks() {\n let (s1, r1) = bounded::(0);\n let (s2, r2) = bounded::(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n s2.send(2).unwrap();\n });\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n match sel.ready_timeout(ms(1000)) {\n Ok(1) => assert_eq!(r2.try_recv(), Ok(2)),\n _ => panic!(),\n }\n })\n .unwrap();\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n assert_eq!(r1.recv().unwrap(), 1);\n });\n\n let mut sel = Select::new();\n let oper1 = sel.send(&s1);\n let oper2 = sel.send(&s2);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => oper.send(&s1, 1).unwrap(),\n i if i == oper2 => panic!(),\n _ => unreachable!(),\n },\n }\n })\n .unwrap();\n}\n\n#[test]\nfn both_ready() {\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n s1.send(1).unwrap();\n assert_eq!(r2.recv().unwrap(), 2);\n });\n\n for _ in 0..2 {\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.send(&s2);\n match sel.ready() {\n 0 => assert_eq!(r1.try_recv(), Ok(1)),\n 1 => s2.try_send(2).unwrap(),\n _ => panic!(),\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn cloning1() {\n scope(|scope| {\n let (s1, r1) = unbounded::();\n let (_s2, r2) = unbounded::();\n let (s3, r3) = unbounded::<()>();\n\n scope.spawn(move |_| {\n r3.recv().unwrap();\n drop(s1.clone());\n assert!(r3.try_recv().is_err());\n s1.send(1).unwrap();\n r3.recv().unwrap();\n });\n\n s3.send(()).unwrap();\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n match sel.ready() {\n 0 => drop(r1.try_recv()),\n 1 => drop(r2.try_recv()),\n _ => panic!(),\n }\n\n s3.send(()).unwrap();\n })\n .unwrap();\n}\n\n#[test]\nfn cloning2() {\n let (s1, r1) = unbounded::<()>();\n let (s2, r2) = unbounded::<()>();\n let (_s3, _r3) = unbounded::<()>();\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n match sel.ready() {\n 0 => panic!(),\n 1 => drop(r2.try_recv()),\n _ => panic!(),\n }\n });\n\n thread::sleep(ms(500));\n drop(s1.clone());\n s2.send(()).unwrap();\n })\n .unwrap();\n}\n\n#[test]\nfn preflight1() {\n let (s, r) = unbounded();\n s.send(()).unwrap();\n\n let mut sel = Select::new();\n sel.recv(&r);\n match sel.ready() {\n 0 => drop(r.try_recv()),\n _ => panic!(),\n }\n}\n\n#[test]\nfn preflight2() {\n let (s, r) = unbounded();\n drop(s.clone());\n s.send(()).unwrap();\n drop(s);\n\n let mut sel = Select::new();\n sel.recv(&r);\n match sel.ready() {\n 0 => assert_eq!(r.try_recv(), Ok(())),\n _ => panic!(),\n }\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected));\n}\n\n#[test]\nfn preflight3() {\n let (s, r) = unbounded();\n drop(s.clone());\n s.send(()).unwrap();\n drop(s);\n r.recv().unwrap();\n\n let mut sel = Select::new();\n sel.recv(&r);\n match sel.ready() {\n 0 => assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected)),\n _ => panic!(),\n }\n}\n\n#[test]\nfn duplicate_operations() {\n let (s, r) = unbounded::();\n let hit = vec![Cell::new(false); 4];\n\n while hit.iter().map(|h| h.get()).any(|hit| !hit) {\n let mut sel = Select::new();\n sel.recv(&r);\n sel.recv(&r);\n sel.send(&s);\n sel.send(&s);\n match sel.ready() {\n 0 => {\n assert!(r.try_recv().is_ok());\n hit[0].set(true);\n }\n 1 => {\n assert!(r.try_recv().is_ok());\n hit[1].set(true);\n }\n 2 => {\n assert!(s.try_send(0).is_ok());\n hit[2].set(true);\n }\n 3 => {\n assert!(s.try_send(0).is_ok());\n hit[3].set(true);\n }\n _ => panic!(),\n }\n }\n}\n\n#[test]\nfn nesting() {\n let (s, r) = unbounded::();\n\n let mut sel = Select::new();\n sel.send(&s);\n match sel.ready() {\n 0 => {\n assert!(s.try_send(0).is_ok());\n\n let mut sel = Select::new();\n sel.recv(&r);\n match sel.ready() {\n 0 => {\n assert_eq!(r.try_recv(), Ok(0));\n\n let mut sel = Select::new();\n sel.send(&s);\n match sel.ready() {\n 0 => {\n assert!(s.try_send(1).is_ok());\n\n let mut sel = Select::new();\n sel.recv(&r);\n match sel.ready() {\n 0 => {\n assert_eq!(r.try_recv(), Ok(1));\n }\n _ => panic!(),\n }\n }\n _ => panic!(),\n }\n }\n _ => panic!(),\n }\n }\n _ => panic!(),\n }\n}\n\n#[test]\nfn stress_recv() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = unbounded();\n let (s2, r2) = bounded(5);\n let (s3, r3) = bounded(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n s1.send(i).unwrap();\n r3.recv().unwrap();\n\n s2.send(i).unwrap();\n r3.recv().unwrap();\n }\n });\n\n for i in 0..COUNT {\n for _ in 0..2 {\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n match sel.ready() {\n 0 => assert_eq!(r1.try_recv(), Ok(i)),\n 1 => assert_eq!(r2.try_recv(), Ok(i)),\n _ => panic!(),\n }\n\n s3.send(()).unwrap();\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_send() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n let (s3, r3) = bounded(100);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n assert_eq!(r1.recv().unwrap(), i);\n assert_eq!(r2.recv().unwrap(), i);\n r3.recv().unwrap();\n }\n });\n\n for i in 0..COUNT {\n for _ in 0..2 {\n let mut sel = Select::new();\n sel.send(&s1);\n sel.send(&s2);\n match sel.ready() {\n 0 => assert!(s1.try_send(i).is_ok()),\n 1 => assert!(s2.try_send(i).is_ok()),\n _ => panic!(),\n }\n }\n s3.send(()).unwrap();\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_mixed() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n let (s3, r3) = bounded(100);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n s1.send(i).unwrap();\n assert_eq!(r2.recv().unwrap(), i);\n r3.recv().unwrap();\n }\n });\n\n for i in 0..COUNT {\n for _ in 0..2 {\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.send(&s2);\n match sel.ready() {\n 0 => assert_eq!(r1.try_recv(), Ok(i)),\n 1 => assert!(s2.try_send(i).is_ok()),\n _ => panic!(),\n }\n }\n s3.send(()).unwrap();\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_timeout_two_threads() {\n const COUNT: usize = 20;\n\n let (s, r) = bounded(2);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(500));\n }\n\n loop {\n let mut sel = Select::new();\n sel.send(&s);\n match sel.ready_timeout(ms(100)) {\n Err(_) => {}\n Ok(0) => {\n assert!(s.try_send(i).is_ok());\n break;\n }\n Ok(_) => panic!(),\n }\n }\n }\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(500));\n }\n\n loop {\n let mut sel = Select::new();\n sel.recv(&r);\n match sel.ready_timeout(ms(100)) {\n Err(_) => {}\n Ok(0) => {\n assert_eq!(r.try_recv(), Ok(i));\n break;\n }\n Ok(_) => panic!(),\n }\n }\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn send_recv_same_channel() {\n let (s, r) = bounded::(0);\n let mut sel = Select::new();\n sel.send(&s);\n sel.recv(&r);\n assert!(sel.ready_timeout(ms(100)).is_err());\n\n let (s, r) = unbounded::();\n let mut sel = Select::new();\n sel.send(&s);\n sel.recv(&r);\n match sel.ready_timeout(ms(100)) {\n Err(_) => panic!(),\n Ok(0) => assert!(s.try_send(0).is_ok()),\n Ok(_) => panic!(),\n }\n}\n\n#[test]\nfn channel_through_channel() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 1000 };\n\n type T = Box;\n\n for cap in 1..4 {\n let (s, r) = bounded::(cap);\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut s = s;\n\n for _ in 0..COUNT {\n let (new_s, new_r) = bounded(cap);\n let new_r: T = Box::new(Some(new_r));\n\n {\n let mut sel = Select::new();\n sel.send(&s);\n match sel.ready() {\n 0 => assert!(s.try_send(new_r).is_ok()),\n _ => panic!(),\n }\n }\n\n s = new_s;\n }\n });\n\n scope.spawn(move |_| {\n let mut r = r;\n\n for _ in 0..COUNT {\n let new = {\n let mut sel = Select::new();\n sel.recv(&r);\n match sel.ready() {\n 0 => r\n .try_recv()\n .unwrap()\n .downcast_mut::>>()\n .unwrap()\n .take()\n .unwrap(),\n _ => panic!(),\n }\n };\n r = new;\n }\n });\n })\n .unwrap();\n }\n}\n\n#[test]\nfn fairness1() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = bounded::<()>(COUNT);\n let (s2, r2) = unbounded::<()>();\n\n for _ in 0..COUNT {\n s1.send(()).unwrap();\n s2.send(()).unwrap();\n }\n\n let hits = vec![Cell::new(0usize); 4];\n for _ in 0..COUNT {\n let after = after(ms(0));\n let tick = tick(ms(0));\n\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n sel.recv(&after);\n sel.recv(&tick);\n match sel.ready() {\n 0 => {\n r1.try_recv().unwrap();\n hits[0].set(hits[0].get() + 1);\n }\n 1 => {\n r2.try_recv().unwrap();\n hits[1].set(hits[1].get() + 1);\n }\n 2 => {\n after.try_recv().unwrap();\n hits[2].set(hits[2].get() + 1);\n }\n 3 => {\n tick.try_recv().unwrap();\n hits[3].set(hits[3].get() + 1);\n }\n _ => panic!(),\n }\n }\n assert!(hits.iter().all(|x| x.get() >= COUNT / hits.len() / 2));\n}\n\n#[test]\nfn fairness2() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 100_000 };\n\n let (s1, r1) = unbounded::<()>();\n let (s2, r2) = bounded::<()>(1);\n let (s3, r3) = bounded::<()>(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n let mut sel = Select::new();\n let mut oper1 = None;\n let mut oper2 = None;\n if s1.is_empty() {\n oper1 = Some(sel.send(&s1));\n }\n if s2.is_empty() {\n oper2 = Some(sel.send(&s2));\n }\n let oper3 = sel.send(&s3);\n let oper = sel.select();\n match oper.index() {\n i if Some(i) == oper1 => assert!(oper.send(&s1, ()).is_ok()),\n i if Some(i) == oper2 => assert!(oper.send(&s2, ()).is_ok()),\n i if i == oper3 => assert!(oper.send(&s3, ()).is_ok()),\n _ => unreachable!(),\n }\n }\n });\n\n let hits = vec![Cell::new(0usize); 3];\n for _ in 0..COUNT {\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n sel.recv(&r3);\n loop {\n match sel.ready() {\n 0 => {\n if r1.try_recv().is_ok() {\n hits[0].set(hits[0].get() + 1);\n break;\n }\n }\n 1 => {\n if r2.try_recv().is_ok() {\n hits[1].set(hits[1].get() + 1);\n break;\n }\n }\n 2 => {\n if r3.try_recv().is_ok() {\n hits[2].set(hits[2].get() + 1);\n break;\n }\n }\n _ => unreachable!(),\n }\n }\n }\n assert!(hits.iter().all(|x| x.get() > 0));\n })\n .unwrap();\n}\n" }, { "path": "crossbeam-channel/tests/same_channel.rs", "content": "#![allow(clippy::redundant_clone)]\n\nuse std::time::Duration;\n\nuse crossbeam_channel::{after, bounded, never, tick, unbounded};\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\nfn after_same_channel() {\n let r = after(ms(50));\n\n let r2 = r.clone();\n assert!(r.same_channel(&r2));\n\n let r3 = after(ms(50));\n assert!(!r.same_channel(&r3));\n assert!(!r2.same_channel(&r3));\n\n let r4 = after(ms(100));\n assert!(!r.same_channel(&r4));\n assert!(!r2.same_channel(&r4));\n}\n\n#[test]\nfn array_same_channel() {\n let (s, r) = bounded::(1);\n\n let s2 = s.clone();\n assert!(s.same_channel(&s2));\n\n let r2 = r.clone();\n assert!(r.same_channel(&r2));\n\n let (s3, r3) = bounded::(1);\n assert!(!s.same_channel(&s3));\n assert!(!s2.same_channel(&s3));\n assert!(!r.same_channel(&r3));\n assert!(!r2.same_channel(&r3));\n}\n\n#[test]\nfn list_same_channel() {\n let (s, r) = unbounded::();\n\n let s2 = s.clone();\n assert!(s.same_channel(&s2));\n\n let r2 = r.clone();\n assert!(r.same_channel(&r2));\n\n let (s3, r3) = unbounded::();\n assert!(!s.same_channel(&s3));\n assert!(!s2.same_channel(&s3));\n assert!(!r.same_channel(&r3));\n assert!(!r2.same_channel(&r3));\n}\n\n#[test]\nfn never_same_channel() {\n let r = never::();\n\n let r2 = r.clone();\n assert!(r.same_channel(&r2));\n\n // Never channel are always equal to one another.\n let r3 = never::();\n assert!(r.same_channel(&r3));\n assert!(r2.same_channel(&r3));\n}\n\n#[test]\nfn tick_same_channel() {\n let r = tick(ms(50));\n\n let r2 = r.clone();\n assert!(r.same_channel(&r2));\n\n let r3 = tick(ms(50));\n assert!(!r.same_channel(&r3));\n assert!(!r2.same_channel(&r3));\n\n let r4 = tick(ms(100));\n assert!(!r.same_channel(&r4));\n assert!(!r2.same_channel(&r4));\n}\n\n#[test]\nfn zero_same_channel() {\n let (s, r) = bounded::(0);\n\n let s2 = s.clone();\n assert!(s.same_channel(&s2));\n\n let r2 = r.clone();\n assert!(r.same_channel(&r2));\n\n let (s3, r3) = bounded::(0);\n assert!(!s.same_channel(&s3));\n assert!(!s2.same_channel(&s3));\n assert!(!r.same_channel(&r3));\n assert!(!r2.same_channel(&r3));\n}\n\n#[test]\nfn different_flavors_same_channel() {\n let (s1, r1) = bounded::(0);\n let (s2, r2) = unbounded::();\n\n assert!(!s1.same_channel(&s2));\n assert!(!r1.same_channel(&r2));\n}\n" }, { "path": "crossbeam-channel/tests/select.rs", "content": "//! Tests for channel selection using the `Select` struct.\n\nuse std::{\n any::Any,\n cell::Cell,\n thread,\n time::{Duration, Instant},\n};\n\nuse crossbeam_channel::{Receiver, Select, TryRecvError, after, bounded, tick, unbounded};\nuse crossbeam_utils::thread::scope;\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\nfn smoke1() {\n let (s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n s1.send(1).unwrap();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => assert_eq!(oper.recv(&r1), Ok(1)),\n i if i == oper2 => panic!(),\n _ => unreachable!(),\n }\n\n s2.send(2).unwrap();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => panic!(),\n i if i == oper2 => assert_eq!(oper.recv(&r2), Ok(2)),\n _ => unreachable!(),\n }\n}\n\n#[test]\nfn smoke2() {\n let (_s1, r1) = unbounded::();\n let (_s2, r2) = unbounded::();\n let (_s3, r3) = unbounded::();\n let (_s4, r4) = unbounded::();\n let (s5, r5) = unbounded::();\n\n s5.send(5).unwrap();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper3 = sel.recv(&r3);\n let oper4 = sel.recv(&r4);\n let oper5 = sel.recv(&r5);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => panic!(),\n i if i == oper2 => panic!(),\n i if i == oper3 => panic!(),\n i if i == oper4 => panic!(),\n i if i == oper5 => assert_eq!(oper.recv(&r5), Ok(5)),\n _ => unreachable!(),\n }\n}\n\n#[test]\nfn disconnected() {\n let (s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n scope(|scope| {\n scope.spawn(|_| {\n drop(s1);\n thread::sleep(ms(500));\n s2.send(5).unwrap();\n });\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert!(oper.recv(&r1).is_err()),\n i if i == oper2 => panic!(),\n _ => unreachable!(),\n },\n }\n\n r2.recv().unwrap();\n })\n .unwrap();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert!(oper.recv(&r1).is_err()),\n i if i == oper2 => panic!(),\n _ => unreachable!(),\n },\n }\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n drop(s2);\n });\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r2);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert!(oper.recv(&r2).is_err()),\n _ => unreachable!(),\n },\n }\n })\n .unwrap();\n}\n\n#[test]\nfn default() {\n let (s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n let mut sel = Select::new();\n let _oper1 = sel.recv(&r1);\n let _oper2 = sel.recv(&r2);\n let oper = sel.try_select();\n match oper {\n Err(_) => {}\n Ok(_) => panic!(),\n }\n\n drop(s1);\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.try_select();\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert!(oper.recv(&r1).is_err()),\n i if i == oper2 => panic!(),\n _ => unreachable!(),\n },\n }\n\n s2.send(2).unwrap();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r2);\n let oper = sel.try_select();\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert_eq!(oper.recv(&r2), Ok(2)),\n _ => unreachable!(),\n },\n }\n\n let mut sel = Select::new();\n let _oper1 = sel.recv(&r2);\n let oper = sel.try_select();\n match oper {\n Err(_) => {}\n Ok(_) => panic!(),\n }\n\n let mut sel = Select::new();\n let oper = sel.try_select();\n match oper {\n Err(_) => {}\n Ok(_) => panic!(),\n }\n}\n\n#[test]\nfn timeout() {\n let (_s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(1500));\n s2.send(2).unwrap();\n });\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => {}\n Ok(oper) => match oper.index() {\n i if i == oper1 => panic!(),\n i if i == oper2 => panic!(),\n _ => unreachable!(),\n },\n }\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => panic!(),\n i if i == oper2 => assert_eq!(oper.recv(&r2), Ok(2)),\n _ => unreachable!(),\n },\n }\n })\n .unwrap();\n\n scope(|scope| {\n let (s, r) = unbounded::();\n\n scope.spawn(move |_| {\n thread::sleep(ms(500));\n drop(s);\n });\n\n let mut sel = Select::new();\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => {\n let mut sel = Select::new();\n let oper1 = sel.recv(&r);\n let oper = sel.try_select();\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert!(oper.recv(&r).is_err()),\n _ => unreachable!(),\n },\n }\n }\n Ok(_) => unreachable!(),\n }\n })\n .unwrap();\n}\n\n#[test]\nfn default_when_disconnected() {\n let (_, r) = unbounded::();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r);\n let oper = sel.try_select();\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert!(oper.recv(&r).is_err()),\n _ => unreachable!(),\n },\n }\n\n let (_, r) = unbounded::();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert!(oper.recv(&r).is_err()),\n _ => unreachable!(),\n },\n }\n\n let (s, _) = bounded::(0);\n\n let mut sel = Select::new();\n let oper1 = sel.send(&s);\n let oper = sel.try_select();\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert!(oper.send(&s, 0).is_err()),\n _ => unreachable!(),\n },\n }\n\n let (s, _) = bounded::(0);\n\n let mut sel = Select::new();\n let oper1 = sel.send(&s);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert!(oper.send(&s, 0).is_err()),\n _ => unreachable!(),\n },\n }\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn default_only() {\n let start = Instant::now();\n\n let mut sel = Select::new();\n let oper = sel.try_select();\n assert!(oper.is_err());\n let now = Instant::now();\n assert!(now - start <= ms(50));\n\n let start = Instant::now();\n let mut sel = Select::new();\n let oper = sel.select_timeout(ms(500));\n assert!(oper.is_err());\n let now = Instant::now();\n assert!(now - start >= ms(450));\n assert!(now - start <= ms(550));\n}\n\n#[test]\nfn unblocks() {\n let (s1, r1) = bounded::(0);\n let (s2, r2) = bounded::(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n s2.send(2).unwrap();\n });\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => panic!(),\n i if i == oper2 => assert_eq!(oper.recv(&r2), Ok(2)),\n _ => unreachable!(),\n },\n }\n })\n .unwrap();\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n assert_eq!(r1.recv().unwrap(), 1);\n });\n\n let mut sel = Select::new();\n let oper1 = sel.send(&s1);\n let oper2 = sel.send(&s2);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n i if i == oper1 => oper.send(&s1, 1).unwrap(),\n i if i == oper2 => panic!(),\n _ => unreachable!(),\n },\n }\n })\n .unwrap();\n}\n\n#[test]\nfn both_ready() {\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n s1.send(1).unwrap();\n assert_eq!(r2.recv().unwrap(), 2);\n });\n\n for _ in 0..2 {\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.send(&s2);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => assert_eq!(oper.recv(&r1), Ok(1)),\n i if i == oper2 => oper.send(&s2, 2).unwrap(),\n _ => unreachable!(),\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn loop_try() {\n const RUNS: usize = 20;\n\n for _ in 0..RUNS {\n let (s1, r1) = bounded::(0);\n let (s2, r2) = bounded::(0);\n let (s_end, r_end) = bounded::<()>(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n loop {\n let mut done = false;\n\n let mut sel = Select::new();\n let oper1 = sel.send(&s1);\n let oper = sel.try_select();\n match oper {\n Err(_) => {}\n Ok(oper) => match oper.index() {\n i if i == oper1 => {\n let _ = oper.send(&s1, 1);\n done = true;\n }\n _ => unreachable!(),\n },\n }\n if done {\n break;\n }\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r_end);\n let oper = sel.try_select();\n match oper {\n Err(_) => {}\n Ok(oper) => match oper.index() {\n i if i == oper1 => {\n let _ = oper.recv(&r_end);\n done = true;\n }\n _ => unreachable!(),\n },\n }\n if done {\n break;\n }\n }\n });\n\n scope.spawn(|_| {\n loop {\n if let Ok(x) = r2.try_recv() {\n assert_eq!(x, 2);\n break;\n }\n\n let mut done = false;\n let mut sel = Select::new();\n let oper1 = sel.recv(&r_end);\n let oper = sel.try_select();\n match oper {\n Err(_) => {}\n Ok(oper) => match oper.index() {\n i if i == oper1 => {\n let _ = oper.recv(&r_end);\n done = true;\n }\n _ => unreachable!(),\n },\n }\n if done {\n break;\n }\n }\n });\n\n scope.spawn(|_| {\n thread::sleep(ms(500));\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.send(&s2);\n let oper = sel.select_timeout(ms(1000));\n match oper {\n Err(_) => {}\n Ok(oper) => match oper.index() {\n i if i == oper1 => assert_eq!(oper.recv(&r1), Ok(1)),\n i if i == oper2 => assert!(oper.send(&s2, 2).is_ok()),\n _ => unreachable!(),\n },\n }\n\n drop(s_end);\n });\n })\n .unwrap();\n }\n}\n\n#[test]\nfn cloning1() {\n scope(|scope| {\n let (s1, r1) = unbounded::();\n let (_s2, r2) = unbounded::();\n let (s3, r3) = unbounded::<()>();\n\n scope.spawn(move |_| {\n r3.recv().unwrap();\n drop(s1.clone());\n assert!(r3.try_recv().is_err());\n s1.send(1).unwrap();\n r3.recv().unwrap();\n });\n\n s3.send(()).unwrap();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => drop(oper.recv(&r1)),\n i if i == oper2 => drop(oper.recv(&r2)),\n _ => unreachable!(),\n }\n\n s3.send(()).unwrap();\n })\n .unwrap();\n}\n\n#[test]\nfn cloning2() {\n let (s1, r1) = unbounded::<()>();\n let (s2, r2) = unbounded::<()>();\n let (_s3, _r3) = unbounded::<()>();\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => panic!(),\n i if i == oper2 => drop(oper.recv(&r2)),\n _ => unreachable!(),\n }\n });\n\n thread::sleep(ms(500));\n drop(s1.clone());\n s2.send(()).unwrap();\n })\n .unwrap();\n}\n\n#[test]\nfn preflight1() {\n let (s, r) = unbounded();\n s.send(()).unwrap();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => drop(oper.recv(&r)),\n _ => unreachable!(),\n }\n}\n\n#[test]\nfn preflight2() {\n let (s, r) = unbounded();\n drop(s.clone());\n s.send(()).unwrap();\n drop(s);\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => assert_eq!(oper.recv(&r), Ok(())),\n _ => unreachable!(),\n }\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected));\n}\n\n#[test]\nfn preflight3() {\n let (s, r) = unbounded();\n drop(s.clone());\n s.send(()).unwrap();\n drop(s);\n r.recv().unwrap();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => assert!(oper.recv(&r).is_err()),\n _ => unreachable!(),\n }\n}\n\n#[test]\nfn duplicate_operations() {\n let (s, r) = unbounded::();\n let hit = vec![Cell::new(false); 4];\n\n while hit.iter().map(|h| h.get()).any(|hit| !hit) {\n let mut sel = Select::new();\n let oper0 = sel.recv(&r);\n let oper1 = sel.recv(&r);\n let oper2 = sel.send(&s);\n let oper3 = sel.send(&s);\n let oper = sel.select();\n match oper.index() {\n i if i == oper0 => {\n assert!(oper.recv(&r).is_ok());\n hit[0].set(true);\n }\n i if i == oper1 => {\n assert!(oper.recv(&r).is_ok());\n hit[1].set(true);\n }\n i if i == oper2 => {\n assert!(oper.send(&s, 0).is_ok());\n hit[2].set(true);\n }\n i if i == oper3 => {\n assert!(oper.send(&s, 0).is_ok());\n hit[3].set(true);\n }\n _ => unreachable!(),\n }\n }\n}\n\n#[test]\nfn nesting() {\n let (s, r) = unbounded::();\n\n let mut sel = Select::new();\n let oper1 = sel.send(&s);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => {\n assert!(oper.send(&s, 0).is_ok());\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => {\n assert_eq!(oper.recv(&r), Ok(0));\n\n let mut sel = Select::new();\n let oper1 = sel.send(&s);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => {\n assert!(oper.send(&s, 1).is_ok());\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => {\n assert_eq!(oper.recv(&r), Ok(1));\n }\n _ => unreachable!(),\n }\n }\n _ => unreachable!(),\n }\n }\n _ => unreachable!(),\n }\n }\n _ => unreachable!(),\n }\n}\n\n#[test]\nfn stress_recv() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 10_000 };\n\n let (s1, r1) = unbounded();\n let (s2, r2) = bounded(5);\n let (s3, r3) = bounded(100);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n s1.send(i).unwrap();\n r3.recv().unwrap();\n\n s2.send(i).unwrap();\n r3.recv().unwrap();\n }\n });\n\n for i in 0..COUNT {\n for _ in 0..2 {\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select();\n match oper.index() {\n ix if ix == oper1 => assert_eq!(oper.recv(&r1), Ok(i)),\n ix if ix == oper2 => assert_eq!(oper.recv(&r2), Ok(i)),\n _ => unreachable!(),\n }\n\n s3.send(()).unwrap();\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_send() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 10_000 };\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n let (s3, r3) = bounded(100);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n assert_eq!(r1.recv().unwrap(), i);\n assert_eq!(r2.recv().unwrap(), i);\n r3.recv().unwrap();\n }\n });\n\n for i in 0..COUNT {\n for _ in 0..2 {\n let mut sel = Select::new();\n let oper1 = sel.send(&s1);\n let oper2 = sel.send(&s2);\n let oper = sel.select();\n match oper.index() {\n ix if ix == oper1 => assert!(oper.send(&s1, i).is_ok()),\n ix if ix == oper2 => assert!(oper.send(&s2, i).is_ok()),\n _ => unreachable!(),\n }\n }\n s3.send(()).unwrap();\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_mixed() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n let (s3, r3) = bounded(100);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n s1.send(i).unwrap();\n assert_eq!(r2.recv().unwrap(), i);\n r3.recv().unwrap();\n }\n });\n\n for i in 0..COUNT {\n for _ in 0..2 {\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.send(&s2);\n let oper = sel.select();\n match oper.index() {\n ix if ix == oper1 => assert_eq!(oper.recv(&r1), Ok(i)),\n ix if ix == oper2 => assert!(oper.send(&s2, i).is_ok()),\n _ => unreachable!(),\n }\n }\n s3.send(()).unwrap();\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_timeout_two_threads() {\n const COUNT: usize = 20;\n\n let (s, r) = bounded(2);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(500));\n }\n\n loop {\n let mut sel = Select::new();\n let oper1 = sel.send(&s);\n let oper = sel.select_timeout(ms(100));\n match oper {\n Err(_) => {}\n Ok(oper) => match oper.index() {\n ix if ix == oper1 => {\n assert!(oper.send(&s, i).is_ok());\n break;\n }\n _ => unreachable!(),\n },\n }\n }\n }\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(500));\n }\n\n loop {\n let mut sel = Select::new();\n let oper1 = sel.recv(&r);\n let oper = sel.select_timeout(ms(100));\n match oper {\n Err(_) => {}\n Ok(oper) => match oper.index() {\n ix if ix == oper1 => {\n assert_eq!(oper.recv(&r), Ok(i));\n break;\n }\n _ => unreachable!(),\n },\n }\n }\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn send_recv_same_channel() {\n let (s, r) = bounded::(0);\n let mut sel = Select::new();\n let oper1 = sel.send(&s);\n let oper2 = sel.recv(&r);\n let oper = sel.select_timeout(ms(100));\n match oper {\n Err(_) => {}\n Ok(oper) => match oper.index() {\n ix if ix == oper1 => panic!(),\n ix if ix == oper2 => panic!(),\n _ => unreachable!(),\n },\n }\n\n let (s, r) = unbounded::();\n let mut sel = Select::new();\n let oper1 = sel.send(&s);\n let oper2 = sel.recv(&r);\n let oper = sel.select_timeout(ms(100));\n match oper {\n Err(_) => panic!(),\n Ok(oper) => match oper.index() {\n ix if ix == oper1 => assert!(oper.send(&s, 0).is_ok()),\n ix if ix == oper2 => panic!(),\n _ => unreachable!(),\n },\n }\n}\n\n#[test]\nfn matching() {\n const THREADS: usize = 44;\n\n let (s, r) = &bounded::(0);\n\n scope(|scope| {\n for i in 0..THREADS {\n scope.spawn(move |_| {\n let mut sel = Select::new();\n let oper1 = sel.recv(r);\n let oper2 = sel.send(s);\n let oper = sel.select();\n match oper.index() {\n ix if ix == oper1 => assert_ne!(oper.recv(r), Ok(i)),\n ix if ix == oper2 => assert!(oper.send(s, i).is_ok()),\n _ => unreachable!(),\n }\n });\n }\n })\n .unwrap();\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\nfn matching_with_leftover() {\n const THREADS: usize = 55;\n\n let (s, r) = &bounded::(0);\n\n scope(|scope| {\n for i in 0..THREADS {\n scope.spawn(move |_| {\n let mut sel = Select::new();\n let oper1 = sel.recv(r);\n let oper2 = sel.send(s);\n let oper = sel.select();\n match oper.index() {\n ix if ix == oper1 => assert_ne!(oper.recv(r), Ok(i)),\n ix if ix == oper2 => assert!(oper.send(s, i).is_ok()),\n _ => unreachable!(),\n }\n });\n }\n s.send(!0).unwrap();\n })\n .unwrap();\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\nfn channel_through_channel() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 1000 };\n\n type T = Box;\n\n for cap in 0..3 {\n let (s, r) = bounded::(cap);\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut s = s;\n\n for _ in 0..COUNT {\n let (new_s, new_r) = bounded(cap);\n let new_r: T = Box::new(Some(new_r));\n\n {\n let mut sel = Select::new();\n let oper1 = sel.send(&s);\n let oper = sel.select();\n match oper.index() {\n ix if ix == oper1 => assert!(oper.send(&s, new_r).is_ok()),\n _ => unreachable!(),\n }\n }\n\n s = new_s;\n }\n });\n\n scope.spawn(move |_| {\n let mut r = r;\n\n for _ in 0..COUNT {\n let new = {\n let mut sel = Select::new();\n let oper1 = sel.recv(&r);\n let oper = sel.select();\n match oper.index() {\n ix if ix == oper1 => oper\n .recv(&r)\n .unwrap()\n .downcast_mut::>>()\n .unwrap()\n .take()\n .unwrap(),\n _ => unreachable!(),\n }\n };\n r = new;\n }\n });\n })\n .unwrap();\n }\n}\n\n#[test]\nfn linearizable_try() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 100_000 };\n\n for step in 0..2 {\n let (start_s, start_r) = bounded::<()>(0);\n let (end_s, end_r) = bounded::<()>(0);\n\n let ((s1, r1), (s2, r2)) = if step == 0 {\n (bounded::(1), bounded::(1))\n } else {\n (unbounded::(), unbounded::())\n };\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n start_s.send(()).unwrap();\n\n s1.send(1).unwrap();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.try_select();\n match oper {\n Err(_) => unreachable!(),\n Ok(oper) => match oper.index() {\n ix if ix == oper1 => assert!(oper.recv(&r1).is_ok()),\n ix if ix == oper2 => assert!(oper.recv(&r2).is_ok()),\n _ => unreachable!(),\n },\n }\n\n end_s.send(()).unwrap();\n let _ = r2.try_recv();\n }\n });\n\n for _ in 0..COUNT {\n start_r.recv().unwrap();\n\n s2.send(1).unwrap();\n let _ = r1.try_recv();\n\n end_r.recv().unwrap();\n }\n })\n .unwrap();\n }\n}\n\n#[test]\nfn linearizable_timeout() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 100_000 };\n\n for step in 0..2 {\n let (start_s, start_r) = bounded::<()>(0);\n let (end_s, end_r) = bounded::<()>(0);\n\n let ((s1, r1), (s2, r2)) = if step == 0 {\n (bounded::(1), bounded::(1))\n } else {\n (unbounded::(), unbounded::())\n };\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n start_s.send(()).unwrap();\n\n s1.send(1).unwrap();\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper = sel.select_timeout(ms(0));\n match oper {\n Err(_) => unreachable!(),\n Ok(oper) => match oper.index() {\n ix if ix == oper1 => assert!(oper.recv(&r1).is_ok()),\n ix if ix == oper2 => assert!(oper.recv(&r2).is_ok()),\n _ => unreachable!(),\n },\n }\n\n end_s.send(()).unwrap();\n let _ = r2.try_recv();\n }\n });\n\n for _ in 0..COUNT {\n start_r.recv().unwrap();\n\n s2.send(1).unwrap();\n let _ = r1.try_recv();\n\n end_r.recv().unwrap();\n }\n })\n .unwrap();\n }\n}\n\n#[test]\nfn fairness1() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 10_000 };\n\n let (s1, r1) = bounded::<()>(COUNT);\n let (s2, r2) = unbounded::<()>();\n\n for _ in 0..COUNT {\n s1.send(()).unwrap();\n s2.send(()).unwrap();\n }\n\n let hits = vec![Cell::new(0usize); 4];\n for _ in 0..COUNT {\n let after = after(ms(0));\n let tick = tick(ms(0));\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper3 = sel.recv(&after);\n let oper4 = sel.recv(&tick);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => {\n oper.recv(&r1).unwrap();\n hits[0].set(hits[0].get() + 1);\n }\n i if i == oper2 => {\n oper.recv(&r2).unwrap();\n hits[1].set(hits[1].get() + 1);\n }\n i if i == oper3 => {\n oper.recv(&after).unwrap();\n hits[2].set(hits[2].get() + 1);\n }\n i if i == oper4 => {\n oper.recv(&tick).unwrap();\n hits[3].set(hits[3].get() + 1);\n }\n _ => unreachable!(),\n }\n }\n assert!(hits.iter().all(|x| x.get() >= COUNT / hits.len() / 2));\n}\n\n#[test]\nfn fairness2() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 10_000 };\n\n let (s1, r1) = unbounded::<()>();\n let (s2, r2) = bounded::<()>(1);\n let (s3, r3) = bounded::<()>(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n let mut sel = Select::new();\n let mut oper1 = None;\n let mut oper2 = None;\n if s1.is_empty() {\n oper1 = Some(sel.send(&s1));\n }\n if s2.is_empty() {\n oper2 = Some(sel.send(&s2));\n }\n let oper3 = sel.send(&s3);\n let oper = sel.select();\n match oper.index() {\n i if Some(i) == oper1 => assert!(oper.send(&s1, ()).is_ok()),\n i if Some(i) == oper2 => assert!(oper.send(&s2, ()).is_ok()),\n i if i == oper3 => assert!(oper.send(&s3, ()).is_ok()),\n _ => unreachable!(),\n }\n }\n });\n\n let hits = vec![Cell::new(0usize); 3];\n for _ in 0..COUNT {\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper3 = sel.recv(&r3);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => {\n oper.recv(&r1).unwrap();\n hits[0].set(hits[0].get() + 1);\n }\n i if i == oper2 => {\n oper.recv(&r2).unwrap();\n hits[1].set(hits[1].get() + 1);\n }\n i if i == oper3 => {\n oper.recv(&r3).unwrap();\n hits[2].set(hits[2].get() + 1);\n }\n _ => unreachable!(),\n }\n }\n assert!(hits.iter().all(|x| x.get() >= COUNT / hits.len() / 50));\n })\n .unwrap();\n}\n\n#[test]\nfn sync_and_clone() {\n const THREADS: usize = 20;\n\n let (s, r) = &bounded::(0);\n\n let mut sel = Select::new();\n let oper1 = sel.recv(r);\n let oper2 = sel.send(s);\n let sel = &sel;\n\n scope(|scope| {\n for i in 0..THREADS {\n scope.spawn(move |_| {\n let mut sel = sel.clone();\n let oper = sel.select();\n match oper.index() {\n ix if ix == oper1 => assert_ne!(oper.recv(r), Ok(i)),\n ix if ix == oper2 => assert!(oper.send(s, i).is_ok()),\n _ => unreachable!(),\n }\n });\n }\n })\n .unwrap();\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\nfn send_and_clone() {\n const THREADS: usize = 20;\n\n let (s, r) = &bounded::(0);\n\n let mut sel = Select::new();\n let oper1 = sel.recv(r);\n let oper2 = sel.send(s);\n\n scope(|scope| {\n for i in 0..THREADS {\n let mut sel = sel.clone();\n scope.spawn(move |_| {\n let oper = sel.select();\n match oper.index() {\n ix if ix == oper1 => assert_ne!(oper.recv(r), Ok(i)),\n ix if ix == oper2 => assert!(oper.send(s, i).is_ok()),\n _ => unreachable!(),\n }\n });\n }\n })\n .unwrap();\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\nfn reuse() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 10_000 };\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n let (s3, r3) = bounded(100);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n s1.send(i).unwrap();\n assert_eq!(r2.recv().unwrap(), i);\n r3.recv().unwrap();\n }\n });\n\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.send(&s2);\n\n for i in 0..COUNT {\n for _ in 0..2 {\n let oper = sel.select();\n match oper.index() {\n ix if ix == oper1 => assert_eq!(oper.recv(&r1), Ok(i)),\n ix if ix == oper2 => assert!(oper.send(&s2, i).is_ok()),\n _ => unreachable!(),\n }\n }\n s3.send(()).unwrap();\n }\n })\n .unwrap();\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/1096\n#[test]\nfn issue_1096() {\n // unbounded recv\n let (tx, rx) = unbounded::();\n tx.send(123).unwrap();\n let mut select = Select::new();\n select.recv(&rx);\n let operation = select.select();\n assert_eq!(operation.recv(&rx.clone()).unwrap(), 123);\n\n // unbounded send\n let (tx, rx) = unbounded::();\n let mut select = Select::new();\n select.send(&tx);\n let operation = select.select();\n operation.send(&tx.clone(), 124).unwrap();\n assert_eq!(rx.recv().unwrap(), 124);\n\n // bounded(2) recv\n let (tx, rx) = bounded::(2);\n tx.send(123).unwrap();\n let mut select = Select::new();\n select.recv(&rx);\n let operation = select.select();\n assert_eq!(operation.recv(&rx.clone()).unwrap(), 123);\n\n // bounded(2) send\n let (tx, rx) = bounded::(2);\n let mut select = Select::new();\n select.send(&tx);\n let operation = select.select();\n operation.send(&tx.clone(), 124).unwrap();\n assert_eq!(rx.recv().unwrap(), 124);\n\n // zero recv\n let (tx, rx) = bounded::(0);\n scope(|scope| {\n scope.spawn(|_| tx.send(123).unwrap());\n let mut select = Select::new();\n select.recv(&rx);\n let operation = select.select();\n assert_eq!(operation.recv(&rx.clone()).unwrap(), 123);\n })\n .unwrap();\n\n // zero send\n let (tx, rx) = bounded::(0);\n scope(|scope| {\n scope.spawn(|_| assert_eq!(rx.recv().unwrap(), 124));\n let mut select = Select::new();\n select.send(&tx);\n let operation = select.select();\n operation.send(&tx.clone(), 124).unwrap();\n })\n .unwrap();\n}\n" }, { "path": "crossbeam-channel/tests/select_macro.rs", "content": "//! Tests for the `select!` macro.\n\n#![forbid(unsafe_code)] // select! is safe.\n#![allow(clippy::match_single_binding)]\n\nuse std::{\n any::Any,\n cell::Cell,\n ops::Deref,\n thread,\n time::{Duration, Instant},\n};\n\nuse crossbeam_channel::{\n Receiver, RecvError, SendError, Sender, TryRecvError, after, bounded, never, select,\n select_biased, tick, unbounded,\n};\nuse crossbeam_utils::thread::scope;\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\nfn smoke1() {\n let (s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n s1.send(1).unwrap();\n\n select! {\n recv(r1) -> v => assert_eq!(v, Ok(1)),\n recv(r2) -> _ => panic!(),\n }\n\n s2.send(2).unwrap();\n\n select! {\n recv(r1) -> _ => panic!(),\n recv(r2) -> v => assert_eq!(v, Ok(2)),\n }\n}\n\n#[test]\nfn smoke2() {\n let (_s1, r1) = unbounded::();\n let (_s2, r2) = unbounded::();\n let (_s3, r3) = unbounded::();\n let (_s4, r4) = unbounded::();\n let (s5, r5) = unbounded::();\n\n s5.send(5).unwrap();\n\n select! {\n recv(r1) -> _ => panic!(),\n recv(r2) -> _ => panic!(),\n recv(r3) -> _ => panic!(),\n recv(r4) -> _ => panic!(),\n recv(r5) -> v => assert_eq!(v, Ok(5)),\n }\n}\n\n#[test]\nfn disconnected() {\n let (s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n scope(|scope| {\n scope.spawn(|_| {\n drop(s1);\n thread::sleep(ms(500));\n s2.send(5).unwrap();\n });\n\n select! {\n recv(r1) -> v => assert!(v.is_err()),\n recv(r2) -> _ => panic!(),\n default(ms(1000)) => panic!(),\n }\n\n r2.recv().unwrap();\n })\n .unwrap();\n\n select! {\n recv(r1) -> v => assert!(v.is_err()),\n recv(r2) -> _ => panic!(),\n default(ms(1000)) => panic!(),\n }\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n drop(s2);\n });\n\n select! {\n recv(r2) -> v => assert!(v.is_err()),\n default(ms(1000)) => panic!(),\n }\n })\n .unwrap();\n}\n\n#[test]\nfn default() {\n let (s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n select! {\n recv(r1) -> _ => panic!(),\n recv(r2) -> _ => panic!(),\n default => {}\n }\n\n drop(s1);\n\n select! {\n recv(r1) -> v => assert!(v.is_err()),\n recv(r2) -> _ => panic!(),\n default => panic!(),\n }\n\n s2.send(2).unwrap();\n\n select! {\n recv(r2) -> v => assert_eq!(v, Ok(2)),\n default => panic!(),\n }\n\n select! {\n recv(r2) -> _ => panic!(),\n default => {},\n }\n\n select! {\n default => {},\n }\n}\n\n#[test]\nfn timeout() {\n let (_s1, r1) = unbounded::();\n let (s2, r2) = unbounded::();\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(1500));\n s2.send(2).unwrap();\n });\n\n select! {\n recv(r1) -> _ => panic!(),\n recv(r2) -> _ => panic!(),\n default(ms(1000)) => {},\n }\n\n select! {\n recv(r1) -> _ => panic!(),\n recv(r2) -> v => assert_eq!(v, Ok(2)),\n default(ms(1000)) => panic!(),\n }\n })\n .unwrap();\n\n scope(|scope| {\n let (s, r) = unbounded::();\n\n scope.spawn(move |_| {\n thread::sleep(ms(500));\n drop(s);\n });\n\n select! {\n default(ms(1000)) => {\n select! {\n recv(r) -> v => assert!(v.is_err()),\n default => panic!(),\n }\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn default_when_disconnected() {\n let (_, r) = unbounded::();\n\n select! {\n recv(r) -> res => assert!(res.is_err()),\n default => panic!(),\n }\n\n let (_, r) = unbounded::();\n\n select! {\n recv(r) -> res => assert!(res.is_err()),\n default(ms(1000)) => panic!(),\n }\n\n let (s, _) = bounded::(0);\n\n select! {\n send(s, 0) -> res => assert!(res.is_err()),\n default => panic!(),\n }\n\n let (s, _) = bounded::(0);\n\n select! {\n send(s, 0) -> res => assert!(res.is_err()),\n default(ms(1000)) => panic!(),\n }\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn default_only() {\n let start = Instant::now();\n select! {\n default => {}\n }\n let now = Instant::now();\n assert!(now - start <= ms(50));\n\n let start = Instant::now();\n select! {\n default(ms(500)) => {}\n }\n let now = Instant::now();\n assert!(now - start >= ms(450));\n assert!(now - start <= ms(550));\n}\n\n#[test]\nfn unblocks() {\n let (s1, r1) = bounded::(0);\n let (s2, r2) = bounded::(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n s2.send(2).unwrap();\n });\n\n select! {\n recv(r1) -> _ => panic!(),\n recv(r2) -> v => assert_eq!(v, Ok(2)),\n default(ms(1000)) => panic!(),\n }\n })\n .unwrap();\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n assert_eq!(r1.recv().unwrap(), 1);\n });\n\n select! {\n send(s1, 1) -> _ => {},\n send(s2, 2) -> _ => panic!(),\n default(ms(1000)) => panic!(),\n }\n })\n .unwrap();\n}\n\n#[test]\nfn both_ready() {\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(500));\n s1.send(1).unwrap();\n assert_eq!(r2.recv().unwrap(), 2);\n });\n\n for _ in 0..2 {\n select! {\n recv(r1) -> v => assert_eq!(v, Ok(1)),\n send(s2, 2) -> _ => {},\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn loop_try() {\n const RUNS: usize = 20;\n\n for _ in 0..RUNS {\n let (s1, r1) = bounded::(0);\n let (s2, r2) = bounded::(0);\n let (s_end, r_end) = bounded::<()>(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n loop {\n select! {\n send(s1, 1) -> _ => break,\n default => {}\n }\n\n select! {\n recv(r_end) -> _ => break,\n default => {}\n }\n }\n });\n\n scope.spawn(|_| {\n loop {\n if let Ok(x) = r2.try_recv() {\n assert_eq!(x, 2);\n break;\n }\n\n select! {\n recv(r_end) -> _ => break,\n default => {}\n }\n }\n });\n\n scope.spawn(|_| {\n thread::sleep(ms(500));\n\n select! {\n recv(r1) -> v => assert_eq!(v, Ok(1)),\n send(s2, 2) -> _ => {},\n default(ms(500)) => panic!(),\n }\n\n drop(s_end);\n });\n })\n .unwrap();\n }\n}\n\n#[test]\nfn cloning1() {\n scope(|scope| {\n let (s1, r1) = unbounded::();\n let (_s2, r2) = unbounded::();\n let (s3, r3) = unbounded::<()>();\n\n scope.spawn(move |_| {\n r3.recv().unwrap();\n drop(s1.clone());\n assert_eq!(r3.try_recv(), Err(TryRecvError::Empty));\n s1.send(1).unwrap();\n r3.recv().unwrap();\n });\n\n s3.send(()).unwrap();\n\n select! {\n recv(r1) -> _ => {},\n recv(r2) -> _ => {},\n }\n\n s3.send(()).unwrap();\n })\n .unwrap();\n}\n\n#[test]\nfn cloning2() {\n let (s1, r1) = unbounded::<()>();\n let (s2, r2) = unbounded::<()>();\n let (_s3, _r3) = unbounded::<()>();\n\n scope(|scope| {\n scope.spawn(move |_| {\n select! {\n recv(r1) -> _ => panic!(),\n recv(r2) -> _ => {},\n }\n });\n\n thread::sleep(ms(500));\n drop(s1.clone());\n s2.send(()).unwrap();\n })\n .unwrap();\n}\n\n#[test]\nfn preflight1() {\n let (s, r) = unbounded();\n s.send(()).unwrap();\n\n select! {\n recv(r) -> _ => {}\n }\n}\n\n#[test]\nfn preflight2() {\n let (s, r) = unbounded();\n drop(s.clone());\n s.send(()).unwrap();\n drop(s);\n\n select! {\n recv(r) -> v => assert!(v.is_ok()),\n }\n assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected));\n}\n\n#[test]\nfn preflight3() {\n let (s, r) = unbounded();\n drop(s.clone());\n s.send(()).unwrap();\n drop(s);\n r.recv().unwrap();\n\n select! {\n recv(r) -> v => assert!(v.is_err())\n }\n}\n\n#[test]\nfn duplicate_operations() {\n let (s, r) = unbounded::();\n let mut hit = [false; 4];\n\n while hit.iter().any(|hit| !hit) {\n select! {\n recv(r) -> _ => hit[0] = true,\n recv(r) -> _ => hit[1] = true,\n send(s, 0) -> _ => hit[2] = true,\n send(s, 0) -> _ => hit[3] = true,\n }\n }\n}\n\n#[test]\nfn nesting() {\n let (s, r) = unbounded::();\n\n select! {\n send(s, 0) -> _ => {\n select! {\n recv(r) -> v => {\n assert_eq!(v, Ok(0));\n select! {\n send(s, 1) -> _ => {\n select! {\n recv(r) -> v => {\n assert_eq!(v, Ok(1));\n }\n }\n }\n }\n }\n }\n }\n }\n}\n\n#[test]\n#[should_panic(expected = \"send panicked\")]\nfn panic_sender() {\n fn get() -> Sender {\n panic!(\"send panicked\")\n }\n\n #[allow(unreachable_code, clippy::diverging_sub_expression)]\n {\n select! {\n send(get(), panic!()) -> _ => {}\n }\n }\n}\n\n#[test]\n#[should_panic(expected = \"recv panicked\")]\nfn panic_receiver() {\n fn get() -> Receiver {\n panic!(\"recv panicked\")\n }\n\n select! {\n recv(get()) -> _ => {}\n }\n}\n\n#[test]\nfn stress_recv() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 10_000 };\n\n let (s1, r1) = unbounded();\n let (s2, r2) = bounded(5);\n let (s3, r3) = bounded(100);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n s1.send(i).unwrap();\n r3.recv().unwrap();\n\n s2.send(i).unwrap();\n r3.recv().unwrap();\n }\n });\n\n for i in 0..COUNT {\n for _ in 0..2 {\n select! {\n recv(r1) -> v => assert_eq!(v, Ok(i)),\n recv(r2) -> v => assert_eq!(v, Ok(i)),\n }\n\n s3.send(()).unwrap();\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_send() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n let (s3, r3) = bounded(100);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n assert_eq!(r1.recv().unwrap(), i);\n assert_eq!(r2.recv().unwrap(), i);\n r3.recv().unwrap();\n }\n });\n\n for i in 0..COUNT {\n for _ in 0..2 {\n select! {\n send(s1, i) -> _ => {},\n send(s2, i) -> _ => {},\n }\n }\n s3.send(()).unwrap();\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_mixed() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = bounded(0);\n let (s2, r2) = bounded(0);\n let (s3, r3) = bounded(100);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n s1.send(i).unwrap();\n assert_eq!(r2.recv().unwrap(), i);\n r3.recv().unwrap();\n }\n });\n\n for i in 0..COUNT {\n for _ in 0..2 {\n select! {\n recv(r1) -> v => assert_eq!(v, Ok(i)),\n send(s2, i) -> _ => {},\n }\n }\n s3.send(()).unwrap();\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_timeout_two_threads() {\n const COUNT: usize = 20;\n\n let (s, r) = bounded(2);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(500));\n }\n\n loop {\n select! {\n send(s, i) -> _ => break,\n default(ms(100)) => {}\n }\n }\n }\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(500));\n }\n\n loop {\n select! {\n recv(r) -> v => {\n assert_eq!(v, Ok(i));\n break;\n }\n default(ms(100)) => {}\n }\n }\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn send_recv_same_channel() {\n let (s, r) = bounded::(0);\n select! {\n send(s, 0) -> _ => panic!(),\n recv(r) -> _ => panic!(),\n default(ms(500)) => {}\n }\n\n let (s, r) = unbounded::();\n select! {\n send(s, 0) -> _ => {},\n recv(r) -> _ => panic!(),\n default(ms(500)) => panic!(),\n }\n}\n\n#[test]\nfn matching() {\n const THREADS: usize = 44;\n\n let (s, r) = &bounded::(0);\n\n scope(|scope| {\n for i in 0..THREADS {\n scope.spawn(move |_| {\n select! {\n recv(r) -> v => assert_ne!(v.unwrap(), i),\n send(s, i) -> _ => {},\n }\n });\n }\n })\n .unwrap();\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\nfn matching_with_leftover() {\n const THREADS: usize = 55;\n\n let (s, r) = &bounded::(0);\n\n scope(|scope| {\n for i in 0..THREADS {\n scope.spawn(move |_| {\n select! {\n recv(r) -> v => assert_ne!(v.unwrap(), i),\n send(s, i) -> _ => {},\n }\n });\n }\n s.send(!0).unwrap();\n })\n .unwrap();\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\nfn channel_through_channel() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 1000 };\n\n type T = Box;\n\n for cap in 0..3 {\n let (s, r) = bounded::(cap);\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut s = s;\n\n for _ in 0..COUNT {\n let (new_s, new_r) = bounded(cap);\n let new_r: T = Box::new(Some(new_r));\n\n select! {\n send(s, new_r) -> _ => {}\n }\n\n s = new_s;\n }\n });\n\n scope.spawn(move |_| {\n let mut r = r;\n\n for _ in 0..COUNT {\n r = select! {\n recv(r) -> msg => {\n msg.unwrap()\n .downcast_mut::>>()\n .unwrap()\n .take()\n .unwrap()\n }\n }\n }\n });\n })\n .unwrap();\n }\n}\n\n#[test]\nfn linearizable_default() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 100_000 };\n\n for step in 0..2 {\n let (start_s, start_r) = bounded::<()>(0);\n let (end_s, end_r) = bounded::<()>(0);\n\n let ((s1, r1), (s2, r2)) = if step == 0 {\n (bounded::(1), bounded::(1))\n } else {\n (unbounded::(), unbounded::())\n };\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n start_s.send(()).unwrap();\n\n s1.send(1).unwrap();\n select! {\n recv(r1) -> _ => {}\n recv(r2) -> _ => {}\n default => unreachable!()\n }\n\n end_s.send(()).unwrap();\n let _ = r2.try_recv();\n }\n });\n\n for _ in 0..COUNT {\n start_r.recv().unwrap();\n\n s2.send(1).unwrap();\n let _ = r1.try_recv();\n\n end_r.recv().unwrap();\n }\n })\n .unwrap();\n }\n}\n\n#[test]\nfn linearizable_timeout() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 100_000 };\n\n for step in 0..2 {\n let (start_s, start_r) = bounded::<()>(0);\n let (end_s, end_r) = bounded::<()>(0);\n\n let ((s1, r1), (s2, r2)) = if step == 0 {\n (bounded::(1), bounded::(1))\n } else {\n (unbounded::(), unbounded::())\n };\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n start_s.send(()).unwrap();\n\n s1.send(1).unwrap();\n select! {\n recv(r1) -> _ => {}\n recv(r2) -> _ => {}\n default(ms(0)) => unreachable!()\n }\n\n end_s.send(()).unwrap();\n let _ = r2.try_recv();\n }\n });\n\n for _ in 0..COUNT {\n start_r.recv().unwrap();\n\n s2.send(1).unwrap();\n let _ = r1.try_recv();\n\n end_r.recv().unwrap();\n }\n })\n .unwrap();\n }\n}\n\n#[test]\nfn fairness1() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = bounded::<()>(COUNT);\n let (s2, r2) = unbounded::<()>();\n\n for _ in 0..COUNT {\n s1.send(()).unwrap();\n s2.send(()).unwrap();\n }\n\n let mut hits = [0usize; 4];\n for _ in 0..COUNT {\n select! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n recv(after(ms(0))) -> _ => hits[2] += 1,\n recv(tick(ms(0))) -> _ => hits[3] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n}\n\n#[test]\n#[cfg_attr(crossbeam_sanitize, ignore)] // TODO: flaky: https://github.com/crossbeam-rs/crossbeam/issues/1094\nfn fairness2() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = unbounded::<()>();\n let (s2, r2) = bounded::<()>(1);\n let (s3, r3) = bounded::<()>(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n let (hole, _r) = bounded(0);\n\n for _ in 0..COUNT {\n let s1 = if s1.is_empty() { &s1 } else { &hole };\n let s2 = if s2.is_empty() { &s2 } else { &hole };\n\n select! {\n send(s1, ()) -> res => assert!(res.is_ok()),\n send(s2, ()) -> res => assert!(res.is_ok()),\n send(s3, ()) -> res => assert!(res.is_ok()),\n }\n }\n });\n\n let hits = vec![Cell::new(0usize); 3];\n for _ in 0..COUNT {\n select! {\n recv(r1) -> _ => hits[0].set(hits[0].get() + 1),\n recv(r2) -> _ => hits[1].set(hits[1].get() + 1),\n recv(r3) -> _ => hits[2].set(hits[2].get() + 1),\n }\n }\n assert!(hits.iter().all(|x| x.get() >= COUNT / hits.len() / 50));\n })\n .unwrap();\n}\n\n#[test]\nfn fairness_recv() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = bounded::<()>(COUNT);\n let (s2, r2) = unbounded::<()>();\n\n for _ in 0..COUNT {\n s1.send(()).unwrap();\n s2.send(()).unwrap();\n }\n\n let mut hits = [0usize; 2];\n while hits[0] + hits[1] < COUNT {\n select! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / 4));\n}\n\n#[test]\nfn fairness_send() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, _r1) = bounded::<()>(COUNT);\n let (s2, _r2) = unbounded::<()>();\n\n let mut hits = [0usize; 2];\n for _ in 0..COUNT {\n select! {\n send(s1, ()) -> _ => hits[0] += 1,\n send(s2, ()) -> _ => hits[1] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / 4));\n}\n\n#[test]\nfn unfairness() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = unbounded::<()>();\n let (s2, r2) = unbounded::<()>();\n let (s3, r3) = unbounded::<()>();\n\n for _ in 0..COUNT {\n s1.send(()).unwrap();\n s2.send(()).unwrap();\n }\n s3.send(()).unwrap();\n\n let mut hits = [0usize; 3];\n for _ in 0..COUNT {\n select_biased! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n recv(r3) -> _ => hits[2] += 1,\n }\n }\n assert_eq!(hits, [COUNT, 0, 0]);\n\n for _ in 0..COUNT {\n select_biased! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n recv(r3) -> _ => hits[2] += 1,\n }\n }\n assert_eq!(hits, [COUNT, COUNT, 0]);\n}\n\n#[test]\nfn unfairness_timeout() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = unbounded::<()>();\n let (s2, r2) = unbounded::<()>();\n let (s3, r3) = unbounded::<()>();\n\n for _ in 0..COUNT {\n s1.send(()).unwrap();\n s2.send(()).unwrap();\n }\n s3.send(()).unwrap();\n\n let mut hits = [0usize; 3];\n for _ in 0..COUNT {\n select_biased! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n recv(r3) -> _ => hits[2] += 1,\n default(ms(1000)) => unreachable!(),\n }\n }\n assert_eq!(hits, [COUNT, 0, 0]);\n\n for _ in 0..COUNT {\n select_biased! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n recv(r3) -> _ => hits[2] += 1,\n default(ms(1000)) => unreachable!(),\n }\n }\n assert_eq!(hits, [COUNT, COUNT, 0]);\n}\n\n#[test]\nfn unfairness_try() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s1, r1) = unbounded::<()>();\n let (s2, r2) = unbounded::<()>();\n let (s3, r3) = unbounded::<()>();\n\n for _ in 0..COUNT {\n s1.send(()).unwrap();\n s2.send(()).unwrap();\n }\n s3.send(()).unwrap();\n\n let mut hits = [0usize; 3];\n for _ in 0..COUNT {\n select_biased! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n recv(r3) -> _ => hits[2] += 1,\n default() => unreachable!(),\n }\n }\n assert_eq!(hits, [COUNT, 0, 0]);\n\n for _ in 0..COUNT {\n select_biased! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n recv(r3) -> _ => hits[2] += 1,\n default() => unreachable!(),\n }\n }\n assert_eq!(hits, [COUNT, COUNT, 0]);\n}\n\n#[allow(clippy::or_fun_call, clippy::unnecessary_literal_unwrap)] // This is intentional.\n#[test]\nfn references() {\n let (s, r) = unbounded::();\n select! {\n send(s, 0) -> _ => {}\n recv(r) -> _ => {}\n }\n select! {\n send(&&&&s, 0) -> _ => {}\n recv(&&&&r) -> _ => {}\n }\n select! {\n recv(Some(&r).unwrap_or(&never())) -> _ => {},\n default => {}\n }\n select! {\n recv(Some(r).unwrap_or(never())) -> _ => {},\n default => {}\n }\n}\n\n#[allow(clippy::never_loop)] // This is intentional.\n#[test]\nfn case_blocks() {\n let (s, r) = unbounded::();\n\n select! {\n recv(r) -> _ => 3.0,\n recv(r) -> _ => loop {\n unreachable!()\n },\n recv(r) -> _ => match 7 + 3 {\n _ => unreachable!()\n },\n default => 7.\n };\n\n select! {\n recv(r) -> msg => if msg.is_ok() {\n unreachable!()\n },\n default => ()\n }\n\n drop(s);\n}\n\n#[allow(clippy::redundant_closure_call)] // This is intentional.\n#[test]\nfn move_handles() {\n let (s, r) = unbounded::();\n select! {\n recv((move || r)()) -> _ => {}\n send((move || s)(), 0) -> _ => {}\n }\n}\n\n#[test]\nfn infer_types() {\n let (s, r) = unbounded();\n select! {\n recv(r) -> _ => {}\n default => {}\n }\n s.send(()).unwrap();\n\n let (s, r) = unbounded();\n select! {\n send(s, ()) -> _ => {}\n }\n r.recv().unwrap();\n}\n\n#[test]\nfn default_syntax() {\n let (s, r) = bounded::(0);\n\n select! {\n recv(r) -> _ => panic!(),\n default => {}\n }\n select! {\n send(s, 0) -> _ => panic!(),\n default() => {}\n }\n select! {\n default => {}\n }\n select! {\n default() => {}\n }\n}\n\n#[test]\nfn same_variable_name() {\n let (_, r) = unbounded::();\n select! {\n recv(r) -> r => assert!(r.is_err()),\n }\n}\n\n#[test]\nfn handles_on_heap() {\n let (s, r) = unbounded::();\n let (s, r) = (Box::new(s), Box::new(r));\n\n select! {\n send(*s, 0) -> _ => {}\n recv(*r) -> _ => {}\n default => {}\n }\n\n drop(s);\n drop(r);\n}\n\n#[test]\nfn once_blocks() {\n let (s, r) = unbounded::();\n\n let once = Box::new(());\n select! {\n send(s, 0) -> _ => drop(once),\n }\n\n let once = Box::new(());\n select! {\n recv(r) -> _ => drop(once),\n }\n\n let once1 = Box::new(());\n let once2 = Box::new(());\n select! {\n send(s, 0) -> _ => drop(once1),\n default => drop(once2),\n }\n\n let once1 = Box::new(());\n let once2 = Box::new(());\n select! {\n recv(r) -> _ => drop(once1),\n default => drop(once2),\n }\n\n let once1 = Box::new(());\n let once2 = Box::new(());\n select! {\n recv(r) -> _ => drop(once1),\n send(s, 0) -> _ => drop(once2),\n }\n}\n\n#[test]\nfn once_receiver() {\n let (_, r) = unbounded::();\n\n let once = Box::new(());\n let get = move || {\n drop(once);\n r\n };\n\n select! {\n recv(get()) -> _ => {}\n }\n}\n\n#[test]\nfn once_sender() {\n let (s, _) = unbounded::();\n\n let once = Box::new(());\n let get = move || {\n drop(once);\n s\n };\n\n select! {\n send(get(), 5) -> _ => {}\n }\n}\n\n#[test]\nfn parse_nesting() {\n let (_, r) = unbounded::();\n\n select! {\n recv(r) -> _ => {}\n recv(r) -> _ => {\n select! {\n recv(r) -> _ => {}\n recv(r) -> _ => {\n select! {\n recv(r) -> _ => {}\n recv(r) -> _ => {\n select! {\n default => {}\n }\n }\n }\n }\n }\n }\n }\n}\n\n#[test]\nfn evaluate() {\n let (s, r) = unbounded::();\n\n let v = select! {\n recv(r) -> _ => \"foo\".into(),\n send(s, 0) -> _ => \"bar\".to_owned(),\n default => \"baz\".to_string(),\n };\n assert_eq!(v, \"bar\");\n\n let v = select! {\n recv(r) -> _ => \"foo\".into(),\n default => \"baz\".to_string(),\n };\n assert_eq!(v, \"foo\");\n\n let v = select! {\n recv(r) -> _ => \"foo\".into(),\n default => \"baz\".to_string(),\n };\n assert_eq!(v, \"baz\");\n}\n\n#[test]\nfn deref() {\n use crossbeam_channel as cc;\n\n struct Sender(cc::Sender);\n struct Receiver(cc::Receiver);\n\n impl Deref for Receiver {\n type Target = cc::Receiver;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n }\n\n impl Deref for Sender {\n type Target = cc::Sender;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n }\n\n let (s, r) = bounded::(0);\n let (s, r) = (Sender(s), Receiver(r));\n\n select! {\n send(s, 0) -> _ => panic!(),\n recv(r) -> _ => panic!(),\n default => {}\n }\n}\n\n#[test]\nfn result_types() {\n let (s, _) = bounded::(0);\n let (_, r) = bounded::(0);\n\n select! {\n recv(r) -> res => { let _: Result = res; },\n }\n select! {\n recv(r) -> res => { let _: Result = res; },\n default => {}\n }\n select! {\n recv(r) -> res => { let _: Result = res; },\n default(ms(0)) => {}\n }\n\n select! {\n send(s, 0) -> res => { let _: Result<(), SendError> = res; },\n }\n select! {\n send(s, 0) -> res => { let _: Result<(), SendError> = res; },\n default => {}\n }\n select! {\n send(s, 0) -> res => { let _: Result<(), SendError> = res; },\n default(ms(0)) => {}\n }\n\n select! {\n send(s, 0) -> res => { let _: Result<(), SendError> = res; },\n recv(r) -> res => { let _: Result = res; },\n }\n}\n\n#[test]\nfn try_recv() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n select! {\n recv(r) -> _ => panic!(),\n default => {}\n }\n thread::sleep(ms(1500));\n select! {\n recv(r) -> v => assert_eq!(v, Ok(7)),\n default => panic!(),\n }\n thread::sleep(ms(500));\n select! {\n recv(r) -> v => assert_eq!(v, Err(RecvError)),\n default => panic!(),\n }\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n select! {\n send(s, 7) -> res => res.unwrap(),\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn recv() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n select! {\n recv(r) -> v => assert_eq!(v, Ok(7)),\n }\n thread::sleep(ms(1000));\n select! {\n recv(r) -> v => assert_eq!(v, Ok(8)),\n }\n thread::sleep(ms(1000));\n select! {\n recv(r) -> v => assert_eq!(v, Ok(9)),\n }\n select! {\n recv(r) -> v => assert_eq!(v, Err(RecvError)),\n }\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n select! {\n send(s, 7) -> res => res.unwrap(),\n }\n select! {\n send(s, 8) -> res => res.unwrap(),\n }\n select! {\n send(s, 9) -> res => res.unwrap(),\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn recv_timeout() {\n let (s, r) = bounded::(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n select! {\n recv(r) -> _ => panic!(),\n default(ms(1000)) => {}\n }\n select! {\n recv(r) -> v => assert_eq!(v, Ok(7)),\n default(ms(1000)) => panic!(),\n }\n select! {\n recv(r) -> v => assert_eq!(v, Err(RecvError)),\n default(ms(1000)) => panic!(),\n }\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n select! {\n send(s, 7) -> res => res.unwrap(),\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn try_send() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n select! {\n send(s, 7) -> _ => panic!(),\n default => {}\n }\n thread::sleep(ms(1500));\n select! {\n send(s, 8) -> res => res.unwrap(),\n default => panic!(),\n }\n thread::sleep(ms(500));\n select! {\n send(s, 8) -> res => assert_eq!(res, Err(SendError(8))),\n default => panic!(),\n }\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n select! {\n recv(r) -> v => assert_eq!(v, Ok(8)),\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn send() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n select! {\n send(s, 7) -> res => res.unwrap(),\n }\n thread::sleep(ms(1000));\n select! {\n send(s, 8) -> res => res.unwrap(),\n }\n thread::sleep(ms(1000));\n select! {\n send(s, 9) -> res => res.unwrap(),\n }\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n select! {\n recv(r) -> v => assert_eq!(v, Ok(7)),\n }\n select! {\n recv(r) -> v => assert_eq!(v, Ok(8)),\n }\n select! {\n recv(r) -> v => assert_eq!(v, Ok(9)),\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn send_timeout() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n select! {\n send(s, 7) -> _ => panic!(),\n default(ms(1000)) => {}\n }\n select! {\n send(s, 8) -> res => res.unwrap(),\n default(ms(1000)) => panic!(),\n }\n select! {\n send(s, 9) -> res => assert_eq!(res, Err(SendError(9))),\n default(ms(1000)) => panic!(),\n }\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n select! {\n recv(r) -> v => assert_eq!(v, Ok(8)),\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn disconnect_wakes_sender() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n select! {\n send(s, ()) -> res => assert_eq!(res, Err(SendError(()))),\n }\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n drop(r);\n });\n })\n .unwrap();\n}\n\n#[test]\nfn disconnect_wakes_receiver() {\n let (s, r) = bounded::<()>(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n select! {\n recv(r) -> res => assert_eq!(res, Err(RecvError)),\n }\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n drop(s);\n });\n })\n .unwrap();\n}\n\n#[test]\nfn trailing_comma() {\n let (s, r) = unbounded::();\n\n select! {\n send(s, 1,) -> _ => {},\n recv(r,) -> _ => {},\n default(ms(1000),) => {},\n }\n}\n" }, { "path": "crossbeam-channel/tests/thread_locals.rs", "content": "//! Tests that make sure accessing thread-locals while exiting the thread doesn't cause panics.\n\n#![cfg(not(miri))] // Miri detects that this test is buggy: the destructor of `FOO` uses `std::thread::current()`!\n\nuse std::{thread, time::Duration};\n\nuse crossbeam_channel::{select, unbounded};\nuse crossbeam_utils::thread::scope;\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\n#[cfg_attr(target_os = \"macos\", ignore = \"TLS is destroyed too early on macOS\")]\nfn use_while_exiting() {\n struct Foo;\n\n impl Drop for Foo {\n fn drop(&mut self) {\n // A blocking operation after the thread-locals have been dropped. This will attempt to\n // use the thread-locals and must not panic.\n let (_s, r) = unbounded::<()>();\n select! {\n recv(r) -> _ => {}\n default(ms(100)) => {}\n }\n }\n }\n\n thread_local! {\n static FOO: Foo = const { Foo };\n }\n\n let (s, r) = unbounded::<()>();\n\n scope(|scope| {\n scope.spawn(|_| {\n // First initialize `FOO`, then the thread-locals related to crossbeam-channel.\n FOO.with(|_| ());\n r.recv().unwrap();\n // At thread exit, thread-locals related to crossbeam-channel get dropped first and\n // `FOO` is dropped last.\n });\n\n scope.spawn(|_| {\n thread::sleep(ms(100));\n s.send(()).unwrap();\n });\n })\n .unwrap();\n}\n" }, { "path": "crossbeam-channel/tests/tick.rs", "content": "//! Tests for the tick channel flavor.\n\n#![cfg(not(miri))] // TODO: many assertions failed due to Miri is slow\n\nuse std::{\n sync::atomic::{AtomicUsize, Ordering},\n thread,\n time::{Duration, Instant},\n};\n\nuse crossbeam_channel::{Select, TryRecvError, after, select, tick};\nuse crossbeam_utils::thread::scope;\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\nfn fire() {\n let start = Instant::now();\n let r = tick(ms(50));\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n thread::sleep(ms(100));\n\n let fired = r.try_recv().unwrap();\n assert!(start < fired);\n assert!(fired - start >= ms(50));\n\n let now = Instant::now();\n assert!(fired < now);\n assert!(now - fired >= ms(50));\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n\n select! {\n recv(r) -> _ => panic!(),\n default => {}\n }\n\n select! {\n recv(r) -> _ => {}\n recv(tick(ms(200))) -> _ => panic!(),\n }\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn intervals() {\n let start = Instant::now();\n let r = tick(ms(50));\n\n let t1 = r.recv().unwrap();\n assert!(start + ms(50) <= t1);\n assert!(start + ms(100) > t1);\n\n thread::sleep(ms(300));\n let t2 = r.try_recv().unwrap();\n assert!(start + ms(100) <= t2);\n assert!(start + ms(150) > t2);\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n let t3 = r.recv().unwrap();\n assert!(start + ms(400) <= t3);\n assert!(start + ms(450) > t3);\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\nfn capacity() {\n const COUNT: usize = 10;\n\n for i in 0..COUNT {\n let r = tick(ms(i as u64));\n assert_eq!(r.capacity(), Some(1));\n }\n}\n\n#[test]\nfn len_empty_full() {\n let r = tick(ms(50));\n\n assert_eq!(r.len(), 0);\n assert!(r.is_empty());\n assert!(!r.is_full());\n\n thread::sleep(ms(100));\n\n assert_eq!(r.len(), 1);\n assert!(!r.is_empty());\n assert!(r.is_full());\n\n r.try_recv().unwrap();\n\n assert_eq!(r.len(), 0);\n assert!(r.is_empty());\n assert!(!r.is_full());\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn try_recv() {\n let r = tick(ms(200));\n assert!(r.try_recv().is_err());\n\n thread::sleep(ms(100));\n assert!(r.try_recv().is_err());\n\n thread::sleep(ms(200));\n assert!(r.try_recv().is_ok());\n assert!(r.try_recv().is_err());\n\n thread::sleep(ms(200));\n assert!(r.try_recv().is_ok());\n assert!(r.try_recv().is_err());\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn recv() {\n let start = Instant::now();\n let r = tick(ms(50));\n\n let fired = r.recv().unwrap();\n assert!(start < fired);\n assert!(fired - start >= ms(50));\n\n let now = Instant::now();\n assert!(fired < now);\n assert!(now - fired < fired - start);\n\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\n#[cfg_attr(crossbeam_sanitize, ignore)] // TODO: assertions failed due to tsan is slow\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn recv_timeout() {\n let start = Instant::now();\n let r = tick(ms(200));\n\n assert!(r.recv_timeout(ms(100)).is_err());\n let now = Instant::now();\n assert!(now - start >= ms(100));\n assert!(now - start <= ms(150));\n\n let fired = r.recv_timeout(ms(200)).unwrap();\n assert!(fired - start >= ms(200));\n assert!(fired - start <= ms(250));\n\n assert!(r.recv_timeout(ms(100)).is_err());\n let now = Instant::now();\n assert!(now - start >= ms(300));\n assert!(now - start <= ms(350));\n\n let fired = r.recv_timeout(ms(200)).unwrap();\n assert!(fired - start >= ms(400));\n assert!(fired - start <= ms(450));\n}\n\n#[test]\nfn recv_two() {\n let r1 = tick(ms(50));\n let r2 = tick(ms(50));\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..10 {\n select! {\n recv(r1) -> _ => {}\n recv(r2) -> _ => {}\n }\n }\n });\n scope.spawn(|_| {\n for _ in 0..10 {\n select! {\n recv(r1) -> _ => {}\n recv(r2) -> _ => {}\n }\n }\n });\n })\n .unwrap();\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn recv_race() {\n select! {\n recv(tick(ms(50))) -> _ => {}\n recv(tick(ms(100))) -> _ => panic!(),\n }\n\n select! {\n recv(tick(ms(100))) -> _ => panic!(),\n recv(tick(ms(50))) -> _ => {}\n }\n}\n\n#[test]\nfn stress_default() {\n const COUNT: usize = 10;\n\n for _ in 0..COUNT {\n select! {\n recv(tick(ms(0))) -> _ => {}\n default => panic!(),\n }\n }\n\n for _ in 0..COUNT {\n select! {\n recv(tick(ms(100))) -> _ => panic!(),\n default => {}\n }\n }\n}\n\n#[test]\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn select() {\n const THREADS: usize = 4;\n\n let hits = AtomicUsize::new(0);\n let r1 = tick(ms(200));\n let r2 = tick(ms(300));\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n let timeout = after(ms(1100));\n loop {\n let mut sel = Select::new();\n let oper1 = sel.recv(&r1);\n let oper2 = sel.recv(&r2);\n let oper3 = sel.recv(&timeout);\n let oper = sel.select();\n match oper.index() {\n i if i == oper1 => {\n oper.recv(&r1).unwrap();\n hits.fetch_add(1, Ordering::SeqCst);\n }\n i if i == oper2 => {\n oper.recv(&r2).unwrap();\n hits.fetch_add(1, Ordering::SeqCst);\n }\n i if i == oper3 => {\n oper.recv(&timeout).unwrap();\n break;\n }\n _ => unreachable!(),\n }\n }\n });\n }\n })\n .unwrap();\n\n assert_eq!(hits.load(Ordering::SeqCst), 8);\n}\n\n#[test]\n#[cfg_attr(crossbeam_sanitize, ignore)] // TODO: assertions failed due to tsan is slow\n#[cfg_attr(gha_macos_runner, ignore = \"GitHub-hosted macOS runner is slow\")]\nfn ready() {\n const THREADS: usize = 4;\n\n let hits = AtomicUsize::new(0);\n let r1 = tick(ms(200));\n let r2 = tick(ms(300));\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n let timeout = after(ms(1100));\n 'outer: loop {\n let mut sel = Select::new();\n sel.recv(&r1);\n sel.recv(&r2);\n sel.recv(&timeout);\n loop {\n match sel.ready() {\n 0 => {\n if r1.try_recv().is_ok() {\n hits.fetch_add(1, Ordering::SeqCst);\n break;\n }\n }\n 1 => {\n if r2.try_recv().is_ok() {\n hits.fetch_add(1, Ordering::SeqCst);\n break;\n }\n }\n 2 => {\n if timeout.try_recv().is_ok() {\n break 'outer;\n }\n }\n _ => unreachable!(),\n }\n }\n }\n });\n }\n })\n .unwrap();\n\n assert_eq!(hits.load(Ordering::SeqCst), 8);\n}\n\n#[test]\nfn fairness() {\n const COUNT: usize = 30;\n\n for &dur in &[0, 1] {\n let mut hits = [0usize; 2];\n\n for _ in 0..COUNT {\n let r1 = tick(ms(dur));\n let r2 = tick(ms(dur));\n\n for _ in 0..COUNT {\n select! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n }\n }\n }\n\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n }\n}\n\n#[test]\nfn fairness_duplicates() {\n const COUNT: usize = 30;\n\n for &dur in &[0, 1] {\n let mut hits = [0usize; 5];\n\n for _ in 0..COUNT {\n let r = tick(ms(dur));\n\n for _ in 0..COUNT {\n select! {\n recv(r) -> _ => hits[0] += 1,\n recv(r) -> _ => hits[1] += 1,\n recv(r) -> _ => hits[2] += 1,\n recv(r) -> _ => hits[3] += 1,\n recv(r) -> _ => hits[4] += 1,\n }\n }\n }\n\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n }\n}\n" }, { "path": "crossbeam-channel/tests/zero.rs", "content": "//! Tests for the zero channel flavor.\n\nuse std::{\n any::Any,\n sync::atomic::{AtomicUsize, Ordering},\n thread,\n time::Duration,\n};\n\nuse crossbeam_channel::{\n Receiver, RecvError, RecvTimeoutError, SendError, SendTimeoutError, TryRecvError, TrySendError,\n bounded, select,\n};\nuse crossbeam_utils::thread::scope;\n\nfn ms(ms: u64) -> Duration {\n Duration::from_millis(ms)\n}\n\n#[test]\nfn smoke() {\n let (s, r) = bounded(0);\n assert_eq!(s.try_send(7), Err(TrySendError::Full(7)));\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n}\n\n#[test]\nfn capacity() {\n let (s, r) = bounded::<()>(0);\n assert_eq!(s.capacity(), Some(0));\n assert_eq!(r.capacity(), Some(0));\n}\n\n#[test]\nfn len_empty_full() {\n let (s, r) = bounded(0);\n\n assert_eq!(s.len(), 0);\n assert!(s.is_empty());\n assert!(s.is_full());\n assert_eq!(r.len(), 0);\n assert!(r.is_empty());\n assert!(r.is_full());\n\n scope(|scope| {\n scope.spawn(|_| s.send(0).unwrap());\n scope.spawn(|_| r.recv().unwrap());\n })\n .unwrap();\n\n assert_eq!(s.len(), 0);\n assert!(s.is_empty());\n assert!(s.is_full());\n assert_eq!(r.len(), 0);\n assert!(r.is_empty());\n assert!(r.is_full());\n}\n\n#[test]\nfn try_recv() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.try_recv(), Err(TryRecvError::Empty));\n thread::sleep(ms(1500));\n assert_eq!(r.try_recv(), Ok(7));\n thread::sleep(ms(500));\n assert_eq!(r.try_recv(), Err(TryRecvError::Disconnected));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n s.send(7).unwrap();\n });\n })\n .unwrap();\n}\n\n#[test]\nfn recv() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.recv(), Ok(7));\n thread::sleep(ms(1000));\n assert_eq!(r.recv(), Ok(8));\n thread::sleep(ms(1000));\n assert_eq!(r.recv(), Ok(9));\n assert_eq!(r.recv(), Err(RecvError));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n s.send(7).unwrap();\n s.send(8).unwrap();\n s.send(9).unwrap();\n });\n })\n .unwrap();\n}\n\n#[test]\nfn recv_timeout() {\n let (s, r) = bounded::(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.recv_timeout(ms(1000)), Err(RecvTimeoutError::Timeout));\n assert_eq!(r.recv_timeout(ms(1000)), Ok(7));\n assert_eq!(\n r.recv_timeout(ms(1000)),\n Err(RecvTimeoutError::Disconnected)\n );\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n s.send(7).unwrap();\n });\n })\n .unwrap();\n}\n\n#[test]\nfn try_send() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(s.try_send(7), Err(TrySendError::Full(7)));\n thread::sleep(ms(1500));\n assert_eq!(s.try_send(8), Ok(()));\n thread::sleep(ms(500));\n assert_eq!(s.try_send(9), Err(TrySendError::Disconnected(9)));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n assert_eq!(r.recv(), Ok(8));\n });\n })\n .unwrap();\n}\n\n#[test]\nfn send() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n s.send(7).unwrap();\n thread::sleep(ms(1000));\n s.send(8).unwrap();\n thread::sleep(ms(1000));\n s.send(9).unwrap();\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n assert_eq!(r.recv(), Ok(7));\n assert_eq!(r.recv(), Ok(8));\n assert_eq!(r.recv(), Ok(9));\n });\n })\n .unwrap();\n}\n\n#[test]\nfn send_timeout() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(\n s.send_timeout(7, ms(1000)),\n Err(SendTimeoutError::Timeout(7))\n );\n assert_eq!(s.send_timeout(8, ms(1000)), Ok(()));\n assert_eq!(\n s.send_timeout(9, ms(1000)),\n Err(SendTimeoutError::Disconnected(9))\n );\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1500));\n assert_eq!(r.recv(), Ok(8));\n });\n })\n .unwrap();\n}\n\n#[test]\nfn len() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 25_000 };\n\n let (s, r) = bounded(0);\n\n assert_eq!(s.len(), 0);\n assert_eq!(r.len(), 0);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n assert_eq!(r.recv(), Ok(i));\n assert_eq!(r.len(), 0);\n }\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n s.send(i).unwrap();\n assert_eq!(s.len(), 0);\n }\n });\n })\n .unwrap();\n\n assert_eq!(s.len(), 0);\n assert_eq!(r.len(), 0);\n}\n\n#[test]\nfn disconnect_wakes_sender() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(s.send(()), Err(SendError(())));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n drop(r);\n });\n })\n .unwrap();\n}\n\n#[test]\nfn disconnect_wakes_receiver() {\n let (s, r) = bounded::<()>(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n assert_eq!(r.recv(), Err(RecvError));\n });\n scope.spawn(move |_| {\n thread::sleep(ms(1000));\n drop(s);\n });\n })\n .unwrap();\n}\n\n#[test]\nfn spsc() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 100_000 };\n\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n for i in 0..COUNT {\n assert_eq!(r.recv(), Ok(i));\n }\n assert_eq!(r.recv(), Err(RecvError));\n });\n scope.spawn(move |_| {\n for i in 0..COUNT {\n s.send(i).unwrap();\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn mpmc() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 25_000 };\n const THREADS: usize = 4;\n\n let (s, r) = bounded::(0);\n let v = (0..COUNT).map(|_| AtomicUsize::new(0)).collect::>();\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n let n = r.recv().unwrap();\n v[n].fetch_add(1, Ordering::SeqCst);\n }\n });\n }\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..COUNT {\n s.send(i).unwrap();\n }\n });\n }\n })\n .unwrap();\n\n for c in v {\n assert_eq!(c.load(Ordering::SeqCst), THREADS);\n }\n}\n\n#[test]\nfn stress_oneshot() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 10_000 };\n\n for _ in 0..COUNT {\n let (s, r) = bounded(1);\n\n scope(|scope| {\n scope.spawn(|_| r.recv().unwrap());\n scope.spawn(|_| s.send(0).unwrap());\n })\n .unwrap();\n }\n}\n\n#[test]\nfn stress_iter() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 1000 };\n\n let (request_s, request_r) = bounded(0);\n let (response_s, response_r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut count = 0;\n loop {\n for x in response_r.try_iter() {\n count += x;\n if count == COUNT {\n return;\n }\n }\n let _ = request_s.try_send(());\n }\n });\n\n for _ in request_r.iter() {\n if response_s.send(1).is_err() {\n break;\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress_timeout_two_threads() {\n const COUNT: usize = 100;\n\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(50));\n }\n loop {\n if let Ok(()) = s.send_timeout(i, ms(10)) {\n break;\n }\n }\n }\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n if i % 2 == 0 {\n thread::sleep(ms(50));\n }\n loop {\n if let Ok(x) = r.recv_timeout(ms(10)) {\n assert_eq!(x, i);\n break;\n }\n }\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn drops() {\n const RUNS: usize = if cfg!(miri) { 20 } else { 100 };\n const STEPS: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n static DROPS: AtomicUsize = AtomicUsize::new(0);\n\n #[derive(Debug, PartialEq)]\n struct DropCounter;\n\n impl Drop for DropCounter {\n fn drop(&mut self) {\n DROPS.fetch_add(1, Ordering::SeqCst);\n }\n }\n\n let mut rng = fastrand::Rng::new();\n\n for _ in 0..RUNS {\n let steps = rng.usize(0..STEPS);\n\n DROPS.store(0, Ordering::SeqCst);\n let (s, r) = bounded::(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..steps {\n r.recv().unwrap();\n }\n });\n\n scope.spawn(|_| {\n for _ in 0..steps {\n s.send(DropCounter).unwrap();\n }\n });\n })\n .unwrap();\n\n assert_eq!(DROPS.load(Ordering::SeqCst), steps);\n drop(s);\n drop(r);\n assert_eq!(DROPS.load(Ordering::SeqCst), steps);\n }\n}\n\n#[test]\nfn fairness() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 10_000 };\n\n let (s1, r1) = bounded::<()>(0);\n let (s2, r2) = bounded::<()>(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n let mut hits = [0usize; 2];\n for _ in 0..COUNT {\n select! {\n recv(r1) -> _ => hits[0] += 1,\n recv(r2) -> _ => hits[1] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n });\n\n let mut hits = [0usize; 2];\n for _ in 0..COUNT {\n select! {\n send(s1, ()) -> _ => hits[0] += 1,\n send(s2, ()) -> _ => hits[1] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n })\n .unwrap();\n}\n\n#[test]\nfn fairness_duplicates() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 10_000 };\n\n let (s, r) = bounded::<()>(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n let mut hits = [0usize; 5];\n for _ in 0..COUNT {\n select! {\n recv(r) -> _ => hits[0] += 1,\n recv(r) -> _ => hits[1] += 1,\n recv(r) -> _ => hits[2] += 1,\n recv(r) -> _ => hits[3] += 1,\n recv(r) -> _ => hits[4] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n });\n\n let mut hits = [0usize; 5];\n for _ in 0..COUNT {\n select! {\n send(s, ()) -> _ => hits[0] += 1,\n send(s, ()) -> _ => hits[1] += 1,\n send(s, ()) -> _ => hits[2] += 1,\n send(s, ()) -> _ => hits[3] += 1,\n send(s, ()) -> _ => hits[4] += 1,\n }\n }\n assert!(hits.iter().all(|x| *x >= COUNT / hits.len() / 2));\n })\n .unwrap();\n}\n\n#[test]\nfn recv_in_send() {\n let (s, r) = bounded(0);\n\n scope(|scope| {\n scope.spawn(|_| {\n thread::sleep(ms(100));\n r.recv()\n });\n\n scope.spawn(|_| {\n thread::sleep(ms(500));\n s.send(()).unwrap();\n });\n\n select! {\n send(s, r.recv().unwrap()) -> _ => {}\n }\n })\n .unwrap();\n}\n\n#[test]\nfn channel_through_channel() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 1000 };\n\n type T = Box;\n\n let (s, r) = bounded::(0);\n\n scope(|scope| {\n scope.spawn(move |_| {\n let mut s = s;\n\n for _ in 0..COUNT {\n let (new_s, new_r) = bounded(0);\n let new_r: T = Box::new(Some(new_r));\n\n s.send(new_r).unwrap();\n s = new_s;\n }\n });\n\n scope.spawn(move |_| {\n let mut r = r;\n\n for _ in 0..COUNT {\n r = r\n .recv()\n .unwrap()\n .downcast_mut::>>()\n .unwrap()\n .take()\n .unwrap()\n }\n });\n })\n .unwrap();\n}\n" }, { "path": "crossbeam-deque/CHANGELOG.md", "content": "# Version 0.8.6\n\n- Fix stack overflow when pushing large value to `Injector`. (#1146, #1147, #1159)\n\n# Version 0.8.5\n\n- Remove dependency on `cfg-if`. (#1072)\n\n# Version 0.8.4\n\n- Bump the minimum supported Rust version to 1.61. (#1037)\n\n# Version 0.8.3\n\n- Add `Stealer::{steal_batch_with_limit, steal_batch_with_limit_and_pop}` methods. (#903)\n- Add `Injector::{steal_batch_with_limit, steal_batch_with_limit_and_pop}` methods. (#903)\n\n# Version 0.8.2\n\n- Bump the minimum supported Rust version to 1.38. (#877)\n\n# Version 0.8.1\n\n- Fix deque steal race condition. (#726)\n- Add `Stealer::len` method. (#708)\n\n# Version 0.8.0\n\n**Note:** This release has been yanked. See [GHSA-pqqp-xmhj-wgcw](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-pqqp-xmhj-wgcw) for details.\n\n- Bump the minimum supported Rust version to 1.36.\n- Add `Worker::len()` and `Injector::len()` methods.\n- Add `std` (enabled by default) feature for forward compatibility.\n\n# Version 0.7.4\n\n- Fix deque steal race condition.\n\n# Version 0.7.3\n\n**Note:** This release has been yanked. See [GHSA-pqqp-xmhj-wgcw](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-pqqp-xmhj-wgcw) for details.\n\n- Stop stealing from the same deque. (#448)\n- Fix unsoundness issues by adopting `MaybeUninit`. (#458)\n\n# Version 0.7.2\n\n**Note:** This release has been yanked. See [GHSA-pqqp-xmhj-wgcw](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-pqqp-xmhj-wgcw) for details.\n\n- Bump `crossbeam-epoch` to `0.8`.\n- Bump `crossbeam-utils` to `0.7`.\n\n# Version 0.7.1\n\n**Note:** This release has been yanked. See [GHSA-pqqp-xmhj-wgcw](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-pqqp-xmhj-wgcw) for details.\n\n- Bump the minimum required version of `crossbeam-utils`.\n\n# Version 0.7.0\n\n**Note:** This release has been yanked. See [GHSA-pqqp-xmhj-wgcw](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-pqqp-xmhj-wgcw) for details.\n\n- Make `Worker::pop()` faster in the FIFO case.\n- Replace `fifo()` nad `lifo()` with `Worker::new_fifo()` and `Worker::new_lifo()`.\n- Add more batched steal methods.\n- Introduce `Injector`, a MPMC queue.\n- Rename `Steal::Data` to `Steal::Success`.\n- Add `Steal::or_else()` and implement `FromIterator` for `Steal`.\n- Add `#[must_use]` to `Steal`.\n\n# Version 0.6.3\n\n- Bump `crossbeam-epoch` to `0.7`.\n\n# Version 0.6.2\n\n- Update `crosbeam-utils` to `0.6`.\n\n# Version 0.6.1\n\n- Change a few `Relaxed` orderings to `Release` in order to fix false positives by tsan.\n\n# Version 0.6.0\n\n- Add `Stealer::steal_many` for batched stealing.\n- Change the return type of `pop` to `Pop` so that spinning can be handled manually.\n\n# Version 0.5.2\n\n- Update `crossbeam-utils` to `0.5.0`.\n\n# Version 0.5.1\n\n- Minor optimizations.\n\n# Version 0.5.0\n\n- Add two deque constructors : `fifo()` and `lifo()`.\n- Update `rand` to `0.5.3`.\n- Rename `Deque` to `Worker`.\n- Return `Option` from `Stealer::steal`.\n- Remove methods `Deque::len` and `Stealer::len`.\n- Remove method `Deque::stealer`.\n- Remove method `Deque::steal`.\n\n# Version 0.4.1\n\n- Update `crossbeam-epoch` to `0.5.0`.\n\n# Version 0.4.0\n\n- Update `crossbeam-epoch` to `0.4.2`.\n- Update `crossbeam-utils` to `0.4.0`.\n- Require minimum Rust version 1.25.\n\n# Version 0.3.1\n\n- Add `Deque::capacity`.\n- Add `Deque::min_capacity`.\n- Add `Deque::shrink_to_fit`.\n- Update `crossbeam-epoch` to `0.3.0`.\n- Support Rust 1.20.\n- Shrink the buffer in `Deque::push` if necessary.\n\n# Version 0.3.0\n\n- Update `crossbeam-epoch` to `0.4.0`.\n- Drop support for Rust 1.13.\n\n# Version 0.2.0\n\n- Update `crossbeam-epoch` to `0.3.0`.\n- Support Rust 1.13.\n\n# Version 0.1.1\n\n- Update `crossbeam-epoch` to `0.2.0`.\n\n# Version 0.1.0\n\n- First implementation of the Chase-Lev deque.\n" }, { "path": "crossbeam-deque/Cargo.toml", "content": "[package]\nname = \"crossbeam-deque\"\n# When publishing a new version:\n# - Update CHANGELOG.md\n# - Update README.md (when increasing major or minor version)\n# - Run './tools/publish.sh crossbeam-deque '\nversion = \"0.8.6\"\nedition = \"2021\"\n# NB: Sync with msrv badge and \"Compatibility\" section in README.md\nrust-version = \"1.74\"\nlicense = \"MIT OR Apache-2.0\"\nrepository = \"https://github.com/crossbeam-rs/crossbeam\"\nhomepage = \"https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-deque\"\ndescription = \"Concurrent work-stealing deque\"\nkeywords = [\"chase-lev\", \"lock-free\", \"scheduler\", \"scheduling\"]\ncategories = [\"algorithms\", \"concurrency\", \"data-structures\"]\n\n[package.metadata.docs.rs]\ntargets = [\"x86_64-unknown-linux-gnu\"]\n\n[package.metadata.cargo_check_external_types]\n# The following are external types that are allowed to be exposed in our public API.\nallowed_external_types = [\n]\n\n[features]\ndefault = [\"std\"]\n\n# Enable to use APIs that require `std`.\n# This is enabled by default.\n#\n# NOTE: Disabling `std` feature is not supported yet.\nstd = [\"crossbeam-epoch/std\", \"crossbeam-utils/std\"]\n\n[dependencies]\ncrossbeam-epoch = { version = \"0.9.17\", path = \"../crossbeam-epoch\", default-features = false }\ncrossbeam-utils = { version = \"0.8.18\", path = \"../crossbeam-utils\", default-features = false }\n\n[dev-dependencies]\nfastrand = \"2\"\n\n[lints]\nworkspace = true\n" }, { "path": "crossbeam-deque/LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "crossbeam-deque/LICENSE-MIT", "content": "The MIT License (MIT)\n\nCopyright (c) 2019 The Crossbeam Project Developers\n\nPermission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "crossbeam-deque/README.md", "content": "# Crossbeam Deque\n\n[![Build Status](https://github.com/crossbeam-rs/crossbeam/workflows/CI/badge.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/actions)\n[![License](https://img.shields.io/badge/license-MIT_OR_Apache--2.0-blue.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-deque#license)\n[![Cargo](https://img.shields.io/crates/v/crossbeam-deque.svg)](\nhttps://crates.io/crates/crossbeam-deque)\n[![Documentation](https://docs.rs/crossbeam-deque/badge.svg)](\nhttps://docs.rs/crossbeam-deque)\n[![Rust 1.74+](https://img.shields.io/badge/rust-1.74+-lightgray.svg)](\nhttps://www.rust-lang.org)\n[![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ)\n\nThis crate provides work-stealing deques, which are primarily intended for\nbuilding task schedulers.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\ncrossbeam-deque = \"0.8\"\n```\n\n## Compatibility\n\nCrossbeam Deque supports stable Rust releases going back at least six months,\nand every time the minimum supported Rust version is increased, a new minor\nversion is released. Currently, the minimum supported Rust version is 1.74.\n\n## License\n\nLicensed under either of\n\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n#### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "path": "crossbeam-deque/build.rs", "content": "// The rustc-cfg emitted by the build script are *not* public API.\n\nuse std::env;\n\nfn main() {\n println!(\"cargo:rerun-if-changed=build.rs\");\n println!(\"cargo:rustc-check-cfg=cfg(crossbeam_sanitize_thread)\");\n\n // `cfg(sanitize = \"..\")` is not stabilized.\n let sanitize = env::var(\"CARGO_CFG_SANITIZE\").unwrap_or_default();\n if sanitize.contains(\"thread\") {\n println!(\"cargo:rustc-cfg=crossbeam_sanitize_thread\");\n }\n}\n" }, { "path": "crossbeam-deque/src/deque.rs", "content": "use alloc::{alloc::handle_alloc_error, boxed::Box, sync::Arc};\nuse core::{\n alloc::Layout,\n cell::{Cell, UnsafeCell},\n cmp, fmt,\n marker::PhantomData,\n mem::{self, MaybeUninit},\n ptr,\n sync::atomic::{self, AtomicIsize, AtomicPtr, AtomicUsize, Ordering},\n};\n\nuse crossbeam_epoch::{self as epoch, Atomic, Owned};\nuse crossbeam_utils::{Backoff, CachePadded};\n\nuse crate::alloc_helper::Global;\n\n// Minimum buffer capacity.\nconst MIN_CAP: usize = 64;\n// Maximum number of tasks that can be stolen in `steal_batch()` and `steal_batch_and_pop()`.\nconst MAX_BATCH: usize = 32;\n// If a buffer of at least this size is retired, thread-local garbage is flushed so that it gets\n// deallocated as soon as possible.\nconst FLUSH_THRESHOLD_BYTES: usize = 1 << 10;\n\n/// A buffer that holds tasks in a worker queue.\n///\n/// This is just a pointer to the buffer and its length - dropping an instance of this struct will\n/// *not* deallocate the buffer.\nstruct Buffer {\n /// Pointer to the allocated memory.\n ptr: *mut T,\n\n /// Capacity of the buffer. Always a power of two.\n cap: usize,\n}\n\nunsafe impl Send for Buffer {}\n\nimpl Buffer {\n /// Allocates a new buffer with the specified capacity.\n fn alloc(cap: usize) -> Self {\n debug_assert_eq!(cap, cap.next_power_of_two());\n\n let ptr = Box::into_raw(\n (0..cap)\n .map(|_| MaybeUninit::::uninit())\n .collect::>(),\n )\n .cast::();\n\n Self { ptr, cap }\n }\n\n /// Deallocates the buffer.\n unsafe fn dealloc(self) {\n drop(unsafe {\n Box::from_raw(ptr::slice_from_raw_parts_mut(\n self.ptr.cast::>(),\n self.cap,\n ))\n });\n }\n\n /// Returns a pointer to the task at the specified `index`.\n unsafe fn at(&self, index: isize) -> *mut T {\n // `self.cap` is always a power of two.\n // We do all the loads at `MaybeUninit` because we might realize, after loading, that we\n // don't actually have the right to access this memory.\n unsafe { self.ptr.offset(index & (self.cap - 1) as isize) }\n }\n\n /// Writes `task` into the specified `index`.\n ///\n /// This method might be concurrently called with another `read` at the same index, which is\n /// technically speaking a data race and therefore UB. We should use an atomic store here, but\n /// that would be more expensive and difficult to implement generically for all types `T`.\n /// Hence, as a hack, we use a volatile write instead.\n unsafe fn write(&self, index: isize, task: MaybeUninit) {\n unsafe { ptr::write_volatile(self.at(index).cast::>(), task) }\n }\n\n /// Reads a task from the specified `index`.\n ///\n /// This method might be concurrently called with another `write` at the same index, which is\n /// technically speaking a data race and therefore UB. We should use an atomic load here, but\n /// that would be more expensive and difficult to implement generically for all types `T`.\n /// Hence, as a hack, we use a volatile load instead.\n unsafe fn read(&self, index: isize) -> MaybeUninit {\n unsafe { ptr::read_volatile(self.at(index).cast::>()) }\n }\n}\n\nimpl Clone for Buffer {\n fn clone(&self) -> Self {\n *self\n }\n}\n\nimpl Copy for Buffer {}\n\n/// Internal queue data shared between the worker and stealers.\n///\n/// The implementation is based on the following work:\n///\n/// 1. [Chase and Lev. Dynamic circular work-stealing deque. SPAA 2005.][chase-lev]\n/// 2. [Le, Pop, Cohen, and Nardelli. Correct and efficient work-stealing for weak memory models.\n/// PPoPP 2013.][weak-mem]\n/// 3. [Norris and Demsky. CDSchecker: checking concurrent data structures written with C/C++\n/// atomics. OOPSLA 2013.][checker]\n///\n/// [chase-lev]: https://dl.acm.org/citation.cfm?id=1073974\n/// [weak-mem]: https://dl.acm.org/citation.cfm?id=2442524\n/// [checker]: https://dl.acm.org/citation.cfm?id=2509514\nstruct Inner {\n /// The front index.\n front: AtomicIsize,\n\n /// The back index.\n back: AtomicIsize,\n\n /// The underlying buffer.\n buffer: CachePadded>>,\n}\n\nimpl Drop for Inner {\n fn drop(&mut self) {\n // Load the back index, front index, and buffer.\n let b = *self.back.get_mut();\n let f = *self.front.get_mut();\n\n unsafe {\n let buffer = self.buffer.load(Ordering::Relaxed, epoch::unprotected());\n\n // Go through the buffer from front to back and drop all tasks in the queue.\n let mut i = f;\n while i != b {\n buffer.deref().at(i).drop_in_place();\n i = i.wrapping_add(1);\n }\n\n // Free the memory allocated by the buffer.\n buffer.into_owned().into_box().dealloc();\n }\n }\n}\n\n/// Worker queue flavor: FIFO or LIFO.\n#[derive(Clone, Copy, Debug, Eq, PartialEq)]\nenum Flavor {\n /// The first-in first-out flavor.\n Fifo,\n\n /// The last-in first-out flavor.\n Lifo,\n}\n\n/// A worker queue.\n///\n/// This is a FIFO or LIFO queue that is owned by a single thread, but other threads may steal\n/// tasks from it. Task schedulers typically create a single worker queue per thread.\n///\n/// # Examples\n///\n/// A FIFO worker:\n///\n/// ```\n/// use crossbeam_deque::{Steal, Worker};\n///\n/// let w = Worker::new_fifo();\n/// let s = w.stealer();\n///\n/// w.push(1);\n/// w.push(2);\n/// w.push(3);\n///\n/// assert_eq!(s.steal(), Steal::Success(1));\n/// assert_eq!(w.pop(), Some(2));\n/// assert_eq!(w.pop(), Some(3));\n/// ```\n///\n/// A LIFO worker:\n///\n/// ```\n/// use crossbeam_deque::{Steal, Worker};\n///\n/// let w = Worker::new_lifo();\n/// let s = w.stealer();\n///\n/// w.push(1);\n/// w.push(2);\n/// w.push(3);\n///\n/// assert_eq!(s.steal(), Steal::Success(1));\n/// assert_eq!(w.pop(), Some(3));\n/// assert_eq!(w.pop(), Some(2));\n/// ```\npub struct Worker {\n /// A reference to the inner representation of the queue.\n inner: Arc>>,\n\n /// A copy of `inner.buffer` for quick access.\n buffer: Cell>,\n\n /// The flavor of the queue.\n flavor: Flavor,\n\n /// Indicates that the worker cannot be shared among threads.\n _marker: PhantomData<*mut ()>, // !Send + !Sync\n}\n\nunsafe impl Send for Worker {}\n\nimpl Worker {\n /// Creates a FIFO worker queue.\n ///\n /// Tasks are pushed and popped from opposite ends.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w = Worker::::new_fifo();\n /// ```\n pub fn new_fifo() -> Self {\n let buffer = Buffer::alloc(MIN_CAP);\n\n let inner = Arc::new(CachePadded::new(Inner {\n front: AtomicIsize::new(0),\n back: AtomicIsize::new(0),\n buffer: CachePadded::new(Atomic::new(buffer)),\n }));\n\n Self {\n inner,\n buffer: Cell::new(buffer),\n flavor: Flavor::Fifo,\n _marker: PhantomData,\n }\n }\n\n /// Creates a LIFO worker queue.\n ///\n /// Tasks are pushed and popped from the same end.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w = Worker::::new_lifo();\n /// ```\n pub fn new_lifo() -> Self {\n let buffer = Buffer::alloc(MIN_CAP);\n\n let inner = Arc::new(CachePadded::new(Inner {\n front: AtomicIsize::new(0),\n back: AtomicIsize::new(0),\n buffer: CachePadded::new(Atomic::new(buffer)),\n }));\n\n Self {\n inner,\n buffer: Cell::new(buffer),\n flavor: Flavor::Lifo,\n _marker: PhantomData,\n }\n }\n\n /// Creates a stealer for this queue.\n ///\n /// The returned stealer can be shared among threads and cloned.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w = Worker::::new_lifo();\n /// let s = w.stealer();\n /// ```\n pub fn stealer(&self) -> Stealer {\n Stealer {\n inner: self.inner.clone(),\n flavor: self.flavor,\n }\n }\n\n /// Resizes the internal buffer to the new capacity of `new_cap`.\n #[cold]\n unsafe fn resize(&self, new_cap: usize) {\n // Load the back index, front index, and buffer.\n let b = self.inner.back.load(Ordering::Relaxed);\n let f = self.inner.front.load(Ordering::Relaxed);\n let buffer = self.buffer.get();\n\n // Allocate a new buffer and copy data from the old buffer to the new one.\n let new = Buffer::alloc(new_cap);\n let mut i = f;\n while i != b {\n unsafe { ptr::copy_nonoverlapping(buffer.at(i), new.at(i), 1) }\n i = i.wrapping_add(1);\n }\n\n let guard = &epoch::pin();\n\n // Replace the old buffer with the new one.\n self.buffer.replace(new);\n let old =\n self.inner\n .buffer\n .swap(Owned::new(new).into_shared(guard), Ordering::Release, guard);\n\n // Destroy the old buffer later.\n unsafe { guard.defer_unchecked(move || old.into_owned().into_box().dealloc()) }\n\n // If the buffer is very large, then flush the thread-local garbage in order to deallocate\n // it as soon as possible.\n if mem::size_of::() * new_cap >= FLUSH_THRESHOLD_BYTES {\n guard.flush();\n }\n }\n\n /// Reserves enough capacity so that `reserve_cap` tasks can be pushed without growing the\n /// buffer.\n fn reserve(&self, reserve_cap: usize) {\n if reserve_cap > 0 {\n // Compute the current length.\n let b = self.inner.back.load(Ordering::Relaxed);\n let f = self.inner.front.load(Ordering::SeqCst);\n let len = b.wrapping_sub(f) as usize;\n\n // The current capacity.\n let cap = self.buffer.get().cap;\n\n // Is there enough capacity to push `reserve_cap` tasks?\n if cap - len < reserve_cap {\n // Keep doubling the capacity as much as is needed.\n let mut new_cap = cap * 2;\n while new_cap - len < reserve_cap {\n new_cap *= 2;\n }\n\n // Resize the buffer.\n unsafe {\n self.resize(new_cap);\n }\n }\n }\n }\n\n /// Returns `true` if the queue is empty.\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w = Worker::new_lifo();\n ///\n /// assert!(w.is_empty());\n /// w.push(1);\n /// assert!(!w.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n let b = self.inner.back.load(Ordering::Relaxed);\n let f = self.inner.front.load(Ordering::SeqCst);\n b.wrapping_sub(f) <= 0\n }\n\n /// Returns the number of tasks in the deque.\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w = Worker::new_lifo();\n ///\n /// assert_eq!(w.len(), 0);\n /// w.push(1);\n /// assert_eq!(w.len(), 1);\n /// w.push(1);\n /// assert_eq!(w.len(), 2);\n /// ```\n pub fn len(&self) -> usize {\n let b = self.inner.back.load(Ordering::Relaxed);\n let f = self.inner.front.load(Ordering::SeqCst);\n b.wrapping_sub(f).max(0) as usize\n }\n\n /// Pushes a task into the queue.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w = Worker::new_lifo();\n /// w.push(1);\n /// w.push(2);\n /// ```\n pub fn push(&self, task: T) {\n // Load the back index, front index, and buffer.\n let b = self.inner.back.load(Ordering::Relaxed);\n let f = self.inner.front.load(Ordering::Acquire);\n let mut buffer = self.buffer.get();\n\n // Calculate the length of the queue.\n let len = b.wrapping_sub(f);\n\n // Is the queue full?\n if len >= buffer.cap as isize {\n // Yes. Grow the underlying buffer.\n unsafe {\n self.resize(2 * buffer.cap);\n }\n buffer = self.buffer.get();\n }\n\n // Write `task` into the slot.\n unsafe {\n buffer.write(b, MaybeUninit::new(task));\n }\n\n // ThreadSanitizer does not understand fences, so we omit fence and do store with Release ordering.\n #[cfg(not(crossbeam_sanitize_thread))]\n atomic::fence(Ordering::Release);\n let store_order = if cfg!(crossbeam_sanitize_thread) {\n Ordering::Release\n } else {\n Ordering::Relaxed\n };\n\n // Increment the back index.\n self.inner.back.store(b.wrapping_add(1), store_order);\n }\n\n /// Pops a task from the queue.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w = Worker::new_fifo();\n /// w.push(1);\n /// w.push(2);\n ///\n /// assert_eq!(w.pop(), Some(1));\n /// assert_eq!(w.pop(), Some(2));\n /// assert_eq!(w.pop(), None);\n /// ```\n pub fn pop(&self) -> Option {\n // Load the back and front index.\n let b = self.inner.back.load(Ordering::Relaxed);\n let f = self.inner.front.load(Ordering::Relaxed);\n\n // Calculate the length of the queue.\n let len = b.wrapping_sub(f);\n\n // Is the queue empty?\n if len <= 0 {\n return None;\n }\n\n match self.flavor {\n // Pop from the front of the queue.\n Flavor::Fifo => {\n // Try incrementing the front index to pop the task.\n let f = self.inner.front.fetch_add(1, Ordering::SeqCst);\n let new_f = f.wrapping_add(1);\n\n if b.wrapping_sub(new_f) < 0 {\n self.inner.front.store(f, Ordering::Relaxed);\n return None;\n }\n\n unsafe {\n // Read the popped task.\n let buffer = self.buffer.get();\n let task = buffer.read(f).assume_init();\n\n // Shrink the buffer if `len - 1` is less than one fourth of the capacity.\n if buffer.cap > MIN_CAP && len <= buffer.cap as isize / 4 {\n self.resize(buffer.cap / 2);\n }\n\n Some(task)\n }\n }\n\n // Pop from the back of the queue.\n Flavor::Lifo => {\n // Decrement the back index.\n let b = b.wrapping_sub(1);\n self.inner.back.store(b, Ordering::Relaxed);\n\n atomic::fence(Ordering::SeqCst);\n\n // Load the front index.\n let f = self.inner.front.load(Ordering::Relaxed);\n\n // Compute the length after the back index was decremented.\n let len = b.wrapping_sub(f);\n\n if len < 0 {\n // The queue is empty. Restore the back index to the original task.\n self.inner.back.store(b.wrapping_add(1), Ordering::Relaxed);\n None\n } else {\n // Read the task to be popped.\n let buffer = self.buffer.get();\n let mut task = unsafe { Some(buffer.read(b)) };\n\n // Are we popping the last task from the queue?\n if len == 0 {\n // Try incrementing the front index.\n if self\n .inner\n .front\n .compare_exchange(\n f,\n f.wrapping_add(1),\n Ordering::SeqCst,\n Ordering::Relaxed,\n )\n .is_err()\n {\n // Failed. We didn't pop anything. Reset to `None`.\n task.take();\n }\n\n // Restore the back index to the original task.\n self.inner.back.store(b.wrapping_add(1), Ordering::Relaxed);\n } else {\n // Shrink the buffer if `len` is less than one fourth of the capacity.\n if buffer.cap > MIN_CAP && len < buffer.cap as isize / 4 {\n unsafe {\n self.resize(buffer.cap / 2);\n }\n }\n }\n\n task.map(|t| unsafe { t.assume_init() })\n }\n }\n }\n }\n}\n\nimpl fmt::Debug for Worker {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Worker { .. }\")\n }\n}\n\n/// A stealer handle of a worker queue.\n///\n/// Stealers can be shared among threads.\n///\n/// Task schedulers typically have a single worker queue per worker thread.\n///\n/// # Examples\n///\n/// ```\n/// use crossbeam_deque::{Steal, Worker};\n///\n/// let w = Worker::new_lifo();\n/// w.push(1);\n/// w.push(2);\n///\n/// let s = w.stealer();\n/// assert_eq!(s.steal(), Steal::Success(1));\n/// assert_eq!(s.steal(), Steal::Success(2));\n/// assert_eq!(s.steal(), Steal::Empty);\n/// ```\npub struct Stealer {\n /// A reference to the inner representation of the queue.\n inner: Arc>>,\n\n /// The flavor of the queue.\n flavor: Flavor,\n}\n\nunsafe impl Send for Stealer {}\nunsafe impl Sync for Stealer {}\n\nimpl Stealer {\n /// Returns `true` if the queue is empty.\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w = Worker::new_lifo();\n /// let s = w.stealer();\n ///\n /// assert!(s.is_empty());\n /// w.push(1);\n /// assert!(!s.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n let f = self.inner.front.load(Ordering::Acquire);\n atomic::fence(Ordering::SeqCst);\n let b = self.inner.back.load(Ordering::Acquire);\n b.wrapping_sub(f) <= 0\n }\n\n /// Returns the number of tasks in the deque.\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w = Worker::new_lifo();\n /// let s = w.stealer();\n ///\n /// assert_eq!(s.len(), 0);\n /// w.push(1);\n /// assert_eq!(s.len(), 1);\n /// w.push(2);\n /// assert_eq!(s.len(), 2);\n /// ```\n pub fn len(&self) -> usize {\n let f = self.inner.front.load(Ordering::Acquire);\n atomic::fence(Ordering::SeqCst);\n let b = self.inner.back.load(Ordering::Acquire);\n b.wrapping_sub(f).max(0) as usize\n }\n\n /// Steals a task from the queue.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::{Steal, Worker};\n ///\n /// let w = Worker::new_lifo();\n /// w.push(1);\n /// w.push(2);\n ///\n /// let s = w.stealer();\n /// assert_eq!(s.steal(), Steal::Success(1));\n /// assert_eq!(s.steal(), Steal::Success(2));\n /// ```\n pub fn steal(&self) -> Steal {\n // Load the front index.\n let f = self.inner.front.load(Ordering::Acquire);\n\n // A SeqCst fence is needed here.\n //\n // If the current thread is already pinned (reentrantly), we must manually issue the\n // fence. Otherwise, the following pinning will issue the fence anyway, so we don't\n // have to.\n if epoch::is_pinned() {\n atomic::fence(Ordering::SeqCst);\n }\n\n let guard = &epoch::pin();\n\n // Load the back index.\n let b = self.inner.back.load(Ordering::Acquire);\n\n // Is the queue empty?\n if b.wrapping_sub(f) <= 0 {\n return Steal::Empty;\n }\n\n // Load the buffer and read the task at the front.\n let buffer = self.inner.buffer.load(Ordering::Acquire, guard);\n let task = unsafe { buffer.deref().read(f) };\n\n // Try incrementing the front index to steal the task.\n // If the buffer has been swapped or the increment fails, we retry.\n if self.inner.buffer.load(Ordering::Acquire, guard) != buffer\n || self\n .inner\n .front\n .compare_exchange(f, f.wrapping_add(1), Ordering::SeqCst, Ordering::Relaxed)\n .is_err()\n {\n // We didn't steal this task, forget it.\n return Steal::Retry;\n }\n\n // Return the stolen task.\n Steal::Success(unsafe { task.assume_init() })\n }\n\n /// Steals a batch of tasks and pushes them into another worker.\n ///\n /// How many tasks exactly will be stolen is not specified. That said, this method will try to\n /// steal around half of the tasks in the queue, but also not more than some constant limit.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w1 = Worker::new_fifo();\n /// w1.push(1);\n /// w1.push(2);\n /// w1.push(3);\n /// w1.push(4);\n ///\n /// let s = w1.stealer();\n /// let w2 = Worker::new_fifo();\n ///\n /// let _ = s.steal_batch(&w2);\n /// assert_eq!(w2.pop(), Some(1));\n /// assert_eq!(w2.pop(), Some(2));\n /// ```\n pub fn steal_batch(&self, dest: &Worker) -> Steal<()> {\n self.steal_batch_with_limit(dest, MAX_BATCH)\n }\n\n /// Steals no more than `limit` of tasks and pushes them into another worker.\n ///\n /// How many tasks exactly will be stolen is not specified. That said, this method will try to\n /// steal around half of the tasks in the queue, but also not more than the given limit.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Worker;\n ///\n /// let w1 = Worker::new_fifo();\n /// w1.push(1);\n /// w1.push(2);\n /// w1.push(3);\n /// w1.push(4);\n /// w1.push(5);\n /// w1.push(6);\n ///\n /// let s = w1.stealer();\n /// let w2 = Worker::new_fifo();\n ///\n /// let _ = s.steal_batch_with_limit(&w2, 2);\n /// assert_eq!(w2.pop(), Some(1));\n /// assert_eq!(w2.pop(), Some(2));\n /// assert_eq!(w2.pop(), None);\n ///\n /// w1.push(7);\n /// w1.push(8);\n /// // Setting a large limit does not guarantee that all elements will be popped. In this case,\n /// // half of the elements are currently popped, but the number of popped elements is considered\n /// // an implementation detail that may be changed in the future.\n /// let _ = s.steal_batch_with_limit(&w2, usize::MAX);\n /// assert_eq!(w2.len(), 3);\n /// ```\n pub fn steal_batch_with_limit(&self, dest: &Worker, limit: usize) -> Steal<()> {\n assert!(limit > 0);\n if Arc::ptr_eq(&self.inner, &dest.inner) {\n if dest.is_empty() {\n return Steal::Empty;\n } else {\n return Steal::Success(());\n }\n }\n\n // Load the front index.\n let mut f = self.inner.front.load(Ordering::Acquire);\n\n // A SeqCst fence is needed here.\n //\n // If the current thread is already pinned (reentrantly), we must manually issue the\n // fence. Otherwise, the following pinning will issue the fence anyway, so we don't\n // have to.\n if epoch::is_pinned() {\n atomic::fence(Ordering::SeqCst);\n }\n\n let guard = &epoch::pin();\n\n // Load the back index.\n let b = self.inner.back.load(Ordering::Acquire);\n\n // Is the queue empty?\n let len = b.wrapping_sub(f);\n if len <= 0 {\n return Steal::Empty;\n }\n\n // Reserve capacity for the stolen batch.\n let batch_size = cmp::min((len as usize).div_ceil(2), limit);\n dest.reserve(batch_size);\n let mut batch_size = batch_size as isize;\n\n // Get the destination buffer and back index.\n let dest_buffer = dest.buffer.get();\n let mut dest_b = dest.inner.back.load(Ordering::Relaxed);\n\n // Load the buffer.\n let buffer = self.inner.buffer.load(Ordering::Acquire, guard);\n\n match self.flavor {\n // Steal a batch of tasks from the front at once.\n Flavor::Fifo => {\n // Copy the batch from the source to the destination buffer.\n match dest.flavor {\n Flavor::Fifo => {\n for i in 0..batch_size {\n unsafe {\n let task = buffer.deref().read(f.wrapping_add(i));\n dest_buffer.write(dest_b.wrapping_add(i), task);\n }\n }\n }\n Flavor::Lifo => {\n for i in 0..batch_size {\n unsafe {\n let task = buffer.deref().read(f.wrapping_add(i));\n dest_buffer.write(dest_b.wrapping_add(batch_size - 1 - i), task);\n }\n }\n }\n }\n\n // Try incrementing the front index to steal the batch.\n // If the buffer has been swapped or the increment fails, we retry.\n if self.inner.buffer.load(Ordering::Acquire, guard) != buffer\n || self\n .inner\n .front\n .compare_exchange(\n f,\n f.wrapping_add(batch_size),\n Ordering::SeqCst,\n Ordering::Relaxed,\n )\n .is_err()\n {\n return Steal::Retry;\n }\n\n dest_b = dest_b.wrapping_add(batch_size);\n }\n\n // Steal a batch of tasks from the front one by one.\n Flavor::Lifo => {\n // This loop may modify the batch_size, which triggers a clippy lint warning.\n // Use a new variable to avoid the warning, and to make it clear we aren't\n // modifying the loop exit condition during iteration.\n let original_batch_size = batch_size;\n\n for i in 0..original_batch_size {\n // If this is not the first steal, check whether the queue is empty.\n if i > 0 {\n // We've already got the current front index. Now execute the fence to\n // synchronize with other threads.\n atomic::fence(Ordering::SeqCst);\n\n // Load the back index.\n let b = self.inner.back.load(Ordering::Acquire);\n\n // Is the queue empty?\n if b.wrapping_sub(f) <= 0 {\n batch_size = i;\n break;\n }\n }\n\n // Read the task at the front.\n let task = unsafe { buffer.deref().read(f) };\n\n // Try incrementing the front index to steal the task.\n // If the buffer has been swapped or the increment fails, we retry.\n if self.inner.buffer.load(Ordering::Acquire, guard) != buffer\n || self\n .inner\n .front\n .compare_exchange(\n f,\n f.wrapping_add(1),\n Ordering::SeqCst,\n Ordering::Relaxed,\n )\n .is_err()\n {\n // We didn't steal this task, forget it and break from the loop.\n batch_size = i;\n break;\n }\n\n // Write the stolen task into the destination buffer.\n unsafe {\n dest_buffer.write(dest_b, task);\n }\n\n // Move the source front index and the destination back index one step forward.\n f = f.wrapping_add(1);\n dest_b = dest_b.wrapping_add(1);\n }\n\n // If we didn't steal anything, the operation needs to be retried.\n if batch_size == 0 {\n return Steal::Retry;\n }\n\n // If stealing into a FIFO queue, stolen tasks need to be reversed.\n if dest.flavor == Flavor::Fifo {\n for i in 0..batch_size / 2 {\n unsafe {\n let i1 = dest_b.wrapping_sub(batch_size - i);\n let i2 = dest_b.wrapping_sub(i + 1);\n let t1 = dest_buffer.read(i1);\n let t2 = dest_buffer.read(i2);\n dest_buffer.write(i1, t2);\n dest_buffer.write(i2, t1);\n }\n }\n }\n }\n }\n\n // ThreadSanitizer does not understand fences, so we omit fence and do store with Release ordering.\n #[cfg(not(crossbeam_sanitize_thread))]\n atomic::fence(Ordering::Release);\n let store_order = if cfg!(crossbeam_sanitize_thread) {\n Ordering::Release\n } else {\n Ordering::Relaxed\n };\n\n // Update the back index in the destination queue.\n dest.inner.back.store(dest_b, store_order);\n\n // Return with success.\n Steal::Success(())\n }\n\n /// Steals a batch of tasks, pushes them into another worker, and pops a task from that worker.\n ///\n /// How many tasks exactly will be stolen is not specified. That said, this method will try to\n /// steal around half of the tasks in the queue, but also not more than some constant limit.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::{Steal, Worker};\n ///\n /// let w1 = Worker::new_fifo();\n /// w1.push(1);\n /// w1.push(2);\n /// w1.push(3);\n /// w1.push(4);\n ///\n /// let s = w1.stealer();\n /// let w2 = Worker::new_fifo();\n ///\n /// assert_eq!(s.steal_batch_and_pop(&w2), Steal::Success(1));\n /// assert_eq!(w2.pop(), Some(2));\n /// ```\n pub fn steal_batch_and_pop(&self, dest: &Worker) -> Steal {\n self.steal_batch_with_limit_and_pop(dest, MAX_BATCH)\n }\n\n /// Steals no more than `limit` of tasks, pushes them into another worker, and pops a task from\n /// that worker.\n ///\n /// How many tasks exactly will be stolen is not specified. That said, this method will try to\n /// steal around half of the tasks in the queue, but also not more than the given limit.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::{Steal, Worker};\n ///\n /// let w1 = Worker::new_fifo();\n /// w1.push(1);\n /// w1.push(2);\n /// w1.push(3);\n /// w1.push(4);\n /// w1.push(5);\n /// w1.push(6);\n ///\n /// let s = w1.stealer();\n /// let w2 = Worker::new_fifo();\n ///\n /// assert_eq!(s.steal_batch_with_limit_and_pop(&w2, 2), Steal::Success(1));\n /// assert_eq!(w2.pop(), Some(2));\n /// assert_eq!(w2.pop(), None);\n ///\n /// w1.push(7);\n /// w1.push(8);\n /// // Setting a large limit does not guarantee that all elements will be popped. In this case,\n /// // half of the elements are currently popped, but the number of popped elements is considered\n /// // an implementation detail that may be changed in the future.\n /// assert_eq!(s.steal_batch_with_limit_and_pop(&w2, usize::MAX), Steal::Success(3));\n /// assert_eq!(w2.pop(), Some(4));\n /// assert_eq!(w2.pop(), Some(5));\n /// assert_eq!(w2.pop(), None);\n /// ```\n pub fn steal_batch_with_limit_and_pop(&self, dest: &Worker, limit: usize) -> Steal {\n assert!(limit > 0);\n if Arc::ptr_eq(&self.inner, &dest.inner) {\n match dest.pop() {\n None => return Steal::Empty,\n Some(task) => return Steal::Success(task),\n }\n }\n\n // Load the front index.\n let mut f = self.inner.front.load(Ordering::Acquire);\n\n // A SeqCst fence is needed here.\n //\n // If the current thread is already pinned (reentrantly), we must manually issue the\n // fence. Otherwise, the following pinning will issue the fence anyway, so we don't\n // have to.\n if epoch::is_pinned() {\n atomic::fence(Ordering::SeqCst);\n }\n\n let guard = &epoch::pin();\n\n // Load the back index.\n let b = self.inner.back.load(Ordering::Acquire);\n\n // Is the queue empty?\n let len = b.wrapping_sub(f);\n if len <= 0 {\n return Steal::Empty;\n }\n\n // Reserve capacity for the stolen batch.\n let batch_size = cmp::min((len as usize - 1) / 2, limit - 1);\n dest.reserve(batch_size);\n let mut batch_size = batch_size as isize;\n\n // Get the destination buffer and back index.\n let dest_buffer = dest.buffer.get();\n let mut dest_b = dest.inner.back.load(Ordering::Relaxed);\n\n // Load the buffer\n let buffer = self.inner.buffer.load(Ordering::Acquire, guard);\n\n // Read the task at the front.\n let mut task = unsafe { buffer.deref().read(f) };\n\n match self.flavor {\n // Steal a batch of tasks from the front at once.\n Flavor::Fifo => {\n // Copy the batch from the source to the destination buffer.\n match dest.flavor {\n Flavor::Fifo => {\n for i in 0..batch_size {\n unsafe {\n let task = buffer.deref().read(f.wrapping_add(i + 1));\n dest_buffer.write(dest_b.wrapping_add(i), task);\n }\n }\n }\n Flavor::Lifo => {\n for i in 0..batch_size {\n unsafe {\n let task = buffer.deref().read(f.wrapping_add(i + 1));\n dest_buffer.write(dest_b.wrapping_add(batch_size - 1 - i), task);\n }\n }\n }\n }\n\n // Try incrementing the front index to steal the task.\n // If the buffer has been swapped or the increment fails, we retry.\n if self.inner.buffer.load(Ordering::Acquire, guard) != buffer\n || self\n .inner\n .front\n .compare_exchange(\n f,\n f.wrapping_add(batch_size + 1),\n Ordering::SeqCst,\n Ordering::Relaxed,\n )\n .is_err()\n {\n // We didn't steal this task, forget it.\n return Steal::Retry;\n }\n\n dest_b = dest_b.wrapping_add(batch_size);\n }\n\n // Steal a batch of tasks from the front one by one.\n Flavor::Lifo => {\n // Try incrementing the front index to steal the task.\n if self\n .inner\n .front\n .compare_exchange(f, f.wrapping_add(1), Ordering::SeqCst, Ordering::Relaxed)\n .is_err()\n {\n // We didn't steal this task, forget it.\n return Steal::Retry;\n }\n\n // Move the front index one step forward.\n f = f.wrapping_add(1);\n\n // Repeat the same procedure for the batch steals.\n //\n // This loop may modify the batch_size, which triggers a clippy lint warning.\n // Use a new variable to avoid the warning, and to make it clear we aren't\n // modifying the loop exit condition during iteration.\n let original_batch_size = batch_size;\n for i in 0..original_batch_size {\n // We've already got the current front index. Now execute the fence to\n // synchronize with other threads.\n atomic::fence(Ordering::SeqCst);\n\n // Load the back index.\n let b = self.inner.back.load(Ordering::Acquire);\n\n // Is the queue empty?\n if b.wrapping_sub(f) <= 0 {\n batch_size = i;\n break;\n }\n\n // Read the task at the front.\n let tmp = unsafe { buffer.deref().read(f) };\n\n // Try incrementing the front index to steal the task.\n // If the buffer has been swapped or the increment fails, we retry.\n if self.inner.buffer.load(Ordering::Acquire, guard) != buffer\n || self\n .inner\n .front\n .compare_exchange(\n f,\n f.wrapping_add(1),\n Ordering::SeqCst,\n Ordering::Relaxed,\n )\n .is_err()\n {\n // We didn't steal this task, forget it and break from the loop.\n batch_size = i;\n break;\n }\n\n // Write the previously stolen task into the destination buffer.\n unsafe {\n dest_buffer.write(dest_b, mem::replace(&mut task, tmp));\n }\n\n // Move the source front index and the destination back index one step forward.\n f = f.wrapping_add(1);\n dest_b = dest_b.wrapping_add(1);\n }\n\n // If stealing into a FIFO queue, stolen tasks need to be reversed.\n if dest.flavor == Flavor::Fifo {\n for i in 0..batch_size / 2 {\n unsafe {\n let i1 = dest_b.wrapping_sub(batch_size - i);\n let i2 = dest_b.wrapping_sub(i + 1);\n let t1 = dest_buffer.read(i1);\n let t2 = dest_buffer.read(i2);\n dest_buffer.write(i1, t2);\n dest_buffer.write(i2, t1);\n }\n }\n }\n }\n }\n\n // ThreadSanitizer does not understand fences, so we omit fence and do store with Release ordering.\n #[cfg(not(crossbeam_sanitize_thread))]\n atomic::fence(Ordering::Release);\n let store_order = if cfg!(crossbeam_sanitize_thread) {\n Ordering::Release\n } else {\n Ordering::Relaxed\n };\n\n // Update the back index in the destination queue.\n dest.inner.back.store(dest_b, store_order);\n\n // Return with success.\n Steal::Success(unsafe { task.assume_init() })\n }\n}\n\nimpl Clone for Stealer {\n fn clone(&self) -> Self {\n Self {\n inner: self.inner.clone(),\n flavor: self.flavor,\n }\n }\n}\n\nimpl fmt::Debug for Stealer {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Stealer { .. }\")\n }\n}\n\n// Bits indicating the state of a slot:\n// * If a task has been written into the slot, `WRITE` is set.\n// * If a task has been read from the slot, `READ` is set.\n// * If the block is being destroyed, `DESTROY` is set.\nconst WRITE: usize = 1;\nconst READ: usize = 2;\nconst DESTROY: usize = 4;\n\n// Each block covers one \"lap\" of indices.\nconst LAP: usize = 64;\n// The maximum number of values a block can hold.\nconst BLOCK_CAP: usize = LAP - 1;\n// How many lower bits are reserved for metadata.\nconst SHIFT: usize = 1;\n// Indicates that the block is not the last one.\nconst HAS_NEXT: usize = 1;\n\n/// A slot in a block.\nstruct Slot {\n /// The task.\n task: UnsafeCell>,\n\n /// The state of the slot.\n state: AtomicUsize,\n}\n\nimpl Slot {\n /// Waits until a task is written into the slot.\n fn wait_write(&self) {\n let backoff = Backoff::new();\n while self.state.load(Ordering::Acquire) & WRITE == 0 {\n backoff.snooze();\n }\n }\n}\n\n/// A block in a linked list.\n///\n/// Each block in the list can hold up to `BLOCK_CAP` values.\nstruct Block {\n /// The next block in the linked list.\n next: AtomicPtr>,\n\n /// Slots for values.\n slots: [Slot; BLOCK_CAP],\n}\n\nimpl Block {\n const LAYOUT: Layout = {\n let layout = Layout::new::();\n assert!(\n layout.size() != 0,\n \"Block should never be zero-sized, as it has an AtomicPtr field\"\n );\n layout\n };\n\n /// Creates an empty block.\n fn new() -> Box {\n // unsafe { Box::new_zeroed().assume_init() } requires Rust 1.92\n match Global.allocate_zeroed(Self::LAYOUT) {\n Some(ptr) => {\n // SAFETY: This is safe because:\n // [1] `Block::next` (AtomicPtr) may be safely zero initialized.\n // [2] `Block::slots` (Array) may be safely zero initialized because of [3, 4].\n // [3] `Slot::task` (UnsafeCell) may be safely zero initialized because it\n // holds a MaybeUninit.\n // [4] `Slot::state` (AtomicUsize) may be safely zero initialized.\n unsafe { Box::from_raw(ptr.as_ptr().cast()) }\n }\n // Handle allocation failure\n None => handle_alloc_error(Self::LAYOUT),\n }\n }\n\n /// Waits until the next pointer is set.\n fn wait_next(&self) -> *mut Self {\n let backoff = Backoff::new();\n loop {\n let next = self.next.load(Ordering::Acquire);\n if !next.is_null() {\n return next;\n }\n backoff.snooze();\n }\n }\n\n /// Sets the `DESTROY` bit in slots starting from `start` and destroys the block.\n unsafe fn destroy(this: *mut Self, count: usize) {\n // It is not necessary to set the `DESTROY` bit in the last slot because that slot has\n // begun destruction of the block.\n for i in (0..count).rev() {\n let slot = unsafe { (*this).slots.get_unchecked(i) };\n\n // Mark the `DESTROY` bit if a thread is still using the slot.\n if slot.state.load(Ordering::Acquire) & READ == 0\n && slot.state.fetch_or(DESTROY, Ordering::AcqRel) & READ == 0\n {\n // If a thread is still using the slot, it will continue destruction of the block.\n return;\n }\n }\n\n // No thread is using the block, now it is safe to destroy it.\n drop(unsafe { Box::from_raw(this) });\n }\n}\n\n/// A position in a queue.\nstruct Position {\n /// The index in the queue.\n index: AtomicUsize,\n\n /// The block in the linked list.\n block: AtomicPtr>,\n}\n\n/// An injector queue.\n///\n/// This is a FIFO queue that can be shared among multiple threads. Task schedulers typically have\n/// a single injector queue, which is the entry point for new tasks.\n///\n/// # Examples\n///\n/// ```\n/// # if option_env!(\"MIRI_FALLIBLE_WEAK_CAS\").is_some() { return; } // see ci/miri.sh\n/// use crossbeam_deque::{Injector, Steal};\n///\n/// let q = Injector::new();\n/// q.push(1);\n/// q.push(2);\n///\n/// assert_eq!(q.steal(), Steal::Success(1));\n/// assert_eq!(q.steal(), Steal::Success(2));\n/// assert_eq!(q.steal(), Steal::Empty);\n/// ```\npub struct Injector {\n /// The head of the queue.\n head: CachePadded>,\n\n /// The tail of the queue.\n tail: CachePadded>,\n\n /// Indicates that dropping a `Injector` may drop values of type `T`.\n _marker: PhantomData,\n}\n\nunsafe impl Send for Injector {}\nunsafe impl Sync for Injector {}\n\nimpl Default for Injector {\n fn default() -> Self {\n let block = Box::into_raw(Block::::new());\n Self {\n head: CachePadded::new(Position {\n block: AtomicPtr::new(block),\n index: AtomicUsize::new(0),\n }),\n tail: CachePadded::new(Position {\n block: AtomicPtr::new(block),\n index: AtomicUsize::new(0),\n }),\n _marker: PhantomData,\n }\n }\n}\n\nimpl Injector {\n /// Creates a new injector queue.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Injector;\n ///\n /// let q = Injector::::new();\n /// ```\n pub fn new() -> Self {\n Self::default()\n }\n\n /// Pushes a task into the queue.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Injector;\n ///\n /// let w = Injector::new();\n /// w.push(1);\n /// w.push(2);\n /// ```\n pub fn push(&self, task: T) {\n let backoff = Backoff::new();\n let mut tail = self.tail.index.load(Ordering::Acquire);\n let mut block = self.tail.block.load(Ordering::Acquire);\n let mut next_block = None;\n\n loop {\n // Calculate the offset of the index into the block.\n let offset = (tail >> SHIFT) % LAP;\n\n // If we reached the end of the block, wait until the next one is installed.\n if offset == BLOCK_CAP {\n backoff.snooze();\n tail = self.tail.index.load(Ordering::Acquire);\n block = self.tail.block.load(Ordering::Acquire);\n continue;\n }\n\n // If we're going to have to install the next block, allocate it in advance in order to\n // make the wait for other threads as short as possible.\n if offset + 1 == BLOCK_CAP && next_block.is_none() {\n next_block = Some(Block::::new());\n }\n\n let new_tail = tail + (1 << SHIFT);\n\n // Try advancing the tail forward.\n match self.tail.index.compare_exchange_weak(\n tail,\n new_tail,\n Ordering::SeqCst,\n Ordering::Acquire,\n ) {\n Ok(_) => unsafe {\n // If we've reached the end of the block, install the next one.\n if offset + 1 == BLOCK_CAP {\n let next_block = Box::into_raw(next_block.unwrap());\n let next_index = new_tail.wrapping_add(1 << SHIFT);\n\n self.tail.block.store(next_block, Ordering::Release);\n self.tail.index.store(next_index, Ordering::Release);\n (*block).next.store(next_block, Ordering::Release);\n }\n\n // Write the task into the slot.\n let slot = (*block).slots.get_unchecked(offset);\n slot.task.get().write(MaybeUninit::new(task));\n slot.state.fetch_or(WRITE, Ordering::Release);\n\n return;\n },\n Err(t) => {\n tail = t;\n block = self.tail.block.load(Ordering::Acquire);\n backoff.spin();\n }\n }\n }\n }\n\n /// Steals a task from the queue.\n ///\n /// # Examples\n ///\n /// ```\n /// # if option_env!(\"MIRI_FALLIBLE_WEAK_CAS\").is_some() { return; } // see ci/miri.sh\n /// use crossbeam_deque::{Injector, Steal};\n ///\n /// let q = Injector::new();\n /// q.push(1);\n /// q.push(2);\n ///\n /// assert_eq!(q.steal(), Steal::Success(1));\n /// assert_eq!(q.steal(), Steal::Success(2));\n /// assert_eq!(q.steal(), Steal::Empty);\n /// ```\n pub fn steal(&self) -> Steal {\n let mut head;\n let mut block;\n let mut offset;\n\n let backoff = Backoff::new();\n loop {\n head = self.head.index.load(Ordering::Acquire);\n block = self.head.block.load(Ordering::Acquire);\n\n // Calculate the offset of the index into the block.\n offset = (head >> SHIFT) % LAP;\n\n // If we reached the end of the block, wait until the next one is installed.\n if offset == BLOCK_CAP {\n backoff.snooze();\n } else {\n break;\n }\n }\n\n let mut new_head = head + (1 << SHIFT);\n\n if new_head & HAS_NEXT == 0 {\n atomic::fence(Ordering::SeqCst);\n let tail = self.tail.index.load(Ordering::Relaxed);\n\n // If the tail equals the head, that means the queue is empty.\n if head >> SHIFT == tail >> SHIFT {\n return Steal::Empty;\n }\n\n // If head and tail are not in the same block, set `HAS_NEXT` in head.\n if (head >> SHIFT) / LAP != (tail >> SHIFT) / LAP {\n new_head |= HAS_NEXT;\n }\n }\n\n // Try moving the head index forward.\n if self\n .head\n .index\n .compare_exchange_weak(head, new_head, Ordering::SeqCst, Ordering::Acquire)\n .is_err()\n {\n return Steal::Retry;\n }\n\n unsafe {\n // If we've reached the end of the block, move to the next one.\n if offset + 1 == BLOCK_CAP {\n let next = (*block).wait_next();\n let mut next_index = (new_head & !HAS_NEXT).wrapping_add(1 << SHIFT);\n if !(*next).next.load(Ordering::Relaxed).is_null() {\n next_index |= HAS_NEXT;\n }\n\n self.head.block.store(next, Ordering::Release);\n self.head.index.store(next_index, Ordering::Release);\n }\n\n // Read the task.\n let slot = (*block).slots.get_unchecked(offset);\n slot.wait_write();\n let task = slot.task.get().read().assume_init();\n\n // Destroy the block if we've reached the end, or if another thread wanted to destroy\n // but couldn't because we were busy reading from the slot.\n if (offset + 1 == BLOCK_CAP)\n || (slot.state.fetch_or(READ, Ordering::AcqRel) & DESTROY != 0)\n {\n Block::destroy(block, offset);\n }\n\n Steal::Success(task)\n }\n }\n\n /// Steals a batch of tasks and pushes them into a worker.\n ///\n /// How many tasks exactly will be stolen is not specified. That said, this method will try to\n /// steal around half of the tasks in the queue, but also not more than some constant limit.\n ///\n /// # Examples\n ///\n /// ```\n /// # if option_env!(\"MIRI_FALLIBLE_WEAK_CAS\").is_some() { return; } // see ci/miri.sh\n /// use crossbeam_deque::{Injector, Worker};\n ///\n /// let q = Injector::new();\n /// q.push(1);\n /// q.push(2);\n /// q.push(3);\n /// q.push(4);\n ///\n /// let w = Worker::new_fifo();\n /// let _ = q.steal_batch(&w);\n /// assert_eq!(w.pop(), Some(1));\n /// assert_eq!(w.pop(), Some(2));\n /// ```\n pub fn steal_batch(&self, dest: &Worker) -> Steal<()> {\n self.steal_batch_with_limit(dest, MAX_BATCH)\n }\n\n /// Steals no more than `limit` of tasks and pushes them into a worker.\n ///\n /// How many tasks exactly will be stolen is not specified. That said, this method will try to\n /// steal around half of the tasks in the queue, but also not more than some constant limit.\n ///\n /// # Examples\n ///\n /// ```\n /// # if option_env!(\"MIRI_FALLIBLE_WEAK_CAS\").is_some() { return; } // see ci/miri.sh\n /// use crossbeam_deque::{Injector, Worker};\n ///\n /// let q = Injector::new();\n /// q.push(1);\n /// q.push(2);\n /// q.push(3);\n /// q.push(4);\n /// q.push(5);\n /// q.push(6);\n ///\n /// let w = Worker::new_fifo();\n /// let _ = q.steal_batch_with_limit(&w, 2);\n /// assert_eq!(w.pop(), Some(1));\n /// assert_eq!(w.pop(), Some(2));\n /// assert_eq!(w.pop(), None);\n ///\n /// q.push(7);\n /// q.push(8);\n /// // Setting a large limit does not guarantee that all elements will be popped. In this case,\n /// // half of the elements are currently popped, but the number of popped elements is considered\n /// // an implementation detail that may be changed in the future.\n /// let _ = q.steal_batch_with_limit(&w, usize::MAX);\n /// assert_eq!(w.len(), 3);\n /// ```\n pub fn steal_batch_with_limit(&self, dest: &Worker, limit: usize) -> Steal<()> {\n assert!(limit > 0);\n let mut head;\n let mut block;\n let mut offset;\n\n let backoff = Backoff::new();\n loop {\n head = self.head.index.load(Ordering::Acquire);\n block = self.head.block.load(Ordering::Acquire);\n\n // Calculate the offset of the index into the block.\n offset = (head >> SHIFT) % LAP;\n\n // If we reached the end of the block, wait until the next one is installed.\n if offset == BLOCK_CAP {\n backoff.snooze();\n } else {\n break;\n }\n }\n\n let mut new_head = head;\n let advance;\n\n if new_head & HAS_NEXT == 0 {\n atomic::fence(Ordering::SeqCst);\n let tail = self.tail.index.load(Ordering::Relaxed);\n\n // If the tail equals the head, that means the queue is empty.\n if head >> SHIFT == tail >> SHIFT {\n return Steal::Empty;\n }\n\n // If head and tail are not in the same block, set `HAS_NEXT` in head. Also, calculate\n // the right batch size to steal.\n if (head >> SHIFT) / LAP != (tail >> SHIFT) / LAP {\n new_head |= HAS_NEXT;\n // We can steal all tasks till the end of the block.\n advance = (BLOCK_CAP - offset).min(limit);\n } else {\n let len = (tail - head) >> SHIFT;\n // Steal half of the available tasks.\n advance = len.div_ceil(2).min(limit);\n }\n } else {\n // We can steal all tasks till the end of the block.\n advance = (BLOCK_CAP - offset).min(limit);\n }\n\n new_head += advance << SHIFT;\n let new_offset = offset + advance;\n\n // Try moving the head index forward.\n if self\n .head\n .index\n .compare_exchange_weak(head, new_head, Ordering::SeqCst, Ordering::Acquire)\n .is_err()\n {\n return Steal::Retry;\n }\n\n // Reserve capacity for the stolen batch.\n let batch_size = new_offset - offset;\n dest.reserve(batch_size);\n\n // Get the destination buffer and back index.\n let dest_buffer = dest.buffer.get();\n let dest_b = dest.inner.back.load(Ordering::Relaxed);\n\n unsafe {\n // If we've reached the end of the block, move to the next one.\n if new_offset == BLOCK_CAP {\n let next = (*block).wait_next();\n let mut next_index = (new_head & !HAS_NEXT).wrapping_add(1 << SHIFT);\n if !(*next).next.load(Ordering::Relaxed).is_null() {\n next_index |= HAS_NEXT;\n }\n\n self.head.block.store(next, Ordering::Release);\n self.head.index.store(next_index, Ordering::Release);\n }\n\n // Copy values from the injector into the destination queue.\n match dest.flavor {\n Flavor::Fifo => {\n for i in 0..batch_size {\n // Read the task.\n let slot = (*block).slots.get_unchecked(offset + i);\n slot.wait_write();\n let task = slot.task.get().read();\n\n // Write it into the destination queue.\n dest_buffer.write(dest_b.wrapping_add(i as isize), task);\n }\n }\n\n Flavor::Lifo => {\n for i in 0..batch_size {\n // Read the task.\n let slot = (*block).slots.get_unchecked(offset + i);\n slot.wait_write();\n let task = slot.task.get().read();\n\n // Write it into the destination queue.\n dest_buffer.write(dest_b.wrapping_add((batch_size - 1 - i) as isize), task);\n }\n }\n }\n\n // ThreadSanitizer does not understand fences, so we omit fence and do store with Release ordering.\n #[cfg(not(crossbeam_sanitize_thread))]\n atomic::fence(Ordering::Release);\n let store_order = if cfg!(crossbeam_sanitize_thread) {\n Ordering::Release\n } else {\n Ordering::Relaxed\n };\n\n // Update the back index in the destination queue.\n dest.inner\n .back\n .store(dest_b.wrapping_add(batch_size as isize), store_order);\n\n // Destroy the block if we've reached the end, or if another thread wanted to destroy\n // but couldn't because we were busy reading from the slot.\n if new_offset == BLOCK_CAP {\n Block::destroy(block, offset);\n } else {\n for i in offset..new_offset {\n let slot = (*block).slots.get_unchecked(i);\n\n if slot.state.fetch_or(READ, Ordering::AcqRel) & DESTROY != 0 {\n Block::destroy(block, offset);\n break;\n }\n }\n }\n\n Steal::Success(())\n }\n }\n\n /// Steals a batch of tasks, pushes them into a worker, and pops a task from that worker.\n ///\n /// How many tasks exactly will be stolen is not specified. That said, this method will try to\n /// steal around half of the tasks in the queue, but also not more than some constant limit.\n ///\n /// # Examples\n ///\n /// ```\n /// # if option_env!(\"MIRI_FALLIBLE_WEAK_CAS\").is_some() { return; } // see ci/miri.sh\n /// use crossbeam_deque::{Injector, Steal, Worker};\n ///\n /// let q = Injector::new();\n /// q.push(1);\n /// q.push(2);\n /// q.push(3);\n /// q.push(4);\n ///\n /// let w = Worker::new_fifo();\n /// assert_eq!(q.steal_batch_and_pop(&w), Steal::Success(1));\n /// assert_eq!(w.pop(), Some(2));\n /// ```\n pub fn steal_batch_and_pop(&self, dest: &Worker) -> Steal {\n // TODO: we use `MAX_BATCH + 1` as the hard limit for Injecter as the performance is slightly\n // better, but we may change it in the future to be compatible with the same method in Stealer.\n self.steal_batch_with_limit_and_pop(dest, MAX_BATCH + 1)\n }\n\n /// Steals no more than `limit` of tasks, pushes them into a worker, and pops a task from that worker.\n ///\n /// How many tasks exactly will be stolen is not specified. That said, this method will try to\n /// steal around half of the tasks in the queue, but also not more than the given limit.\n ///\n /// # Examples\n ///\n /// ```\n /// # if option_env!(\"MIRI_FALLIBLE_WEAK_CAS\").is_some() { return; } // see ci/miri.sh\n /// use crossbeam_deque::{Injector, Steal, Worker};\n ///\n /// let q = Injector::new();\n /// q.push(1);\n /// q.push(2);\n /// q.push(3);\n /// q.push(4);\n /// q.push(5);\n /// q.push(6);\n ///\n /// let w = Worker::new_fifo();\n /// assert_eq!(q.steal_batch_with_limit_and_pop(&w, 2), Steal::Success(1));\n /// assert_eq!(w.pop(), Some(2));\n /// assert_eq!(w.pop(), None);\n ///\n /// q.push(7);\n /// // Setting a large limit does not guarantee that all elements will be popped. In this case,\n /// // half of the elements are currently popped, but the number of popped elements is considered\n /// // an implementation detail that may be changed in the future.\n /// assert_eq!(q.steal_batch_with_limit_and_pop(&w, usize::MAX), Steal::Success(3));\n /// assert_eq!(w.pop(), Some(4));\n /// assert_eq!(w.pop(), Some(5));\n /// assert_eq!(w.pop(), None);\n /// ```\n pub fn steal_batch_with_limit_and_pop(&self, dest: &Worker, limit: usize) -> Steal {\n assert!(limit > 0);\n let mut head;\n let mut block;\n let mut offset;\n\n let backoff = Backoff::new();\n loop {\n head = self.head.index.load(Ordering::Acquire);\n block = self.head.block.load(Ordering::Acquire);\n\n // Calculate the offset of the index into the block.\n offset = (head >> SHIFT) % LAP;\n\n // If we reached the end of the block, wait until the next one is installed.\n if offset == BLOCK_CAP {\n backoff.snooze();\n } else {\n break;\n }\n }\n\n let mut new_head = head;\n let advance;\n\n if new_head & HAS_NEXT == 0 {\n atomic::fence(Ordering::SeqCst);\n let tail = self.tail.index.load(Ordering::Relaxed);\n\n // If the tail equals the head, that means the queue is empty.\n if head >> SHIFT == tail >> SHIFT {\n return Steal::Empty;\n }\n\n // If head and tail are not in the same block, set `HAS_NEXT` in head.\n if (head >> SHIFT) / LAP != (tail >> SHIFT) / LAP {\n new_head |= HAS_NEXT;\n // We can steal all tasks till the end of the block.\n advance = (BLOCK_CAP - offset).min(limit);\n } else {\n let len = (tail - head) >> SHIFT;\n // Steal half of the available tasks.\n advance = len.div_ceil(2).min(limit);\n }\n } else {\n // We can steal all tasks till the end of the block.\n advance = (BLOCK_CAP - offset).min(limit);\n }\n\n new_head += advance << SHIFT;\n let new_offset = offset + advance;\n\n // Try moving the head index forward.\n if self\n .head\n .index\n .compare_exchange_weak(head, new_head, Ordering::SeqCst, Ordering::Acquire)\n .is_err()\n {\n return Steal::Retry;\n }\n\n // Reserve capacity for the stolen batch.\n let batch_size = new_offset - offset - 1;\n dest.reserve(batch_size);\n\n // Get the destination buffer and back index.\n let dest_buffer = dest.buffer.get();\n let dest_b = dest.inner.back.load(Ordering::Relaxed);\n\n unsafe {\n // If we've reached the end of the block, move to the next one.\n if new_offset == BLOCK_CAP {\n let next = (*block).wait_next();\n let mut next_index = (new_head & !HAS_NEXT).wrapping_add(1 << SHIFT);\n if !(*next).next.load(Ordering::Relaxed).is_null() {\n next_index |= HAS_NEXT;\n }\n\n self.head.block.store(next, Ordering::Release);\n self.head.index.store(next_index, Ordering::Release);\n }\n\n // Read the task.\n let slot = (*block).slots.get_unchecked(offset);\n slot.wait_write();\n let task = slot.task.get().read();\n\n match dest.flavor {\n Flavor::Fifo => {\n // Copy values from the injector into the destination queue.\n for i in 0..batch_size {\n // Read the task.\n let slot = (*block).slots.get_unchecked(offset + i + 1);\n slot.wait_write();\n let task = slot.task.get().read();\n\n // Write it into the destination queue.\n dest_buffer.write(dest_b.wrapping_add(i as isize), task);\n }\n }\n\n Flavor::Lifo => {\n // Copy values from the injector into the destination queue.\n for i in 0..batch_size {\n // Read the task.\n let slot = (*block).slots.get_unchecked(offset + i + 1);\n slot.wait_write();\n let task = slot.task.get().read();\n\n // Write it into the destination queue.\n dest_buffer.write(dest_b.wrapping_add((batch_size - 1 - i) as isize), task);\n }\n }\n }\n\n // ThreadSanitizer does not understand fences, so we omit fence and do store with Release ordering.\n #[cfg(not(crossbeam_sanitize_thread))]\n atomic::fence(Ordering::Release);\n let store_order = if cfg!(crossbeam_sanitize_thread) {\n Ordering::Release\n } else {\n Ordering::Relaxed\n };\n\n // Update the back index in the destination queue.\n dest.inner\n .back\n .store(dest_b.wrapping_add(batch_size as isize), store_order);\n\n // Destroy the block if we've reached the end, or if another thread wanted to destroy\n // but couldn't because we were busy reading from the slot.\n if new_offset == BLOCK_CAP {\n Block::destroy(block, offset);\n } else {\n for i in offset..new_offset {\n let slot = (*block).slots.get_unchecked(i);\n\n if slot.state.fetch_or(READ, Ordering::AcqRel) & DESTROY != 0 {\n Block::destroy(block, offset);\n break;\n }\n }\n }\n\n Steal::Success(task.assume_init())\n }\n }\n\n /// Returns `true` if the queue is empty.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Injector;\n ///\n /// let q = Injector::new();\n ///\n /// assert!(q.is_empty());\n /// q.push(1);\n /// assert!(!q.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n let head = self.head.index.load(Ordering::SeqCst);\n let tail = self.tail.index.load(Ordering::SeqCst);\n head >> SHIFT == tail >> SHIFT\n }\n\n /// Returns the number of tasks in the queue.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Injector;\n ///\n /// let q = Injector::new();\n ///\n /// assert_eq!(q.len(), 0);\n /// q.push(1);\n /// assert_eq!(q.len(), 1);\n /// q.push(1);\n /// assert_eq!(q.len(), 2);\n /// ```\n pub fn len(&self) -> usize {\n loop {\n // Load the tail index, then load the head index.\n let mut tail = self.tail.index.load(Ordering::SeqCst);\n let mut head = self.head.index.load(Ordering::SeqCst);\n\n // If the tail index didn't change, we've got consistent indices to work with.\n if self.tail.index.load(Ordering::SeqCst) == tail {\n // Erase the lower bits.\n tail &= !((1 << SHIFT) - 1);\n head &= !((1 << SHIFT) - 1);\n\n // Fix up indices if they fall onto block ends.\n if (tail >> SHIFT) & (LAP - 1) == LAP - 1 {\n tail = tail.wrapping_add(1 << SHIFT);\n }\n if (head >> SHIFT) & (LAP - 1) == LAP - 1 {\n head = head.wrapping_add(1 << SHIFT);\n }\n\n // Rotate indices so that head falls into the first block.\n let lap = (head >> SHIFT) / LAP;\n tail = tail.wrapping_sub((lap * LAP) << SHIFT);\n head = head.wrapping_sub((lap * LAP) << SHIFT);\n\n // Remove the lower bits.\n tail >>= SHIFT;\n head >>= SHIFT;\n\n // Return the difference minus the number of blocks between tail and head.\n return tail - head - tail / LAP;\n }\n }\n }\n}\n\nimpl Drop for Injector {\n fn drop(&mut self) {\n let mut head = *self.head.index.get_mut();\n let mut tail = *self.tail.index.get_mut();\n let mut block = *self.head.block.get_mut();\n\n // Erase the lower bits.\n head &= !((1 << SHIFT) - 1);\n tail &= !((1 << SHIFT) - 1);\n\n unsafe {\n // Drop all values between `head` and `tail` and deallocate the heap-allocated blocks.\n while head != tail {\n let offset = (head >> SHIFT) % LAP;\n\n if offset < BLOCK_CAP {\n // Drop the task in the slot.\n let slot = (*block).slots.get_unchecked(offset);\n (*slot.task.get()).assume_init_drop();\n } else {\n // Deallocate the block and move to the next one.\n let next = *(*block).next.get_mut();\n drop(Box::from_raw(block));\n block = next;\n }\n\n head = head.wrapping_add(1 << SHIFT);\n }\n\n // Deallocate the last remaining block.\n drop(Box::from_raw(block));\n }\n }\n}\n\nimpl fmt::Debug for Injector {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Worker { .. }\")\n }\n}\n\n/// Possible outcomes of a steal operation.\n///\n/// # Examples\n///\n/// There are lots of ways to chain results of steal operations together:\n///\n/// ```\n/// use crossbeam_deque::Steal::{self, Empty, Retry, Success};\n///\n/// let collect = |v: Vec>| v.into_iter().collect::>();\n///\n/// assert_eq!(collect(vec![Empty, Empty, Empty]), Empty);\n/// assert_eq!(collect(vec![Empty, Retry, Empty]), Retry);\n/// assert_eq!(collect(vec![Retry, Success(1), Empty]), Success(1));\n///\n/// assert_eq!(collect(vec![Empty, Empty]).or_else(|| Retry), Retry);\n/// assert_eq!(collect(vec![Retry, Empty]).or_else(|| Success(1)), Success(1));\n/// ```\n#[must_use]\n#[derive(PartialEq, Eq, Copy, Clone)]\npub enum Steal {\n /// The queue was empty at the time of stealing.\n Empty,\n\n /// At least one task was successfully stolen.\n Success(T),\n\n /// The steal operation needs to be retried.\n Retry,\n}\n\nimpl Steal {\n /// Returns `true` if the queue was empty at the time of stealing.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Steal::{Empty, Retry, Success};\n ///\n /// assert!(!Success(7).is_empty());\n /// assert!(!Retry::.is_empty());\n ///\n /// assert!(Empty::.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n matches!(self, Self::Empty)\n }\n\n /// Returns `true` if at least one task was stolen.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Steal::{Empty, Retry, Success};\n ///\n /// assert!(!Empty::.is_success());\n /// assert!(!Retry::.is_success());\n ///\n /// assert!(Success(7).is_success());\n /// ```\n pub fn is_success(&self) -> bool {\n matches!(self, Self::Success(_))\n }\n\n /// Returns `true` if the steal operation needs to be retried.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Steal::{Empty, Retry, Success};\n ///\n /// assert!(!Empty::.is_retry());\n /// assert!(!Success(7).is_retry());\n ///\n /// assert!(Retry::.is_retry());\n /// ```\n pub fn is_retry(&self) -> bool {\n matches!(self, Self::Retry)\n }\n\n /// Returns the result of the operation, if successful.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Steal::{Empty, Retry, Success};\n ///\n /// assert_eq!(Empty::.success(), None);\n /// assert_eq!(Retry::.success(), None);\n ///\n /// assert_eq!(Success(7).success(), Some(7));\n /// ```\n pub fn success(self) -> Option {\n match self {\n Self::Success(res) => Some(res),\n _ => None,\n }\n }\n\n /// If no task was stolen, attempts another steal operation.\n ///\n /// Returns this steal result if it is `Success`. Otherwise, closure `f` is invoked and then:\n ///\n /// * If the second steal resulted in `Success`, it is returned.\n /// * If both steals were unsuccessful but any resulted in `Retry`, then `Retry` is returned.\n /// * If both resulted in `None`, then `None` is returned.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_deque::Steal::{Empty, Retry, Success};\n ///\n /// assert_eq!(Success(1).or_else(|| Success(2)), Success(1));\n /// assert_eq!(Retry.or_else(|| Success(2)), Success(2));\n ///\n /// assert_eq!(Retry.or_else(|| Empty), Retry::);\n /// assert_eq!(Empty.or_else(|| Retry), Retry::);\n ///\n /// assert_eq!(Empty.or_else(|| Empty), Empty::);\n /// ```\n pub fn or_else(self, f: F) -> Self\n where\n F: FnOnce() -> Self,\n {\n match self {\n Self::Empty => f(),\n Self::Success(_) => self,\n Self::Retry => {\n if let Self::Success(res) = f() {\n Self::Success(res)\n } else {\n Self::Retry\n }\n }\n }\n }\n}\n\nimpl fmt::Debug for Steal {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match self {\n Self::Empty => f.pad(\"Empty\"),\n Self::Success(_) => f.pad(\"Success(..)\"),\n Self::Retry => f.pad(\"Retry\"),\n }\n }\n}\n\nimpl FromIterator for Steal {\n /// Consumes items until a `Success` is found and returns it.\n ///\n /// If no `Success` was found, but there was at least one `Retry`, then returns `Retry`.\n /// Otherwise, `Empty` is returned.\n fn from_iter(iter: I) -> Self\n where\n I: IntoIterator,\n {\n let mut retry = false;\n for s in iter {\n match &s {\n Self::Empty => {}\n Self::Success(_) => return s,\n Self::Retry => retry = true,\n }\n }\n\n if retry { Self::Retry } else { Self::Empty }\n }\n}\n" }, { "path": "crossbeam-deque/src/lib.rs", "content": "//! Concurrent work-stealing deques.\n//!\n//! These data structures are most commonly used in work-stealing schedulers. The typical setup\n//! involves a number of threads, each having its own FIFO or LIFO queue (*worker*). There is also\n//! one global FIFO queue (*injector*) and a list of references to *worker* queues that are able to\n//! steal tasks (*stealers*).\n//!\n//! We spawn a new task onto the scheduler by pushing it into the *injector* queue. Each worker\n//! thread waits in a loop until it finds the next task to run and then runs it. To find a task, it\n//! first looks into its local *worker* queue, and then into the *injector* and *stealers*.\n//!\n//! # Queues\n//!\n//! [`Injector`] is a FIFO queue, where tasks are pushed and stolen from opposite ends. It is\n//! shared among threads and is usually the entry point for new tasks.\n//!\n//! [`Worker`] has two constructors:\n//!\n//! * [`new_fifo()`] - Creates a FIFO queue, in which tasks are pushed and popped from opposite\n//! ends.\n//! * [`new_lifo()`] - Creates a LIFO queue, in which tasks are pushed and popped from the same\n//! end.\n//!\n//! Each [`Worker`] is owned by a single thread and supports only push and pop operations.\n//!\n//! Method [`stealer()`] creates a [`Stealer`] that may be shared among threads and can only steal\n//! tasks from its [`Worker`]. Tasks are stolen from the end opposite to where they get pushed.\n//!\n//! # Stealing\n//!\n//! Steal operations come in three flavors:\n//!\n//! 1. [`steal()`] - Steals one task.\n//! 2. [`steal_batch()`] - Steals a batch of tasks and moves them into another worker.\n//! 3. [`steal_batch_and_pop()`] - Steals a batch of tasks, moves them into another queue, and pops\n//! one task from that worker.\n//!\n//! In contrast to push and pop operations, stealing can spuriously fail with [`Steal::Retry`], in\n//! which case the steal operation needs to be retried.\n//!\n//! # Examples\n//!\n//! Suppose a thread in a work-stealing scheduler is idle and looking for the next task to run. To\n//! find an available task, it might do the following:\n//!\n//! 1. Try popping one task from the local worker queue.\n//! 2. Try stealing a batch of tasks from the global injector queue.\n//! 3. Try stealing one task from another thread using the stealer list.\n//!\n//! An implementation of this work-stealing strategy:\n//!\n//! ```\n//! use crossbeam_deque::{Injector, Stealer, Worker};\n//! use std::iter;\n//!\n//! fn find_task(\n//! local: &Worker,\n//! global: &Injector,\n//! stealers: &[Stealer],\n//! ) -> Option {\n//! // Pop a task from the local queue, if not empty.\n//! local.pop().or_else(|| {\n//! // Otherwise, we need to look for a task elsewhere.\n//! iter::repeat_with(|| {\n//! // Try stealing a batch of tasks from the global queue.\n//! global.steal_batch_and_pop(local)\n//! // Or try stealing a task from one of the other threads.\n//! .or_else(|| stealers.iter().map(|s| s.steal()).collect())\n//! })\n//! // Loop while no task was stolen and any steal operation needs to be retried.\n//! .find(|s| !s.is_retry())\n//! // Extract the stolen task, if there is one.\n//! .and_then(|s| s.success())\n//! })\n//! }\n//! ```\n//!\n//! [`new_fifo()`]: Worker::new_fifo\n//! [`new_lifo()`]: Worker::new_lifo\n//! [`stealer()`]: Worker::stealer\n//! [`steal()`]: Stealer::steal\n//! [`steal_batch()`]: Stealer::steal_batch\n//! [`steal_batch_and_pop()`]: Stealer::steal_batch_and_pop\n\n#![no_std]\n#![doc(test(\n no_crate_inject,\n attr(allow(dead_code, unused_assignments, unused_variables))\n))]\n#![warn(\n missing_docs,\n unsafe_op_in_unsafe_fn,\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\n\n#[cfg(feature = \"std\")]\nextern crate alloc;\n#[cfg(feature = \"std\")]\nextern crate std;\n\n#[cfg(feature = \"std\")]\nmod alloc_helper;\n\n#[cfg(feature = \"std\")]\nmod deque;\n#[cfg(feature = \"std\")]\npub use crate::deque::{Injector, Steal, Stealer, Worker};\n" }, { "path": "crossbeam-deque/tests/fifo.rs", "content": "use std::sync::{\n Arc, Mutex,\n atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst},\n};\n\nuse crossbeam_deque::{\n Steal::{Empty, Success},\n Worker,\n};\nuse crossbeam_utils::thread::scope;\n\n#[test]\nfn smoke() {\n let w = Worker::new_fifo();\n let s = w.stealer();\n assert_eq!(w.pop(), None);\n assert_eq!(s.steal(), Empty);\n\n w.push(1);\n assert_eq!(w.pop(), Some(1));\n assert_eq!(w.pop(), None);\n assert_eq!(s.steal(), Empty);\n\n w.push(2);\n assert_eq!(s.steal(), Success(2));\n assert_eq!(s.steal(), Empty);\n assert_eq!(w.pop(), None);\n\n w.push(3);\n w.push(4);\n w.push(5);\n assert_eq!(s.steal(), Success(3));\n assert_eq!(s.steal(), Success(4));\n assert_eq!(s.steal(), Success(5));\n assert_eq!(s.steal(), Empty);\n\n w.push(6);\n w.push(7);\n w.push(8);\n w.push(9);\n assert_eq!(w.pop(), Some(6));\n assert_eq!(s.steal(), Success(7));\n assert_eq!(w.pop(), Some(8));\n assert_eq!(w.pop(), Some(9));\n assert_eq!(w.pop(), None);\n}\n\n#[test]\nfn is_empty() {\n let w = Worker::new_fifo();\n let s = w.stealer();\n\n assert!(w.is_empty());\n w.push(1);\n assert!(!w.is_empty());\n w.push(2);\n assert!(!w.is_empty());\n let _ = w.pop();\n assert!(!w.is_empty());\n let _ = w.pop();\n assert!(w.is_empty());\n\n assert!(s.is_empty());\n w.push(1);\n assert!(!s.is_empty());\n w.push(2);\n assert!(!s.is_empty());\n let _ = s.steal();\n assert!(!s.is_empty());\n let _ = s.steal();\n assert!(s.is_empty());\n}\n\n#[test]\nfn spsc() {\n const STEPS: usize = if cfg!(miri) { 500 } else { 50_000 };\n\n let w = Worker::new_fifo();\n let s = w.stealer();\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..STEPS {\n loop {\n if let Success(v) = s.steal() {\n assert_eq!(i, v);\n break;\n }\n }\n }\n\n assert_eq!(s.steal(), Empty);\n });\n\n for i in 0..STEPS {\n w.push(i);\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stampede() {\n const THREADS: usize = 8;\n const COUNT: usize = if cfg!(miri) { 500 } else { 50_000 };\n\n let w = Worker::new_fifo();\n\n for i in 0..COUNT {\n w.push(Box::new(i + 1));\n }\n let remaining = Arc::new(AtomicUsize::new(COUNT));\n\n scope(|scope| {\n for _ in 0..THREADS {\n let s = w.stealer();\n let remaining = remaining.clone();\n\n scope.spawn(move |_| {\n let mut last = 0;\n while remaining.load(SeqCst) > 0 {\n if let Success(x) = s.steal() {\n assert!(last < *x);\n last = *x;\n remaining.fetch_sub(1, SeqCst);\n }\n }\n });\n }\n\n let mut last = 0;\n while remaining.load(SeqCst) > 0 {\n if let Some(x) = w.pop() {\n assert!(last < *x);\n last = *x;\n remaining.fetch_sub(1, SeqCst);\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress() {\n const THREADS: usize = 8;\n const COUNT: usize = if cfg!(miri) { 500 } else { 50_000 };\n\n let w = Worker::new_fifo();\n let done = Arc::new(AtomicBool::new(false));\n let hits = Arc::new(AtomicUsize::new(0));\n\n scope(|scope| {\n for _ in 0..THREADS {\n let s = w.stealer();\n let done = done.clone();\n let hits = hits.clone();\n\n scope.spawn(move |_| {\n let w2 = Worker::new_fifo();\n\n while !done.load(SeqCst) {\n if let Success(_) = s.steal() {\n hits.fetch_add(1, SeqCst);\n }\n\n let _ = s.steal_batch(&w2);\n\n if let Success(_) = s.steal_batch_and_pop(&w2) {\n hits.fetch_add(1, SeqCst);\n }\n\n while w2.pop().is_some() {\n hits.fetch_add(1, SeqCst);\n }\n }\n });\n }\n\n let mut rng = fastrand::Rng::new();\n let mut expected = 0;\n while expected < COUNT {\n if rng.u8(0..3) == 0 {\n while w.pop().is_some() {\n hits.fetch_add(1, SeqCst);\n }\n } else {\n w.push(expected);\n expected += 1;\n }\n }\n\n while hits.load(SeqCst) < COUNT {\n while w.pop().is_some() {\n hits.fetch_add(1, SeqCst);\n }\n }\n done.store(true, SeqCst);\n })\n .unwrap();\n}\n\n#[cfg_attr(miri, ignore)] // Miri is too slow\n#[test]\nfn no_starvation() {\n const THREADS: usize = 8;\n const COUNT: usize = 50_000;\n\n let w = Worker::new_fifo();\n let done = Arc::new(AtomicBool::new(false));\n let mut all_hits = Vec::new();\n\n scope(|scope| {\n for _ in 0..THREADS {\n let s = w.stealer();\n let done = done.clone();\n let hits = Arc::new(AtomicUsize::new(0));\n all_hits.push(hits.clone());\n\n scope.spawn(move |_| {\n let w2 = Worker::new_fifo();\n\n while !done.load(SeqCst) {\n if let Success(_) = s.steal() {\n hits.fetch_add(1, SeqCst);\n }\n\n let _ = s.steal_batch(&w2);\n\n if let Success(_) = s.steal_batch_and_pop(&w2) {\n hits.fetch_add(1, SeqCst);\n }\n\n while w2.pop().is_some() {\n hits.fetch_add(1, SeqCst);\n }\n }\n });\n }\n\n let mut rng = fastrand::Rng::new();\n let mut my_hits = 0;\n loop {\n for i in 0..rng.usize(0..COUNT) {\n if rng.u8(0..3) == 0 && my_hits == 0 {\n while w.pop().is_some() {\n my_hits += 1;\n }\n } else {\n w.push(i);\n }\n }\n\n if my_hits > 0 && all_hits.iter().all(|h| h.load(SeqCst) > 0) {\n break;\n }\n }\n done.store(true, SeqCst);\n })\n .unwrap();\n}\n\n#[test]\nfn destructors() {\n const THREADS: usize = if cfg!(miri) { 2 } else { 8 };\n const COUNT: usize = if cfg!(miri) { 500 } else { 50_000 };\n const STEPS: usize = if cfg!(miri) { 100 } else { 1000 };\n\n struct Elem(usize, Arc>>);\n\n impl Drop for Elem {\n fn drop(&mut self) {\n self.1.lock().unwrap().push(self.0);\n }\n }\n\n let w = Worker::new_fifo();\n let dropped = Arc::new(Mutex::new(Vec::new()));\n let remaining = Arc::new(AtomicUsize::new(COUNT));\n\n for i in 0..COUNT {\n w.push(Elem(i, dropped.clone()));\n }\n\n scope(|scope| {\n for _ in 0..THREADS {\n let remaining = remaining.clone();\n let s = w.stealer();\n\n scope.spawn(move |_| {\n let w2 = Worker::new_fifo();\n let mut cnt = 0;\n\n while cnt < STEPS {\n if let Success(_) = s.steal() {\n cnt += 1;\n remaining.fetch_sub(1, SeqCst);\n }\n\n let _ = s.steal_batch(&w2);\n\n if let Success(_) = s.steal_batch_and_pop(&w2) {\n cnt += 1;\n remaining.fetch_sub(1, SeqCst);\n }\n\n while w2.pop().is_some() {\n cnt += 1;\n remaining.fetch_sub(1, SeqCst);\n }\n }\n });\n }\n\n for _ in 0..STEPS {\n if w.pop().is_some() {\n remaining.fetch_sub(1, SeqCst);\n }\n }\n })\n .unwrap();\n\n let rem = remaining.load(SeqCst);\n assert!(rem > 0);\n\n {\n let mut v = dropped.lock().unwrap();\n assert_eq!(v.len(), COUNT - rem);\n v.clear();\n }\n\n drop(w);\n\n {\n let mut v = dropped.lock().unwrap();\n assert_eq!(v.len(), rem);\n v.sort_unstable();\n for pair in v.windows(2) {\n assert_eq!(pair[0] + 1, pair[1]);\n }\n }\n}\n" }, { "path": "crossbeam-deque/tests/injector.rs", "content": "use std::sync::{\n Arc, Mutex,\n atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst},\n};\n\nuse crossbeam_deque::{\n Injector,\n Steal::{self, Empty, Success},\n Worker,\n};\nuse crossbeam_utils::thread::scope;\n\nfn busy_retry(mut f: impl FnMut() -> Steal) -> Steal {\n loop {\n let s = f();\n if !s.is_retry() {\n return s;\n }\n }\n}\n\n#[test]\nfn smoke() {\n let q = Injector::new();\n assert_eq!(busy_retry(|| q.steal()), Empty);\n\n q.push(1);\n q.push(2);\n assert_eq!(busy_retry(|| q.steal()), Success(1));\n assert_eq!(busy_retry(|| q.steal()), Success(2));\n assert_eq!(busy_retry(|| q.steal()), Empty);\n\n q.push(3);\n assert_eq!(busy_retry(|| q.steal()), Success(3));\n assert_eq!(busy_retry(|| q.steal()), Empty);\n}\n\n#[test]\nfn is_empty() {\n let q = Injector::new();\n assert!(q.is_empty());\n\n q.push(1);\n assert!(!q.is_empty());\n q.push(2);\n assert!(!q.is_empty());\n\n let _ = busy_retry(|| q.steal());\n assert!(!q.is_empty());\n let _ = busy_retry(|| q.steal());\n assert!(q.is_empty());\n\n q.push(3);\n assert!(!q.is_empty());\n let _ = busy_retry(|| q.steal());\n assert!(q.is_empty());\n}\n\n#[test]\nfn spsc() {\n const COUNT: usize = if cfg!(miri) { 500 } else { 100_000 };\n\n let q = Injector::new();\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n loop {\n if let Success(v) = q.steal() {\n assert_eq!(i, v);\n break;\n }\n #[cfg(miri)]\n std::hint::spin_loop();\n }\n }\n\n assert_eq!(busy_retry(|| q.steal()), Empty);\n });\n\n for i in 0..COUNT {\n q.push(i);\n }\n })\n .unwrap();\n}\n\n#[test]\nfn mpmc() {\n const COUNT: usize = if cfg!(miri) { 500 } else { 25_000 };\n const THREADS: usize = 4;\n\n let q = Injector::new();\n let v = (0..COUNT).map(|_| AtomicUsize::new(0)).collect::>();\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..COUNT {\n q.push(i);\n }\n });\n }\n\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n loop {\n if let Success(n) = q.steal() {\n v[n].fetch_add(1, SeqCst);\n break;\n }\n #[cfg(miri)]\n std::hint::spin_loop();\n }\n }\n });\n }\n })\n .unwrap();\n\n for c in v {\n assert_eq!(c.load(SeqCst), THREADS);\n }\n}\n\n#[test]\nfn stampede() {\n const THREADS: usize = 8;\n const COUNT: usize = if cfg!(miri) { 500 } else { 50_000 };\n\n let q = Injector::new();\n\n for i in 0..COUNT {\n q.push(Box::new(i + 1));\n }\n let remaining = Arc::new(AtomicUsize::new(COUNT));\n\n scope(|scope| {\n for _ in 0..THREADS {\n let remaining = remaining.clone();\n let q = &q;\n\n scope.spawn(move |_| {\n let mut last = 0;\n while remaining.load(SeqCst) > 0 {\n if let Success(x) = q.steal() {\n assert!(last < *x);\n last = *x;\n remaining.fetch_sub(1, SeqCst);\n }\n }\n });\n }\n\n let mut last = 0;\n while remaining.load(SeqCst) > 0 {\n if let Success(x) = q.steal() {\n assert!(last < *x);\n last = *x;\n remaining.fetch_sub(1, SeqCst);\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress() {\n const THREADS: usize = 8;\n const COUNT: usize = if cfg!(miri) { 500 } else { 50_000 };\n\n if option_env!(\"MIRI_FALLIBLE_WEAK_CAS\").is_some() {\n return; // see ci/miri.sh\n }\n\n let q = Injector::new();\n let done = Arc::new(AtomicBool::new(false));\n let hits = Arc::new(AtomicUsize::new(0));\n\n scope(|scope| {\n for _ in 0..THREADS {\n let done = done.clone();\n let hits = hits.clone();\n let q = &q;\n\n scope.spawn(move |_| {\n let w2 = Worker::new_fifo();\n\n while !done.load(SeqCst) {\n if let Success(_) = q.steal() {\n hits.fetch_add(1, SeqCst);\n }\n\n let _ = q.steal_batch(&w2);\n\n if let Success(_) = q.steal_batch_and_pop(&w2) {\n hits.fetch_add(1, SeqCst);\n }\n\n while w2.pop().is_some() {\n hits.fetch_add(1, SeqCst);\n }\n }\n });\n }\n\n let mut rng = fastrand::Rng::new();\n let mut expected = 0;\n while expected < COUNT {\n if rng.u8(0..3) == 0 {\n while let Success(_) = q.steal() {\n hits.fetch_add(1, SeqCst);\n }\n } else {\n q.push(expected);\n expected += 1;\n }\n }\n\n while hits.load(SeqCst) < COUNT {\n while let Success(_) = q.steal() {\n hits.fetch_add(1, SeqCst);\n }\n }\n done.store(true, SeqCst);\n })\n .unwrap();\n}\n\n#[cfg_attr(miri, ignore)] // Miri is too slow\n#[test]\nfn no_starvation() {\n const THREADS: usize = 8;\n const COUNT: usize = 50_000;\n\n let q = Injector::new();\n let done = Arc::new(AtomicBool::new(false));\n let mut all_hits = Vec::new();\n\n scope(|scope| {\n for _ in 0..THREADS {\n let done = done.clone();\n let hits = Arc::new(AtomicUsize::new(0));\n all_hits.push(hits.clone());\n let q = &q;\n\n scope.spawn(move |_| {\n let w2 = Worker::new_fifo();\n\n while !done.load(SeqCst) {\n if let Success(_) = q.steal() {\n hits.fetch_add(1, SeqCst);\n }\n\n let _ = q.steal_batch(&w2);\n\n if let Success(_) = q.steal_batch_and_pop(&w2) {\n hits.fetch_add(1, SeqCst);\n }\n\n while w2.pop().is_some() {\n hits.fetch_add(1, SeqCst);\n }\n }\n });\n }\n\n let mut rng = fastrand::Rng::new();\n let mut my_hits = 0;\n loop {\n for i in 0..rng.usize(0..COUNT) {\n if rng.u8(0..3) == 0 && my_hits == 0 {\n while let Success(_) = q.steal() {\n my_hits += 1;\n }\n } else {\n q.push(i);\n }\n }\n\n if my_hits > 0 && all_hits.iter().all(|h| h.load(SeqCst) > 0) {\n break;\n }\n }\n done.store(true, SeqCst);\n })\n .unwrap();\n}\n\n#[test]\nfn destructors() {\n const THREADS: usize = if cfg!(miri) { 2 } else { 8 };\n const COUNT: usize = if cfg!(miri) { 500 } else { 50_000 };\n const STEPS: usize = if cfg!(miri) { 100 } else { 1000 };\n\n struct Elem(usize, Arc>>);\n\n impl Drop for Elem {\n fn drop(&mut self) {\n self.1.lock().unwrap().push(self.0);\n }\n }\n\n let q = Injector::new();\n let dropped = Arc::new(Mutex::new(Vec::new()));\n let remaining = Arc::new(AtomicUsize::new(COUNT));\n\n for i in 0..COUNT {\n q.push(Elem(i, dropped.clone()));\n }\n\n scope(|scope| {\n for _ in 0..THREADS {\n let remaining = remaining.clone();\n let q = &q;\n\n scope.spawn(move |_| {\n let w2 = Worker::new_fifo();\n let mut cnt = 0;\n\n while cnt < STEPS {\n if let Success(_) = q.steal() {\n cnt += 1;\n remaining.fetch_sub(1, SeqCst);\n }\n\n let _ = q.steal_batch(&w2);\n\n if let Success(_) = q.steal_batch_and_pop(&w2) {\n cnt += 1;\n remaining.fetch_sub(1, SeqCst);\n }\n\n while w2.pop().is_some() {\n cnt += 1;\n remaining.fetch_sub(1, SeqCst);\n }\n }\n });\n }\n\n for _ in 0..STEPS {\n if let Success(_) = q.steal() {\n remaining.fetch_sub(1, SeqCst);\n }\n }\n })\n .unwrap();\n\n let rem = remaining.load(SeqCst);\n assert!(rem > 0);\n\n {\n let mut v = dropped.lock().unwrap();\n assert_eq!(v.len(), COUNT - rem);\n v.clear();\n }\n\n drop(q);\n\n {\n let mut v = dropped.lock().unwrap();\n assert_eq!(v.len(), rem);\n v.sort_unstable();\n for pair in v.windows(2) {\n assert_eq!(pair[0] + 1, pair[1]);\n }\n }\n}\n\n// If `Block` is created on the stack, the array of slots will multiply this `BigStruct` and\n// probably overflow the thread stack. It's now directly created on the heap to avoid this.\n#[test]\nfn stack_overflow() {\n const N: usize = 32_768;\n struct BigStruct {\n _data: [u8; N],\n }\n\n let q = Injector::new();\n\n q.push(BigStruct { _data: [0u8; N] });\n\n while !matches!(q.steal(), Empty) {}\n}\n" }, { "path": "crossbeam-deque/tests/lifo.rs", "content": "use std::sync::{\n Arc, Mutex,\n atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst},\n};\n\nuse crossbeam_deque::{\n Steal::{Empty, Success},\n Worker,\n};\nuse crossbeam_utils::thread::scope;\n\n#[test]\nfn smoke() {\n let w = Worker::new_lifo();\n let s = w.stealer();\n assert_eq!(w.pop(), None);\n assert_eq!(s.steal(), Empty);\n\n w.push(1);\n assert_eq!(w.pop(), Some(1));\n assert_eq!(w.pop(), None);\n assert_eq!(s.steal(), Empty);\n\n w.push(2);\n assert_eq!(s.steal(), Success(2));\n assert_eq!(s.steal(), Empty);\n assert_eq!(w.pop(), None);\n\n w.push(3);\n w.push(4);\n w.push(5);\n assert_eq!(s.steal(), Success(3));\n assert_eq!(s.steal(), Success(4));\n assert_eq!(s.steal(), Success(5));\n assert_eq!(s.steal(), Empty);\n\n w.push(6);\n w.push(7);\n w.push(8);\n w.push(9);\n assert_eq!(w.pop(), Some(9));\n assert_eq!(s.steal(), Success(6));\n assert_eq!(w.pop(), Some(8));\n assert_eq!(w.pop(), Some(7));\n assert_eq!(w.pop(), None);\n}\n\n#[test]\nfn is_empty() {\n let w = Worker::new_lifo();\n let s = w.stealer();\n\n assert!(w.is_empty());\n w.push(1);\n assert!(!w.is_empty());\n w.push(2);\n assert!(!w.is_empty());\n let _ = w.pop();\n assert!(!w.is_empty());\n let _ = w.pop();\n assert!(w.is_empty());\n\n assert!(s.is_empty());\n w.push(1);\n assert!(!s.is_empty());\n w.push(2);\n assert!(!s.is_empty());\n let _ = s.steal();\n assert!(!s.is_empty());\n let _ = s.steal();\n assert!(s.is_empty());\n}\n\n#[test]\nfn spsc() {\n const STEPS: usize = if cfg!(miri) { 500 } else { 50_000 };\n\n let w = Worker::new_lifo();\n let s = w.stealer();\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..STEPS {\n loop {\n if let Success(v) = s.steal() {\n assert_eq!(i, v);\n break;\n }\n #[cfg(miri)]\n std::hint::spin_loop();\n }\n }\n\n assert_eq!(s.steal(), Empty);\n });\n\n for i in 0..STEPS {\n w.push(i);\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stampede() {\n const THREADS: usize = 8;\n const COUNT: usize = if cfg!(miri) { 500 } else { 50_000 };\n\n let w = Worker::new_lifo();\n\n for i in 0..COUNT {\n w.push(Box::new(i + 1));\n }\n let remaining = Arc::new(AtomicUsize::new(COUNT));\n\n scope(|scope| {\n for _ in 0..THREADS {\n let s = w.stealer();\n let remaining = remaining.clone();\n\n scope.spawn(move |_| {\n let mut last = 0;\n while remaining.load(SeqCst) > 0 {\n if let Success(x) = s.steal() {\n assert!(last < *x);\n last = *x;\n remaining.fetch_sub(1, SeqCst);\n }\n }\n });\n }\n\n let mut last = COUNT + 1;\n while remaining.load(SeqCst) > 0 {\n if let Some(x) = w.pop() {\n assert!(last > *x);\n last = *x;\n remaining.fetch_sub(1, SeqCst);\n }\n }\n })\n .unwrap();\n}\n\n#[test]\nfn stress() {\n const THREADS: usize = 8;\n const COUNT: usize = if cfg!(miri) { 500 } else { 50_000 };\n\n let w = Worker::new_lifo();\n let done = Arc::new(AtomicBool::new(false));\n let hits = Arc::new(AtomicUsize::new(0));\n\n scope(|scope| {\n for _ in 0..THREADS {\n let s = w.stealer();\n let done = done.clone();\n let hits = hits.clone();\n\n scope.spawn(move |_| {\n let w2 = Worker::new_lifo();\n\n while !done.load(SeqCst) {\n if let Success(_) = s.steal() {\n hits.fetch_add(1, SeqCst);\n }\n\n let _ = s.steal_batch(&w2);\n\n if let Success(_) = s.steal_batch_and_pop(&w2) {\n hits.fetch_add(1, SeqCst);\n }\n\n while w2.pop().is_some() {\n hits.fetch_add(1, SeqCst);\n }\n }\n });\n }\n\n let mut rng = fastrand::Rng::new();\n let mut expected = 0;\n while expected < COUNT {\n if rng.u8(0..3) == 0 {\n while w.pop().is_some() {\n hits.fetch_add(1, SeqCst);\n }\n } else {\n w.push(expected);\n expected += 1;\n }\n }\n\n while hits.load(SeqCst) < COUNT {\n while w.pop().is_some() {\n hits.fetch_add(1, SeqCst);\n }\n }\n done.store(true, SeqCst);\n })\n .unwrap();\n}\n\n#[cfg_attr(miri, ignore)] // Miri is too slow\n#[test]\nfn no_starvation() {\n const THREADS: usize = 8;\n const COUNT: usize = 50_000;\n\n let w = Worker::new_lifo();\n let done = Arc::new(AtomicBool::new(false));\n let mut all_hits = Vec::new();\n\n scope(|scope| {\n for _ in 0..THREADS {\n let s = w.stealer();\n let done = done.clone();\n let hits = Arc::new(AtomicUsize::new(0));\n all_hits.push(hits.clone());\n\n scope.spawn(move |_| {\n let w2 = Worker::new_lifo();\n\n while !done.load(SeqCst) {\n if let Success(_) = s.steal() {\n hits.fetch_add(1, SeqCst);\n }\n\n let _ = s.steal_batch(&w2);\n\n if let Success(_) = s.steal_batch_and_pop(&w2) {\n hits.fetch_add(1, SeqCst);\n }\n\n while w2.pop().is_some() {\n hits.fetch_add(1, SeqCst);\n }\n }\n });\n }\n\n let mut rng = fastrand::Rng::new();\n let mut my_hits = 0;\n loop {\n for i in 0..rng.usize(0..COUNT) {\n if rng.u8(0..3) == 0 && my_hits == 0 {\n while w.pop().is_some() {\n my_hits += 1;\n }\n } else {\n w.push(i);\n }\n }\n\n if my_hits > 0 && all_hits.iter().all(|h| h.load(SeqCst) > 0) {\n break;\n }\n }\n done.store(true, SeqCst);\n })\n .unwrap();\n}\n\n#[test]\nfn destructors() {\n const THREADS: usize = if cfg!(miri) { 2 } else { 8 };\n const COUNT: usize = if cfg!(miri) { 500 } else { 50_000 };\n const STEPS: usize = if cfg!(miri) { 100 } else { 1000 };\n\n struct Elem(usize, Arc>>);\n\n impl Drop for Elem {\n fn drop(&mut self) {\n self.1.lock().unwrap().push(self.0);\n }\n }\n\n let w = Worker::new_lifo();\n let dropped = Arc::new(Mutex::new(Vec::new()));\n let remaining = Arc::new(AtomicUsize::new(COUNT));\n\n for i in 0..COUNT {\n w.push(Elem(i, dropped.clone()));\n }\n\n scope(|scope| {\n for _ in 0..THREADS {\n let remaining = remaining.clone();\n let s = w.stealer();\n\n scope.spawn(move |_| {\n let w2 = Worker::new_lifo();\n let mut cnt = 0;\n\n while cnt < STEPS {\n if let Success(_) = s.steal() {\n cnt += 1;\n remaining.fetch_sub(1, SeqCst);\n }\n\n let _ = s.steal_batch(&w2);\n\n if let Success(_) = s.steal_batch_and_pop(&w2) {\n cnt += 1;\n remaining.fetch_sub(1, SeqCst);\n }\n\n while w2.pop().is_some() {\n cnt += 1;\n remaining.fetch_sub(1, SeqCst);\n }\n }\n });\n }\n\n for _ in 0..STEPS {\n if w.pop().is_some() {\n remaining.fetch_sub(1, SeqCst);\n }\n }\n })\n .unwrap();\n\n let rem = remaining.load(SeqCst);\n assert!(rem > 0);\n\n {\n let mut v = dropped.lock().unwrap();\n assert_eq!(v.len(), COUNT - rem);\n v.clear();\n }\n\n drop(w);\n\n {\n let mut v = dropped.lock().unwrap();\n assert_eq!(v.len(), rem);\n v.sort_unstable();\n for pair in v.windows(2) {\n assert_eq!(pair[0] + 1, pair[1]);\n }\n }\n}\n" }, { "path": "crossbeam-deque/tests/steal.rs", "content": "use crossbeam_deque::{\n Injector,\n Steal::{self, Success},\n Worker,\n};\n\nfn busy_retry(mut f: impl FnMut() -> Steal) -> Steal {\n loop {\n let s = f();\n if !s.is_retry() {\n return s;\n }\n }\n}\n\n#[test]\nfn steal_fifo() {\n let w = Worker::new_fifo();\n for i in 1..=3 {\n w.push(i);\n }\n\n let s = w.stealer();\n assert_eq!(s.steal(), Success(1));\n assert_eq!(s.steal(), Success(2));\n assert_eq!(s.steal(), Success(3));\n}\n\n#[test]\nfn steal_lifo() {\n let w = Worker::new_lifo();\n for i in 1..=3 {\n w.push(i);\n }\n\n let s = w.stealer();\n assert_eq!(s.steal(), Success(1));\n assert_eq!(s.steal(), Success(2));\n assert_eq!(s.steal(), Success(3));\n}\n\n#[test]\nfn steal_injector() {\n let q = Injector::new();\n for i in 1..=3 {\n q.push(i);\n }\n\n assert_eq!(busy_retry(|| q.steal()), Success(1));\n assert_eq!(busy_retry(|| q.steal()), Success(2));\n assert_eq!(busy_retry(|| q.steal()), Success(3));\n}\n\n#[test]\nfn steal_batch_fifo_fifo() {\n let w = Worker::new_fifo();\n for i in 1..=4 {\n w.push(i);\n }\n\n let s = w.stealer();\n let w2 = Worker::new_fifo();\n\n assert_eq!(s.steal_batch(&w2), Success(()));\n assert_eq!(w2.pop(), Some(1));\n assert_eq!(w2.pop(), Some(2));\n}\n\n#[test]\nfn steal_batch_lifo_lifo() {\n let w = Worker::new_lifo();\n for i in 1..=4 {\n w.push(i);\n }\n\n let s = w.stealer();\n let w2 = Worker::new_lifo();\n\n assert_eq!(s.steal_batch(&w2), Success(()));\n assert_eq!(w2.pop(), Some(2));\n assert_eq!(w2.pop(), Some(1));\n}\n\n#[test]\nfn steal_batch_fifo_lifo() {\n let w = Worker::new_fifo();\n for i in 1..=4 {\n w.push(i);\n }\n\n let s = w.stealer();\n let w2 = Worker::new_lifo();\n\n assert_eq!(s.steal_batch(&w2), Success(()));\n assert_eq!(w2.pop(), Some(1));\n assert_eq!(w2.pop(), Some(2));\n}\n\n#[test]\nfn steal_batch_lifo_fifo() {\n let w = Worker::new_lifo();\n for i in 1..=4 {\n w.push(i);\n }\n\n let s = w.stealer();\n let w2 = Worker::new_fifo();\n\n assert_eq!(s.steal_batch(&w2), Success(()));\n assert_eq!(w2.pop(), Some(2));\n assert_eq!(w2.pop(), Some(1));\n}\n\n#[test]\nfn steal_batch_injector_fifo() {\n let q = Injector::new();\n for i in 1..=4 {\n q.push(i);\n }\n\n let w2 = Worker::new_fifo();\n assert_eq!(busy_retry(|| q.steal_batch(&w2)), Success(()));\n assert_eq!(w2.pop(), Some(1));\n assert_eq!(w2.pop(), Some(2));\n}\n\n#[test]\nfn steal_batch_injector_lifo() {\n let q = Injector::new();\n for i in 1..=4 {\n q.push(i);\n }\n\n let w2 = Worker::new_lifo();\n assert_eq!(busy_retry(|| q.steal_batch(&w2)), Success(()));\n assert_eq!(w2.pop(), Some(1));\n assert_eq!(w2.pop(), Some(2));\n}\n\n#[test]\nfn steal_batch_and_pop_fifo_fifo() {\n let w = Worker::new_fifo();\n for i in 1..=6 {\n w.push(i);\n }\n\n let s = w.stealer();\n let w2 = Worker::new_fifo();\n\n assert_eq!(s.steal_batch_and_pop(&w2), Success(1));\n assert_eq!(w2.pop(), Some(2));\n assert_eq!(w2.pop(), Some(3));\n}\n\n#[test]\nfn steal_batch_and_pop_lifo_lifo() {\n let w = Worker::new_lifo();\n for i in 1..=6 {\n w.push(i);\n }\n\n let s = w.stealer();\n let w2 = Worker::new_lifo();\n\n assert_eq!(s.steal_batch_and_pop(&w2), Success(3));\n assert_eq!(w2.pop(), Some(2));\n assert_eq!(w2.pop(), Some(1));\n}\n\n#[test]\nfn steal_batch_and_pop_fifo_lifo() {\n let w = Worker::new_fifo();\n for i in 1..=6 {\n w.push(i);\n }\n\n let s = w.stealer();\n let w2 = Worker::new_lifo();\n\n assert_eq!(s.steal_batch_and_pop(&w2), Success(1));\n assert_eq!(w2.pop(), Some(2));\n assert_eq!(w2.pop(), Some(3));\n}\n\n#[test]\nfn steal_batch_and_pop_lifo_fifo() {\n let w = Worker::new_lifo();\n for i in 1..=6 {\n w.push(i);\n }\n\n let s = w.stealer();\n let w2 = Worker::new_fifo();\n\n assert_eq!(s.steal_batch_and_pop(&w2), Success(3));\n assert_eq!(w2.pop(), Some(2));\n assert_eq!(w2.pop(), Some(1));\n}\n\n#[test]\nfn steal_batch_and_pop_injector_fifo() {\n let q = Injector::new();\n for i in 1..=6 {\n q.push(i);\n }\n\n let w2 = Worker::new_fifo();\n assert_eq!(busy_retry(|| q.steal_batch_and_pop(&w2)), Success(1));\n assert_eq!(w2.pop(), Some(2));\n assert_eq!(w2.pop(), Some(3));\n}\n\n#[test]\nfn steal_batch_and_pop_injector_lifo() {\n let q = Injector::new();\n for i in 1..=6 {\n q.push(i);\n }\n\n let w2 = Worker::new_lifo();\n assert_eq!(busy_retry(|| q.steal_batch_and_pop(&w2)), Success(1));\n assert_eq!(w2.pop(), Some(2));\n assert_eq!(w2.pop(), Some(3));\n}\n" }, { "path": "crossbeam-epoch/CHANGELOG.md", "content": "# Version 0.9.18\n\n- Remove dependency on `cfg-if`. (#1072)\n- Remove dependency on `autocfg`. (#1071)\n\n# Version 0.9.17\n\n- Remove dependency on `memoffset`. (#1058)\n\n# Version 0.9.16\n\n- Bump the minimum supported Rust version to 1.61. (#1037)\n- Improve support for targets without atomic CAS. (#1037)\n- Remove build script. (#1037)\n- Remove dependency on `scopeguard`. (#1045)\n- Update `loom` dependency to 0.7.\n\n# Version 0.9.15\n\n- Update `memoffset` to 0.9. (#981)\n\n# Version 0.9.14\n\n- Update `memoffset` to 0.8. (#955)\n\n# Version 0.9.13\n\n- Fix build script bug introduced in 0.9.12. (#932)\n\n# Version 0.9.12\n\n**Note:** This release has been yanked due to regression fixed in 0.9.13.\n\n- Update `memoffset` to 0.7. (#926)\n- Improve support for custom targets. (#922)\n\n# Version 0.9.11\n\n- Removes the dependency on the `once_cell` crate to restore the MSRV. (#913)\n- Work around [rust-lang#98302](https://github.com/rust-lang/rust/issues/98302), which causes compile error on windows-gnu when LTO is enabled. (#913)\n\n# Version 0.9.10\n\n- Bump the minimum supported Rust version to 1.38. (#877)\n- Mitigate the risk of segmentation faults in buggy downstream implementations. (#879)\n- Add `{Atomic, Shared}::try_into_owned` (#701)\n\n# Version 0.9.9\n\n- Replace lazy_static with once_cell. (#817)\n\n# Version 0.9.8\n\n- Make `Atomic::null()` const function at 1.61+. (#797)\n\n# Version 0.9.7\n\n- Fix Miri error when `-Zmiri-check-number-validity` is enabled. (#779)\n\n# Version 0.9.6\n\n- Add `Atomic::fetch_update`. (#706)\n\n# Version 0.9.5\n\n- Fix UB in `Pointable` impl of `[MaybeUninit]`. (#694)\n- Support targets that do not have atomic CAS on stable Rust. (#698)\n- Fix breakage with nightly feature due to rust-lang/rust#84510. (#692)\n\n# Version 0.9.4\n\n**Note**: This release has been yanked. See [#693](https://github.com/crossbeam-rs/crossbeam/issues/693) for details.\n\n- Fix UB in `<[MaybeUninit] as Pointable>::init` when global allocator failed allocation. (#690)\n- Bump `loom` dependency to version 0.5. (#686)\n\n# Version 0.9.3\n\n**Note**: This release has been yanked. See [#693](https://github.com/crossbeam-rs/crossbeam/issues/693) for details.\n\n- Make `loom` dependency optional. (#666)\n\n# Version 0.9.2\n\n**Note**: This release has been yanked. See [#693](https://github.com/crossbeam-rs/crossbeam/issues/693) for details.\n\n- Add `Atomic::compare_exchange` and `Atomic::compare_exchange_weak`. (#628)\n- Deprecate `Atomic::compare_and_set` and `Atomic::compare_and_set_weak`. Use `Atomic::compare_exchange` or `Atomic::compare_exchange_weak` instead. (#628)\n- Make `const_fn` dependency optional. (#611)\n- Add unstable support for `loom`. (#487)\n\n# Version 0.9.1\n\n**Note**: This release has been yanked. See [#693](https://github.com/crossbeam-rs/crossbeam/issues/693) for details.\n\n- Bump `memoffset` dependency to version 0.6. (#592)\n\n# Version 0.9.0\n\n**Note**: This release has been yanked. See [#693](https://github.com/crossbeam-rs/crossbeam/issues/693) for details.\n\n- Bump the minimum supported Rust version to 1.36.\n- Support dynamically sized types.\n\n# Version 0.8.2\n\n- Fix bug in release (yanking 0.8.1)\n\n# Version 0.8.1\n\n- Bump `autocfg` dependency to version 1.0. (#460)\n- Reduce stall in list iteration. (#376)\n- Stop stealing from the same deque. (#448)\n- Fix unsoundness issues by adopting `MaybeUninit`. (#458)\n- Fix use-after-free in lock-free queue. (#466)\n\n# Version 0.8.0\n\n- Bump the minimum required version to 1.28.\n- Fix breakage with nightly feature due to rust-lang/rust#65214.\n- Make `Atomic::null()` const function at 1.31+.\n- Bump `crossbeam-utils` to `0.7`.\n\n# Version 0.7.2\n\n- Add `Atomic::into_owned()`.\n- Update `memoffset` dependency.\n\n# Version 0.7.1\n\n- Add `Shared::deref_mut()`.\n- Add a Treiber stack to examples.\n\n# Version 0.7.0\n\n- Remove `Guard::clone()`.\n- Bump dependencies.\n\n# Version 0.6.1\n\n- Update `crossbeam-utils` to `0.6`.\n\n# Version 0.6.0\n\n- `defer` now requires `F: Send + 'static`.\n- Bump the minimum Rust version to 1.26.\n- Pinning while TLS is tearing down does not fail anymore.\n- Rename `Handle` to `LocalHandle`.\n- Add `defer_unchecked` and `defer_destroy`.\n- Remove `Clone` impl for `LocalHandle`.\n\n# Version 0.5.2\n\n- Update `crossbeam-utils` to `0.5`.\n\n# Version 0.5.1\n\n- Fix compatibility with the latest Rust nightly.\n\n# Version 0.5.0\n\n- Update `crossbeam-utils` to `0.4`.\n- Specify the minimum Rust version to `1.25.0`.\n\n# Version 0.4.3\n\n- Downgrade `crossbeam-utils` to `0.3` because it was a breaking change.\n\n# Version 0.4.2\n\n- Expose the `Pointer` trait.\n- Warn missing docs and missing debug impls.\n- Update `crossbeam-utils` to `0.4`.\n\n# Version 0.4.1\n\n- Add `Debug` impls for `Collector`, `Handle`, and `Guard`.\n- Add `load_consume` to `Atomic`.\n- Rename `Collector::handle` to `Collector::register`.\n- Remove the `Send` implementation for `Handle` (this was a bug). Only\n `Collector`s can be shared among multiple threads, while `Handle`s and\n `Guard`s must stay within the thread in which they were created.\n\n# Version 0.4.0\n\n- Update dependencies.\n- Remove support for Rust 1.13.\n\n# Version 0.3.0\n\n- Add support for Rust 1.13.\n- Improve documentation for CAS.\n\n# Version 0.2.0\n\n- Add method `Owned::into_box`.\n- Fix a use-after-free bug in `Local::finalize`.\n- Fix an ordering bug in `Global::push_bag`.\n- Fix a bug in calculating distance between epochs.\n- Remove `impl Into> for Owned`.\n\n# Version 0.1.0\n\n- First version of the new epoch-based GC.\n" }, { "path": "crossbeam-epoch/Cargo.toml", "content": "[package]\nname = \"crossbeam-epoch\"\n# When publishing a new version:\n# - Update CHANGELOG.md\n# - Update README.md (when increasing major or minor version)\n# - Run './tools/publish.sh crossbeam-epoch '\nversion = \"0.9.18\"\nedition = \"2021\"\n# NB: Sync with msrv badge and \"Compatibility\" section in README.md\nrust-version = \"1.74\"\nlicense = \"MIT OR Apache-2.0\"\nrepository = \"https://github.com/crossbeam-rs/crossbeam\"\nhomepage = \"https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-epoch\"\ndescription = \"Epoch-based garbage collection\"\nkeywords = [\"lock-free\", \"rcu\", \"atomic\", \"garbage\"]\ncategories = [\"concurrency\", \"memory-management\", \"no-std\"]\n\n[package.metadata.docs.rs]\ntargets = [\"x86_64-unknown-linux-gnu\"]\n\n[package.metadata.cargo_check_external_types]\n# The following are external types that are allowed to be exposed in our public API.\nallowed_external_types = [\n]\n\n[features]\ndefault = [\"std\"]\n\n# Enable to use APIs that require `std`.\n# This is enabled by default.\nstd = [\"alloc\", \"crossbeam-utils/std\"]\n\n# Enable to use APIs that require `alloc`.\n# This is enabled by default and also enabled if the `std` feature is enabled.\n#\n# NOTE: Disabling both `std` *and* `alloc` features is not supported yet.\nalloc = []\n\n# Enable the use of loom for concurrency testing.\n#\n# NOTE: This feature is outside of the normal semver guarantees and minor or\n# patch versions of crossbeam may make breaking changes to them at any time.\nloom = [\"loom-crate\", \"crossbeam-utils/loom\"]\n\n[dependencies]\ncrossbeam-utils = { version = \"0.8.18\", path = \"../crossbeam-utils\", default-features = false, features = [\"atomic\"] }\n\n# Enable the use of loom for concurrency testing.\n#\n# NOTE: This feature is outside of the normal semver guarantees and minor or\n# patch versions of crossbeam may make breaking changes to them at any time.\n[target.'cfg(crossbeam_loom)'.dependencies]\nloom-crate = { package = \"loom\", version = \"0.7.1\", optional = true }\n\n[dev-dependencies]\nfastrand = \"2\"\n\n[lints]\nworkspace = true\n" }, { "path": "crossbeam-epoch/LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "crossbeam-epoch/LICENSE-MIT", "content": "The MIT License (MIT)\n\nCopyright (c) 2019 The Crossbeam Project Developers\n\nPermission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "crossbeam-epoch/README.md", "content": "# Crossbeam Epoch\n\n[![Build Status](https://github.com/crossbeam-rs/crossbeam/workflows/CI/badge.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/actions)\n[![License](https://img.shields.io/badge/license-MIT_OR_Apache--2.0-blue.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-epoch#license)\n[![Cargo](https://img.shields.io/crates/v/crossbeam-epoch.svg)](\nhttps://crates.io/crates/crossbeam-epoch)\n[![Documentation](https://docs.rs/crossbeam-epoch/badge.svg)](\nhttps://docs.rs/crossbeam-epoch)\n[![Rust 1.74+](https://img.shields.io/badge/rust-1.74+-lightgray.svg)](\nhttps://www.rust-lang.org)\n[![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ)\n\nThis crate provides epoch-based garbage collection for building concurrent data structures.\n\nWhen a thread removes an object from a concurrent data structure, other threads\nmay be still using pointers to it at the same time, so it cannot be destroyed\nimmediately. Epoch-based GC is an efficient mechanism for deferring destruction of\nshared objects until no pointers to them can exist.\n\nEverything in this crate except the global GC can be used in `no_std` environments, provided that\n`alloc` feature is enabled.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\ncrossbeam-epoch = \"0.9\"\n```\n\n## Compatibility\n\nCrossbeam Epoch supports stable Rust releases going back at least six months,\nand every time the minimum supported Rust version is increased, a new minor\nversion is released. Currently, the minimum supported Rust version is 1.74.\n\n## License\n\nLicensed under either of\n\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n#### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "path": "crossbeam-epoch/benches/defer.rs", "content": "#![feature(test)]\n\nextern crate test;\n\nuse crossbeam_epoch::{self as epoch, Owned};\nuse crossbeam_utils::thread::scope;\nuse test::Bencher;\n\n#[bench]\nfn single_alloc_defer_free(b: &mut Bencher) {\n b.iter(|| {\n let guard = &epoch::pin();\n let p = Owned::new(1).into_shared(guard);\n unsafe {\n guard.defer_destroy(p);\n }\n });\n}\n\n#[bench]\nfn single_defer(b: &mut Bencher) {\n b.iter(|| {\n let guard = &epoch::pin();\n guard.defer(move || ());\n });\n}\n\n#[bench]\nfn multi_alloc_defer_free(b: &mut Bencher) {\n const THREADS: usize = 16;\n const STEPS: usize = 10_000;\n\n b.iter(|| {\n scope(|s| {\n for _ in 0..THREADS {\n s.spawn(|_| {\n for _ in 0..STEPS {\n let guard = &epoch::pin();\n let p = Owned::new(1).into_shared(guard);\n unsafe {\n guard.defer_destroy(p);\n }\n }\n });\n }\n })\n .unwrap();\n });\n}\n\n#[bench]\nfn multi_defer(b: &mut Bencher) {\n const THREADS: usize = 16;\n const STEPS: usize = 10_000;\n\n b.iter(|| {\n scope(|s| {\n for _ in 0..THREADS {\n s.spawn(|_| {\n for _ in 0..STEPS {\n let guard = &epoch::pin();\n guard.defer(move || ());\n }\n });\n }\n })\n .unwrap();\n });\n}\n" }, { "path": "crossbeam-epoch/benches/flush.rs", "content": "#![feature(test)]\n\nextern crate test;\n\nuse std::sync::Barrier;\n\nuse crossbeam_epoch as epoch;\nuse crossbeam_utils::thread::scope;\nuse test::Bencher;\n\n#[bench]\nfn single_flush(b: &mut Bencher) {\n const THREADS: usize = 16;\n\n let start = Barrier::new(THREADS + 1);\n let end = Barrier::new(THREADS + 1);\n\n scope(|s| {\n for _ in 0..THREADS {\n s.spawn(|_| {\n epoch::pin();\n start.wait();\n end.wait();\n });\n }\n\n start.wait();\n b.iter(|| epoch::pin().flush());\n end.wait();\n })\n .unwrap();\n}\n\n#[bench]\nfn multi_flush(b: &mut Bencher) {\n const THREADS: usize = 16;\n const STEPS: usize = 10_000;\n\n b.iter(|| {\n scope(|s| {\n for _ in 0..THREADS {\n s.spawn(|_| {\n for _ in 0..STEPS {\n let guard = &epoch::pin();\n guard.flush();\n }\n });\n }\n })\n .unwrap();\n });\n}\n" }, { "path": "crossbeam-epoch/benches/pin.rs", "content": "#![feature(test)]\n\nextern crate test;\n\nuse crossbeam_epoch as epoch;\nuse crossbeam_utils::thread::scope;\nuse test::Bencher;\n\n#[bench]\nfn single_pin(b: &mut Bencher) {\n b.iter(epoch::pin);\n}\n\n#[bench]\nfn multi_pin(b: &mut Bencher) {\n const THREADS: usize = 16;\n const STEPS: usize = 100_000;\n\n b.iter(|| {\n scope(|s| {\n for _ in 0..THREADS {\n s.spawn(|_| {\n for _ in 0..STEPS {\n epoch::pin();\n }\n });\n }\n })\n .unwrap();\n });\n}\n" }, { "path": "crossbeam-epoch/build.rs", "content": "// The rustc-cfg emitted by the build script are *not* public API.\n\nuse std::env;\n\nfn main() {\n println!(\"cargo:rerun-if-changed=build.rs\");\n println!(\"cargo:rustc-check-cfg=cfg(crossbeam_sanitize_thread)\");\n\n // `cfg(sanitize = \"..\")` is not stabilized.\n let sanitize = env::var(\"CARGO_CFG_SANITIZE\").unwrap_or_default();\n if sanitize.contains(\"thread\") {\n println!(\"cargo:rustc-cfg=crossbeam_sanitize_thread\");\n }\n}\n" }, { "path": "crossbeam-epoch/examples/sanitize.rs", "content": "use std::{\n sync::{\n Arc,\n atomic::{\n AtomicUsize,\n Ordering::{AcqRel, Acquire, Relaxed},\n },\n },\n thread,\n time::{Duration, Instant},\n};\n\nuse crossbeam_epoch::{self as epoch, Atomic, Collector, LocalHandle, Owned, Shared};\n\nfn worker(a: Arc>, handle: LocalHandle) -> usize {\n let mut rng = fastrand::Rng::new();\n let mut sum = 0;\n\n if rng.bool() {\n thread::sleep(Duration::from_millis(1));\n }\n let timeout = Duration::from_millis(rng.u64(0..10));\n let now = Instant::now();\n\n while now.elapsed() < timeout {\n for _ in 0..100 {\n let guard = &handle.pin();\n guard.flush();\n\n let val = if rng.bool() {\n let p = a.swap(Owned::new(AtomicUsize::new(sum)), AcqRel, guard);\n unsafe {\n guard.defer_destroy(p);\n guard.flush();\n p.deref().load(Relaxed)\n }\n } else {\n let p = a.load(Acquire, guard);\n unsafe { p.deref().fetch_add(sum, Relaxed) }\n };\n\n sum = sum.wrapping_add(val);\n }\n }\n\n sum\n}\n\nfn main() {\n for _ in 0..100 {\n let collector = Collector::new();\n let a = Arc::new(Atomic::new(AtomicUsize::new(777)));\n\n let threads = (0..16)\n .map(|_| {\n let a = a.clone();\n let c = collector.clone();\n thread::spawn(move || worker(a, c.register()))\n })\n .collect::>();\n\n for t in threads {\n t.join().unwrap();\n }\n\n unsafe {\n a.swap(Shared::null(), AcqRel, epoch::unprotected())\n .into_owned();\n }\n }\n}\n" }, { "path": "crossbeam-epoch/src/atomic.rs", "content": "use alloc::boxed::Box;\nuse core::{\n alloc::Layout,\n borrow::{Borrow, BorrowMut},\n cmp, fmt,\n marker::PhantomData,\n mem::{self, MaybeUninit},\n ops::{Deref, DerefMut},\n ptr::{self, NonNull},\n};\n\nuse crossbeam_utils::atomic::AtomicConsume;\n\n#[cfg(not(miri))]\nuse crate::primitive::sync::atomic::AtomicUsize;\nuse crate::{\n alloc_helper::Global,\n guard::Guard,\n primitive::sync::atomic::{AtomicPtr, Ordering},\n};\n\n/// The value returned from a compare-and-swap operation.\npub struct CompareExchangeValue<'g, T: ?Sized + Pointable> {\n /// The previous value that was in the atomic pointer.\n pub old: Shared<'g, T>,\n\n /// The new value that was stored.\n pub new: Shared<'g, T>,\n}\n\nimpl fmt::Debug for CompareExchangeValue<'_, T> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"CompareExchangeValue\")\n .field(\"old\", &self.old)\n .field(\"new\", &self.new)\n .finish()\n }\n}\n\n/// The error returned on failed compare-and-swap operation.\npub struct CompareExchangeError<'g, T: ?Sized + Pointable, P: Pointer> {\n /// The value in the atomic pointer at the time of the failed operation.\n pub current: Shared<'g, T>,\n\n /// The new value, which the operation failed to store.\n pub new: P,\n}\n\nimpl + fmt::Debug> fmt::Debug for CompareExchangeError<'_, T, P> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"CompareExchangeError\")\n .field(\"current\", &self.current)\n .field(\"new\", &self.new)\n .finish()\n }\n}\n\n/// Returns a bitmask containing the unused least significant bits of an aligned pointer to `T`.\n#[inline]\nfn low_bits() -> usize {\n (1 << T::ALIGN.trailing_zeros()) - 1\n}\n\n/// Panics if the pointer is not properly unaligned.\n#[inline]\nfn ensure_aligned(raw: *mut ()) {\n assert_eq!(raw as usize & low_bits::(), 0, \"unaligned pointer\");\n}\n\n/// Given a tagged pointer `data`, returns the same pointer, but tagged with `tag`.\n///\n/// `tag` is truncated to fit into the unused bits of the pointer to `T`.\n#[inline]\nfn compose_tag(ptr: *mut (), tag: usize) -> *mut () {\n map_addr(ptr, |a| (a & !low_bits::()) | (tag & low_bits::()))\n}\n\n/// Decomposes a tagged pointer `data` into the pointer and the tag.\n#[inline]\nfn decompose_tag(ptr: *mut ()) -> (*mut (), usize) {\n (\n map_addr(ptr, |a| a & !low_bits::()),\n ptr as usize & low_bits::(),\n )\n}\n\n// FIXME: This is exactly ,\n// which we cannot use yet due to the MSRV.\n#[inline]\nfn map_addr(ptr: *mut T, f: impl FnOnce(usize) -> usize) -> *mut T {\n let new_addr = f(ptr as usize);\n ptr.cast::()\n .wrapping_add(new_addr.wrapping_sub(ptr as usize))\n .cast::()\n}\n\n/// Types that are pointed to by a single word.\n///\n/// In concurrent programming, it is necessary to represent an object within a word because atomic\n/// operations (e.g., reads, writes, read-modify-writes) support only single words. This trait\n/// qualifies such types that are pointed to by a single word.\n///\n/// The trait generalizes `Box` for a sized type `T`. In a box, an object of type `T` is\n/// allocated in heap and it is owned by a single-word pointer. This trait is also implemented for\n/// `[MaybeUninit]` by storing its size along with its elements and pointing to the pair of array\n/// size and elements.\n///\n/// Pointers to `Pointable` types can be stored in [`Atomic`], [`Owned`], and [`Shared`]. In\n/// particular, Crossbeam supports dynamically sized slices as follows.\n///\n/// ```\n/// use std::mem::MaybeUninit;\n/// use crossbeam_epoch::Owned;\n///\n/// let o = Owned::<[MaybeUninit]>::init(10); // allocating [i32; 10]\n/// ```\npub trait Pointable {\n /// The alignment of pointer.\n const ALIGN: usize;\n\n /// The type for initializers.\n type Init;\n\n /// Initializes a with the given initializer.\n ///\n /// # Safety\n ///\n /// The result should be a multiple of `ALIGN`.\n unsafe fn init(init: Self::Init) -> *mut ();\n\n /// Gets the raw pointer to this type that allows immutable access from the given pointer.\n ///\n /// # Safety\n ///\n /// - The given `ptr` should have been initialized with [`Pointable::init`].\n /// - `ptr` should not have yet been dropped by [`Pointable::drop`].\n /// - `ptr` should not be mutably dereferenced by [`Pointable::as_mut_ptr`] concurrently.\n unsafe fn as_ptr(ptr: *mut ()) -> *const Self;\n\n /// Gets the raw pointer to this type that allows mutable access from the given pointer.\n ///\n /// # Safety\n ///\n /// - The given `ptr` should have been initialized with [`Pointable::init`].\n /// - `ptr` should not have yet been dropped by [`Pointable::drop`].\n /// - `ptr` should not be dereferenced by [`Pointable::as_ptr`] or [`Pointable::as_mut_ptr`]\n /// concurrently.\n unsafe fn as_mut_ptr(ptr: *mut ()) -> *mut Self;\n\n /// Drops the object pointed to by the given pointer.\n ///\n /// # Safety\n ///\n /// - The given `ptr` should have been initialized with [`Pointable::init`].\n /// - `ptr` should not have yet been dropped by [`Pointable::drop`].\n /// - `ptr` should not be dereferenced by [`Pointable::as_ptr`] or [`Pointable::as_mut_ptr`]\n /// concurrently.\n unsafe fn drop(ptr: *mut ());\n}\n\nimpl Pointable for T {\n const ALIGN: usize = mem::align_of::();\n\n type Init = T;\n\n unsafe fn init(init: Self::Init) -> *mut () {\n Box::into_raw(Box::new(init)).cast::<()>()\n }\n\n unsafe fn as_ptr(ptr: *mut ()) -> *const Self {\n ptr as *const T\n }\n\n unsafe fn as_mut_ptr(ptr: *mut ()) -> *mut Self {\n ptr.cast::()\n }\n\n unsafe fn drop(ptr: *mut ()) {\n drop(unsafe { Box::from_raw(ptr.cast::()) });\n }\n}\n\n/// Array with size.\n///\n/// # Memory layout\n///\n/// An array consisting of size and elements:\n///\n/// ```text\n/// elements\n/// |\n/// |\n/// ------------------------------------\n/// | size | 0 | 1 | 2 | 3 | 4 | 5 | 6 |\n/// ------------------------------------\n/// ```\n///\n/// Its memory layout is different from that of `Box<[T]>` in that size is in the allocation (not\n/// along with pointer as in `Box<[T]>`).\n///\n/// Elements are not present in the type, but they will be in the allocation.\n#[repr(C)]\nstruct Array {\n /// The number of elements (not the number of bytes).\n len: usize,\n elements: [MaybeUninit; 0],\n}\n\nimpl Array {\n fn layout(len: usize) -> Layout {\n Layout::new::()\n .extend(Layout::array::>(len).unwrap())\n .unwrap()\n .0\n .pad_to_align()\n }\n}\n\nimpl Pointable for [MaybeUninit] {\n const ALIGN: usize = mem::align_of::>();\n\n type Init = usize;\n\n #[inline]\n unsafe fn init(len: Self::Init) -> *mut () {\n let layout = Array::::layout(len);\n match Global.allocate(layout) {\n Some(ptr) => unsafe {\n let ptr = ptr.as_ptr().cast::>();\n ptr::addr_of_mut!((*ptr).len).write(len);\n ptr.cast::<()>()\n },\n None => alloc::alloc::handle_alloc_error(layout),\n }\n }\n\n unsafe fn as_ptr(ptr: *mut ()) -> *const Self {\n unsafe {\n let len = (*ptr.cast::>()).len;\n // Use addr_of_mut for stacked borrows: https://github.com/rust-lang/miri/issues/1976\n let elements =\n ptr::addr_of_mut!((*ptr.cast::>()).elements).cast::>();\n ptr::slice_from_raw_parts(elements, len)\n }\n }\n\n unsafe fn as_mut_ptr(ptr: *mut ()) -> *mut Self {\n unsafe {\n let len = (*ptr.cast::>()).len;\n let elements =\n ptr::addr_of_mut!((*ptr.cast::>()).elements).cast::>();\n ptr::slice_from_raw_parts_mut(elements, len)\n }\n }\n\n unsafe fn drop(ptr: *mut ()) {\n unsafe {\n let len = (*ptr.cast::>()).len;\n let layout = Array::::layout(len);\n Global.deallocate(NonNull::new_unchecked(ptr.cast::()), layout);\n }\n }\n}\n\n/// An atomic pointer that can be safely shared between threads.\n///\n/// The pointer must be properly aligned. Since it is aligned, a tag can be stored into the unused\n/// least significant bits of the address. For example, the tag for a pointer to a sized type `T`\n/// should be less than `(1 << mem::align_of::().trailing_zeros())`.\n///\n/// Any method that loads the pointer must be passed a reference to a [`Guard`].\n///\n/// Crossbeam supports dynamically sized types. See [`Pointable`] for details.\npub struct Atomic {\n data: AtomicPtr<()>,\n _marker: PhantomData<*mut T>,\n}\n\nunsafe impl Send for Atomic {}\nunsafe impl Sync for Atomic {}\n\nimpl Atomic {\n /// Allocates `value` on the heap and returns a new atomic pointer pointing to it.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Atomic;\n ///\n /// let a = Atomic::new(1234);\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn new(init: T) -> Self {\n Self::init(init)\n }\n}\n\nimpl Atomic {\n /// Allocates `value` on the heap and returns a new atomic pointer pointing to it.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Atomic;\n ///\n /// let a = Atomic::::init(1234);\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn init(init: T::Init) -> Self {\n Self::from(Owned::init(init))\n }\n\n /// Returns a new atomic pointer pointing to the tagged pointer `data`.\n fn from_ptr(data: *mut ()) -> Self {\n Self {\n data: AtomicPtr::new(data),\n _marker: PhantomData,\n }\n }\n\n const_fn! {\n const_if: #[cfg(not(crossbeam_loom))];\n /// Returns a new null atomic pointer.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Atomic;\n ///\n /// let a = Atomic::::null();\n /// ```\n pub const fn null() -> Self {\n Self {\n data: AtomicPtr::new(ptr::null_mut()),\n _marker: PhantomData,\n }\n }\n }\n\n /// Loads a `Shared` from the atomic pointer.\n ///\n /// This method takes an [`Ordering`] argument which describes the memory ordering of this\n /// operation.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(1234);\n /// let guard = &epoch::pin();\n /// let p = a.load(SeqCst, guard);\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn load<'g>(&self, order: Ordering, _: &'g Guard) -> Shared<'g, T> {\n unsafe { Shared::from_ptr(self.data.load(order)) }\n }\n\n /// Loads a `Shared` from the atomic pointer using a \"consume\" memory ordering.\n ///\n /// This is similar to the \"acquire\" ordering, except that an ordering is\n /// only guaranteed with operations that \"depend on\" the result of the load.\n /// However consume loads are usually much faster than acquire loads on\n /// architectures with a weak memory model since they don't require memory\n /// fence instructions.\n ///\n /// The exact definition of \"depend on\" is a bit vague, but it works as you\n /// would expect in practice since a lot of software, especially the Linux\n /// kernel, rely on this behavior.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n ///\n /// let a = Atomic::new(1234);\n /// let guard = &epoch::pin();\n /// let p = a.load_consume(guard);\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn load_consume<'g>(&self, _: &'g Guard) -> Shared<'g, T> {\n unsafe { Shared::from_ptr(self.data.load_consume()) }\n }\n\n /// Stores a `Shared` or `Owned` pointer into the atomic pointer.\n ///\n /// This method takes an [`Ordering`] argument which describes the memory ordering of this\n /// operation.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{Atomic, Owned, Shared};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(1234);\n /// # unsafe { drop(a.load(SeqCst, &crossbeam_epoch::pin()).into_owned()); } // avoid leak\n /// a.store(Shared::null(), SeqCst);\n /// a.store(Owned::new(1234), SeqCst);\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn store>(&self, new: P, order: Ordering) {\n self.data.store(new.into_ptr(), order);\n }\n\n /// Stores a `Shared` or `Owned` pointer into the atomic pointer, returning the previous\n /// `Shared`.\n ///\n /// This method takes an [`Ordering`] argument which describes the memory ordering of this\n /// operation.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Shared};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(1234);\n /// let guard = &epoch::pin();\n /// let p = a.swap(Shared::null(), SeqCst, guard);\n /// # unsafe { drop(p.into_owned()); } // avoid leak\n /// ```\n pub fn swap<'g, P: Pointer>(&self, new: P, order: Ordering, _: &'g Guard) -> Shared<'g, T> {\n unsafe { Shared::from_ptr(self.data.swap(new.into_ptr(), order)) }\n }\n\n /// Stores the pointer `new` (either `Shared` or `Owned`) into the atomic pointer if the current\n /// value is the same as `current`. The tag is also taken into account, so two pointers to the\n /// same object, but with different tags, will not be considered equal.\n ///\n /// The return value contains both the previous value and the new value that was written.\n /// On success, `old` contains the previous value (which equals `current`) and `new` contains\n /// the value that was stored. On failure, `old` contains the actual current value and `new`\n /// contains the value that was attempted to be stored.\n ///\n /// This method takes two `Ordering` arguments to describe the memory\n /// ordering of this operation. `success` describes the required ordering for the\n /// read-modify-write operation that takes place if the comparison with `current` succeeds.\n /// `failure` describes the required ordering for the load operation that takes place when\n /// the comparison fails. Using `Acquire` as success ordering makes the store part\n /// of this operation `Relaxed`, and using `Release` makes the successful load\n /// `Relaxed`. The failure ordering can only be `SeqCst`, `Acquire` or `Relaxed`\n /// and must be equivalent to or weaker than the success ordering.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Owned, Shared};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(1234);\n ///\n /// let guard = &epoch::pin();\n /// let curr = a.load(SeqCst, guard);\n /// let res1 = a.compare_exchange(curr, Shared::null(), SeqCst, SeqCst, guard);\n /// let res2 = a.compare_exchange(curr, Owned::new(5678), SeqCst, SeqCst, guard);\n /// # unsafe { drop(curr.into_owned()); } // avoid leak\n /// ```\n pub fn compare_exchange<'g, P>(\n &self,\n current: Shared<'_, T>,\n new: P,\n success: Ordering,\n failure: Ordering,\n _: &'g Guard,\n ) -> Result, CompareExchangeError<'g, T, P>>\n where\n P: Pointer,\n {\n let new = new.into_ptr();\n self.data\n .compare_exchange(current.into_ptr(), new, success, failure)\n .map(|old| unsafe {\n CompareExchangeValue {\n old: Shared::from_ptr(old),\n new: Shared::from_ptr(new),\n }\n })\n .map_err(|current| unsafe {\n CompareExchangeError {\n current: Shared::from_ptr(current),\n new: P::from_ptr(new),\n }\n })\n }\n\n /// Stores the pointer `new` (either `Shared` or `Owned`) into the atomic pointer if the current\n /// value is the same as `current`. The tag is also taken into account, so two pointers to the\n /// same object, but with different tags, will not be considered equal.\n ///\n /// Unlike [`compare_exchange`], this method is allowed to spuriously fail even when comparison\n /// succeeds, which can result in more efficient code on some platforms. The return value\n /// contains both the previous value and the new value that was written. On success, `old`\n /// contains the previous value (which equals `current`) and `new` contains the value that was\n /// stored. On failure, `old` contains the actual current value and `new` contains the value\n /// that was attempted to be stored.\n ///\n /// This method takes two `Ordering` arguments to describe the memory\n /// ordering of this operation. `success` describes the required ordering for the\n /// read-modify-write operation that takes place if the comparison with `current` succeeds.\n /// `failure` describes the required ordering for the load operation that takes place when\n /// the comparison fails. Using `Acquire` as success ordering makes the store part\n /// of this operation `Relaxed`, and using `Release` makes the successful load\n /// `Relaxed`. The failure ordering can only be `SeqCst`, `Acquire` or `Relaxed`\n /// and must be equivalent to or weaker than the success ordering.\n ///\n /// [`compare_exchange`]: Atomic::compare_exchange\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Owned, Shared};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(1234);\n /// let guard = &epoch::pin();\n ///\n /// let mut new = Owned::new(5678);\n /// let mut ptr = a.load(SeqCst, guard);\n /// # unsafe { drop(a.load(SeqCst, guard).into_owned()); } // avoid leak\n /// loop {\n /// match a.compare_exchange_weak(ptr, new, SeqCst, SeqCst, guard) {\n /// Ok(result) => {\n /// ptr = result.old;\n /// break;\n /// }\n /// Err(err) => {\n /// ptr = err.current;\n /// new = err.new;\n /// }\n /// }\n /// }\n ///\n /// let mut curr = a.load(SeqCst, guard);\n /// loop {\n /// match a.compare_exchange_weak(curr, Shared::null(), SeqCst, SeqCst, guard) {\n /// Ok(_) => break,\n /// Err(err) => curr = err.current,\n /// }\n /// }\n /// # unsafe { drop(curr.into_owned()); } // avoid leak\n /// ```\n pub fn compare_exchange_weak<'g, P>(\n &self,\n current: Shared<'_, T>,\n new: P,\n success: Ordering,\n failure: Ordering,\n _: &'g Guard,\n ) -> Result, CompareExchangeError<'g, T, P>>\n where\n P: Pointer,\n {\n let new = new.into_ptr();\n self.data\n .compare_exchange_weak(current.into_ptr(), new, success, failure)\n .map(|old| unsafe {\n CompareExchangeValue {\n old: Shared::from_ptr(old),\n new: Shared::from_ptr(new),\n }\n })\n .map_err(|current| unsafe {\n CompareExchangeError {\n current: Shared::from_ptr(current),\n new: P::from_ptr(new),\n }\n })\n }\n\n /// Fetches the pointer, and then applies a function to it that returns a new value.\n /// Returns a `Result` of `Ok(previous_value)` if the function returned `Some`, else `Err(_)`.\n ///\n /// Note that the given function may be called multiple times if the value has been changed by\n /// other threads in the meantime, as long as the function returns `Some(_)`, but the function\n /// will have been applied only once to the stored value.\n ///\n /// `fetch_update` takes two [`Ordering`] arguments to describe the memory\n /// ordering of this operation. The first describes the required ordering for\n /// when the operation finally succeeds while the second describes the\n /// required ordering for loads. These correspond to the success and failure\n /// orderings of [`Atomic::compare_exchange`] respectively.\n ///\n /// Using [`Acquire`] as success ordering makes the store part of this\n /// operation [`Relaxed`], and using [`Release`] makes the final successful\n /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`],\n /// [`Acquire`] or [`Relaxed`] and must be equivalent to or weaker than the\n /// success ordering.\n ///\n /// [`Relaxed`]: Ordering::Relaxed\n /// [`Acquire`]: Ordering::Acquire\n /// [`Release`]: Ordering::Release\n /// [`SeqCst`]: Ordering::SeqCst\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(1234);\n /// let guard = &epoch::pin();\n ///\n /// let res1 = a.fetch_update(SeqCst, SeqCst, guard, |x| Some(x.with_tag(1)));\n /// assert!(res1.is_ok());\n ///\n /// let res2 = a.fetch_update(SeqCst, SeqCst, guard, |x| None);\n /// assert!(res2.is_err());\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn fetch_update<'g, F>(\n &self,\n set_order: Ordering,\n fail_order: Ordering,\n guard: &'g Guard,\n mut func: F,\n ) -> Result, Shared<'g, T>>\n where\n F: FnMut(Shared<'g, T>) -> Option>,\n {\n let mut prev = self.load(fail_order, guard);\n while let Some(next) = func(prev) {\n match self.compare_exchange_weak(prev, next, set_order, fail_order, guard) {\n Ok(result) => return Ok(result.old),\n Err(next_prev) => prev = next_prev.current,\n }\n }\n Err(prev)\n }\n\n /// Bitwise \"and\" with the current tag.\n ///\n /// Performs a bitwise \"and\" operation on the current tag and the argument `val`, and sets the\n /// new tag to the result. Returns the previous pointer.\n ///\n /// This method takes an [`Ordering`] argument which describes the memory ordering of this\n /// operation.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Shared};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::::from(Shared::null().with_tag(3));\n /// let guard = &epoch::pin();\n /// assert_eq!(a.fetch_and(2, SeqCst, guard).tag(), 3);\n /// assert_eq!(a.load(SeqCst, guard).tag(), 2);\n /// ```\n pub fn fetch_and<'g>(&self, val: usize, order: Ordering, _: &'g Guard) -> Shared<'g, T> {\n let val = val | !low_bits::();\n // Ideally, we would always use AtomicPtr::fetch_* since it is strict-provenance\n // compatible, but it requires Rust 1.91. So, for now use it only on cfg(miri).\n // Code using AtomicUsize::fetch_* via casts is still permissive-provenance\n // compatible and is sound.\n #[cfg(miri)]\n unsafe {\n Shared::from_ptr(self.data.fetch_and(val, order))\n }\n #[cfg(not(miri))]\n unsafe {\n Shared::from_ptr(\n (*(&self.data as *const AtomicPtr<_> as *const AtomicUsize)).fetch_and(val, order)\n as *mut (),\n )\n }\n }\n\n /// Bitwise \"or\" with the current tag.\n ///\n /// Performs a bitwise \"or\" operation on the current tag and the argument `val`, and sets the\n /// new tag to the result. Returns the previous pointer.\n ///\n /// This method takes an [`Ordering`] argument which describes the memory ordering of this\n /// operation.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Shared};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::::from(Shared::null().with_tag(1));\n /// let guard = &epoch::pin();\n /// assert_eq!(a.fetch_or(2, SeqCst, guard).tag(), 1);\n /// assert_eq!(a.load(SeqCst, guard).tag(), 3);\n /// ```\n pub fn fetch_or<'g>(&self, val: usize, order: Ordering, _: &'g Guard) -> Shared<'g, T> {\n let val = val & low_bits::();\n // Ideally, we would always use AtomicPtr::fetch_* since it is strict-provenance\n // compatible, but it requires Rust 1.91. So, for now use it only on cfg(miri).\n // Code using AtomicUsize::fetch_* via casts is still permissive-provenance\n // compatible and is sound.\n #[cfg(miri)]\n unsafe {\n Shared::from_ptr(self.data.fetch_or(val, order))\n }\n #[cfg(not(miri))]\n unsafe {\n Shared::from_ptr(\n (*(&self.data as *const AtomicPtr<_> as *const AtomicUsize)).fetch_or(val, order)\n as *mut (),\n )\n }\n }\n\n /// Bitwise \"xor\" with the current tag.\n ///\n /// Performs a bitwise \"xor\" operation on the current tag and the argument `val`, and sets the\n /// new tag to the result. Returns the previous pointer.\n ///\n /// This method takes an [`Ordering`] argument which describes the memory ordering of this\n /// operation.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Shared};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::::from(Shared::null().with_tag(1));\n /// let guard = &epoch::pin();\n /// assert_eq!(a.fetch_xor(3, SeqCst, guard).tag(), 1);\n /// assert_eq!(a.load(SeqCst, guard).tag(), 2);\n /// ```\n pub fn fetch_xor<'g>(&self, val: usize, order: Ordering, _: &'g Guard) -> Shared<'g, T> {\n let val = val & low_bits::();\n // Ideally, we would always use AtomicPtr::fetch_* since it is strict-provenance\n // compatible, but it requires Rust 1.91. So, for now use it only on cfg(miri).\n // Code using AtomicUsize::fetch_* via casts is still permissive-provenance\n // compatible and is sound.\n #[cfg(miri)]\n unsafe {\n Shared::from_ptr(self.data.fetch_xor(val, order) as *mut ())\n }\n #[cfg(not(miri))]\n unsafe {\n Shared::from_ptr(\n (*(&self.data as *const AtomicPtr<_> as *const AtomicUsize)).fetch_xor(val, order)\n as *mut (),\n )\n }\n }\n\n /// Takes ownership of the pointee.\n ///\n /// This consumes the atomic and converts it into [`Owned`]. As [`Atomic`] doesn't have a\n /// destructor and doesn't drop the pointee while [`Owned`] does, this is suitable for\n /// destructors of data structures.\n ///\n /// # Panics\n ///\n /// Panics if this pointer is null, but only in debug mode.\n ///\n /// # Safety\n ///\n /// This method may be called only if the pointer is valid and nobody else is holding a\n /// reference to the same object.\n ///\n /// # Examples\n ///\n /// ```rust\n /// # use std::mem;\n /// # use crossbeam_epoch::Atomic;\n /// struct DataStructure {\n /// ptr: Atomic,\n /// }\n ///\n /// impl Drop for DataStructure {\n /// fn drop(&mut self) {\n /// // By now the DataStructure lives only in our thread and we are sure we don't hold\n /// // any Shared or & to it ourselves.\n /// unsafe {\n /// drop(mem::take(&mut self.ptr).into_owned());\n /// }\n /// }\n /// }\n /// ```\n pub unsafe fn into_owned(self) -> Owned {\n unsafe { Owned::from_ptr(self.data.into_inner()) }\n }\n\n /// Takes ownership of the pointee if it is non-null.\n ///\n /// This consumes the atomic and converts it into [`Owned`]. As [`Atomic`] doesn't have a\n /// destructor and doesn't drop the pointee while [`Owned`] does, this is suitable for\n /// destructors of data structures.\n ///\n /// # Safety\n ///\n /// This method may be called only if the pointer is valid and nobody else is holding a\n /// reference to the same object, or the pointer is null.\n ///\n /// # Examples\n ///\n /// ```rust\n /// # use std::mem;\n /// # use crossbeam_epoch::Atomic;\n /// struct DataStructure {\n /// ptr: Atomic,\n /// }\n ///\n /// impl Drop for DataStructure {\n /// fn drop(&mut self) {\n /// // By now the DataStructure lives only in our thread and we are sure we don't hold\n /// // any Shared or & to it ourselves, but it may be null, so we have to be careful.\n /// let old = mem::take(&mut self.ptr);\n /// unsafe {\n /// if let Some(x) = old.try_into_owned() {\n /// drop(x)\n /// }\n /// }\n /// }\n /// }\n /// ```\n pub unsafe fn try_into_owned(self) -> Option> {\n let data = self.data.into_inner();\n if decompose_tag::(data).0.is_null() {\n None\n } else {\n Some(unsafe { Owned::from_ptr(data) })\n }\n }\n}\n\nimpl fmt::Debug for Atomic {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n let data = self.data.load(Ordering::SeqCst);\n let (raw, tag) = decompose_tag::(data);\n\n f.debug_struct(\"Atomic\")\n .field(\"raw\", &raw)\n .field(\"tag\", &tag)\n .finish()\n }\n}\n\nimpl fmt::Pointer for Atomic {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n let data = self.data.load(Ordering::SeqCst);\n let (raw, _) = decompose_tag::(data);\n fmt::Pointer::fmt(&(unsafe { T::as_ptr(raw) }), f)\n }\n}\n\nimpl Clone for Atomic {\n /// Returns a copy of the atomic value.\n ///\n /// Note that a `Relaxed` load is used here. If you need synchronization, use it with other\n /// atomics or fences.\n fn clone(&self) -> Self {\n let data = self.data.load(Ordering::Relaxed);\n Self::from_ptr(data)\n }\n}\n\nimpl Default for Atomic {\n fn default() -> Self {\n Self::null()\n }\n}\n\nimpl From> for Atomic {\n /// Returns a new atomic pointer pointing to `owned`.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{Atomic, Owned};\n ///\n /// let a = Atomic::::from(Owned::new(1234));\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n fn from(owned: Owned) -> Self {\n let data = owned.data;\n mem::forget(owned);\n Self::from_ptr(data)\n }\n}\n\nimpl From> for Atomic {\n fn from(b: Box) -> Self {\n Self::from(Owned::from(b))\n }\n}\n\nimpl From for Atomic {\n fn from(t: T) -> Self {\n Self::new(t)\n }\n}\n\nimpl<'g, T: ?Sized + Pointable> From> for Atomic {\n /// Returns a new atomic pointer pointing to `ptr`.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{Atomic, Shared};\n ///\n /// let a = Atomic::::from(Shared::::null());\n /// ```\n fn from(ptr: Shared<'g, T>) -> Self {\n Self::from_ptr(ptr.data)\n }\n}\n\nimpl From<*const T> for Atomic {\n /// Returns a new atomic pointer pointing to `raw`.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::ptr;\n /// use crossbeam_epoch::Atomic;\n ///\n /// let a = Atomic::::from(ptr::null::());\n /// ```\n fn from(raw: *const T) -> Self {\n Self::from_ptr(raw as *mut ())\n }\n}\n\n/// A trait for either `Owned` or `Shared` pointers.\n///\n/// This trait is sealed and cannot be implemented for types outside of `crossbeam-epoch`.\npub trait Pointer: crate::sealed::Sealed {\n /// Returns the machine representation of the pointer.\n fn into_ptr(self) -> *mut ();\n\n /// Returns a new pointer pointing to the tagged pointer `data`.\n ///\n /// # Safety\n ///\n /// The given `data` should have been created by `Pointer::into_ptr()`, and one `data` should\n /// not be converted back by `Pointer::from_ptr()` multiple times.\n unsafe fn from_ptr(data: *mut ()) -> Self;\n}\n\n/// An owned heap-allocated object.\n///\n/// This type is very similar to `Box`.\n///\n/// The pointer must be properly aligned. Since it is aligned, a tag can be stored into the unused\n/// least significant bits of the address.\npub struct Owned {\n data: *mut (),\n _marker: PhantomData>,\n}\n\nimpl crate::sealed::Sealed for Owned {}\nimpl Pointer for Owned {\n #[inline]\n fn into_ptr(self) -> *mut () {\n let data = self.data;\n mem::forget(self);\n data\n }\n\n /// Returns a new pointer pointing to the tagged pointer `data`.\n ///\n /// # Panics\n ///\n /// Panics if the pointer is null, but only in debug mode.\n #[inline]\n unsafe fn from_ptr(data: *mut ()) -> Self {\n debug_assert!(!data.is_null(), \"converting null into `Owned`\");\n Self {\n data,\n _marker: PhantomData,\n }\n }\n}\n\nimpl Owned {\n /// Returns a new owned pointer pointing to `raw`.\n ///\n /// This function is unsafe because improper use may lead to memory problems. Argument `raw`\n /// must be a valid pointer. Also, a double-free may occur if the function is called twice on\n /// the same raw pointer.\n ///\n /// # Panics\n ///\n /// Panics if `raw` is not properly aligned.\n ///\n /// # Safety\n ///\n /// The given `raw` should have been derived from `Owned`, and one `raw` should not be converted\n /// back by `Owned::from_raw()` multiple times.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Owned;\n ///\n /// let o = unsafe { Owned::from_raw(Box::into_raw(Box::new(1234))) };\n /// ```\n pub unsafe fn from_raw(raw: *mut T) -> Self {\n let raw = raw.cast::<()>();\n ensure_aligned::(raw);\n unsafe { Self::from_ptr(raw) }\n }\n\n /// Converts the owned pointer into a `Box`.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Owned;\n ///\n /// let o = Owned::new(1234);\n /// let b: Box = o.into_box();\n /// assert_eq!(*b, 1234);\n /// ```\n pub fn into_box(self) -> Box {\n let (raw, _) = decompose_tag::(self.data);\n mem::forget(self);\n unsafe { Box::from_raw(raw.cast::()) }\n }\n\n /// Allocates `value` on the heap and returns a new owned pointer pointing to it.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Owned;\n ///\n /// let o = Owned::new(1234);\n /// ```\n pub fn new(init: T) -> Self {\n Self::init(init)\n }\n}\n\nimpl Owned {\n /// Allocates `value` on the heap and returns a new owned pointer pointing to it.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Owned;\n ///\n /// let o = Owned::::init(1234);\n /// ```\n pub fn init(init: T::Init) -> Self {\n unsafe { Self::from_ptr(T::init(init)) }\n }\n\n /// Converts the owned pointer into a [`Shared`].\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Owned};\n ///\n /// let o = Owned::new(1234);\n /// let guard = &epoch::pin();\n /// let p = o.into_shared(guard);\n /// # unsafe { drop(p.into_owned()); } // avoid leak\n /// ```\n #[allow(clippy::needless_lifetimes)]\n pub fn into_shared<'g>(self, _: &'g Guard) -> Shared<'g, T> {\n unsafe { Shared::from_ptr(self.into_ptr()) }\n }\n\n /// Returns the tag stored within the pointer.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Owned;\n ///\n /// assert_eq!(Owned::new(1234).tag(), 0);\n /// ```\n pub fn tag(&self) -> usize {\n let (_, tag) = decompose_tag::(self.data);\n tag\n }\n\n /// Returns the same pointer, but tagged with `tag`. `tag` is truncated to be fit into the\n /// unused bits of the pointer to `T`.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Owned;\n ///\n /// let o = Owned::new(0u64);\n /// assert_eq!(o.tag(), 0);\n /// let o = o.with_tag(2);\n /// assert_eq!(o.tag(), 2);\n /// ```\n pub fn with_tag(self, tag: usize) -> Self {\n let data = self.into_ptr();\n unsafe { Self::from_ptr(compose_tag::(data, tag)) }\n }\n}\n\nimpl Drop for Owned {\n fn drop(&mut self) {\n let (raw, _) = decompose_tag::(self.data);\n unsafe {\n T::drop(raw);\n }\n }\n}\n\nimpl fmt::Debug for Owned {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n let (raw, tag) = decompose_tag::(self.data);\n\n f.debug_struct(\"Owned\")\n .field(\"raw\", &raw)\n .field(\"tag\", &tag)\n .finish()\n }\n}\n\nimpl Clone for Owned {\n fn clone(&self) -> Self {\n Self::new((**self).clone()).with_tag(self.tag())\n }\n}\n\nimpl Deref for Owned {\n type Target = T;\n\n fn deref(&self) -> &T {\n let (raw, _) = decompose_tag::(self.data);\n unsafe { &*T::as_ptr(raw) }\n }\n}\n\nimpl DerefMut for Owned {\n fn deref_mut(&mut self) -> &mut T {\n let (raw, _) = decompose_tag::(self.data);\n unsafe { &mut *T::as_mut_ptr(raw) }\n }\n}\n\nimpl From for Owned {\n fn from(t: T) -> Self {\n Self::new(t)\n }\n}\n\nimpl From> for Owned {\n /// Returns a new owned pointer pointing to `b`.\n ///\n /// # Panics\n ///\n /// Panics if the pointer (the `Box`) is not properly aligned.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Owned;\n ///\n /// let o = unsafe { Owned::from_raw(Box::into_raw(Box::new(1234))) };\n /// ```\n fn from(b: Box) -> Self {\n unsafe { Self::from_raw(Box::into_raw(b)) }\n }\n}\n\nimpl Borrow for Owned {\n fn borrow(&self) -> &T {\n self.deref()\n }\n}\n\nimpl BorrowMut for Owned {\n fn borrow_mut(&mut self) -> &mut T {\n self.deref_mut()\n }\n}\n\nimpl AsRef for Owned {\n fn as_ref(&self) -> &T {\n self.deref()\n }\n}\n\nimpl AsMut for Owned {\n fn as_mut(&mut self) -> &mut T {\n self.deref_mut()\n }\n}\n\n/// A pointer to an object protected by the epoch GC.\n///\n/// The pointer is valid for use only during the lifetime `'g`.\n///\n/// The pointer must be properly aligned. Since it is aligned, a tag can be stored into the unused\n/// least significant bits of the address.\npub struct Shared<'g, T: 'g + ?Sized + Pointable> {\n data: *mut (),\n _marker: PhantomData<(&'g (), *const T)>,\n}\n\nimpl Clone for Shared<'_, T> {\n fn clone(&self) -> Self {\n *self\n }\n}\n\nimpl Copy for Shared<'_, T> {}\n\nimpl crate::sealed::Sealed for Shared<'_, T> {}\nimpl Pointer for Shared<'_, T> {\n #[inline]\n fn into_ptr(self) -> *mut () {\n self.data\n }\n\n #[inline]\n unsafe fn from_ptr(data: *mut ()) -> Self {\n Shared {\n data,\n _marker: PhantomData,\n }\n }\n}\n\nimpl Shared<'_, T> {\n /// Converts the pointer to a raw pointer (without the tag).\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Owned};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let o = Owned::new(1234);\n /// let raw = &*o as *const _;\n /// let a = Atomic::from(o);\n ///\n /// let guard = &epoch::pin();\n /// let p = a.load(SeqCst, guard);\n /// assert_eq!(p.as_raw(), raw);\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn as_raw(&self) -> *const T {\n let (raw, _) = decompose_tag::(self.data);\n raw as *const _\n }\n}\n\nimpl<'g, T: ?Sized + Pointable> Shared<'g, T> {\n /// Returns a new null pointer.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Shared;\n ///\n /// let p = Shared::::null();\n /// assert!(p.is_null());\n /// ```\n pub const fn null() -> Self {\n Self {\n data: ptr::null_mut(),\n _marker: PhantomData,\n }\n }\n\n /// Returns `true` if the pointer is null.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Owned};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::null();\n /// let guard = &epoch::pin();\n /// assert!(a.load(SeqCst, guard).is_null());\n /// a.store(Owned::new(1234), SeqCst);\n /// assert!(!a.load(SeqCst, guard).is_null());\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn is_null(&self) -> bool {\n let (raw, _) = decompose_tag::(self.data);\n raw.is_null()\n }\n\n pub(crate) unsafe fn as_ptr(&self) -> *const T {\n let (raw, _) = decompose_tag::(self.data);\n unsafe { T::as_ptr(raw) }\n }\n\n pub(crate) unsafe fn as_mut_ptr(&self) -> *mut T {\n let (raw, _) = decompose_tag::(self.data);\n unsafe { T::as_mut_ptr(raw) }\n }\n\n /// Dereferences the pointer.\n ///\n /// Returns a reference to the pointee that is valid during the lifetime `'g`.\n ///\n /// # Safety\n ///\n /// Dereferencing a pointer is unsafe because it could be pointing to invalid memory.\n ///\n /// Another concern is the possibility of data races due to lack of proper synchronization.\n /// For example, consider the following scenario:\n ///\n /// 1. A thread creates a new object: `a.store(Owned::new(10), Relaxed)`\n /// 2. Another thread reads it: `*a.load(Relaxed, guard).as_ref().unwrap()`\n ///\n /// The problem is that relaxed orderings don't synchronize initialization of the object with\n /// the read from the second thread. This is a data race. A possible solution would be to use\n /// `Release` and `Acquire` orderings.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(1234);\n /// let guard = &epoch::pin();\n /// let p = a.load(SeqCst, guard);\n /// unsafe {\n /// assert_eq!(p.deref(), &1234);\n /// }\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub unsafe fn deref(&self) -> &'g T {\n unsafe { &*self.as_ptr() }\n }\n\n /// Dereferences the pointer.\n ///\n /// Returns a mutable reference to the pointee that is valid during the lifetime `'g`.\n ///\n /// # Safety\n ///\n /// * There is no guarantee that there are no more threads attempting to read/write from/to the\n /// actual object at the same time.\n ///\n /// The user must know that there are no concurrent accesses towards the object itself.\n ///\n /// * Other than the above, all safety concerns of `deref()` applies here.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(vec![1, 2, 3, 4]);\n /// let guard = &epoch::pin();\n ///\n /// let mut p = a.load(SeqCst, guard);\n /// unsafe {\n /// assert!(!p.is_null());\n /// let b = p.deref_mut();\n /// assert_eq!(b, &vec![1, 2, 3, 4]);\n /// b.push(5);\n /// assert_eq!(b, &vec![1, 2, 3, 4, 5]);\n /// }\n ///\n /// let p = a.load(SeqCst, guard);\n /// unsafe {\n /// assert_eq!(p.deref(), &vec![1, 2, 3, 4, 5]);\n /// }\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub unsafe fn deref_mut(&mut self) -> &'g mut T {\n unsafe { &mut *self.as_mut_ptr() }\n }\n\n /// Converts the pointer to a reference.\n ///\n /// Returns `None` if the pointer is null, or else a reference to the object wrapped in `Some`.\n ///\n /// # Safety\n ///\n /// Dereferencing a pointer is unsafe because it could be pointing to invalid memory.\n ///\n /// Another concern is the possibility of data races due to lack of proper synchronization.\n /// For example, consider the following scenario:\n ///\n /// 1. A thread creates a new object: `a.store(Owned::new(10), Relaxed)`\n /// 2. Another thread reads it: `*a.load(Relaxed, guard).as_ref().unwrap()`\n ///\n /// The problem is that relaxed orderings don't synchronize initialization of the object with\n /// the read from the second thread. This is a data race. A possible solution would be to use\n /// `Release` and `Acquire` orderings.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(1234);\n /// let guard = &epoch::pin();\n /// let p = a.load(SeqCst, guard);\n /// unsafe {\n /// assert_eq!(p.as_ref(), Some(&1234));\n /// }\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub unsafe fn as_ref(&self) -> Option<&'g T> {\n let (raw, _) = decompose_tag::(self.data);\n if raw.is_null() {\n None\n } else {\n Some(unsafe { &*T::as_ptr(raw) })\n }\n }\n\n /// Takes ownership of the pointee.\n ///\n /// # Panics\n ///\n /// Panics if this pointer is null, but only in debug mode.\n ///\n /// # Safety\n ///\n /// This method may be called only if the pointer is valid and nobody else is holding a\n /// reference to the same object.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(1234);\n /// unsafe {\n /// let guard = &epoch::unprotected();\n /// let p = a.load(SeqCst, guard);\n /// drop(p.into_owned());\n /// }\n /// ```\n pub unsafe fn into_owned(self) -> Owned {\n debug_assert!(!self.is_null(), \"converting a null `Shared` into `Owned`\");\n unsafe { Owned::from_ptr(self.data) }\n }\n\n /// Takes ownership of the pointee if it is not null.\n ///\n /// # Safety\n ///\n /// This method may be called only if the pointer is valid and nobody else is holding a\n /// reference to the same object, or if the pointer is null.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(1234);\n /// unsafe {\n /// let guard = &epoch::unprotected();\n /// let p = a.load(SeqCst, guard);\n /// if let Some(x) = p.try_into_owned() {\n /// drop(x);\n /// }\n /// }\n /// ```\n pub unsafe fn try_into_owned(self) -> Option> {\n if self.is_null() {\n None\n } else {\n Some(unsafe { Owned::from_ptr(self.data) })\n }\n }\n\n /// Returns the tag stored within the pointer.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Owned};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::::from(Owned::new(0u64).with_tag(2));\n /// let guard = &epoch::pin();\n /// let p = a.load(SeqCst, guard);\n /// assert_eq!(p.tag(), 2);\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn tag(&self) -> usize {\n let (_, tag) = decompose_tag::(self.data);\n tag\n }\n\n /// Returns the same pointer, but tagged with `tag`. `tag` is truncated to be fit into the\n /// unused bits of the pointer to `T`.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(0u64);\n /// let guard = &epoch::pin();\n /// let p1 = a.load(SeqCst, guard);\n /// let p2 = p1.with_tag(2);\n ///\n /// assert_eq!(p1.tag(), 0);\n /// assert_eq!(p2.tag(), 2);\n /// assert_eq!(p1.as_raw(), p2.as_raw());\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn with_tag(&self, tag: usize) -> Self {\n unsafe { Self::from_ptr(compose_tag::(self.data, tag)) }\n }\n}\n\nimpl From<*const T> for Shared<'_, T> {\n /// Returns a new pointer pointing to `raw`.\n ///\n /// # Panics\n ///\n /// Panics if `raw` is not properly aligned.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::Shared;\n ///\n /// let p = Shared::from(Box::into_raw(Box::new(1234)) as *const _);\n /// assert!(!p.is_null());\n /// # unsafe { drop(p.into_owned()); } // avoid leak\n /// ```\n fn from(raw: *const T) -> Self {\n let raw = raw as *mut ();\n ensure_aligned::(raw);\n unsafe { Self::from_ptr(raw) }\n }\n}\n\nimpl PartialEq for Shared<'_, T> {\n fn eq(&self, other: &Self) -> bool {\n self.data == other.data\n }\n}\n\nimpl Eq for Shared<'_, T> {}\n\nimpl PartialOrd for Shared<'_, T> {\n fn partial_cmp(&self, other: &Self) -> Option {\n Some(self.cmp(other))\n }\n}\n\nimpl Ord for Shared<'_, T> {\n fn cmp(&self, other: &Self) -> cmp::Ordering {\n self.data.cmp(&other.data)\n }\n}\n\nimpl fmt::Debug for Shared<'_, T> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n let (raw, tag) = decompose_tag::(self.data);\n\n f.debug_struct(\"Shared\")\n .field(\"raw\", &raw)\n .field(\"tag\", &tag)\n .finish()\n }\n}\n\nimpl fmt::Pointer for Shared<'_, T> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n fmt::Pointer::fmt(&(unsafe { self.as_ptr() }), f)\n }\n}\n\nimpl Default for Shared<'_, T> {\n fn default() -> Self {\n Self::null()\n }\n}\n\n#[cfg(all(test, not(crossbeam_loom)))]\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\nmod tests {\n use std::{mem::MaybeUninit, sync::atomic::Ordering};\n\n use super::{Atomic, Owned, Shared};\n use crate::pin;\n\n #[test]\n fn valid_tag_i8() {\n Shared::::null().with_tag(0);\n }\n\n #[test]\n fn valid_tag_i64() {\n Shared::::null().with_tag(7);\n }\n\n #[test]\n fn const_null() {\n use super::{Atomic, Shared};\n static _A: Atomic = Atomic::::null();\n static _S: () = {\n let _shared = Shared::::null();\n };\n }\n\n #[test]\n fn array_init() {\n let owned = Owned::<[MaybeUninit]>::init(10);\n let arr: &[MaybeUninit] = &owned;\n assert_eq!(arr.len(), 10);\n }\n\n #[test]\n fn compare_exchange_success() {\n let atomic = Atomic::new(42);\n let guard = &pin();\n\n let current = atomic.load(Ordering::SeqCst, guard);\n let new_value = Owned::new(100);\n\n let result = atomic\n .compare_exchange(\n current,\n new_value,\n Ordering::SeqCst,\n Ordering::SeqCst,\n guard,\n )\n .unwrap();\n // On success, `old` should equal the current value we loaded\n assert_eq!(unsafe { result.old.deref() }, &42);\n // `new` should equal the value we stored\n assert_eq!(unsafe { result.new.deref() }, &100);\n\n // Verify the atomic actually contains the new value\n let current = atomic.load(Ordering::SeqCst, guard);\n assert_eq!(unsafe { current.deref() }, &100);\n\n unsafe {\n drop(result.old.into_owned());\n drop(atomic.into_owned());\n }\n }\n\n #[test]\n fn compare_exchange_failure() {\n let atomic = Atomic::new(42);\n let guard = &pin();\n\n // Load the current value\n let current = atomic.load(Ordering::SeqCst, guard);\n\n let old_value = atomic.swap(Owned::new(200), Ordering::SeqCst, guard);\n unsafe {\n drop(old_value.into_owned());\n }\n\n // Now try to compare_exchange with the old current value - this should fail\n let new_value = Owned::new(300);\n let error = atomic\n .compare_exchange(\n current,\n new_value,\n Ordering::SeqCst,\n Ordering::SeqCst,\n guard,\n )\n .unwrap_err();\n // On failure, `current` should contain the actual current value (200)\n assert_eq!(unsafe { error.current.deref() }, &200);\n // `new` should contain the value we tried to store (300)\n assert_eq!(&*error.new, &300);\n\n // Verify the atomic still contains the value we set (200), not the failed attempt (300)\n let current = atomic.load(Ordering::SeqCst, guard);\n assert_eq!(unsafe { current.deref() }, &200);\n\n unsafe {\n drop(atomic.into_owned());\n }\n }\n\n #[test]\n fn compare_exchange_weak_success() {\n let atomic = Atomic::new(42);\n let guard = &pin();\n\n let mut current = atomic.load(Ordering::SeqCst, guard);\n let mut new_value = Owned::new(100);\n\n loop {\n match atomic.compare_exchange_weak(\n current,\n new_value,\n Ordering::SeqCst,\n Ordering::SeqCst,\n guard,\n ) {\n Ok(result) => {\n // On success, `old` should equal the current value we loaded\n assert_eq!(unsafe { result.old.deref() }, &42);\n // `new` should equal the value we stored\n assert_eq!(unsafe { result.new.deref() }, &100);\n\n // Verify the atomic actually contains the new value\n let current = atomic.load(Ordering::SeqCst, guard);\n assert_eq!(unsafe { current.deref() }, &100);\n\n unsafe {\n drop(result.old.into_owned());\n }\n break;\n }\n Err(e) => {\n current = e.current;\n new_value = e.new;\n assert_eq!(unsafe { current.deref() }, &42);\n }\n }\n }\n\n unsafe {\n drop(atomic.into_owned());\n }\n }\n}\n" }, { "path": "crossbeam-epoch/src/collector.rs", "content": "/// Epoch-based garbage collector.\n///\n/// # Examples\n///\n/// ```\n/// use crossbeam_epoch::Collector;\n///\n/// let collector = Collector::new();\n///\n/// let handle = collector.register();\n/// drop(collector); // `handle` still works after dropping `collector`\n///\n/// handle.pin().flush();\n/// ```\nuse core::fmt;\n\nuse crate::{\n guard::Guard,\n internal::{Global, Local},\n primitive::sync::Arc,\n};\n\n/// An epoch-based garbage collector.\npub struct Collector {\n pub(crate) global: Arc,\n}\n\nunsafe impl Send for Collector {}\nunsafe impl Sync for Collector {}\n\nimpl Default for Collector {\n #[allow(clippy::arc_with_non_send_sync)] // https://github.com/rust-lang/rust-clippy/issues/11382\n fn default() -> Self {\n Self {\n global: Arc::new(Global::new()),\n }\n }\n}\n\nimpl Collector {\n /// Creates a new collector.\n pub fn new() -> Self {\n Self::default()\n }\n\n /// Registers a new handle for the collector.\n pub fn register(&self) -> LocalHandle {\n Local::register(self)\n }\n}\n\nimpl Clone for Collector {\n /// Creates another reference to the same garbage collector.\n fn clone(&self) -> Self {\n Self {\n global: self.global.clone(),\n }\n }\n}\n\nimpl fmt::Debug for Collector {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Collector { .. }\")\n }\n}\n\nimpl PartialEq for Collector {\n /// Checks if both handles point to the same collector.\n fn eq(&self, rhs: &Self) -> bool {\n Arc::ptr_eq(&self.global, &rhs.global)\n }\n}\nimpl Eq for Collector {}\n\n/// A handle to a garbage collector.\npub struct LocalHandle {\n pub(crate) local: *const Local,\n}\n\nimpl LocalHandle {\n /// Pins the handle.\n #[inline]\n pub fn pin(&self) -> Guard {\n unsafe { (*self.local).pin() }\n }\n\n /// Returns `true` if the handle is pinned.\n #[inline]\n pub fn is_pinned(&self) -> bool {\n unsafe { (*self.local).is_pinned() }\n }\n\n /// Returns the `Collector` associated with this handle.\n #[inline]\n pub fn collector(&self) -> &Collector {\n unsafe { (*self.local).collector() }\n }\n}\n\nimpl Drop for LocalHandle {\n #[inline]\n fn drop(&mut self) {\n unsafe {\n Local::release_handle(&*self.local);\n }\n }\n}\n\nimpl fmt::Debug for LocalHandle {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"LocalHandle { .. }\")\n }\n}\n\n#[cfg(all(test, not(crossbeam_loom)))]\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\nmod tests {\n use std::{\n mem::ManuallyDrop,\n sync::atomic::{AtomicUsize, Ordering},\n vec::Vec,\n };\n\n use crossbeam_utils::thread;\n\n use crate::{Collector, Owned};\n\n const NUM_THREADS: usize = 8;\n\n #[test]\n fn pin_reentrant() {\n let collector = Collector::new();\n let handle = collector.register();\n drop(collector);\n\n assert!(!handle.is_pinned());\n {\n let _guard = &handle.pin();\n assert!(handle.is_pinned());\n {\n let _guard = &handle.pin();\n assert!(handle.is_pinned());\n }\n assert!(handle.is_pinned());\n }\n assert!(!handle.is_pinned());\n }\n\n #[test]\n fn flush_local_bag() {\n let collector = Collector::new();\n let handle = collector.register();\n drop(collector);\n\n for _ in 0..100 {\n let guard = &handle.pin();\n unsafe {\n let a = Owned::new(7).into_shared(guard);\n guard.defer_destroy(a);\n\n assert!(!(*guard.local).bag.with(|b| (*b).is_empty()));\n\n while !(*guard.local).bag.with(|b| (*b).is_empty()) {\n guard.flush();\n }\n }\n }\n }\n\n #[test]\n fn garbage_buffering() {\n let collector = Collector::new();\n let handle = collector.register();\n drop(collector);\n\n let guard = &handle.pin();\n unsafe {\n for _ in 0..10 {\n let a = Owned::new(7).into_shared(guard);\n guard.defer_destroy(a);\n }\n assert!(!(*guard.local).bag.with(|b| (*b).is_empty()));\n }\n }\n\n #[test]\n fn pin_holds_advance() {\n const N: usize = if cfg!(miri) { 500 } else { 500_000 };\n\n let collector = Collector::new();\n\n thread::scope(|scope| {\n for _ in 0..NUM_THREADS {\n scope.spawn(|_| {\n let handle = collector.register();\n for _ in 0..N {\n let guard = &handle.pin();\n\n let before = collector.global.epoch.load(Ordering::Relaxed);\n collector.global.collect(guard);\n let after = collector.global.epoch.load(Ordering::Relaxed);\n\n assert!(after.wrapping_sub(before) <= 2);\n }\n });\n }\n })\n .unwrap();\n }\n\n #[test]\n // TODO: assertions failed due to `cfg(crossbeam_sanitize)` reduce `internal::MAX_OBJECTS`: https://github.com/crossbeam-rs/crossbeam/issues/662\n #[cfg_attr(crossbeam_sanitize, ignore)]\n fn incremental() {\n const COUNT: usize = if cfg!(miri) { 500 } else { 100_000 };\n static DESTROYS: AtomicUsize = AtomicUsize::new(0);\n\n let collector = Collector::new();\n let handle = collector.register();\n\n unsafe {\n let guard = &handle.pin();\n for _ in 0..COUNT {\n let a = Owned::new(7i32).into_shared(guard);\n guard.defer_unchecked(move || {\n drop(a.into_owned());\n DESTROYS.fetch_add(1, Ordering::Relaxed);\n });\n }\n guard.flush();\n }\n\n let mut last = 0;\n\n while last < COUNT {\n let curr = DESTROYS.load(Ordering::Relaxed);\n assert!(curr - last <= 1024);\n last = curr;\n\n let guard = &handle.pin();\n collector.global.collect(guard);\n }\n assert!(DESTROYS.load(Ordering::Relaxed) == COUNT);\n }\n\n #[test]\n fn buffering() {\n const COUNT: usize = 10;\n const N: usize = if cfg!(miri) { 500 } else { 100_000 };\n static DESTROYS: AtomicUsize = AtomicUsize::new(0);\n\n let collector = Collector::new();\n let handle = collector.register();\n\n unsafe {\n let guard = &handle.pin();\n for _ in 0..COUNT {\n let a = Owned::new(7i32).into_shared(guard);\n guard.defer_unchecked(move || {\n drop(a.into_owned());\n DESTROYS.fetch_add(1, Ordering::Relaxed);\n });\n }\n }\n\n for _ in 0..N {\n collector.global.collect(&handle.pin());\n }\n assert!(DESTROYS.load(Ordering::Relaxed) < COUNT);\n\n handle.pin().flush();\n\n while DESTROYS.load(Ordering::Relaxed) < COUNT {\n let guard = &handle.pin();\n collector.global.collect(guard);\n }\n assert_eq!(DESTROYS.load(Ordering::Relaxed), COUNT);\n }\n\n #[test]\n fn count_drops() {\n const COUNT: usize = if cfg!(miri) { 500 } else { 100_000 };\n static DROPS: AtomicUsize = AtomicUsize::new(0);\n\n struct Elem(#[allow(dead_code)] i32);\n\n impl Drop for Elem {\n fn drop(&mut self) {\n DROPS.fetch_add(1, Ordering::Relaxed);\n }\n }\n\n let collector = Collector::new();\n let handle = collector.register();\n\n unsafe {\n let guard = &handle.pin();\n\n for _ in 0..COUNT {\n let a = Owned::new(Elem(7i32)).into_shared(guard);\n guard.defer_destroy(a);\n }\n guard.flush();\n }\n\n while DROPS.load(Ordering::Relaxed) < COUNT {\n let guard = &handle.pin();\n collector.global.collect(guard);\n }\n assert_eq!(DROPS.load(Ordering::Relaxed), COUNT);\n }\n\n #[test]\n fn count_destroy() {\n const COUNT: usize = if cfg!(miri) { 500 } else { 100_000 };\n static DESTROYS: AtomicUsize = AtomicUsize::new(0);\n\n let collector = Collector::new();\n let handle = collector.register();\n\n unsafe {\n let guard = &handle.pin();\n\n for _ in 0..COUNT {\n let a = Owned::new(7i32).into_shared(guard);\n guard.defer_unchecked(move || {\n drop(a.into_owned());\n DESTROYS.fetch_add(1, Ordering::Relaxed);\n });\n }\n guard.flush();\n }\n\n while DESTROYS.load(Ordering::Relaxed) < COUNT {\n let guard = &handle.pin();\n collector.global.collect(guard);\n }\n assert_eq!(DESTROYS.load(Ordering::Relaxed), COUNT);\n }\n\n #[test]\n fn drop_array() {\n const COUNT: usize = 700;\n static DROPS: AtomicUsize = AtomicUsize::new(0);\n\n struct Elem(#[allow(dead_code)] i32);\n\n impl Drop for Elem {\n fn drop(&mut self) {\n DROPS.fetch_add(1, Ordering::Relaxed);\n }\n }\n\n let collector = Collector::new();\n let handle = collector.register();\n\n let mut guard = handle.pin();\n\n let mut v = Vec::with_capacity(COUNT);\n for i in 0..COUNT {\n v.push(Elem(i as i32));\n }\n\n {\n let a = Owned::new(v).into_shared(&guard);\n unsafe {\n guard.defer_destroy(a);\n }\n guard.flush();\n }\n\n while DROPS.load(Ordering::Relaxed) < COUNT {\n guard.repin();\n collector.global.collect(&guard);\n }\n assert_eq!(DROPS.load(Ordering::Relaxed), COUNT);\n }\n\n #[test]\n fn destroy_array() {\n const COUNT: usize = if cfg!(miri) { 500 } else { 100_000 };\n static DESTROYS: AtomicUsize = AtomicUsize::new(0);\n\n let collector = Collector::new();\n let handle = collector.register();\n\n unsafe {\n let guard = &handle.pin();\n\n let mut v = Vec::with_capacity(COUNT);\n for i in 0..COUNT {\n v.push(i as i32);\n }\n\n let len = v.len();\n let cap = v.capacity();\n let ptr = ManuallyDrop::new(v).as_mut_ptr();\n guard.defer_unchecked(move || {\n drop(Vec::from_raw_parts(ptr, len, cap));\n DESTROYS.fetch_add(len, Ordering::Relaxed);\n });\n guard.flush();\n }\n\n while DESTROYS.load(Ordering::Relaxed) < COUNT {\n let guard = &handle.pin();\n collector.global.collect(guard);\n }\n assert_eq!(DESTROYS.load(Ordering::Relaxed), COUNT);\n }\n\n #[test]\n fn stress() {\n const THREADS: usize = 8;\n const COUNT: usize = if cfg!(miri) { 500 } else { 100_000 };\n static DROPS: AtomicUsize = AtomicUsize::new(0);\n\n struct Elem(#[allow(dead_code)] i32);\n\n impl Drop for Elem {\n fn drop(&mut self) {\n DROPS.fetch_add(1, Ordering::Relaxed);\n }\n }\n\n let collector = Collector::new();\n\n thread::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n let handle = collector.register();\n for _ in 0..COUNT {\n let guard = &handle.pin();\n unsafe {\n let a = Owned::new(Elem(7i32)).into_shared(guard);\n guard.defer_destroy(a);\n }\n }\n });\n }\n })\n .unwrap();\n\n let handle = collector.register();\n while DROPS.load(Ordering::Relaxed) < COUNT * THREADS {\n let guard = &handle.pin();\n collector.global.collect(guard);\n }\n assert_eq!(DROPS.load(Ordering::Relaxed), COUNT * THREADS);\n }\n}\n" }, { "path": "crossbeam-epoch/src/default.rs", "content": "//! The default garbage collector.\n//!\n//! For each thread, a participant is lazily initialized on its first use, when the current thread\n//! is registered in the default collector. If initialized, the thread's participant will get\n//! destructed on thread exit, which in turn unregisters the thread.\n\n#[cfg(not(crossbeam_loom))]\nuse std::sync::OnceLock;\n\nuse crate::{\n collector::{Collector, LocalHandle},\n guard::Guard,\n primitive::thread_local,\n};\n\nfn collector() -> &'static Collector {\n #[cfg(not(crossbeam_loom))]\n {\n /// The global data for the default garbage collector.\n static COLLECTOR: OnceLock = OnceLock::new();\n COLLECTOR.get_or_init(Collector::new)\n }\n // FIXME: loom does not currently provide the equivalent of Lazy:\n // https://github.com/tokio-rs/loom/issues/263\n #[cfg(crossbeam_loom)]\n {\n loom::lazy_static! {\n /// The global data for the default garbage collector.\n static ref COLLECTOR: Collector = Collector::new();\n }\n &COLLECTOR\n }\n}\n\nthread_local! {\n /// The per-thread participant for the default garbage collector.\n static HANDLE: LocalHandle = collector().register();\n}\n\n/// Pins the current thread.\n#[inline]\npub fn pin() -> Guard {\n with_handle(|handle| handle.pin())\n}\n\n/// Returns `true` if the current thread is pinned.\n#[inline]\npub fn is_pinned() -> bool {\n with_handle(|handle| handle.is_pinned())\n}\n\n/// Returns the default global collector.\npub fn default_collector() -> &'static Collector {\n collector()\n}\n\n#[inline]\nfn with_handle(mut f: F) -> R\nwhere\n F: FnMut(&LocalHandle) -> R,\n{\n HANDLE\n .try_with(|h| f(h))\n .unwrap_or_else(|_| f(&collector().register()))\n}\n\n#[cfg(all(test, not(crossbeam_loom)))]\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\nmod tests {\n use crossbeam_utils::thread;\n\n #[test]\n fn pin_while_exiting() {\n struct Foo;\n\n impl Drop for Foo {\n fn drop(&mut self) {\n // Pin after `HANDLE` has been dropped. This must not panic.\n super::pin();\n }\n }\n\n std::thread_local! {\n static FOO: Foo = const { Foo };\n }\n\n thread::scope(|scope| {\n scope.spawn(|_| {\n // Initialize `FOO` and then `HANDLE`.\n FOO.with(|_| ());\n super::pin();\n // At thread exit, `HANDLE` gets dropped first and `FOO` second.\n });\n })\n .unwrap();\n }\n}\n" }, { "path": "crossbeam-epoch/src/deferred.rs", "content": "use alloc::boxed::Box;\nuse core::{\n fmt,\n marker::PhantomData,\n mem::{self, MaybeUninit},\n ptr,\n};\n\n/// Number of words a piece of `Data` can hold.\n///\n/// Three words should be enough for the majority of cases. For example, you can fit inside it the\n/// function pointer together with a fat pointer representing an object that needs to be destroyed.\nconst DATA_WORDS: usize = 3;\n\n/// Some space to keep a `FnOnce()` object on the stack.\ntype Data = [usize; DATA_WORDS];\n\n/// A `FnOnce()` that is stored inline if small, or otherwise boxed on the heap.\n///\n/// This is a handy way of keeping an unsized `FnOnce()` within a sized structure.\npub(crate) struct Deferred {\n call: unsafe fn(*mut u8),\n data: MaybeUninit,\n _marker: PhantomData<*mut ()>, // !Send + !Sync\n}\n\nimpl fmt::Debug for Deferred {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {\n f.pad(\"Deferred { .. }\")\n }\n}\n\nimpl Deferred {\n pub(crate) const NO_OP: Self = {\n fn no_op_call(_raw: *mut u8) {}\n Self {\n call: no_op_call,\n data: MaybeUninit::uninit(),\n _marker: PhantomData,\n }\n };\n\n /// Constructs a new `Deferred` from a `FnOnce()`.\n pub(crate) fn new(f: F) -> Self {\n let size = mem::size_of::();\n let align = mem::align_of::();\n\n unsafe {\n if size <= mem::size_of::() && align <= mem::align_of::() {\n let mut data = MaybeUninit::::uninit();\n ptr::write(data.as_mut_ptr().cast::(), f);\n\n unsafe fn call(raw: *mut u8) {\n let f: F = unsafe { ptr::read(raw.cast::()) };\n f();\n }\n\n Self {\n call: call::,\n data,\n _marker: PhantomData,\n }\n } else {\n let b: Box = Box::new(f);\n let mut data = MaybeUninit::::uninit();\n ptr::write(data.as_mut_ptr().cast::>(), b);\n\n unsafe fn call(raw: *mut u8) {\n // It's safe to cast `raw` from `*mut u8` to `*mut Box`, because `raw` is\n // originally derived from `*mut Box`.\n let b: Box = unsafe { ptr::read(raw.cast::>()) };\n (*b)();\n }\n\n Self {\n call: call::,\n data,\n _marker: PhantomData,\n }\n }\n }\n }\n\n /// Calls the function.\n #[inline]\n pub(crate) fn call(mut self) {\n let call = self.call;\n unsafe { call(self.data.as_mut_ptr().cast::()) };\n }\n}\n\n#[cfg(all(test, not(crossbeam_loom)))]\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\nmod tests {\n use std::{boxed::Box, cell::Cell, convert::identity, string::ToString, vec};\n\n use super::Deferred;\n\n #[test]\n fn on_stack() {\n let fired = &Cell::new(false);\n let a = [0usize; 1];\n\n let d = Deferred::new(move || {\n let _ = identity(a);\n fired.set(true);\n });\n\n assert!(!fired.get());\n d.call();\n assert!(fired.get());\n }\n\n #[test]\n fn on_heap() {\n let fired = &Cell::new(false);\n let a = [0usize; 10];\n\n let d = Deferred::new(move || {\n let _ = identity(a);\n fired.set(true);\n });\n\n assert!(!fired.get());\n d.call();\n assert!(fired.get());\n }\n\n #[test]\n fn string() {\n let a = \"hello\".to_string();\n let d = Deferred::new(move || assert_eq!(a, \"hello\"));\n d.call();\n }\n\n #[test]\n fn boxed_slice_i32() {\n let a: Box<[i32]> = vec![2, 3, 5, 7].into_boxed_slice();\n let d = Deferred::new(move || assert_eq!(*a, [2, 3, 5, 7]));\n d.call();\n }\n\n #[test]\n fn long_slice_usize() {\n let a: [usize; 5] = [2, 3, 5, 7, 11];\n let d = Deferred::new(move || assert_eq!(a, [2, 3, 5, 7, 11]));\n d.call();\n }\n}\n" }, { "path": "crossbeam-epoch/src/epoch.rs", "content": "//! The global epoch\n//!\n//! The last bit in this number is unused and is always zero. Every so often the global epoch is\n//! incremented, i.e. we say it \"advances\". A pinned participant may advance the global epoch only\n//! if all currently pinned participants have been pinned in the current epoch.\n//!\n//! If an object became garbage in some epoch, then we can be sure that after two advancements no\n//! participant will hold a reference to it. That is the crux of safe memory reclamation.\n\nuse crate::primitive::sync::atomic::Ordering;\n\n// Ideally, we want to always use AtomicU64, but since it is not available on all platforms,\n// we only use it when it is available for now.\n// TODO: On platforms where AtomicU64 is unavailable, we may want to use AtomicCell instead of AtomicUsize.\n#[cfg(target_has_atomic = \"64\")]\ntype AtomicEpochRepr = crate::primitive::sync::atomic::AtomicU64;\n#[cfg(not(target_has_atomic = \"64\"))]\ntype AtomicEpochRepr = crate::primitive::sync::atomic::AtomicUsize;\n#[cfg(target_has_atomic = \"64\")]\ntype EpochRepr = u64;\n#[cfg(not(target_has_atomic = \"64\"))]\ntype EpochRepr = usize;\n#[cfg(target_has_atomic = \"64\")]\ntype EpochReprSigned = i64;\n#[cfg(not(target_has_atomic = \"64\"))]\ntype EpochReprSigned = isize;\n\n/// An epoch that can be marked as pinned or unpinned.\n///\n/// Internally, the epoch is represented as an integer that wraps around at some unspecified point\n/// and a flag that represents whether it is pinned or unpinned.\n#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)]\npub(crate) struct Epoch {\n /// The least significant bit is set if pinned. The rest of the bits hold the epoch.\n data: EpochRepr,\n}\n\nimpl Epoch {\n /// Returns the starting epoch in unpinned state.\n #[inline]\n pub(crate) fn starting() -> Self {\n Self::default()\n }\n\n /// Returns the number of epochs `self` is ahead of `rhs`.\n ///\n /// Internally, epochs are represented as numbers in the range `(isize::MIN / 2) .. (isize::MAX\n /// / 2)`, so the returned distance will be in the same interval.\n pub(crate) fn wrapping_sub(self, rhs: Self) -> EpochReprSigned {\n // The result is the same with `(self.data & !1).wrapping_sub(rhs.data & !1) as isize >> 1`,\n // because the possible difference of LSB in `(self.data & !1).wrapping_sub(rhs.data & !1)`\n // will be ignored in the shift operation.\n self.data.wrapping_sub(rhs.data & !1) as EpochReprSigned >> 1\n }\n\n /// Returns `true` if the epoch is marked as pinned.\n #[inline]\n pub(crate) fn is_pinned(self) -> bool {\n (self.data & 1) == 1\n }\n\n /// Returns the same epoch, but marked as pinned.\n #[inline]\n pub(crate) fn pinned(self) -> Self {\n Self {\n data: self.data | 1,\n }\n }\n\n /// Returns the same epoch, but marked as unpinned.\n #[inline]\n pub(crate) fn unpinned(self) -> Self {\n Self {\n data: self.data & !1,\n }\n }\n\n /// Returns the successor epoch.\n ///\n /// The returned epoch will be marked as pinned only if the previous one was as well.\n #[inline]\n pub(crate) fn successor(self) -> Self {\n Self {\n data: self.data.wrapping_add(2),\n }\n }\n}\n\n/// An atomic value that holds an `Epoch`.\n#[derive(Default, Debug)]\npub(crate) struct AtomicEpoch {\n /// Since `Epoch` is just a wrapper around `usize`, an `AtomicEpoch` is similarly represented\n /// using an `AtomicUsize`.\n data: AtomicEpochRepr,\n}\n\nimpl AtomicEpoch {\n /// Creates a new atomic epoch.\n #[inline]\n pub(crate) fn new(epoch: Epoch) -> Self {\n let data = AtomicEpochRepr::new(epoch.data);\n Self { data }\n }\n\n /// Loads a value from the atomic epoch.\n #[inline]\n pub(crate) fn load(&self, ord: Ordering) -> Epoch {\n Epoch {\n data: self.data.load(ord),\n }\n }\n\n /// Stores a value into the atomic epoch.\n #[inline]\n pub(crate) fn store(&self, epoch: Epoch, ord: Ordering) {\n self.data.store(epoch.data, ord);\n }\n\n /// Stores a value into the atomic epoch if the current value is the same as `current`.\n ///\n /// The return value is a result indicating whether the new value was written and containing\n /// the previous value. On success this value is guaranteed to be equal to `current`.\n ///\n /// This method takes two `Ordering` arguments to describe the memory\n /// ordering of this operation. `success` describes the required ordering for the\n /// read-modify-write operation that takes place if the comparison with `current` succeeds.\n /// `failure` describes the required ordering for the load operation that takes place when\n /// the comparison fails. Using `Acquire` as success ordering makes the store part\n /// of this operation `Relaxed`, and using `Release` makes the successful load\n /// `Relaxed`. The failure ordering can only be `SeqCst`, `Acquire` or `Relaxed`\n /// and must be equivalent to or weaker than the success ordering.\n #[inline]\n pub(crate) fn compare_exchange(\n &self,\n current: Epoch,\n new: Epoch,\n success: Ordering,\n failure: Ordering,\n ) -> Result {\n match self\n .data\n .compare_exchange(current.data, new.data, success, failure)\n {\n Ok(data) => Ok(Epoch { data }),\n Err(data) => Err(Epoch { data }),\n }\n }\n}\n" }, { "path": "crossbeam-epoch/src/guard.rs", "content": "use core::{fmt, mem};\n\nuse crate::{\n atomic::{Pointable, Shared},\n collector::Collector,\n deferred::Deferred,\n internal::Local,\n};\n\n/// A guard that keeps the current thread pinned.\n///\n/// # Pinning\n///\n/// The current thread is pinned by calling [`pin`], which returns a new guard:\n///\n/// ```\n/// use crossbeam_epoch as epoch;\n///\n/// // It is often convenient to prefix a call to `pin` with a `&` in order to create a reference.\n/// // This is not really necessary, but makes passing references to the guard a bit easier.\n/// let guard = &epoch::pin();\n/// ```\n///\n/// When a guard gets dropped, the current thread is automatically unpinned.\n///\n/// # Pointers on the stack\n///\n/// Having a guard allows us to create pointers on the stack to heap-allocated objects.\n/// For example:\n///\n/// ```\n/// use crossbeam_epoch::{self as epoch, Atomic};\n/// use std::sync::atomic::Ordering::SeqCst;\n///\n/// // Create a heap-allocated number.\n/// let a = Atomic::new(777);\n///\n/// // Pin the current thread.\n/// let guard = &epoch::pin();\n///\n/// // Load the heap-allocated object and create pointer `p` on the stack.\n/// let p = a.load(SeqCst, guard);\n///\n/// // Dereference the pointer and print the value:\n/// if let Some(num) = unsafe { p.as_ref() } {\n/// println!(\"The number is {}.\", num);\n/// }\n/// # unsafe { drop(a.into_owned()); } // avoid leak\n/// ```\n///\n/// # Multiple guards\n///\n/// Pinning is reentrant and it is perfectly legal to create multiple guards. In that case, the\n/// thread will actually be pinned only when the first guard is created and unpinned when the last\n/// one is dropped:\n///\n/// ```\n/// use crossbeam_epoch as epoch;\n///\n/// let guard1 = epoch::pin();\n/// let guard2 = epoch::pin();\n/// assert!(epoch::is_pinned());\n/// drop(guard1);\n/// assert!(epoch::is_pinned());\n/// drop(guard2);\n/// assert!(!epoch::is_pinned());\n/// ```\n///\n/// [`pin`]: super::pin\npub struct Guard {\n pub(crate) local: *const Local,\n}\n\nimpl Guard {\n /// Stores a function so that it can be executed at some point after all currently pinned\n /// threads get unpinned.\n ///\n /// This method first stores `f` into the thread-local (or handle-local) cache. If this cache\n /// becomes full, some functions are moved into the global cache. At the same time, some\n /// functions from both local and global caches may get executed in order to incrementally\n /// clean up the caches as they fill up.\n ///\n /// There is no guarantee when exactly `f` will be executed. The only guarantee is that it\n /// won't be executed until all currently pinned threads get unpinned. In theory, `f` might\n /// never run, but the epoch-based garbage collection will make an effort to execute it\n /// reasonably soon.\n ///\n /// If this method is called from an [`unprotected`] guard, the function will simply be\n /// executed immediately.\n pub fn defer(&self, f: F)\n where\n F: FnOnce() -> R,\n F: Send + 'static,\n {\n unsafe {\n self.defer_unchecked(f);\n }\n }\n\n /// Stores a function so that it can be executed at some point after all currently pinned\n /// threads get unpinned.\n ///\n /// This method first stores `f` into the thread-local (or handle-local) cache. If this cache\n /// becomes full, some functions are moved into the global cache. At the same time, some\n /// functions from both local and global caches may get executed in order to incrementally\n /// clean up the caches as they fill up.\n ///\n /// There is no guarantee when exactly `f` will be executed. The only guarantee is that it\n /// won't be executed until all currently pinned threads get unpinned. In theory, `f` might\n /// never run, but the epoch-based garbage collection will make an effort to execute it\n /// reasonably soon.\n ///\n /// If this method is called from an [`unprotected`] guard, the function will simply be\n /// executed immediately.\n ///\n /// # Safety\n ///\n /// The given function must not hold reference onto the stack. It is highly recommended that\n /// the passed function is **always** marked with `move` in order to prevent accidental\n /// borrows.\n ///\n /// ```\n /// use crossbeam_epoch as epoch;\n ///\n /// let guard = &epoch::pin();\n /// let message = \"Hello!\";\n /// unsafe {\n /// // ALWAYS use `move` when sending a closure into `defer_unchecked`.\n /// guard.defer_unchecked(move || {\n /// println!(\"{}\", message);\n /// });\n /// }\n /// ```\n ///\n /// Apart from that, keep in mind that another thread may execute `f`, so anything accessed by\n /// the closure must be `Send`.\n ///\n /// We intentionally didn't require `F: Send`, because Rust's type systems usually cannot prove\n /// `F: Send` for typical use cases. For example, consider the following code snippet, which\n /// exemplifies the typical use case of deferring the deallocation of a shared reference:\n ///\n /// ```ignore\n /// let shared = Owned::new(7i32).into_shared(guard);\n /// guard.defer_unchecked(move || shared.into_owned()); // `Shared` is not `Send`!\n /// ```\n ///\n /// While `Shared` is not `Send`, it's safe for another thread to call the deferred function,\n /// because it's called only after the grace period and `shared` is no longer shared with other\n /// threads. But we don't expect type systems to prove this.\n ///\n /// # Examples\n ///\n /// When a heap-allocated object in a data structure becomes unreachable, it has to be\n /// deallocated. However, the current thread and other threads may be still holding references\n /// on the stack to that same object. Therefore it cannot be deallocated before those references\n /// get dropped. This method can defer deallocation until all those threads get unpinned and\n /// consequently drop all their references on the stack.\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Owned};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(\"foo\");\n ///\n /// // Now suppose that `a` is shared among multiple threads and concurrently\n /// // accessed and modified...\n ///\n /// // Pin the current thread.\n /// let guard = &epoch::pin();\n ///\n /// // Steal the object currently stored in `a` and swap it with another one.\n /// let p = a.swap(Owned::new(\"bar\").into_shared(guard), SeqCst, guard);\n ///\n /// if !p.is_null() {\n /// // The object `p` is pointing to is now unreachable.\n /// // Defer its deallocation until all currently pinned threads get unpinned.\n /// unsafe {\n /// // ALWAYS use `move` when sending a closure into `defer_unchecked`.\n /// guard.defer_unchecked(move || {\n /// println!(\"{} is now being deallocated.\", p.deref());\n /// // Now we have unique access to the object pointed to by `p` and can turn it\n /// // into an `Owned`. Dropping the `Owned` will deallocate the object.\n /// drop(p.into_owned());\n /// });\n /// }\n /// }\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub unsafe fn defer_unchecked(&self, f: F)\n where\n F: FnOnce() -> R,\n {\n unsafe {\n if let Some(local) = self.local.as_ref() {\n local.defer(Deferred::new(move || drop(f())), self);\n } else {\n drop(f());\n }\n }\n }\n\n /// Stores a destructor for an object so that it can be deallocated and dropped at some point\n /// after all currently pinned threads get unpinned.\n ///\n /// This method first stores the destructor into the thread-local (or handle-local) cache. If\n /// this cache becomes full, some destructors are moved into the global cache. At the same\n /// time, some destructors from both local and global caches may get executed in order to\n /// incrementally clean up the caches as they fill up.\n ///\n /// There is no guarantee when exactly the destructor will be executed. The only guarantee is\n /// that it won't be executed until all currently pinned threads get unpinned. In theory, the\n /// destructor might never run, but the epoch-based garbage collection will make an effort to\n /// execute it reasonably soon.\n ///\n /// If this method is called from an [`unprotected`] guard, the destructor will simply be\n /// executed immediately.\n ///\n /// # Safety\n ///\n /// The object must not be reachable by other threads anymore, otherwise it might be still in\n /// use when the destructor runs.\n ///\n /// Apart from that, keep in mind that another thread may execute the destructor, so the object\n /// must be sendable to other threads.\n ///\n /// We intentionally didn't require `T: Send`, because Rust's type systems usually cannot prove\n /// `T: Send` for typical use cases. For example, consider the following code snippet, which\n /// exemplifies the typical use case of deferring the deallocation of a shared reference:\n ///\n /// ```ignore\n /// let shared = Owned::new(7i32).into_shared(guard);\n /// guard.defer_destroy(shared); // `Shared` is not `Send`!\n /// ```\n ///\n /// While `Shared` is not `Send`, it's safe for another thread to call the destructor, because\n /// it's called only after the grace period and `shared` is no longer shared with other\n /// threads. But we don't expect type systems to prove this.\n ///\n /// # Examples\n ///\n /// When a heap-allocated object in a data structure becomes unreachable, it has to be\n /// deallocated. However, the current thread and other threads may be still holding references\n /// on the stack to that same object. Therefore it cannot be deallocated before those references\n /// get dropped. This method can defer deallocation until all those threads get unpinned and\n /// consequently drop all their references on the stack.\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic, Owned};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(\"foo\");\n ///\n /// // Now suppose that `a` is shared among multiple threads and concurrently\n /// // accessed and modified...\n ///\n /// // Pin the current thread.\n /// let guard = &epoch::pin();\n ///\n /// // Steal the object currently stored in `a` and swap it with another one.\n /// let p = a.swap(Owned::new(\"bar\").into_shared(guard), SeqCst, guard);\n ///\n /// if !p.is_null() {\n /// // The object `p` is pointing to is now unreachable.\n /// // Defer its deallocation until all currently pinned threads get unpinned.\n /// unsafe {\n /// guard.defer_destroy(p);\n /// }\n /// }\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub unsafe fn defer_destroy(&self, ptr: Shared<'_, T>) {\n unsafe { self.defer_unchecked(move || ptr.into_owned()) }\n }\n\n /// Clears up the thread-local cache of deferred functions by executing them or moving into the\n /// global cache.\n ///\n /// Call this method after deferring execution of a function if you want to get it executed as\n /// soon as possible. Flushing will make sure it is residing in the global cache, so that\n /// any thread has a chance of taking the function and executing it.\n ///\n /// If this method is called from an [`unprotected`] guard, it is a no-op (nothing happens).\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch as epoch;\n ///\n /// let guard = &epoch::pin();\n /// guard.defer(move || {\n /// println!(\"This better be printed as soon as possible!\");\n /// });\n /// guard.flush();\n /// ```\n pub fn flush(&self) {\n if let Some(local) = unsafe { self.local.as_ref() } {\n local.flush(self);\n }\n }\n\n /// Unpins and then immediately re-pins the thread.\n ///\n /// This method is useful when you don't want delay the advancement of the global epoch by\n /// holding an old epoch. For safety, you should not maintain any guard-based reference across\n /// the call (the latter is enforced by `&mut self`). The thread will only be repinned if this\n /// is the only active guard for the current thread.\n ///\n /// If this method is called from an [`unprotected`] guard, then the call will be just no-op.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// let a = Atomic::new(777);\n /// let mut guard = epoch::pin();\n /// {\n /// let p = a.load(SeqCst, &guard);\n /// assert_eq!(unsafe { p.as_ref() }, Some(&777));\n /// }\n /// guard.repin();\n /// {\n /// let p = a.load(SeqCst, &guard);\n /// assert_eq!(unsafe { p.as_ref() }, Some(&777));\n /// }\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn repin(&mut self) {\n if let Some(local) = unsafe { self.local.as_ref() } {\n local.repin();\n }\n }\n\n /// Temporarily unpins the thread, executes the given function and then re-pins the thread.\n ///\n /// This method is useful when you need to perform a long-running operation (e.g. sleeping)\n /// and don't need to maintain any guard-based reference across the call (the latter is enforced\n /// by `&mut self`). The thread will only be unpinned if this is the only active guard for the\n /// current thread.\n ///\n /// If this method is called from an [`unprotected`] guard, then the passed function is called\n /// directly without unpinning the thread.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch::{self as epoch, Atomic};\n /// use std::sync::atomic::Ordering::SeqCst;\n /// use std::thread;\n /// use std::time::Duration;\n ///\n /// let a = Atomic::new(777);\n /// let mut guard = epoch::pin();\n /// {\n /// let p = a.load(SeqCst, &guard);\n /// assert_eq!(unsafe { p.as_ref() }, Some(&777));\n /// }\n /// guard.repin_after(|| thread::sleep(Duration::from_millis(50)));\n /// {\n /// let p = a.load(SeqCst, &guard);\n /// assert_eq!(unsafe { p.as_ref() }, Some(&777));\n /// }\n /// # unsafe { drop(a.into_owned()); } // avoid leak\n /// ```\n pub fn repin_after(&mut self, f: F) -> R\n where\n F: FnOnce() -> R,\n {\n // Ensure the Guard is re-pinned even if the function panics\n struct ScopeGuard(*const Local);\n impl Drop for ScopeGuard {\n fn drop(&mut self) {\n if let Some(local) = unsafe { self.0.as_ref() } {\n mem::forget(local.pin());\n unsafe {\n Local::release_handle(local);\n }\n }\n }\n }\n\n if let Some(local) = unsafe { self.local.as_ref() } {\n // We need to acquire a handle here to ensure the Local doesn't\n // disappear from under us.\n local.acquire_handle();\n local.unpin();\n }\n\n let _guard = ScopeGuard(self.local);\n\n f()\n }\n\n /// Returns the `Collector` associated with this guard.\n ///\n /// This method is useful when you need to ensure that all guards used with\n /// a data structure come from the same collector.\n ///\n /// If this method is called from an [`unprotected`] guard, then `None` is returned.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_epoch as epoch;\n ///\n /// let guard1 = epoch::pin();\n /// let guard2 = epoch::pin();\n /// assert!(guard1.collector() == guard2.collector());\n /// ```\n pub fn collector(&self) -> Option<&Collector> {\n unsafe { self.local.as_ref().map(|local| local.collector()) }\n }\n}\n\nimpl Drop for Guard {\n #[inline]\n fn drop(&mut self) {\n if let Some(local) = unsafe { self.local.as_ref() } {\n local.unpin();\n }\n }\n}\n\nimpl fmt::Debug for Guard {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Guard { .. }\")\n }\n}\n\n/// Returns a reference to a dummy guard that allows unprotected access to [`Atomic`]s.\n///\n/// This guard should be used in special occasions only. Note that it doesn't actually keep any\n/// thread pinned - it's just a fake guard that allows loading from [`Atomic`]s unsafely.\n///\n/// Note that calling [`defer`] with a dummy guard will not defer the function - it will just\n/// execute the function immediately.\n///\n/// If necessary, it's possible to create more dummy guards by cloning: `unprotected().clone()`.\n///\n/// # Safety\n///\n/// Loading and dereferencing data from an [`Atomic`] using this guard is safe only if the\n/// [`Atomic`] is not being concurrently modified by other threads.\n///\n/// # Examples\n///\n/// ```\n/// use crossbeam_epoch::{self as epoch, Atomic};\n/// use std::sync::atomic::Ordering::Relaxed;\n///\n/// let a = Atomic::new(7);\n///\n/// unsafe {\n/// // Load `a` without pinning the current thread.\n/// a.load(Relaxed, epoch::unprotected());\n///\n/// // It's possible to create more dummy guards.\n/// let dummy = epoch::unprotected();\n///\n/// dummy.defer(move || {\n/// println!(\"This gets executed immediately.\");\n/// });\n///\n/// // Dropping `dummy` doesn't affect the current thread - it's just a noop.\n/// }\n/// # unsafe { drop(a.into_owned()); } // avoid leak\n/// ```\n///\n/// The most common use of this function is when constructing or destructing a data structure.\n///\n/// For example, we can use a dummy guard in the destructor of a Treiber stack because at that\n/// point no other thread could concurrently modify the [`Atomic`]s we are accessing.\n///\n/// If we were to actually pin the current thread during destruction, that would just unnecessarily\n/// delay garbage collection and incur some performance cost, so in cases like these `unprotected`\n/// is very helpful.\n///\n/// ```\n/// use crossbeam_epoch::{self as epoch, Atomic};\n/// use std::mem::ManuallyDrop;\n/// use std::sync::atomic::Ordering::Relaxed;\n///\n/// struct Stack {\n/// head: Atomic>,\n/// }\n///\n/// struct Node {\n/// data: ManuallyDrop,\n/// next: Atomic>,\n/// }\n///\n/// impl Drop for Stack {\n/// fn drop(&mut self) {\n/// unsafe {\n/// // Unprotected load.\n/// let mut node = self.head.load(Relaxed, epoch::unprotected());\n///\n/// while let Some(n) = node.as_ref() {\n/// // Unprotected load.\n/// let next = n.next.load(Relaxed, epoch::unprotected());\n///\n/// // Take ownership of the node, then drop its data and deallocate it.\n/// let mut o = node.into_owned();\n/// ManuallyDrop::drop(&mut o.data);\n/// drop(o);\n///\n/// node = next;\n/// }\n/// }\n/// }\n/// }\n/// ```\n///\n/// [`Atomic`]: super::Atomic\n/// [`defer`]: Guard::defer\n#[inline]\npub unsafe fn unprotected() -> &'static Guard {\n // An unprotected guard is just a `Guard` with its field `local` set to null.\n // We make a newtype over `Guard` because `Guard` isn't `Sync`, so can't be directly stored in\n // a `static`\n struct GuardWrapper(Guard);\n unsafe impl Sync for GuardWrapper {}\n static UNPROTECTED: GuardWrapper = GuardWrapper(Guard {\n local: core::ptr::null(),\n });\n &UNPROTECTED.0\n}\n" }, { "path": "crossbeam-epoch/src/internal.rs", "content": "//! The global data and participant for garbage collection.\n//!\n//! # Registration\n//!\n//! In order to track all participants in one place, we need some form of participant\n//! registration. When a participant is created, it is registered to a global lock-free\n//! singly-linked list of registries; and when a participant is leaving, it is unregistered from the\n//! list.\n//!\n//! # Pinning\n//!\n//! Every participant contains an integer that tells whether the participant is pinned and if so,\n//! what was the global epoch at the time it was pinned. Participants also hold a pin counter that\n//! aids in periodic global epoch advancement.\n//!\n//! When a participant is pinned, a `Guard` is returned as a witness that the participant is pinned.\n//! Guards are necessary for performing atomic operations, and for freeing/dropping locations.\n//!\n//! # Thread-local bag\n//!\n//! Objects that get unlinked from concurrent data structures must be stashed away until the global\n//! epoch sufficiently advances so that they become safe for destruction. Pointers to such objects\n//! are pushed into a thread-local bag, and when it becomes full, the bag is marked with the current\n//! global epoch and pushed into the global queue of bags. We store objects in thread-local storages\n//! for amortizing the synchronization cost of pushing the garbages to a global queue.\n//!\n//! # Global queue\n//!\n//! Whenever a bag is pushed into a queue, the objects in some bags in the queue are collected and\n//! destroyed along the way. This design reduces contention on data structures. The global queue\n//! cannot be explicitly accessed: the only way to interact with it is by calling functions\n//! `defer()` that adds an object to the thread-local bag, or `collect()` that manually triggers\n//! garbage collection.\n//!\n//! Ideally each instance of concurrent data structure may have its own queue that gets fully\n//! destroyed as soon as the data structure gets dropped.\n\nuse core::{\n cell::Cell,\n fmt,\n mem::{self, ManuallyDrop},\n num::Wrapping,\n ptr,\n};\n\nuse crossbeam_utils::CachePadded;\n\nuse crate::{\n atomic::{Owned, Shared},\n collector::{Collector, LocalHandle},\n deferred::Deferred,\n epoch::{AtomicEpoch, Epoch},\n guard::{Guard, unprotected},\n primitive::{\n cell::UnsafeCell,\n sync::atomic::{self, Ordering},\n },\n sync::{\n list::{Entry, IsElement, IterError, List},\n queue::Queue,\n },\n};\n\n/// Maximum number of objects a bag can contain.\n#[cfg(not(any(crossbeam_sanitize, miri)))]\nconst MAX_OBJECTS: usize = 64;\n// Makes it more likely to trigger any potential data races.\n#[cfg(any(crossbeam_sanitize, miri))]\nconst MAX_OBJECTS: usize = 4;\n\n/// A bag of deferred functions.\npub(crate) struct Bag {\n /// Stashed objects.\n deferreds: [Deferred; MAX_OBJECTS],\n len: usize,\n}\n\n/// `Bag::try_push()` requires that it is safe for another thread to execute the given functions.\nunsafe impl Send for Bag {}\n\nimpl Bag {\n /// Returns a new, empty bag.\n pub(crate) fn new() -> Self {\n Self::default()\n }\n\n /// Returns `true` if the bag is empty.\n pub(crate) fn is_empty(&self) -> bool {\n self.len == 0\n }\n\n /// Attempts to insert a deferred function into the bag.\n ///\n /// Returns `Ok(())` if successful, and `Err(deferred)` for the given `deferred` if the bag is\n /// full.\n ///\n /// # Safety\n ///\n /// It should be safe for another thread to execute the given function.\n pub(crate) unsafe fn try_push(&mut self, deferred: Deferred) -> Result<(), Deferred> {\n if self.len < MAX_OBJECTS {\n self.deferreds[self.len] = deferred;\n self.len += 1;\n Ok(())\n } else {\n Err(deferred)\n }\n }\n\n /// Seals the bag with the given epoch.\n fn seal(self, epoch: Epoch) -> SealedBag {\n SealedBag { epoch, _bag: self }\n }\n}\n\nimpl Default for Bag {\n fn default() -> Self {\n Self {\n len: 0,\n deferreds: [Deferred::NO_OP; MAX_OBJECTS],\n }\n }\n}\n\nimpl Drop for Bag {\n fn drop(&mut self) {\n // Call all deferred functions.\n for deferred in &mut self.deferreds[..self.len] {\n let no_op = Deferred::NO_OP;\n let owned_deferred = mem::replace(deferred, no_op);\n owned_deferred.call();\n }\n }\n}\n\n// can't #[derive(Debug)] because Debug is not implemented for arrays 64 items long\nimpl fmt::Debug for Bag {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"Bag\")\n .field(\"deferreds\", &&self.deferreds[..self.len])\n .finish()\n }\n}\n\n/// A pair of an epoch and a bag.\n#[derive(Default, Debug)]\nstruct SealedBag {\n epoch: Epoch,\n _bag: Bag,\n}\n\n/// It is safe to share `SealedBag` because `is_expired` only inspects the epoch.\nunsafe impl Sync for SealedBag {}\n\nimpl SealedBag {\n /// Checks if it is safe to drop the bag w.r.t. the given global epoch.\n fn is_expired(&self, global_epoch: Epoch) -> bool {\n // A pinned participant can witness at most one epoch advancement. Therefore, any bag that\n // is within one epoch of the current one cannot be destroyed yet.\n global_epoch.wrapping_sub(self.epoch) >= 2\n }\n}\n\n/// The global data for a garbage collector.\npub(crate) struct Global {\n /// The intrusive linked list of `Local`s.\n locals: List,\n\n /// The global queue of bags of deferred functions.\n queue: Queue,\n\n /// The global epoch.\n pub(crate) epoch: CachePadded,\n}\n\nimpl Global {\n /// Number of bags to destroy.\n const COLLECT_STEPS: usize = 8;\n\n /// Creates a new global data for garbage collection.\n #[inline]\n pub(crate) fn new() -> Self {\n Self {\n locals: List::new(),\n queue: Queue::new(),\n epoch: CachePadded::new(AtomicEpoch::new(Epoch::starting())),\n }\n }\n\n /// Pushes the bag into the global queue and replaces the bag with a new empty bag.\n pub(crate) fn push_bag(&self, bag: &mut Bag, guard: &Guard) {\n let bag = mem::replace(bag, Bag::new());\n\n atomic::fence(Ordering::SeqCst);\n\n let epoch = self.epoch.load(Ordering::Relaxed);\n self.queue.push(bag.seal(epoch), guard);\n }\n\n /// Collects several bags from the global queue and executes deferred functions in them.\n ///\n /// Note: This may itself produce garbage and in turn allocate new bags.\n ///\n /// `pin()` rarely calls `collect()`, so we want the compiler to place that call on a cold\n /// path. In other words, we want the compiler to optimize branching for the case when\n /// `collect()` is not called.\n #[cold]\n pub(crate) fn collect(&self, guard: &Guard) {\n let global_epoch = self.try_advance(guard);\n\n let steps = if cfg!(crossbeam_sanitize) {\n usize::MAX\n } else {\n Self::COLLECT_STEPS\n };\n\n for _ in 0..steps {\n match self.queue.try_pop_if(\n |sealed_bag: &SealedBag| sealed_bag.is_expired(global_epoch),\n guard,\n ) {\n None => break,\n Some(sealed_bag) => drop(sealed_bag),\n }\n }\n }\n\n /// Attempts to advance the global epoch.\n ///\n /// The global epoch can advance only if all currently pinned participants have been pinned in\n /// the current epoch.\n ///\n /// Returns the current global epoch.\n ///\n /// `try_advance()` is annotated `#[cold]` because it is rarely called.\n #[cold]\n pub(crate) fn try_advance(&self, guard: &Guard) -> Epoch {\n let global_epoch = self.epoch.load(Ordering::Relaxed);\n atomic::fence(Ordering::SeqCst);\n\n // For ThreadSanitizer that does not understand fences, we simulate the equivalent effect.\n // It is unfortunate that allocation is required, but without it, synchronization might\n // occur in cases where it should not, potentially causing false positives.\n #[cfg(crossbeam_sanitize_thread)]\n let mut locals = alloc::vec![];\n // TODO(stjepang): `Local`s are stored in a linked list because linked lists are fairly\n // easy to implement in a lock-free manner. However, traversal can be slow due to cache\n // misses and data dependencies. We should experiment with other data structures as well.\n for local in self.locals.iter(guard) {\n match local {\n Err(IterError::Stalled) => {\n // A concurrent thread stalled this iteration. That thread might also try to\n // advance the epoch, in which case we leave the job to it. Otherwise, the\n // epoch will not be advanced.\n return global_epoch;\n }\n Ok(local) => {\n let local_epoch = local.epoch.load(Ordering::Relaxed);\n\n // If the participant was pinned in a different epoch, we cannot advance the\n // global epoch just yet.\n if local_epoch.is_pinned() && local_epoch.unpinned() != global_epoch {\n return global_epoch;\n }\n\n #[cfg(crossbeam_sanitize_thread)]\n locals.push(local);\n }\n }\n }\n #[cfg(crossbeam_sanitize_thread)]\n for local in locals {\n local.epoch.load(Ordering::Acquire);\n }\n #[cfg(not(crossbeam_sanitize_thread))]\n atomic::fence(Ordering::Acquire);\n\n // All pinned participants were pinned in the current global epoch.\n // Now let's advance the global epoch...\n //\n // Note that if another thread already advanced it before us, this store will simply\n // overwrite the global epoch with the same value. This is true because `try_advance` was\n // called from a thread that was pinned in `global_epoch`, and the global epoch cannot be\n // advanced two steps ahead of it.\n let new_epoch = global_epoch.successor();\n self.epoch.store(new_epoch, Ordering::Release);\n new_epoch\n }\n}\n\n/// Participant for garbage collection.\n#[repr(C)] // Note: `entry` must be the first field\npub(crate) struct Local {\n /// A node in the intrusive linked list of `Local`s.\n entry: Entry,\n\n /// A reference to the global data.\n ///\n /// When all guards and handles get dropped, this reference is destroyed.\n collector: UnsafeCell>,\n\n /// The local bag of deferred functions.\n pub(crate) bag: UnsafeCell,\n\n /// The number of guards keeping this participant pinned.\n guard_count: Cell,\n\n /// The number of active handles.\n handle_count: Cell,\n\n /// Total number of pinnings performed.\n ///\n /// This is just an auxiliary counter that sometimes kicks off collection.\n pin_count: Cell>,\n\n /// The local epoch.\n epoch: CachePadded,\n}\n\n// Make sure `Local` is less than or equal to 2048 bytes.\n// https://github.com/crossbeam-rs/crossbeam/issues/551\n#[cfg(not(any(crossbeam_sanitize, miri)))] // `crossbeam_sanitize` and `miri` reduce the size of `Local`\n#[test]\nfn local_size() {\n // TODO: https://github.com/crossbeam-rs/crossbeam/issues/869\n // assert!(\n // core::mem::size_of::() <= 2048,\n // \"An allocation of `Local` should be <= 2048 bytes.\"\n // );\n}\n\nimpl Local {\n /// Number of pinnings after which a participant will execute some deferred functions from the\n /// global queue.\n const PINNINGS_BETWEEN_COLLECT: usize = 128;\n\n /// Registers a new `Local` in the provided `Global`.\n pub(crate) fn register(collector: &Collector) -> LocalHandle {\n unsafe {\n // Since we dereference no pointers in this block, it is safe to use `unprotected`.\n\n let local = Owned::new(Self {\n entry: Entry::default(),\n collector: UnsafeCell::new(ManuallyDrop::new(collector.clone())),\n bag: UnsafeCell::new(Bag::new()),\n guard_count: Cell::new(0),\n handle_count: Cell::new(1),\n pin_count: Cell::new(Wrapping(0)),\n epoch: CachePadded::new(AtomicEpoch::new(Epoch::starting())),\n })\n .into_shared(unprotected());\n collector.global.locals.insert(local, unprotected());\n LocalHandle {\n local: local.as_raw(),\n }\n }\n }\n\n /// Returns a reference to the `Global` in which this `Local` resides.\n #[inline]\n pub(crate) fn global(&self) -> &Global {\n &self.collector().global\n }\n\n /// Returns a reference to the `Collector` in which this `Local` resides.\n #[inline]\n pub(crate) fn collector(&self) -> &Collector {\n self.collector.with(|c| unsafe { &**c })\n }\n\n /// Returns `true` if the current participant is pinned.\n #[inline]\n pub(crate) fn is_pinned(&self) -> bool {\n self.guard_count.get() > 0\n }\n\n /// Adds `deferred` to the thread-local bag.\n ///\n /// # Safety\n ///\n /// It should be safe for another thread to execute the given function.\n pub(crate) unsafe fn defer(&self, mut deferred: Deferred, guard: &Guard) {\n let bag = self.bag.with_mut(|b| unsafe { &mut *b });\n\n while let Err(d) = unsafe { bag.try_push(deferred) } {\n self.global().push_bag(bag, guard);\n deferred = d;\n }\n }\n\n pub(crate) fn flush(&self, guard: &Guard) {\n let bag = self.bag.with_mut(|b| unsafe { &mut *b });\n\n if !bag.is_empty() {\n self.global().push_bag(bag, guard);\n }\n\n self.global().collect(guard);\n }\n\n /// Pins the `Local`.\n #[inline]\n pub(crate) fn pin(&self) -> Guard {\n let guard = Guard { local: self };\n\n let guard_count = self.guard_count.get();\n self.guard_count.set(guard_count.checked_add(1).unwrap());\n\n if guard_count == 0 {\n let global_epoch = self.global().epoch.load(Ordering::Relaxed);\n let new_epoch = global_epoch.pinned();\n\n // Now we must store `new_epoch` into `self.epoch` and execute a `SeqCst` fence.\n // The fence makes sure that any future loads from `Atomic`s will not happen before\n // this store.\n if cfg!(all(\n any(target_arch = \"x86\", target_arch = \"x86_64\"),\n not(miri)\n )) {\n // HACK(stjepang): On x86 architectures there are two different ways of executing\n // a `SeqCst` fence.\n //\n // 1. `atomic::fence(SeqCst)`, which compiles into a `mfence` instruction.\n // 2. `_.compare_exchange(_, _, SeqCst, SeqCst)`, which compiles into a `lock cmpxchg`\n // instruction.\n //\n // Both instructions have the effect of a full barrier, but benchmarks have shown\n // that the second one makes pinning faster in this particular case. It is not\n // clear that this is permitted by the C++ memory model (SC fences work very\n // differently from SC accesses), but experimental evidence suggests that this\n // works fine. Using inline assembly would be a viable (and correct) alternative,\n // but alas, that is not possible on stable Rust.\n let current = Epoch::starting();\n let res = self.epoch.compare_exchange(\n current,\n new_epoch,\n Ordering::SeqCst,\n Ordering::SeqCst,\n );\n debug_assert!(res.is_ok(), \"participant was expected to be unpinned\");\n // We add a compiler fence to make it less likely for LLVM to do something wrong\n // here. Formally, this is not enough to get rid of data races; practically,\n // it should go a long way.\n atomic::compiler_fence(Ordering::SeqCst);\n } else {\n self.epoch.store(new_epoch, Ordering::Relaxed);\n atomic::fence(Ordering::SeqCst);\n }\n\n // Increment the pin counter.\n let count = self.pin_count.get();\n self.pin_count.set(count + Wrapping(1));\n\n // After every `PINNINGS_BETWEEN_COLLECT` try advancing the epoch and collecting\n // some garbage.\n if count.0 % Self::PINNINGS_BETWEEN_COLLECT == 0 {\n self.global().collect(&guard);\n }\n }\n\n guard\n }\n\n /// Unpins the `Local`.\n #[inline]\n pub(crate) fn unpin(&self) {\n let guard_count = self.guard_count.get();\n self.guard_count.set(guard_count - 1);\n\n if guard_count == 1 {\n self.epoch.store(Epoch::starting(), Ordering::Release);\n\n if self.handle_count.get() == 0 {\n unsafe {\n Self::finalize(self);\n }\n }\n }\n }\n\n /// Unpins and then pins the `Local`.\n #[inline]\n pub(crate) fn repin(&self) {\n let guard_count = self.guard_count.get();\n\n // Update the local epoch only if there's only one guard.\n if guard_count == 1 {\n let epoch = self.epoch.load(Ordering::Relaxed);\n let global_epoch = self.global().epoch.load(Ordering::Relaxed).pinned();\n\n // Update the local epoch only if the global epoch is greater than the local epoch.\n if epoch != global_epoch {\n // We store the new epoch with `Release` because we need to ensure any memory\n // accesses from the previous epoch do not leak into the new one.\n self.epoch.store(global_epoch, Ordering::Release);\n\n // However, we don't need a following `SeqCst` fence, because it is safe for memory\n // accesses from the new epoch to be executed before updating the local epoch. At\n // worse, other threads will see the new epoch late and delay GC slightly.\n }\n }\n }\n\n /// Increments the handle count.\n #[inline]\n pub(crate) fn acquire_handle(&self) {\n let handle_count = self.handle_count.get();\n debug_assert!(handle_count >= 1);\n self.handle_count.set(handle_count + 1);\n }\n\n /// Decrements the handle count.\n #[inline]\n pub(crate) unsafe fn release_handle(this: *const Self) {\n let guard_count = unsafe { (*this).guard_count.get() };\n let handle_count = unsafe { (*this).handle_count.get() };\n debug_assert!(handle_count >= 1);\n unsafe {\n (*this).handle_count.set(handle_count - 1);\n }\n\n if guard_count == 0 && handle_count == 1 {\n unsafe {\n Self::finalize(this);\n }\n }\n }\n\n /// Removes the `Local` from the global linked list.\n #[cold]\n unsafe fn finalize(this: *const Self) {\n unsafe {\n debug_assert_eq!((*this).guard_count.get(), 0);\n debug_assert_eq!((*this).handle_count.get(), 0);\n }\n\n // Temporarily increment handle count. This is required so that the following call to `pin`\n // doesn't call `finalize` again.\n unsafe {\n (*this).handle_count.set(1);\n }\n unsafe {\n // Pin and move the local bag into the global queue. It's important that `push_bag`\n // doesn't defer destruction on any new garbage.\n let guard = &(*this).pin();\n (*this)\n .global()\n .push_bag((*this).bag.with_mut(|b| &mut *b), guard);\n }\n // Revert the handle count back to zero.\n unsafe {\n (*this).handle_count.set(0);\n }\n\n unsafe {\n // Take the reference to the `Global` out of this `Local`. Since we're not protected\n // by a guard at this time, it's crucial that the reference is read before marking the\n // `Local` as deleted.\n let collector: Collector = ptr::read((*this).collector.with(|c| &*(*c)));\n\n // Mark this node in the linked list as deleted.\n (*this).entry.delete(unprotected());\n\n // Finally, drop the reference to the global. Note that this might be the last reference\n // to the `Global`. If so, the global data will be destroyed and all deferred functions\n // in its queue will be executed.\n drop(collector);\n }\n }\n}\n\nimpl IsElement for Local {\n fn entry_of(local: *const Self) -> *const Entry {\n // SAFETY: `Local` is `repr(C)` and `entry` is the first field of it.\n local.cast::()\n }\n\n unsafe fn element_of(entry: *const Entry) -> *const Self {\n // SAFETY: `Local` is `repr(C)` and `entry` is the first field of it.\n entry.cast::()\n }\n\n unsafe fn finalize(entry: *const Entry, guard: &Guard) {\n unsafe { guard.defer_destroy(Shared::from(Self::element_of(entry))) }\n }\n}\n\n#[cfg(all(test, not(crossbeam_loom)))]\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\nmod tests {\n use std::sync::atomic::AtomicUsize;\n\n use super::*;\n\n #[test]\n fn check_defer() {\n static FLAG: AtomicUsize = AtomicUsize::new(0);\n fn set() {\n FLAG.store(42, Ordering::Relaxed);\n }\n\n let d = Deferred::new(set);\n assert_eq!(FLAG.load(Ordering::Relaxed), 0);\n d.call();\n assert_eq!(FLAG.load(Ordering::Relaxed), 42);\n }\n\n #[test]\n fn check_bag() {\n static FLAG: AtomicUsize = AtomicUsize::new(0);\n fn incr() {\n FLAG.fetch_add(1, Ordering::Relaxed);\n }\n\n let mut bag = Bag::new();\n assert!(bag.is_empty());\n\n for _ in 0..MAX_OBJECTS {\n assert!(unsafe { bag.try_push(Deferred::new(incr)).is_ok() });\n assert!(!bag.is_empty());\n assert_eq!(FLAG.load(Ordering::Relaxed), 0);\n }\n\n let result = unsafe { bag.try_push(Deferred::new(incr)) };\n assert!(result.is_err());\n assert!(!bag.is_empty());\n assert_eq!(FLAG.load(Ordering::Relaxed), 0);\n\n drop(bag);\n assert_eq!(FLAG.load(Ordering::Relaxed), MAX_OBJECTS);\n }\n}\n" }, { "path": "crossbeam-epoch/src/lib.rs", "content": "//! Epoch-based memory reclamation.\n//!\n//! An interesting problem concurrent collections deal with comes from the remove operation.\n//! Suppose that a thread removes an element from a lock-free map, while another thread is reading\n//! that same element at the same time. The first thread must wait until the second thread stops\n//! reading the element. Only then it is safe to destruct it.\n//!\n//! Programming languages that come with garbage collectors solve this problem trivially. The\n//! garbage collector will destruct the removed element when no thread can hold a reference to it\n//! anymore.\n//!\n//! This crate implements a basic memory reclamation mechanism, which is based on epochs. When an\n//! element gets removed from a concurrent collection, it is inserted into a pile of garbage and\n//! marked with the current epoch. Every time a thread accesses a collection, it checks the current\n//! epoch, attempts to increment it, and destructs some garbage that became so old that no thread\n//! can be referencing it anymore.\n//!\n//! That is the general mechanism behind epoch-based memory reclamation, but the details are a bit\n//! more complicated. Anyhow, memory reclamation is designed to be fully automatic and something\n//! users of concurrent collections don't have to worry much about.\n//!\n//! # Pointers\n//!\n//! Concurrent collections are built using atomic pointers. This module provides [`Atomic`], which\n//! is just a shared atomic pointer to a heap-allocated object. Loading an [`Atomic`] yields a\n//! [`Shared`], which is an epoch-protected pointer through which the loaded object can be safely\n//! read.\n//!\n//! # Pinning\n//!\n//! Before an [`Atomic`] can be loaded, a participant must be [`pin`]ned. By pinning a participant\n//! we declare that any object that gets removed from now on must not be destructed just\n//! yet. Garbage collection of newly removed objects is suspended until the participant gets\n//! unpinned.\n//!\n//! # Garbage\n//!\n//! Objects that get removed from concurrent collections must be stashed away until all currently\n//! pinned participants get unpinned. Such objects can be stored into a thread-local or global\n//! storage, where they are kept until the right time for their destruction comes.\n//!\n//! There is a global shared instance of garbage queue. You can [`defer`](Guard::defer) the execution of an\n//! arbitrary function until the global epoch is advanced enough. Most notably, concurrent data\n//! structures may defer the deallocation of an object.\n//!\n//! # APIs\n//!\n//! For majority of use cases, just use the default garbage collector by invoking [`pin`]. If you\n//! want to create your own garbage collector, use the [`Collector`] API.\n\n#![no_std]\n#![doc(test(\n no_crate_inject,\n attr(allow(dead_code, unused_assignments, unused_variables))\n))]\n#![warn(\n missing_docs,\n unsafe_op_in_unsafe_fn,\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\n\n#[cfg(crossbeam_loom)]\nextern crate loom_crate as loom;\n#[cfg(feature = \"std\")]\nextern crate std;\n\n#[cfg(crossbeam_loom)]\n#[allow(unused_imports, dead_code)]\nmod primitive {\n pub(crate) mod cell {\n pub(crate) use loom::cell::UnsafeCell;\n }\n pub(crate) mod sync {\n pub(crate) mod atomic {\n #[cfg(target_has_atomic = \"64\")]\n pub(crate) use loom::sync::atomic::AtomicU64;\n pub(crate) use loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering, fence};\n\n // FIXME: loom does not support compiler_fence at the moment.\n // https://github.com/tokio-rs/loom/issues/117\n // we use fence as a stand-in for compiler_fence for the time being.\n // this may miss some races since fence is stronger than compiler_fence,\n // but it's the best we can do for the time being.\n pub(crate) use self::fence as compiler_fence;\n }\n pub(crate) use loom::sync::Arc;\n }\n pub(crate) use loom::thread_local;\n}\n#[cfg(target_has_atomic = \"ptr\")]\n#[cfg(not(crossbeam_loom))]\n#[allow(unused_imports, dead_code)]\nmod primitive {\n pub(crate) mod cell {\n #[derive(Debug)]\n #[repr(transparent)]\n pub(crate) struct UnsafeCell(::core::cell::UnsafeCell);\n\n // loom's UnsafeCell has a slightly different API than the standard library UnsafeCell.\n // Since we want the rest of the code to be agnostic to whether it's running under loom or\n // not, we write this small wrapper that provides the loom-supported API for the standard\n // library UnsafeCell. This is also what the loom documentation recommends:\n // https://github.com/tokio-rs/loom#handling-loom-api-differences\n impl UnsafeCell {\n #[inline]\n pub(crate) const fn new(data: T) -> Self {\n Self(::core::cell::UnsafeCell::new(data))\n }\n\n #[inline]\n pub(crate) fn with(&self, f: impl FnOnce(*const T) -> R) -> R {\n f(self.0.get())\n }\n\n #[inline]\n pub(crate) fn with_mut(&self, f: impl FnOnce(*mut T) -> R) -> R {\n f(self.0.get())\n }\n }\n }\n pub(crate) mod sync {\n #[cfg(feature = \"alloc\")]\n pub(crate) use alloc::sync::Arc;\n pub(crate) use core::sync::atomic;\n }\n\n #[cfg(feature = \"std\")]\n pub(crate) use std::thread_local;\n}\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nextern crate alloc;\n\n/// Make the given function const if the given condition is true.\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmacro_rules! const_fn {\n (\n const_if: #[cfg($($cfg:tt)+)];\n $(#[$($attr:tt)*])*\n $vis:vis const $($rest:tt)*\n ) => {\n #[cfg($($cfg)+)]\n $(#[$($attr)*])*\n $vis const $($rest)*\n #[cfg(not($($cfg)+))]\n $(#[$($attr)*])*\n $vis $($rest)*\n };\n}\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod alloc_helper;\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod atomic;\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod collector;\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod deferred;\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod epoch;\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod guard;\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod internal;\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod sync;\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\npub use crate::{\n atomic::{\n Atomic, CompareExchangeError, CompareExchangeValue, Owned, Pointable, Pointer, Shared,\n },\n collector::{Collector, LocalHandle},\n guard::{Guard, unprotected},\n};\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod sealed {\n pub trait Sealed {}\n}\n\n#[cfg(feature = \"std\")]\nmod default;\n#[cfg(feature = \"std\")]\npub use crate::default::{default_collector, is_pinned, pin};\n" }, { "path": "crossbeam-epoch/src/sync/list.rs", "content": "//! Lock-free intrusive linked list.\n//!\n//! Ideas from Michael. High Performance Dynamic Lock-Free Hash Tables and List-Based Sets. SPAA\n//! 2002. \n\nuse core::{\n marker::PhantomData,\n ptr::NonNull,\n sync::atomic::Ordering::{Acquire, Relaxed, Release},\n};\n\nuse crate::{Atomic, Guard, Shared, unprotected};\n\n/// An entry in a linked list.\n///\n/// An Entry is accessed from multiple threads, so it would be beneficial to put it in a different\n/// cache-line than thread-local data in terms of performance.\n#[derive(Debug)]\npub(crate) struct Entry {\n /// The next entry in the linked list.\n /// If the tag is 1, this entry is marked as deleted.\n next: Atomic,\n}\n\n/// Implementing this trait asserts that the type `T` can be used as an element in the intrusive\n/// linked list defined in this module. `T` has to contain (or otherwise be linked to) an instance\n/// of `Entry`.\n///\n/// # Example\n///\n/// ```ignore\n/// struct A {\n/// entry: Entry,\n/// data: usize,\n/// }\n///\n/// impl IsElement for A {\n/// fn entry_of(a: &A) -> &Entry {\n/// let entry_ptr = ((a as usize) + offset_of!(A, entry)) as *const Entry;\n/// unsafe { &*entry_ptr }\n/// }\n///\n/// unsafe fn element_of(entry: &Entry) -> &T {\n/// let elem_ptr = ((entry as usize) - offset_of!(A, entry)) as *const T;\n/// &*elem_ptr\n/// }\n///\n/// unsafe fn finalize(entry: &Entry, guard: &Guard) {\n/// guard.defer_destroy(Shared::from(Self::element_of(entry) as *const _));\n/// }\n/// }\n/// ```\n///\n/// This trait is implemented on a type separate from `T` (although it can be just `T`), because\n/// one type might be placeable into multiple lists, in which case it would require multiple\n/// implementations of `IsElement`. In such cases, each struct implementing `IsElement`\n/// represents a distinct `Entry` in `T`.\n///\n/// For example, we can insert the following struct into two lists using `entry1` for one\n/// and `entry2` for the other:\n///\n/// ```ignore\n/// struct B {\n/// entry1: Entry,\n/// entry2: Entry,\n/// data: usize,\n/// }\n/// ```\n///\npub(crate) trait IsElement {\n /// Returns a pointer to this element's `Entry`.\n fn entry_of(_: *const T) -> *const Entry;\n\n /// Given a pointer to an element's entry, returns that element.\n ///\n /// ```ignore\n /// let elem = ListElement::new();\n /// assert_eq!(elem.entry_of(),\n /// unsafe { ListElement::element_of(elem.entry_of()) } );\n /// ```\n ///\n /// # Safety\n ///\n /// The caller has to guarantee that the `Entry` is called with was retrieved from an instance\n /// of the element type (`T`).\n unsafe fn element_of(_: *const Entry) -> *const T;\n\n /// The function that is called when an entry is unlinked from list.\n ///\n /// # Safety\n ///\n /// The caller has to guarantee that the `Entry` is called with was retrieved from an instance\n /// of the element type (`T`).\n unsafe fn finalize(_: *const Entry, _: &Guard);\n}\n\n/// A lock-free, intrusive linked list of type `T`.\n#[derive(Debug)]\npub(crate) struct List = T> {\n /// The head of the linked list.\n head: Atomic,\n\n /// The phantom data for using `T` and `C`.\n _marker: PhantomData<(T, C)>,\n}\n\n/// An iterator used for retrieving values from the list.\npub(crate) struct Iter<'g, T, C: IsElement> {\n /// The guard that protects the iteration.\n guard: &'g Guard,\n\n /// Pointer from the predecessor to the current entry.\n pred: &'g Atomic,\n\n /// The current entry.\n curr: Shared<'g, Entry>,\n\n /// The list head, needed for restarting iteration.\n head: &'g Atomic,\n\n /// Logically, we store a borrow of an instance of `T` and\n /// use the type information from `C`.\n _marker: PhantomData<(&'g T, C)>,\n}\n\n/// An error that occurs during iteration over the list.\n#[derive(PartialEq, Debug)]\npub(crate) enum IterError {\n /// A concurrent thread modified the state of the list at the same place that this iterator\n /// was inspecting. Subsequent iteration will restart from the beginning of the list.\n Stalled,\n}\n\nimpl Default for Entry {\n /// Returns the empty entry.\n fn default() -> Self {\n Self {\n next: Atomic::null(),\n }\n }\n}\n\nimpl Entry {\n /// Marks this entry as deleted, deferring the actual deallocation to a later iteration.\n ///\n /// # Safety\n ///\n /// The entry should be a member of a linked list, and it should not have been deleted.\n /// It should be safe to call `C::finalize` on the entry after the `guard` is dropped, where `C`\n /// is the associated helper for the linked list.\n pub(crate) unsafe fn delete(&self, guard: &Guard) {\n self.next.fetch_or(1, Release, guard);\n }\n}\n\nimpl> List {\n /// Returns a new, empty linked list.\n pub(crate) fn new() -> Self {\n Self {\n head: Atomic::null(),\n _marker: PhantomData,\n }\n }\n\n /// Inserts `entry` into the head of the list.\n ///\n /// # Safety\n ///\n /// You should guarantee that:\n ///\n /// - `container` is not null\n /// - `container` is immovable, e.g. inside an `Owned`\n /// - the same `Entry` is not inserted more than once\n /// - the inserted object will be removed before the list is dropped\n pub(crate) unsafe fn insert<'g>(&'g self, container: Shared<'g, T>, guard: &'g Guard) {\n // Insert right after head, i.e. at the beginning of the list.\n let to = &self.head;\n // Get the intrusively stored Entry of the new element to insert.\n let entry: *const Entry = C::entry_of(unsafe { container.as_ptr() });\n // Make a Shared ptr to that Entry.\n let entry_ptr = Shared::from(entry);\n // Read the current successor of where we want to insert.\n let mut next = to.load(Relaxed, guard);\n\n loop {\n // Set the Entry of the to-be-inserted element to point to the previous successor of\n // `to`.\n unsafe { (*entry).next.store(next, Relaxed) }\n match to.compare_exchange_weak(next, entry_ptr, Release, Relaxed, guard) {\n Ok(_) => break,\n // We lost the race or weak CAS failed spuriously. Update the successor and try\n // again.\n Err(err) => next = err.current,\n }\n }\n }\n\n /// Returns an iterator over all objects.\n ///\n /// # Caveat\n ///\n /// Every object that is inserted at the moment this function is called and persists at least\n /// until the end of iteration will be returned. Since this iterator traverses a lock-free\n /// linked list that may be concurrently modified, some additional caveats apply:\n ///\n /// 1. If a new object is inserted during iteration, it may or may not be returned.\n /// 2. If an object is deleted during iteration, it may or may not be returned.\n /// 3. The iteration may be aborted when it lost in a race condition. In this case, the winning\n /// thread will continue to iterate over the same list.\n pub(crate) fn iter<'g>(&'g self, guard: &'g Guard) -> Iter<'g, T, C> {\n Iter {\n guard,\n pred: &self.head,\n curr: self.head.load(Acquire, guard),\n head: &self.head,\n _marker: PhantomData,\n }\n }\n}\n\nimpl> Drop for List {\n fn drop(&mut self) {\n unsafe {\n let guard = unprotected();\n let mut curr = self.head.load(Relaxed, guard);\n while let Some(c) = NonNull::new(curr.as_ptr() as *mut Entry) {\n let succ = c.as_ref().next.load(Relaxed, guard);\n // Verify that all elements have been removed from the list.\n assert_eq!(succ.tag(), 1);\n\n C::finalize(curr.as_ptr(), guard);\n curr = succ;\n }\n }\n }\n}\n\nimpl<'g, T: 'g, C: IsElement> Iterator for Iter<'g, T, C> {\n type Item = Result<&'g T, IterError>;\n\n fn next(&mut self) -> Option {\n while let Some(c) = unsafe { NonNull::new(self.curr.as_ptr() as *mut Entry) } {\n let succ = unsafe { c.as_ref().next.load(Acquire, self.guard) };\n\n if succ.tag() == 1 {\n // This entry was removed. Try unlinking it from the list.\n let succ = succ.with_tag(0);\n\n // The tag should always be zero, because removing a node after a logically deleted\n // node leaves the list in an invalid state.\n debug_assert!(self.curr.tag() == 0);\n\n // Try to unlink `curr` from the list, and get the new value of `self.pred`.\n let succ = match self\n .pred\n .compare_exchange(self.curr, succ, Acquire, Acquire, self.guard)\n {\n Ok(_) => {\n // We succeeded in unlinking `curr`, so we have to schedule\n // deallocation. Deferred drop is okay, because `list.delete()` can only be\n // called if `T: 'static`.\n unsafe {\n C::finalize(self.curr.as_ptr(), self.guard);\n }\n\n // `succ` is the new value of `self.pred`.\n succ\n }\n Err(e) => {\n // `e.current` is the current value of `self.pred`.\n e.current\n }\n };\n\n // If the predecessor node is already marked as deleted, we need to restart from\n // `head`.\n if succ.tag() != 0 {\n self.pred = self.head;\n self.curr = self.head.load(Acquire, self.guard);\n\n return Some(Err(IterError::Stalled));\n }\n\n // Move over the removed by only advancing `curr`, not `pred`.\n self.curr = succ;\n continue;\n }\n\n // Move one step forward.\n self.pred = unsafe { &(*c.as_ptr()).next };\n self.curr = succ;\n\n return Some(Ok(unsafe { &*C::element_of(c.as_ptr()) }));\n }\n\n // We reached the end of the list.\n None\n }\n}\n\n#[cfg(all(test, not(crossbeam_loom)))]\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\nmod tests {\n use std::{ptr, sync::Barrier, vec::Vec};\n\n use crossbeam_utils::thread;\n\n use super::*;\n use crate::{Collector, Owned};\n\n impl IsElement for Entry {\n fn entry_of(entry: *const Self) -> *const Entry {\n entry\n }\n\n unsafe fn element_of(entry: *const Entry) -> *const Self {\n entry\n }\n\n unsafe fn finalize(entry: *const Entry, guard: &Guard) {\n unsafe { guard.defer_destroy(Shared::from(Self::element_of(entry))) }\n }\n }\n\n /// Checks whether the list retains inserted elements\n /// and returns them in the correct order.\n #[test]\n fn insert() {\n let collector = Collector::new();\n let handle = collector.register();\n let guard = handle.pin();\n\n let l: List = List::new();\n\n let e1 = Owned::new(Entry::default()).into_shared(&guard);\n let e2 = Owned::new(Entry::default()).into_shared(&guard);\n let e3 = Owned::new(Entry::default()).into_shared(&guard);\n\n unsafe {\n l.insert(e1, &guard);\n l.insert(e2, &guard);\n l.insert(e3, &guard);\n }\n\n let mut iter = l.iter(&guard);\n let maybe_e3 = iter.next();\n assert!(maybe_e3.is_some());\n assert!(ptr::eq(maybe_e3.unwrap().unwrap(), e3.as_raw()));\n let maybe_e2 = iter.next();\n assert!(maybe_e2.is_some());\n assert!(ptr::eq(maybe_e2.unwrap().unwrap(), e2.as_raw()));\n let maybe_e1 = iter.next();\n assert!(maybe_e1.is_some());\n assert!(ptr::eq(maybe_e1.unwrap().unwrap(), e1.as_raw()));\n assert!(iter.next().is_none());\n\n unsafe {\n e1.as_ref().unwrap().delete(&guard);\n e2.as_ref().unwrap().delete(&guard);\n e3.as_ref().unwrap().delete(&guard);\n }\n }\n\n /// Checks whether elements can be removed from the list and whether\n /// the correct elements are removed.\n #[test]\n fn delete() {\n let collector = Collector::new();\n let handle = collector.register();\n let guard = handle.pin();\n\n let l: List = List::new();\n\n let e1 = Owned::new(Entry::default()).into_shared(&guard);\n let e2 = Owned::new(Entry::default()).into_shared(&guard);\n let e3 = Owned::new(Entry::default()).into_shared(&guard);\n unsafe {\n l.insert(e1, &guard);\n l.insert(e2, &guard);\n l.insert(e3, &guard);\n e2.as_ref().unwrap().delete(&guard);\n }\n\n let mut iter = l.iter(&guard);\n let maybe_e3 = iter.next();\n assert!(maybe_e3.is_some());\n assert!(ptr::eq(maybe_e3.unwrap().unwrap(), e3.as_raw()));\n let maybe_e1 = iter.next();\n assert!(maybe_e1.is_some());\n assert!(ptr::eq(maybe_e1.unwrap().unwrap(), e1.as_raw()));\n assert!(iter.next().is_none());\n\n unsafe {\n e1.as_ref().unwrap().delete(&guard);\n e3.as_ref().unwrap().delete(&guard);\n }\n\n let mut iter = l.iter(&guard);\n assert!(iter.next().is_none());\n }\n\n const THREADS: usize = 8;\n const ITERS: usize = 512;\n\n /// Contends the list on insert and delete operations to make sure they can run concurrently.\n #[test]\n fn insert_delete_multi() {\n let collector = Collector::new();\n\n let l: List = List::new();\n let b = Barrier::new(THREADS);\n\n thread::scope(|s| {\n for _ in 0..THREADS {\n s.spawn(|_| {\n b.wait();\n\n let handle = collector.register();\n let guard: Guard = handle.pin();\n let mut v = Vec::with_capacity(ITERS);\n\n for _ in 0..ITERS {\n let e = Owned::new(Entry::default()).into_shared(&guard);\n v.push(e);\n unsafe {\n l.insert(e, &guard);\n }\n }\n\n for e in v {\n unsafe {\n e.as_ref().unwrap().delete(&guard);\n }\n }\n });\n }\n })\n .unwrap();\n\n let handle = collector.register();\n let guard = handle.pin();\n\n let mut iter = l.iter(&guard);\n assert!(iter.next().is_none());\n }\n\n /// Contends the list on iteration to make sure that it can be iterated over concurrently.\n #[test]\n fn iter_multi() {\n let collector = Collector::new();\n\n let l: List = List::new();\n let b = Barrier::new(THREADS);\n\n thread::scope(|s| {\n for _ in 0..THREADS {\n s.spawn(|_| {\n b.wait();\n\n let handle = collector.register();\n let guard: Guard = handle.pin();\n let mut v = Vec::with_capacity(ITERS);\n\n for _ in 0..ITERS {\n let e = Owned::new(Entry::default()).into_shared(&guard);\n v.push(e);\n unsafe {\n l.insert(e, &guard);\n }\n }\n\n let mut iter = l.iter(&guard);\n for _ in 0..ITERS {\n assert!(iter.next().is_some());\n }\n\n for e in v {\n unsafe {\n e.as_ref().unwrap().delete(&guard);\n }\n }\n });\n }\n })\n .unwrap();\n\n let handle = collector.register();\n let guard = handle.pin();\n\n let mut iter = l.iter(&guard);\n assert!(iter.next().is_none());\n }\n}\n" }, { "path": "crossbeam-epoch/src/sync/mod.rs", "content": "//! Synchronization primitives.\n\npub(crate) mod list;\npub(crate) mod queue;\n" }, { "path": "crossbeam-epoch/src/sync/queue.rs", "content": "//! Michael-Scott lock-free queue.\n//!\n//! Usable with any number of producers and consumers.\n//!\n//! Michael and Scott. Simple, Fast, and Practical Non-Blocking and Blocking Concurrent Queue\n//! Algorithms. PODC 1996. \n//!\n//! Simon Doherty, Lindsay Groves, Victor Luchangco, and Mark Moir. 2004b. Formal Verification of a\n//! Practical Lock-Free Queue Algorithm. \n\nuse core::{\n mem::MaybeUninit,\n sync::atomic::Ordering::{Acquire, Relaxed, Release},\n};\n\nuse crossbeam_utils::CachePadded;\n\nuse crate::{Atomic, Guard, Owned, Shared, unprotected};\n\n// The representation here is a singly-linked list, with a sentinel node at the front. In general\n// the `tail` pointer may lag behind the actual tail. Non-sentinel nodes are either all `Data` or\n// all `Blocked` (requests for data from blocked threads).\n#[derive(Debug)]\npub(crate) struct Queue {\n head: CachePadded>>,\n tail: CachePadded>>,\n}\n\nstruct Node {\n /// The slot in which a value of type `T` can be stored.\n ///\n /// The type of `data` is `MaybeUninit` because a `Node` doesn't always contain a `T`.\n /// For example, the sentinel node in a queue never contains a value: its slot is always empty.\n /// Other nodes start their life with a push operation and contain a value until it gets popped\n /// out. After that such empty nodes get added to the collector for destruction.\n data: MaybeUninit,\n\n next: Atomic>,\n}\n\n// Any particular `T` should never be accessed concurrently, so no need for `Sync`.\nunsafe impl Sync for Queue {}\nunsafe impl Send for Queue {}\n\nimpl Queue {\n /// Create a new, empty queue.\n pub(crate) fn new() -> Self {\n let q = Self {\n head: CachePadded::new(Atomic::null()),\n tail: CachePadded::new(Atomic::null()),\n };\n let sentinel = Owned::new(Node {\n data: MaybeUninit::uninit(),\n next: Atomic::null(),\n });\n unsafe {\n let guard = unprotected();\n let sentinel = sentinel.into_shared(guard);\n q.head.store(sentinel, Relaxed);\n q.tail.store(sentinel, Relaxed);\n q\n }\n }\n\n /// Attempts to atomically place `n` into the `next` pointer of `onto`, and returns `true` on\n /// success. The queue's `tail` pointer may be updated.\n #[inline(always)]\n fn push_internal(\n &self,\n onto: Shared<'_, Node>,\n new: Shared<'_, Node>,\n guard: &Guard,\n ) -> bool {\n // is `onto` the actual tail?\n let o = unsafe { onto.deref() };\n let next = o.next.load(Acquire, guard);\n if unsafe { next.as_ref().is_some() } {\n // if not, try to \"help\" by moving the tail pointer forward\n let _ = self\n .tail\n .compare_exchange(onto, next, Release, Relaxed, guard);\n false\n } else {\n // looks like the actual tail; attempt to link in `n`\n let result = o\n .next\n .compare_exchange(Shared::null(), new, Release, Relaxed, guard)\n .is_ok();\n if result {\n // try to move the tail pointer forward\n let _ = self\n .tail\n .compare_exchange(onto, new, Release, Relaxed, guard);\n }\n result\n }\n }\n\n /// Adds `t` to the back of the queue, possibly waking up threads blocked on `pop`.\n pub(crate) fn push(&self, t: T, guard: &Guard) {\n let new = Owned::new(Node {\n data: MaybeUninit::new(t),\n next: Atomic::null(),\n });\n let new = Owned::into_shared(new, guard);\n\n loop {\n // We push onto the tail, so we'll start optimistically by looking there first.\n let tail = self.tail.load(Acquire, guard);\n\n // Attempt to push onto the `tail` snapshot; fails if `tail.next` has changed.\n if self.push_internal(tail, new, guard) {\n break;\n }\n }\n }\n\n /// Attempts to pop a data node. `Ok(None)` if queue is empty; `Err(())` if lost race to pop.\n #[inline(always)]\n fn pop_internal(&self, guard: &Guard) -> Result, ()> {\n let head = self.head.load(Acquire, guard);\n let h = unsafe { head.deref() };\n let next = h.next.load(Acquire, guard);\n match unsafe { next.as_ref() } {\n Some(n) => unsafe {\n self.head\n .compare_exchange(head, next, Release, Relaxed, guard)\n .map(|_| {\n let tail = self.tail.load(Relaxed, guard);\n // Advance the tail so that we don't retire a pointer to a reachable node.\n if head == tail {\n let _ = self\n .tail\n .compare_exchange(tail, next, Release, Relaxed, guard);\n }\n guard.defer_destroy(head);\n Some(n.data.assume_init_read())\n })\n .map_err(|_| ())\n },\n None => Ok(None),\n }\n }\n\n /// Attempts to pop a data node, if the data satisfies the given condition. `Ok(None)` if queue\n /// is empty or the data does not satisfy the condition; `Err(())` if lost race to pop.\n #[inline(always)]\n fn pop_if_internal(&self, condition: F, guard: &Guard) -> Result, ()>\n where\n T: Sync,\n F: Fn(&T) -> bool,\n {\n let head = self.head.load(Acquire, guard);\n let h = unsafe { head.deref() };\n let next = h.next.load(Acquire, guard);\n match unsafe { next.as_ref() } {\n Some(n) if condition(unsafe { &*n.data.as_ptr() }) => unsafe {\n self.head\n .compare_exchange(head, next, Release, Relaxed, guard)\n .map(|_| {\n let tail = self.tail.load(Relaxed, guard);\n // Advance the tail so that we don't retire a pointer to a reachable node.\n if head == tail {\n let _ = self\n .tail\n .compare_exchange(tail, next, Release, Relaxed, guard);\n }\n guard.defer_destroy(head);\n Some(n.data.assume_init_read())\n })\n .map_err(|_| ())\n },\n None | Some(_) => Ok(None),\n }\n }\n\n /// Attempts to dequeue from the front.\n ///\n /// Returns `None` if the queue is observed to be empty.\n pub(crate) fn try_pop(&self, guard: &Guard) -> Option {\n loop {\n if let Ok(head) = self.pop_internal(guard) {\n return head;\n }\n }\n }\n\n /// Attempts to dequeue from the front, if the item satisfies the given condition.\n ///\n /// Returns `None` if the queue is observed to be empty, or the head does not satisfy the given\n /// condition.\n pub(crate) fn try_pop_if(&self, condition: F, guard: &Guard) -> Option\n where\n T: Sync,\n F: Fn(&T) -> bool,\n {\n loop {\n if let Ok(head) = self.pop_if_internal(&condition, guard) {\n return head;\n }\n }\n }\n}\n\nimpl Drop for Queue {\n fn drop(&mut self) {\n unsafe {\n let guard = unprotected();\n\n while self.try_pop(guard).is_some() {}\n\n // Destroy the remaining sentinel node.\n let sentinel = self.head.load(Relaxed, guard);\n drop(sentinel.into_owned());\n }\n }\n}\n\n#[cfg(all(test, not(crossbeam_loom)))]\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\nmod test {\n use std::vec;\n\n use crossbeam_utils::thread;\n\n use super::*;\n use crate::pin;\n\n struct Queue {\n queue: super::Queue,\n }\n\n impl Queue {\n pub(crate) fn new() -> Self {\n Self {\n queue: super::Queue::new(),\n }\n }\n\n pub(crate) fn push(&self, t: T) {\n let guard = &pin();\n self.queue.push(t, guard);\n }\n\n pub(crate) fn is_empty(&self) -> bool {\n let guard = &pin();\n let head = self.queue.head.load(Acquire, guard);\n let h = unsafe { head.deref() };\n h.next.load(Acquire, guard).is_null()\n }\n\n pub(crate) fn try_pop(&self) -> Option {\n let guard = &pin();\n self.queue.try_pop(guard)\n }\n\n pub(crate) fn pop(&self) -> T {\n loop {\n match self.try_pop() {\n None => continue,\n Some(t) => return t,\n }\n }\n }\n }\n\n const CONC_COUNT: i64 = if cfg!(miri) { 1000 } else { 1000000 };\n\n #[test]\n fn push_try_pop_1() {\n let q: Queue = Queue::new();\n assert!(q.is_empty());\n q.push(37);\n assert!(!q.is_empty());\n assert_eq!(q.try_pop(), Some(37));\n assert!(q.is_empty());\n }\n\n #[test]\n fn push_try_pop_2() {\n let q: Queue = Queue::new();\n assert!(q.is_empty());\n q.push(37);\n q.push(48);\n assert_eq!(q.try_pop(), Some(37));\n assert!(!q.is_empty());\n assert_eq!(q.try_pop(), Some(48));\n assert!(q.is_empty());\n }\n\n #[test]\n fn push_try_pop_many_seq() {\n let q: Queue = Queue::new();\n assert!(q.is_empty());\n for i in 0..200 {\n q.push(i)\n }\n assert!(!q.is_empty());\n for i in 0..200 {\n assert_eq!(q.try_pop(), Some(i));\n }\n assert!(q.is_empty());\n }\n\n #[test]\n fn push_pop_1() {\n let q: Queue = Queue::new();\n assert!(q.is_empty());\n q.push(37);\n assert!(!q.is_empty());\n assert_eq!(q.pop(), 37);\n assert!(q.is_empty());\n }\n\n #[test]\n fn push_pop_2() {\n let q: Queue = Queue::new();\n q.push(37);\n q.push(48);\n assert_eq!(q.pop(), 37);\n assert_eq!(q.pop(), 48);\n }\n\n #[test]\n fn push_pop_many_seq() {\n let q: Queue = Queue::new();\n assert!(q.is_empty());\n for i in 0..200 {\n q.push(i)\n }\n assert!(!q.is_empty());\n for i in 0..200 {\n assert_eq!(q.pop(), i);\n }\n assert!(q.is_empty());\n }\n\n #[test]\n fn push_try_pop_many_spsc() {\n let q: Queue = Queue::new();\n assert!(q.is_empty());\n\n thread::scope(|scope| {\n scope.spawn(|_| {\n let mut next = 0;\n\n while next < CONC_COUNT {\n if let Some(elem) = q.try_pop() {\n assert_eq!(elem, next);\n next += 1;\n }\n }\n });\n\n for i in 0..CONC_COUNT {\n q.push(i)\n }\n })\n .unwrap();\n }\n\n #[test]\n fn push_try_pop_many_spmc() {\n fn recv(_t: i32, q: &Queue) {\n let mut cur = -1;\n for _i in 0..CONC_COUNT {\n if let Some(elem) = q.try_pop() {\n assert!(elem > cur);\n cur = elem;\n\n if cur == CONC_COUNT - 1 {\n break;\n }\n }\n }\n }\n\n let q: Queue = Queue::new();\n assert!(q.is_empty());\n thread::scope(|scope| {\n for i in 0..3 {\n let q = &q;\n scope.spawn(move |_| recv(i, q));\n }\n\n scope.spawn(|_| {\n for i in 0..CONC_COUNT {\n q.push(i);\n }\n });\n })\n .unwrap();\n }\n\n #[test]\n fn push_try_pop_many_mpmc() {\n enum LR {\n Left(i64),\n Right(i64),\n }\n\n let q: Queue = Queue::new();\n assert!(q.is_empty());\n\n thread::scope(|scope| {\n for _t in 0..2 {\n scope.spawn(|_| {\n for i in CONC_COUNT - 1..CONC_COUNT {\n q.push(LR::Left(i))\n }\n });\n scope.spawn(|_| {\n for i in CONC_COUNT - 1..CONC_COUNT {\n q.push(LR::Right(i))\n }\n });\n scope.spawn(|_| {\n let mut vl = vec![];\n let mut vr = vec![];\n for _i in 0..CONC_COUNT {\n match q.try_pop() {\n Some(LR::Left(x)) => vl.push(x),\n Some(LR::Right(x)) => vr.push(x),\n _ => {}\n }\n }\n\n let mut vl2 = vl.clone();\n let mut vr2 = vr.clone();\n vl2.sort_unstable();\n vr2.sort_unstable();\n\n assert_eq!(vl, vl2);\n assert_eq!(vr, vr2);\n });\n }\n })\n .unwrap();\n }\n\n #[test]\n fn push_pop_many_spsc() {\n let q: Queue = Queue::new();\n\n thread::scope(|scope| {\n scope.spawn(|_| {\n let mut next = 0;\n while next < CONC_COUNT {\n assert_eq!(q.pop(), next);\n next += 1;\n }\n });\n\n for i in 0..CONC_COUNT {\n q.push(i)\n }\n })\n .unwrap();\n assert!(q.is_empty());\n }\n\n #[test]\n fn is_empty_dont_pop() {\n let q: Queue = Queue::new();\n q.push(20);\n q.push(20);\n assert!(!q.is_empty());\n assert!(!q.is_empty());\n assert!(q.try_pop().is_some());\n }\n}\n" }, { "path": "crossbeam-epoch/tests/loom.rs", "content": "// Put in module instead of using #![cfg(..)] to work around rustc/cargo bug around -Z crate-attr.\n#[cfg(crossbeam_loom)]\nmod tests {\n use std::{mem::ManuallyDrop, ptr};\n\n use crossbeam_epoch as epoch;\n use epoch::{Atomic, Owned, *};\n use loom::{\n sync::{\n Arc,\n atomic::Ordering::{self, Acquire, Relaxed, Release},\n },\n thread::spawn,\n };\n use loom_crate as loom;\n\n #[test]\n fn it_works() {\n loom::model(|| {\n let collector = Collector::new();\n let item: Atomic = Atomic::from(Owned::new(String::from(\"boom\")));\n let item2 = item.clone();\n let collector2 = collector.clone();\n let guard = collector.register().pin();\n\n let jh = loom::thread::spawn(move || {\n let guard = collector2.register().pin();\n guard.defer(move || {\n // this isn't really safe, since other threads may still have pointers to the\n // value, but in this limited test scenario it's okay, since we know the test won't\n // access item after all the pins are released.\n let mut item = unsafe { item2.into_owned() };\n // mutate it as a second measure to make sure the assert_eq below would fail\n item.retain(|c| c == 'o');\n drop(item);\n });\n });\n\n let item = item.load(Ordering::SeqCst, &guard);\n // we pinned strictly before the call to defer_destroy,\n // so item cannot have been dropped yet\n assert_eq!(*unsafe { item.deref() }, \"boom\");\n drop(guard);\n\n jh.join().unwrap();\n\n drop(collector);\n })\n }\n\n #[test]\n fn treiber_stack() {\n /// Treiber's lock-free stack.\n ///\n /// Usable with any number of producers and consumers.\n #[derive(Debug)]\n struct TreiberStack {\n head: Atomic>,\n }\n\n #[derive(Debug)]\n struct Node {\n data: ManuallyDrop,\n next: Atomic>,\n }\n\n impl TreiberStack {\n /// Creates a new, empty stack.\n fn new() -> Self {\n Self {\n head: Atomic::null(),\n }\n }\n\n /// Pushes a value on top of the stack.\n fn push(&self, t: T) {\n let mut n = Owned::new(Node {\n data: ManuallyDrop::new(t),\n next: Atomic::null(),\n });\n\n let guard = epoch::pin();\n\n loop {\n let head = self.head.load(Relaxed, &guard);\n n.next.store(head, Relaxed);\n\n match self\n .head\n .compare_exchange(head, n, Release, Relaxed, &guard)\n {\n Ok(_) => break,\n Err(e) => n = e.new,\n }\n }\n }\n\n /// Attempts to pop the top element from the stack.\n ///\n /// Returns `None` if the stack is empty.\n fn pop(&self) -> Option {\n let guard = epoch::pin();\n loop {\n let head = self.head.load(Acquire, &guard);\n\n match unsafe { head.as_ref() } {\n Some(h) => {\n let next = h.next.load(Relaxed, &guard);\n\n if self\n .head\n .compare_exchange(head, next, Relaxed, Relaxed, &guard)\n .is_ok()\n {\n unsafe {\n guard.defer_destroy(head);\n return Some(ManuallyDrop::into_inner(ptr::read(&(*h).data)));\n }\n }\n }\n None => return None,\n }\n }\n }\n\n /// Returns `true` if the stack is empty.\n fn is_empty(&self) -> bool {\n let guard = epoch::pin();\n self.head.load(Acquire, &guard).is_null()\n }\n }\n\n impl Drop for TreiberStack {\n fn drop(&mut self) {\n while self.pop().is_some() {}\n }\n }\n\n loom::model(|| {\n let stack1 = Arc::new(TreiberStack::new());\n let stack2 = Arc::clone(&stack1);\n\n // use 5 since it's greater than the 4 used for the sanitize feature\n let jh = spawn(move || {\n for i in 0..5 {\n stack2.push(i);\n assert!(stack2.pop().is_some());\n }\n });\n\n for i in 0..5 {\n stack1.push(i);\n assert!(stack1.pop().is_some());\n }\n\n jh.join().unwrap();\n assert!(stack1.pop().is_none());\n assert!(stack1.is_empty());\n });\n }\n}\n" }, { "path": "crossbeam-queue/CHANGELOG.md", "content": "# Version 0.3.12\n\n- Fix stack overflow when pushing large value to `SegQueue`. (#1146, #1147, #1159)\n\n# Version 0.3.11\n\n- Remove dependency on `cfg-if`. (#1072)\n\n# Version 0.3.10\n\n- Relax the minimum supported Rust version to 1.60. (#1056)\n- Implement `UnwindSafe` and `RefUnwindSafe` for `ArrayQueue` and `SegQueue`. (#1053)\n- Optimize `Drop` implementation of `ArrayQueue`. (#1057)\n\n# Version 0.3.9\n\n- Bump the minimum supported Rust version to 1.61. (#1037)\n- Improve support for targets without atomic CAS. (#1037)\n- Remove build script. (#1037)\n\n# Version 0.3.8\n\n- Fix build script bug introduced in 0.3.7. (#932)\n\n# Version 0.3.7\n\n**Note:** This release has been yanked due to regression fixed in 0.3.8.\n\n- Improve support for custom targets. (#922)\n\n# Version 0.3.6\n\n- Bump the minimum supported Rust version to 1.38. (#877)\n\n# Version 0.3.5\n\n- Add `ArrayQueue::force_push`. (#789)\n\n# Version 0.3.4\n\n- Implement `IntoIterator` for `ArrayQueue` and `SegQueue`. (#772)\n\n# Version 0.3.3\n\n- Fix stacked borrows violation in `ArrayQueue` when `-Zmiri-tag-raw-pointers` is enabled. (#763)\n\n# Version 0.3.2\n\n- Support targets that do not have atomic CAS on stable Rust. (#698)\n\n# Version 0.3.1\n\n- Make `SegQueue::new` const fn. (#584)\n- Change license to \"MIT OR Apache-2.0\".\n\n# Version 0.3.0\n\n- Bump the minimum supported Rust version to 1.36.\n- Remove `PushError` and `PopError`.\n\n# Version 0.2.3\n\n- Fix bug in release (yanking 0.2.2)\n\n# Version 0.2.2\n\n- Fix unsoundness issues by adopting `MaybeUninit`. (#458)\n\n# Version 0.2.1\n\n- Add `no_std` support.\n\n# Version 0.2.0\n\n- Bump the minimum required version to 1.28.\n- Bump `crossbeam-utils` to `0.7`.\n\n# Version 0.1.2\n\n- Update `crossbeam-utils` to `0.6.5`.\n\n# Version 0.1.1\n\n- Update `crossbeam-utils` to `0.6.4`.\n\n# Version 0.1.0\n\n- Initial version with `ArrayQueue` and `SegQueue`.\n" }, { "path": "crossbeam-queue/Cargo.toml", "content": "[package]\nname = \"crossbeam-queue\"\n# When publishing a new version:\n# - Update CHANGELOG.md\n# - Update README.md (when increasing major or minor version)\n# - Run './tools/publish.sh crossbeam-queue '\nversion = \"0.3.12\"\nedition = \"2021\"\n# NB: Sync with msrv badge and \"Compatibility\" section in README.md\nrust-version = \"1.60\"\nlicense = \"MIT OR Apache-2.0\"\nrepository = \"https://github.com/crossbeam-rs/crossbeam\"\nhomepage = \"https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-queue\"\ndescription = \"Concurrent queues\"\nkeywords = [\"queue\", \"mpmc\", \"lock-free\", \"producer\", \"consumer\"]\ncategories = [\"concurrency\", \"data-structures\", \"no-std\"]\n\n[package.metadata.docs.rs]\ntargets = [\"x86_64-unknown-linux-gnu\"]\n\n[package.metadata.cargo_check_external_types]\n# The following are external types that are allowed to be exposed in our public API.\nallowed_external_types = [\n]\n\n[features]\ndefault = [\"std\"]\n\n# Enable to use APIs that require `std`.\n# This is enabled by default.\nstd = [\"alloc\", \"crossbeam-utils/std\"]\n\n# Enable to use APIs that require `alloc`.\n# This is enabled by default and also enabled if the `std` feature is enabled.\n#\n# NOTE: Disabling both `std` *and* `alloc` features is not supported yet.\nalloc = []\n\n[dependencies]\ncrossbeam-utils = { version = \"0.8.18\", path = \"../crossbeam-utils\", default-features = false }\n\n[dev-dependencies]\nfastrand = \"2\"\n\n[lints]\nworkspace = true\n" }, { "path": "crossbeam-queue/LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "crossbeam-queue/LICENSE-MIT", "content": "The MIT License (MIT)\n\nCopyright (c) 2019 The Crossbeam Project Developers\n\nPermission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "crossbeam-queue/README.md", "content": "# Crossbeam Queue\n\n[![Build Status](https://github.com/crossbeam-rs/crossbeam/workflows/CI/badge.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/actions)\n[![License](https://img.shields.io/badge/license-MIT_OR_Apache--2.0-blue.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-queue#license)\n[![Cargo](https://img.shields.io/crates/v/crossbeam-queue.svg)](\nhttps://crates.io/crates/crossbeam-queue)\n[![Documentation](https://docs.rs/crossbeam-queue/badge.svg)](\nhttps://docs.rs/crossbeam-queue)\n[![Rust 1.60+](https://img.shields.io/badge/rust-1.60+-lightgray.svg)](\nhttps://www.rust-lang.org)\n[![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ)\n\nThis crate provides concurrent queues that can be shared among threads:\n\n* [`ArrayQueue`], a bounded MPMC queue that allocates a fixed-capacity buffer on construction.\n* [`SegQueue`], an unbounded MPMC queue that allocates small buffers, segments, on demand.\n\nEverything in this crate can be used in `no_std` environments, provided that `alloc` feature is\nenabled.\n\n[`ArrayQueue`]: https://docs.rs/crossbeam-queue/latest/crossbeam_queue/struct.ArrayQueue.html\n[`SegQueue`]: https://docs.rs/crossbeam-queue/latest/crossbeam_queue/struct.SegQueue.html\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\ncrossbeam-queue = \"0.3\"\n```\n\n## Compatibility\n\nCrossbeam Queue supports stable Rust releases going back at least six months,\nand every time the minimum supported Rust version is increased, a new minor\nversion is released. Currently, the minimum supported Rust version is 1.60.\n\n## License\n\nLicensed under either of\n\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n#### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "path": "crossbeam-queue/src/array_queue.rs", "content": "//! The implementation is based on Dmitry Vyukov's bounded MPMC queue.\n//!\n//! Source:\n//! - \n\nuse alloc::boxed::Box;\nuse core::{\n cell::UnsafeCell,\n fmt,\n mem::{self, MaybeUninit},\n panic::{RefUnwindSafe, UnwindSafe},\n sync::atomic::{self, AtomicUsize, Ordering},\n};\n\nuse crossbeam_utils::{Backoff, CachePadded};\n\n/// A slot in a queue.\nstruct Slot {\n /// The current stamp.\n ///\n /// If the stamp equals the tail, this node will be next written to. If it equals head + 1,\n /// this node will be next read from.\n stamp: AtomicUsize,\n\n /// The value in this slot.\n value: UnsafeCell>,\n}\n\n/// A bounded multi-producer multi-consumer queue.\n///\n/// This queue allocates a fixed-capacity buffer on construction, which is used to store pushed\n/// elements. The queue cannot hold more elements than the buffer allows. Attempting to push an\n/// element into a full queue will fail. Alternatively, [`force_push`] makes it possible for\n/// this queue to be used as a ring-buffer. Having a buffer allocated upfront makes this queue\n/// a bit faster than [`SegQueue`].\n///\n/// [`force_push`]: ArrayQueue::force_push\n/// [`SegQueue`]: super::SegQueue\n///\n/// # Examples\n///\n/// ```\n/// use crossbeam_queue::ArrayQueue;\n///\n/// let q = ArrayQueue::new(2);\n///\n/// assert_eq!(q.push('a'), Ok(()));\n/// assert_eq!(q.push('b'), Ok(()));\n/// assert_eq!(q.push('c'), Err('c'));\n/// assert_eq!(q.pop(), Some('a'));\n/// ```\npub struct ArrayQueue {\n /// The head of the queue.\n ///\n /// This value is a \"stamp\" consisting of an index into the buffer and a lap, but packed into a\n /// single `usize`. The lower bits represent the index, while the upper bits represent the lap.\n ///\n /// Elements are popped from the head of the queue.\n head: CachePadded,\n\n /// The tail of the queue.\n ///\n /// This value is a \"stamp\" consisting of an index into the buffer and a lap, but packed into a\n /// single `usize`. The lower bits represent the index, while the upper bits represent the lap.\n ///\n /// Elements are pushed into the tail of the queue.\n tail: CachePadded,\n\n /// The buffer holding slots.\n buffer: Box<[Slot]>,\n\n /// A stamp with the value of `{ lap: 1, index: 0 }`.\n one_lap: usize,\n}\n\nunsafe impl Sync for ArrayQueue {}\nunsafe impl Send for ArrayQueue {}\n\nimpl UnwindSafe for ArrayQueue {}\nimpl RefUnwindSafe for ArrayQueue {}\n\nimpl ArrayQueue {\n /// Creates a new bounded queue with the given capacity.\n ///\n /// # Panics\n ///\n /// Panics if the capacity is zero.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::ArrayQueue;\n ///\n /// let q = ArrayQueue::::new(100);\n /// ```\n pub fn new(cap: usize) -> Self {\n assert!(cap > 0, \"capacity must be non-zero\");\n\n // Head is initialized to `{ lap: 0, index: 0 }`.\n // Tail is initialized to `{ lap: 0, index: 0 }`.\n let head = 0;\n let tail = 0;\n\n // Allocate a buffer of `cap` slots initialized\n // with stamps.\n let buffer: Box<[Slot]> = (0..cap)\n .map(|i| {\n // Set the stamp to `{ lap: 0, index: i }`.\n Slot {\n stamp: AtomicUsize::new(i),\n value: UnsafeCell::new(MaybeUninit::uninit()),\n }\n })\n .collect();\n\n // One lap is the smallest power of two greater than `cap`.\n let one_lap = (cap + 1).next_power_of_two();\n\n Self {\n buffer,\n one_lap,\n head: CachePadded::new(AtomicUsize::new(head)),\n tail: CachePadded::new(AtomicUsize::new(tail)),\n }\n }\n\n fn push_or_else(&self, mut value: T, f: F) -> Result<(), T>\n where\n F: Fn(T, usize, usize, &Slot) -> Result,\n {\n let backoff = Backoff::new();\n let mut tail = self.tail.load(Ordering::Relaxed);\n\n loop {\n // Deconstruct the tail.\n let index = tail & (self.one_lap - 1);\n let lap = tail & !(self.one_lap - 1);\n\n let new_tail = if index + 1 < self.capacity() {\n // Same lap, incremented index.\n // Set to `{ lap: lap, index: index + 1 }`.\n tail + 1\n } else {\n // One lap forward, index wraps around to zero.\n // Set to `{ lap: lap.wrapping_add(1), index: 0 }`.\n lap.wrapping_add(self.one_lap)\n };\n\n // Inspect the corresponding slot.\n debug_assert!(index < self.buffer.len());\n let slot = unsafe { self.buffer.get_unchecked(index) };\n let stamp = slot.stamp.load(Ordering::Acquire);\n\n // If the tail and the stamp match, we may attempt to push.\n if tail == stamp {\n // Try moving the tail.\n match self.tail.compare_exchange_weak(\n tail,\n new_tail,\n Ordering::SeqCst,\n Ordering::Relaxed,\n ) {\n Ok(_) => {\n // Write the value into the slot and update the stamp.\n unsafe {\n slot.value.get().write(MaybeUninit::new(value));\n }\n slot.stamp.store(tail + 1, Ordering::Release);\n return Ok(());\n }\n Err(t) => {\n tail = t;\n backoff.spin();\n }\n }\n } else if stamp.wrapping_add(self.one_lap) == tail + 1 {\n atomic::fence(Ordering::SeqCst);\n value = f(value, tail, new_tail, slot)?;\n backoff.spin();\n tail = self.tail.load(Ordering::Relaxed);\n } else {\n // Snooze because we need to wait for the stamp to get updated.\n backoff.snooze();\n tail = self.tail.load(Ordering::Relaxed);\n }\n }\n }\n\n /// Attempts to push an element into the queue.\n ///\n /// If the queue is full, the element is returned back as an error.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::ArrayQueue;\n ///\n /// let q = ArrayQueue::new(1);\n ///\n /// assert_eq!(q.push(10), Ok(()));\n /// assert_eq!(q.push(20), Err(20));\n /// ```\n pub fn push(&self, value: T) -> Result<(), T> {\n self.push_or_else(value, |v, tail, _, _| {\n let head = self.head.load(Ordering::Relaxed);\n\n // If the head lags one lap behind the tail as well...\n if head.wrapping_add(self.one_lap) == tail {\n // ...then the queue is full.\n Err(v)\n } else {\n Ok(v)\n }\n })\n }\n\n /// Attempts to push an element using an exclusive reference of the queue.\n ///\n /// Atomic operations and checks are omitted\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::ArrayQueue;\n ///\n /// let mut q = ArrayQueue::new(1);\n ///\n /// assert_eq!(q.push_mut(10), Ok(()));\n /// assert_eq!(q.push_mut(20), Err(20));\n /// ```\n pub fn push_mut(&mut self, value: T) -> Result<(), T> {\n let tail = *self.tail.get_mut();\n let head = *self.head.get_mut();\n\n if head.wrapping_add(self.one_lap) == tail {\n return Err(value);\n }\n\n let index = tail & (self.one_lap - 1);\n let lap = tail & !(self.one_lap - 1);\n let new_tail = if index + 1 < self.capacity() {\n tail + 1\n } else {\n lap.wrapping_add(self.one_lap)\n };\n\n *self.tail.get_mut() = new_tail;\n\n let slot = unsafe { self.buffer.get_unchecked_mut(index) };\n unsafe {\n slot.value.get().write(MaybeUninit::new(value));\n }\n *slot.stamp.get_mut() = tail + 1;\n\n Ok(())\n }\n\n /// Pushes an element into the queue, replacing the oldest element if necessary.\n ///\n /// If the queue is full, the oldest element is replaced and returned,\n /// otherwise `None` is returned.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::ArrayQueue;\n ///\n /// let q = ArrayQueue::new(2);\n ///\n /// assert_eq!(q.force_push(10), None);\n /// assert_eq!(q.force_push(20), None);\n /// assert_eq!(q.force_push(30), Some(10));\n /// assert_eq!(q.pop(), Some(20));\n /// ```\n pub fn force_push(&self, value: T) -> Option {\n self.push_or_else(value, |v, tail, new_tail, slot| {\n let head = tail.wrapping_sub(self.one_lap);\n let new_head = new_tail.wrapping_sub(self.one_lap);\n\n // Try moving the head.\n if self\n .head\n .compare_exchange_weak(head, new_head, Ordering::SeqCst, Ordering::Relaxed)\n .is_ok()\n {\n // Move the tail.\n self.tail.store(new_tail, Ordering::SeqCst);\n\n // Swap the previous value.\n let old = unsafe { slot.value.get().replace(MaybeUninit::new(v)).assume_init() };\n\n // Update the stamp.\n slot.stamp.store(tail + 1, Ordering::Release);\n\n Err(old)\n } else {\n Ok(v)\n }\n })\n .err()\n }\n\n /// Attempts to pop an element from the queue.\n ///\n /// If the queue is empty, `None` is returned.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::ArrayQueue;\n ///\n /// let q = ArrayQueue::new(1);\n /// assert_eq!(q.push(10), Ok(()));\n ///\n /// assert_eq!(q.pop(), Some(10));\n /// assert!(q.pop().is_none());\n /// ```\n pub fn pop(&self) -> Option {\n let backoff = Backoff::new();\n let mut head = self.head.load(Ordering::Relaxed);\n\n loop {\n // Deconstruct the head.\n let index = head & (self.one_lap - 1);\n let lap = head & !(self.one_lap - 1);\n\n // Inspect the corresponding slot.\n debug_assert!(index < self.buffer.len());\n let slot = unsafe { self.buffer.get_unchecked(index) };\n let stamp = slot.stamp.load(Ordering::Acquire);\n\n // If the stamp is ahead of the head by 1, we may attempt to pop.\n if head + 1 == stamp {\n let new = if index + 1 < self.capacity() {\n // Same lap, incremented index.\n // Set to `{ lap: lap, index: index + 1 }`.\n head + 1\n } else {\n // One lap forward, index wraps around to zero.\n // Set to `{ lap: lap.wrapping_add(1), index: 0 }`.\n lap.wrapping_add(self.one_lap)\n };\n\n // Try moving the head.\n match self.head.compare_exchange_weak(\n head,\n new,\n Ordering::SeqCst,\n Ordering::Relaxed,\n ) {\n Ok(_) => {\n // Read the value from the slot and update the stamp.\n let msg = unsafe { slot.value.get().read().assume_init() };\n slot.stamp\n .store(head.wrapping_add(self.one_lap), Ordering::Release);\n return Some(msg);\n }\n Err(h) => {\n head = h;\n backoff.spin();\n }\n }\n } else if stamp == head {\n atomic::fence(Ordering::SeqCst);\n let tail = self.tail.load(Ordering::Relaxed);\n\n // If the tail equals the head, that means the channel is empty.\n if tail == head {\n return None;\n }\n\n backoff.spin();\n head = self.head.load(Ordering::Relaxed);\n } else {\n // Snooze because we need to wait for the stamp to get updated.\n backoff.snooze();\n head = self.head.load(Ordering::Relaxed);\n }\n }\n }\n\n /// Attempts to pop an element using an exclusive reference of the queue.\n ///\n /// Due to having an exclusive reference, atomic operations and checks are omitted\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::ArrayQueue;\n ///\n /// let mut q = ArrayQueue::new(1);\n /// assert_eq!(q.push(10), Ok(()));\n ///\n /// assert_eq!(q.pop_mut(), Some(10));\n /// assert!(q.pop_mut().is_none());\n /// ```\n pub fn pop_mut(&mut self) -> Option {\n let head = *self.head.get_mut();\n let tail = *self.tail.get_mut();\n\n // If the tail equals the head, that means the channel is empty.\n if tail == head {\n return None;\n }\n let index = head & (self.one_lap - 1);\n let lap = head & !(self.one_lap - 1);\n\n // Inspect the corresponding slot.\n debug_assert!(index < self.buffer.len());\n\n let new = if index + 1 < self.capacity() {\n // Same lap, incremented index.\n // Set to `{ lap: lap, index: index + 1 }`.\n head + 1\n } else {\n // One lap forward, index wraps around to zero.\n // Set to `{ lap: lap.wrapping_add(1), index: 0 }`.\n lap.wrapping_add(self.one_lap)\n };\n\n let slot = unsafe { self.buffer.get_unchecked_mut(index) };\n\n let msg = unsafe { slot.value.get().read().assume_init() };\n *slot.stamp.get_mut() = head.wrapping_add(self.one_lap);\n *self.head.get_mut() = new;\n Some(msg)\n }\n\n /// Returns the capacity of the queue.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::ArrayQueue;\n ///\n /// let q = ArrayQueue::::new(100);\n ///\n /// assert_eq!(q.capacity(), 100);\n /// ```\n #[inline]\n pub fn capacity(&self) -> usize {\n self.buffer.len()\n }\n\n /// Returns `true` if the queue is empty.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::ArrayQueue;\n ///\n /// let q = ArrayQueue::new(100);\n ///\n /// assert!(q.is_empty());\n /// q.push(1).unwrap();\n /// assert!(!q.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n let head = self.head.load(Ordering::SeqCst);\n let tail = self.tail.load(Ordering::SeqCst);\n\n // Is the tail lagging one lap behind head?\n // Is the tail equal to the head?\n //\n // Note: If the head changes just before we load the tail, that means there was a moment\n // when the channel was not empty, so it is safe to just return `false`.\n tail == head\n }\n\n /// Returns `true` if the queue is full.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::ArrayQueue;\n ///\n /// let q = ArrayQueue::new(1);\n ///\n /// assert!(!q.is_full());\n /// q.push(1).unwrap();\n /// assert!(q.is_full());\n /// ```\n pub fn is_full(&self) -> bool {\n let tail = self.tail.load(Ordering::SeqCst);\n let head = self.head.load(Ordering::SeqCst);\n\n // Is the head lagging one lap behind tail?\n //\n // Note: If the tail changes just before we load the head, that means there was a moment\n // when the queue was not full, so it is safe to just return `false`.\n head.wrapping_add(self.one_lap) == tail\n }\n\n /// Returns the number of elements in the queue.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::ArrayQueue;\n ///\n /// let q = ArrayQueue::new(100);\n /// assert_eq!(q.len(), 0);\n ///\n /// q.push(10).unwrap();\n /// assert_eq!(q.len(), 1);\n ///\n /// q.push(20).unwrap();\n /// assert_eq!(q.len(), 2);\n /// ```\n pub fn len(&self) -> usize {\n loop {\n // Load the tail, then load the head.\n let tail = self.tail.load(Ordering::SeqCst);\n let head = self.head.load(Ordering::SeqCst);\n\n // If the tail didn't change, we've got consistent values to work with.\n if self.tail.load(Ordering::SeqCst) == tail {\n let hix = head & (self.one_lap - 1);\n let tix = tail & (self.one_lap - 1);\n\n return if hix < tix {\n tix - hix\n } else if hix > tix {\n self.capacity() - hix + tix\n } else if tail == head {\n 0\n } else {\n self.capacity()\n };\n }\n }\n }\n}\n\nimpl Drop for ArrayQueue {\n fn drop(&mut self) {\n if mem::needs_drop::() {\n // Get the index of the head.\n let head = *self.head.get_mut();\n let tail = *self.tail.get_mut();\n\n let hix = head & (self.one_lap - 1);\n let tix = tail & (self.one_lap - 1);\n\n let len = if hix < tix {\n tix - hix\n } else if hix > tix {\n self.capacity() - hix + tix\n } else if tail == head {\n 0\n } else {\n self.capacity()\n };\n\n // Loop over all slots that hold a message and drop them.\n for i in 0..len {\n // Compute the index of the next slot holding a message.\n let index = if hix + i < self.capacity() {\n hix + i\n } else {\n hix + i - self.capacity()\n };\n\n unsafe {\n debug_assert!(index < self.buffer.len());\n let slot = self.buffer.get_unchecked_mut(index);\n (*slot.value.get()).assume_init_drop();\n }\n }\n }\n }\n}\n\nimpl fmt::Debug for ArrayQueue {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"ArrayQueue { .. }\")\n }\n}\n\nimpl IntoIterator for ArrayQueue {\n type Item = T;\n\n type IntoIter = IntoIter;\n\n fn into_iter(self) -> Self::IntoIter {\n IntoIter { value: self }\n }\n}\n\n#[derive(Debug)]\npub struct IntoIter {\n value: ArrayQueue,\n}\n\nimpl Iterator for IntoIter {\n type Item = T;\n\n fn next(&mut self) -> Option {\n let value = &mut self.value;\n let head = *value.head.get_mut();\n if value.head.get_mut() != value.tail.get_mut() {\n let index = head & (value.one_lap - 1);\n let lap = head & !(value.one_lap - 1);\n // SAFETY: We have mutable access to this, so we can read without\n // worrying about concurrency. Furthermore, we know this is\n // initialized because it is the value pointed at by `value.head`\n // and this is a non-empty queue.\n let val = unsafe {\n debug_assert!(index < value.buffer.len());\n let slot = value.buffer.get_unchecked_mut(index);\n slot.value.get().read().assume_init()\n };\n let new = if index + 1 < value.capacity() {\n // Same lap, incremented index.\n // Set to `{ lap: lap, index: index + 1 }`.\n head + 1\n } else {\n // One lap forward, index wraps around to zero.\n // Set to `{ lap: lap.wrapping_add(1), index: 0 }`.\n lap.wrapping_add(value.one_lap)\n };\n *value.head.get_mut() = new;\n Some(val)\n } else {\n None\n }\n }\n}\n" }, { "path": "crossbeam-queue/src/lib.rs", "content": "//! Concurrent queues.\n//!\n//! This crate provides concurrent queues that can be shared among threads:\n//!\n//! * [`ArrayQueue`], a bounded MPMC queue that allocates a fixed-capacity buffer on construction.\n//! * [`SegQueue`], an unbounded MPMC queue that allocates small buffers, segments, on demand.\n\n#![no_std]\n#![doc(test(\n no_crate_inject,\n attr(allow(dead_code, unused_assignments, unused_variables))\n))]\n#![warn(\n missing_docs,\n unsafe_op_in_unsafe_fn,\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nextern crate alloc;\n#[cfg(feature = \"std\")]\nextern crate std;\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod alloc_helper;\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod array_queue;\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod seg_queue;\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\npub use crate::{array_queue::ArrayQueue, seg_queue::SegQueue};\n" }, { "path": "crossbeam-queue/src/seg_queue.rs", "content": "use alloc::{alloc::handle_alloc_error, boxed::Box};\nuse core::{\n alloc::Layout,\n cell::UnsafeCell,\n fmt,\n marker::PhantomData,\n mem::MaybeUninit,\n panic::{RefUnwindSafe, UnwindSafe},\n ptr,\n sync::atomic::{self, AtomicPtr, AtomicUsize, Ordering},\n};\n\nuse crossbeam_utils::{Backoff, CachePadded};\n\nuse crate::alloc_helper::Global;\n\n// Bits indicating the state of a slot:\n// * If a value has been written into the slot, `WRITE` is set.\n// * If a value has been read from the slot, `READ` is set.\n// * If the block is being destroyed, `DESTROY` is set.\nconst WRITE: usize = 1;\nconst READ: usize = 2;\nconst DESTROY: usize = 4;\n\n// Each block covers one \"lap\" of indices.\nconst LAP: usize = 32;\n// The maximum number of values a block can hold.\nconst BLOCK_CAP: usize = LAP - 1;\n// How many lower bits are reserved for metadata.\nconst SHIFT: usize = 1;\n// Indicates that the block is not the last one.\nconst HAS_NEXT: usize = 1;\n\n/// A slot in a block.\nstruct Slot {\n /// The value.\n value: UnsafeCell>,\n\n /// The state of the slot.\n state: AtomicUsize,\n}\n\nimpl Slot {\n /// Waits until a value is written into the slot.\n fn wait_write(&self) {\n let backoff = Backoff::new();\n while self.state.load(Ordering::Acquire) & WRITE == 0 {\n backoff.snooze();\n }\n }\n}\n\n/// A block in a linked list.\n///\n/// Each block in the list can hold up to `BLOCK_CAP` values.\nstruct Block {\n /// The next block in the linked list.\n next: AtomicPtr>,\n\n /// Slots for values.\n slots: [Slot; BLOCK_CAP],\n}\n\nimpl Block {\n const LAYOUT: Layout = {\n let layout = Layout::new::();\n assert!(\n layout.size() != 0,\n \"Block should never be zero-sized, as it has an AtomicPtr field\"\n );\n layout\n };\n\n /// Creates an empty block.\n fn new() -> Box {\n // unsafe { Box::new_zeroed().assume_init() } requires Rust 1.92\n match Global.allocate_zeroed(Self::LAYOUT) {\n Some(ptr) => {\n // SAFETY: This is safe because:\n // [1] `Block::next` (AtomicPtr) may be safely zero initialized.\n // [2] `Block::slots` (Array) may be safely zero initialized because of [3, 4].\n // [3] `Slot::value` (UnsafeCell) may be safely zero initialized because it\n // holds a MaybeUninit.\n // [4] `Slot::state` (AtomicUsize) may be safely zero initialized.\n unsafe { Box::from_raw(ptr.as_ptr().cast()) }\n }\n // Handle allocation failure\n None => handle_alloc_error(Self::LAYOUT),\n }\n }\n\n /// Waits until the next pointer is set.\n fn wait_next(&self) -> *mut Self {\n let backoff = Backoff::new();\n loop {\n let next = self.next.load(Ordering::Acquire);\n if !next.is_null() {\n return next;\n }\n backoff.snooze();\n }\n }\n\n /// Sets the `DESTROY` bit in slots starting from `start` and destroys the block.\n unsafe fn destroy(this: *mut Self, start: usize) {\n // It is not necessary to set the `DESTROY` bit in the last slot because that slot has\n // begun destruction of the block.\n for i in start..BLOCK_CAP - 1 {\n let slot = unsafe { (*this).slots.get_unchecked(i) };\n\n // Mark the `DESTROY` bit if a thread is still using the slot.\n if slot.state.load(Ordering::Acquire) & READ == 0\n && slot.state.fetch_or(DESTROY, Ordering::AcqRel) & READ == 0\n {\n // If a thread is still using the slot, it will continue destruction of the block.\n return;\n }\n }\n // No thread is using the block, now it is safe to destroy it.\n drop(unsafe { Box::from_raw(this) });\n }\n\n /// Destroys the block. Only safe to call with exclusive access, when no other thread is using it.\n unsafe fn destroy_mut(this: *mut Self) {\n drop(unsafe { Box::from_raw(this) });\n }\n}\n\n/// A position in a queue.\nstruct Position {\n /// The index in the queue.\n index: AtomicUsize,\n\n /// The block in the linked list.\n block: AtomicPtr>,\n}\n\n/// An unbounded multi-producer multi-consumer queue.\n///\n/// This queue is implemented as a linked list of segments, where each segment is a small buffer\n/// that can hold a handful of elements. There is no limit to how many elements can be in the queue\n/// at a time. However, since segments need to be dynamically allocated as elements get pushed,\n/// this queue is somewhat slower than [`ArrayQueue`].\n///\n/// [`ArrayQueue`]: super::ArrayQueue\n///\n/// # Examples\n///\n/// ```\n/// use crossbeam_queue::SegQueue;\n///\n/// let q = SegQueue::new();\n///\n/// q.push('a');\n/// q.push('b');\n///\n/// assert_eq!(q.pop(), Some('a'));\n/// assert_eq!(q.pop(), Some('b'));\n/// assert!(q.pop().is_none());\n/// ```\npub struct SegQueue {\n /// The head of the queue.\n head: CachePadded>,\n\n /// The tail of the queue.\n tail: CachePadded>,\n\n /// Indicates that dropping a `SegQueue` may drop values of type `T`.\n _marker: PhantomData,\n}\n\nunsafe impl Send for SegQueue {}\nunsafe impl Sync for SegQueue {}\n\nimpl UnwindSafe for SegQueue {}\nimpl RefUnwindSafe for SegQueue {}\n\nimpl SegQueue {\n /// Creates a new unbounded queue.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::SegQueue;\n ///\n /// let q = SegQueue::::new();\n /// ```\n pub const fn new() -> Self {\n Self {\n head: CachePadded::new(Position {\n block: AtomicPtr::new(ptr::null_mut()),\n index: AtomicUsize::new(0),\n }),\n tail: CachePadded::new(Position {\n block: AtomicPtr::new(ptr::null_mut()),\n index: AtomicUsize::new(0),\n }),\n _marker: PhantomData,\n }\n }\n\n /// Pushes back an element to the tail.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::SegQueue;\n ///\n /// let q = SegQueue::new();\n ///\n /// q.push(10);\n /// q.push(20);\n /// ```\n pub fn push(&self, value: T) {\n let backoff = Backoff::new();\n let mut tail = self.tail.index.load(Ordering::Acquire);\n let mut block = self.tail.block.load(Ordering::Acquire);\n let mut next_block = None;\n\n loop {\n // Calculate the offset of the index into the block.\n let offset = (tail >> SHIFT) % LAP;\n\n // If we reached the end of the block, wait until the next one is installed.\n if offset == BLOCK_CAP {\n backoff.snooze();\n tail = self.tail.index.load(Ordering::Acquire);\n block = self.tail.block.load(Ordering::Acquire);\n continue;\n }\n\n // If we're going to have to install the next block, allocate it in advance in order to\n // make the wait for other threads as short as possible.\n if offset + 1 == BLOCK_CAP && next_block.is_none() {\n next_block = Some(Block::::new());\n }\n\n // If this is the first push operation, we need to allocate the first block.\n if block.is_null() {\n let new = Box::into_raw(Block::::new());\n\n if self\n .tail\n .block\n .compare_exchange(block, new, Ordering::Release, Ordering::Relaxed)\n .is_ok()\n {\n self.head.block.store(new, Ordering::Release);\n block = new;\n } else {\n next_block = unsafe { Some(Box::from_raw(new)) };\n tail = self.tail.index.load(Ordering::Acquire);\n block = self.tail.block.load(Ordering::Acquire);\n continue;\n }\n }\n\n let new_tail = tail + (1 << SHIFT);\n\n // Try advancing the tail forward.\n match self.tail.index.compare_exchange_weak(\n tail,\n new_tail,\n Ordering::SeqCst,\n Ordering::Acquire,\n ) {\n Ok(_) => unsafe {\n // If we've reached the end of the block, install the next one.\n if offset + 1 == BLOCK_CAP {\n let next_block = Box::into_raw(next_block.unwrap());\n let next_index = new_tail.wrapping_add(1 << SHIFT);\n\n self.tail.block.store(next_block, Ordering::Release);\n self.tail.index.store(next_index, Ordering::Release);\n (*block).next.store(next_block, Ordering::Release);\n }\n\n // Write the value into the slot.\n let slot = (*block).slots.get_unchecked(offset);\n slot.value.get().write(MaybeUninit::new(value));\n slot.state.fetch_or(WRITE, Ordering::Release);\n\n return;\n },\n Err(t) => {\n tail = t;\n block = self.tail.block.load(Ordering::Acquire);\n backoff.spin();\n }\n }\n }\n }\n\n /// Pushes an element to the queue with exclusive mutable access.\n ///\n /// Avoids atomic operations and synchronization, assuming\n /// no other threads access the queue concurrently.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::SegQueue;\n ///\n /// let mut q = SegQueue::new();\n ///\n /// q.push_mut(10);\n /// q.push_mut(20);\n /// ```\n pub fn push_mut(&mut self, value: T) {\n let tail = *self.tail.index.get_mut();\n let mut block = *self.tail.block.get_mut();\n\n // Calculate the offset of the index into the block.\n let offset = (tail >> SHIFT) % LAP;\n\n // If this is the first push operation, we need to allocate the first block.\n if block.is_null() {\n let new = Box::into_raw(Block::::new());\n *self.head.block.get_mut() = new;\n *self.tail.block.get_mut() = new;\n\n block = new;\n }\n\n let new_tail = tail + (1 << SHIFT);\n\n *self.tail.index.get_mut() = new_tail;\n\n unsafe {\n // If we've reached the end of the block, install the next one.\n if offset + 1 == BLOCK_CAP {\n let next_block = Box::into_raw(Block::::new());\n let next_index = new_tail.wrapping_add(1 << SHIFT);\n\n *self.tail.block.get_mut() = next_block;\n *self.tail.index.get_mut() = next_index;\n *(*block).next.get_mut() = next_block;\n }\n\n // Write the value into the slot.\n let slot = (*block).slots.get_unchecked(offset);\n slot.value.get().write(MaybeUninit::new(value));\n *(*block).slots.get_unchecked_mut(offset).state.get_mut() |= WRITE;\n }\n }\n\n /// Pops the head element from the queue.\n ///\n /// If the queue is empty, `None` is returned.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::SegQueue;\n ///\n /// let q = SegQueue::new();\n ///\n /// q.push(10);\n /// q.push(20);\n /// assert_eq!(q.pop(), Some(10));\n /// assert_eq!(q.pop(), Some(20));\n /// assert!(q.pop().is_none());\n /// ```\n pub fn pop(&self) -> Option {\n let backoff = Backoff::new();\n let mut head = self.head.index.load(Ordering::Acquire);\n let mut block = self.head.block.load(Ordering::Acquire);\n\n loop {\n // Calculate the offset of the index into the block.\n let offset = (head >> SHIFT) % LAP;\n\n // If we reached the end of the block, wait until the next one is installed.\n if offset == BLOCK_CAP {\n backoff.snooze();\n head = self.head.index.load(Ordering::Acquire);\n block = self.head.block.load(Ordering::Acquire);\n continue;\n }\n\n let mut new_head = head + (1 << SHIFT);\n\n if new_head & HAS_NEXT == 0 {\n atomic::fence(Ordering::SeqCst);\n let tail = self.tail.index.load(Ordering::Relaxed);\n\n // If the tail equals the head, that means the queue is empty.\n if head >> SHIFT == tail >> SHIFT {\n return None;\n }\n\n // If head and tail are not in the same block, set `HAS_NEXT` in head.\n if (head >> SHIFT) / LAP != (tail >> SHIFT) / LAP {\n new_head |= HAS_NEXT;\n }\n }\n\n // The block can be null here only if the first push operation is in progress. In that\n // case, just wait until it gets initialized.\n if block.is_null() {\n backoff.snooze();\n head = self.head.index.load(Ordering::Acquire);\n block = self.head.block.load(Ordering::Acquire);\n continue;\n }\n\n // Try moving the head index forward.\n match self.head.index.compare_exchange_weak(\n head,\n new_head,\n Ordering::SeqCst,\n Ordering::Acquire,\n ) {\n Ok(_) => unsafe {\n // If we've reached the end of the block, move to the next one.\n if offset + 1 == BLOCK_CAP {\n let next = (*block).wait_next();\n let mut next_index = (new_head & !HAS_NEXT).wrapping_add(1 << SHIFT);\n if !(*next).next.load(Ordering::Relaxed).is_null() {\n next_index |= HAS_NEXT;\n }\n\n self.head.block.store(next, Ordering::Release);\n self.head.index.store(next_index, Ordering::Release);\n }\n\n // Read the value.\n let slot = (*block).slots.get_unchecked(offset);\n slot.wait_write();\n let value = slot.value.get().read().assume_init();\n\n // Destroy the block if we've reached the end, or if another thread wanted to\n // destroy but couldn't because we were busy reading from the slot.\n if offset + 1 == BLOCK_CAP {\n Block::destroy(block, 0);\n } else if slot.state.fetch_or(READ, Ordering::AcqRel) & DESTROY != 0 {\n Block::destroy(block, offset + 1);\n }\n\n return Some(value);\n },\n Err(h) => {\n head = h;\n block = self.head.block.load(Ordering::Acquire);\n backoff.spin();\n }\n }\n }\n }\n\n /// Pops the head element from the queue using an exclusive reference.\n ///\n /// Avoids atomic operations and synchronization, assuming\n /// no other threads access the queue concurrently.\n ///\n /// If the queue is empty, `None` is returned.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::SegQueue;\n ///\n /// let mut q = SegQueue::new();\n ///\n /// q.push(10);\n /// q.push(20);\n /// assert_eq!(q.pop_mut(), Some(10));\n /// assert_eq!(q.pop_mut(), Some(20));\n /// assert!(q.pop_mut().is_none());\n /// ```\n pub fn pop_mut(&mut self) -> Option {\n let head = *self.head.index.get_mut();\n let block = *self.head.block.get_mut();\n\n // Calculate the offset of the index into the block.\n let offset = (head >> SHIFT) % LAP;\n\n let mut new_head = head + (1 << SHIFT);\n\n if new_head & HAS_NEXT == 0 {\n let tail = *self.tail.index.get_mut();\n\n // If the tail equals the head, that means the queue is empty.\n if head >> SHIFT == tail >> SHIFT {\n return None;\n }\n\n // If head and tail are not in the same block, set `HAS_NEXT` in head.\n if (head >> SHIFT) / LAP != (tail >> SHIFT) / LAP {\n new_head |= HAS_NEXT;\n }\n }\n\n *self.head.index.get_mut() = new_head;\n\n unsafe {\n // If we've reached the end of the block, move to the next one.\n if offset + 1 == BLOCK_CAP {\n let next = *(*block).next.get_mut();\n let mut next_index = (new_head & !HAS_NEXT).wrapping_add(1 << SHIFT);\n if !(*next).next.get_mut().is_null() {\n next_index |= HAS_NEXT;\n }\n\n *self.head.block.get_mut() = next;\n *self.head.index.get_mut() = next_index;\n }\n\n // Read the value.\n let slot = (*block).slots.get_unchecked(offset);\n let value = slot.value.get().read().assume_init();\n\n // Destroy the block if we've reached the end\n if offset + 1 == BLOCK_CAP {\n Block::destroy_mut(block);\n } else {\n let state = *(*block).slots.get_unchecked_mut(offset).state.get_mut();\n *(*block).slots.get_unchecked_mut(offset).state.get_mut() = state | READ;\n if state & DESTROY != 0 {\n Block::destroy(block, offset + 1);\n }\n }\n\n Some(value)\n }\n }\n\n /// Returns `true` if the queue is empty.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::SegQueue;\n ///\n /// let q = SegQueue::new();\n ///\n /// assert!(q.is_empty());\n /// q.push(1);\n /// assert!(!q.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n let head = self.head.index.load(Ordering::SeqCst);\n let tail = self.tail.index.load(Ordering::SeqCst);\n head >> SHIFT == tail >> SHIFT\n }\n\n /// Returns the number of elements in the queue.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_queue::SegQueue;\n ///\n /// let q = SegQueue::new();\n /// assert_eq!(q.len(), 0);\n ///\n /// q.push(10);\n /// assert_eq!(q.len(), 1);\n ///\n /// q.push(20);\n /// assert_eq!(q.len(), 2);\n /// ```\n pub fn len(&self) -> usize {\n loop {\n // Load the tail index, then load the head index.\n let mut tail = self.tail.index.load(Ordering::SeqCst);\n let mut head = self.head.index.load(Ordering::SeqCst);\n\n // If the tail index didn't change, we've got consistent indices to work with.\n if self.tail.index.load(Ordering::SeqCst) == tail {\n // Erase the lower bits.\n tail &= !((1 << SHIFT) - 1);\n head &= !((1 << SHIFT) - 1);\n\n // Fix up indices if they fall onto block ends.\n if (tail >> SHIFT) & (LAP - 1) == LAP - 1 {\n tail = tail.wrapping_add(1 << SHIFT);\n }\n if (head >> SHIFT) & (LAP - 1) == LAP - 1 {\n head = head.wrapping_add(1 << SHIFT);\n }\n\n // Rotate indices so that head falls into the first block.\n let lap = (head >> SHIFT) / LAP;\n tail = tail.wrapping_sub((lap * LAP) << SHIFT);\n head = head.wrapping_sub((lap * LAP) << SHIFT);\n\n // Remove the lower bits.\n tail >>= SHIFT;\n head >>= SHIFT;\n\n // Return the difference minus the number of blocks between tail and head.\n return tail - head - tail / LAP;\n }\n }\n }\n}\n\nimpl Drop for SegQueue {\n fn drop(&mut self) {\n let mut head = *self.head.index.get_mut();\n let mut tail = *self.tail.index.get_mut();\n let mut block = *self.head.block.get_mut();\n\n // Erase the lower bits.\n head &= !((1 << SHIFT) - 1);\n tail &= !((1 << SHIFT) - 1);\n\n unsafe {\n // Drop all values between `head` and `tail` and deallocate the heap-allocated blocks.\n while head != tail {\n let offset = (head >> SHIFT) % LAP;\n\n if offset < BLOCK_CAP {\n // Drop the value in the slot.\n let slot = (*block).slots.get_unchecked(offset);\n (*slot.value.get()).assume_init_drop();\n } else {\n // Deallocate the block and move to the next one.\n let next = *(*block).next.get_mut();\n drop(Box::from_raw(block));\n block = next;\n }\n\n head = head.wrapping_add(1 << SHIFT);\n }\n\n // Deallocate the last remaining block.\n if !block.is_null() {\n drop(Box::from_raw(block));\n }\n }\n }\n}\n\nimpl fmt::Debug for SegQueue {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"SegQueue { .. }\")\n }\n}\n\nimpl Default for SegQueue {\n fn default() -> Self {\n Self::new()\n }\n}\n\nimpl IntoIterator for SegQueue {\n type Item = T;\n\n type IntoIter = IntoIter;\n\n fn into_iter(self) -> Self::IntoIter {\n IntoIter { value: self }\n }\n}\n\n#[derive(Debug)]\npub struct IntoIter {\n value: SegQueue,\n}\n\nimpl Iterator for IntoIter {\n type Item = T;\n\n fn next(&mut self) -> Option {\n let value = &mut self.value;\n let head = *value.head.index.get_mut();\n let tail = *value.tail.index.get_mut();\n if head >> SHIFT == tail >> SHIFT {\n None\n } else {\n let block = *value.head.block.get_mut();\n let offset = (head >> SHIFT) % LAP;\n\n // SAFETY: We have mutable access to this, so we can read without\n // worrying about concurrency. Furthermore, we know this is\n // initialized because it is the value pointed at by `value.head`\n // and this is a non-empty queue.\n let item = unsafe {\n let slot = (*block).slots.get_unchecked(offset);\n slot.value.get().read().assume_init()\n };\n if offset + 1 == BLOCK_CAP {\n // Deallocate the block and move to the next one.\n // SAFETY: The block is initialized because we've been reading\n // from it this entire time. We can drop it b/c everything has\n // been read out of it, so nothing is pointing to it anymore.\n unsafe {\n let next = *(*block).next.get_mut();\n drop(Box::from_raw(block));\n *value.head.block.get_mut() = next;\n }\n // The last value in a block is empty, so skip it\n *value.head.index.get_mut() = head.wrapping_add(2 << SHIFT);\n // Double-check that we're pointing to the first item in a block.\n debug_assert_eq!((*value.head.index.get_mut() >> SHIFT) % LAP, 0);\n } else {\n *value.head.index.get_mut() = head.wrapping_add(1 << SHIFT);\n }\n Some(item)\n }\n }\n}\n" }, { "path": "crossbeam-queue/tests/array_queue.rs", "content": "use std::sync::atomic::{AtomicUsize, Ordering};\n\nuse crossbeam_queue::ArrayQueue;\nuse crossbeam_utils::thread::scope;\n\n#[test]\nfn smoke() {\n let q = ArrayQueue::new(1);\n\n q.push(7).unwrap();\n assert_eq!(q.pop(), Some(7));\n\n q.push(8).unwrap();\n assert_eq!(q.pop(), Some(8));\n assert!(q.pop().is_none());\n}\n\n#[test]\nfn capacity() {\n for i in 1..10 {\n let q = ArrayQueue::::new(i);\n assert_eq!(q.capacity(), i);\n }\n}\n\n#[test]\n#[should_panic(expected = \"capacity must be non-zero\")]\nfn zero_capacity() {\n let _ = ArrayQueue::::new(0);\n}\n\n#[test]\nfn len_empty_full() {\n let q = ArrayQueue::new(2);\n\n assert_eq!(q.len(), 0);\n assert!(q.is_empty());\n assert!(!q.is_full());\n\n q.push(()).unwrap();\n\n assert_eq!(q.len(), 1);\n assert!(!q.is_empty());\n assert!(!q.is_full());\n\n q.push(()).unwrap();\n\n assert_eq!(q.len(), 2);\n assert!(!q.is_empty());\n assert!(q.is_full());\n\n q.pop().unwrap();\n\n assert_eq!(q.len(), 1);\n assert!(!q.is_empty());\n assert!(!q.is_full());\n}\n\n#[test]\nfn exclusive_reference() {\n let mut q = ArrayQueue::new(2);\n\n assert_eq!(q.len(), 0);\n assert!(q.is_empty());\n\n q.push_mut(()).unwrap();\n\n assert_eq!(q.len(), 1);\n assert!(!q.is_empty());\n assert!(!q.is_full());\n\n q.push_mut(()).unwrap();\n\n assert_eq!(q.len(), 2);\n assert!(!q.is_empty());\n assert!(q.is_full());\n\n q.pop_mut().unwrap();\n\n assert_eq!(q.len(), 1);\n assert!(!q.is_empty());\n assert!(!q.is_full());\n\n q.pop().unwrap();\n\n assert_eq!(q.len(), 0);\n}\n\n#[test]\nfn len() {\n const COUNT: usize = if cfg!(miri) { 30 } else { 25_000 };\n const CAP: usize = if cfg!(miri) { 40 } else { 1000 };\n const ITERS: usize = CAP / 20;\n\n let q = ArrayQueue::new(CAP);\n assert_eq!(q.len(), 0);\n\n for _ in 0..CAP / 10 {\n for i in 0..ITERS {\n q.push(i).unwrap();\n assert_eq!(q.len(), i + 1);\n }\n\n for i in 0..ITERS {\n q.pop().unwrap();\n assert_eq!(q.len(), ITERS - i - 1);\n }\n }\n assert_eq!(q.len(), 0);\n\n for i in 0..CAP {\n q.push(i).unwrap();\n assert_eq!(q.len(), i + 1);\n }\n\n for _ in 0..CAP {\n q.pop().unwrap();\n }\n assert_eq!(q.len(), 0);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n loop {\n if let Some(x) = q.pop() {\n assert_eq!(x, i);\n break;\n }\n }\n let len = q.len();\n assert!(len <= CAP);\n }\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n while q.push(i).is_err() {}\n let len = q.len();\n assert!(len <= CAP);\n }\n });\n })\n .unwrap();\n assert_eq!(q.len(), 0);\n}\n\n#[test]\nfn spsc() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 100_000 };\n\n let q = ArrayQueue::new(3);\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n loop {\n if let Some(x) = q.pop() {\n assert_eq!(x, i);\n break;\n }\n }\n }\n assert!(q.pop().is_none());\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n while q.push(i).is_err() {}\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn spsc_ring_buffer() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 100_000 };\n\n let t = AtomicUsize::new(1);\n let q = ArrayQueue::::new(3);\n let v = (0..COUNT).map(|_| AtomicUsize::new(0)).collect::>();\n\n scope(|scope| {\n scope.spawn(|_| {\n loop {\n match t.load(Ordering::SeqCst) {\n 0 if q.is_empty() => break,\n\n _ => {\n while let Some(n) = q.pop() {\n v[n].fetch_add(1, Ordering::SeqCst);\n }\n }\n }\n }\n });\n\n scope.spawn(|_| {\n for i in 0..COUNT {\n if let Some(n) = q.force_push(i) {\n v[n].fetch_add(1, Ordering::SeqCst);\n }\n }\n\n t.fetch_sub(1, Ordering::SeqCst);\n });\n })\n .unwrap();\n\n for c in v {\n assert_eq!(c.load(Ordering::SeqCst), 1);\n }\n}\n\n#[test]\nfn mpmc() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 25_000 };\n const THREADS: usize = 4;\n\n let q = ArrayQueue::::new(3);\n let v = (0..COUNT).map(|_| AtomicUsize::new(0)).collect::>();\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n let n = loop {\n if let Some(x) = q.pop() {\n break x;\n }\n };\n v[n].fetch_add(1, Ordering::SeqCst);\n }\n });\n }\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..COUNT {\n while q.push(i).is_err() {}\n }\n });\n }\n })\n .unwrap();\n\n for c in v {\n assert_eq!(c.load(Ordering::SeqCst), THREADS);\n }\n}\n\n#[test]\nfn mpmc_ring_buffer() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 25_000 };\n const THREADS: usize = 4;\n\n let t = AtomicUsize::new(THREADS);\n let q = ArrayQueue::::new(3);\n let v = (0..COUNT).map(|_| AtomicUsize::new(0)).collect::>();\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n loop {\n match t.load(Ordering::SeqCst) {\n 0 if q.is_empty() => break,\n\n _ => {\n while let Some(n) = q.pop() {\n v[n].fetch_add(1, Ordering::SeqCst);\n }\n }\n }\n }\n });\n }\n\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..COUNT {\n if let Some(n) = q.force_push(i) {\n v[n].fetch_add(1, Ordering::SeqCst);\n }\n }\n\n t.fetch_sub(1, Ordering::SeqCst);\n });\n }\n })\n .unwrap();\n\n for c in v {\n assert_eq!(c.load(Ordering::SeqCst), THREADS);\n }\n}\n\n#[test]\nfn drops() {\n let runs: usize = if cfg!(miri) { 3 } else { 100 };\n let steps: usize = if cfg!(miri) { 50 } else { 10_000 };\n let additional: usize = if cfg!(miri) { 10 } else { 50 };\n\n static DROPS: AtomicUsize = AtomicUsize::new(0);\n\n #[derive(Debug, PartialEq)]\n struct DropCounter;\n\n impl Drop for DropCounter {\n fn drop(&mut self) {\n DROPS.fetch_add(1, Ordering::SeqCst);\n }\n }\n\n let mut rng = fastrand::Rng::new();\n\n for _ in 0..runs {\n let steps = rng.usize(0..steps);\n let additional = rng.usize(0..additional);\n\n DROPS.store(0, Ordering::SeqCst);\n let q = ArrayQueue::new(50);\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..steps {\n while q.pop().is_none() {}\n }\n });\n\n scope.spawn(|_| {\n for _ in 0..steps {\n while q.push(DropCounter).is_err() {\n DROPS.fetch_sub(1, Ordering::SeqCst);\n }\n }\n });\n })\n .unwrap();\n\n for _ in 0..additional {\n q.push(DropCounter).unwrap();\n }\n\n assert_eq!(DROPS.load(Ordering::SeqCst), steps);\n drop(q);\n assert_eq!(DROPS.load(Ordering::SeqCst), steps + additional);\n }\n}\n\n#[test]\nfn linearizable() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 25_000 };\n const THREADS: usize = 4;\n\n let q = ArrayQueue::new(THREADS);\n\n scope(|scope| {\n for _ in 0..THREADS / 2 {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n while q.push(0).is_err() {}\n q.pop().unwrap();\n }\n });\n\n scope.spawn(|_| {\n for _ in 0..COUNT {\n if q.force_push(0).is_none() {\n q.pop().unwrap();\n }\n }\n });\n }\n })\n .unwrap();\n}\n\n#[test]\nfn into_iter() {\n let q = ArrayQueue::new(100);\n for i in 0..100 {\n q.push(i).unwrap();\n }\n for (i, j) in q.into_iter().enumerate() {\n assert_eq!(i, j);\n }\n}\n" }, { "path": "crossbeam-queue/tests/seg_queue.rs", "content": "use std::sync::atomic::{AtomicUsize, Ordering};\n\nuse crossbeam_queue::SegQueue;\nuse crossbeam_utils::thread::scope;\n\n#[test]\nfn smoke() {\n let q = SegQueue::new();\n q.push(7);\n assert_eq!(q.pop(), Some(7));\n\n q.push(8);\n assert_eq!(q.pop(), Some(8));\n assert!(q.pop().is_none());\n}\n\n#[test]\nfn len_empty_full() {\n let q = SegQueue::new();\n\n assert_eq!(q.len(), 0);\n assert!(q.is_empty());\n\n q.push(());\n\n assert_eq!(q.len(), 1);\n assert!(!q.is_empty());\n\n q.pop().unwrap();\n\n assert_eq!(q.len(), 0);\n assert!(q.is_empty());\n}\n\n#[test]\nfn len() {\n let q = SegQueue::new();\n\n assert_eq!(q.len(), 0);\n\n for i in 0..50 {\n q.push(i);\n assert_eq!(q.len(), i + 1);\n }\n\n for i in 0..50 {\n q.pop().unwrap();\n assert_eq!(q.len(), 50 - i - 1);\n }\n\n assert_eq!(q.len(), 0);\n}\n\n#[test]\nfn exclusive_reference() {\n let mut q = SegQueue::new();\n\n assert_eq!(q.len(), 0);\n\n for i in 0..50 {\n q.push_mut(i);\n assert_eq!(q.len(), i + 1);\n }\n\n for i in 0..50 {\n q.pop_mut().unwrap();\n assert_eq!(q.len(), 50 - i - 1);\n }\n\n assert_eq!(q.len(), 0);\n assert!(q.is_empty());\n\n for i in 0..35 {\n q.push(i);\n assert_eq!(q.len(), i + 1);\n }\n\n for i in 0..5 {\n q.push_mut(i);\n assert_eq!(q.len(), 35 + i + 1);\n }\n\n for i in 0..5 {\n q.pop_mut().unwrap();\n assert_eq!(q.len(), 40 - i - 1);\n }\n\n for i in 0..35 {\n q.pop().unwrap();\n assert_eq!(q.len(), 35 - i - 1);\n }\n\n assert_eq!(q.len(), 0);\n assert!(q.is_empty());\n\n q.push_mut(1);\n\n assert!(!q.is_empty());\n}\n\n#[test]\nfn spsc() {\n const COUNT: usize = if cfg!(miri) { 100 } else { 100_000 };\n\n let q = SegQueue::new();\n\n scope(|scope| {\n scope.spawn(|_| {\n for i in 0..COUNT {\n loop {\n if let Some(x) = q.pop() {\n assert_eq!(x, i);\n break;\n }\n }\n }\n assert!(q.pop().is_none());\n });\n scope.spawn(|_| {\n for i in 0..COUNT {\n q.push(i);\n }\n });\n })\n .unwrap();\n}\n\n#[test]\nfn mpmc() {\n const COUNT: usize = if cfg!(miri) { 50 } else { 25_000 };\n const THREADS: usize = 4;\n\n let q = SegQueue::::new();\n let v = (0..COUNT).map(|_| AtomicUsize::new(0)).collect::>();\n\n scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for _ in 0..COUNT {\n let n = loop {\n if let Some(x) = q.pop() {\n break x;\n }\n };\n v[n].fetch_add(1, Ordering::SeqCst);\n }\n });\n }\n for _ in 0..THREADS {\n scope.spawn(|_| {\n for i in 0..COUNT {\n q.push(i);\n }\n });\n }\n })\n .unwrap();\n\n for c in v {\n assert_eq!(c.load(Ordering::SeqCst), THREADS);\n }\n}\n\n#[test]\nfn drops() {\n let runs: usize = if cfg!(miri) { 5 } else { 100 };\n let steps: usize = if cfg!(miri) { 50 } else { 10_000 };\n let additional: usize = if cfg!(miri) { 100 } else { 1_000 };\n\n static DROPS: AtomicUsize = AtomicUsize::new(0);\n\n #[derive(Debug, PartialEq)]\n struct DropCounter;\n\n impl Drop for DropCounter {\n fn drop(&mut self) {\n DROPS.fetch_add(1, Ordering::SeqCst);\n }\n }\n\n let mut rng = fastrand::Rng::new();\n\n for _ in 0..runs {\n let steps = rng.usize(0..steps);\n let additional = rng.usize(0..additional);\n\n DROPS.store(0, Ordering::SeqCst);\n let q = SegQueue::new();\n\n scope(|scope| {\n scope.spawn(|_| {\n for _ in 0..steps {\n while q.pop().is_none() {}\n }\n });\n\n scope.spawn(|_| {\n for _ in 0..steps {\n q.push(DropCounter);\n }\n });\n })\n .unwrap();\n\n for _ in 0..additional {\n q.push(DropCounter);\n }\n\n assert_eq!(DROPS.load(Ordering::SeqCst), steps);\n drop(q);\n assert_eq!(DROPS.load(Ordering::SeqCst), steps + additional);\n }\n}\n\n#[test]\nfn into_iter() {\n let q = SegQueue::new();\n for i in 0..100 {\n q.push(i);\n }\n for (i, j) in q.into_iter().enumerate() {\n assert_eq!(i, j);\n }\n}\n\n#[test]\nfn into_iter_drop() {\n let q = SegQueue::new();\n for i in 0..100 {\n q.push(i);\n }\n for (i, j) in q.into_iter().enumerate().take(50) {\n assert_eq!(i, j);\n }\n}\n\n// If `Block` is created on the stack, the array of slots will multiply this `BigStruct` and\n// probably overflow the thread stack. It's now directly created on the heap to avoid this.\n#[test]\nfn stack_overflow() {\n const N: usize = 32_768;\n struct BigStruct {\n _data: [u8; N],\n }\n\n let q = SegQueue::new();\n q.push(BigStruct { _data: [0u8; N] });\n\n for _data in q.into_iter() {}\n}\n" }, { "path": "crossbeam-skiplist/CHANGELOG.md", "content": "# Version 0.1.3\n\n- Remove dependency on `cfg-if`. (#1072)\n\n# Version 0.1.2\n\n- Bump the minimum supported Rust version to 1.61. (#1037)\n- Add `compare_insert`. (#976)\n- Improve support for targets without atomic CAS. (#1037)\n- Remove build script. (#1037)\n- Remove dependency on `scopeguard`. (#1045)\n\n# Version 0.1.1\n\n- Fix `get_unchecked` panic by raw pointer calculation. (#940)\n\n# Version 0.1.0\n\n**Note:** This release has been yanked due to bug fixed in 0.1.1.\n\n- Initial implementation.\n" }, { "path": "crossbeam-skiplist/Cargo.toml", "content": "[package]\nname = \"crossbeam-skiplist\"\n# When publishing a new version:\n# - Update CHANGELOG.md\n# - Update README.md (when increasing major or minor version)\n# - Run './tools/publish.sh crossbeam-skiplist '\nversion = \"0.1.3\"\nedition = \"2021\"\n# NB: Sync with msrv badge and \"Compatibility\" section in README.md\nrust-version = \"1.74\"\nlicense = \"MIT OR Apache-2.0\"\nrepository = \"https://github.com/crossbeam-rs/crossbeam\"\nhomepage = \"https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-skiplist\"\ndescription = \"A concurrent skip list\"\nkeywords = [\"map\", \"set\", \"skiplist\", \"lock-free\"]\ncategories = [\"algorithms\", \"concurrency\", \"data-structures\", \"no-std\"]\n\n[package.metadata.docs.rs]\ntargets = [\"x86_64-unknown-linux-gnu\"]\n\n[package.metadata.cargo_check_external_types]\n# The following are external types that are allowed to be exposed in our public API.\nallowed_external_types = [\n \"crossbeam_epoch::*\",\n]\n\n[features]\ndefault = [\"std\"]\n\n# Enable to use APIs that require `std`.\n# This is enabled by default.\nstd = [\"alloc\", \"crossbeam-epoch/std\", \"crossbeam-utils/std\"]\n\n# Enable to use APIs that require `alloc`.\n# This is enabled by default and also enabled if the `std` feature is enabled.\n#\n# NOTE: Disabling both `std` *and* `alloc` features is not supported yet.\nalloc = [\"crossbeam-epoch/alloc\"]\n\n[dependencies]\ncrossbeam-epoch = { version = \"0.9.17\", path = \"../crossbeam-epoch\", default-features = false }\ncrossbeam-utils = { version = \"0.8.18\", path = \"../crossbeam-utils\", default-features = false }\n\n[dev-dependencies]\nfastrand = \"2\"\n\n[lints]\nworkspace = true\n" }, { "path": "crossbeam-skiplist/LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "crossbeam-skiplist/LICENSE-MIT", "content": "The MIT License (MIT)\n\nCopyright (c) 2019 The Crossbeam Project Developers\n\nPermission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "crossbeam-skiplist/README.md", "content": "# Crossbeam Skiplist\n\n[![Build Status](https://github.com/crossbeam-rs/crossbeam/workflows/CI/badge.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/actions)\n[![License](https://img.shields.io/badge/license-MIT_OR_Apache--2.0-blue.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-skiplist#license)\n[![Cargo](https://img.shields.io/crates/v/crossbeam-skiplist.svg)](\nhttps://crates.io/crates/crossbeam-skiplist)\n[![Documentation](https://docs.rs/crossbeam-skiplist/badge.svg)](\nhttps://docs.rs/crossbeam-skiplist)\n[![Rust 1.74+](https://img.shields.io/badge/rust-1.74+-lightgray.svg)](\nhttps://www.rust-lang.org)\n[![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ)\n\nThis crate provides the types [`SkipMap`] and [`SkipSet`].\nThese data structures provide an interface similar to `BTreeMap` and `BTreeSet`,\nrespectively, except they support safe concurrent access across multiple threads.\n\nThis crate can be used in `no_std` environments that implement `alloc`. The `alloc` feature of this crate needs to be enabled in `no_std` environments.\n\n[`SkipMap`]: https://docs.rs/crossbeam-skiplist/latest/crossbeam_skiplist/struct.SkipMap.html\n[`SkipSet`]: https://docs.rs/crossbeam-skiplist/latest/crossbeam_skiplist/struct.SkipSet.html\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\ncrossbeam-skiplist = \"0.1\"\n```\n\n## Compatibility\n\nCrossbeam Skiplist supports stable Rust releases going back at least six months,\nand every time the minimum supported Rust version is increased, a new minor\nversion is released. Currently, the minimum supported Rust version is 1.74.\n\n## License\n\nLicensed under either of\n\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n#### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "path": "crossbeam-skiplist/benches/btree.rs", "content": "#![feature(test)]\n\nextern crate test;\n\nuse std::collections::BTreeMap as Map;\n\nuse test::{Bencher, black_box};\n\n#[bench]\nfn insert(b: &mut Bencher) {\n b.iter(|| {\n let mut map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n });\n}\n\n#[bench]\nfn iter(b: &mut Bencher) {\n let mut map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n b.iter(|| {\n for x in map.iter() {\n black_box(x);\n }\n });\n}\n\n#[bench]\nfn rev_iter(b: &mut Bencher) {\n let mut map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n b.iter(|| {\n for x in map.iter().rev() {\n black_box(x);\n }\n });\n}\n\n#[bench]\nfn lookup(b: &mut Bencher) {\n let mut map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n b.iter(|| {\n let mut num = 0u64;\n\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n black_box(map.get(&num));\n }\n });\n}\n\n#[bench]\nfn insert_remove(b: &mut Bencher) {\n b.iter(|| {\n let mut map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n black_box(map.remove(&num).unwrap());\n }\n });\n}\n" }, { "path": "crossbeam-skiplist/benches/hash.rs", "content": "#![feature(test)]\n\nextern crate test;\n\nuse std::collections::HashMap as Map;\n\nuse test::{Bencher, black_box};\n\n#[bench]\nfn insert(b: &mut Bencher) {\n b.iter(|| {\n let mut map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n });\n}\n\n#[bench]\nfn iter(b: &mut Bencher) {\n let mut map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n b.iter(|| {\n for x in map.iter() {\n black_box(x);\n }\n });\n}\n\n#[bench]\nfn lookup(b: &mut Bencher) {\n let mut map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n b.iter(|| {\n let mut num = 0u64;\n\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n black_box(map.get(&num));\n }\n });\n}\n\n#[bench]\nfn insert_remove(b: &mut Bencher) {\n b.iter(|| {\n let mut map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n black_box(map.remove(&num).unwrap());\n }\n });\n}\n" }, { "path": "crossbeam-skiplist/benches/skiplist.rs", "content": "#![feature(test)]\n#![allow(clippy::unit_arg)]\n\nextern crate test;\n\nuse crossbeam_epoch as epoch;\nuse crossbeam_skiplist::SkipList;\nuse test::{Bencher, black_box};\n\n#[bench]\nfn insert(b: &mut Bencher) {\n let guard = &epoch::pin();\n\n b.iter(|| {\n let map = SkipList::new(epoch::default_collector().clone());\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num, guard);\n }\n });\n}\n\n#[bench]\nfn iter(b: &mut Bencher) {\n let guard = &epoch::pin();\n let map = SkipList::new(epoch::default_collector().clone());\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num, guard).release(guard);\n }\n\n b.iter(|| {\n for x in map.iter(guard) {\n black_box(x.key());\n }\n });\n}\n\n#[bench]\nfn rev_iter(b: &mut Bencher) {\n let guard = &epoch::pin();\n let map = SkipList::new(epoch::default_collector().clone());\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num, guard).release(guard);\n }\n\n b.iter(|| {\n for x in map.iter(guard).rev() {\n black_box(x.key());\n }\n });\n}\n\n#[bench]\nfn lookup(b: &mut Bencher) {\n let guard = &epoch::pin();\n let map = SkipList::new(epoch::default_collector().clone());\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num, guard).release(guard);\n }\n\n b.iter(|| {\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n black_box(map.get(&num, guard));\n }\n });\n}\n\n#[bench]\nfn insert_remove(b: &mut Bencher) {\n let guard = &epoch::pin();\n\n b.iter(|| {\n let map = SkipList::new(epoch::default_collector().clone());\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num, guard).release(guard);\n }\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n black_box(map.remove(&num, guard).unwrap().release(guard));\n }\n });\n}\n" }, { "path": "crossbeam-skiplist/benches/skipmap.rs", "content": "#![feature(test)]\n\nextern crate test;\n\nuse crossbeam_skiplist::SkipMap as Map;\nuse test::{Bencher, black_box};\n\n#[bench]\nfn insert(b: &mut Bencher) {\n b.iter(|| {\n let map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n });\n}\n\n#[bench]\nfn iter(b: &mut Bencher) {\n let map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n b.iter(|| {\n for x in map.iter() {\n black_box(x);\n }\n });\n}\n\n#[bench]\nfn rev_iter(b: &mut Bencher) {\n let map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n b.iter(|| {\n for x in map.iter().rev() {\n black_box(x);\n }\n });\n}\n\n#[bench]\nfn lookup(b: &mut Bencher) {\n let map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n b.iter(|| {\n let mut num = 0u64;\n\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n black_box(map.get(&num));\n }\n });\n}\n\n#[bench]\nfn insert_remove(b: &mut Bencher) {\n b.iter(|| {\n let map = Map::new();\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n map.insert(num, !num);\n }\n\n let mut num = 0u64;\n for _ in 0..1_000 {\n num = num.wrapping_mul(17).wrapping_add(255);\n black_box(map.remove(&num).unwrap());\n }\n });\n}\n" }, { "path": "crossbeam-skiplist/examples/simple.rs", "content": "// use std::time::Instant;\n\nfn main() {\n // let map = crossbeam_skiplist::SkipMap::new();\n // // let mut map = std::collections::BTreeMap::new();\n // // let mut map = std::collections::HashMap::new();\n //\n // let now = Instant::now();\n //\n // let mut num = 0u64;\n // for _ in 0..1_000_000 {\n // num = num.wrapping_mul(17).wrapping_add(255);\n // map.insert(num, !num);\n // }\n //\n // let dur = Instant::now() - now;\n // println!(\"insert: {} sec\", dur.as_secs() as f64 + dur.subsec_nanos() as f64 * 1e-9);\n //\n // let now = Instant::now();\n //\n // for _ in map.iter() {}\n //\n // let dur = Instant::now() - now;\n // println!(\"iterate: {} sec\", dur.as_secs() as f64 + dur.subsec_nanos() as f64 * 1e-9);\n}\n" }, { "path": "crossbeam-skiplist/src/base.rs", "content": "//! A lock-free skip list. See [`SkipList`].\n\nuse alloc::alloc::handle_alloc_error;\nuse core::{\n alloc::Layout,\n cmp, fmt,\n marker::PhantomData,\n mem,\n ops::{Bound, Deref, RangeBounds},\n ptr,\n ptr::NonNull,\n sync::atomic::{AtomicUsize, Ordering, fence},\n};\n\nuse crossbeam_epoch::{self as epoch, Atomic, Collector, Guard, Shared};\nuse crossbeam_utils::CachePadded;\n\nuse crate::{\n alloc_helper::Global,\n comparator::{BasicComparator, Comparator},\n};\n\n/// Number of bits needed to store height.\nconst HEIGHT_BITS: usize = 5;\n\n/// Maximum height of a skip list tower.\nconst MAX_HEIGHT: usize = 1 << HEIGHT_BITS;\n\n/// The bits of `refs_and_height` that keep the height.\nconst HEIGHT_MASK: usize = (1 << HEIGHT_BITS) - 1;\n\n/// The tower of atomic pointers.\n///\n/// The actual size of the tower will vary depending on the height that a node\n/// was allocated with.\n#[repr(C)]\nstruct Tower {\n pointers: [Atomic>; 0],\n}\n\n/// A \"reference\" to a Tower that preserves provenance for accessing the dynamically sized tower.\n///\n/// A regular `&'a Tower` would not have permission to access any bytes under stacked borrows\n/// since Tower is a placeholder ZST.\n///\n/// Note, under tree borrows this isn't necessary.\nstruct TowerRef<'a, K, V> {\n ptr: NonNull>,\n _marker: PhantomData<&'a Tower>,\n}\n\nimpl Clone for TowerRef<'_, K, V> {\n fn clone(&self) -> Self {\n *self\n }\n}\nimpl Copy for TowerRef<'_, K, V> {}\n\nimpl<'a, K, V> TowerRef<'a, K, V> {\n /// Creates a TowerRef.\n ///\n /// # Safety\n ///\n /// Same as NonNull::as_ref, except the pointer must be valid for accessing the actual\n /// size of the tower.\n #[inline]\n unsafe fn new(ptr: NonNull>) -> Self {\n Self {\n ptr,\n _marker: PhantomData,\n }\n }\n\n /// Gets the atomic node pointer at the specified level of the tower.\n ///\n /// # Safety\n ///\n /// Index must be in bounds.\n #[inline]\n unsafe fn get_level(self, index: usize) -> &'a Atomic> {\n // SAFETY: Requirements passed to caller.\n unsafe { &*(self.ptr.as_ptr() as *const Atomic>).add(index) }\n }\n}\n\n/// Tower at the head of a skip list.\n///\n/// This is located in the `SkipList` struct itself and holds a full height\n/// tower.\n#[repr(C)]\nstruct Head {\n pointers: [Atomic>; MAX_HEIGHT],\n}\n\nimpl Head {\n /// Initializes a `Head`.\n #[inline]\n fn new() -> Self {\n // Initializing arrays in rust is a pain...\n Self {\n pointers: Default::default(),\n }\n }\n\n /// Gets `TowerRef`\n #[inline]\n fn as_tower(&self) -> TowerRef<'_, K, V> {\n unsafe { TowerRef::new(NonNull::from(self).cast::>()) }\n }\n\n /// Gets the atomic node pointer at the specified level.\n ///\n /// # Safety\n ///\n /// Index must be in bounds.\n #[inline]\n unsafe fn get_level(&self, index: usize) -> &Atomic> {\n // SAFETY: Requirements passed to caller.\n unsafe { self.pointers.get_unchecked(index) }\n }\n}\n\n/// A skip list node.\n///\n/// This struct is marked with `repr(C)` so that the specific order of fields is enforced.\n/// It is important that the tower is the last field since it is dynamically sized. The key,\n/// reference count, and height are kept close to the tower to improve cache locality during\n/// skip list traversal.\n#[repr(C)]\nstruct Node {\n /// The value.\n value: V,\n\n /// The key.\n key: K,\n\n /// Keeps the reference count and the height of its tower.\n ///\n /// The reference count is equal to the number of `Entry`s pointing to this node, plus the\n /// number of levels in which this node is installed.\n refs_and_height: AtomicUsize,\n\n /// The tower of atomic pointers.\n tower: Tower,\n}\n\n/// A \"reference\" to a Node that preserves provenance for accessing the dynamically sized tower at\n/// the end of the node allocation.\n///\n/// Note, in a few situations below, we also rely on this for preserving write permissions.\nstruct NodeRef<'a, K, V> {\n ptr: NonNull>,\n _marker: PhantomData<&'a Node>,\n}\n\nimpl Clone for NodeRef<'_, K, V> {\n fn clone(&self) -> Self {\n *self\n }\n}\nimpl Copy for NodeRef<'_, K, V> {}\n\nimpl Node {\n /// Allocates a node.\n ///\n /// The returned node will start with reference count of `ref_count` and the tower will be initialized\n /// with null pointers. However, the key and the value will be left uninitialized, and that is\n /// why this function is unsafe.\n unsafe fn alloc(height: usize, ref_count: usize) -> *mut Self {\n let layout = Self::get_layout(height);\n unsafe {\n let ptr = match Global.allocate(layout) {\n Some(ptr) => ptr.as_ptr().cast::(),\n None => handle_alloc_error(layout),\n };\n\n ptr::addr_of_mut!((*ptr).refs_and_height)\n .write(AtomicUsize::new((height - 1) | (ref_count << HEIGHT_BITS)));\n ptr::addr_of_mut!((*ptr).tower.pointers)\n .cast::>()\n .write_bytes(0, height);\n ptr\n }\n }\n\n /// Deallocates a node.\n ///\n /// This function will not run any destructors.\n unsafe fn dealloc(ptr: *mut Self) {\n unsafe {\n let height = (*ptr).height();\n let layout = Self::get_layout(height);\n Global.deallocate(NonNull::new_unchecked(ptr.cast::()), layout);\n }\n }\n\n /// Returns the layout of a node with the given `height`.\n fn get_layout(height: usize) -> Layout {\n assert!((1..=MAX_HEIGHT).contains(&height));\n\n Layout::new::()\n .extend(Layout::array::>(height).unwrap())\n .unwrap()\n .0\n .pad_to_align()\n }\n\n /// Returns the height of this node's tower.\n #[inline]\n fn height(&self) -> usize {\n (self.refs_and_height.load(Ordering::Relaxed) & HEIGHT_MASK) + 1\n }\n\n /// Attempts to increment the reference count of a node and returns `true` on success.\n ///\n /// The reference count can be incremented only if it is non-zero.\n ///\n /// # Panics\n ///\n /// Panics if the reference count overflows.\n #[inline]\n unsafe fn try_increment(&self) -> bool {\n let mut refs_and_height = self.refs_and_height.load(Ordering::Relaxed);\n\n loop {\n // If the reference count is zero, then the node has already been\n // queued for deletion. Incrementing it again could lead to a\n // double-free.\n if refs_and_height & !HEIGHT_MASK == 0 {\n return false;\n }\n\n // If all bits in the reference count are ones, we're about to overflow it.\n let new_refs_and_height = refs_and_height\n .checked_add(1 << HEIGHT_BITS)\n .expect(\"SkipList reference count overflow\");\n\n // Try incrementing the count.\n match self.refs_and_height.compare_exchange_weak(\n refs_and_height,\n new_refs_and_height,\n Ordering::Relaxed,\n Ordering::Relaxed,\n ) {\n Ok(_) => return true,\n Err(current) => refs_and_height = current,\n }\n }\n }\n\n /// Drops the key and value of a node, then deallocates it.\n #[cold]\n unsafe fn finalize(ptr: *mut Self) {\n unsafe {\n // Call destructors: drop the key and the value.\n ptr::drop_in_place(&mut (*ptr).key);\n ptr::drop_in_place(&mut (*ptr).value);\n\n // Finally, deallocate the memory occupied by the node.\n Self::dealloc(ptr);\n }\n }\n}\n\nimpl<'a, K, V> NodeRef<'a, K, V> {\n /// Creates a NodeRef.\n ///\n /// # Safety\n ///\n /// Same as NonNull::as_ref, except the pointer must also be valid for accessing the actual\n /// size of the tower at the end of the node.\n #[inline]\n unsafe fn new(ptr: NonNull>) -> Self {\n Self {\n ptr,\n _marker: PhantomData,\n }\n }\n\n /// # Safety\n ///\n /// See [`Shared::as_ref`].\n #[inline]\n unsafe fn from_shared(shared: Shared<'a, Node>) -> Option {\n if let Some(ptr) = NonNull::new(shared.as_raw() as *mut Node) {\n Some(unsafe { Self::new(ptr) })\n } else {\n None\n }\n }\n\n /// Decrements the reference count of a node, destroying it if the count becomes zero.\n #[inline]\n unsafe fn decrement(self, guard: &Guard) {\n if self\n .refs_and_height\n .fetch_sub(1 << HEIGHT_BITS, Ordering::Release)\n >> HEIGHT_BITS\n == 1\n {\n fence(Ordering::Acquire);\n unsafe { guard.defer_unchecked(move || Node::finalize(self.ptr.as_ptr())) }\n }\n }\n\n /// Decrements the reference count of a node, pinning the thread and destroying the node\n /// if the count become zero.\n #[inline]\n unsafe fn decrement_with_pin(self, parent: &SkipList, pin: F)\n where\n F: FnOnce() -> Guard,\n {\n if self\n .refs_and_height\n .fetch_sub(1 << HEIGHT_BITS, Ordering::Release)\n >> HEIGHT_BITS\n == 1\n {\n fence(Ordering::Acquire);\n let guard = &pin();\n parent.check_guard(guard);\n unsafe { guard.defer_unchecked(move || Node::finalize(self.ptr.as_ptr())) }\n }\n }\n\n /// Marks all pointers in the tower and returns `true` if the level 0 was not marked.\n fn mark_tower(self) -> bool {\n let height = self.height();\n\n for level in (0..height).rev() {\n let tag = unsafe {\n // We're loading the pointer only for the tag, so it's okay to use\n // `epoch::unprotected()` in this situation.\n // TODO(Amanieu): can we use release ordering here?\n self.get_level(level)\n .fetch_or(1, Ordering::SeqCst, epoch::unprotected())\n .tag()\n };\n\n // If the level 0 pointer was already marked, somebody else removed the node.\n if level == 0 && tag == 1 {\n return false;\n }\n }\n\n // We marked the level 0 pointer, therefore we removed the node.\n true\n }\n\n /// Returns `true` if the node is removed.\n #[inline]\n fn is_removed(self) -> bool {\n let tag = unsafe {\n // We're loading the pointer only for the tag, so it's okay to use\n // `epoch::unprotected()` in this situation.\n self.get_level(0)\n .load(Ordering::Relaxed, epoch::unprotected())\n .tag()\n };\n tag == 1\n }\n\n /// Creates a TowerRef to the atomic pointers at the end of this Node allocation.\n #[inline]\n fn as_tower(self) -> TowerRef<'a, K, V> {\n // SAFETY: self.ptr has provenance to access the tower.\n unsafe {\n TowerRef::new(NonNull::new_unchecked(\n ptr::addr_of!((*self.ptr.as_ptr()).tower) as *mut Tower,\n ))\n }\n }\n\n /// Gets a plain reference to the node which can be used for anything that doesn't need to access\n /// the tower.\n #[inline]\n fn as_ref(self) -> &'a Node {\n // SAFETY: Self::new requires the conditions for creating a Node reference.\n unsafe { self.ptr.as_ref() }\n }\n\n /// Gets the atomic node pointer at the specified level of the tower.\n ///\n /// # Safety\n ///\n /// Index must be in bounds.\n #[inline]\n unsafe fn get_level(&self, index: usize) -> &Atomic> {\n // SAFETY: Requirements passed to caller.\n unsafe { self.as_tower().get_level(index) }\n }\n}\n\nimpl Deref for NodeRef<'_, K, V> {\n type Target = Node;\n #[inline]\n fn deref(&self) -> &Node {\n self.as_ref()\n }\n}\n\nimpl fmt::Debug for Node\nwhere\n K: fmt::Debug,\n V: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_tuple(\"Node\")\n .field(&self.key)\n .field(&self.value)\n .finish()\n }\n}\n\nimpl fmt::Debug for NodeRef<'_, K, V>\nwhere\n K: fmt::Debug,\n V: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n as fmt::Debug>::fmt(self, f)\n }\n}\n\n/// A search result.\n///\n/// The result indicates whether the key was found, as well as what were the adjacent nodes to the\n/// key on each level of the skip list.\nstruct Position<'a, K, V> {\n /// Reference to a node with the given key, if found.\n ///\n /// If this is `Some` then it will point to the same node as `right[0]`.\n found: Option>,\n\n /// Adjacent nodes with smaller keys (predecessors).\n left: [TowerRef<'a, K, V>; MAX_HEIGHT],\n\n /// Adjacent nodes with equal or greater keys (successors).\n right: [Shared<'a, Node>; MAX_HEIGHT],\n}\n\n/// Frequently modified data associated with a skip list.\nstruct HotData {\n /// The seed for random height generation.\n seed: AtomicUsize,\n\n /// The number of entries in the skip list.\n len: AtomicUsize,\n\n /// Highest tower currently in use. This value is used as a hint for where\n /// to start lookups and never decreases.\n max_height: AtomicUsize,\n}\n\n/// A lock-free skip list.\n// TODO(stjepang): Embed a custom `epoch::Collector` inside `SkipList`. Instead of adding\n// garbage to the default global collector, we should add it to a local collector tied to the\n// particular skip list instance.\n//\n// Since global collector might destroy garbage arbitrarily late in the future, some skip list\n// methods have `K: 'static` and `V: 'static` bounds. But a local collector embedded in the skip\n// list would destroy all remaining garbage when the skip list is dropped, so in that case we'd be\n// able to remove those bounds on types `K` and `V`.\n//\n// As a further future optimization, if `!mem::needs_drop::() && !mem::needs_drop::()`\n// (neither key nor the value have destructors), there's no point in creating a new local\n// collector, so we should simply use the global one.\npub struct SkipList {\n /// The head of the skip list (just a dummy node, not a real entry).\n head: Head,\n\n /// The `Collector` associated with this skip list.\n collector: Collector,\n\n /// Hot data associated with the skip list, stored in a dedicated cache line.\n hot_data: CachePadded,\n\n /// The `Comparator` used to determine key ordering.\n comparator: C,\n}\n\nunsafe impl Send for SkipList {}\nunsafe impl Sync for SkipList {}\n\nimpl SkipList {\n /// Returns a new, empty skip list.\n pub fn new(collector: Collector) -> Self {\n Self::with_comparator(collector, Default::default())\n }\n}\n\nimpl SkipList {\n /// Returns a new, empty skip list using the given comparator.\n pub fn with_comparator(collector: Collector, comparator: C) -> Self {\n Self {\n head: Head::new(),\n collector,\n hot_data: CachePadded::new(HotData {\n seed: AtomicUsize::new(1),\n len: AtomicUsize::new(0),\n max_height: AtomicUsize::new(1),\n }),\n comparator,\n }\n }\n\n /// Returns `true` if the skip list is empty.\n pub fn is_empty(&self) -> bool {\n self.len() == 0\n }\n\n /// Returns the number of entries in the skip list.\n ///\n /// If the skip list is being concurrently modified, consider the returned number just an\n /// approximation without any guarantees.\n pub fn len(&self) -> usize {\n let len = self.hot_data.len.load(Ordering::Relaxed);\n\n // Due to the relaxed memory ordering, the length counter may sometimes\n // underflow and produce a very large value. We treat such values as 0.\n if len > isize::MAX as usize { 0 } else { len }\n }\n\n /// Ensures that all `Guard`s used with the skip list come from the same\n /// `Collector`.\n fn check_guard(&self, guard: &Guard) {\n if let Some(c) = guard.collector() {\n assert!(c == &self.collector);\n }\n }\n}\n\nimpl SkipList\nwhere\n C: Comparator,\n{\n /// Returns the entry with the smallest key.\n pub fn front<'a: 'g, 'g>(&'a self, guard: &'g Guard) -> Option> {\n self.check_guard(guard);\n let n = self.next_node(self.head.as_tower(), Bound::Unbounded, guard)?;\n Some(Entry {\n parent: self,\n node: n,\n guard,\n })\n }\n\n /// Returns the entry with the largest key.\n pub fn back<'a: 'g, 'g>(&'a self, guard: &'g Guard) -> Option> {\n self.check_guard(guard);\n let n = self.search_bound::(Bound::Unbounded, true, guard)?;\n Some(Entry {\n parent: self,\n node: n,\n guard,\n })\n }\n\n /// Returns `true` if the map contains a value for the specified key.\n pub fn contains_key(&self, key: &Q, guard: &Guard) -> bool\n where\n C: Comparator,\n Q: ?Sized,\n {\n self.get(key, guard).is_some()\n }\n\n /// Returns an entry with the specified `key`.\n pub fn get<'a: 'g, 'g, Q>(&'a self, key: &Q, guard: &'g Guard) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n self.check_guard(guard);\n let n = self.search_bound(Bound::Included(key), false, guard)?;\n if !self.comparator.equivalent(&n.key, key) {\n return None;\n }\n\n Some(Entry {\n parent: self,\n node: n,\n guard,\n })\n }\n\n /// Returns an `Entry` pointing to the lowest element whose key is above\n /// the given bound. If no such element is found then `None` is\n /// returned.\n pub fn lower_bound<'a: 'g, 'g, Q>(\n &'a self,\n bound: Bound<&Q>,\n guard: &'g Guard,\n ) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n self.check_guard(guard);\n let n = self.search_bound(bound, false, guard)?;\n Some(Entry {\n parent: self,\n node: n,\n guard,\n })\n }\n\n /// Returns an `Entry` pointing to the highest element whose key is below\n /// the given bound. If no such element is found then `None` is\n /// returned.\n pub fn upper_bound<'a: 'g, 'g, Q>(\n &'a self,\n bound: Bound<&Q>,\n guard: &'g Guard,\n ) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n self.check_guard(guard);\n let n = self.search_bound(bound, true, guard)?;\n Some(Entry {\n parent: self,\n node: n,\n guard,\n })\n }\n\n /// Finds an entry with the specified key, or inserts a new `key`-`value` pair if none exist.\n pub fn get_or_insert(&self, key: K, value: V, guard: &Guard) -> RefEntry<'_, K, V, C> {\n self.insert_internal(key, || value, |_| false, guard)\n }\n\n /// Finds an entry with the specified key, or inserts a new `key`-`value` pair if none exist,\n /// where value is calculated with a function.\n ///\n /// Note: Another thread may write key value first, leading to the result of this closure\n /// discarded. If closure is modifying some other state (such as shared counters or shared\n /// objects), it may lead to undesired behaviour such as counters being changed without\n /// result of closure inserted\n pub fn get_or_insert_with(&self, key: K, value: F, guard: &Guard) -> RefEntry<'_, K, V, C>\n where\n F: FnOnce() -> V,\n {\n self.insert_internal(key, value, |_| false, guard)\n }\n\n /// Returns an iterator over all entries in the skip list.\n pub fn iter<'a: 'g, 'g>(&'a self, guard: &'g Guard) -> Iter<'a, 'g, K, V, C> {\n self.check_guard(guard);\n Iter {\n parent: self,\n head: None,\n tail: None,\n guard,\n }\n }\n\n /// Returns an iterator over all entries in the skip list.\n pub fn ref_iter(&self) -> RefIter<'_, K, V, C> {\n RefIter {\n parent: self,\n head: None,\n tail: None,\n }\n }\n\n /// Returns an iterator over a subset of entries in the skip list.\n pub fn range<'a: 'g, 'g, Q, R>(\n &'a self,\n range: R,\n guard: &'g Guard,\n ) -> Range<'a, 'g, Q, R, K, V, C>\n where\n C: Comparator,\n R: RangeBounds,\n Q: ?Sized,\n {\n self.check_guard(guard);\n Range {\n parent: self,\n head: None,\n tail: None,\n range,\n guard,\n _marker: PhantomData,\n }\n }\n\n /// Returns an iterator over a subset of entries in the skip list.\n #[allow(clippy::needless_lifetimes)]\n pub fn ref_range<'a, Q, R>(&'a self, range: R) -> RefRange<'a, Q, R, K, V, C>\n where\n C: Comparator,\n R: RangeBounds,\n Q: ?Sized,\n {\n RefRange {\n parent: self,\n range,\n head: None,\n tail: None,\n _marker: PhantomData,\n }\n }\n\n /// Generates a random height and returns it.\n fn random_height(&self) -> usize {\n // Pseudorandom number generation from \"Xorshift RNGs\" by George Marsaglia.\n //\n // This particular set of operations generates 32-bit integers. See:\n // https://en.wikipedia.org/wiki/Xorshift#Example_implementation\n let mut num = self.hot_data.seed.load(Ordering::Relaxed);\n num ^= num << 13;\n num ^= num >> 17;\n num ^= num << 5;\n self.hot_data.seed.store(num, Ordering::Relaxed);\n\n let mut height = cmp::min(MAX_HEIGHT, num.trailing_zeros() as usize + 1);\n unsafe {\n // Keep decreasing the height while it's much larger than all towers currently in the\n // skip list.\n //\n // Note that we're loading the pointer only to check whether it is null, so it's okay\n // to use `epoch::unprotected()` in this situation.\n while height >= 4\n && self\n .head\n .get_level(height - 2)\n .load(Ordering::Relaxed, epoch::unprotected())\n .is_null()\n {\n height -= 1;\n }\n }\n\n // Track the max height to speed up lookups\n let mut max_height = self.hot_data.max_height.load(Ordering::Relaxed);\n while height > max_height {\n match self.hot_data.max_height.compare_exchange_weak(\n max_height,\n height,\n Ordering::Relaxed,\n Ordering::Relaxed,\n ) {\n Ok(_) => break,\n Err(h) => max_height = h,\n }\n }\n height\n }\n\n /// If we encounter a deleted node while searching, help with the deletion\n /// by attempting to unlink the node from the list.\n ///\n /// If the unlinking is successful then this function returns the next node\n /// with which the search should continue on the current level.\n #[cold]\n unsafe fn help_unlink<'a>(\n &'a self,\n pred: &'a Atomic>,\n curr: NodeRef<'a, K, V>,\n succ: Shared<'a, Node>,\n guard: &'a Guard,\n ) -> Option>> {\n // If `succ` is marked, that means `curr` is removed. Let's try\n // unlinking it from the skip list at this level.\n match pred.compare_exchange(\n Shared::from(curr.ptr.as_ptr() as *const Node),\n succ.with_tag(0),\n Ordering::Release,\n Ordering::Relaxed,\n guard,\n ) {\n Ok(_) => {\n unsafe { curr.decrement(guard) }\n Some(succ.with_tag(0))\n }\n Err(_) => None,\n }\n }\n\n /// Returns the successor of a node.\n ///\n /// This will keep searching until a non-deleted node is found. If a deleted\n /// node is reached then a search is performed using the given key.\n fn next_node<'a>(\n &'a self,\n pred: TowerRef<'a, K, V>,\n lower_bound: Bound<&K>,\n guard: &'a Guard,\n ) -> Option> {\n unsafe {\n // Load the level 0 successor of the current node.\n let mut curr = pred.get_level(0).load_consume(guard);\n\n // If `curr` is marked, that means `pred` is removed and we have to use\n // a key search.\n if curr.tag() == 1 {\n return self.search_bound(lower_bound, false, guard);\n }\n\n while let Some(c) = NodeRef::from_shared(curr) {\n let succ = c.get_level(0).load_consume(guard);\n\n if succ.tag() == 1 {\n if let Some(c) = self.help_unlink(pred.get_level(0), c, succ, guard) {\n // On success, continue searching through the current level.\n curr = c;\n continue;\n } else {\n // On failure, we cannot do anything reasonable to continue\n // searching from the current position. Restart the search.\n return self.search_bound(lower_bound, false, guard);\n }\n }\n\n return Some(c);\n }\n\n None\n }\n }\n\n /// Searches for first/last node that is greater/less/equal to a key in the skip list.\n ///\n /// If `upper_bound == true`: the last node less than (or equal to) the key.\n ///\n /// If `upper_bound == false`: the first node greater than (or equal to) the key.\n ///\n /// This is unsafe because the returned nodes are bound to the lifetime of\n /// the `SkipList`, not the `Guard`.\n fn search_bound<'a, Q>(\n &'a self,\n bound: Bound<&Q>,\n upper_bound: bool,\n guard: &'a Guard,\n ) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n unsafe {\n 'search: loop {\n // The current level we're at.\n let mut level = self.hot_data.max_height.load(Ordering::Relaxed);\n\n // Fast loop to skip empty tower levels.\n while level >= 1\n && self\n .head\n .get_level(level - 1)\n .load(Ordering::Relaxed, guard)\n .is_null()\n {\n level -= 1;\n }\n\n // The current best node\n let mut result = None;\n\n // The predecessor node\n let mut pred = self.head.as_tower();\n\n while level >= 1 {\n level -= 1;\n\n // Two adjacent nodes at the current level.\n let mut curr = pred.get_level(level).load_consume(guard);\n\n // If `curr` is marked, that means `pred` is removed and we have to restart the\n // search.\n if curr.tag() == 1 {\n continue 'search;\n }\n\n // Iterate through the current level until we reach a node with a key greater\n // than or equal to `key`.\n while let Some(c) = NodeRef::from_shared(curr) {\n let succ = c.get_level(level).load_consume(guard);\n\n if succ.tag() == 1 {\n if let Some(c) = self.help_unlink(pred.get_level(level), c, succ, guard)\n {\n // On success, continue searching through the current level.\n curr = c;\n continue;\n } else {\n // On failure, we cannot do anything reasonable to continue\n // searching from the current position. Restart the search.\n continue 'search;\n }\n }\n\n // If `curr` contains a key that is greater than (or equal) to `key`, we're\n // done with this level.\n //\n // The condition determines whether we should stop the search. For the upper\n // bound, we return the last node before the condition became true. For the\n // lower bound, we return the first node after the condition became true.\n if upper_bound {\n if !below_upper_bound(&self.comparator, &bound, &c.key) {\n break;\n }\n result = Some(c);\n } else if above_lower_bound(&self.comparator, &bound, &c.key) {\n result = Some(c);\n break;\n }\n\n // Move one step forward.\n pred = c.as_tower();\n curr = succ;\n }\n }\n\n return result;\n }\n }\n }\n\n /// Searches for a key in the skip list and returns a list of all adjacent nodes.\n fn search_position<'a, Q>(&'a self, key: &Q, guard: &'a Guard) -> Position<'a, K, V>\n where\n C: Comparator,\n Q: ?Sized,\n {\n unsafe {\n 'search: loop {\n // The result of this search.\n let mut result = Position {\n found: None,\n left: [self.head.as_tower(); MAX_HEIGHT],\n right: [Shared::null(); MAX_HEIGHT],\n };\n\n // The current level we're at.\n let mut level = self.hot_data.max_height.load(Ordering::Relaxed);\n\n // Fast loop to skip empty tower levels.\n while level >= 1\n && self\n .head\n .get_level(level - 1)\n .load(Ordering::Relaxed, guard)\n .is_null()\n {\n level -= 1;\n }\n\n // The predecessor node\n let mut pred = self.head.as_tower();\n\n while level >= 1 {\n level -= 1;\n\n // Two adjacent nodes at the current level.\n let mut curr = pred.get_level(level).load_consume(guard);\n\n // If `curr` is marked, that means `pred` is removed and we have to restart the\n // search.\n if curr.tag() == 1 {\n continue 'search;\n }\n\n // Iterate through the current level until we reach a node with a key greater\n // than or equal to `key`.\n while let Some(c) = NodeRef::from_shared(curr) {\n let succ = c.get_level(level).load_consume(guard);\n\n if succ.tag() == 1 {\n if let Some(c) = self.help_unlink(pred.get_level(level), c, succ, guard)\n {\n // On success, continue searching through the current level.\n curr = c;\n continue;\n } else {\n // On failure, we cannot do anything reasonable to continue\n // searching from the current position. Restart the search.\n continue 'search;\n }\n }\n\n // If `curr` contains a key that is greater than or equal to `key`, we're\n // done with this level.\n match self.comparator.compare(&c.key, key) {\n cmp::Ordering::Greater => break,\n cmp::Ordering::Equal => {\n result.found = Some(c);\n break;\n }\n cmp::Ordering::Less => {}\n }\n\n // Move one step forward.\n pred = c.as_tower();\n curr = succ;\n }\n\n // Store the position at the current level into the result.\n result.left[level] = pred;\n result.right[level] = curr;\n }\n\n return result;\n }\n }\n }\n\n /// Inserts an entry with the specified `key` and `value`.\n ///\n /// If `replace` is `true`, then any existing entry with this key will first be removed.\n fn insert_internal(\n &self,\n key: K,\n value: F,\n replace: CompareF,\n guard: &Guard,\n ) -> RefEntry<'_, K, V, C>\n where\n C: Comparator,\n F: FnOnce() -> V,\n CompareF: Fn(&V) -> bool,\n {\n self.check_guard(guard);\n\n unsafe {\n // Rebind the guard to the lifetime of self. This is a bit of a\n // hack but it allows us to return references that are not bound to\n // the lifetime of the guard.\n let guard = &*(guard as *const _);\n\n // First try searching for the key.\n // Note that the `Ord` implementation for `K` may panic during the search.\n let mut search = self.search_position(&key, guard);\n if let Some(r) = search.found {\n let replace = replace(&r.value);\n if !replace {\n // If a node with the key was found and we're not going to replace it, let's\n // try returning it as an entry.\n if let Some(e) = RefEntry::try_acquire(self, r) {\n return e;\n }\n }\n }\n\n // create value before creating node, so extra allocation doesn't happen if value() function panics\n let value = value();\n // Create a new node.\n let height = self.random_height();\n let (node, n) = {\n // The reference count is initially two to account for:\n // 1. The entry that will be returned.\n // 2. The link at the level 0 of the tower.\n let n = Node::::alloc(height, 2);\n\n // Write the key and the value into the node.\n ptr::addr_of_mut!((*n).key).write(key);\n ptr::addr_of_mut!((*n).value).write(value);\n\n (\n Shared::>::from(n as *const _),\n NodeRef::new(NonNull::new_unchecked(n)),\n )\n };\n\n // Optimistically increment `len`.\n self.hot_data.len.fetch_add(1, Ordering::Relaxed);\n\n loop {\n // Set the lowest successor of `n` to `search.right[0]`.\n n.get_level(0).store(search.right[0], Ordering::Relaxed);\n\n // Try installing the new node into the skip list (at level 0).\n // TODO(Amanieu): can we use release ordering here?\n if search.left[0]\n .get_level(0)\n .compare_exchange(\n search.right[0],\n node,\n Ordering::SeqCst,\n Ordering::SeqCst,\n guard,\n )\n .is_ok()\n {\n // This node has been abandoned\n if let Some(r) = search.found {\n if r.mark_tower() {\n self.hot_data.len.fetch_sub(1, Ordering::Relaxed);\n }\n }\n break;\n }\n\n // We failed. Let's search for the key and try again.\n {\n // Create a guard that destroys the new node in case search panics.\n struct ScopeGuard(*const Node);\n impl Drop for ScopeGuard {\n fn drop(&mut self) {\n unsafe { Node::finalize(self.0 as *mut Node) }\n }\n }\n let sg = ScopeGuard(node.as_raw());\n search = self.search_position(&n.key, guard);\n mem::forget(sg);\n }\n\n if let Some(r) = search.found {\n let replace = replace(&r.value);\n if !replace {\n // If a node with the key was found and we're not going to replace it,\n // let's try returning it as an entry.\n if let Some(e) = RefEntry::try_acquire(self, r) {\n // Destroy the new node.\n Node::finalize(node.as_raw() as *mut Node);\n self.hot_data.len.fetch_sub(1, Ordering::Relaxed);\n\n return e;\n }\n\n // If we couldn't increment the reference count, that means someone has\n // just now removed the node.\n }\n }\n }\n\n // The new node was successfully installed. Let's create an entry associated with it.\n let entry = RefEntry {\n parent: self,\n node: n,\n };\n\n // Build the rest of the tower above level 0.\n 'build: for level in 1..height {\n loop {\n // Obtain the predecessor and successor at the current level.\n let pred = search.left[level];\n let succ = search.right[level];\n\n // Load the current value of the pointer in the tower at this level.\n // TODO(Amanieu): can we use relaxed ordering here?\n let next = n.get_level(level).load(Ordering::SeqCst, guard);\n\n // If the current pointer is marked, that means another thread is already\n // removing the node we've just inserted. In that case, let's just stop\n // building the tower.\n if next.tag() == 1 {\n break 'build;\n }\n\n // When searching for `key` and traversing the skip list from the highest level\n // to the lowest, it is possible to observe a node with an equal key at higher\n // levels and then find it missing at the lower levels if it gets removed\n // during traversal. Even worse, it is possible to observe completely different\n // nodes with the exact same key at different levels.\n //\n // Linking the new node to a dead successor with an equal key could create\n // subtle corner cases that would require special care. It's much easier to\n // simply prohibit linking two nodes with equal keys.\n //\n // If the successor has the same key as the new node, that means it is marked\n // as removed and should be unlinked from the skip list. In that case, let's\n // repeat the search to make sure it gets unlinked and try again.\n //\n // If this comparison or the following search panics, we simply stop building\n // the tower without breaking any invariants. Note that building higher levels\n // is completely optional. Only the lowest level really matters, and all the\n // higher levels are there just to make searching faster.\n if succ\n .as_ref()\n .is_some_and(|s| self.comparator.equivalent(&s.key, &n.key))\n {\n search = self.search_position(&n.key, guard);\n continue;\n }\n\n // Change the pointer at the current level from `next` to `succ`. If this CAS\n // operation fails, that means another thread has marked the pointer and we\n // should stop building the tower.\n // TODO(Amanieu): can we use release ordering here?\n if n.get_level(level)\n .compare_exchange(next, succ, Ordering::SeqCst, Ordering::SeqCst, guard)\n .is_err()\n {\n break 'build;\n }\n\n // Increment the reference count. The current value will always be at least 1\n // because we are holding `entry`.\n n.refs_and_height\n .fetch_add(1 << HEIGHT_BITS, Ordering::Relaxed);\n\n // Try installing the new node at the current level.\n // TODO(Amanieu): can we use release ordering here?\n if pred\n .get_level(level)\n .compare_exchange(succ, node, Ordering::SeqCst, Ordering::SeqCst, guard)\n .is_ok()\n {\n // Success! Continue on the next level.\n break;\n }\n\n // Installation failed. Decrement the reference count.\n n.refs_and_height\n .fetch_sub(1 << HEIGHT_BITS, Ordering::Relaxed);\n\n // We don't have the most up-to-date search results. Repeat the search.\n //\n // If this search panics, we simply stop building the tower without breaking\n // any invariants. Note that building higher levels is completely optional.\n // Only the lowest level really matters, and all the higher levels are there\n // just to make searching faster.\n search = self.search_position(&n.key, guard);\n }\n }\n\n // If any pointer in the tower is marked, that means our node is in the process of\n // removal or already removed. It is possible that another thread (either partially or\n // completely) removed the new node while we were building the tower, and just after\n // that we installed the new node at one of the higher levels. In order to undo that\n // installation, we must repeat the search, which will unlink the new node at that\n // level.\n // TODO(Amanieu): can we use relaxed ordering here?\n if n.get_level(height - 1).load(Ordering::SeqCst, guard).tag() == 1 {\n self.search_bound(Bound::Included(&n.key), false, guard);\n }\n\n // Finally, return the new entry.\n entry\n }\n }\n}\n\nimpl SkipList\nwhere\n C: Comparator,\n K: Send + 'static,\n V: Send + 'static,\n{\n /// Inserts a `key`-`value` pair into the skip list and returns the new entry.\n ///\n /// If there is an existing entry with this key, it will be removed before inserting the new\n /// one.\n pub fn insert(&self, key: K, value: V, guard: &Guard) -> RefEntry<'_, K, V, C> {\n self.insert_internal(key, || value, |_| true, guard)\n }\n\n /// Inserts a `key`-`value` pair into the skip list and returns the new entry.\n ///\n /// If there is an existing entry with this key and compare(entry.value) returns true,\n /// it will be removed before inserting the new one.\n /// The closure will not be called if the key is not present.\n pub fn compare_insert(\n &self,\n key: K,\n value: V,\n compare_fn: F,\n guard: &Guard,\n ) -> RefEntry<'_, K, V, C>\n where\n F: Fn(&V) -> bool,\n {\n self.insert_internal(key, || value, compare_fn, guard)\n }\n\n /// Removes an entry with the specified `key` from the map and returns it.\n pub fn remove(&self, key: &Q, guard: &Guard) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n self.check_guard(guard);\n\n unsafe {\n // Rebind the guard to the lifetime of self. This is a bit of a\n // hack but it allows us to return references that are not bound to\n // the lifetime of the guard.\n let guard = &*(guard as *const _);\n\n loop {\n // Try searching for the key.\n let search = self.search_position(key, guard);\n\n let n = search.found?;\n\n // First try incrementing the reference count because we have to return the node as\n // an entry. If this fails, repeat the search.\n let entry = match RefEntry::try_acquire(self, n) {\n Some(e) => e,\n None => continue,\n };\n\n // Try removing the node by marking its tower.\n if n.mark_tower() {\n // Success! Decrement `len`.\n self.hot_data.len.fetch_sub(1, Ordering::Relaxed);\n\n // Unlink the node at each level of the skip list. We could do this by simply\n // repeating the search, but it's usually faster to unlink it manually using\n // the `left` and `right` lists.\n for level in (0..n.height()).rev() {\n // TODO(Amanieu): can we use relaxed ordering here?\n let succ = n.get_level(level).load(Ordering::SeqCst, guard).with_tag(0);\n\n // Try linking the predecessor and successor at this level.\n // TODO(Amanieu): can we use release ordering here?\n if search.left[level]\n .get_level(level)\n .compare_exchange(\n Shared::from(n.ptr.as_ptr() as *const Node),\n succ,\n Ordering::SeqCst,\n Ordering::SeqCst,\n guard,\n )\n .is_ok()\n {\n // Success! Decrement the reference count.\n n.decrement(guard);\n } else {\n // Failed! Just repeat the search to completely unlink the node.\n self.search_bound(Bound::Included(key), false, guard);\n break;\n }\n }\n return Some(entry);\n } else {\n // The node has already been marked.\n n.decrement(guard);\n return None;\n }\n }\n }\n }\n\n /// Removes an entry from the front of the skip list.\n pub fn pop_front(&self, guard: &Guard) -> Option> {\n self.check_guard(guard);\n loop {\n let e = self.front(guard)?;\n if let Some(e) = e.pin() {\n if e.remove(guard) {\n return Some(e);\n } else {\n e.release(guard);\n }\n }\n }\n }\n\n /// Removes an entry from the back of the skip list.\n pub fn pop_back(&self, guard: &Guard) -> Option> {\n self.check_guard(guard);\n loop {\n let e = self.back(guard)?;\n if let Some(e) = e.pin() {\n if e.remove(guard) {\n return Some(e);\n } else {\n e.release(guard);\n }\n }\n }\n }\n\n /// Iterates over the map and removes every entry.\n pub fn clear(&self, guard: &mut Guard) {\n self.check_guard(guard);\n\n /// Number of steps after which we repin the current thread and unlink removed nodes.\n const BATCH_SIZE: usize = 100;\n\n loop {\n {\n // Search for the first entry in order to unlink all the preceding entries\n // we have removed.\n //\n // By unlinking nodes in batches we make sure that the final search doesn't\n // unlink all nodes at once, which could keep the current thread pinned for a\n // long time.\n let mut entry = self.lower_bound::(Bound::Unbounded, guard);\n\n for _ in 0..BATCH_SIZE {\n // Stop if we have reached the end of the list.\n let e = match entry {\n None => return,\n Some(e) => e,\n };\n\n // Before removing the current entry, first obtain the following one.\n let next = e.next();\n\n // Try removing the current entry.\n if e.node.mark_tower() {\n // Success! Decrement `len`.\n self.hot_data.len.fetch_sub(1, Ordering::Relaxed);\n }\n\n entry = next;\n }\n }\n\n // Repin the current thread because we don't want to keep it pinned in the same\n // epoch for a too long time.\n guard.repin();\n }\n }\n}\n\nimpl Drop for SkipList {\n fn drop(&mut self) {\n unsafe {\n let mut node = NodeRef::from_shared(\n self.head\n .get_level(0)\n .load(Ordering::Relaxed, epoch::unprotected()),\n );\n\n // Iterate through the whole skip list and destroy every node.\n while let Some(n) = node {\n // Unprotected loads are okay because this function is the only one currently using\n // the skip list.\n let next = NodeRef::from_shared(\n n.get_level(0).load(Ordering::Relaxed, epoch::unprotected()),\n );\n\n // Deallocate every node.\n Node::finalize(n.ptr.as_ptr());\n\n node = next;\n }\n }\n }\n}\n\nimpl fmt::Debug for SkipList\nwhere\n K: fmt::Debug,\n V: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"SkipList { .. }\")\n }\n}\n\nimpl IntoIterator for SkipList {\n type Item = (K, V);\n type IntoIter = IntoIter;\n\n fn into_iter(self) -> Self::IntoIter {\n unsafe {\n // Load the front node.\n //\n // Unprotected loads are okay because this function is the only one currently using\n // the skip list.\n let front = self\n .head\n .get_level(0)\n .load(Ordering::Relaxed, epoch::unprotected())\n .as_raw();\n\n // Clear the skip list by setting all pointers in head to null.\n for level in 0..MAX_HEIGHT {\n self.head\n .get_level(level)\n .store(Shared::null(), Ordering::Relaxed);\n }\n\n IntoIter {\n node: front as *mut Node,\n }\n }\n }\n}\n\n/// An entry in a skip list, protected by a `Guard`.\n///\n/// The lifetimes of the key and value are the same as that of the `Guard`\n/// used when creating the `Entry` (`'g`). This lifetime is also constrained to\n/// not outlive the `SkipList`.\npub struct Entry<'a: 'g, 'g, K, V, C = BasicComparator> {\n parent: &'a SkipList,\n node: NodeRef<'g, K, V>,\n guard: &'g Guard,\n}\n\nimpl<'a: 'g, 'g, K: 'a, V: 'a, C> Entry<'a, 'g, K, V, C> {\n /// Returns `true` if the entry is removed from the skip list.\n pub fn is_removed(&self) -> bool {\n self.node.is_removed()\n }\n\n /// Returns a reference to the key.\n pub fn key(&self) -> &'g K {\n &self.node.as_ref().key\n }\n\n /// Returns a reference to the value.\n pub fn value(&self) -> &'g V {\n &self.node.as_ref().value\n }\n\n /// Returns a reference to the parent `SkipList`\n pub fn skiplist(&self) -> &'a SkipList {\n self.parent\n }\n\n /// Attempts to pin the entry with a reference count, ensuring that it\n /// remains accessible even after the `Guard` is dropped.\n ///\n /// This method may return `None` if the reference count is already 0 and\n /// the node has been queued for deletion.\n pub fn pin(&self) -> Option> {\n unsafe { RefEntry::try_acquire(self.parent, self.node) }\n }\n}\n\nimpl Entry<'_, '_, K, V, C>\nwhere\n C: Comparator,\n K: Send + 'static,\n V: Send + 'static,\n{\n /// Removes the entry from the skip list.\n ///\n /// Returns `true` if this call removed the entry and `false` if it was already removed.\n pub fn remove(&self) -> bool {\n // Try marking the tower.\n if self.node.mark_tower() {\n // Success - the entry is removed. Now decrement `len`.\n self.parent.hot_data.len.fetch_sub(1, Ordering::Relaxed);\n\n // Search for the key to unlink the node from the skip list.\n self.parent\n .search_bound(Bound::Included(&self.node.key), false, self.guard);\n\n true\n } else {\n false\n }\n }\n}\n\nimpl Clone for Entry<'_, '_, K, V, C> {\n fn clone(&self) -> Self {\n Self {\n parent: self.parent,\n node: self.node,\n guard: self.guard,\n }\n }\n}\n\nimpl fmt::Debug for Entry<'_, '_, K, V, C>\nwhere\n K: fmt::Debug,\n V: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_tuple(\"Entry\")\n .field(self.key())\n .field(self.value())\n .finish()\n }\n}\n\nimpl Entry<'_, '_, K, V, C>\nwhere\n C: Comparator,\n{\n /// Moves to the next entry in the skip list.\n pub fn move_next(&mut self) -> bool {\n match self.next() {\n None => false,\n Some(n) => {\n *self = n;\n true\n }\n }\n }\n\n /// Returns the next entry in the skip list.\n pub fn next(&self) -> Option {\n let n = self.parent.next_node(\n self.node.as_tower(),\n Bound::Excluded(&self.node.key),\n self.guard,\n )?;\n Some(Entry {\n parent: self.parent,\n node: n,\n guard: self.guard,\n })\n }\n\n /// Moves to the previous entry in the skip list.\n pub fn move_prev(&mut self) -> bool {\n match self.prev() {\n None => false,\n Some(n) => {\n *self = n;\n true\n }\n }\n }\n\n /// Returns the previous entry in the skip list.\n pub fn prev(&self) -> Option {\n let n = self\n .parent\n .search_bound(Bound::Excluded(&self.node.key), true, self.guard)?;\n Some(Entry {\n parent: self.parent,\n node: n,\n guard: self.guard,\n })\n }\n}\n\n/// A reference-counted entry in a skip list.\n///\n/// You *must* call `release` to free this type, otherwise the node will be\n/// leaked. This is because releasing the entry requires a `Guard`.\npub struct RefEntry<'a, K, V, C = BasicComparator> {\n parent: &'a SkipList,\n node: NodeRef<'a, K, V>,\n}\n\nimpl<'a, K: 'a, V: 'a, C: 'a> RefEntry<'a, K, V, C> {\n /// Returns `true` if the entry is removed from the skip list.\n pub fn is_removed(&self) -> bool {\n self.node.is_removed()\n }\n\n /// Returns a reference to the key.\n pub fn key(&self) -> &'a K {\n &self.node.as_ref().key\n }\n\n /// Returns a reference to the value.\n pub fn value(&self) -> &'a V {\n &self.node.as_ref().value\n }\n\n /// Returns a reference to the parent `SkipList`\n pub fn skiplist(&self) -> &'a SkipList {\n self.parent\n }\n\n /// Releases the reference on the entry.\n pub fn release(self, guard: &Guard) {\n self.parent.check_guard(guard);\n unsafe { self.node.decrement(guard) }\n }\n\n /// Releases the reference of the entry, pinning the thread only when\n /// the reference count of the node becomes 0.\n pub fn release_with_pin(self, pin: F)\n where\n F: FnOnce() -> Guard,\n {\n unsafe { self.node.decrement_with_pin(self.parent, pin) }\n }\n\n /// Tries to create a new `RefEntry` by incrementing the reference count of\n /// a node.\n unsafe fn try_acquire(parent: &'a SkipList, node: NodeRef<'_, K, V>) -> Option {\n if unsafe { node.try_increment() } {\n Some(RefEntry {\n parent,\n\n // We re-bind the lifetime of the node here to that of the skip\n // list since we now hold a reference to it.\n node: unsafe { NodeRef::new(node.ptr) },\n })\n } else {\n None\n }\n }\n}\n\nimpl RefEntry<'_, K, V, C>\nwhere\n C: Comparator,\n K: Send + 'static,\n V: Send + 'static,\n{\n /// Removes the entry from the skip list.\n ///\n /// Returns `true` if this call removed the entry and `false` if it was already removed.\n pub fn remove(&self, guard: &Guard) -> bool {\n self.parent.check_guard(guard);\n\n // Try marking the tower.\n if self.node.mark_tower() {\n // Success - the entry is removed. Now decrement `len`.\n self.parent.hot_data.len.fetch_sub(1, Ordering::Relaxed);\n\n // Search for the key to unlink the node from the skip list.\n self.parent\n .search_bound(Bound::Included(&self.node.key), false, guard);\n\n true\n } else {\n false\n }\n }\n}\n\nimpl Clone for RefEntry<'_, K, V, C> {\n fn clone(&self) -> Self {\n unsafe {\n // Incrementing will always succeed since we're already holding a reference to the node.\n Node::try_increment(&*self.node);\n }\n Self {\n parent: self.parent,\n node: self.node,\n }\n }\n}\n\nimpl fmt::Debug for RefEntry<'_, K, V, C>\nwhere\n K: fmt::Debug,\n V: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_tuple(\"RefEntry\")\n .field(self.key())\n .field(self.value())\n .finish()\n }\n}\n\nimpl RefEntry<'_, K, V, C>\nwhere\n C: Comparator,\n{\n /// Moves to the next entry in the skip list.\n pub fn move_next(&mut self, guard: &Guard) -> bool {\n match self.next(guard) {\n None => false,\n Some(e) => {\n mem::replace(self, e).release(guard);\n true\n }\n }\n }\n\n /// Returns the next entry in the skip list.\n pub fn next(&self, guard: &Guard) -> Option {\n self.parent.check_guard(guard);\n unsafe {\n let mut n = self.node;\n loop {\n n = self\n .parent\n .next_node(n.as_tower(), Bound::Excluded(&n.key), guard)?;\n if let Some(e) = RefEntry::try_acquire(self.parent, n) {\n return Some(e);\n }\n }\n }\n }\n\n /// Moves to the previous entry in the skip list.\n pub fn move_prev(&mut self, guard: &Guard) -> bool {\n match self.prev(guard) {\n None => false,\n Some(e) => {\n mem::replace(self, e).release(guard);\n true\n }\n }\n }\n\n /// Returns the previous entry in the skip list.\n pub fn prev(&self, guard: &Guard) -> Option {\n self.parent.check_guard(guard);\n unsafe {\n let mut n = self.node;\n loop {\n n = self\n .parent\n .search_bound(Bound::Excluded(&n.key), true, guard)?;\n if let Some(e) = RefEntry::try_acquire(self.parent, n) {\n return Some(e);\n }\n }\n }\n }\n}\n\n/// An iterator over the entries of a `SkipList`.\npub struct Iter<'a: 'g, 'g, K, V, C = BasicComparator> {\n parent: &'a SkipList,\n head: Option>,\n tail: Option>,\n guard: &'g Guard,\n}\n\nimpl<'a: 'g, 'g, K: 'a, V: 'a, C> Iterator for Iter<'a, 'g, K, V, C>\nwhere\n C: Comparator,\n{\n type Item = Entry<'a, 'g, K, V, C>;\n\n fn next(&mut self) -> Option {\n self.head = match self.head {\n Some(n) => self\n .parent\n .next_node(n.as_tower(), Bound::Excluded(&n.key), self.guard),\n None => {\n self.parent\n .next_node(self.parent.head.as_tower(), Bound::Unbounded, self.guard)\n }\n };\n if let (Some(h), Some(t)) = (self.head, self.tail) {\n if self.parent.comparator.compare(&h.key, &t.key).is_ge() {\n self.head = None;\n self.tail = None;\n }\n }\n self.head.map(|n| Entry {\n parent: self.parent,\n node: n,\n guard: self.guard,\n })\n }\n}\n\nimpl<'a: 'g, 'g, K: 'a, V: 'a, C> DoubleEndedIterator for Iter<'a, 'g, K, V, C>\nwhere\n C: Comparator,\n{\n fn next_back(&mut self) -> Option {\n self.tail = match self.tail {\n Some(n) => self\n .parent\n .search_bound(Bound::Excluded(&n.key), true, self.guard),\n None => self\n .parent\n .search_bound::(Bound::Unbounded, true, self.guard),\n };\n if let (Some(h), Some(t)) = (self.head, self.tail) {\n if self.parent.comparator.compare(&h.key, &t.key).is_ge() {\n self.head = None;\n self.tail = None;\n }\n }\n self.tail.map(|n| Entry {\n parent: self.parent,\n node: n,\n guard: self.guard,\n })\n }\n}\n\nimpl fmt::Debug for Iter<'_, '_, K, V, C>\nwhere\n K: fmt::Debug,\n V: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"Iter\")\n .field(\"head\", &self.head.as_deref().map(|n| (&n.key, &n.value)))\n .field(\"tail\", &self.tail.as_deref().map(|n| (&n.key, &n.value)))\n .finish()\n }\n}\n\n/// An iterator over reference-counted entries of a `SkipList`.\npub struct RefIter<'a, K, V, C = BasicComparator> {\n parent: &'a SkipList,\n head: Option>,\n tail: Option>,\n}\n\nimpl fmt::Debug for RefIter<'_, K, V, C>\nwhere\n K: fmt::Debug,\n V: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n let mut d = f.debug_struct(\"RefIter\");\n match &self.head {\n None => d.field(\"head\", &None::<(&K, &V)>),\n Some(e) => d.field(\"head\", &(e.key(), e.value())),\n };\n match &self.tail {\n None => d.field(\"tail\", &None::<(&K, &V)>),\n Some(e) => d.field(\"tail\", &(e.key(), e.value())),\n };\n d.finish()\n }\n}\n\nimpl<'a, K: 'a, V: 'a, C> RefIter<'a, K, V, C>\nwhere\n C: Comparator,\n{\n /// Advances the iterator and returns the next value.\n pub fn next(&mut self, guard: &Guard) -> Option> {\n self.parent.check_guard(guard);\n let next_head = match &self.head {\n Some(e) => e.next(guard),\n None => try_pin_loop(|| self.parent.front(guard)),\n };\n match (&next_head, &self.tail) {\n // The next key is larger than the latest tail key we observed with this iterator.\n (Some(ref next), Some(t))\n if self.parent.comparator.compare(next.key(), t.key()).is_ge() =>\n {\n unsafe {\n next.node.decrement(guard);\n }\n None\n }\n (Some(_), _) => {\n if let Some(e) = mem::replace(&mut self.head, next_head.clone()) {\n unsafe {\n e.node.decrement(guard);\n }\n }\n next_head\n }\n (None, _) => None,\n }\n }\n\n /// Removes and returns an element from the end of the iterator.\n pub fn next_back(&mut self, guard: &Guard) -> Option> {\n self.parent.check_guard(guard);\n let next_tail = match &self.tail {\n Some(e) => e.prev(guard),\n None => try_pin_loop(|| self.parent.back(guard)),\n };\n match (&self.head, &next_tail) {\n // The prev key is smaller than the latest head key we observed with this iterator.\n (Some(h), Some(next))\n if self.parent.comparator.compare(h.key(), next.key()).is_ge() =>\n {\n unsafe {\n next.node.decrement(guard);\n }\n None\n }\n (_, Some(_)) => {\n if let Some(e) = mem::replace(&mut self.tail, next_tail.clone()) {\n unsafe {\n e.node.decrement(guard);\n }\n }\n next_tail\n }\n (_, None) => None,\n }\n }\n}\n\nimpl<'a, K: 'a, V: 'a, C> RefIter<'a, K, V, C> {\n /// Decrements the reference count of `RefEntry` owned by the iterator.\n pub fn drop_impl(&mut self, guard: &Guard) {\n self.parent.check_guard(guard);\n if let Some(e) = self.head.take() {\n unsafe { e.node.decrement(guard) };\n }\n if let Some(e) = self.tail.take() {\n unsafe { e.node.decrement(guard) };\n }\n }\n}\n\n/// An iterator over a subset of entries of a `SkipList`.\npub struct Range<'a: 'g, 'g, Q, R, K, V, C = BasicComparator>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n parent: &'a SkipList,\n head: Option>,\n tail: Option>,\n range: R,\n guard: &'g Guard,\n _marker: PhantomData Q>, // covariant over `Q`\n}\n\nimpl<'a: 'g, 'g, Q, R, K: 'a, V: 'a, C> Iterator for Range<'a, 'g, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n type Item = Entry<'a, 'g, K, V, C>;\n\n fn next(&mut self) -> Option {\n self.head = match self.head {\n Some(n) => self\n .parent\n .next_node(n.as_tower(), Bound::Excluded(&n.key), self.guard),\n None => self\n .parent\n .search_bound(self.range.start_bound(), false, self.guard),\n };\n if let Some(h) = self.head {\n match self.tail {\n Some(t) => {\n let bound = Bound::Excluded(&t.as_ref().key);\n if !below_upper_bound(&self.parent.comparator, &bound, &h.key) {\n self.head = None;\n self.tail = None;\n }\n }\n None => {\n let bound = self.range.end_bound();\n if !below_upper_bound(&self.parent.comparator, &bound, &h.key) {\n self.head = None;\n self.tail = None;\n }\n }\n };\n }\n self.head.map(|n| Entry {\n parent: self.parent,\n node: n,\n guard: self.guard,\n })\n }\n}\n\nimpl<'a: 'g, 'g, Q, R, K: 'a, V: 'a, C> DoubleEndedIterator for Range<'a, 'g, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n fn next_back(&mut self) -> Option {\n self.tail = match self.tail {\n Some(n) => self\n .parent\n .search_bound::(Bound::Excluded(&n.key), true, self.guard),\n None => self\n .parent\n .search_bound(self.range.end_bound(), true, self.guard),\n };\n if let Some(t) = self.tail {\n match self.head {\n Some(h) => {\n let bound = Bound::Excluded(&h.as_ref().key);\n if !above_lower_bound(&self.parent.comparator, &bound, &t.key) {\n self.head = None;\n self.tail = None;\n }\n }\n None => {\n let bound = self.range.start_bound();\n if !above_lower_bound(&self.parent.comparator, &bound, &t.key) {\n self.head = None;\n self.tail = None;\n }\n }\n };\n }\n self.tail.map(|n| Entry {\n parent: self.parent,\n node: n,\n guard: self.guard,\n })\n }\n}\n\nimpl fmt::Debug for Range<'_, '_, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n K: fmt::Debug,\n V: fmt::Debug,\n R: RangeBounds + fmt::Debug,\n Q: ?Sized,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"Range\")\n .field(\"range\", &self.range)\n .field(\"head\", &self.head)\n .field(\"tail\", &self.tail)\n .finish()\n }\n}\n\n/// An iterator over reference-counted subset of entries of a `SkipList`.\npub struct RefRange<'a, Q, R, K, V, C = BasicComparator>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n parent: &'a SkipList,\n pub(crate) head: Option>,\n pub(crate) tail: Option>,\n pub(crate) range: R,\n _marker: PhantomData Q>, // covariant over `Q`\n}\n\nunsafe impl Send for RefRange<'_, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n}\n\nunsafe impl Sync for RefRange<'_, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n}\n\nimpl fmt::Debug for RefRange<'_, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n K: fmt::Debug,\n V: fmt::Debug,\n R: RangeBounds + fmt::Debug,\n Q: ?Sized,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"RefRange\")\n .field(\"range\", &self.range)\n .field(\"head\", &self.head)\n .field(\"tail\", &self.tail)\n .finish()\n }\n}\n\nimpl<'a, Q, R, K: 'a, V: 'a, C> RefRange<'a, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n /// Advances the iterator and returns the next value.\n pub fn next(&mut self, guard: &Guard) -> Option> {\n self.parent.check_guard(guard);\n let next_head = match self.head {\n Some(ref e) => e.next(guard),\n None => try_pin_loop(|| self.parent.lower_bound(self.range.start_bound(), guard)),\n };\n\n if let Some(ref h) = next_head {\n match self.tail {\n Some(ref t) => {\n let bound = Bound::Excluded(t.key());\n if below_upper_bound(&self.parent.comparator, &bound, h.key()) {\n if let Some(e) = mem::replace(&mut self.head, next_head.clone()) {\n unsafe {\n e.node.decrement(guard);\n }\n }\n next_head\n } else {\n unsafe {\n h.node.decrement(guard);\n }\n None\n }\n }\n None => {\n let bound = self.range.end_bound();\n if below_upper_bound(&self.parent.comparator, &bound, h.key()) {\n if let Some(e) = mem::replace(&mut self.head, next_head.clone()) {\n unsafe {\n e.node.decrement(guard);\n }\n }\n next_head\n } else {\n unsafe {\n h.node.decrement(guard);\n }\n None\n }\n }\n }\n } else {\n None\n }\n }\n\n /// Removes and returns an element from the end of the iterator.\n pub fn next_back(&mut self, guard: &Guard) -> Option> {\n self.parent.check_guard(guard);\n let next_tail = match self.tail {\n Some(ref e) => e.prev(guard),\n None => try_pin_loop(|| self.parent.upper_bound(self.range.end_bound(), guard)),\n };\n\n if let Some(ref t) = next_tail {\n match self.head {\n Some(ref h) => {\n let bound = Bound::Excluded(h.key());\n if above_lower_bound(&self.parent.comparator, &bound, t.key()) {\n if let Some(e) = mem::replace(&mut self.tail, next_tail.clone()) {\n unsafe {\n e.node.decrement(guard);\n }\n }\n next_tail\n } else {\n unsafe {\n t.node.decrement(guard);\n }\n None\n }\n }\n None => {\n let bound = self.range.start_bound();\n if above_lower_bound(&self.parent.comparator, &bound, t.key()) {\n if let Some(e) = mem::replace(&mut self.tail, next_tail.clone()) {\n unsafe {\n e.node.decrement(guard);\n }\n }\n next_tail\n } else {\n unsafe {\n t.node.decrement(guard);\n }\n None\n }\n }\n }\n } else {\n None\n }\n }\n\n /// Decrements a reference count owned by this iterator.\n pub fn drop_impl(&mut self, guard: &Guard) {\n self.parent.check_guard(guard);\n if let Some(e) = self.head.take() {\n unsafe { e.node.decrement(guard) };\n }\n if let Some(e) = self.tail.take() {\n unsafe { e.node.decrement(guard) };\n }\n }\n}\n\n/// An owning iterator over the entries of a `SkipList`.\npub struct IntoIter {\n /// The current node.\n ///\n /// All preceding nods have already been destroyed.\n node: *mut Node,\n}\n\nimpl Drop for IntoIter {\n fn drop(&mut self) {\n // Iterate through the whole chain and destroy every node.\n while let Some(node) = NonNull::new(self.node) {\n unsafe {\n let node = NodeRef::new(node);\n // Unprotected loads are okay because this function is the only one currently using\n // the skip list.\n let next = node\n .get_level(0)\n .load(Ordering::Relaxed, epoch::unprotected());\n\n // We can safely do this without deferring because references to\n // keys & values that we give out never outlive the SkipList.\n Node::finalize(node.ptr.as_ptr());\n\n self.node = next.as_raw() as *mut Node;\n }\n }\n }\n}\n\nimpl Iterator for IntoIter {\n type Item = (K, V);\n\n fn next(&mut self) -> Option {\n loop {\n // Have we reached the end of the skip list?\n if self.node.is_null() {\n return None;\n }\n\n unsafe {\n // Take the key and value out of the node.\n let key = ptr::read(&(*self.node).key);\n let value = ptr::read(&(*self.node).value);\n\n // Get the next node in the skip list.\n //\n // Unprotected loads are okay because this function is the only one currently using\n // the skip list.\n let next = {\n let node = NodeRef::new(NonNull::new_unchecked(self.node));\n node.get_level(0)\n .load(Ordering::Relaxed, epoch::unprotected())\n };\n\n // Deallocate the current node and move to the next one.\n Node::dealloc(self.node);\n self.node = next.as_raw() as *mut Node;\n\n // The current node may be marked. If it is, it's been removed from the skip list\n // and we should just skip it.\n if next.tag() == 0 {\n return Some((key, value));\n }\n }\n }\n }\n}\n\nimpl fmt::Debug for IntoIter {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"IntoIter { .. }\")\n }\n}\n\n/// Helper function to retry an operation until pinning succeeds or `None` is\n/// returned.\npub(crate) fn try_pin_loop<'a: 'g, 'g, F, K, V, C>(mut f: F) -> Option>\nwhere\n F: FnMut() -> Option>,\n{\n loop {\n if let Some(e) = f()?.pin() {\n return Some(e);\n }\n }\n}\n\n/// Helper function to check if a value is above a lower bound\nfn above_lower_bound(comparator: &C, bound: &Bound<&T>, other: &V) -> bool\nwhere\n T: ?Sized,\n C: Comparator,\n{\n match *bound {\n Bound::Unbounded => true,\n Bound::Included(key) => comparator.compare(other, key).is_ge(),\n Bound::Excluded(key) => comparator.compare(other, key).is_gt(),\n }\n}\n\n/// Helper function to check if a value is below an upper bound\nfn below_upper_bound(comparator: &C, bound: &Bound<&T>, other: &V) -> bool\nwhere\n T: ?Sized,\n C: Comparator,\n{\n match *bound {\n Bound::Unbounded => true,\n Bound::Included(key) => comparator.compare(other, key).is_le(),\n Bound::Excluded(key) => comparator.compare(other, key).is_lt(),\n }\n}\n" }, { "path": "crossbeam-skiplist/src/comparator.rs", "content": "//! Traits for key comparison in maps.\n\nuse core::cmp::Ordering;\n\nuse crate::equivalent::{Comparable, Equivalent};\n\n/// Key equality trait.\n///\n/// This trait allows for very flexible comparison of objects. You may\n/// borrow/dereference `L` and `R` using `Borrow` or another trait. The trait\n/// takes a `self` parameter, which you can use to change how the operands are\n/// compared. For example, you can toggle string case-sensitivity on and off at\n/// runtime, or you can use `Box>` to allow the user of\n/// your code to supply a custom comparison function.\n///\n/// Implementations of `Equivalator` don't affect the inherent `Eq` or `Hash`\n/// implementations for a type. Currently there is no companion trait that\n/// allows for custom hashing the way `Equivalator` allows for custom\n/// comparison.\n///\n/// See also `Comparator`.\n///\n/// ## Example\n/// ```\n/// use crossbeam_skiplist::comparator::Equivalator;\n///\n/// struct MyEquivalator {\n/// case_sensitive: bool,\n/// }\n///\n/// impl Equivalator<[u8], [u8]> for MyEquivalator {\n/// fn equivalent(&self, lhs: &[u8], rhs: &[u8]) -> bool {\n/// if self.case_sensitive {\n/// lhs == rhs\n/// } else {\n/// // Case-insensitive ASCII comparison on raw bytes\n/// if lhs.len() != rhs.len() { return false; }\n/// for (&c1, &c2) in lhs.iter().zip(rhs.iter()) {\n/// let c1 = if c1 >= 0x61 && c1 <= 0x7a { c1 - 32 } else { c1 };\n/// let c2 = if c2 >= 0x61 && c2 <= 0x7a { c2 - 32 } else { c2 };\n/// if c1 != c2 { return false; }\n/// }\n/// true\n/// }\n/// }\n/// }\n/// ```\npub trait Equivalator {\n /// Compare `lhs` to `rhs` for equality.\n fn equivalent(&self, lhs: &L, rhs: &R) -> bool;\n}\n\n/// Key ordering trait.\n///\n/// This trait allows for very flexible comparison of objects. You may\n/// borrow/dereference `L` and `R` using `Borrow` or another trait. The trait\n/// takes a `self` parameter, which you can use to change how the operands are\n/// compared. For example, you can toggle string case-sensitivity on and off,\n/// or you can use `Box>` to allow the user of your code\n/// to supply a custom comparison function.\n///\n/// The `Comparator` and `Equivalator` implementations for a comparator must\n/// agree on which inputs are equal.\n///\n/// See also `Equivalator`.\npub trait Comparator: Equivalator {\n /// Compare `lhs` to `rhs` and return their ordering.\n fn compare(&self, lhs: &L, rhs: &R) -> Ordering;\n}\n\n/// This comparator uses the `Equivalent` and `Comparable` traits to perform\n/// comparisons, which themselves fall back on the standard library `Borrow`\n/// and `Ord` traits. When used in a map, this results in the same behavior as\n/// the standard `BTreeMap` interface.\n#[derive(Clone, Copy, Debug, Default)]\npub struct BasicComparator;\n\nimpl Equivalator for BasicComparator\nwhere\n K: Equivalent,\n{\n #[inline]\n fn equivalent(&self, lhs: &K, rhs: &Q) -> bool {\n >::equivalent(lhs, rhs)\n }\n}\n\nimpl Comparator for BasicComparator\nwhere\n K: Comparable,\n{\n #[inline]\n fn compare(&self, lhs: &K, rhs: &Q) -> Ordering {\n >::compare(lhs, rhs)\n }\n}\n" }, { "path": "crossbeam-skiplist/src/equivalent.rs", "content": "// These traits are based on `equivalent` crate, but `K` and `Q` are flipped to avoid type inference issues:\n// https://github.com/indexmap-rs/equivalent/issues/5\n\n//! Traits for key comparison in maps.\n\nuse core::{borrow::Borrow, cmp::Ordering};\n\n/// Key equivalence trait.\n///\n/// This trait allows hash table lookup to be customized. It has one blanket\n/// implementation that uses the regular solution with `Borrow` and `Eq`, just\n/// like `HashMap` does, so that you can pass `&str` to lookup into a map with\n/// `String` keys and so on.\n///\n/// # Contract\n///\n/// The implementor **must** hash like `Q`, if it is hashable.\npub trait Equivalent {\n /// Compare self to `key` and return `true` if they are equal.\n fn equivalent(&self, key: &Q) -> bool;\n}\n\nimpl Equivalent for K\nwhere\n K: Borrow,\n Q: Eq,\n{\n #[inline]\n fn equivalent(&self, key: &Q) -> bool {\n PartialEq::eq(self.borrow(), key)\n }\n}\n\n/// Key ordering trait.\n///\n/// This trait allows ordered map lookup to be customized. It has one blanket\n/// implementation that uses the regular solution with `Borrow` and `Ord`, just\n/// like `BTreeMap` does, so that you can pass `&str` to lookup into a map with\n/// `String` keys and so on.\npub trait Comparable: Equivalent {\n /// Compare self to `key` and return their ordering.\n fn compare(&self, key: &Q) -> Ordering;\n}\n\nimpl Comparable for K\nwhere\n K: Borrow,\n Q: Ord,\n{\n #[inline]\n fn compare(&self, key: &Q) -> Ordering {\n Ord::cmp(self.borrow(), key)\n }\n}\n" }, { "path": "crossbeam-skiplist/src/lib.rs", "content": "//! Concurrent maps and sets based on [skip lists].\n//!\n//! This crate provides the types [`SkipMap`] and [`SkipSet`].\n//! These data structures provide an interface similar to [`BTreeMap`] and [`BTreeSet`],\n//! respectively, except they support safe concurrent access across\n//! multiple threads.\n//!\n//! # Concurrent access\n//! [`SkipMap`] and [`SkipSet`] implement [`Send`] and [`Sync`],\n//! so they can be shared across threads with ease.\n//!\n//! Methods which mutate the map, such as [`insert`],\n//! take `&self` rather than `&mut self`. This allows\n//! them to be invoked concurrently.\n//!\n//! ```\n//! use crossbeam_skiplist::SkipMap;\n//! use crossbeam_utils::thread::scope;\n//!\n//! let person_ages = SkipMap::new();\n//!\n//! scope(|s| {\n//! // Insert entries into the map from multiple threads.\n//! s.spawn(|_| {\n//! person_ages.insert(\"Spike Garrett\", 22);\n//! person_ages.insert(\"Stan Hancock\", 47);\n//! person_ages.insert(\"Rea Bryan\", 234);\n//!\n//! assert_eq!(person_ages.get(\"Spike Garrett\").unwrap().value(), &22);\n//! });\n//! s.spawn(|_| {\n//! person_ages.insert(\"Bryon Conroy\", 65);\n//! person_ages.insert(\"Lauren Reilly\", 2);\n//! });\n//! }).unwrap();\n//!\n//! assert!(person_ages.contains_key(\"Spike Garrett\"));\n//! person_ages.remove(\"Rea Bryan\");\n//! assert!(!person_ages.contains_key(\"Rea Bryan\"));\n//!\n//! ```\n//!\n//! Concurrent access to skip lists is lock-free and sound.\n//! Threads won't get blocked waiting for other threads to finish operating\n//! on the map.\n//!\n//! Be warned that, because of this lock-freedom, it's easy to introduce\n//! race conditions into your code. For example:\n//! ```no_run\n//! use crossbeam_skiplist::SkipSet;\n//! use crossbeam_utils::thread::scope;\n//!\n//! let numbers = SkipSet::new();\n//! scope(|s| {\n//! // Spawn a thread which will remove 5 from the set.\n//! s.spawn(|_| {\n//! numbers.remove(&5);\n//! });\n//!\n//! // While the thread above is running, insert a value into the set.\n//! numbers.insert(5);\n//!\n//! // This check can fail!\n//! // The other thread may remove the value\n//! // before we perform this check.\n//! assert!(numbers.contains(&5));\n//! }).unwrap();\n//! ```\n//!\n//! In effect, a _single_ operation on the map, such as [`insert`],\n//! operates atomically: race conditions are impossible. However,\n//! concurrent calls to functions can become interleaved across\n//! threads, introducing non-determinism.\n//!\n//! To avoid this sort of race condition, never assume that a collection's\n//! state will remain the same across multiple lines of code. For instance,\n//! in the example above, the problem arises from the assumption that\n//! the map won't be mutated between the calls to `insert` and `contains`.\n//! In sequential code, this would be correct. But when multiple\n//! threads are introduced, more care is needed.\n//!\n//! Note that race conditions do not violate Rust's memory safety rules.\n//! A race between multiple threads can never cause memory errors or\n//! segfaults. A race condition is a _logic error_ in its entirety.\n//!\n//! # Mutable access to elements\n//! [`SkipMap`] and [`SkipSet`] provide no way to retrieve a mutable reference\n//! to a value. Since access methods can be called concurrently, providing\n//! e.g. a `get_mut` function could cause data races.\n//!\n//! A solution to the above is to have the implementation wrap\n//! each value in a lock. However, this has some repercussions:\n//! * The map would no longer be lock-free, inhibiting scalability\n//! and allowing for deadlocks.\n//! * If a user of the map doesn't need mutable access, then they pay\n//! the price of locks without actually needing them.\n//!\n//! Instead, the approach taken by this crate gives more control to the user.\n//! If mutable access is needed, then you can use interior mutability,\n//! such as [`RwLock`]: `SkipMap>`.\n//!\n//! # Garbage collection\n//! A problem faced by many concurrent data structures\n//! is choosing when to free unused memory. Care must be\n//! taken to prevent use-after-frees and double-frees, both\n//! of which cause undefined behavior.\n//!\n//! Consider the following sequence of events operating on a [`SkipMap`]:\n//! * Thread A calls [`get`] and holds a reference to a value in the map.\n//! * Thread B removes that key from the map.\n//! * Thread A now attempts to access the value.\n//!\n//! What happens here? If the map implementation frees the memory\n//! belonging to a value when it is\n//! removed, then a use-after-free occurs, resulting in memory corruption.\n//!\n//! To solve the above, this crate uses the _epoch-based memory reclamation_ mechanism\n//! implemented in [`crossbeam-epoch`]. Simplified, a value removed from the map\n//! is not freed until after all references to it have been dropped. This mechanism\n//! is similar to the garbage collection found in some languages, such as Java, except\n//! it operates solely on the values inside the map.\n//!\n//! This garbage collection scheme functions automatically; users don't have to worry about it.\n//! However, keep in mind that holding [`Entry`] handles to entries in the map will prevent\n//! that memory from being freed until at least after the handles are dropped.\n//!\n//! # Performance versus B-trees\n//! In general, when you need concurrent writes\n//! to an ordered collection, skip lists are a reasonable choice.\n//! However, they can be substantially slower than B-trees\n//! in some scenarios.\n//!\n//! The main benefit of a skip list over a `RwLock`\n//! is that it allows concurrent writes to progress without\n//! mutual exclusion. However, when the frequency\n//! of writes is low, this benefit isn't as useful.\n//! In these cases, a shared [`BTreeMap`] may be a faster option.\n//!\n//! These guidelines should be taken with a grain of salt—performance\n//! in practice varies depending on your use case.\n//! In the end, the best way to choose between [`BTreeMap`] and [`SkipMap`]\n//! is to benchmark them in your own application.\n//!\n//! # Alternatives\n//! This crate implements _ordered_ maps and sets, akin to [`BTreeMap`] and [`BTreeSet`].\n//! In many situations, however, a defined order on elements is not required. For these\n//! purposes, unordered maps will suffice. In addition, unordered maps\n//! often have better performance characteristics than their ordered alternatives.\n//!\n//! Crossbeam [does not currently provide a concurrent unordered map](https://github.com/crossbeam-rs/rfcs/issues/32).\n//! That said, here are some other crates which may suit you:\n//! * [`DashMap`](https://docs.rs/dashmap) implements a novel concurrent hash map\n//! with good performance characteristics.\n//! * [`flurry`](https://docs.rs/flurry) is a Rust port of Java's `ConcurrentHashMap`.\n//!\n//! [`insert`]: SkipMap::insert\n//! [`get`]: SkipMap::get\n//! [`Entry`]: map::Entry\n//! [skip lists]: https://en.wikipedia.org/wiki/Skip_list\n//! [`crossbeam-epoch`]: https://docs.rs/crossbeam-epoch\n//! [`BTreeMap`]: std::collections::BTreeMap\n//! [`BTreeSet`]: std::collections::BTreeSet\n//! [`RwLock`]: std::sync::RwLock\n//!\n//! # Examples\n//! [`SkipMap`] basic usage:\n//! ```\n//! use crossbeam_skiplist::SkipMap;\n//!\n//! // Note that the variable doesn't have to be mutable:\n//! // SkipMap methods take &self to support concurrent access.\n//! let movie_reviews = SkipMap::new();\n//!\n//! // Insert some key-value pairs.\n//! movie_reviews.insert(\"Office Space\", \"Deals with real issues in the workplace.\");\n//! movie_reviews.insert(\"Pulp Fiction\", \"Masterpiece.\");\n//! movie_reviews.insert(\"The Godfather\", \"Very enjoyable.\");\n//! movie_reviews.insert(\"The Blues Brothers\", \"Eye lyked it a lot.\");\n//!\n//! // Get the value associated with a key.\n//! // get() returns an Entry, which gives\n//! // references to the key and value.\n//! let pulp_fiction = movie_reviews.get(\"Pulp Fiction\").unwrap();\n//! assert_eq!(*pulp_fiction.key(), \"Pulp Fiction\");\n//! assert_eq!(*pulp_fiction.value(), \"Masterpiece.\");\n//!\n//! // Remove a key-value pair.\n//! movie_reviews.remove(\"The Blues Brothers\");\n//! assert!(movie_reviews.get(\"The Blues Brothers\").is_none());\n//!\n//! // Iterate over the reviews. Since SkipMap\n//! // is an ordered map, the iterator will yield\n//! // keys in lexicographical order.\n//! for entry in &movie_reviews {\n//! let movie = entry.key();\n//! let review = entry.value();\n//! println!(\"{}: \\\"{}\\\"\", movie, review);\n//! }\n//! ```\n//!\n//! [`SkipSet`] basic usage:\n//! ```\n//! use crossbeam_skiplist::SkipSet;\n//!\n//! let books = SkipSet::new();\n//!\n//! // Add some books to the set.\n//! books.insert(\"A Dance With Dragons\");\n//! books.insert(\"To Kill a Mockingbird\");\n//! books.insert(\"The Odyssey\");\n//! books.insert(\"The Great Gatsby\");\n//!\n//! // Check for a specific one.\n//! if !books.contains(\"The Winds of Winter\") {\n//! println!(\"We have {} books, but The Winds of Winter ain't one.\",\n//! books.len());\n//! }\n//!\n//! // Remove a book from the set.\n//! books.remove(\"To Kill a Mockingbird\");\n//! assert!(!books.contains(\"To Kill a Mockingbird\"));\n//!\n//! // Iterate over the books in the set.\n//! // Values are returned in lexicographical order.\n//! for entry in &books {\n//! let book = entry.value();\n//! println!(\"{}\", book);\n//! }\n//! ```\n\n#![no_std]\n#![doc(test(\n no_crate_inject,\n attr(allow(dead_code, unused_assignments, unused_variables))\n))]\n#![warn(\n missing_docs,\n unsafe_op_in_unsafe_fn,\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nextern crate alloc;\n#[cfg(feature = \"std\")]\nextern crate std;\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\nmod alloc_helper;\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\npub mod base;\n\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\n#[doc(inline)]\npub use crate::base::SkipList;\n\n#[cfg(feature = \"std\")]\npub mod map;\n#[cfg(feature = \"std\")]\npub mod set;\n\n#[cfg(feature = \"std\")]\n#[doc(inline)]\npub use crate::{map::SkipMap, set::SkipSet};\n\npub mod comparator;\npub mod equivalent;\n" }, { "path": "crossbeam-skiplist/src/map.rs", "content": "//! An ordered map based on a lock-free skip list. See [`SkipMap`].\n\nuse core::{\n fmt,\n mem::ManuallyDrop,\n ops::{Bound, RangeBounds},\n ptr,\n};\n\nuse crossbeam_epoch as epoch;\n\nuse crate::{\n base::{self, try_pin_loop},\n comparator::{BasicComparator, Comparator},\n};\n\n/// An ordered map based on a lock-free skip list.\n///\n/// This is an alternative to [`BTreeMap`] which supports\n/// concurrent access across multiple threads.\n///\n/// A custom comparator may be provided, causing all keys\n/// to be ordered by the comparison function used instead\n/// of the standard `Ord` impl. See [`Comparator`].\n///\n/// [`BTreeMap`]: std::collections::BTreeMap\n/// [`Comparator`]: crate::comparator::Comparator\npub struct SkipMap {\n inner: base::SkipList,\n}\n\nimpl SkipMap {\n /// Returns a new, empty map with the default comparator.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let map: SkipMap = SkipMap::new();\n /// ```\n pub fn new() -> Self {\n Self {\n inner: base::SkipList::new(epoch::default_collector().clone()),\n }\n }\n}\n\nimpl SkipMap {\n /// Returns a new, empty map with the given comparator.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::{SkipMap, comparator::BasicComparator};\n ///\n /// let map: SkipMap = SkipMap::with_comparator(BasicComparator);\n /// ```\n pub fn with_comparator(comparator: C) -> Self {\n Self {\n inner: base::SkipList::with_comparator(epoch::default_collector().clone(), comparator),\n }\n }\n\n /// Returns `true` if the map is empty.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let map: SkipMap<&str, &str> = SkipMap::new();\n /// assert!(map.is_empty());\n ///\n /// map.insert(\"key\", \"value\");\n /// assert!(!map.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n self.inner.is_empty()\n }\n\n /// Returns the number of entries in the map.\n ///\n /// If the map is being concurrently modified, consider the returned number just an\n /// approximation without any guarantees.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let map = SkipMap::new();\n /// map.insert(0, 1);\n /// assert_eq!(map.len(), 1);\n ///\n /// for x in 1..=5 {\n /// map.insert(x, x + 1);\n /// }\n ///\n /// assert_eq!(map.len(), 6);\n /// ```\n pub fn len(&self) -> usize {\n self.inner.len()\n }\n}\n\nimpl SkipMap\nwhere\n C: Comparator,\n{\n /// Returns the entry with the smallest key.\n ///\n /// This function returns an [`Entry`] which\n /// can be used to access the key's associated value.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let numbers = SkipMap::new();\n /// numbers.insert(5, \"five\");\n /// assert_eq!(*numbers.front().unwrap().value(), \"five\");\n /// numbers.insert(6, \"six\");\n /// assert_eq!(*numbers.front().unwrap().value(), \"five\");\n /// ```\n pub fn front(&self) -> Option> {\n let guard = &epoch::pin();\n try_pin_loop(|| self.inner.front(guard)).map(Entry::new)\n }\n\n /// Returns the entry with the largest key.\n ///\n /// This function returns an [`Entry`] which\n /// can be used to access the key's associated value.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let numbers = SkipMap::new();\n /// numbers.insert(5, \"five\");\n /// assert_eq!(*numbers.back().unwrap().value(), \"five\");\n /// numbers.insert(6, \"six\");\n /// assert_eq!(*numbers.back().unwrap().value(), \"six\");\n /// ```\n pub fn back(&self) -> Option> {\n let guard = &epoch::pin();\n try_pin_loop(|| self.inner.back(guard)).map(Entry::new)\n }\n\n /// Finds an entry with the specified key, or inserts a new `key`-`value` pair if none exist.\n ///\n /// This function returns an [`Entry`] which\n /// can be used to access the key's associated value.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let ages = SkipMap::new();\n /// let gates_age = ages.get_or_insert(\"Bill Gates\", 64);\n /// assert_eq!(*gates_age.value(), 64);\n ///\n /// ages.insert(\"Steve Jobs\", 65);\n /// let jobs_age = ages.get_or_insert(\"Steve Jobs\", -1);\n /// assert_eq!(*jobs_age.value(), 65);\n /// ```\n pub fn get_or_insert(&self, key: K, value: V) -> Entry<'_, K, V, C> {\n let guard = &epoch::pin();\n Entry::new(self.inner.get_or_insert(key, value, guard))\n }\n\n /// Finds an entry with the specified key, or inserts a new `key`-`value` pair if none exist,\n /// where value is calculated with a function.\n ///\n /// Note: Another thread may write key value first, leading to the result of this closure\n /// discarded. If closure is modifying some other state (such as shared counters or shared\n /// objects), it may lead to undesired behaviour such as counters being changed without\n /// result of closure inserted\n ///\n /// This function returns an [`Entry`] which\n /// can be used to access the key's associated value.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let ages = SkipMap::new();\n /// let gates_age = ages.get_or_insert_with(\"Bill Gates\", || 64);\n /// assert_eq!(*gates_age.value(), 64);\n ///\n /// ages.insert(\"Steve Jobs\", 65);\n /// let jobs_age = ages.get_or_insert_with(\"Steve Jobs\", || -1);\n /// assert_eq!(*jobs_age.value(), 65);\n /// ```\n pub fn get_or_insert_with(&self, key: K, value_fn: F) -> Entry<'_, K, V, C>\n where\n F: FnOnce() -> V,\n {\n let guard = &epoch::pin();\n Entry::new(self.inner.get_or_insert_with(key, value_fn, guard))\n }\n\n /// Returns an iterator over all entries in the map,\n /// sorted by key.\n ///\n /// This iterator returns [`Entry`]s which\n /// can be used to access keys and their associated values.\n ///\n /// # Examples\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let numbers = SkipMap::new();\n /// numbers.insert(6, \"six\");\n /// numbers.insert(7, \"seven\");\n /// numbers.insert(12, \"twelve\");\n ///\n /// // Print then numbers from least to greatest\n /// for entry in numbers.iter() {\n /// let number = entry.key();\n /// let number_str = entry.value();\n /// println!(\"{} is {}\", number, number_str);\n /// }\n /// ```\n pub fn iter(&self) -> Iter<'_, K, V, C> {\n Iter {\n inner: self.inner.ref_iter(),\n }\n }\n}\n\nimpl SkipMap\nwhere\n C: Comparator,\n{\n /// Returns `true` if the map contains a value for the specified key.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let ages = SkipMap::new();\n /// ages.insert(\"Bill Gates\", 64);\n ///\n /// assert!(ages.contains_key(&\"Bill Gates\"));\n /// assert!(!ages.contains_key(&\"Steve Jobs\"));\n /// ```\n pub fn contains_key(&self, key: &Q) -> bool\n where\n C: Comparator,\n Q: ?Sized,\n {\n let guard = &epoch::pin();\n self.inner.contains_key(key, guard)\n }\n\n /// Returns an entry with the specified `key`.\n ///\n /// This function returns an [`Entry`] which\n /// can be used to access the key's associated value.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let numbers: SkipMap<&str, i32> = SkipMap::new();\n /// assert!(numbers.get(\"six\").is_none());\n ///\n /// numbers.insert(\"six\", 6);\n /// assert_eq!(*numbers.get(\"six\").unwrap().value(), 6);\n /// ```\n pub fn get(&self, key: &Q) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n let guard = &epoch::pin();\n try_pin_loop(|| self.inner.get(key, guard)).map(Entry::new)\n }\n\n /// Returns an `Entry` pointing to the lowest element whose key is above\n /// the given bound. If no such element is found then `None` is\n /// returned.\n ///\n /// This function returns an [`Entry`] which\n /// can be used to access the key's associated value.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n /// use std::ops::Bound::*;\n ///\n /// let numbers = SkipMap::new();\n /// numbers.insert(6, \"six\");\n /// numbers.insert(7, \"seven\");\n /// numbers.insert(12, \"twelve\");\n ///\n /// let greater_than_five = numbers.lower_bound(Excluded(&5)).unwrap();\n /// assert_eq!(*greater_than_five.value(), \"six\");\n ///\n /// let greater_than_six = numbers.lower_bound(Excluded(&6)).unwrap();\n /// assert_eq!(*greater_than_six.value(), \"seven\");\n ///\n /// let greater_than_thirteen = numbers.lower_bound(Excluded(&13));\n /// assert!(greater_than_thirteen.is_none());\n /// ```\n pub fn lower_bound<'a, Q>(&'a self, bound: Bound<&Q>) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n let guard = &epoch::pin();\n try_pin_loop(|| self.inner.lower_bound(bound, guard)).map(Entry::new)\n }\n\n /// Returns an `Entry` pointing to the highest element whose key is below\n /// the given bound. If no such element is found then `None` is\n /// returned.\n ///\n /// This function returns an [`Entry`] which\n /// can be used to access the key's associated value.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n /// use std::ops::Bound::*;\n ///\n /// let numbers = SkipMap::new();\n /// numbers.insert(6, \"six\");\n /// numbers.insert(7, \"seven\");\n /// numbers.insert(12, \"twelve\");\n ///\n /// let less_than_eight = numbers.upper_bound(Excluded(&8)).unwrap();\n /// assert_eq!(*less_than_eight.value(), \"seven\");\n ///\n /// let less_than_six = numbers.upper_bound(Excluded(&6));\n /// assert!(less_than_six.is_none());\n /// ```\n pub fn upper_bound<'a, Q>(&'a self, bound: Bound<&Q>) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n let guard = &epoch::pin();\n try_pin_loop(|| self.inner.upper_bound(bound, guard)).map(Entry::new)\n }\n\n /// Returns an iterator over a subset of entries in the map.\n ///\n /// This iterator returns [`Entry`]s which\n /// can be used to access keys and their associated values.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let numbers = SkipMap::new();\n /// numbers.insert(6, \"six\");\n /// numbers.insert(7, \"seven\");\n /// numbers.insert(12, \"twelve\");\n ///\n /// // Print all numbers in the map between 5 and 8.\n /// for entry in numbers.range(5..=8) {\n /// let number = entry.key();\n /// let number_str = entry.value();\n /// println!(\"{} is {}\", number, number_str);\n /// }\n /// ```\n pub fn range(&self, range: R) -> Range<'_, Q, R, K, V, C>\n where\n R: RangeBounds,\n C: Comparator,\n Q: ?Sized,\n {\n Range {\n inner: self.inner.ref_range(range),\n }\n }\n}\n\nimpl SkipMap\nwhere\n C: Comparator,\n K: Send + 'static,\n V: Send + 'static,\n{\n /// Inserts a `key`-`value` pair into the map and returns the new entry.\n ///\n /// If there is an existing entry with this key, it will be removed before inserting the new\n /// one.\n ///\n /// This function returns an [`Entry`] which\n /// can be used to access the inserted key's associated value.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let map = SkipMap::new();\n /// map.insert(\"key\", \"value\");\n ///\n /// assert_eq!(*map.get(\"key\").unwrap().value(), \"value\");\n /// ```\n pub fn insert(&self, key: K, value: V) -> Entry<'_, K, V, C> {\n let guard = &epoch::pin();\n Entry::new(self.inner.insert(key, value, guard))\n }\n\n /// Inserts a `key`-`value` pair into the skip list and returns the new entry.\n ///\n /// If there is an existing entry with this key and compare(entry.value) returns true,\n /// it will be removed before inserting the new one.\n /// The closure will not be called if the key is not present.\n ///\n /// This function returns an [`Entry`] which\n /// can be used to access the inserted key's associated value.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let map = SkipMap::new();\n /// map.insert(\"key\", 1);\n /// map.compare_insert(\"key\", 0, |x| x < &0);\n /// assert_eq!(*map.get(\"key\").unwrap().value(), 1);\n /// map.compare_insert(\"key\", 2, |x| x < &2);\n /// assert_eq!(*map.get(\"key\").unwrap().value(), 2);\n /// map.compare_insert(\"absent_key\", 0, |_| false);\n /// assert_eq!(*map.get(\"absent_key\").unwrap().value(), 0);\n /// ```\n pub fn compare_insert(&self, key: K, value: V, compare_fn: F) -> Entry<'_, K, V, C>\n where\n F: Fn(&V) -> bool,\n {\n let guard = &epoch::pin();\n Entry::new(self.inner.compare_insert(key, value, compare_fn, guard))\n }\n\n /// Removes an entry with the specified `key` from the map and returns it.\n ///\n /// The value will not actually be dropped until all references to it have gone\n /// out of scope.\n ///\n /// This function returns an [`Entry`] which\n /// can be used to access the removed key's associated value.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let map: SkipMap<&str, &str> = SkipMap::new();\n /// assert!(map.remove(\"invalid key\").is_none());\n ///\n /// map.insert(\"key\", \"value\");\n /// assert_eq!(*map.remove(\"key\").unwrap().value(), \"value\");\n /// ```\n pub fn remove(&self, key: &Q) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n let guard = &epoch::pin();\n self.inner.remove(key, guard).map(Entry::new)\n }\n\n /// Removes the entry with the lowest key\n /// from the map. Returns the removed entry.\n ///\n /// The value will not actually be dropped until all references to it have gone\n /// out of scope.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let numbers = SkipMap::new();\n /// numbers.insert(6, \"six\");\n /// numbers.insert(7, \"seven\");\n /// numbers.insert(12, \"twelve\");\n ///\n /// assert_eq!(*numbers.pop_front().unwrap().value(), \"six\");\n /// assert_eq!(*numbers.pop_front().unwrap().value(), \"seven\");\n /// assert_eq!(*numbers.pop_front().unwrap().value(), \"twelve\");\n ///\n /// // All entries have been removed now.\n /// assert!(numbers.is_empty());\n /// ```\n pub fn pop_front(&self) -> Option> {\n let guard = &epoch::pin();\n self.inner.pop_front(guard).map(Entry::new)\n }\n\n /// Removes the entry with the greatest key from the map.\n /// Returns the removed entry.\n ///\n /// The value will not actually be dropped until all references to it have gone\n /// out of scope.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let numbers = SkipMap::new();\n /// numbers.insert(6, \"six\");\n /// numbers.insert(7, \"seven\");\n /// numbers.insert(12, \"twelve\");\n ///\n /// assert_eq!(*numbers.pop_back().unwrap().value(), \"twelve\");\n /// assert_eq!(*numbers.pop_back().unwrap().value(), \"seven\");\n /// assert_eq!(*numbers.pop_back().unwrap().value(), \"six\");\n ///\n /// // All entries have been removed now.\n /// assert!(numbers.is_empty());\n /// ```\n pub fn pop_back(&self) -> Option> {\n let guard = &epoch::pin();\n self.inner.pop_back(guard).map(Entry::new)\n }\n\n /// Removes all entries from the map.\n ///\n /// # Example\n /// ```\n /// use crossbeam_skiplist::SkipMap;\n ///\n /// let people = SkipMap::new();\n /// people.insert(\"Bill\", \"Gates\");\n /// people.insert(\"Steve\", \"Jobs\");\n ///\n /// people.clear();\n /// assert!(people.is_empty());\n /// ```\n pub fn clear(&self) {\n let guard = &mut epoch::pin();\n self.inner.clear(guard);\n }\n}\n\nimpl Default for SkipMap\nwhere\n C: Default,\n{\n fn default() -> Self {\n Self::with_comparator(Default::default())\n }\n}\n\nimpl fmt::Debug for SkipMap\nwhere\n K: fmt::Debug,\n V: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"SkipMap { .. }\")\n }\n}\n\nimpl IntoIterator for SkipMap {\n type Item = (K, V);\n type IntoIter = IntoIter;\n\n fn into_iter(self) -> Self::IntoIter {\n IntoIter {\n inner: self.inner.into_iter(),\n }\n }\n}\n\nimpl<'a, K, V, C> IntoIterator for &'a SkipMap\nwhere\n C: Comparator,\n{\n type Item = Entry<'a, K, V, C>;\n type IntoIter = Iter<'a, K, V, C>;\n\n fn into_iter(self) -> Self::IntoIter {\n self.iter()\n }\n}\n\nimpl FromIterator<(K, V)> for SkipMap\nwhere\n C: Comparator + Default,\n{\n fn from_iter(iter: I) -> Self\n where\n I: IntoIterator,\n {\n let s = Self::default();\n for (k, v) in iter {\n s.get_or_insert(k, v);\n }\n s\n }\n}\n\n/// A reference-counted entry in a map.\npub struct Entry<'a, K, V, C = BasicComparator> {\n inner: ManuallyDrop>,\n}\n\nimpl<'a, K, V, C> Entry<'a, K, V, C> {\n fn new(inner: base::RefEntry<'a, K, V, C>) -> Self {\n Self {\n inner: ManuallyDrop::new(inner),\n }\n }\n\n /// Returns a reference to the key.\n pub fn key(&self) -> &'a K {\n self.inner.key()\n }\n\n /// Returns a reference to the value.\n pub fn value(&self) -> &'a V {\n self.inner.value()\n }\n\n /// Returns `true` if the entry is removed from the map.\n pub fn is_removed(&self) -> bool {\n self.inner.is_removed()\n }\n}\n\nimpl Drop for Entry<'_, K, V, C> {\n fn drop(&mut self) {\n unsafe {\n ManuallyDrop::into_inner(ptr::read(&self.inner)).release_with_pin(epoch::pin);\n }\n }\n}\n\nimpl Entry<'_, K, V, C>\nwhere\n C: Comparator,\n{\n /// Moves to the next entry in the map.\n pub fn move_next(&mut self) -> bool {\n let guard = &epoch::pin();\n self.inner.move_next(guard)\n }\n\n /// Moves to the previous entry in the map.\n pub fn move_prev(&mut self) -> bool {\n let guard = &epoch::pin();\n self.inner.move_prev(guard)\n }\n\n /// Returns the next entry in the map.\n pub fn next(&self) -> Option {\n let guard = &epoch::pin();\n self.inner.next(guard).map(Entry::new)\n }\n\n /// Returns the previous entry in the map.\n pub fn prev(&self) -> Option {\n let guard = &epoch::pin();\n self.inner.prev(guard).map(Entry::new)\n }\n}\n\nimpl Entry<'_, K, V, C>\nwhere\n C: Comparator,\n K: Send + 'static,\n V: Send + 'static,\n{\n /// Removes the entry from the map.\n ///\n /// Returns `true` if this call removed the entry and `false` if it was already removed.\n pub fn remove(&self) -> bool {\n let guard = &epoch::pin();\n self.inner.remove(guard)\n }\n}\n\nimpl Clone for Entry<'_, K, V, C> {\n fn clone(&self) -> Self {\n Self {\n inner: self.inner.clone(),\n }\n }\n}\n\nimpl fmt::Debug for Entry<'_, K, V, C>\nwhere\n K: fmt::Debug,\n V: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_tuple(\"Entry\")\n .field(self.key())\n .field(self.value())\n .finish()\n }\n}\n\n/// An owning iterator over the entries of a `SkipMap`.\npub struct IntoIter {\n inner: base::IntoIter,\n}\n\nimpl Iterator for IntoIter {\n type Item = (K, V);\n\n fn next(&mut self) -> Option {\n self.inner.next()\n }\n}\n\nimpl fmt::Debug for IntoIter {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"IntoIter { .. }\")\n }\n}\n\n/// An iterator over the entries of a `SkipMap`.\npub struct Iter<'a, K, V, C = BasicComparator> {\n inner: base::RefIter<'a, K, V, C>,\n}\n\nimpl<'a, K, V, C> Iterator for Iter<'a, K, V, C>\nwhere\n C: Comparator,\n{\n type Item = Entry<'a, K, V, C>;\n\n fn next(&mut self) -> Option> {\n let guard = &epoch::pin();\n self.inner.next(guard).map(Entry::new)\n }\n}\n\nimpl<'a, K, V, C> DoubleEndedIterator for Iter<'a, K, V, C>\nwhere\n C: Comparator,\n{\n fn next_back(&mut self) -> Option> {\n let guard = &epoch::pin();\n self.inner.next_back(guard).map(Entry::new)\n }\n}\n\nimpl fmt::Debug for Iter<'_, K, V, C> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Iter { .. }\")\n }\n}\n\nimpl Drop for Iter<'_, K, V, C> {\n fn drop(&mut self) {\n let guard = &epoch::pin();\n self.inner.drop_impl(guard);\n }\n}\n\n/// An iterator over a subset of entries of a `SkipMap`.\npub struct Range<'a, Q, R, K, V, C = BasicComparator>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n pub(crate) inner: base::RefRange<'a, Q, R, K, V, C>,\n}\n\nimpl<'a, Q, R, K, V, C> Iterator for Range<'a, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n type Item = Entry<'a, K, V, C>;\n\n fn next(&mut self) -> Option> {\n let guard = &epoch::pin();\n self.inner.next(guard).map(Entry::new)\n }\n}\n\nimpl<'a, Q, R, K, V, C> DoubleEndedIterator for Range<'a, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n fn next_back(&mut self) -> Option> {\n let guard = &epoch::pin();\n self.inner.next_back(guard).map(Entry::new)\n }\n}\n\nimpl fmt::Debug for Range<'_, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n K: fmt::Debug,\n V: fmt::Debug,\n R: RangeBounds + fmt::Debug,\n Q: ?Sized,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"Range\")\n .field(\"range\", &self.inner.range)\n .field(\"head\", &self.inner.head)\n .field(\"tail\", &self.inner.tail)\n .finish()\n }\n}\n\nimpl Drop for Range<'_, Q, R, K, V, C>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n fn drop(&mut self) {\n let guard = &epoch::pin();\n self.inner.drop_impl(guard);\n }\n}\n" }, { "path": "crossbeam-skiplist/src/set.rs", "content": "//! A set based on a lock-free skip list. See [`SkipSet`].\n\nuse core::{\n fmt,\n ops::{Bound, Deref, RangeBounds},\n};\n\nuse crate::{\n comparator::{BasicComparator, Comparator},\n map,\n};\n\n/// A set based on a lock-free skip list.\n///\n/// This is an alternative to [`BTreeSet`] which supports\n/// concurrent access across multiple threads.\n///\n/// A custom comparator may be provided, causing all\n/// elements to be ordered by the comparison function used\n/// instead of the standard `Ord` impl. See [`Comparator`].\n///\n/// [`BTreeSet`]: std::collections::BTreeSet\n/// [`Comparator`]: crate::comparator::Comparator\npub struct SkipSet {\n inner: map::SkipMap,\n}\n\nimpl SkipSet {\n /// Returns a new, empty set with the default comparator.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set: SkipSet = SkipSet::new();\n /// ```\n pub fn new() -> Self {\n Self {\n inner: map::SkipMap::new(),\n }\n }\n}\n\nimpl SkipSet {\n /// Returns a new, empty set with the given comparator.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::{SkipSet, comparator::BasicComparator};\n ///\n /// let set: SkipSet = SkipSet::with_comparator(BasicComparator);\n /// ```\n pub fn with_comparator(comparator: C) -> Self {\n Self {\n inner: map::SkipMap::with_comparator(comparator),\n }\n }\n\n /// Returns `true` if the set is empty.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// assert!(set.is_empty());\n ///\n /// set.insert(1);\n /// assert!(!set.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n self.inner.is_empty()\n }\n\n /// Returns the number of entries in the set.\n ///\n /// If the set is being concurrently modified, consider the returned number just an\n /// approximation without any guarantees.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// assert_eq!(set.len(), 0);\n ///\n /// set.insert(1);\n /// assert_eq!(set.len(), 1);\n /// ```\n pub fn len(&self) -> usize {\n self.inner.len()\n }\n}\n\nimpl SkipSet\nwhere\n C: Comparator,\n{\n /// Returns the entry with the smallest key.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// set.insert(1);\n /// assert_eq!(*set.front().unwrap(), 1);\n /// set.insert(2);\n /// assert_eq!(*set.front().unwrap(), 1);\n /// ```\n pub fn front(&self) -> Option> {\n self.inner.front().map(Entry::new)\n }\n\n /// Returns the entry with the largest key.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// set.insert(1);\n /// assert_eq!(*set.back().unwrap(), 1);\n /// set.insert(2);\n /// assert_eq!(*set.back().unwrap(), 2);\n /// ```\n pub fn back(&self) -> Option> {\n self.inner.back().map(Entry::new)\n }\n\n /// Returns `true` if the set contains a value for the specified key.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set: SkipSet<_> = (1..=3).collect();\n /// assert!(set.contains(&1));\n /// assert!(!set.contains(&4));\n /// ```\n pub fn contains(&self, key: &Q) -> bool\n where\n C: Comparator,\n Q: ?Sized,\n {\n self.inner.contains_key(key)\n }\n\n /// Returns an entry with the specified `key`.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set: SkipSet<_> = (1..=3).collect();\n /// assert_eq!(*set.get(&3).unwrap(), 3);\n /// assert!(set.get(&4).is_none());\n /// ```\n pub fn get(&self, key: &Q) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n self.inner.get(key).map(Entry::new)\n }\n\n /// Returns an `Entry` pointing to the lowest element whose key is above\n /// the given bound. If no such element is found then `None` is\n /// returned.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n /// use std::ops::Bound::*;\n ///\n /// let set = SkipSet::new();\n /// set.insert(6);\n /// set.insert(7);\n /// set.insert(12);\n ///\n /// let greater_than_five = set.lower_bound(Excluded(&5)).unwrap();\n /// assert_eq!(*greater_than_five, 6);\n ///\n /// let greater_than_six = set.lower_bound(Excluded(&6)).unwrap();\n /// assert_eq!(*greater_than_six, 7);\n ///\n /// let greater_than_thirteen = set.lower_bound(Excluded(&13));\n /// assert!(greater_than_thirteen.is_none());\n /// ```\n pub fn lower_bound<'a, Q>(&'a self, bound: Bound<&Q>) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n self.inner.lower_bound(bound).map(Entry::new)\n }\n\n /// Returns an `Entry` pointing to the highest element whose key is below\n /// the given bound. If no such element is found then `None` is\n /// returned.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n /// use std::ops::Bound::*;\n ///\n /// let set = SkipSet::new();\n /// set.insert(6);\n /// set.insert(7);\n /// set.insert(12);\n ///\n /// let less_than_eight = set.upper_bound(Excluded(&8)).unwrap();\n /// assert_eq!(*less_than_eight, 7);\n ///\n /// let less_than_six = set.upper_bound(Excluded(&6));\n /// assert!(less_than_six.is_none());\n /// ```\n pub fn upper_bound<'a, Q>(&'a self, bound: Bound<&Q>) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n self.inner.upper_bound(bound).map(Entry::new)\n }\n\n /// Finds an entry with the specified key, or inserts a new `key`-`value` pair if none exist.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// let entry = set.get_or_insert(2);\n /// assert_eq!(*entry, 2);\n /// ```\n pub fn get_or_insert(&self, key: T) -> Entry<'_, T, C> {\n Entry::new(self.inner.get_or_insert(key, ()))\n }\n\n /// Returns an iterator over all entries in the set.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// set.insert(6);\n /// set.insert(7);\n /// set.insert(12);\n ///\n /// let mut set_iter = set.iter();\n /// assert_eq!(*set_iter.next().unwrap(), 6);\n /// assert_eq!(*set_iter.next().unwrap(), 7);\n /// assert_eq!(*set_iter.next().unwrap(), 12);\n /// assert!(set_iter.next().is_none());\n /// ```\n pub fn iter(&self) -> Iter<'_, T, C> {\n Iter {\n inner: self.inner.iter(),\n }\n }\n\n /// Returns an iterator over a subset of entries in the set.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// set.insert(6);\n /// set.insert(7);\n /// set.insert(12);\n ///\n /// let mut set_range = set.range(5..=8);\n /// assert_eq!(*set_range.next().unwrap(), 6);\n /// assert_eq!(*set_range.next().unwrap(), 7);\n /// assert!(set_range.next().is_none());\n /// ```\n pub fn range(&self, range: R) -> Range<'_, Q, R, T, C>\n where\n R: RangeBounds,\n C: Comparator,\n Q: ?Sized,\n {\n Range {\n inner: self.inner.range(range),\n }\n }\n}\n\nimpl SkipSet\nwhere\n C: Comparator,\n T: Send + 'static,\n{\n /// Inserts a `key`-`value` pair into the set and returns the new entry.\n ///\n /// If there is an existing entry with this key, it will be removed before inserting the new\n /// one.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// set.insert(2);\n /// assert_eq!(*set.get(&2).unwrap(), 2);\n /// ```\n pub fn insert(&self, key: T) -> Entry<'_, T, C> {\n Entry::new(self.inner.insert(key, ()))\n }\n\n /// Removes an entry with the specified key from the set and returns it.\n ///\n /// The value will not actually be dropped until all references to it have gone\n /// out of scope.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// set.insert(2);\n /// assert_eq!(*set.remove(&2).unwrap(), 2);\n /// assert!(set.remove(&2).is_none());\n /// ```\n pub fn remove(&self, key: &Q) -> Option>\n where\n C: Comparator,\n Q: ?Sized,\n {\n self.inner.remove(key).map(Entry::new)\n }\n\n /// Removes an entry from the front of the set.\n /// Returns the removed entry.\n ///\n /// The value will not actually be dropped until all references to it have gone\n /// out of scope.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// set.insert(1);\n /// set.insert(2);\n ///\n /// assert_eq!(*set.pop_front().unwrap(), 1);\n /// assert_eq!(*set.pop_front().unwrap(), 2);\n ///\n /// // All entries have been removed now.\n /// assert!(set.is_empty());\n /// ```\n pub fn pop_front(&self) -> Option> {\n self.inner.pop_front().map(Entry::new)\n }\n\n /// Removes an entry from the back of the set.\n /// Returns the removed entry.\n ///\n /// The value will not actually be dropped until all references to it have gone\n /// out of scope.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// set.insert(1);\n /// set.insert(2);\n ///\n /// assert_eq!(*set.pop_back().unwrap(), 2);\n /// assert_eq!(*set.pop_back().unwrap(), 1);\n ///\n /// // All entries have been removed now.\n /// assert!(set.is_empty());\n /// ```\n pub fn pop_back(&self) -> Option> {\n self.inner.pop_back().map(Entry::new)\n }\n\n /// Iterates over the set and removes every entry.\n ///\n /// # Example\n ///\n /// ```\n /// use crossbeam_skiplist::SkipSet;\n ///\n /// let set = SkipSet::new();\n /// set.insert(1);\n /// set.insert(2);\n ///\n /// set.clear();\n /// assert!(set.is_empty());\n /// ```\n pub fn clear(&self) {\n self.inner.clear();\n }\n}\n\nimpl Default for SkipSet\nwhere\n C: Default,\n{\n fn default() -> Self {\n Self::with_comparator(Default::default())\n }\n}\n\nimpl fmt::Debug for SkipSet\nwhere\n C: Comparator,\n T: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"SkipSet { .. }\")\n }\n}\n\nimpl IntoIterator for SkipSet {\n type Item = T;\n type IntoIter = IntoIter;\n\n fn into_iter(self) -> Self::IntoIter {\n IntoIter {\n inner: self.inner.into_iter(),\n }\n }\n}\n\nimpl<'a, T, C> IntoIterator for &'a SkipSet\nwhere\n C: Comparator,\n{\n type Item = Entry<'a, T, C>;\n type IntoIter = Iter<'a, T, C>;\n\n fn into_iter(self) -> Self::IntoIter {\n self.iter()\n }\n}\n\nimpl FromIterator for SkipSet\nwhere\n C: Comparator + Default,\n{\n fn from_iter(iter: I) -> Self\n where\n I: IntoIterator,\n {\n let s = Self::default();\n for t in iter {\n s.get_or_insert(t);\n }\n s\n }\n}\n\n/// A reference-counted entry in a set.\npub struct Entry<'a, T, C = BasicComparator> {\n inner: map::Entry<'a, T, (), C>,\n}\n\nimpl<'a, T, C> Entry<'a, T, C> {\n fn new(inner: map::Entry<'a, T, (), C>) -> Self {\n Self { inner }\n }\n\n /// Returns a reference to the value.\n pub fn value(&self) -> &'a T {\n self.inner.key()\n }\n\n /// Returns `true` if the entry is removed from the set.\n pub fn is_removed(&self) -> bool {\n self.inner.is_removed()\n }\n}\n\nimpl Entry<'_, T, C>\nwhere\n C: Comparator,\n{\n /// Moves to the next entry in the set.\n pub fn move_next(&mut self) -> bool {\n self.inner.move_next()\n }\n\n /// Moves to the previous entry in the set.\n pub fn move_prev(&mut self) -> bool {\n self.inner.move_prev()\n }\n\n /// Returns the next entry in the set.\n pub fn next(&self) -> Option {\n self.inner.next().map(Entry::new)\n }\n\n /// Returns the previous entry in the set.\n pub fn prev(&self) -> Option {\n self.inner.prev().map(Entry::new)\n }\n}\n\nimpl Entry<'_, T, C>\nwhere\n C: Comparator,\n T: Send + 'static,\n{\n /// Removes the entry from the set.\n ///\n /// Returns `true` if this call removed the entry and `false` if it was already removed.\n pub fn remove(&self) -> bool {\n self.inner.remove()\n }\n}\n\nimpl Clone for Entry<'_, T, C> {\n fn clone(&self) -> Self {\n Self {\n inner: self.inner.clone(),\n }\n }\n}\n\nimpl fmt::Debug for Entry<'_, T, C>\nwhere\n T: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"Entry\")\n .field(\"value\", self.value())\n .finish()\n }\n}\n\nimpl Deref for Entry<'_, T, C> {\n type Target = T;\n\n fn deref(&self) -> &Self::Target {\n self.value()\n }\n}\n\n/// An owning iterator over the entries of a `SkipSet`.\npub struct IntoIter {\n inner: map::IntoIter,\n}\n\nimpl Iterator for IntoIter {\n type Item = T;\n\n fn next(&mut self) -> Option {\n self.inner.next().map(|(k, ())| k)\n }\n}\n\nimpl fmt::Debug for IntoIter {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"IntoIter { .. }\")\n }\n}\n\n/// An iterator over the entries of a `SkipSet`.\npub struct Iter<'a, T, C = BasicComparator> {\n inner: map::Iter<'a, T, (), C>,\n}\n\nimpl<'a, T, C> Iterator for Iter<'a, T, C>\nwhere\n C: Comparator,\n{\n type Item = Entry<'a, T, C>;\n\n fn next(&mut self) -> Option {\n self.inner.next().map(Entry::new)\n }\n}\n\nimpl DoubleEndedIterator for Iter<'_, T, C>\nwhere\n C: Comparator,\n{\n fn next_back(&mut self) -> Option {\n self.inner.next_back().map(Entry::new)\n }\n}\n\nimpl fmt::Debug for Iter<'_, T, C> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Iter { .. }\")\n }\n}\n\n/// An iterator over a subset of entries of a `SkipSet`.\npub struct Range<'a, Q, R, T, C = BasicComparator>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n inner: map::Range<'a, Q, R, T, (), C>,\n}\n\nimpl<'a, Q, R, T, C> Iterator for Range<'a, Q, R, T, C>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n type Item = Entry<'a, T, C>;\n\n fn next(&mut self) -> Option {\n self.inner.next().map(Entry::new)\n }\n}\n\nimpl DoubleEndedIterator for Range<'_, Q, R, T, C>\nwhere\n C: Comparator + Comparator,\n R: RangeBounds,\n Q: ?Sized,\n{\n fn next_back(&mut self) -> Option {\n self.inner.next_back().map(Entry::new)\n }\n}\n\nimpl fmt::Debug for Range<'_, Q, R, T, C>\nwhere\n C: Comparator + Comparator,\n T: fmt::Debug,\n R: RangeBounds + fmt::Debug,\n Q: ?Sized,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"Range\")\n .field(\"range\", &self.inner.inner.range)\n .field(\"head\", &self.inner.inner.head.as_ref().map(|e| e.key()))\n .field(\"tail\", &self.inner.inner.tail.as_ref().map(|e| e.key()))\n .finish()\n }\n}\n" }, { "path": "crossbeam-skiplist/tests/base.rs", "content": "#![allow(clippy::redundant_clone)]\n\nuse std::{\n ops::Bound,\n sync::atomic::{AtomicUsize, Ordering},\n};\n\nuse crossbeam_epoch as epoch;\nuse crossbeam_skiplist::{SkipList, base};\n\nfn ref_entry<'a, K, V>(e: impl Into>>) -> Entry<'a, K, V> {\n Entry(e.into())\n}\nstruct Entry<'a, K, V>(Option>);\nimpl Entry<'_, K, V> {\n fn value(&self) -> &V {\n self.0.as_ref().unwrap().value()\n }\n}\nimpl Drop for Entry<'_, K, V> {\n fn drop(&mut self) {\n if let Some(e) = self.0.take() {\n e.release_with_pin(epoch::pin)\n }\n }\n}\n\n#[test]\nfn new() {\n SkipList::::new(epoch::default_collector().clone());\n SkipList::>::new(epoch::default_collector().clone());\n}\n\n#[test]\nfn is_empty() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n assert!(s.is_empty());\n\n s.insert(1, 10, guard).release(guard);\n assert!(!s.is_empty());\n s.insert(2, 20, guard).release(guard);\n s.insert(3, 30, guard).release(guard);\n assert!(!s.is_empty());\n\n s.remove(&2, guard).unwrap().release(guard);\n assert!(!s.is_empty());\n\n s.remove(&1, guard).unwrap().release(guard);\n assert!(!s.is_empty());\n\n s.remove(&3, guard).unwrap().release(guard);\n assert!(s.is_empty());\n}\n\n#[test]\nfn insert() {\n let guard = &epoch::pin();\n let insert = [0, 4, 2, 12, 8, 7, 11, 5];\n let not_present = [1, 3, 6, 9, 10];\n let s = SkipList::new(epoch::default_collector().clone());\n\n for &x in &insert {\n s.insert(x, x * 10, guard).release(guard);\n assert_eq!(*s.get(&x, guard).unwrap().value(), x * 10);\n }\n\n for &x in ¬_present {\n assert!(s.get(&x, guard).is_none());\n }\n}\n\n#[test]\nfn remove() {\n let guard = &epoch::pin();\n let insert = [0, 4, 2, 12, 8, 7, 11, 5];\n let not_present = [1, 3, 6, 9, 10];\n let remove = [2, 12, 8];\n let remaining = [0, 4, 5, 7, 11];\n\n let s = SkipList::new(epoch::default_collector().clone());\n\n for &x in &insert {\n s.insert(x, x * 10, guard).release(guard);\n }\n for x in ¬_present {\n assert!(s.remove(x, guard).is_none());\n }\n for x in &remove {\n s.remove(x, guard).unwrap().release(guard);\n }\n\n let mut v = vec![];\n let mut e = s.front(guard).unwrap();\n loop {\n v.push(*e.key());\n if !e.move_next() {\n break;\n }\n }\n\n assert_eq!(v, remaining);\n for x in &insert {\n ref_entry(s.remove(x, guard));\n }\n assert!(s.is_empty());\n}\n\n#[test]\nfn entry() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n\n assert!(s.front(guard).is_none());\n assert!(s.back(guard).is_none());\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10, guard).release(guard);\n }\n\n let mut e = s.front(guard).unwrap();\n assert_eq!(*e.key(), 2);\n assert!(!e.move_prev());\n assert!(e.move_next());\n assert_eq!(*e.key(), 4);\n\n e = s.back(guard).unwrap();\n assert_eq!(*e.key(), 12);\n assert!(!e.move_next());\n assert!(e.move_prev());\n assert_eq!(*e.key(), 11);\n}\n\n#[test]\nfn entry_remove() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10, guard).release(guard);\n }\n\n let mut e = s.get(&7, guard).unwrap();\n assert!(!e.is_removed());\n assert!(e.remove());\n assert!(e.is_removed());\n\n e.move_prev();\n e.move_next();\n assert_ne!(*e.key(), 7);\n\n for e in s.iter(guard) {\n assert!(!s.is_empty());\n assert_ne!(s.len(), 0);\n e.remove();\n }\n assert!(s.is_empty());\n assert_eq!(s.len(), 0);\n}\n\n#[test]\nfn entry_reposition() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10, guard).release(guard);\n }\n\n let mut e = s.get(&7, guard).unwrap();\n assert!(!e.is_removed());\n assert!(e.remove());\n assert!(e.is_removed());\n\n s.insert(7, 700, guard).release(guard);\n e.move_prev();\n e.move_next();\n assert_eq!(*e.key(), 7);\n}\n\n#[test]\nfn len() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n assert_eq!(s.len(), 0);\n\n for (i, &x) in [4, 2, 12, 8, 7, 11, 5].iter().enumerate() {\n s.insert(x, x * 10, guard).release(guard);\n assert_eq!(s.len(), i + 1);\n }\n\n s.insert(5, 0, guard).release(guard);\n assert_eq!(s.len(), 7);\n s.insert(5, 0, guard).release(guard);\n assert_eq!(s.len(), 7);\n\n assert!(s.remove(&6, guard).is_none());\n assert_eq!(s.len(), 7);\n s.remove(&5, guard).unwrap().release(guard);\n assert_eq!(s.len(), 6);\n s.remove(&12, guard).unwrap().release(guard);\n assert_eq!(s.len(), 5);\n}\n\n#[test]\nfn insert_and_remove() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n let keys = || s.iter(guard).map(|e| *e.key()).collect::>();\n\n s.insert(3, 0, guard).release(guard);\n s.insert(5, 0, guard).release(guard);\n s.insert(1, 0, guard).release(guard);\n s.insert(4, 0, guard).release(guard);\n s.insert(2, 0, guard).release(guard);\n assert_eq!(keys(), [1, 2, 3, 4, 5]);\n\n s.remove(&4, guard).unwrap().release(guard);\n assert_eq!(keys(), [1, 2, 3, 5]);\n s.remove(&3, guard).unwrap().release(guard);\n assert_eq!(keys(), [1, 2, 5]);\n s.remove(&1, guard).unwrap().release(guard);\n assert_eq!(keys(), [2, 5]);\n\n assert!(s.remove(&1, guard).is_none());\n assert_eq!(keys(), [2, 5]);\n assert!(s.remove(&3, guard).is_none());\n assert_eq!(keys(), [2, 5]);\n\n s.remove(&2, guard).unwrap().release(guard);\n assert_eq!(keys(), [5]);\n s.remove(&5, guard).unwrap().release(guard);\n assert_eq!(keys(), []);\n\n s.insert(3, 0, guard).release(guard);\n assert_eq!(keys(), [3]);\n s.insert(1, 0, guard).release(guard);\n assert_eq!(keys(), [1, 3]);\n s.insert(3, 0, guard).release(guard);\n assert_eq!(keys(), [1, 3]);\n s.insert(5, 0, guard).release(guard);\n assert_eq!(keys(), [1, 3, 5]);\n\n s.remove(&3, guard).unwrap().release(guard);\n assert_eq!(keys(), [1, 5]);\n s.remove(&1, guard).unwrap().release(guard);\n assert_eq!(keys(), [5]);\n assert!(s.remove(&3, guard).is_none());\n assert_eq!(keys(), [5]);\n s.remove(&5, guard).unwrap().release(guard);\n assert_eq!(keys(), []);\n}\n\n#[test]\nfn get() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n s.insert(30, 3, guard).release(guard);\n s.insert(50, 5, guard).release(guard);\n s.insert(10, 1, guard).release(guard);\n s.insert(40, 4, guard).release(guard);\n s.insert(20, 2, guard).release(guard);\n\n assert_eq!(*s.get(&10, guard).unwrap().value(), 1);\n assert_eq!(*s.get(&20, guard).unwrap().value(), 2);\n assert_eq!(*s.get(&30, guard).unwrap().value(), 3);\n assert_eq!(*s.get(&40, guard).unwrap().value(), 4);\n assert_eq!(*s.get(&50, guard).unwrap().value(), 5);\n\n assert!(s.get(&7, guard).is_none());\n assert!(s.get(&27, guard).is_none());\n assert!(s.get(&31, guard).is_none());\n assert!(s.get(&97, guard).is_none());\n}\n\n#[test]\nfn lower_bound() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n s.insert(30, 3, guard).release(guard);\n s.insert(50, 5, guard).release(guard);\n s.insert(10, 1, guard).release(guard);\n s.insert(40, 4, guard).release(guard);\n s.insert(20, 2, guard).release(guard);\n\n assert_eq!(*s.lower_bound(Bound::Unbounded, guard).unwrap().value(), 1);\n\n assert_eq!(\n *s.lower_bound(Bound::Included(&10), guard).unwrap().value(),\n 1\n );\n assert_eq!(\n *s.lower_bound(Bound::Included(&20), guard).unwrap().value(),\n 2\n );\n assert_eq!(\n *s.lower_bound(Bound::Included(&30), guard).unwrap().value(),\n 3\n );\n assert_eq!(\n *s.lower_bound(Bound::Included(&40), guard).unwrap().value(),\n 4\n );\n assert_eq!(\n *s.lower_bound(Bound::Included(&50), guard).unwrap().value(),\n 5\n );\n\n assert_eq!(\n *s.lower_bound(Bound::Included(&7), guard).unwrap().value(),\n 1\n );\n assert_eq!(\n *s.lower_bound(Bound::Included(&27), guard).unwrap().value(),\n 3\n );\n assert_eq!(\n *s.lower_bound(Bound::Included(&31), guard).unwrap().value(),\n 4\n );\n assert!(s.lower_bound(Bound::Included(&97), guard).is_none());\n\n assert_eq!(\n *s.lower_bound(Bound::Excluded(&10), guard).unwrap().value(),\n 2\n );\n assert_eq!(\n *s.lower_bound(Bound::Excluded(&20), guard).unwrap().value(),\n 3\n );\n assert_eq!(\n *s.lower_bound(Bound::Excluded(&30), guard).unwrap().value(),\n 4\n );\n assert_eq!(\n *s.lower_bound(Bound::Excluded(&40), guard).unwrap().value(),\n 5\n );\n assert!(s.lower_bound(Bound::Excluded(&50), guard).is_none());\n\n assert_eq!(\n *s.lower_bound(Bound::Excluded(&7), guard).unwrap().value(),\n 1\n );\n assert_eq!(\n *s.lower_bound(Bound::Excluded(&27), guard).unwrap().value(),\n 3\n );\n assert_eq!(\n *s.lower_bound(Bound::Excluded(&31), guard).unwrap().value(),\n 4\n );\n assert!(s.lower_bound(Bound::Excluded(&97), guard).is_none());\n}\n\n#[test]\nfn upper_bound() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n s.insert(30, 3, guard).release(guard);\n s.insert(50, 5, guard).release(guard);\n s.insert(10, 1, guard).release(guard);\n s.insert(40, 4, guard).release(guard);\n s.insert(20, 2, guard).release(guard);\n\n assert_eq!(*s.upper_bound(Bound::Unbounded, guard).unwrap().value(), 5);\n\n assert_eq!(\n *s.upper_bound(Bound::Included(&10), guard).unwrap().value(),\n 1\n );\n assert_eq!(\n *s.upper_bound(Bound::Included(&20), guard).unwrap().value(),\n 2\n );\n assert_eq!(\n *s.upper_bound(Bound::Included(&30), guard).unwrap().value(),\n 3\n );\n assert_eq!(\n *s.upper_bound(Bound::Included(&40), guard).unwrap().value(),\n 4\n );\n assert_eq!(\n *s.upper_bound(Bound::Included(&50), guard).unwrap().value(),\n 5\n );\n\n assert!(s.upper_bound(Bound::Included(&7), guard).is_none());\n assert_eq!(\n *s.upper_bound(Bound::Included(&27), guard).unwrap().value(),\n 2\n );\n assert_eq!(\n *s.upper_bound(Bound::Included(&31), guard).unwrap().value(),\n 3\n );\n assert_eq!(\n *s.upper_bound(Bound::Included(&97), guard).unwrap().value(),\n 5\n );\n\n assert!(s.upper_bound(Bound::Excluded(&10), guard).is_none());\n assert_eq!(\n *s.upper_bound(Bound::Excluded(&20), guard).unwrap().value(),\n 1\n );\n assert_eq!(\n *s.upper_bound(Bound::Excluded(&30), guard).unwrap().value(),\n 2\n );\n assert_eq!(\n *s.upper_bound(Bound::Excluded(&40), guard).unwrap().value(),\n 3\n );\n assert_eq!(\n *s.upper_bound(Bound::Excluded(&50), guard).unwrap().value(),\n 4\n );\n\n assert!(s.upper_bound(Bound::Excluded(&7), guard).is_none());\n assert_eq!(\n *s.upper_bound(Bound::Excluded(&27), guard).unwrap().value(),\n 2\n );\n assert_eq!(\n *s.upper_bound(Bound::Excluded(&31), guard).unwrap().value(),\n 3\n );\n assert_eq!(\n *s.upper_bound(Bound::Excluded(&97), guard).unwrap().value(),\n 5\n );\n}\n\n#[test]\nfn get_or_insert() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n s.insert(3, 3, guard).release(guard);\n s.insert(5, 5, guard).release(guard);\n s.insert(1, 1, guard).release(guard);\n s.insert(4, 4, guard).release(guard);\n s.insert(2, 2, guard).release(guard);\n\n assert_eq!(*s.get(&4, guard).unwrap().value(), 4);\n assert_eq!(*ref_entry(s.insert(4, 40, guard)).value(), 40);\n assert_eq!(*s.get(&4, guard).unwrap().value(), 40);\n\n assert_eq!(*s.get_or_insert(4, 400, guard).value(), 40);\n assert_eq!(*s.get(&4, guard).unwrap().value(), 40);\n assert_eq!(*s.get_or_insert(6, 600, guard).value(), 600);\n}\n\n#[test]\nfn get_or_insert_with() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n s.insert(3, 3, guard).release(guard);\n s.insert(5, 5, guard).release(guard);\n s.insert(1, 1, guard).release(guard);\n s.insert(4, 4, guard).release(guard);\n s.insert(2, 2, guard).release(guard);\n\n assert_eq!(*s.get(&4, guard).unwrap().value(), 4);\n assert_eq!(*ref_entry(s.insert(4, 40, guard)).value(), 40);\n assert_eq!(*s.get(&4, guard).unwrap().value(), 40);\n\n assert_eq!(*s.get_or_insert_with(4, || 400, guard).value(), 40);\n assert_eq!(*s.get(&4, guard).unwrap().value(), 40);\n assert_eq!(*s.get_or_insert_with(6, || 600, guard).value(), 600);\n}\n\n#[test]\nfn get_or_insert_with_panic() {\n use std::panic;\n\n let s = SkipList::new(epoch::default_collector().clone());\n let res = panic::catch_unwind(panic::AssertUnwindSafe(|| {\n let guard = &epoch::pin();\n s.get_or_insert_with(4, || panic!(), guard);\n }));\n assert!(res.is_err());\n assert!(s.is_empty());\n let guard = &epoch::pin();\n assert_eq!(*s.get_or_insert_with(4, || 40, guard).value(), 40);\n assert_eq!(s.len(), 1);\n}\n\n#[test]\nfn get_or_insert_with_parallel_run() {\n use std::sync::{Arc, Mutex};\n\n let s = Arc::new(SkipList::new(epoch::default_collector().clone()));\n let s2 = s.clone();\n let called = Arc::new(Mutex::new(false));\n let called2 = called.clone();\n let handle = std::thread::spawn(move || {\n let guard = &epoch::pin();\n assert_eq!(\n *s2.get_or_insert_with(\n 7,\n || {\n *called2.lock().unwrap() = true;\n\n // allow main thread to run before we return result\n std::thread::sleep(std::time::Duration::from_secs(4));\n 70\n },\n guard,\n )\n .value(),\n 700\n );\n });\n std::thread::sleep(std::time::Duration::from_secs(2));\n let guard = &epoch::pin();\n\n // main thread writes the value first\n assert_eq!(*s.get_or_insert(7, 700, guard).value(), 700);\n handle.join().unwrap();\n assert!(*called.lock().unwrap());\n}\n\n#[test]\nfn get_next_prev() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n s.insert(3, 3, guard).release(guard);\n s.insert(5, 5, guard).release(guard);\n s.insert(1, 1, guard).release(guard);\n s.insert(4, 4, guard).release(guard);\n s.insert(2, 2, guard).release(guard);\n\n let mut e = s.get(&3, guard).unwrap();\n assert_eq!(*e.next().unwrap().value(), 4);\n assert_eq!(*e.prev().unwrap().value(), 2);\n assert_eq!(*e.value(), 3);\n\n e.move_prev();\n assert_eq!(*e.next().unwrap().value(), 3);\n assert_eq!(*e.prev().unwrap().value(), 1);\n assert_eq!(*e.value(), 2);\n\n e.move_prev();\n assert_eq!(*e.next().unwrap().value(), 2);\n assert!(e.prev().is_none());\n assert_eq!(*e.value(), 1);\n\n e.move_next();\n e.move_next();\n e.move_next();\n e.move_next();\n assert!(e.next().is_none());\n assert_eq!(*e.prev().unwrap().value(), 4);\n assert_eq!(*e.value(), 5);\n}\n\n#[test]\nfn front_and_back() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n assert!(s.front(guard).is_none());\n assert!(s.back(guard).is_none());\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10, guard);\n }\n\n assert_eq!(*s.front(guard).unwrap().key(), 2);\n assert_eq!(*s.back(guard).unwrap().key(), 12);\n}\n\n#[test]\nfn iter() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10, guard).release(guard);\n }\n\n assert_eq!(\n s.iter(guard).map(|e| *e.key()).collect::>(),\n &[2, 4, 5, 7, 8, 11, 12]\n );\n\n let mut it = s.iter(guard);\n s.remove(&2, guard).unwrap().release(guard);\n assert_eq!(*it.next().unwrap().key(), 4);\n s.remove(&7, guard).unwrap().release(guard);\n assert_eq!(*it.next().unwrap().key(), 5);\n s.remove(&5, guard).unwrap().release(guard);\n assert_eq!(*it.next().unwrap().key(), 8);\n s.remove(&12, guard).unwrap().release(guard);\n assert_eq!(*it.next().unwrap().key(), 11);\n assert!(it.next().is_none());\n}\n\n#[test]\nfn iter_range() {\n use crate::Bound::*;\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n let v = (0..10).map(|x| x * 10).collect::>();\n for &x in v.iter() {\n s.insert(x, x, guard).release(guard);\n }\n\n assert_eq!(\n s.iter(guard).map(|x| *x.value()).collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.iter(guard).rev().map(|x| *x.value()).collect::>(),\n vec![90, 80, 70, 60, 50, 40, 30, 20, 10, 0]\n );\n assert_eq!(\n s.range(.., guard).map(|x| *x.value()).collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n\n assert_eq!(\n s.range((Included(&0), Unbounded), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Excluded(&0), Unbounded), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Included(&25), Unbounded), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Excluded(&25), Unbounded), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Included(&70), Unbounded), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![70, 80, 90]\n );\n assert_eq!(\n s.range((Excluded(&70), Unbounded), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![80, 90]\n );\n assert_eq!(\n s.range((Included(&100), Unbounded), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&100), Unbounded), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Unbounded, Included(&90)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&90)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80]\n );\n assert_eq!(\n s.range((Unbounded, Included(&25)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&25)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20]\n );\n assert_eq!(\n s.range((Unbounded, Included(&70)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&70)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60]\n );\n assert_eq!(\n s.range((Unbounded, Included(&-1)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&-1)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Included(&25), Included(&80)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80]\n );\n assert_eq!(\n s.range((Included(&25), Excluded(&80)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70]\n );\n assert_eq!(\n s.range((Excluded(&25), Included(&80)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80]\n );\n assert_eq!(\n s.range((Excluded(&25), Excluded(&80)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70]\n );\n\n assert_eq!(\n s.range((Included(&25), Included(&25)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Included(&25), Excluded(&25)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&25), Included(&25)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&25), Excluded(&25)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Included(&50), Included(&50)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![50]\n );\n assert_eq!(\n s.range((Included(&50), Excluded(&50)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&50), Included(&50)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&50), Excluded(&50)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Included(&100), Included(&-2)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Included(&100), Excluded(&-2)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&100), Included(&-2)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&100), Excluded(&-2)), guard)\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n}\n\n#[test]\nfn into_iter() {\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10, guard).release(guard);\n }\n\n assert_eq!(\n s.into_iter().collect::>(),\n &[\n (2, 20),\n (4, 40),\n (5, 50),\n (7, 70),\n (8, 80),\n (11, 110),\n (12, 120),\n ]\n );\n}\n\n#[test]\nfn clear() {\n let guard = &mut epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10, guard).release(guard);\n }\n\n assert!(!s.is_empty());\n assert_ne!(s.len(), 0);\n s.clear(guard);\n assert!(s.is_empty());\n assert_eq!(s.len(), 0);\n}\n\n#[test]\nfn drops() {\n static KEYS: AtomicUsize = AtomicUsize::new(0);\n static VALUES: AtomicUsize = AtomicUsize::new(0);\n\n let collector = epoch::Collector::new();\n let handle = collector.register();\n {\n let guard = &handle.pin();\n\n #[derive(Eq, PartialEq, Ord, PartialOrd)]\n struct Key(i32);\n\n impl Drop for Key {\n fn drop(&mut self) {\n KEYS.fetch_add(1, Ordering::SeqCst);\n }\n }\n\n struct Value;\n\n impl Drop for Value {\n fn drop(&mut self) {\n VALUES.fetch_add(1, Ordering::SeqCst);\n }\n }\n\n let s = SkipList::new(collector.clone());\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(Key(x), Value, guard).release(guard);\n }\n assert_eq!(KEYS.load(Ordering::SeqCst), 0);\n assert_eq!(VALUES.load(Ordering::SeqCst), 0);\n\n let key7 = Key(7);\n s.remove(&key7, guard).unwrap().release(guard);\n assert_eq!(KEYS.load(Ordering::SeqCst), 0);\n assert_eq!(VALUES.load(Ordering::SeqCst), 0);\n\n drop(s);\n }\n\n handle.pin().flush();\n handle.pin().flush();\n assert_eq!(KEYS.load(Ordering::SeqCst), 8);\n assert_eq!(VALUES.load(Ordering::SeqCst), 7);\n}\n\n#[test]\nfn comparable_get() {\n use crossbeam_skiplist::equivalent::{Comparable, Equivalent};\n\n #[derive(PartialEq, Eq, PartialOrd, Ord)]\n struct Foo {\n a: u64,\n b: u32,\n }\n\n #[derive(PartialEq, Eq)]\n struct FooRef<'a> {\n data: &'a [u8],\n }\n\n impl PartialOrd for FooRef<'_> {\n fn partial_cmp(&self, other: &Self) -> Option {\n Some(self.cmp(other))\n }\n }\n\n impl Ord for FooRef<'_> {\n fn cmp(&self, other: &Self) -> std::cmp::Ordering {\n let a = u64::from_be_bytes(self.data[..8].try_into().unwrap());\n let b = u32::from_be_bytes(self.data[8..].try_into().unwrap());\n let other_a = u64::from_be_bytes(other.data[..8].try_into().unwrap());\n let other_b = u32::from_be_bytes(other.data[8..].try_into().unwrap());\n Foo { a, b }.cmp(&Foo {\n a: other_a,\n b: other_b,\n })\n }\n }\n\n impl<'a> Equivalent> for Foo {\n fn equivalent(&self, key: &FooRef<'a>) -> bool {\n let a = u64::from_be_bytes(key.data[..8].try_into().unwrap());\n let b = u32::from_be_bytes(key.data[8..].try_into().unwrap());\n a == self.a && b == self.b\n }\n }\n\n impl<'a> Comparable> for Foo {\n fn compare(&self, key: &FooRef<'a>) -> std::cmp::Ordering {\n let a = u64::from_be_bytes(key.data[..8].try_into().unwrap());\n let b = u32::from_be_bytes(key.data[8..].try_into().unwrap());\n Self { a, b }.cmp(self)\n }\n }\n\n let s = SkipList::new(epoch::default_collector().clone());\n let foo = Foo { a: 1, b: 2 };\n\n let g = &epoch::pin();\n s.insert(foo, 12, g);\n\n let buf = 1u64\n .to_be_bytes()\n .iter()\n .chain(2u32.to_be_bytes().iter())\n .copied()\n .collect::>();\n let foo_ref = FooRef { data: &buf };\n\n let ent = s.get(&foo_ref, g).unwrap();\n assert_eq!(ent.key().a, 1);\n assert_eq!(ent.key().b, 2);\n assert_eq!(*ent.value(), 12);\n}\n\n// https://github.com/crossbeam-rs/crossbeam/pull/1143\n#[test]\nfn remove_race() {\n use std::sync::atomic::{AtomicU32, Ordering};\n\n const NTHREADS: u32 = 16;\n const KEY_RANGE: u32 = if cfg!(miri) { 100 } else { 100_000 };\n\n let guard = &epoch::pin();\n let s = SkipList::new(epoch::default_collector().clone());\n\n for x in 0..KEY_RANGE {\n s.insert(x, (), guard).release(guard);\n }\n\n let barrier1 = AtomicU32::new(NTHREADS);\n let barrier2 = AtomicU32::new(NTHREADS);\n let mut total_removed = AtomicU32::new(0);\n\n std::thread::scope(|scope| {\n for _ in 0..NTHREADS {\n scope.spawn(|| {\n let guard = &epoch::pin();\n let mut removed_entries = Vec::with_capacity(KEY_RANGE as usize);\n\n barrier1.fetch_sub(1, Ordering::Relaxed);\n while barrier1.load(Ordering::Acquire) != 0 {}\n\n for x in 0..KEY_RANGE {\n if let Some(entry) = s.remove(&x, guard) {\n removed_entries.push(entry);\n }\n }\n\n barrier2.fetch_sub(1, Ordering::Relaxed);\n while barrier2.load(Ordering::Acquire) != 0 {}\n\n total_removed.fetch_add(removed_entries.len() as u32, Ordering::Relaxed);\n\n for entry in removed_entries.drain(..) {\n entry.release(guard);\n }\n });\n }\n });\n\n assert_eq!(*total_removed.get_mut(), KEY_RANGE);\n}\n" }, { "path": "crossbeam-skiplist/tests/map.rs", "content": "use std::{\n borrow::Borrow,\n iter,\n ops::Bound,\n sync::{Arc, Barrier},\n};\n\nuse crossbeam_skiplist::SkipMap;\nuse crossbeam_utils::thread;\n\n#[test]\nfn smoke() {\n let m = SkipMap::new();\n m.insert(1, 10);\n m.insert(5, 50);\n m.insert(7, 70);\n}\n\n#[test]\nfn is_empty() {\n let s = SkipMap::new();\n assert!(s.is_empty());\n\n s.insert(1, 10);\n assert!(!s.is_empty());\n s.insert(2, 20);\n s.insert(3, 30);\n assert!(!s.is_empty());\n\n s.remove(&2);\n assert!(!s.is_empty());\n\n s.remove(&1);\n assert!(!s.is_empty());\n\n s.remove(&3);\n assert!(s.is_empty());\n}\n\n#[test]\nfn insert() {\n let insert = [0, 4, 2, 12, 8, 7, 11, 5];\n let not_present = [1, 3, 6, 9, 10];\n let s = SkipMap::new();\n\n for &x in &insert {\n s.insert(x, x * 10);\n assert_eq!(*s.get(&x).unwrap().value(), x * 10);\n }\n\n for &x in ¬_present {\n assert!(s.get(&x).is_none());\n }\n}\n\n#[test]\nfn compare_and_insert() {\n let insert = [0, 4, 2, 12, 8, 7, 11, 5];\n let not_present = [1, 3, 6, 9, 10];\n let s = SkipMap::new();\n\n for &x in &insert {\n s.insert(x, x * 10);\n }\n\n for &x in &insert {\n let value = x * 5;\n let old_value = *s.get(&x).unwrap().value();\n s.compare_insert(x, value, |x| x < &value);\n assert_eq!(*s.get(&x).unwrap().value(), old_value);\n }\n\n for &x in &insert {\n let value = x * 15;\n s.compare_insert(x, value, |x| x < &value);\n assert_eq!(*s.get(&x).unwrap().value(), value);\n }\n\n for &x in ¬_present {\n assert!(s.get(&x).is_none());\n }\n}\n\n#[test]\nfn compare_insert_with_absent_key() {\n let insert = [0, 4, 2, 12, 8, 7, 11, 5];\n let s = SkipMap::new();\n\n // The closure will not be called if the key is not present,\n // so the key-value will be inserted into the map\n for &x in &insert {\n let value = x * 15;\n s.compare_insert(x, value, |_| false);\n assert_eq!(*s.get(&x).unwrap().value(), value);\n }\n}\n\n#[test]\nfn remove() {\n let insert = [0, 4, 2, 12, 8, 7, 11, 5];\n let not_present = [1, 3, 6, 9, 10];\n let remove = [2, 12, 8];\n let remaining = [0, 4, 5, 7, 11];\n\n let s = SkipMap::new();\n\n for &x in &insert {\n s.insert(x, x * 10);\n }\n for x in ¬_present {\n assert!(s.remove(x).is_none());\n }\n for x in &remove {\n assert!(s.remove(x).is_some());\n }\n\n let mut v = vec![];\n let mut e = s.front().unwrap();\n loop {\n v.push(*e.key());\n if !e.move_next() {\n break;\n }\n }\n\n assert_eq!(v, remaining);\n for x in &insert {\n s.remove(x);\n }\n assert!(s.is_empty());\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/672\n#[test]\nfn concurrent_insert() {\n for _ in 0..100 {\n let set: SkipMap = iter::once((1, 1)).collect();\n let barrier = Barrier::new(2);\n thread::scope(|s| {\n s.spawn(|_| {\n barrier.wait();\n set.insert(1, 1);\n });\n s.spawn(|_| {\n barrier.wait();\n set.insert(1, 1);\n });\n })\n .unwrap();\n }\n}\n\n#[test]\nfn concurrent_compare_and_insert() {\n let set: SkipMap = SkipMap::new();\n let set = std::sync::Arc::new(set);\n let len = 100;\n let mut handlers = Vec::with_capacity(len as usize);\n for i in 0..len {\n let set = set.clone();\n let handler = std::thread::spawn(move || {\n set.compare_insert(1, i, |j| j < &i);\n });\n handlers.push(handler);\n }\n for handler in handlers {\n handler.join().unwrap();\n }\n assert_eq!(*set.get(&1).unwrap().value(), len - 1);\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/672\n#[test]\nfn concurrent_remove() {\n for _ in 0..100 {\n let set: SkipMap = iter::once((1, 1)).collect();\n let barrier = Barrier::new(2);\n thread::scope(|s| {\n s.spawn(|_| {\n barrier.wait();\n set.remove(&1);\n });\n s.spawn(|_| {\n barrier.wait();\n set.remove(&1);\n });\n })\n .unwrap();\n }\n}\n\n#[test]\nfn next_memory_leak() {\n let map: SkipMap = iter::once((1, 1)).collect();\n let mut iter = map.iter();\n let e = iter.next_back();\n assert!(e.is_some());\n let e = iter.next();\n assert!(e.is_none());\n map.remove(&1);\n}\n\n#[test]\nfn next_back_memory_leak() {\n let map: SkipMap = iter::once((1, 1)).collect();\n map.insert(1, 1);\n let mut iter = map.iter();\n let e = iter.next();\n assert!(e.is_some());\n let e = iter.next_back();\n assert!(e.is_none());\n map.remove(&1);\n}\n\n#[test]\nfn range_next_memory_leak() {\n let map: SkipMap = iter::once((1, 1)).collect();\n let mut iter = map.range(0..);\n let e = iter.next();\n assert!(e.is_some());\n let e = iter.next_back();\n assert!(e.is_none());\n map.remove(&1);\n}\n\n#[test]\nfn entry() {\n let s = SkipMap::new();\n\n assert!(s.front().is_none());\n assert!(s.back().is_none());\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10);\n }\n\n let mut e = s.front().unwrap();\n assert_eq!(*e.key(), 2);\n assert!(!e.move_prev());\n assert!(e.move_next());\n assert_eq!(*e.key(), 4);\n\n e = s.back().unwrap();\n assert_eq!(*e.key(), 12);\n assert!(!e.move_next());\n assert!(e.move_prev());\n assert_eq!(*e.key(), 11);\n}\n\n#[test]\nfn ordered_iter() {\n let s: SkipMap = SkipMap::new();\n s.insert(1, 1);\n\n let mut iter = s.iter();\n assert!(iter.next().is_some());\n assert!(iter.next().is_none());\n assert!(iter.next().is_none());\n\n s.insert(2, 2);\n assert!(iter.next().is_some());\n assert!(iter.next().is_none());\n assert!(iter.next().is_none());\n\n s.insert(3, 3);\n s.insert(4, 4);\n s.insert(5, 5);\n assert_eq!(*iter.next_back().unwrap().key(), 5);\n assert_eq!(*iter.next().unwrap().key(), 3);\n assert_eq!(*iter.next_back().unwrap().key(), 4);\n assert!(iter.next().is_none());\n assert!(iter.next_back().is_none());\n assert!(iter.next().is_none());\n assert!(iter.next_back().is_none());\n}\n\n#[test]\nfn ordered_range() {\n let s: SkipMap = SkipMap::new();\n s.insert(1, 1);\n\n let mut iter = s.range(0..);\n assert!(iter.next().is_some());\n assert!(iter.next().is_none());\n assert!(iter.next().is_none());\n\n s.insert(2, 2);\n assert!(iter.next().is_some());\n assert!(iter.next().is_none());\n assert!(iter.next().is_none());\n\n s.insert(3, 3);\n s.insert(4, 4);\n s.insert(5, 5);\n assert_eq!(*iter.next_back().unwrap().key(), 5);\n assert_eq!(*iter.next().unwrap().key(), 3);\n assert_eq!(*iter.next_back().unwrap().key(), 4);\n assert!(iter.next().is_none());\n assert!(iter.next_back().is_none());\n assert!(iter.next().is_none());\n assert!(iter.next_back().is_none());\n}\n#[test]\nfn entry_remove() {\n let s = SkipMap::new();\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10);\n }\n\n let mut e = s.get(&7).unwrap();\n assert!(!e.is_removed());\n assert!(e.remove());\n assert!(e.is_removed());\n\n e.move_prev();\n e.move_next();\n assert_ne!(*e.key(), 7);\n\n for e in s.iter() {\n assert!(!s.is_empty());\n assert_ne!(s.len(), 0);\n e.remove();\n }\n assert!(s.is_empty());\n assert_eq!(s.len(), 0);\n}\n\n#[test]\nfn entry_reposition() {\n let s = SkipMap::new();\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10);\n }\n\n let mut e = s.get(&7).unwrap();\n assert!(!e.is_removed());\n assert!(e.remove());\n assert!(e.is_removed());\n\n s.insert(7, 700);\n e.move_prev();\n e.move_next();\n assert_eq!(*e.key(), 7);\n}\n\n#[test]\nfn len() {\n let s = SkipMap::new();\n assert_eq!(s.len(), 0);\n\n for (i, &x) in [4, 2, 12, 8, 7, 11, 5].iter().enumerate() {\n s.insert(x, x * 10);\n assert_eq!(s.len(), i + 1);\n }\n\n s.insert(5, 0);\n assert_eq!(s.len(), 7);\n s.insert(5, 0);\n assert_eq!(s.len(), 7);\n\n s.remove(&6);\n assert_eq!(s.len(), 7);\n s.remove(&5);\n assert_eq!(s.len(), 6);\n s.remove(&12);\n assert_eq!(s.len(), 5);\n}\n\n#[test]\nfn insert_and_remove() {\n let s = SkipMap::new();\n let keys = || s.iter().map(|e| *e.key()).collect::>();\n\n s.insert(3, 0);\n s.insert(5, 0);\n s.insert(1, 0);\n s.insert(4, 0);\n s.insert(2, 0);\n assert_eq!(keys(), [1, 2, 3, 4, 5]);\n\n assert!(s.remove(&4).is_some());\n assert_eq!(keys(), [1, 2, 3, 5]);\n assert!(s.remove(&3).is_some());\n assert_eq!(keys(), [1, 2, 5]);\n assert!(s.remove(&1).is_some());\n assert_eq!(keys(), [2, 5]);\n\n assert!(s.remove(&1).is_none());\n assert_eq!(keys(), [2, 5]);\n assert!(s.remove(&3).is_none());\n assert_eq!(keys(), [2, 5]);\n\n assert!(s.remove(&2).is_some());\n assert_eq!(keys(), [5]);\n assert!(s.remove(&5).is_some());\n assert_eq!(keys(), []);\n\n s.insert(3, 0);\n assert_eq!(keys(), [3]);\n s.insert(1, 0);\n assert_eq!(keys(), [1, 3]);\n s.insert(3, 0);\n assert_eq!(keys(), [1, 3]);\n s.insert(5, 0);\n assert_eq!(keys(), [1, 3, 5]);\n\n assert!(s.remove(&3).is_some());\n assert_eq!(keys(), [1, 5]);\n assert!(s.remove(&1).is_some());\n assert_eq!(keys(), [5]);\n assert!(s.remove(&3).is_none());\n assert_eq!(keys(), [5]);\n assert!(s.remove(&5).is_some());\n assert_eq!(keys(), []);\n}\n\n#[test]\nfn get() {\n let s = SkipMap::new();\n s.insert(30, 3);\n s.insert(50, 5);\n s.insert(10, 1);\n s.insert(40, 4);\n s.insert(20, 2);\n\n assert_eq!(*s.get(&10).unwrap().value(), 1);\n assert_eq!(*s.get(&20).unwrap().value(), 2);\n assert_eq!(*s.get(&30).unwrap().value(), 3);\n assert_eq!(*s.get(&40).unwrap().value(), 4);\n assert_eq!(*s.get(&50).unwrap().value(), 5);\n\n assert!(s.get(&7).is_none());\n assert!(s.get(&27).is_none());\n assert!(s.get(&31).is_none());\n assert!(s.get(&97).is_none());\n}\n\n#[test]\nfn lower_bound() {\n let s = SkipMap::new();\n s.insert(30, 3);\n s.insert(50, 5);\n s.insert(10, 1);\n s.insert(40, 4);\n s.insert(20, 2);\n\n assert_eq!(*s.lower_bound(Bound::Unbounded).unwrap().value(), 1);\n\n assert_eq!(*s.lower_bound(Bound::Included(&10)).unwrap().value(), 1);\n assert_eq!(*s.lower_bound(Bound::Included(&20)).unwrap().value(), 2);\n assert_eq!(*s.lower_bound(Bound::Included(&30)).unwrap().value(), 3);\n assert_eq!(*s.lower_bound(Bound::Included(&40)).unwrap().value(), 4);\n assert_eq!(*s.lower_bound(Bound::Included(&50)).unwrap().value(), 5);\n\n assert_eq!(*s.lower_bound(Bound::Included(&7)).unwrap().value(), 1);\n assert_eq!(*s.lower_bound(Bound::Included(&27)).unwrap().value(), 3);\n assert_eq!(*s.lower_bound(Bound::Included(&31)).unwrap().value(), 4);\n assert!(s.lower_bound(Bound::Included(&97)).is_none());\n\n assert_eq!(*s.lower_bound(Bound::Excluded(&10)).unwrap().value(), 2);\n assert_eq!(*s.lower_bound(Bound::Excluded(&20)).unwrap().value(), 3);\n assert_eq!(*s.lower_bound(Bound::Excluded(&30)).unwrap().value(), 4);\n assert_eq!(*s.lower_bound(Bound::Excluded(&40)).unwrap().value(), 5);\n assert!(s.lower_bound(Bound::Excluded(&50)).is_none());\n\n assert_eq!(*s.lower_bound(Bound::Excluded(&7)).unwrap().value(), 1);\n assert_eq!(*s.lower_bound(Bound::Excluded(&27)).unwrap().value(), 3);\n assert_eq!(*s.lower_bound(Bound::Excluded(&31)).unwrap().value(), 4);\n assert!(s.lower_bound(Bound::Excluded(&97)).is_none());\n}\n\n#[test]\nfn upper_bound() {\n let s = SkipMap::new();\n s.insert(30, 3);\n s.insert(50, 5);\n s.insert(10, 1);\n s.insert(40, 4);\n s.insert(20, 2);\n\n assert_eq!(*s.upper_bound(Bound::Unbounded).unwrap().value(), 5);\n\n assert_eq!(*s.upper_bound(Bound::Included(&10)).unwrap().value(), 1);\n assert_eq!(*s.upper_bound(Bound::Included(&20)).unwrap().value(), 2);\n assert_eq!(*s.upper_bound(Bound::Included(&30)).unwrap().value(), 3);\n assert_eq!(*s.upper_bound(Bound::Included(&40)).unwrap().value(), 4);\n assert_eq!(*s.upper_bound(Bound::Included(&50)).unwrap().value(), 5);\n\n assert!(s.upper_bound(Bound::Included(&7)).is_none());\n assert_eq!(*s.upper_bound(Bound::Included(&27)).unwrap().value(), 2);\n assert_eq!(*s.upper_bound(Bound::Included(&31)).unwrap().value(), 3);\n assert_eq!(*s.upper_bound(Bound::Included(&97)).unwrap().value(), 5);\n\n assert!(s.upper_bound(Bound::Excluded(&10)).is_none());\n assert_eq!(*s.upper_bound(Bound::Excluded(&20)).unwrap().value(), 1);\n assert_eq!(*s.upper_bound(Bound::Excluded(&30)).unwrap().value(), 2);\n assert_eq!(*s.upper_bound(Bound::Excluded(&40)).unwrap().value(), 3);\n assert_eq!(*s.upper_bound(Bound::Excluded(&50)).unwrap().value(), 4);\n\n assert!(s.upper_bound(Bound::Excluded(&7)).is_none());\n assert_eq!(*s.upper_bound(Bound::Excluded(&27)).unwrap().value(), 2);\n assert_eq!(*s.upper_bound(Bound::Excluded(&31)).unwrap().value(), 3);\n assert_eq!(*s.upper_bound(Bound::Excluded(&97)).unwrap().value(), 5);\n}\n\n#[test]\nfn get_or_insert() {\n let s = SkipMap::new();\n s.insert(3, 3);\n s.insert(5, 5);\n s.insert(1, 1);\n s.insert(4, 4);\n s.insert(2, 2);\n\n assert_eq!(*s.get(&4).unwrap().value(), 4);\n assert_eq!(*s.insert(4, 40).value(), 40);\n assert_eq!(*s.get(&4).unwrap().value(), 40);\n\n assert_eq!(*s.get_or_insert(4, 400).value(), 40);\n assert_eq!(*s.get(&4).unwrap().value(), 40);\n assert_eq!(*s.get_or_insert(6, 600).value(), 600);\n}\n\n#[test]\nfn get_or_insert_with() {\n let s = SkipMap::new();\n s.insert(3, 3);\n s.insert(5, 5);\n s.insert(1, 1);\n s.insert(4, 4);\n s.insert(2, 2);\n\n assert_eq!(*s.get(&4).unwrap().value(), 4);\n assert_eq!(*s.insert(4, 40).value(), 40);\n assert_eq!(*s.get(&4).unwrap().value(), 40);\n\n assert_eq!(*s.get_or_insert_with(4, || 400).value(), 40);\n assert_eq!(*s.get(&4).unwrap().value(), 40);\n assert_eq!(*s.get_or_insert_with(6, || 600).value(), 600);\n}\n\n#[test]\nfn get_or_insert_with_panic() {\n use std::panic;\n\n let s = SkipMap::new();\n let res = panic::catch_unwind(panic::AssertUnwindSafe(|| {\n s.get_or_insert_with(4, || panic!());\n }));\n assert!(res.is_err());\n assert!(s.is_empty());\n assert_eq!(*s.get_or_insert_with(4, || 40).value(), 40);\n assert_eq!(s.len(), 1);\n}\n\n#[test]\nfn get_or_insert_with_parallel_run() {\n use std::sync::{Arc, Mutex};\n\n let s = Arc::new(SkipMap::new());\n let s2 = s.clone();\n let called = Arc::new(Mutex::new(false));\n let called2 = called.clone();\n let handle = std::thread::spawn(move || {\n assert_eq!(\n *s2.get_or_insert_with(7, || {\n *called2.lock().unwrap() = true;\n\n // allow main thread to run before we return result\n std::thread::sleep(std::time::Duration::from_secs(4));\n 70\n })\n .value(),\n 700\n );\n });\n std::thread::sleep(std::time::Duration::from_secs(2));\n\n // main thread writes the value first\n assert_eq!(*s.get_or_insert(7, 700).value(), 700);\n handle.join().unwrap();\n assert!(*called.lock().unwrap());\n}\n\n#[test]\nfn get_next_prev() {\n let s = SkipMap::new();\n s.insert(3, 3);\n s.insert(5, 5);\n s.insert(1, 1);\n s.insert(4, 4);\n s.insert(2, 2);\n\n let mut e = s.get(&3).unwrap();\n assert_eq!(*e.next().unwrap().value(), 4);\n assert_eq!(*e.prev().unwrap().value(), 2);\n assert_eq!(*e.value(), 3);\n\n e.move_prev();\n assert_eq!(*e.next().unwrap().value(), 3);\n assert_eq!(*e.prev().unwrap().value(), 1);\n assert_eq!(*e.value(), 2);\n\n e.move_prev();\n assert_eq!(*e.next().unwrap().value(), 2);\n assert!(e.prev().is_none());\n assert_eq!(*e.value(), 1);\n\n e.move_next();\n e.move_next();\n e.move_next();\n e.move_next();\n assert!(e.next().is_none());\n assert_eq!(*e.prev().unwrap().value(), 4);\n assert_eq!(*e.value(), 5);\n}\n\n#[test]\nfn front_and_back() {\n let s = SkipMap::new();\n assert!(s.front().is_none());\n assert!(s.back().is_none());\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10);\n }\n\n assert_eq!(*s.front().unwrap().key(), 2);\n assert_eq!(*s.back().unwrap().key(), 12);\n}\n\n#[test]\nfn iter() {\n let s = SkipMap::new();\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10);\n }\n\n assert_eq!(\n s.iter().map(|e| *e.key()).collect::>(),\n &[2, 4, 5, 7, 8, 11, 12]\n );\n\n let mut it = s.iter();\n s.remove(&2);\n assert_eq!(*it.next().unwrap().key(), 4);\n s.remove(&7);\n assert_eq!(*it.next().unwrap().key(), 5);\n s.remove(&5);\n assert_eq!(*it.next().unwrap().key(), 8);\n s.remove(&12);\n assert_eq!(*it.next().unwrap().key(), 11);\n assert!(it.next().is_none());\n}\n\n#[test]\nfn iter_range() {\n use std::ops::Bound::*;\n let s = SkipMap::new();\n let v = (0..10).map(|x| x * 10).collect::>();\n for &x in v.iter() {\n s.insert(x, x);\n }\n\n assert_eq!(\n s.iter().map(|x| *x.value()).collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.iter().rev().map(|x| *x.value()).collect::>(),\n vec![90, 80, 70, 60, 50, 40, 30, 20, 10, 0]\n );\n assert_eq!(\n s.range(..).map(|x| *x.value()).collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n\n assert_eq!(\n s.range((Included(&0), Unbounded))\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Included(&0), Included(&60)))\n .rev()\n .map(|x| *x.value())\n .collect::>(),\n vec![60, 50, 40, 30, 20, 10, 0]\n );\n assert_eq!(\n s.range((Excluded(&0), Unbounded))\n .map(|x| *x.value())\n .collect::>(),\n vec![10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Included(&25), Unbounded))\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Excluded(&25), Unbounded))\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Included(&70), Unbounded))\n .map(|x| *x.value())\n .collect::>(),\n vec![70, 80, 90]\n );\n assert_eq!(\n s.range((Excluded(&70), Unbounded))\n .map(|x| *x.value())\n .collect::>(),\n vec![80, 90]\n );\n assert_eq!(\n s.range((Included(&100), Unbounded))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&100), Unbounded))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Unbounded, Included(&90)))\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&90)))\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80]\n );\n assert_eq!(\n s.range((Unbounded, Included(&25)))\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&25)))\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20]\n );\n assert_eq!(\n s.range((Unbounded, Included(&70)))\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&70)))\n .map(|x| *x.value())\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60]\n );\n assert_eq!(\n s.range((Unbounded, Included(&-1)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&-1)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Included(&25), Included(&80)))\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80]\n );\n assert_eq!(\n s.range((Included(&25), Excluded(&80)))\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70]\n );\n assert_eq!(\n s.range((Excluded(&25), Included(&80)))\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80]\n );\n assert_eq!(\n s.range((Excluded(&25), Excluded(&80)))\n .map(|x| *x.value())\n .collect::>(),\n vec![30, 40, 50, 60, 70]\n );\n\n assert_eq!(\n s.range((Included(&25), Included(&25)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Included(&25), Excluded(&25)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&25), Included(&25)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&25), Excluded(&25)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Included(&50), Included(&50)))\n .map(|x| *x.value())\n .collect::>(),\n vec![50]\n );\n assert_eq!(\n s.range((Included(&50), Excluded(&50)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&50), Included(&50)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&50), Excluded(&50)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Included(&100), Included(&-2)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Included(&100), Excluded(&-2)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&100), Included(&-2)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&100), Excluded(&-2)))\n .map(|x| *x.value())\n .collect::>(),\n vec![]\n );\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/671\n#[test]\nfn iter_range2() {\n let set: SkipMap<_, _> = [1, 3, 5].iter().map(|x| (*x, *x)).collect();\n assert_eq!(set.range(2..4).count(), 1);\n set.insert(3, 3);\n}\n\n#[test]\nfn into_iter() {\n let s = SkipMap::new();\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10);\n }\n\n assert_eq!(\n s.into_iter().collect::>(),\n &[\n (2, 20),\n (4, 40),\n (5, 50),\n (7, 70),\n (8, 80),\n (11, 110),\n (12, 120),\n ]\n );\n}\n\n#[test]\nfn clear() {\n let s = SkipMap::new();\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x, x * 10);\n }\n\n assert!(!s.is_empty());\n assert_ne!(s.len(), 0);\n s.clear();\n assert!(s.is_empty());\n assert_eq!(s.len(), 0);\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/1023\n#[test]\nfn concurrent_insert_get_same_key() {\n let map: Arc> = Arc::new(SkipMap::new());\n let len = if cfg!(miri) { 100 } else { 10_000 };\n let key = 0;\n map.insert(0, ());\n\n let getter = map.clone();\n let handle = std::thread::spawn(move || {\n for _ in 0..len {\n map.insert(0, ());\n }\n });\n for _ in 0..len {\n assert!(getter.get(&key).is_some());\n }\n handle.join().unwrap()\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/1178\n#[test]\nfn issue_1178() {\n let map = SkipMap::new();\n let _ = map.insert(vec![1u8], vec![1u8]);\n let _ = map.insert(vec![2], vec![2]);\n let _ = map.insert(vec![3], vec![3]);\n\n let kvs: Vec<(Vec, Vec)> = map\n .range(vec![1]..vec![3])\n .map(|e| (e.key().clone(), e.value().clone()))\n .collect();\n\n for (k, v) in kvs {\n map.insert(k, v);\n }\n}\n\n#[test]\nfn comparator() {\n use std::cmp::Ordering;\n\n use crossbeam_skiplist::comparator::{Comparator, Equivalator};\n\n #[derive(Debug)]\n struct CaseComparator {\n case_sensitive: bool,\n }\n\n impl + ?Sized, R: Borrow + ?Sized> Equivalator for CaseComparator {\n fn equivalent(&self, lhs: &L, rhs: &R) -> bool {\n if self.case_sensitive {\n lhs.borrow() == rhs.borrow()\n } else {\n lhs.borrow()\n .chars()\n .flat_map(|c| c.to_lowercase())\n .eq(rhs.borrow().chars().flat_map(|c| c.to_lowercase()))\n }\n }\n }\n\n impl + ?Sized, R: Borrow + ?Sized> Comparator for CaseComparator {\n fn compare(&self, lhs: &L, rhs: &R) -> Ordering {\n if self.case_sensitive {\n lhs.borrow().cmp(rhs.borrow())\n } else {\n lhs.borrow()\n .chars()\n .flat_map(|c| c.to_lowercase())\n .cmp(rhs.borrow().chars().flat_map(|c| c.to_lowercase()))\n }\n }\n }\n\n let s: SkipMap = SkipMap::with_comparator(CaseComparator {\n case_sensitive: true,\n });\n s.insert(\"abc\".to_owned(), 1);\n s.insert(\"ABC\".to_owned(), 2);\n assert_eq!(s.len(), 2);\n assert_eq!(*s.get(\"abc\").unwrap().value(), 1);\n assert_eq!(*s.get(\"ABC\").unwrap().value(), 2);\n assert!(!s.contains_key(\"aBc\"));\n s.remove(\"abc\");\n assert!(!s.contains_key(\"abc\"));\n assert!(s.contains_key(\"ABC\"));\n\n let s: SkipMap = SkipMap::with_comparator(CaseComparator {\n case_sensitive: false,\n });\n s.insert(\"abc\".to_owned(), 1);\n s.insert(\"ABC\".to_owned(), 2);\n assert_eq!(s.len(), 1);\n assert_eq!(*s.get(\"abc\").unwrap().value(), 2);\n assert_eq!(*s.get(\"ABC\").unwrap().value(), 2);\n assert_eq!(*s.get(\"aBc\").unwrap().value(), 2);\n assert_eq!(s.remove(\"AbC\").unwrap().key(), \"ABC\");\n assert!(s.is_empty());\n}\n" }, { "path": "crossbeam-skiplist/tests/set.rs", "content": "use std::{\n iter,\n ops::Bound,\n sync::{Arc, Barrier},\n};\n\nuse crossbeam_skiplist::SkipSet;\nuse crossbeam_utils::thread;\n\n#[test]\nfn smoke() {\n let m = SkipSet::new();\n m.insert(1);\n m.insert(5);\n m.insert(7);\n}\n\n#[test]\nfn is_empty() {\n let s = SkipSet::new();\n assert!(s.is_empty());\n\n s.insert(1);\n assert!(!s.is_empty());\n s.insert(2);\n s.insert(3);\n assert!(!s.is_empty());\n\n s.remove(&2);\n assert!(!s.is_empty());\n\n s.remove(&1);\n assert!(!s.is_empty());\n\n s.remove(&3);\n assert!(s.is_empty());\n}\n\n#[test]\nfn insert() {\n let insert = [0, 4, 2, 12, 8, 7, 11, 5];\n let not_present = [1, 3, 6, 9, 10];\n let s = SkipSet::new();\n\n for &x in &insert {\n s.insert(x);\n assert_eq!(*s.get(&x).unwrap(), x);\n }\n\n for &x in ¬_present {\n assert!(s.get(&x).is_none());\n }\n}\n\n#[test]\nfn remove() {\n let insert = [0, 4, 2, 12, 8, 7, 11, 5];\n let not_present = [1, 3, 6, 9, 10];\n let remove = [2, 12, 8];\n let remaining = [0, 4, 5, 7, 11];\n\n let s = SkipSet::new();\n\n for &x in &insert {\n s.insert(x);\n }\n for x in ¬_present {\n assert!(s.remove(x).is_none());\n }\n for x in &remove {\n assert!(s.remove(x).is_some());\n }\n\n let mut v = vec![];\n let mut e = s.front().unwrap();\n loop {\n v.push(*e);\n if !e.move_next() {\n break;\n }\n }\n\n assert_eq!(v, remaining);\n for x in &insert {\n s.remove(x);\n }\n assert!(s.is_empty());\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/672\n#[test]\nfn concurrent_insert() {\n for _ in 0..100 {\n let set: SkipSet = iter::once(1).collect();\n let barrier = Barrier::new(2);\n thread::scope(|s| {\n s.spawn(|_| {\n barrier.wait();\n set.insert(1);\n });\n s.spawn(|_| {\n barrier.wait();\n set.insert(1);\n });\n })\n .unwrap();\n }\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/672\n#[test]\nfn concurrent_remove() {\n for _ in 0..100 {\n let set: SkipSet = iter::once(1).collect();\n let barrier = Barrier::new(2);\n thread::scope(|s| {\n s.spawn(|_| {\n barrier.wait();\n set.remove(&1);\n });\n s.spawn(|_| {\n barrier.wait();\n set.remove(&1);\n });\n })\n .unwrap();\n }\n}\n\n#[test]\nfn entry() {\n let s = SkipSet::new();\n\n assert!(s.front().is_none());\n assert!(s.back().is_none());\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x);\n }\n\n let mut e = s.front().unwrap();\n assert_eq!(*e, 2);\n assert!(!e.move_prev());\n assert!(e.move_next());\n assert_eq!(*e, 4);\n\n e = s.back().unwrap();\n assert_eq!(*e, 12);\n assert!(!e.move_next());\n assert!(e.move_prev());\n assert_eq!(*e, 11);\n}\n\n#[test]\nfn entry_remove() {\n let s = SkipSet::new();\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x);\n }\n\n let mut e = s.get(&7).unwrap();\n assert!(!e.is_removed());\n assert!(e.remove());\n assert!(e.is_removed());\n\n e.move_prev();\n e.move_next();\n assert_ne!(*e, 7);\n\n for e in s.iter() {\n assert!(!s.is_empty());\n assert_ne!(s.len(), 0);\n e.remove();\n }\n assert!(s.is_empty());\n assert_eq!(s.len(), 0);\n}\n\n#[test]\nfn entry_reposition() {\n let s = SkipSet::new();\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x);\n }\n\n let mut e = s.get(&7).unwrap();\n assert!(!e.is_removed());\n assert!(e.remove());\n assert!(e.is_removed());\n\n s.insert(7);\n e.move_prev();\n e.move_next();\n assert_eq!(*e, 7);\n}\n\n#[test]\nfn len() {\n let s = SkipSet::new();\n assert_eq!(s.len(), 0);\n\n for (i, &x) in [4, 2, 12, 8, 7, 11, 5].iter().enumerate() {\n s.insert(x);\n assert_eq!(s.len(), i + 1);\n }\n\n s.insert(5);\n assert_eq!(s.len(), 7);\n s.insert(5);\n assert_eq!(s.len(), 7);\n\n s.remove(&6);\n assert_eq!(s.len(), 7);\n s.remove(&5);\n assert_eq!(s.len(), 6);\n s.remove(&12);\n assert_eq!(s.len(), 5);\n}\n\n#[test]\nfn insert_and_remove() {\n let s = SkipSet::new();\n let keys = || s.iter().map(|e| *e).collect::>();\n\n s.insert(3);\n s.insert(5);\n s.insert(1);\n s.insert(4);\n s.insert(2);\n assert_eq!(keys(), [1, 2, 3, 4, 5]);\n\n assert!(s.remove(&4).is_some());\n assert_eq!(keys(), [1, 2, 3, 5]);\n assert!(s.remove(&3).is_some());\n assert_eq!(keys(), [1, 2, 5]);\n assert!(s.remove(&1).is_some());\n assert_eq!(keys(), [2, 5]);\n\n assert!(s.remove(&1).is_none());\n assert_eq!(keys(), [2, 5]);\n assert!(s.remove(&3).is_none());\n assert_eq!(keys(), [2, 5]);\n\n assert!(s.remove(&2).is_some());\n assert_eq!(keys(), [5]);\n assert!(s.remove(&5).is_some());\n assert_eq!(keys(), []);\n\n s.insert(3);\n assert_eq!(keys(), [3]);\n s.insert(1);\n assert_eq!(keys(), [1, 3]);\n s.insert(3);\n assert_eq!(keys(), [1, 3]);\n s.insert(5);\n assert_eq!(keys(), [1, 3, 5]);\n\n assert!(s.remove(&3).is_some());\n assert_eq!(keys(), [1, 5]);\n assert!(s.remove(&1).is_some());\n assert_eq!(keys(), [5]);\n assert!(s.remove(&3).is_none());\n assert_eq!(keys(), [5]);\n assert!(s.remove(&5).is_some());\n assert_eq!(keys(), []);\n}\n\n#[test]\nfn get() {\n let s = SkipSet::new();\n s.insert(30);\n s.insert(50);\n s.insert(10);\n s.insert(40);\n s.insert(20);\n\n assert_eq!(*s.get(&10).unwrap(), 10);\n assert_eq!(*s.get(&20).unwrap(), 20);\n assert_eq!(*s.get(&30).unwrap(), 30);\n assert_eq!(*s.get(&40).unwrap(), 40);\n assert_eq!(*s.get(&50).unwrap(), 50);\n\n assert!(s.get(&7).is_none());\n assert!(s.get(&27).is_none());\n assert!(s.get(&31).is_none());\n assert!(s.get(&97).is_none());\n}\n\n#[test]\nfn lower_bound() {\n let s = SkipSet::new();\n s.insert(30);\n s.insert(50);\n s.insert(10);\n s.insert(40);\n s.insert(20);\n\n assert_eq!(*s.lower_bound(Bound::Unbounded).unwrap(), 10);\n\n assert_eq!(*s.lower_bound(Bound::Included(&10)).unwrap(), 10);\n assert_eq!(*s.lower_bound(Bound::Included(&20)).unwrap(), 20);\n assert_eq!(*s.lower_bound(Bound::Included(&30)).unwrap(), 30);\n assert_eq!(*s.lower_bound(Bound::Included(&40)).unwrap(), 40);\n assert_eq!(*s.lower_bound(Bound::Included(&50)).unwrap(), 50);\n\n assert_eq!(*s.lower_bound(Bound::Included(&7)).unwrap(), 10);\n assert_eq!(*s.lower_bound(Bound::Included(&27)).unwrap(), 30);\n assert_eq!(*s.lower_bound(Bound::Included(&31)).unwrap(), 40);\n assert!(s.lower_bound(Bound::Included(&97)).is_none());\n\n assert_eq!(*s.lower_bound(Bound::Excluded(&10)).unwrap(), 20);\n assert_eq!(*s.lower_bound(Bound::Excluded(&20)).unwrap(), 30);\n assert_eq!(*s.lower_bound(Bound::Excluded(&30)).unwrap(), 40);\n assert_eq!(*s.lower_bound(Bound::Excluded(&40)).unwrap(), 50);\n assert!(s.lower_bound(Bound::Excluded(&50)).is_none());\n\n assert_eq!(*s.lower_bound(Bound::Excluded(&7)).unwrap(), 10);\n assert_eq!(*s.lower_bound(Bound::Excluded(&27)).unwrap(), 30);\n assert_eq!(*s.lower_bound(Bound::Excluded(&31)).unwrap(), 40);\n assert!(s.lower_bound(Bound::Excluded(&97)).is_none());\n}\n\n#[test]\nfn upper_bound() {\n let s = SkipSet::new();\n s.insert(30);\n s.insert(50);\n s.insert(10);\n s.insert(40);\n s.insert(20);\n\n assert_eq!(*s.upper_bound(Bound::Unbounded).unwrap(), 50);\n\n assert_eq!(*s.upper_bound(Bound::Included(&10)).unwrap(), 10);\n assert_eq!(*s.upper_bound(Bound::Included(&20)).unwrap(), 20);\n assert_eq!(*s.upper_bound(Bound::Included(&30)).unwrap(), 30);\n assert_eq!(*s.upper_bound(Bound::Included(&40)).unwrap(), 40);\n assert_eq!(*s.upper_bound(Bound::Included(&50)).unwrap(), 50);\n\n assert!(s.upper_bound(Bound::Included(&7)).is_none());\n assert_eq!(*s.upper_bound(Bound::Included(&27)).unwrap(), 20);\n assert_eq!(*s.upper_bound(Bound::Included(&31)).unwrap(), 30);\n assert_eq!(*s.upper_bound(Bound::Included(&97)).unwrap(), 50);\n\n assert!(s.upper_bound(Bound::Excluded(&10)).is_none());\n assert_eq!(*s.upper_bound(Bound::Excluded(&20)).unwrap(), 10);\n assert_eq!(*s.upper_bound(Bound::Excluded(&30)).unwrap(), 20);\n assert_eq!(*s.upper_bound(Bound::Excluded(&40)).unwrap(), 30);\n assert_eq!(*s.upper_bound(Bound::Excluded(&50)).unwrap(), 40);\n\n assert!(s.upper_bound(Bound::Excluded(&7)).is_none());\n assert_eq!(*s.upper_bound(Bound::Excluded(&27)).unwrap(), 20);\n assert_eq!(*s.upper_bound(Bound::Excluded(&31)).unwrap(), 30);\n assert_eq!(*s.upper_bound(Bound::Excluded(&97)).unwrap(), 50);\n}\n\n#[test]\nfn get_or_insert() {\n let s = SkipSet::new();\n s.insert(3);\n s.insert(5);\n s.insert(1);\n s.insert(4);\n s.insert(2);\n\n assert_eq!(*s.get(&4).unwrap(), 4);\n assert_eq!(*s.insert(4), 4);\n assert_eq!(*s.get(&4).unwrap(), 4);\n\n assert_eq!(*s.get_or_insert(4), 4);\n assert_eq!(*s.get(&4).unwrap(), 4);\n assert_eq!(*s.get_or_insert(6), 6);\n}\n\n#[test]\nfn get_next_prev() {\n let s = SkipSet::new();\n s.insert(3);\n s.insert(5);\n s.insert(1);\n s.insert(4);\n s.insert(2);\n\n let mut e = s.get(&3).unwrap();\n assert_eq!(*e.next().unwrap(), 4);\n assert_eq!(*e.prev().unwrap(), 2);\n assert_eq!(*e, 3);\n\n e.move_prev();\n assert_eq!(*e.next().unwrap(), 3);\n assert_eq!(*e.prev().unwrap(), 1);\n assert_eq!(*e, 2);\n\n e.move_prev();\n assert_eq!(*e.next().unwrap(), 2);\n assert!(e.prev().is_none());\n assert_eq!(*e, 1);\n\n e.move_next();\n e.move_next();\n e.move_next();\n e.move_next();\n assert!(e.next().is_none());\n assert_eq!(*e.prev().unwrap(), 4);\n assert_eq!(*e, 5);\n}\n\n#[test]\nfn front_and_back() {\n let s = SkipSet::new();\n assert!(s.front().is_none());\n assert!(s.back().is_none());\n\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x);\n }\n\n assert_eq!(*s.front().unwrap(), 2);\n assert_eq!(*s.back().unwrap(), 12);\n}\n\n#[test]\nfn iter() {\n let s = SkipSet::new();\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x);\n }\n\n assert_eq!(\n s.iter().map(|e| *e).collect::>(),\n &[2, 4, 5, 7, 8, 11, 12]\n );\n\n let mut it = s.iter();\n s.remove(&2);\n assert_eq!(*it.next().unwrap(), 4);\n s.remove(&7);\n assert_eq!(*it.next().unwrap(), 5);\n s.remove(&5);\n assert_eq!(*it.next().unwrap(), 8);\n s.remove(&12);\n assert_eq!(*it.next().unwrap(), 11);\n assert!(it.next().is_none());\n}\n\n#[test]\nfn iter_range() {\n use std::ops::Bound::*;\n let s = SkipSet::new();\n let v = (0..10).map(|x| x * 10).collect::>();\n for &x in v.iter() {\n s.insert(x);\n }\n\n assert_eq!(\n s.iter().map(|x| *x).collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.iter().rev().map(|x| *x).collect::>(),\n vec![90, 80, 70, 60, 50, 40, 30, 20, 10, 0]\n );\n assert_eq!(\n s.range(..).map(|x| *x).collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n\n assert_eq!(\n s.range((Included(&0), Unbounded))\n .map(|x| *x)\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Excluded(&0), Unbounded))\n .map(|x| *x)\n .collect::>(),\n vec![10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Included(&25), Unbounded))\n .map(|x| *x)\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Excluded(&25), Unbounded))\n .map(|x| *x)\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Included(&70), Unbounded))\n .map(|x| *x)\n .collect::>(),\n vec![70, 80, 90]\n );\n assert_eq!(\n s.range((Excluded(&70), Unbounded))\n .map(|x| *x)\n .collect::>(),\n vec![80, 90]\n );\n assert_eq!(\n s.range((Included(&100), Unbounded))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&100), Unbounded))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Unbounded, Included(&90)))\n .map(|x| *x)\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80, 90]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&90)))\n .map(|x| *x)\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70, 80]\n );\n assert_eq!(\n s.range((Unbounded, Included(&25)))\n .map(|x| *x)\n .collect::>(),\n vec![0, 10, 20]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&25)))\n .map(|x| *x)\n .collect::>(),\n vec![0, 10, 20]\n );\n assert_eq!(\n s.range((Unbounded, Included(&70)))\n .map(|x| *x)\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60, 70]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&70)))\n .map(|x| *x)\n .collect::>(),\n vec![0, 10, 20, 30, 40, 50, 60]\n );\n assert_eq!(\n s.range((Unbounded, Included(&-1)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Unbounded, Excluded(&-1)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Included(&25), Included(&80)))\n .map(|x| *x)\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80]\n );\n assert_eq!(\n s.range((Included(&25), Excluded(&80)))\n .map(|x| *x)\n .collect::>(),\n vec![30, 40, 50, 60, 70]\n );\n assert_eq!(\n s.range((Excluded(&25), Included(&80)))\n .map(|x| *x)\n .collect::>(),\n vec![30, 40, 50, 60, 70, 80]\n );\n assert_eq!(\n s.range((Excluded(&25), Excluded(&80)))\n .map(|x| *x)\n .collect::>(),\n vec![30, 40, 50, 60, 70]\n );\n\n assert_eq!(\n s.range((Included(&25), Included(&25)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Included(&25), Excluded(&25)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&25), Included(&25)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&25), Excluded(&25)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Included(&50), Included(&50)))\n .map(|x| *x)\n .collect::>(),\n vec![50]\n );\n assert_eq!(\n s.range((Included(&50), Excluded(&50)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&50), Included(&50)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&50), Excluded(&50)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n\n assert_eq!(\n s.range((Included(&100), Included(&-2)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Included(&100), Excluded(&-2)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&100), Included(&-2)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n assert_eq!(\n s.range((Excluded(&100), Excluded(&-2)))\n .map(|x| *x)\n .collect::>(),\n vec![]\n );\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/671\n#[test]\nfn iter_range2() {\n let set: SkipSet<_> = [1, 3, 5].iter().cloned().collect();\n assert_eq!(set.range(2..4).count(), 1);\n set.insert(3);\n}\n\n#[test]\nfn into_iter() {\n let s = SkipSet::new();\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x);\n }\n\n assert_eq!(s.into_iter().collect::>(), &[2, 4, 5, 7, 8, 11, 12]);\n}\n\n#[test]\nfn clear() {\n let s = SkipSet::new();\n for &x in &[4, 2, 12, 8, 7, 11, 5] {\n s.insert(x);\n }\n\n assert!(!s.is_empty());\n assert_ne!(s.len(), 0);\n s.clear();\n assert!(s.is_empty());\n assert_eq!(s.len(), 0);\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/1023\n#[test]\nfn concurrent_insert_get_same_key() {\n let set: Arc> = Arc::new(SkipSet::new());\n let len = if cfg!(miri) { 100 } else { 10_000 };\n let key = 0;\n set.insert(0);\n\n let getter = set.clone();\n let handle = std::thread::spawn(move || {\n for _ in 0..len {\n set.insert(0);\n }\n });\n for _ in 0..len {\n assert!(getter.get(&key).is_some());\n }\n handle.join().unwrap()\n}\n\n#[test]\nfn comparator() {\n use std::cmp::Ordering;\n\n use crossbeam_skiplist::comparator::{Comparator, Equivalator};\n\n struct DynComparator {\n inner: Box Ordering>,\n }\n\n impl Equivalator for DynComparator\n where\n L: std::borrow::Borrow<[u8]>,\n R: std::borrow::Borrow<[u8]>,\n {\n fn equivalent(&self, lhs: &L, rhs: &R) -> bool {\n (self.inner)(lhs.borrow(), rhs.borrow()).is_eq()\n }\n }\n\n impl Comparator for DynComparator\n where\n L: std::borrow::Borrow<[u8]>,\n R: std::borrow::Borrow<[u8]>,\n {\n fn compare(&self, lhs: &L, rhs: &R) -> Ordering {\n (self.inner)(lhs.borrow(), rhs.borrow())\n }\n }\n\n #[derive(Clone, Copy, Debug)]\n struct OrderedF32(f32);\n\n impl PartialEq for OrderedF32 {\n fn eq(&self, other: &Self) -> bool {\n self.cmp(other).is_eq()\n }\n }\n\n impl Eq for OrderedF32 {}\n\n impl PartialOrd for OrderedF32 {\n fn partial_cmp(&self, other: &Self) -> Option {\n Some(self.cmp(other))\n }\n }\n\n impl Ord for OrderedF32 {\n fn cmp(&self, other: &Self) -> Ordering {\n match self.0.partial_cmp(&other.0) {\n Some(o) => o,\n None => Ordering::Equal,\n }\n }\n }\n\n fn encode(slice: &[f32]) -> Vec {\n let mut bytes = vec![];\n for &f in slice {\n bytes.extend_from_slice(&f.to_le_bytes());\n }\n bytes\n }\n\n fn decode(slice: &[u8]) -> impl Iterator + '_ {\n slice\n .chunks_exact(4)\n .map(|b| OrderedF32(f32::from_le_bytes(b.try_into().unwrap())))\n }\n\n fn compare(lhs: &[u8], rhs: &[u8]) -> Ordering {\n decode(lhs).cmp(decode(rhs))\n }\n\n let s = SkipSet::with_comparator(DynComparator {\n inner: Box::new(compare),\n });\n s.insert(encode(&[0f32]));\n s.insert(encode(&[1f32]));\n s.insert(encode(&[0f32, 1f32]));\n s.insert(encode(&[-0f32]));\n\n assert_eq!(s.len(), 3);\n assert!(s.contains(&encode(&[0f32])));\n assert!(s.contains(&encode(&[-0f32])));\n assert!(s.contains(&encode(&[1f32])));\n assert!(s.contains(&encode(&[0f32, 1f32])));\n\n let elems: Vec<_> = s.iter().map(|x| &x.value()[..]).collect();\n assert_eq!(\n elems,\n [&encode(&[-0f32]), &encode(&[0f32, 1f32]), &encode(&[1f32])],\n );\n\n s.remove(&encode(&[0f32]));\n assert!(!s.contains(&encode(&[-0f32])));\n}\n" }, { "path": "crossbeam-utils/CHANGELOG.md", "content": "# Version 0.8.21\n\n- Improve implementation of `CachePadded`. (#1152)\n\n# Version 0.8.20\n\n- Implement `Display` for `CachePadded`. (#1097)\n\n# Version 0.8.19\n\n- Remove dependency on `cfg-if`. (#1072)\n\n# Version 0.8.18\n\n- Relax the minimum supported Rust version to 1.60. (#1056)\n- Improve scalability of `AtomicCell` fallback. (#1055)\n\n# Version 0.8.17\n\n- Bump the minimum supported Rust version to 1.61. (#1037)\n- Improve support for targets without atomic CAS or 64-bit atomic. (#1037)\n- Always implement `UnwindSafe` and `RefUnwindSafe` for `AtomicCell`. (#1045)\n- Improve compatibility with Miri, TSan, and loom. (#995, #1003)\n- Improve compatibility with unstable `oom=panic`. (#1045)\n- Improve implementation of `CachePadded`. (#1014, #1025)\n- Update `loom` dependency to 0.7.\n\n# Version 0.8.16\n\n- Improve implementation of `CachePadded`. (#967)\n\n# Version 0.8.15\n\n- Add `#[clippy::has_significant_drop]` to `ShardedLock{Read,Write}Guard`. (#958)\n- Improve handling of very large timeout. (#953)\n- Soft-deprecate `thread::scope()` in favor of the more efficient `std::thread::scope` that stabilized in Rust 1.63. (#954)\n\n# Version 0.8.14\n\n- Fix build script bug introduced in 0.8.13. (#932)\n\n# Version 0.8.13\n\n**Note:** This release has been yanked due to regression fixed in 0.8.14.\n\n- Improve support for custom targets. (#922)\n\n# Version 0.8.12\n\n- Removes the dependency on the `once_cell` crate to restore the MSRV. (#913)\n- Work around [rust-lang#98302](https://github.com/rust-lang/rust/issues/98302), which causes compile error on windows-gnu when LTO is enabled. (#913)\n\n# Version 0.8.11\n\n- Bump the minimum supported Rust version to 1.38. (#877)\n\n# Version 0.8.10\n\n- Fix unsoundness of `AtomicCell` on types containing niches. (#834)\n This fix contains breaking changes, but they are allowed because this is a soundness bug fix. See #834 for more.\n\n# Version 0.8.9\n\n- Replace lazy_static with once_cell. (#817)\n\n# Version 0.8.8\n\n- Fix a bug when unstable `loom` support is enabled. (#787)\n\n# Version 0.8.7\n\n- Add `AtomicCell<{i*,u*}>::{fetch_max,fetch_min}`. (#785)\n- Add `AtomicCell<{i*,u*,bool}>::fetch_nand`. (#785)\n- Fix unsoundness of `AtomicCell<{i,u}64>` arithmetics on 32-bit targets that support `Atomic{I,U}64` (#781)\n\n# Version 0.8.6\n\n**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details.\n\n- Re-add `AtomicCell<{i,u}64>::{fetch_add,fetch_sub,fetch_and,fetch_or,fetch_xor}` that were accidentally removed in 0.8.0 on targets that do not support `Atomic{I,U}64`. (#767)\n- Re-add `AtomicCell<{i,u}128>::{fetch_add,fetch_sub,fetch_and,fetch_or,fetch_xor}` that were accidentally removed in 0.8.0. (#767)\n\n# Version 0.8.5\n\n**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details.\n\n- Add `AtomicCell::fetch_update`. (#704)\n- Support targets that do not have atomic CAS on stable Rust. (#698)\n\n# Version 0.8.4\n\n**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details.\n\n- Bump `loom` dependency to version 0.5. (#686)\n\n# Version 0.8.3\n\n**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details.\n\n- Make `loom` dependency optional. (#666)\n\n# Version 0.8.2\n\n**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details.\n\n- Deprecate `AtomicCell::compare_and_swap`. Use `AtomicCell::compare_exchange` instead. (#619)\n- Add `Parker::park_deadline`. (#563)\n- Improve implementation of `CachePadded`. (#636)\n- Add unstable support for `loom`. (#487)\n\n# Version 0.8.1\n\n**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details.\n\n- Make `AtomicCell::is_lock_free` always const fn. (#600)\n- Fix a bug in `seq_lock_wide`. (#596)\n- Remove `const_fn` dependency. (#600)\n- `crossbeam-utils` no longer fails to compile if unable to determine rustc version. Instead, it now displays a warning. (#604)\n\n# Version 0.8.0\n\n**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details.\n\n- Bump the minimum supported Rust version to 1.36.\n- Remove deprecated `AtomicCell::get_mut()` and `Backoff::is_complete()` methods.\n- Remove `alloc` feature.\n- Make `CachePadded::new()` const function.\n- Make `AtomicCell::is_lock_free()` const function at 1.46+.\n- Implement `From` for `AtomicCell`.\n\n# Version 0.7.2\n\n- Fix bug in release (yanking 0.7.1)\n\n# Version 0.7.1\n\n- Bump `autocfg` dependency to version 1.0. (#460)\n- Make `AtomicCell` lockfree for u8, u16, u32, u64 sized values at 1.34+. (#454)\n\n# Version 0.7.0\n\n- Bump the minimum required version to 1.28.\n- Fix breakage with nightly feature due to rust-lang/rust#65214.\n- Apply `#[repr(transparent)]` to `AtomicCell`.\n- Make `AtomicCell::new()` const function at 1.31+.\n\n# Version 0.6.6\n\n- Add `UnwindSafe` and `RefUnwindSafe` impls for `AtomicCell`.\n- Add `AtomicCell::as_ptr()`.\n- Add `AtomicCell::take()`.\n- Fix a bug in `AtomicCell::compare_exchange()` and `AtomicCell::compare_and_swap()`.\n- Various documentation improvements.\n\n# Version 0.6.5\n\n- Rename `Backoff::is_complete()` to `Backoff::is_completed()`.\n\n# Version 0.6.4\n\n- Add `WaitGroup`, `ShardedLock`, and `Backoff`.\n- Add `fetch_*` methods for `AtomicCell` and `AtomicCell`.\n- Expand documentation.\n\n# Version 0.6.3\n\n- Add `AtomicCell`.\n- Improve documentation.\n\n# Version 0.6.2\n\n- Add `Parker`.\n- Improve documentation.\n\n# Version 0.6.1\n\n- Fix a soundness bug in `Scope::spawn()`.\n- Remove the `T: 'scope` bound on `ScopedJoinHandle`.\n\n# Version 0.6.0\n\n- Move `AtomicConsume` to `atomic` module.\n- `scope()` returns a `Result` of thread joins.\n- Remove `spawn_unchecked`.\n- Fix a soundness bug due to incorrect lifetimes.\n- Improve documentation.\n- Support nested scoped spawns.\n- Implement `Copy`, `Hash`, `PartialEq`, and `Eq` for `CachePadded`.\n- Add `CachePadded::into_inner()`.\n\n# Version 0.5.0\n\n- Reorganize sub-modules and rename functions.\n\n# Version 0.4.1\n\n- Fix a documentation link.\n\n# Version 0.4.0\n\n- `CachePadded` supports types bigger than 64 bytes.\n- Fix a bug in scoped threads where unitialized memory was being dropped.\n- Minimum required Rust version is now 1.25.\n\n# Version 0.3.2\n\n- Mark `load_consume` with `#[inline]`.\n\n# Version 0.3.1\n\n- `load_consume` on ARM and AArch64.\n\n# Version 0.3.0\n\n- Add `join` for scoped thread API.\n- Add `load_consume` for atomic load-consume memory ordering.\n- Remove `AtomicOption`.\n\n# Version 0.2.2\n\n- Support Rust 1.12.1.\n- Call `T::clone` when cloning a `CachePadded`.\n\n# Version 0.2.1\n\n- Add `use_std` feature.\n\n# Version 0.2.0\n\n- Add `nightly` feature.\n- Use `repr(align(64))` on `CachePadded` with the `nightly` feature.\n- Implement `Drop` for `CachePadded`.\n- Implement `Clone` for `CachePadded`.\n- Implement `From` for `CachePadded`.\n- Implement better `Debug` for `CachePadded`.\n- Write more tests.\n- Add this changelog.\n- Change cache line length to 64 bytes.\n- Remove `ZerosValid`.\n\n# Version 0.1.0\n\n- Old implementation of `CachePadded` from `crossbeam` version 0.3.0\n" }, { "path": "crossbeam-utils/Cargo.toml", "content": "[package]\nname = \"crossbeam-utils\"\n# When publishing a new version:\n# - Update CHANGELOG.md\n# - Update README.md (when increasing major or minor version)\n# - Run './tools/publish.sh crossbeam-utils '\nversion = \"0.8.21\"\nedition = \"2021\"\n# NB: Sync with msrv badge and \"Compatibility\" section in README.md\nrust-version = \"1.56\"\nlicense = \"MIT OR Apache-2.0\"\nrepository = \"https://github.com/crossbeam-rs/crossbeam\"\nhomepage = \"https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils\"\ndescription = \"Utilities for concurrent programming\"\nkeywords = [\"scoped\", \"thread\", \"atomic\", \"cache\"]\ncategories = [\"algorithms\", \"concurrency\", \"data-structures\", \"no-std\"]\n\n[package.metadata.docs.rs]\nall-features = true\ntargets = [\"x86_64-unknown-linux-gnu\"]\n\n[package.metadata.cargo_check_external_types]\n# The following are external types that are allowed to be exposed in our public API.\nallowed_external_types = [\n]\n\n[features]\ndefault = [\"std\"]\n\n# Enable to use APIs that require `std`.\n# This is enabled by default.\nstd = []\n\n# Enable `atomic` module.\n# This requires Rust 1.74.\natomic = [\"atomic-maybe-uninit\"]\n\n[dependencies]\natomic-maybe-uninit = { version = \"0.3.4\", optional = true }\n\n# Enable the use of loom for concurrency testing.\n#\n# NOTE: This feature is outside of the normal semver guarantees and minor or\n# patch versions of crossbeam may make breaking changes to them at any time.\n[target.'cfg(crossbeam_loom)'.dependencies]\nloom = { version = \"0.7.1\", optional = true }\n\n[dev-dependencies]\nfastrand = \"2\"\nrustversion = \"1\"\n\n[lints]\nworkspace = true\n" }, { "path": "crossbeam-utils/LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "crossbeam-utils/LICENSE-MIT", "content": "The MIT License (MIT)\n\nCopyright (c) 2019 The Crossbeam Project Developers\n\nPermission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "crossbeam-utils/README.md", "content": "# Crossbeam Utils\n\n[![Build Status](https://github.com/crossbeam-rs/crossbeam/workflows/CI/badge.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/actions)\n[![License](https://img.shields.io/badge/license-MIT_OR_Apache--2.0-blue.svg)](\nhttps://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils#license)\n[![Cargo](https://img.shields.io/crates/v/crossbeam-utils.svg)](\nhttps://crates.io/crates/crossbeam-utils)\n[![Documentation](https://docs.rs/crossbeam-utils/badge.svg)](\nhttps://docs.rs/crossbeam-utils)\n[![Rust 1.56+](https://img.shields.io/badge/rust-1.56+-lightgray.svg)](\nhttps://www.rust-lang.org)\n[![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ)\n\nThis crate provides miscellaneous tools for concurrent programming:\n\n#### Atomics\n\n* [`AtomicCell`], a thread-safe mutable memory location.(no_std)\n* [`AtomicConsume`], for reading from primitive atomic types with \"consume\" ordering.(no_std)\n\n#### Thread synchronization\n\n* [`Parker`], a thread parking primitive.\n* [`ShardedLock`], a sharded reader-writer lock with fast concurrent reads.\n* [`WaitGroup`], for synchronizing the beginning or end of some computation.\n\n#### Utilities\n\n* [`Backoff`], for exponential backoff in spin loops.(no_std)\n* [`CachePadded`], for padding and aligning a value to the length of a cache line.(no_std)\n* [`scope`], for spawning threads that borrow local variables from the stack.\n\n*Features marked with (no_std) can be used in `no_std` environments.*
\n\n[`AtomicCell`]: https://docs.rs/crossbeam-utils/latest/crossbeam_utils/atomic/struct.AtomicCell.html\n[`AtomicConsume`]: https://docs.rs/crossbeam-utils/latest/crossbeam_utils/atomic/trait.AtomicConsume.html\n[`Parker`]: https://docs.rs/crossbeam-utils/latest/crossbeam_utils/sync/struct.Parker.html\n[`ShardedLock`]: https://docs.rs/crossbeam-utils/latest/crossbeam_utils/sync/struct.ShardedLock.html\n[`WaitGroup`]: https://docs.rs/crossbeam-utils/latest/crossbeam_utils/sync/struct.WaitGroup.html\n[`Backoff`]: https://docs.rs/crossbeam-utils/latest/crossbeam_utils/struct.Backoff.html\n[`CachePadded`]: https://docs.rs/crossbeam-utils/latest/crossbeam_utils/struct.CachePadded.html\n[`scope`]: https://docs.rs/crossbeam-utils/latest/crossbeam_utils/thread/fn.scope.html\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\ncrossbeam-utils = \"0.8\"\n```\n\n## Compatibility\n\nCrossbeam Utils supports stable Rust releases going back at least six months,\nand every time the minimum supported Rust version is increased, a new minor\nversion is released. Currently, the minimum supported Rust version is 1.56.\n\n## License\n\nLicensed under either of\n\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n#### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "path": "crossbeam-utils/benches/atomic_cell.rs", "content": "#![feature(test)]\n\nextern crate test;\n\nuse std::sync::Barrier;\n\nuse crossbeam_utils::{atomic::AtomicCell, thread};\n\n#[bench]\nfn load_u8(b: &mut test::Bencher) {\n let a = AtomicCell::new(0u8);\n let mut sum = 0;\n b.iter(|| sum += a.load());\n test::black_box(sum);\n}\n\n#[bench]\nfn store_u8(b: &mut test::Bencher) {\n let a = AtomicCell::new(0u8);\n b.iter(|| a.store(1));\n}\n\n#[bench]\nfn fetch_add_u8(b: &mut test::Bencher) {\n let a = AtomicCell::new(0u8);\n b.iter(|| a.fetch_add(1));\n}\n\n#[bench]\nfn compare_exchange_u8(b: &mut test::Bencher) {\n let a = AtomicCell::new(0u8);\n let mut i = 0;\n b.iter(|| {\n let _ = a.compare_exchange(i, i.wrapping_add(1));\n i = i.wrapping_add(1);\n });\n}\n\n#[bench]\nfn concurrent_load_u8(b: &mut test::Bencher) {\n const THREADS: usize = 2;\n const STEPS: usize = 1_000_000;\n\n let start = Barrier::new(THREADS + 1);\n let end = Barrier::new(THREADS + 1);\n let exit = AtomicCell::new(false);\n\n let a = AtomicCell::new(0u8);\n\n thread::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n loop {\n start.wait();\n\n let mut sum = 0;\n for _ in 0..STEPS {\n sum += a.load();\n }\n test::black_box(sum);\n\n end.wait();\n if exit.load() {\n break;\n }\n }\n });\n }\n\n start.wait();\n end.wait();\n\n b.iter(|| {\n start.wait();\n end.wait();\n });\n\n start.wait();\n exit.store(true);\n end.wait();\n })\n .unwrap();\n}\n\n#[bench]\nfn load_usize(b: &mut test::Bencher) {\n let a = AtomicCell::new(0usize);\n let mut sum = 0;\n b.iter(|| sum += a.load());\n test::black_box(sum);\n}\n\n#[bench]\nfn store_usize(b: &mut test::Bencher) {\n let a = AtomicCell::new(0usize);\n b.iter(|| a.store(1));\n}\n\n#[bench]\nfn fetch_add_usize(b: &mut test::Bencher) {\n let a = AtomicCell::new(0usize);\n b.iter(|| a.fetch_add(1));\n}\n\n#[bench]\nfn compare_exchange_usize(b: &mut test::Bencher) {\n let a = AtomicCell::new(0usize);\n let mut i = 0;\n b.iter(|| {\n let _ = a.compare_exchange(i, i.wrapping_add(1));\n i = i.wrapping_add(1);\n });\n}\n\n#[bench]\nfn concurrent_load_usize(b: &mut test::Bencher) {\n const THREADS: usize = 2;\n const STEPS: usize = 1_000_000;\n\n let start = Barrier::new(THREADS + 1);\n let end = Barrier::new(THREADS + 1);\n let exit = AtomicCell::new(false);\n\n let a = AtomicCell::new(0usize);\n\n thread::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n loop {\n start.wait();\n\n let mut sum = 0;\n for _ in 0..STEPS {\n sum += a.load();\n }\n test::black_box(sum);\n\n end.wait();\n if exit.load() {\n break;\n }\n }\n });\n }\n\n start.wait();\n end.wait();\n\n b.iter(|| {\n start.wait();\n end.wait();\n });\n\n start.wait();\n exit.store(true);\n end.wait();\n })\n .unwrap();\n}\n" }, { "path": "crossbeam-utils/build.rs", "content": "// The rustc-cfg listed below are considered public API, but it is *unstable*\n// and outside of the normal semver guarantees:\n//\n// - `crossbeam_no_atomic`\n// Assume the target does *not* support any atomic operations.\n// This is usually detected automatically by the build script, but you may\n// need to enable it manually when building for custom targets or using\n// non-cargo build systems that don't run the build script.\n//\n// With the exceptions mentioned above, the rustc-cfg emitted by the build\n// script are *not* public API.\n\nuse std::env;\n\ninclude!(\"no_atomic.rs\");\ninclude!(\"build-common.rs\");\n\nfn main() {\n println!(\"cargo:rerun-if-changed=no_atomic.rs\");\n println!(\n \"cargo:rustc-check-cfg=cfg(crossbeam_no_atomic,crossbeam_sanitize_thread,crossbeam_atomic_cell_force_fallback)\"\n );\n\n let target = match env::var(\"TARGET\") {\n Ok(target) => convert_custom_linux_target(target),\n Err(e) => {\n println!(\n \"cargo:warning={}: unable to get TARGET environment variable: {}\",\n env!(\"CARGO_PKG_NAME\"),\n e\n );\n return;\n }\n };\n\n // Note that this is `no_`*, not `has_*`. This allows treating as the latest\n // stable rustc is used when the build script doesn't run. This is useful\n // for non-cargo build systems that don't run the build script.\n if NO_ATOMIC.contains(&&*target) {\n println!(\"cargo:rustc-cfg=crossbeam_no_atomic\");\n }\n\n // `cfg(sanitize = \"..\")` is not stabilized.\n if let Ok(sanitize) = env::var(\"CARGO_CFG_SANITIZE\") {\n if sanitize.contains(\"thread\") {\n println!(\"cargo:rustc-cfg=crossbeam_sanitize_thread\");\n }\n println!(\"cargo:rustc-cfg=crossbeam_atomic_cell_force_fallback\");\n }\n}\n" }, { "path": "crossbeam-utils/src/alloc_helper.rs", "content": "use core::{alloc::Layout, ptr::NonNull};\n\n// Based on unstable alloc::alloc::Global.\n//\n// Note: unlike alloc::alloc::Global that returns NonNull<[u8]>,\n// this returns NonNull.\npub(crate) struct Global;\n#[allow(clippy::unused_self)]\nimpl Global {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Option> {\n // Layout::dangling is unstable\n #[inline]\n #[must_use]\n fn dangling(layout: Layout) -> NonNull {\n // SAFETY: align is guaranteed to be non-zero\n unsafe { NonNull::new_unchecked(without_provenance_mut::(layout.align())) }\n }\n\n match layout.size() {\n 0 => Some(dangling(layout)),\n // SAFETY: `layout` is non-zero in size,\n _size => unsafe {\n #[allow(clippy::disallowed_methods)] // we are in alloc_helper\n let raw_ptr = if zeroed {\n alloc::alloc::alloc_zeroed(layout)\n } else {\n alloc::alloc::alloc(layout)\n };\n NonNull::new(raw_ptr)\n },\n }\n }\n #[allow(dead_code)]\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn allocate(self, layout: Layout) -> Option> {\n self.alloc_impl(layout, false)\n }\n #[allow(dead_code)]\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn allocate_zeroed(self, layout: Layout) -> Option> {\n self.alloc_impl(layout, true)\n }\n #[allow(dead_code)]\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) unsafe fn deallocate(self, ptr: NonNull, layout: Layout) {\n if layout.size() != 0 {\n // SAFETY:\n // * We have checked that `layout` is non-zero in size.\n // * The caller is obligated to provide a layout that \"fits\", and in this case,\n // \"fit\" always means a layout that is equal to the original, because our\n // `allocate()`, `grow()`, and `shrink()` implementations never returns a larger\n // allocation than requested.\n // * Other conditions must be upheld by the caller, as per `Allocator::deallocate()`'s\n // safety documentation.\n #[allow(clippy::disallowed_methods)] // we are in alloc_helper\n unsafe {\n alloc::alloc::dealloc(ptr.as_ptr(), layout)\n }\n }\n }\n}\n\n#[inline(always)]\n#[must_use]\nconst fn without_provenance_mut(addr: usize) -> *mut T {\n // An int-to-pointer transmute currently has exactly the intended semantics: it creates a\n // pointer without provenance. Note that this is *not* a stable guarantee about transmute\n // semantics, it relies on sysroot crates having special status.\n // SAFETY: every valid integer is also a valid pointer (as long as you don't dereference that\n // pointer).\n #[cfg(miri)]\n unsafe {\n core::mem::transmute(addr)\n }\n // Using transmute doesn't work with CHERI: https://github.com/kent-weak-memory/rust/blob/0c0ca909de877f889629057e1ddf139527446d75/library/core/src/ptr/mod.rs#L607\n #[cfg(not(miri))]\n {\n addr as *mut T\n }\n}\n" }, { "path": "crossbeam-utils/src/atomic/atomic_cell.rs", "content": "// Necessary for implementing atomic methods for `AtomicUnit`\n#![allow(clippy::unit_arg)]\n\nuse core::{\n cell::UnsafeCell,\n cmp, fmt,\n mem::{self, ManuallyDrop, MaybeUninit},\n panic::{RefUnwindSafe, UnwindSafe},\n ptr,\n};\n\nuse super::seq_lock::SeqLock;\nuse crate::{\n CachePadded,\n primitive::sync::atomic::{self, Ordering},\n};\n\n/// A thread-safe mutable memory location.\n///\n/// This type is equivalent to [`Cell`], except it can also be shared among multiple threads.\n///\n/// Operations on `AtomicCell`s use atomic instructions whenever possible, and synchronize using\n/// global locks otherwise. You can call [`AtomicCell::::is_lock_free()`] to check whether\n/// atomic instructions or locks will be used.\n///\n/// Atomic loads use the [`Acquire`] ordering and atomic stores use the [`Release`] ordering.\n///\n/// [`Cell`]: std::cell::Cell\n/// [`AtomicCell::::is_lock_free()`]: AtomicCell::is_lock_free\n/// [`Acquire`]: std::sync::atomic::Ordering::Acquire\n/// [`Release`]: std::sync::atomic::Ordering::Release\n#[repr(transparent)]\npub struct AtomicCell {\n /// The inner value.\n ///\n /// If this value can be transmuted into a primitive atomic type, it will be treated as such.\n /// Otherwise, all potentially concurrent operations on this data will be protected by a global\n /// lock.\n ///\n /// Using MaybeUninit to prevent code outside the cell from observing partially initialized state:\n /// \n /// (This rustc bug has been fixed in Rust 1.64.)\n ///\n /// Note:\n /// - we'll never store uninitialized `T` due to our API only using initialized `T`.\n /// - this `MaybeUninit` does *not* fix .\n value: UnsafeCell>,\n}\n\nunsafe impl Send for AtomicCell {}\nunsafe impl Sync for AtomicCell {}\n\nimpl UnwindSafe for AtomicCell {}\nimpl RefUnwindSafe for AtomicCell {}\n\nimpl AtomicCell {\n /// Creates a new atomic cell initialized with `val`.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(7);\n /// ```\n pub const fn new(val: T) -> Self {\n Self {\n value: UnsafeCell::new(MaybeUninit::new(val)),\n }\n }\n\n /// Consumes the atomic and returns the contained value.\n ///\n /// This is safe because passing `self` by value guarantees that no other threads are\n /// concurrently accessing the atomic data.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(7);\n /// let v = a.into_inner();\n ///\n /// assert_eq!(v, 7);\n /// ```\n pub const fn into_inner(self) -> T {\n // HACK: This is equivalent to transmute_copy by value, but available in const\n // context even on older rustc (const transmute_copy requires Rust 1.74), and\n // can work around \"cannot borrow here, since the borrowed element may contain\n // interior mutability\" error occurs (until const_refs_to_cell stabilized, i.e.,\n // Rust 1.83) when using transmute_copy with generic type in const context\n // (because this is a by-value transmutation that doesn't create a reference to\n // the source value).\n /// # Safety\n ///\n /// This function has the same safety requirements as [`core::mem::transmute_copy`].\n ///\n /// Since this is a by-value transmutation, it copies the bits from the source value\n /// into the destination value, then forgets the original, as with the [`core::mem::transmute`].\n #[inline]\n #[must_use]\n const unsafe fn transmute_copy_by_val(src: Src) -> Dst {\n #[repr(C)]\n union ConstHack {\n src: ManuallyDrop,\n dst: ManuallyDrop,\n }\n assert!(mem::size_of::() >= mem::size_of::()); // assertion copied from transmute_copy\n // SAFETY: ConstHack is #[repr(C)] union, and the caller must guarantee that\n // transmuting Src to Dst is safe.\n ManuallyDrop::into_inner(unsafe {\n ConstHack:: {\n src: ManuallyDrop::new(src),\n }\n .dst\n })\n }\n\n // SAFETY:\n // - Self is repr(transparent) over `UnsafeCell>` and\n // `UnsafeCell>` and `T` has the same layout.\n // - passing `self` by value guarantees that no other threads are concurrently\n // accessing the atomic data\n // (Equivalent to UnsafeCell::into_inner which is unstable in const context.)\n unsafe { transmute_copy_by_val(self) }\n }\n\n /// Returns `true` if operations on values of this type are lock-free.\n ///\n /// If the compiler or the platform doesn't support the necessary atomic instructions,\n /// `AtomicCell` will use global locks for every potentially concurrent atomic operation.\n ///\n /// # Examples\n ///\n /// ```\n /// # // Always use fallback for now on environments that do not support inline assembly.\n /// # if cfg!(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)) { return; }\n /// # atomic_maybe_uninit::cfg_no_atomic_cas! { if true { return; } }\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// // This type is internally represented as `AtomicUsize` so we can just use atomic\n /// // operations provided by it.\n /// assert_eq!(AtomicCell::::is_lock_free(), true);\n ///\n /// // A wrapper struct around `isize`.\n /// struct Foo {\n /// bar: isize,\n /// }\n /// // `AtomicCell` will be internally represented as `AtomicIsize`.\n /// assert_eq!(AtomicCell::::is_lock_free(), true);\n ///\n /// // Operations on zero-sized types are always lock-free.\n /// assert_eq!(AtomicCell::<()>::is_lock_free(), true);\n ///\n /// // Very large types cannot be represented as any of the standard atomic types, so atomic\n /// // operations on them will have to use global locks for synchronization.\n /// assert_eq!(AtomicCell::<[u8; 1000]>::is_lock_free(), false);\n /// ```\n pub const fn is_lock_free() -> bool {\n atomic_is_lock_free::()\n }\n\n /// Stores `val` into the atomic cell.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(7);\n ///\n /// assert_eq!(a.load(), 7);\n /// a.store(8);\n /// assert_eq!(a.load(), 8);\n /// ```\n pub fn store(&self, val: T) {\n if mem::needs_drop::() {\n drop(self.swap(val));\n } else {\n unsafe {\n atomic_store(self.as_ptr(), val);\n }\n }\n }\n\n /// Stores `val` into the atomic cell and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(7);\n ///\n /// assert_eq!(a.load(), 7);\n /// assert_eq!(a.swap(8), 7);\n /// assert_eq!(a.load(), 8);\n /// ```\n pub fn swap(&self, val: T) -> T {\n unsafe { atomic_swap(self.as_ptr(), val) }\n }\n\n /// Returns a raw pointer to the underlying data in this atomic cell.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(5);\n ///\n /// let ptr = a.as_ptr();\n /// ```\n #[inline]\n pub const fn as_ptr(&self) -> *mut T {\n self.value.get().cast::()\n }\n}\n\nimpl AtomicCell {\n /// Takes the value of the atomic cell, leaving `Default::default()` in its place.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(5);\n /// let five = a.take();\n ///\n /// assert_eq!(five, 5);\n /// assert_eq!(a.into_inner(), 0);\n /// ```\n pub fn take(&self) -> T {\n self.swap(Default::default())\n }\n}\n\nimpl AtomicCell {\n /// Loads a value from the atomic cell.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(7);\n ///\n /// assert_eq!(a.load(), 7);\n /// ```\n pub fn load(&self) -> T {\n unsafe { atomic_load(self.as_ptr()) }\n }\n}\n\nimpl AtomicCell {\n /// If the current value equals `current`, stores `new` into the atomic cell.\n ///\n /// The return value is a result indicating whether the new value was written and containing\n /// the previous value. On success this value is guaranteed to be equal to `current`.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(1);\n ///\n /// assert_eq!(a.compare_exchange(2, 3), Err(1));\n /// assert_eq!(a.load(), 1);\n ///\n /// assert_eq!(a.compare_exchange(1, 2), Ok(1));\n /// assert_eq!(a.load(), 2);\n /// ```\n pub fn compare_exchange(&self, current: T, new: T) -> Result {\n unsafe { atomic_compare_exchange_weak(self.as_ptr(), current, new) }\n }\n\n /// Fetches the value, and applies a function to it that returns an optional\n /// new value. Returns a `Result` of `Ok(previous_value)` if the function returned `Some(_)`, else\n /// `Err(previous_value)`.\n ///\n /// Note: This may call the function multiple times if the value has been changed from other threads in\n /// the meantime, as long as the function returns `Some(_)`, but the function will have been applied\n /// only once to the stored value.\n ///\n /// # Examples\n ///\n /// ```rust\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(7);\n /// assert_eq!(a.fetch_update(|_| None), Err(7));\n /// assert_eq!(a.fetch_update(|a| Some(a + 1)), Ok(7));\n /// assert_eq!(a.fetch_update(|a| Some(a + 1)), Ok(8));\n /// assert_eq!(a.load(), 9);\n /// ```\n #[inline]\n pub fn fetch_update(&self, mut f: F) -> Result\n where\n F: FnMut(T) -> Option,\n {\n let mut prev = self.load();\n while let Some(next) = f(prev) {\n match self.compare_exchange(prev, next) {\n x @ Ok(_) => return x,\n Err(next_prev) => prev = next_prev,\n }\n }\n Err(prev)\n }\n}\n\n// `MaybeUninit` prevents `T` from being dropped, so we need to implement `Drop`\n// for `AtomicCell` to avoid leaks of non-`Copy` types.\nimpl Drop for AtomicCell {\n fn drop(&mut self) {\n if mem::needs_drop::() {\n // SAFETY:\n // - the mutable reference guarantees that no other threads are concurrently accessing the atomic data\n // - the raw pointer passed in is valid because we got it from a reference\n // - `MaybeUninit` prevents double dropping `T`\n unsafe {\n self.as_ptr().drop_in_place();\n }\n }\n }\n}\n\nmacro_rules! atomic {\n // If values of type `$t` can be transmuted into values of the primitive atomic type `$atomic`,\n // declares variable `$a` of type `$atomic` and executes `$atomic_op`, breaking out of the loop.\n (@check, $t:ty, $atomic:ty, $a:ident, $atomic_op:expr) => {\n if can_transmute::<$t, $atomic>() {\n let $a: &$atomic;\n break $atomic_op;\n }\n };\n\n // If values of type `$t` can be transmuted into values of a primitive atomic type, declares\n // variable `$a` of that type and executes `$atomic_op`. Otherwise, just executes\n // `$fallback_op`.\n ($t:ty, $a:ident, $atomic_op:expr, $fallback_op:expr) => {\n loop {\n atomic!(@check, $t, AtomicUnit, $a, $atomic_op);\n\n // Always use fallback for now on environments that do not support inline assembly.\n #[cfg(not(any(\n miri,\n crossbeam_loom,\n crossbeam_atomic_cell_force_fallback,\n )))]\n atomic_maybe_uninit::cfg_has_atomic_cas! {\n atomic_maybe_uninit::cfg_has_atomic_8! {\n atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op);\n }\n atomic_maybe_uninit::cfg_has_atomic_16! {\n atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op);\n }\n atomic_maybe_uninit::cfg_has_atomic_32! {\n atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op);\n }\n atomic_maybe_uninit::cfg_has_atomic_64! {\n atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op);\n }\n atomic_maybe_uninit::cfg_has_atomic_128! {\n atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op);\n }\n }\n\n break $fallback_op;\n }\n };\n}\n\nmacro_rules! impl_arithmetic {\n ($t:ty, #[cfg($target_has_atomic:meta)] $atomic:ident, $example:tt) => {\n impl AtomicCell<$t> {\n /// Increments the current value by `val` and returns the previous value.\n ///\n /// The addition wraps on overflow.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n #[doc = $example]\n ///\n /// assert_eq!(a.fetch_add(3), 7);\n /// assert_eq!(a.load(), 10);\n /// ```\n #[inline]\n pub fn fetch_add(&self, val: $t) -> $t {\n atomic! {\n $t, _a,\n {\n #[cfg($target_has_atomic)]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) };\n a.fetch_add(val, Ordering::AcqRel)\n }\n #[cfg(not($target_has_atomic))]\n {\n self.fetch_update(|old| Some(old.wrapping_add(val))).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value = value.wrapping_add(val);\n old\n }\n }\n }\n\n /// Decrements the current value by `val` and returns the previous value.\n ///\n /// The subtraction wraps on overflow.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n #[doc = $example]\n ///\n /// assert_eq!(a.fetch_sub(3), 7);\n /// assert_eq!(a.load(), 4);\n /// ```\n #[inline]\n pub fn fetch_sub(&self, val: $t) -> $t {\n atomic! {\n $t, _a,\n {\n #[cfg($target_has_atomic)]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) };\n a.fetch_sub(val, Ordering::AcqRel)\n }\n #[cfg(not($target_has_atomic))]\n {\n self.fetch_update(|old| Some(old.wrapping_sub(val))).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value = value.wrapping_sub(val);\n old\n }\n }\n }\n\n /// Applies bitwise \"and\" to the current value and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n #[doc = $example]\n ///\n /// assert_eq!(a.fetch_and(3), 7);\n /// assert_eq!(a.load(), 3);\n /// ```\n #[inline]\n pub fn fetch_and(&self, val: $t) -> $t {\n atomic! {\n $t, _a,\n {\n #[cfg($target_has_atomic)]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) };\n a.fetch_and(val, Ordering::AcqRel)\n }\n #[cfg(not($target_has_atomic))]\n {\n self.fetch_update(|old| Some(old & val)).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value &= val;\n old\n }\n }\n }\n\n /// Applies bitwise \"nand\" to the current value and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n #[doc = $example]\n ///\n /// assert_eq!(a.fetch_nand(3), 7);\n /// assert_eq!(a.load(), !(7 & 3));\n /// ```\n #[inline]\n pub fn fetch_nand(&self, val: $t) -> $t {\n atomic! {\n $t, _a,\n {\n #[cfg($target_has_atomic)]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) };\n a.fetch_nand(val, Ordering::AcqRel)\n }\n #[cfg(not($target_has_atomic))]\n {\n self.fetch_update(|old| Some(!(old & val))).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value = !(old & val);\n old\n }\n }\n }\n\n /// Applies bitwise \"or\" to the current value and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n #[doc = $example]\n ///\n /// assert_eq!(a.fetch_or(16), 7);\n /// assert_eq!(a.load(), 23);\n /// ```\n #[inline]\n pub fn fetch_or(&self, val: $t) -> $t {\n atomic! {\n $t, _a,\n {\n #[cfg($target_has_atomic)]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) };\n a.fetch_or(val, Ordering::AcqRel)\n }\n #[cfg(not($target_has_atomic))]\n {\n self.fetch_update(|old| Some(old | val)).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value |= val;\n old\n }\n }\n }\n\n /// Applies bitwise \"xor\" to the current value and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n #[doc = $example]\n ///\n /// assert_eq!(a.fetch_xor(2), 7);\n /// assert_eq!(a.load(), 5);\n /// ```\n #[inline]\n pub fn fetch_xor(&self, val: $t) -> $t {\n atomic! {\n $t, _a,\n {\n #[cfg($target_has_atomic)]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) };\n a.fetch_xor(val, Ordering::AcqRel)\n }\n #[cfg(not($target_has_atomic))]\n {\n self.fetch_update(|old| Some(old ^ val)).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value ^= val;\n old\n }\n }\n }\n\n /// Compares and sets the maximum of the current value and `val`,\n /// and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n #[doc = $example]\n ///\n /// assert_eq!(a.fetch_max(9), 7);\n /// assert_eq!(a.load(), 9);\n /// ```\n #[inline]\n pub fn fetch_max(&self, val: $t) -> $t {\n atomic! {\n $t, _a,\n {\n #[cfg($target_has_atomic)]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) };\n a.fetch_max(val, Ordering::AcqRel)\n }\n #[cfg(not($target_has_atomic))]\n {\n self.fetch_update(|old| Some(cmp::max(old, val))).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value = cmp::max(old, val);\n old\n }\n }\n }\n\n /// Compares and sets the minimum of the current value and `val`,\n /// and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n #[doc = $example]\n ///\n /// assert_eq!(a.fetch_min(2), 7);\n /// assert_eq!(a.load(), 2);\n /// ```\n #[inline]\n pub fn fetch_min(&self, val: $t) -> $t {\n atomic! {\n $t, _a,\n {\n #[cfg($target_has_atomic)]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) };\n a.fetch_min(val, Ordering::AcqRel)\n }\n #[cfg(not($target_has_atomic))]\n {\n self.fetch_update(|old| Some(cmp::min(old, val))).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value = cmp::min(old, val);\n old\n }\n }\n }\n }\n };\n}\n\nimpl_arithmetic!(\n u8,\n #[cfg(target_has_atomic = \"8\")]\n AtomicU8,\n \"let a = AtomicCell::new(7u8);\"\n);\nimpl_arithmetic!(\n i8,\n #[cfg(target_has_atomic = \"8\")]\n AtomicI8,\n \"let a = AtomicCell::new(7i8);\"\n);\nimpl_arithmetic!(\n u16,\n #[cfg(target_has_atomic = \"16\")]\n AtomicU16,\n \"let a = AtomicCell::new(7u16);\"\n);\nimpl_arithmetic!(\n i16,\n #[cfg(target_has_atomic = \"16\")]\n AtomicI16,\n \"let a = AtomicCell::new(7i16);\"\n);\n\nimpl_arithmetic!(\n u32,\n #[cfg(target_has_atomic = \"32\")]\n AtomicU32,\n \"let a = AtomicCell::new(7u32);\"\n);\nimpl_arithmetic!(\n i32,\n #[cfg(target_has_atomic = \"32\")]\n AtomicI32,\n \"let a = AtomicCell::new(7i32);\"\n);\n\nimpl_arithmetic!(\n u64,\n #[cfg(target_has_atomic = \"64\")]\n AtomicU64,\n \"let a = AtomicCell::new(7u64);\"\n);\nimpl_arithmetic!(\n i64,\n #[cfg(target_has_atomic = \"64\")]\n AtomicI64,\n \"let a = AtomicCell::new(7i64);\"\n);\n\n// TODO: core::sync::atomic::AtomicU128 is unstable\nimpl_arithmetic!(\n u128,\n #[cfg(any(/* always false */))]\n AtomicU128,\n \"let a = AtomicCell::new(7u128);\"\n);\nimpl_arithmetic!(\n i128,\n #[cfg(any(/* always false */))]\n AtomicI128,\n \"let a = AtomicCell::new(7i128);\"\n);\n\nimpl_arithmetic!(\n usize,\n #[cfg(target_has_atomic = \"ptr\")]\n AtomicUsize,\n \"let a = AtomicCell::new(7usize);\"\n);\nimpl_arithmetic!(\n isize,\n #[cfg(target_has_atomic = \"ptr\")]\n AtomicIsize,\n \"let a = AtomicCell::new(7isize);\"\n);\n\nimpl AtomicCell {\n /// Applies logical \"and\" to the current value and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(true);\n ///\n /// assert_eq!(a.fetch_and(true), true);\n /// assert_eq!(a.load(), true);\n ///\n /// assert_eq!(a.fetch_and(false), true);\n /// assert_eq!(a.load(), false);\n /// ```\n #[inline]\n pub fn fetch_and(&self, val: bool) -> bool {\n atomic! {\n bool, _a,\n {\n #[cfg(target_has_atomic = \"8\")]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) };\n a.fetch_and(val, Ordering::AcqRel)\n }\n #[cfg(not(target_has_atomic = \"8\"))]\n {\n self.fetch_update(|old| Some(old & val)).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value &= val;\n old\n }\n }\n }\n\n /// Applies logical \"nand\" to the current value and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(true);\n ///\n /// assert_eq!(a.fetch_nand(false), true);\n /// assert_eq!(a.load(), true);\n ///\n /// assert_eq!(a.fetch_nand(true), true);\n /// assert_eq!(a.load(), false);\n ///\n /// assert_eq!(a.fetch_nand(false), false);\n /// assert_eq!(a.load(), true);\n /// ```\n #[inline]\n pub fn fetch_nand(&self, val: bool) -> bool {\n atomic! {\n bool, _a,\n {\n #[cfg(target_has_atomic = \"8\")]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) };\n a.fetch_nand(val, Ordering::AcqRel)\n }\n #[cfg(not(target_has_atomic = \"8\"))]\n {\n self.fetch_update(|old| Some(!(old & val))).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value = !(old & val);\n old\n }\n }\n }\n\n /// Applies logical \"or\" to the current value and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(false);\n ///\n /// assert_eq!(a.fetch_or(false), false);\n /// assert_eq!(a.load(), false);\n ///\n /// assert_eq!(a.fetch_or(true), false);\n /// assert_eq!(a.load(), true);\n /// ```\n #[inline]\n pub fn fetch_or(&self, val: bool) -> bool {\n atomic! {\n bool, _a,\n {\n #[cfg(target_has_atomic = \"8\")]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) };\n a.fetch_or(val, Ordering::AcqRel)\n }\n #[cfg(not(target_has_atomic = \"8\"))]\n {\n self.fetch_update(|old| Some(old | val)).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value |= val;\n old\n }\n }\n }\n\n /// Applies logical \"xor\" to the current value and returns the previous value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::atomic::AtomicCell;\n ///\n /// let a = AtomicCell::new(true);\n ///\n /// assert_eq!(a.fetch_xor(false), true);\n /// assert_eq!(a.load(), true);\n ///\n /// assert_eq!(a.fetch_xor(true), true);\n /// assert_eq!(a.load(), false);\n /// ```\n #[inline]\n pub fn fetch_xor(&self, val: bool) -> bool {\n atomic! {\n bool, _a,\n {\n #[cfg(target_has_atomic = \"8\")]\n {\n let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) };\n a.fetch_xor(val, Ordering::AcqRel)\n }\n #[cfg(not(target_has_atomic = \"8\"))]\n {\n self.fetch_update(|old| Some(old ^ val)).unwrap()\n }\n },\n {\n let _guard = lock(self.as_ptr() as usize).write();\n let value = unsafe { &mut *(self.as_ptr()) };\n let old = *value;\n *value ^= val;\n old\n }\n }\n }\n}\n\nimpl Default for AtomicCell {\n fn default() -> Self {\n Self::new(T::default())\n }\n}\n\nimpl From for AtomicCell {\n #[inline]\n fn from(val: T) -> Self {\n Self::new(val)\n }\n}\n\nimpl fmt::Debug for AtomicCell {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"AtomicCell\")\n .field(\"value\", &self.load())\n .finish()\n }\n}\n\n/// Returns `true` if values of type `A` can be transmuted into values of type `B`.\nconst fn can_transmute() -> bool {\n // Sizes must be equal, but alignment of `A` must be greater or equal than that of `B`.\n (mem::size_of::() == mem::size_of::()) & (mem::align_of::() >= mem::align_of::())\n}\n\n/// Returns a reference to the global lock associated with the `AtomicCell` at address `addr`.\n///\n/// This function is used to protect atomic data which doesn't fit into any of the primitive atomic\n/// types in `std::sync::atomic`. Operations on such atomics must therefore use a global lock.\n///\n/// However, there is not only one global lock but an array of many locks, and one of them is\n/// picked based on the given address. Having many locks reduces contention and improves\n/// scalability.\n#[inline]\n#[must_use]\nfn lock(addr: usize) -> &'static SeqLock {\n // The number of locks is a prime number because we want to make sure `addr % LEN` gets\n // dispersed across all locks.\n //\n // Note that addresses are always aligned to some power of 2, depending on type `T` in\n // `AtomicCell`. If `LEN` was an even number, then `addr % LEN` would be an even number,\n // too, which means only half of the locks would get utilized!\n //\n // It is also possible for addresses to accidentally get aligned to a number that is not a\n // power of 2. Consider this example:\n //\n // ```\n // #[repr(C)]\n // struct Foo {\n // a: AtomicCell,\n // b: u8,\n // c: u8,\n // }\n // ```\n //\n // Now, if we have a slice of type `&[Foo]`, it is possible that field `a` in all items gets\n // stored at addresses that are multiples of 3. It'd be too bad if `LEN` was divisible by 3.\n // In order to protect from such cases, we simply choose a large prime number for `LEN`.\n const LEN: usize = 67;\n const L: CachePadded = CachePadded::new(SeqLock::new());\n static LOCKS: [CachePadded; LEN] = [L; LEN];\n\n // If the modulus is a constant number, the compiler will use crazy math to transform this into\n // a sequence of cheap arithmetic operations rather than using the slow modulo instruction.\n &LOCKS[addr % LEN]\n}\n\n/// An atomic `()`.\n///\n/// All operations are noops.\nstruct AtomicUnit;\n\nimpl AtomicUnit {\n #[inline]\n fn load(&self, _order: Ordering) {}\n\n #[inline]\n fn store(&self, _val: (), _order: Ordering) {}\n\n #[inline]\n fn swap(&self, _val: (), _order: Ordering) {}\n\n #[inline]\n fn compare_exchange_weak(\n &self,\n _current: (),\n _new: (),\n _success: Ordering,\n _failure: Ordering,\n ) -> Result<(), ()> {\n Ok(())\n }\n}\n\n/// Returns `true` if operations on `AtomicCell` are lock-free.\nconst fn atomic_is_lock_free() -> bool {\n atomic! { T, _a, true, false }\n}\n\n/// Atomically reads data from `src`.\n///\n/// This operation uses the `Acquire` ordering. If possible, an atomic instructions is used, and a\n/// global lock otherwise.\nunsafe fn atomic_load(src: *mut T) -> T\nwhere\n T: Copy,\n{\n atomic! {\n T, a,\n {\n a = unsafe { &*(src as *const _ as *const _) };\n unsafe { mem::transmute_copy(&a.load(Ordering::Acquire)) }\n },\n {\n let lock = lock(src as usize);\n\n // Try doing an optimistic read first.\n if let Some(stamp) = lock.optimistic_read() {\n // We need a volatile read here because other threads might concurrently modify the\n // value. In theory, data races are *always* UB, even if we use volatile reads and\n // discard the data when a data race is detected. The proper solution would be to\n // do atomic reads and atomic writes, but we can't atomically read and write all\n // kinds of data since `AtomicU8` is not available on stable Rust yet.\n // Load as `MaybeUninit` because we may load a value that is not valid as `T`.\n let val = unsafe { ptr::read_volatile(src.cast::>()) };\n\n if lock.validate_read(stamp) {\n return unsafe { val.assume_init() };\n }\n }\n\n // Grab a regular write lock so that writers don't starve this load.\n let guard = lock.write();\n let val = unsafe { ptr::read(src) };\n // The value hasn't been changed. Drop the guard without incrementing the stamp.\n guard.abort();\n val\n }\n }\n}\n\n/// Atomically writes `val` to `dst`.\n///\n/// This operation uses the `Release` ordering. If possible, an atomic instructions is used, and a\n/// global lock otherwise.\nunsafe fn atomic_store(dst: *mut T, val: T) {\n atomic! {\n T, a,\n {\n a = unsafe { &*(dst as *const _ as *const _) };\n a.store(unsafe { mem::transmute_copy(&val) }, Ordering::Release);\n mem::forget(val);\n },\n {\n let _guard = lock(dst as usize).write();\n unsafe { ptr::write(dst, val) }\n }\n }\n}\n\n/// Atomically swaps data at `dst` with `val`.\n///\n/// This operation uses the `AcqRel` ordering. If possible, an atomic instructions is used, and a\n/// global lock otherwise.\nunsafe fn atomic_swap(dst: *mut T, val: T) -> T {\n atomic! {\n T, a,\n {\n a = unsafe { &*(dst as *const _ as *const _) };\n let res = unsafe { mem::transmute_copy(&a.swap(mem::transmute_copy(&val), Ordering::AcqRel)) };\n mem::forget(val);\n res\n },\n {\n let _guard = lock(dst as usize).write();\n unsafe { ptr::replace(dst, val) }\n }\n }\n}\n\n/// Atomically compares data at `dst` to `current` and, if equal byte-for-byte, exchanges data at\n/// `dst` with `new`.\n///\n/// Returns the old value on success, or the current value at `dst` on failure.\n///\n/// This operation uses the `AcqRel` ordering. If possible, an atomic instructions is used, and a\n/// global lock otherwise.\n#[allow(clippy::let_unit_value)]\nunsafe fn atomic_compare_exchange_weak(dst: *mut T, mut current: T, new: T) -> Result\nwhere\n T: Copy + Eq,\n{\n atomic! {\n T, a,\n {\n a = unsafe { &*(dst as *const _ as *const _) };\n let mut current_raw = unsafe { mem::transmute_copy(¤t) };\n let new_raw = unsafe { mem::transmute_copy(&new) };\n\n loop {\n match a.compare_exchange_weak(\n current_raw,\n new_raw,\n Ordering::AcqRel,\n Ordering::Acquire,\n ) {\n Ok(_) => break Ok(current),\n Err(previous_raw) => {\n let previous = unsafe { mem::transmute_copy(&previous_raw) };\n\n if !T::eq(&previous, ¤t) {\n break Err(previous);\n }\n\n // The compare-exchange operation has failed and didn't store `new`. The\n // failure is either spurious, or `previous` was semantically equal to\n // `current` but not byte-equal. Let's retry with `previous` as the new\n // `current`.\n current = previous;\n current_raw = previous_raw;\n }\n }\n }\n },\n {\n let guard = lock(dst as usize).write();\n\n let old = unsafe { ptr::read(dst) };\n if T::eq(&old, ¤t) {\n unsafe { ptr::write(dst, new) }\n Ok(old)\n } else {\n // The value hasn't been changed. Drop the guard without incrementing the stamp.\n guard.abort();\n Err(old)\n }\n }\n }\n}\n" }, { "path": "crossbeam-utils/src/atomic/consume.rs", "content": "#[cfg(not(crossbeam_no_atomic))]\nuse core::sync::atomic::Ordering;\n\n/// Trait which allows reading from primitive atomic types with \"consume\" ordering.\npub trait AtomicConsume {\n /// Type returned by `load_consume`.\n type Val;\n\n /// Loads a value from the atomic using a \"consume\" memory ordering.\n ///\n /// This is similar to the \"acquire\" ordering, except that an ordering is\n /// only guaranteed with operations that \"depend on\" the result of the load.\n /// However consume loads are usually much faster than acquire loads on\n /// architectures with a weak memory model since they don't require memory\n /// fence instructions.\n ///\n /// The exact definition of \"depend on\" is a bit vague, but it works as you\n /// would expect in practice since a lot of software, especially the Linux\n /// kernel, rely on this behavior.\n ///\n /// This is currently only implemented on ARM and AArch64, where a fence\n /// can be avoided. On other architectures this will fall back to a simple\n /// `load(Ordering::Acquire)`.\n fn load_consume(&self) -> Self::Val;\n}\n\n#[cfg(not(crossbeam_no_atomic))]\n// Miri and Loom don't support \"consume\" ordering and ThreadSanitizer doesn't treat\n// load(Relaxed) + compiler_fence(Acquire) as \"consume\" load.\n// LLVM generates machine code equivalent to fence(Acquire) in compiler_fence(Acquire)\n// on PowerPC, MIPS, etc. (https://godbolt.org/z/hffvjvW7h), so for now the fence\n// can be actually avoided here only on ARM and AArch64. See also\n// https://github.com/rust-lang/rust/issues/62256.\n#[cfg(all(\n any(target_arch = \"arm\", target_arch = \"aarch64\"),\n not(any(miri, crossbeam_loom, crossbeam_sanitize_thread)),\n))]\nmacro_rules! impl_consume {\n () => {\n #[inline]\n fn load_consume(&self) -> Self::Val {\n use crate::primitive::sync::atomic::compiler_fence;\n let result = self.load(Ordering::Relaxed);\n compiler_fence(Ordering::Acquire);\n result\n }\n };\n}\n\n#[cfg(not(crossbeam_no_atomic))]\n#[cfg(not(all(\n any(target_arch = \"arm\", target_arch = \"aarch64\"),\n not(any(miri, crossbeam_loom, crossbeam_sanitize_thread)),\n)))]\nmacro_rules! impl_consume {\n () => {\n #[inline]\n fn load_consume(&self) -> Self::Val {\n self.load(Ordering::Acquire)\n }\n };\n}\n\nmacro_rules! impl_atomic {\n ($atomic:ident, $val:ty) => {\n #[cfg(not(crossbeam_no_atomic))]\n impl AtomicConsume for core::sync::atomic::$atomic {\n type Val = $val;\n impl_consume!();\n }\n #[cfg(crossbeam_loom)]\n impl AtomicConsume for loom::sync::atomic::$atomic {\n type Val = $val;\n impl_consume!();\n }\n };\n}\n\nimpl_atomic!(AtomicBool, bool);\nimpl_atomic!(AtomicUsize, usize);\nimpl_atomic!(AtomicIsize, isize);\nimpl_atomic!(AtomicU8, u8);\nimpl_atomic!(AtomicI8, i8);\nimpl_atomic!(AtomicU16, u16);\nimpl_atomic!(AtomicI16, i16);\n#[cfg(any(target_has_atomic = \"32\", not(target_pointer_width = \"16\")))]\nimpl_atomic!(AtomicU32, u32);\n#[cfg(any(target_has_atomic = \"32\", not(target_pointer_width = \"16\")))]\nimpl_atomic!(AtomicI32, i32);\n#[cfg(any(\n target_has_atomic = \"64\",\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n))]\nimpl_atomic!(AtomicU64, u64);\n#[cfg(any(\n target_has_atomic = \"64\",\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n))]\nimpl_atomic!(AtomicI64, i64);\n\n#[cfg(not(crossbeam_no_atomic))]\nimpl AtomicConsume for core::sync::atomic::AtomicPtr {\n type Val = *mut T;\n impl_consume!();\n}\n\n#[cfg(crossbeam_loom)]\nimpl AtomicConsume for loom::sync::atomic::AtomicPtr {\n type Val = *mut T;\n impl_consume!();\n}\n" }, { "path": "crossbeam-utils/src/atomic/mod.rs", "content": "//! Atomic types.\n//!\n//! * [`AtomicCell`], a thread-safe mutable memory location.\n//! * [`AtomicConsume`], for reading from primitive atomic types with \"consume\" ordering.\n\n#[cfg(target_has_atomic = \"ptr\")]\n#[cfg(not(crossbeam_loom))]\n// Use \"wide\" sequence lock if the pointer width <= 32 for preventing its counter against wrap\n// around.\n//\n// In narrow architectures (pointer width <= 16), the counter is still <= 32-bit and may be\n// vulnerable to wrap around. But it's mostly okay, since in such a primitive hardware, the\n// counter will not be increased that fast.\n// Note that Rust (and C99) pointers must be at least 16-bit (i.e., 8-bit targets are impossible): https://github.com/rust-lang/rust/pull/49305\n#[cfg_attr(\n any(target_pointer_width = \"16\", target_pointer_width = \"32\"),\n path = \"seq_lock_wide.rs\"\n)]\nmod seq_lock;\n\n#[cfg(target_has_atomic = \"ptr\")]\n// We cannot provide AtomicCell under cfg(crossbeam_loom) because loom's atomic\n// types have a different in-memory representation than the underlying type.\n// TODO: The latest loom supports fences, so fallback using seqlock may be available.\n#[cfg(not(crossbeam_loom))]\nmod atomic_cell;\n#[cfg(target_has_atomic = \"ptr\")]\n#[cfg(not(crossbeam_loom))]\npub use self::atomic_cell::AtomicCell;\n\nmod consume;\npub use self::consume::AtomicConsume;\n" }, { "path": "crossbeam-utils/src/atomic/seq_lock.rs", "content": "use core::{\n mem,\n sync::atomic::{self, AtomicUsize, Ordering},\n};\n\nuse crate::Backoff;\n\n/// A simple stamped lock.\npub(crate) struct SeqLock {\n /// The current state of the lock.\n ///\n /// All bits except the least significant one hold the current stamp. When locked, the state\n /// equals 1 and doesn't contain a valid stamp.\n state: AtomicUsize,\n}\n\nimpl SeqLock {\n pub(crate) const fn new() -> Self {\n Self {\n state: AtomicUsize::new(0),\n }\n }\n\n /// If not locked, returns the current stamp.\n ///\n /// This method should be called before optimistic reads.\n #[inline]\n pub(crate) fn optimistic_read(&self) -> Option {\n let state = self.state.load(Ordering::Acquire);\n if state == 1 { None } else { Some(state) }\n }\n\n /// Returns `true` if the current stamp is equal to `stamp`.\n ///\n /// This method should be called after optimistic reads to check whether they are valid. The\n /// argument `stamp` should correspond to the one returned by method `optimistic_read`.\n #[inline]\n pub(crate) fn validate_read(&self, stamp: usize) -> bool {\n atomic::fence(Ordering::Acquire);\n self.state.load(Ordering::Relaxed) == stamp\n }\n\n /// Grabs the lock for writing.\n #[inline]\n pub(crate) fn write(&'static self) -> SeqLockWriteGuard {\n let backoff = Backoff::new();\n loop {\n let previous = self.state.swap(1, Ordering::Acquire);\n\n if previous != 1 {\n atomic::fence(Ordering::Release);\n\n return SeqLockWriteGuard {\n lock: self,\n state: previous,\n };\n }\n\n backoff.snooze();\n }\n }\n}\n\n/// An RAII guard that releases the lock and increments the stamp when dropped.\npub(crate) struct SeqLockWriteGuard {\n /// The parent lock.\n lock: &'static SeqLock,\n\n /// The stamp before locking.\n state: usize,\n}\n\nimpl SeqLockWriteGuard {\n /// Releases the lock without incrementing the stamp.\n #[inline]\n pub(crate) fn abort(self) {\n self.lock.state.store(self.state, Ordering::Release);\n\n // We specifically don't want to call drop(), since that's\n // what increments the stamp.\n mem::forget(self);\n }\n}\n\nimpl Drop for SeqLockWriteGuard {\n #[inline]\n fn drop(&mut self) {\n // Release the lock and increment the stamp.\n self.lock\n .state\n .store(self.state.wrapping_add(2), Ordering::Release);\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::SeqLock;\n\n #[test]\n fn test_abort() {\n static LK: SeqLock = SeqLock::new();\n let before = LK.optimistic_read().unwrap();\n {\n let guard = LK.write();\n guard.abort();\n }\n let after = LK.optimistic_read().unwrap();\n assert_eq!(before, after, \"aborted write does not update the stamp\");\n }\n}\n" }, { "path": "crossbeam-utils/src/atomic/seq_lock_wide.rs", "content": "use core::{\n mem,\n sync::atomic::{self, AtomicUsize, Ordering},\n};\n\nuse crate::Backoff;\n\n/// A simple stamped lock.\n///\n/// The state is represented as two `AtomicUsize`: `state_hi` for high bits and `state_lo` for low\n/// bits.\npub(crate) struct SeqLock {\n /// The high bits of the current state of the lock.\n state_hi: AtomicUsize,\n\n /// The low bits of the current state of the lock.\n ///\n /// All bits except the least significant one hold the current stamp. When locked, the state_lo\n /// equals 1 and doesn't contain a valid stamp.\n state_lo: AtomicUsize,\n}\n\nimpl SeqLock {\n pub(crate) const fn new() -> Self {\n Self {\n state_hi: AtomicUsize::new(0),\n state_lo: AtomicUsize::new(0),\n }\n }\n\n /// If not locked, returns the current stamp.\n ///\n /// This method should be called before optimistic reads.\n #[inline]\n pub(crate) fn optimistic_read(&self) -> Option<(usize, usize)> {\n // The acquire loads from `state_hi` and `state_lo` synchronize with the release stores in\n // `SeqLockWriteGuard::drop`.\n //\n // As a consequence, we can make sure that (1) all writes within the era of `state_hi - 1`\n // happens before now; and therefore, (2) if `state_lo` is even, all writes within the\n // critical section of (`state_hi`, `state_lo`) happens before now.\n let state_hi = self.state_hi.load(Ordering::Acquire);\n let state_lo = self.state_lo.load(Ordering::Acquire);\n if state_lo == 1 {\n None\n } else {\n Some((state_hi, state_lo))\n }\n }\n\n /// Returns `true` if the current stamp is equal to `stamp`.\n ///\n /// This method should be called after optimistic reads to check whether they are valid. The\n /// argument `stamp` should correspond to the one returned by method `optimistic_read`.\n #[inline]\n pub(crate) fn validate_read(&self, stamp: (usize, usize)) -> bool {\n // Thanks to the fence, if we're noticing any modification to the data at the critical\n // section of `(a, b)`, then the critical section's write of 1 to state_lo should be\n // visible.\n atomic::fence(Ordering::Acquire);\n\n // So if `state_lo` coincides with `stamp.1`, then either (1) we're noticing no modification\n // to the data after the critical section of `(stamp.0, stamp.1)`, or (2) `state_lo` wrapped\n // around.\n //\n // If (2) is the case, the acquire ordering ensures we see the new value of `state_hi`.\n let state_lo = self.state_lo.load(Ordering::Acquire);\n\n // If (2) is the case and `state_hi` coincides with `stamp.0`, then `state_hi` also wrapped\n // around, which we give up to correctly validate the read.\n let state_hi = self.state_hi.load(Ordering::Relaxed);\n\n // Except for the case that both `state_hi` and `state_lo` wrapped around, the following\n // condition implies that we're noticing no modification to the data after the critical\n // section of `(stamp.0, stamp.1)`.\n (state_hi, state_lo) == stamp\n }\n\n /// Grabs the lock for writing.\n #[inline]\n pub(crate) fn write(&'static self) -> SeqLockWriteGuard {\n let backoff = Backoff::new();\n loop {\n let previous = self.state_lo.swap(1, Ordering::Acquire);\n\n if previous != 1 {\n // To synchronize with the acquire fence in `validate_read` via any modification to\n // the data at the critical section of `(state_hi, previous)`.\n atomic::fence(Ordering::Release);\n\n return SeqLockWriteGuard {\n lock: self,\n state_lo: previous,\n };\n }\n\n backoff.snooze();\n }\n }\n}\n\n/// An RAII guard that releases the lock and increments the stamp when dropped.\npub(crate) struct SeqLockWriteGuard {\n /// The parent lock.\n lock: &'static SeqLock,\n\n /// The stamp before locking.\n state_lo: usize,\n}\n\nimpl SeqLockWriteGuard {\n /// Releases the lock without incrementing the stamp.\n #[inline]\n pub(crate) fn abort(self) {\n self.lock.state_lo.store(self.state_lo, Ordering::Release);\n mem::forget(self);\n }\n}\n\nimpl Drop for SeqLockWriteGuard {\n #[inline]\n fn drop(&mut self) {\n let state_lo = self.state_lo.wrapping_add(2);\n\n // Increase the high bits if the low bits wrap around.\n //\n // Release ordering for synchronizing with `optimistic_read`.\n if state_lo == 0 {\n let state_hi = self.lock.state_hi.load(Ordering::Relaxed);\n self.lock\n .state_hi\n .store(state_hi.wrapping_add(1), Ordering::Release);\n }\n\n // Release the lock and increment the stamp.\n //\n // Release ordering for synchronizing with `optimistic_read`.\n self.lock.state_lo.store(state_lo, Ordering::Release);\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::SeqLock;\n\n #[test]\n fn test_abort() {\n static LK: SeqLock = SeqLock::new();\n let before = LK.optimistic_read().unwrap();\n {\n let guard = LK.write();\n guard.abort();\n }\n let after = LK.optimistic_read().unwrap();\n assert_eq!(before, after, \"aborted write does not update the stamp\");\n }\n}\n" }, { "path": "crossbeam-utils/src/backoff.rs", "content": "use core::{cell::Cell, fmt};\n\nuse crate::primitive::hint;\n\nconst SPIN_LIMIT: u32 = 6;\nconst YIELD_LIMIT: u32 = 10;\n\n/// Performs exponential backoff in spin loops.\n///\n/// Backing off in spin loops reduces contention and improves overall performance.\n///\n/// This primitive can execute *YIELD* and *PAUSE* instructions, yield the current thread to the OS\n/// scheduler, and tell when is a good time to block the thread using a different synchronization\n/// mechanism. Each step of the back off procedure takes roughly twice as long as the previous\n/// step.\n///\n/// # Examples\n///\n/// Backing off in a lock-free loop:\n///\n/// ```\n/// use crossbeam_utils::Backoff;\n/// use std::sync::atomic::AtomicUsize;\n/// use std::sync::atomic::Ordering::SeqCst;\n///\n/// fn fetch_mul(a: &AtomicUsize, b: usize) -> usize {\n/// let backoff = Backoff::new();\n/// loop {\n/// let val = a.load(SeqCst);\n/// if a.compare_exchange(val, val.wrapping_mul(b), SeqCst, SeqCst).is_ok() {\n/// return val;\n/// }\n/// backoff.spin();\n/// }\n/// }\n/// ```\n///\n/// Waiting for an [`AtomicBool`] to become `true`:\n///\n/// ```\n/// use crossbeam_utils::Backoff;\n/// use std::sync::atomic::AtomicBool;\n/// use std::sync::atomic::Ordering::SeqCst;\n///\n/// fn spin_wait(ready: &AtomicBool) {\n/// let backoff = Backoff::new();\n/// while !ready.load(SeqCst) {\n/// backoff.snooze();\n/// }\n/// }\n/// ```\n///\n/// Waiting for an [`AtomicBool`] to become `true` and parking the thread after a long wait.\n/// Note that whoever sets the atomic variable to `true` must notify the parked thread by calling\n/// [`unpark()`]:\n///\n/// ```\n/// use crossbeam_utils::Backoff;\n/// use std::sync::atomic::AtomicBool;\n/// use std::sync::atomic::Ordering::SeqCst;\n/// use std::thread;\n///\n/// fn blocking_wait(ready: &AtomicBool) {\n/// let backoff = Backoff::new();\n/// while !ready.load(SeqCst) {\n/// if backoff.is_completed() {\n/// thread::park();\n/// } else {\n/// backoff.snooze();\n/// }\n/// }\n/// }\n/// ```\n///\n/// [`is_completed`]: Backoff::is_completed\n/// [`std::thread::park()`]: std::thread::park\n/// [`Condvar`]: std::sync::Condvar\n/// [`AtomicBool`]: std::sync::atomic::AtomicBool\n/// [`unpark()`]: std::thread::Thread::unpark\npub struct Backoff {\n step: Cell,\n}\n\nimpl Backoff {\n /// Creates a new `Backoff`.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::Backoff;\n ///\n /// let backoff = Backoff::new();\n /// ```\n #[inline]\n pub fn new() -> Self {\n Self { step: Cell::new(0) }\n }\n\n /// Resets the `Backoff`.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::Backoff;\n ///\n /// let backoff = Backoff::new();\n /// backoff.reset();\n /// ```\n #[inline]\n pub fn reset(&self) {\n self.step.set(0);\n }\n\n /// Backs off in a lock-free loop.\n ///\n /// This method should be used when we need to retry an operation because another thread made\n /// progress.\n ///\n /// The processor may yield using the *YIELD* or *PAUSE* instruction.\n ///\n /// # Examples\n ///\n /// Backing off in a lock-free loop:\n ///\n /// ```\n /// use crossbeam_utils::Backoff;\n /// use std::sync::atomic::AtomicUsize;\n /// use std::sync::atomic::Ordering::SeqCst;\n ///\n /// fn fetch_mul(a: &AtomicUsize, b: usize) -> usize {\n /// let backoff = Backoff::new();\n /// loop {\n /// let val = a.load(SeqCst);\n /// if a.compare_exchange(val, val.wrapping_mul(b), SeqCst, SeqCst).is_ok() {\n /// return val;\n /// }\n /// backoff.spin();\n /// }\n /// }\n ///\n /// let a = AtomicUsize::new(7);\n /// assert_eq!(fetch_mul(&a, 8), 7);\n /// assert_eq!(a.load(SeqCst), 56);\n /// ```\n #[inline]\n pub fn spin(&self) {\n for _ in 0..1 << self.step.get().min(SPIN_LIMIT) {\n hint::spin_loop();\n }\n\n if self.step.get() <= SPIN_LIMIT {\n self.step.set(self.step.get() + 1);\n }\n }\n\n /// Backs off in a blocking loop.\n ///\n /// This method should be used when we need to wait for another thread to make progress.\n ///\n /// The processor may yield using the *YIELD* or *PAUSE* instruction and the current thread\n /// may yield by giving up a timeslice to the OS scheduler.\n ///\n /// In `#[no_std]` environments, this method is equivalent to [`spin`].\n ///\n /// If possible, use [`is_completed`] to check when it is advised to stop using backoff and\n /// block the current thread using a different synchronization mechanism instead.\n ///\n /// [`spin`]: Backoff::spin\n /// [`is_completed`]: Backoff::is_completed\n ///\n /// # Examples\n ///\n /// Waiting for an [`AtomicBool`] to become `true`:\n ///\n /// ```\n /// use crossbeam_utils::Backoff;\n /// use std::sync::Arc;\n /// use std::sync::atomic::AtomicBool;\n /// use std::sync::atomic::Ordering::SeqCst;\n /// use std::thread;\n /// use std::time::Duration;\n ///\n /// fn spin_wait(ready: &AtomicBool) {\n /// let backoff = Backoff::new();\n /// while !ready.load(SeqCst) {\n /// backoff.snooze();\n /// }\n /// }\n ///\n /// let ready = Arc::new(AtomicBool::new(false));\n /// let ready2 = ready.clone();\n ///\n /// # let t =\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_millis(100));\n /// ready2.store(true, SeqCst);\n /// });\n ///\n /// assert_eq!(ready.load(SeqCst), false);\n /// spin_wait(&ready);\n /// assert_eq!(ready.load(SeqCst), true);\n /// # t.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n ///\n /// [`AtomicBool`]: std::sync::atomic::AtomicBool\n #[inline]\n pub fn snooze(&self) {\n if self.step.get() <= SPIN_LIMIT {\n for _ in 0..1 << self.step.get() {\n hint::spin_loop();\n }\n } else {\n #[cfg(not(feature = \"std\"))]\n for _ in 0..1 << self.step.get() {\n hint::spin_loop();\n }\n\n #[cfg(feature = \"std\")]\n ::std::thread::yield_now();\n }\n\n if self.step.get() <= YIELD_LIMIT {\n self.step.set(self.step.get() + 1);\n }\n }\n\n /// Returns `true` if exponential backoff has completed and blocking the thread is advised.\n ///\n /// # Examples\n ///\n /// Waiting for an [`AtomicBool`] to become `true` and parking the thread after a long wait:\n ///\n /// ```\n /// use crossbeam_utils::Backoff;\n /// use std::sync::Arc;\n /// use std::sync::atomic::AtomicBool;\n /// use std::sync::atomic::Ordering::SeqCst;\n /// use std::thread;\n /// use std::time::Duration;\n ///\n /// fn blocking_wait(ready: &AtomicBool) {\n /// let backoff = Backoff::new();\n /// while !ready.load(SeqCst) {\n /// if backoff.is_completed() {\n /// thread::park();\n /// } else {\n /// backoff.snooze();\n /// }\n /// }\n /// }\n ///\n /// let ready = Arc::new(AtomicBool::new(false));\n /// let ready2 = ready.clone();\n /// let waiter = thread::current();\n ///\n /// # let t =\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_millis(100));\n /// ready2.store(true, SeqCst);\n /// waiter.unpark();\n /// });\n ///\n /// assert_eq!(ready.load(SeqCst), false);\n /// blocking_wait(&ready);\n /// assert_eq!(ready.load(SeqCst), true);\n /// # t.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n ///\n /// [`AtomicBool`]: std::sync::atomic::AtomicBool\n #[inline]\n pub fn is_completed(&self) -> bool {\n self.step.get() > YIELD_LIMIT\n }\n}\n\nimpl fmt::Debug for Backoff {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"Backoff\")\n .field(\"step\", &self.step)\n .field(\"is_completed\", &self.is_completed())\n .finish()\n }\n}\n\nimpl Default for Backoff {\n fn default() -> Self {\n Self::new()\n }\n}\n" }, { "path": "crossbeam-utils/src/cache_padded.rs", "content": "use core::{\n fmt,\n ops::{Deref, DerefMut},\n};\n\n/// Pads and aligns a value to the length of a cache line.\n///\n/// In concurrent programming, sometimes it is desirable to make sure commonly accessed pieces of\n/// data are not placed into the same cache line. Updating an atomic value invalidates the whole\n/// cache line it belongs to, which makes the next access to the same cache line slower for other\n/// CPU cores. Use `CachePadded` to ensure updating one piece of data doesn't invalidate other\n/// cached data.\n///\n/// # Size and alignment\n///\n/// Cache lines are assumed to be N bytes long, depending on the architecture:\n///\n/// * On x86-64, aarch64, and powerpc64, N = 128.\n/// * On arm, mips, mips64, sparc, and hexagon, N = 32.\n/// * On m68k, N = 16.\n/// * On s390x, N = 256.\n/// * On all others, N = 64.\n///\n/// Note that N is just a reasonable guess and is not guaranteed to match the actual cache line\n/// length of the machine the program is running on. On modern Intel architectures, spatial\n/// prefetcher is pulling pairs of 64-byte cache lines at a time, so we pessimistically assume that\n/// cache lines are 128 bytes long.\n///\n/// The size of `CachePadded` is the smallest multiple of N bytes large enough to accommodate\n/// a value of type `T`.\n///\n/// The alignment of `CachePadded` is the maximum of N bytes and the alignment of `T`.\n///\n/// # Examples\n///\n/// Alignment and padding:\n///\n/// ```\n/// use crossbeam_utils::CachePadded;\n///\n/// let array = [CachePadded::new(1i8), CachePadded::new(2i8)];\n/// let addr1 = &*array[0] as *const i8 as usize;\n/// let addr2 = &*array[1] as *const i8 as usize;\n///\n/// assert!(addr2 - addr1 >= 32);\n/// assert_eq!(addr1 % 32, 0);\n/// assert_eq!(addr2 % 32, 0);\n/// ```\n///\n/// When building a concurrent queue with a head and a tail index, it is wise to place them in\n/// different cache lines so that concurrent threads pushing and popping elements don't invalidate\n/// each other's cache lines:\n///\n/// ```\n/// use crossbeam_utils::CachePadded;\n/// use std::sync::atomic::AtomicUsize;\n///\n/// struct Queue {\n/// head: CachePadded,\n/// tail: CachePadded,\n/// buffer: *mut T,\n/// }\n/// ```\n#[derive(Clone, Copy, Default, Hash, PartialEq, Eq)]\n// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache\n// lines at a time, so we have to align to 128 bytes rather than 64.\n//\n// Sources:\n// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf\n// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107\n//\n// aarch64/arm64ec's big.LITTLE architecture has asymmetric cores and \"big\" cores have 128-byte cache line size.\n//\n// Sources:\n// - https://www.mono-project.com/news/2016/09/12/arm64-icache/\n//\n// powerpc64 has 128-byte cache line size.\n//\n// Sources:\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/powerpc/include/asm/cache.h#L26\n#[cfg_attr(\n any(\n target_arch = \"x86_64\",\n target_arch = \"aarch64\",\n target_arch = \"arm64ec\",\n target_arch = \"powerpc64\",\n ),\n repr(align(128))\n)]\n// arm, mips, mips64, sparc, and hexagon have 32-byte cache line size.\n//\n// Sources:\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L17\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/hexagon/include/asm/cache.h#L12\n#[cfg_attr(\n any(\n target_arch = \"arm\",\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"sparc\",\n target_arch = \"hexagon\",\n ),\n repr(align(32))\n)]\n// m68k has 16-byte cache line size.\n//\n// Sources:\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/m68k/include/asm/cache.h#L9\n#[cfg_attr(target_arch = \"m68k\", repr(align(16)))]\n// s390x has 256-byte cache line size.\n//\n// Sources:\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/s390/include/asm/cache.h#L13\n#[cfg_attr(target_arch = \"s390x\", repr(align(256)))]\n// x86, wasm, riscv, and sparc64 have 64-byte cache line size.\n//\n// Sources:\n// - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/riscv/include/asm/cache.h#L10\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L19\n//\n// All others are assumed to have 64-byte cache line size.\n#[cfg_attr(\n not(any(\n target_arch = \"x86_64\",\n target_arch = \"aarch64\",\n target_arch = \"arm64ec\",\n target_arch = \"powerpc64\",\n target_arch = \"arm\",\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"sparc\",\n target_arch = \"hexagon\",\n target_arch = \"m68k\",\n target_arch = \"s390x\",\n )),\n repr(align(64))\n)]\npub struct CachePadded {\n value: T,\n}\n\nunsafe impl Send for CachePadded {}\nunsafe impl Sync for CachePadded {}\n\nimpl CachePadded {\n /// Pads and aligns a value to the length of a cache line.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::CachePadded;\n ///\n /// let padded_value = CachePadded::new(1);\n /// ```\n pub const fn new(t: T) -> Self {\n Self { value: t }\n }\n\n /// Returns the inner value.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::CachePadded;\n ///\n /// let padded_value = CachePadded::new(7);\n /// let value = padded_value.into_inner();\n /// assert_eq!(value, 7);\n /// ```\n pub fn into_inner(self) -> T {\n self.value\n }\n}\n\nimpl Deref for CachePadded {\n type Target = T;\n\n fn deref(&self) -> &T {\n &self.value\n }\n}\n\nimpl DerefMut for CachePadded {\n fn deref_mut(&mut self) -> &mut T {\n &mut self.value\n }\n}\n\nimpl fmt::Debug for CachePadded {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"CachePadded\")\n .field(\"value\", &self.value)\n .finish()\n }\n}\n\nimpl From for CachePadded {\n fn from(t: T) -> Self {\n Self::new(t)\n }\n}\n\nimpl fmt::Display for CachePadded {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n fmt::Display::fmt(&self.value, f)\n }\n}\n" }, { "path": "crossbeam-utils/src/lib.rs", "content": "//! Miscellaneous tools for concurrent programming.\n//!\n//! ## Atomics\n//!\n//! * [`AtomicCell`], a thread-safe mutable memory location.\n//! * [`AtomicConsume`], for reading from primitive atomic types with \"consume\" ordering.\n//!\n//! ## Thread synchronization\n//!\n//! * [`Parker`], a thread parking primitive.\n//! * [`ShardedLock`], a sharded reader-writer lock with fast concurrent reads.\n//! * [`WaitGroup`], for synchronizing the beginning or end of some computation.\n//!\n//! ## Utilities\n//!\n//! * [`Backoff`], for exponential backoff in spin loops.\n//! * [`CachePadded`], for padding and aligning a value to the length of a cache line.\n//! * [`scope`], for spawning threads that borrow local variables from the stack.\n//!\n//! [`AtomicCell`]: atomic::AtomicCell\n//! [`AtomicConsume`]: atomic::AtomicConsume\n//! [`Parker`]: sync::Parker\n//! [`ShardedLock`]: sync::ShardedLock\n//! [`WaitGroup`]: sync::WaitGroup\n//! [`scope`]: thread::scope\n\n#![no_std]\n#![doc(test(\n no_crate_inject,\n attr(allow(dead_code, unused_assignments, unused_variables))\n))]\n#![warn(\n missing_docs,\n unsafe_op_in_unsafe_fn,\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\n#![cfg_attr(docsrs, feature(doc_cfg))]\n\n#[cfg(feature = \"std\")]\n#[cfg(not(crossbeam_loom))]\nextern crate alloc;\n#[cfg(feature = \"std\")]\nextern crate std;\n\n#[cfg(crossbeam_loom)]\n#[allow(unused_imports)]\nmod primitive {\n pub(crate) mod hint {\n pub(crate) use loom::hint::spin_loop;\n }\n pub(crate) mod sync {\n pub(crate) mod atomic {\n pub(crate) use loom::sync::atomic::{\n AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicI64, AtomicIsize, AtomicU8,\n AtomicU16, AtomicU32, AtomicU64, AtomicUsize, Ordering, fence,\n };\n\n // FIXME: loom does not support compiler_fence at the moment.\n // https://github.com/tokio-rs/loom/issues/117\n // we use fence as a stand-in for compiler_fence for the time being.\n // this may miss some races since fence is stronger than compiler_fence,\n // but it's the best we can do for the time being.\n pub(crate) use self::fence as compiler_fence;\n }\n pub(crate) use loom::sync::{Arc, Condvar, Mutex};\n }\n}\n#[cfg(not(crossbeam_loom))]\n#[allow(unused_imports)]\nmod primitive {\n pub(crate) mod hint {\n pub(crate) use core::hint::spin_loop;\n }\n pub(crate) mod sync {\n #[cfg(feature = \"std\")]\n pub(crate) use alloc::sync::Arc;\n pub(crate) use core::sync::atomic;\n #[cfg(feature = \"std\")]\n pub(crate) use std::sync::{Condvar, Mutex};\n }\n}\n\n#[cfg(feature = \"atomic\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"atomic\")))]\npub mod atomic;\n\nmod cache_padded;\npub use crate::cache_padded::CachePadded;\n\nmod backoff;\npub use crate::backoff::Backoff;\n\n#[cfg(feature = \"std\")]\npub mod sync;\n\n#[cfg(feature = \"std\")]\n#[cfg(not(crossbeam_loom))]\npub mod thread;\n" }, { "path": "crossbeam-utils/src/sync/mod.rs", "content": "//! Thread synchronization primitives.\n//!\n//! * [`Parker`], a thread parking primitive.\n//! * [`ShardedLock`], a sharded reader-writer lock with fast concurrent reads.\n//! * [`WaitGroup`], for synchronizing the beginning or end of some computation.\n\n#[cfg(not(crossbeam_loom))]\nmod once_lock;\nmod parker;\n#[cfg(not(crossbeam_loom))]\nmod sharded_lock;\nmod wait_group;\n\n#[cfg(not(crossbeam_loom))]\npub use self::sharded_lock::{ShardedLock, ShardedLockReadGuard, ShardedLockWriteGuard};\npub use self::{\n parker::{Parker, UnparkReason, Unparker},\n wait_group::WaitGroup,\n};\n" }, { "path": "crossbeam-utils/src/sync/once_lock.rs", "content": "// Based on unstable std::sync::OnceLock.\n//\n// Source: https://github.com/rust-lang/rust/blob/8e9c93df464b7ada3fc7a1c8ccddd9dcb24ee0a0/library/std/src/sync/once_lock.rs\n\nuse core::{cell::UnsafeCell, mem::MaybeUninit};\nuse std::sync::Once;\n\npub(crate) struct OnceLock {\n once: Once,\n value: UnsafeCell>,\n // Unlike std::sync::OnceLock, we don't need PhantomData here because\n // we don't use #[may_dangle].\n}\n\nunsafe impl Sync for OnceLock {}\nunsafe impl Send for OnceLock {}\n\nimpl OnceLock {\n /// Creates a new empty cell.\n #[must_use]\n pub(crate) const fn new() -> Self {\n Self {\n once: Once::new(),\n value: UnsafeCell::new(MaybeUninit::uninit()),\n }\n }\n\n /// Gets the contents of the cell, initializing it with `f` if the cell\n /// was empty.\n ///\n /// Many threads may call `get_or_init` concurrently with different\n /// initializing functions, but it is guaranteed that only one function\n /// will be executed.\n ///\n /// # Panics\n ///\n /// If `f` panics, the panic is propagated to the caller, and the cell\n /// remains uninitialized.\n ///\n /// It is an error to reentrantly initialize the cell from `f`. The\n /// exact outcome is unspecified. Current implementation deadlocks, but\n /// this may be changed to a panic in the future.\n pub(crate) fn get_or_init(&self, f: F) -> &T\n where\n F: FnOnce() -> T,\n {\n // Fast path check\n if self.once.is_completed() {\n // SAFETY: The inner value has been initialized\n return unsafe { self.get_unchecked() };\n }\n self.initialize(f);\n\n // SAFETY: The inner value has been initialized\n unsafe { self.get_unchecked() }\n }\n\n #[cold]\n fn initialize(&self, f: F)\n where\n F: FnOnce() -> T,\n {\n let slot = self.value.get();\n\n self.once.call_once(|| {\n let value = f();\n unsafe { slot.write(MaybeUninit::new(value)) }\n });\n }\n\n /// # Safety\n ///\n /// The value must be initialized\n unsafe fn get_unchecked(&self) -> &T {\n debug_assert!(self.once.is_completed());\n unsafe { (*self.value.get()).assume_init_ref() }\n }\n}\n\nimpl Drop for OnceLock {\n fn drop(&mut self) {\n if self.once.is_completed() {\n // SAFETY: The inner value has been initialized\n // assume_init_drop requires Rust 1.60\n // unsafe { (*self.value.get()).assume_init_drop() };\n unsafe { self.value.get().cast::().drop_in_place() };\n }\n }\n}\n" }, { "path": "crossbeam-utils/src/sync/parker.rs", "content": "use core::{fmt, marker::PhantomData, time::Duration};\nuse std::time::Instant;\n\nuse crate::primitive::sync::{\n Arc, Condvar, Mutex,\n atomic::{AtomicUsize, Ordering::SeqCst},\n};\n\n/// A thread parking primitive.\n///\n/// Conceptually, each `Parker` has an associated token which is initially not present:\n///\n/// * The [`park`] method blocks the current thread unless or until the token is available, at\n/// which point it automatically consumes the token.\n///\n/// * The [`park_timeout`] and [`park_deadline`] methods work the same as [`park`], but block for\n/// a specified maximum time.\n///\n/// * The [`unpark`] method atomically makes the token available if it wasn't already. Because the\n/// token is initially absent, [`unpark`] followed by [`park`] will result in the second call\n/// returning immediately.\n///\n/// In other words, each `Parker` acts a bit like a spinlock that can be locked and unlocked using\n/// [`park`] and [`unpark`].\n///\n/// # Examples\n///\n/// ```\n/// use std::thread;\n/// use std::time::Duration;\n/// use crossbeam_utils::sync::Parker;\n///\n/// let p = Parker::new();\n/// let u = p.unparker().clone();\n///\n/// // Make the token available.\n/// u.unpark();\n/// // Wakes up immediately and consumes the token.\n/// p.park();\n///\n/// # let t =\n/// thread::spawn(move || {\n/// thread::sleep(Duration::from_millis(500));\n/// u.unpark();\n/// });\n///\n/// // Wakes up when `u.unpark()` provides the token.\n/// p.park();\n/// # t.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n/// ```\n///\n/// [`park`]: Parker::park\n/// [`park_timeout`]: Parker::park_timeout\n/// [`park_deadline`]: Parker::park_deadline\n/// [`unpark`]: Unparker::unpark\npub struct Parker {\n unparker: Unparker,\n _marker: PhantomData<*const ()>,\n}\n\nunsafe impl Send for Parker {}\n\nimpl Default for Parker {\n fn default() -> Self {\n Self {\n unparker: Unparker {\n inner: Arc::new(Inner {\n state: AtomicUsize::new(EMPTY),\n lock: Mutex::new(()),\n cvar: Condvar::new(),\n }),\n },\n _marker: PhantomData,\n }\n }\n}\n\nimpl Parker {\n /// Creates a new `Parker`.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::Parker;\n ///\n /// let p = Parker::new();\n /// ```\n ///\n pub fn new() -> Self {\n Self::default()\n }\n\n /// Blocks the current thread until the token is made available.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::Parker;\n ///\n /// let p = Parker::new();\n /// let u = p.unparker().clone();\n ///\n /// // Make the token available.\n /// u.unpark();\n ///\n /// // Wakes up immediately and consumes the token.\n /// p.park();\n /// ```\n pub fn park(&self) {\n self.unparker.inner.park(None);\n }\n\n /// Blocks the current thread until the token is made available, but only for a limited time.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::time::Duration;\n /// use crossbeam_utils::sync::Parker;\n ///\n /// let p = Parker::new();\n ///\n /// // Waits for the token to become available, but will not wait longer than 500 ms.\n /// p.park_timeout(Duration::from_millis(500));\n /// ```\n pub fn park_timeout(&self, timeout: Duration) -> UnparkReason {\n match Instant::now().checked_add(timeout) {\n Some(deadline) => self.park_deadline(deadline),\n None => {\n self.park();\n UnparkReason::Unparked\n }\n }\n }\n\n /// Blocks the current thread until the token is made available, or until a certain deadline.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::time::{Duration, Instant};\n /// use crossbeam_utils::sync::Parker;\n ///\n /// let p = Parker::new();\n /// let deadline = Instant::now() + Duration::from_millis(500);\n ///\n /// // Waits for the token to become available, but will not wait longer than 500 ms.\n /// p.park_deadline(deadline);\n /// ```\n pub fn park_deadline(&self, deadline: Instant) -> UnparkReason {\n self.unparker.inner.park(Some(deadline))\n }\n\n /// Returns a reference to an associated [`Unparker`].\n ///\n /// The returned [`Unparker`] doesn't have to be used by reference - it can also be cloned.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::Parker;\n ///\n /// let p = Parker::new();\n /// let u = p.unparker().clone();\n ///\n /// // Make the token available.\n /// u.unpark();\n /// // Wakes up immediately and consumes the token.\n /// p.park();\n /// ```\n ///\n /// [`park`]: Parker::park\n /// [`park_timeout`]: Parker::park_timeout\n pub fn unparker(&self) -> &Unparker {\n &self.unparker\n }\n\n /// Converts a `Parker` into a raw pointer.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::Parker;\n ///\n /// let p = Parker::new();\n /// let raw = Parker::into_raw(p);\n /// # let _ = unsafe { Parker::from_raw(raw) };\n /// ```\n pub fn into_raw(this: Self) -> *const () {\n Unparker::into_raw(this.unparker)\n }\n\n /// Converts a raw pointer into a `Parker`.\n ///\n /// # Safety\n ///\n /// This method is safe to use only with pointers returned by [`Parker::into_raw`].\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::Parker;\n ///\n /// let p = Parker::new();\n /// let raw = Parker::into_raw(p);\n /// let p = unsafe { Parker::from_raw(raw) };\n /// ```\n pub unsafe fn from_raw(ptr: *const ()) -> Self {\n Self {\n unparker: unsafe { Unparker::from_raw(ptr) },\n _marker: PhantomData,\n }\n }\n}\n\nimpl fmt::Debug for Parker {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Parker { .. }\")\n }\n}\n\n/// Unparks a thread parked by the associated [`Parker`].\npub struct Unparker {\n inner: Arc,\n}\n\nunsafe impl Send for Unparker {}\nunsafe impl Sync for Unparker {}\n\nimpl Unparker {\n /// Atomically makes the token available if it is not already.\n ///\n /// This method will wake up the thread blocked on [`park`] or [`park_timeout`], if there is\n /// any.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::thread;\n /// use std::time::Duration;\n /// use crossbeam_utils::sync::Parker;\n ///\n /// let p = Parker::new();\n /// let u = p.unparker().clone();\n ///\n /// # let t =\n /// thread::spawn(move || {\n /// thread::sleep(Duration::from_millis(500));\n /// u.unpark();\n /// });\n ///\n /// // Wakes up when `u.unpark()` provides the token.\n /// p.park();\n /// # t.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n ///\n /// [`park`]: Parker::park\n /// [`park_timeout`]: Parker::park_timeout\n pub fn unpark(&self) {\n self.inner.unpark()\n }\n\n /// Converts an `Unparker` into a raw pointer.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::{Parker, Unparker};\n ///\n /// let p = Parker::new();\n /// let u = p.unparker().clone();\n /// let raw = Unparker::into_raw(u);\n /// # let _ = unsafe { Unparker::from_raw(raw) };\n /// ```\n pub fn into_raw(this: Self) -> *const () {\n Arc::into_raw(this.inner).cast::<()>()\n }\n\n /// Converts a raw pointer into an `Unparker`.\n ///\n /// # Safety\n ///\n /// This method is safe to use only with pointers returned by [`Unparker::into_raw`].\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::{Parker, Unparker};\n ///\n /// let p = Parker::new();\n /// let u = p.unparker().clone();\n ///\n /// let raw = Unparker::into_raw(u);\n /// let u = unsafe { Unparker::from_raw(raw) };\n /// ```\n pub unsafe fn from_raw(ptr: *const ()) -> Self {\n Self {\n inner: unsafe { Arc::from_raw(ptr.cast::()) },\n }\n }\n}\n\nimpl fmt::Debug for Unparker {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Unparker { .. }\")\n }\n}\n\nimpl Clone for Unparker {\n fn clone(&self) -> Self {\n Self {\n inner: self.inner.clone(),\n }\n }\n}\n\n/// An enum that reports whether a `Parker::park_timeout` or\n/// `Parker::park_deadline` returned because another thread called `unpark` or\n/// because of a timeout.\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub enum UnparkReason {\n /// The park method returned due to a call to `unpark`.\n Unparked,\n\n /// The park method returned due to a timeout.\n Timeout,\n}\n\nconst EMPTY: usize = 0;\nconst PARKED: usize = 1;\nconst NOTIFIED: usize = 2;\n\nstruct Inner {\n state: AtomicUsize,\n lock: Mutex<()>,\n cvar: Condvar,\n}\n\nimpl Inner {\n fn park(&self, deadline: Option) -> UnparkReason {\n // If we were previously notified then we consume this notification and return quickly.\n if self\n .state\n .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst)\n .is_ok()\n {\n return UnparkReason::Unparked;\n }\n\n // If the timeout is zero, then there is no need to actually block.\n if let Some(deadline) = deadline {\n if deadline <= Instant::now() {\n return UnparkReason::Timeout;\n }\n }\n\n // Otherwise we need to coordinate going to sleep.\n let mut m = self.lock.lock().unwrap();\n\n match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) {\n Ok(_) => {}\n // Consume this notification to avoid spurious wakeups in the next park.\n Err(NOTIFIED) => {\n // We must read `state` here, even though we know it will be `NOTIFIED`. This is\n // because `unpark` may have been called again since we read `NOTIFIED` in the\n // `compare_exchange` above. We must perform an acquire operation that synchronizes\n // with that `unpark` to observe any writes it made before the call to `unpark`. To\n // do that we must read from the write it made to `state`.\n let old = self.state.swap(EMPTY, SeqCst);\n assert_eq!(old, NOTIFIED, \"park state changed unexpectedly\");\n return UnparkReason::Unparked;\n }\n Err(n) => panic!(\"inconsistent park_timeout state: {}\", n),\n }\n\n loop {\n // Block the current thread on the conditional variable.\n m = match deadline {\n None => self.cvar.wait(m).unwrap(),\n Some(deadline) => {\n let now = Instant::now();\n if now < deadline {\n // We could check for a timeout here, in the return value of wait_timeout,\n // but in the case that a timeout and an unpark arrive simultaneously, we\n // prefer to report the former.\n self.cvar.wait_timeout(m, deadline - now).unwrap().0\n } else {\n // We've timed out; swap out the state back to empty on our way out\n return match self.state.swap(EMPTY, SeqCst) {\n NOTIFIED => UnparkReason::Unparked, // got a notification\n PARKED => UnparkReason::Timeout, // no notification\n n => panic!(\"inconsistent park_timeout state: {}\", n),\n };\n }\n }\n };\n\n if self\n .state\n .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst)\n .is_ok()\n {\n // got a notification\n return UnparkReason::Unparked;\n }\n\n // Spurious wakeup, go back to sleep. Alternatively, if we timed out, it will be caught\n // in the branch above, when we discover the deadline is in the past\n }\n }\n\n pub(crate) fn unpark(&self) {\n // To ensure the unparked thread will observe any writes we made before this call, we must\n // perform a release operation that `park` can synchronize with. To do that we must write\n // `NOTIFIED` even if `state` is already `NOTIFIED`. That is why this must be a swap rather\n // than a compare-and-swap that returns if it reads `NOTIFIED` on failure.\n match self.state.swap(NOTIFIED, SeqCst) {\n EMPTY => return, // no one was waiting\n NOTIFIED => return, // already unparked\n PARKED => {} // gotta go wake someone up\n _ => panic!(\"inconsistent state in unpark\"),\n }\n\n // There is a period between when the parked thread sets `state` to `PARKED` (or last\n // checked `state` in the case of a spurious wakeup) and when it actually waits on `cvar`.\n // If we were to notify during this period it would be ignored and then when the parked\n // thread went to sleep it would never wake up. Fortunately, it has `lock` locked at this\n // stage so we can acquire `lock` to wait until it is ready to receive the notification.\n //\n // Releasing `lock` before the call to `notify_one` means that when the parked thread wakes\n // it doesn't get woken only to have to wait for us to release `lock`.\n drop(self.lock.lock().unwrap());\n self.cvar.notify_one();\n }\n}\n" }, { "path": "crossbeam-utils/src/sync/sharded_lock.rs", "content": "use alloc::{boxed::Box, vec::Vec};\nuse core::{\n cell::UnsafeCell,\n fmt,\n marker::PhantomData,\n mem,\n ops::{Deref, DerefMut},\n panic::{RefUnwindSafe, UnwindSafe},\n};\nuse std::{\n collections::HashMap,\n sync::{\n LockResult, Mutex, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard, TryLockError,\n TryLockResult,\n },\n thread::{self, ThreadId},\n};\n\nuse crate::{CachePadded, sync::once_lock::OnceLock};\n\n/// The number of shards per sharded lock. Must be a power of two.\nconst NUM_SHARDS: usize = 8;\n\n/// A shard containing a single reader-writer lock.\nstruct Shard {\n /// The inner reader-writer lock.\n lock: RwLock<()>,\n\n /// The write-guard keeping this shard locked.\n ///\n /// Write operations will lock each shard and store the guard here. These guards get dropped at\n /// the same time the big guard is dropped.\n write_guard: UnsafeCell>>,\n}\n\n/// A sharded reader-writer lock.\n///\n/// This lock is equivalent to [`RwLock`], except read operations are faster and write operations\n/// are slower.\n///\n/// A `ShardedLock` is internally made of a list of *shards*, each being a [`RwLock`] occupying a\n/// single cache line. Read operations will pick one of the shards depending on the current thread\n/// and lock it. Write operations need to lock all shards in succession.\n///\n/// By splitting the lock into shards, concurrent read operations will in most cases choose\n/// different shards and thus update different cache lines, which is good for scalability. However,\n/// write operations need to do more work and are therefore slower than usual.\n///\n/// The priority policy of the lock is dependent on the underlying operating system's\n/// implementation, and this type does not guarantee that any particular policy will be used.\n///\n/// # Poisoning\n///\n/// A `ShardedLock`, like [`RwLock`], will become poisoned on a panic. Note that it may only be\n/// poisoned if a panic occurs while a write operation is in progress. If a panic occurs in any\n/// read operation, the lock will not be poisoned.\n///\n/// # Examples\n///\n/// ```\n/// use crossbeam_utils::sync::ShardedLock;\n///\n/// let lock = ShardedLock::new(5);\n///\n/// // Any number of read locks can be held at once.\n/// {\n/// let r1 = lock.read().unwrap();\n/// let r2 = lock.read().unwrap();\n/// assert_eq!(*r1, 5);\n/// assert_eq!(*r2, 5);\n/// } // Read locks are dropped at this point.\n///\n/// // However, only one write lock may be held.\n/// {\n/// let mut w = lock.write().unwrap();\n/// *w += 1;\n/// assert_eq!(*w, 6);\n/// } // Write lock is dropped here.\n/// ```\n///\n/// [`RwLock`]: std::sync::RwLock\npub struct ShardedLock {\n /// A list of locks protecting the internal data.\n shards: Box<[CachePadded]>,\n\n /// The internal data.\n value: UnsafeCell,\n}\n\nunsafe impl Send for ShardedLock {}\nunsafe impl Sync for ShardedLock {}\n\nimpl UnwindSafe for ShardedLock {}\nimpl RefUnwindSafe for ShardedLock {}\n\nimpl ShardedLock {\n /// Creates a new sharded reader-writer lock.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::ShardedLock;\n ///\n /// let lock = ShardedLock::new(5);\n /// ```\n pub fn new(value: T) -> Self {\n Self {\n shards: (0..NUM_SHARDS)\n .map(|_| {\n CachePadded::new(Shard {\n lock: RwLock::new(()),\n write_guard: UnsafeCell::new(None),\n })\n })\n .collect::>(),\n value: UnsafeCell::new(value),\n }\n }\n\n /// Consumes this lock, returning the underlying data.\n ///\n /// # Errors\n ///\n /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write\n /// operation panics.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::ShardedLock;\n ///\n /// let lock = ShardedLock::new(String::new());\n /// {\n /// let mut s = lock.write().unwrap();\n /// *s = \"modified\".to_owned();\n /// }\n /// assert_eq!(lock.into_inner().unwrap(), \"modified\");\n /// ```\n pub fn into_inner(self) -> LockResult {\n let is_poisoned = self.is_poisoned();\n let inner = self.value.into_inner();\n\n if is_poisoned {\n Err(PoisonError::new(inner))\n } else {\n Ok(inner)\n }\n }\n}\n\nimpl ShardedLock {\n /// Returns `true` if the lock is poisoned.\n ///\n /// If another thread can still access the lock, it may become poisoned at any time. A `false`\n /// result should not be trusted without additional synchronization.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::ShardedLock;\n /// use std::sync::Arc;\n /// use std::thread;\n ///\n /// let lock = Arc::new(ShardedLock::new(0));\n /// let c_lock = lock.clone();\n ///\n /// let _: Result<(), _> = thread::spawn(move || {\n /// let _lock = c_lock.write().unwrap();\n /// panic!(); // the lock gets poisoned\n /// }).join();\n /// assert_eq!(lock.is_poisoned(), true);\n /// ```\n pub fn is_poisoned(&self) -> bool {\n self.shards[0].lock.is_poisoned()\n }\n\n /// Returns a mutable reference to the underlying data.\n ///\n /// Since this call borrows the lock mutably, no actual locking needs to take place.\n ///\n /// # Errors\n ///\n /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write\n /// operation panics.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::ShardedLock;\n ///\n /// let mut lock = ShardedLock::new(0);\n /// *lock.get_mut().unwrap() = 10;\n /// assert_eq!(*lock.read().unwrap(), 10);\n /// ```\n pub fn get_mut(&mut self) -> LockResult<&mut T> {\n let is_poisoned = self.is_poisoned();\n let inner = unsafe { &mut *self.value.get() };\n\n if is_poisoned {\n Err(PoisonError::new(inner))\n } else {\n Ok(inner)\n }\n }\n\n /// Attempts to acquire this lock with shared read access.\n ///\n /// If the access could not be granted at this time, an error is returned. Otherwise, a guard\n /// is returned which will release the shared access when it is dropped. This method does not\n /// provide any guarantees with respect to the ordering of whether contentious readers or\n /// writers will acquire the lock first.\n ///\n /// # Errors\n ///\n /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write\n /// operation panics.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::ShardedLock;\n ///\n /// let lock = ShardedLock::new(1);\n ///\n /// match lock.try_read() {\n /// Ok(n) => assert_eq!(*n, 1),\n /// Err(_) => unreachable!(),\n /// };\n /// ```\n pub fn try_read(&self) -> TryLockResult> {\n // Take the current thread index and map it to a shard index. Thread indices will tend to\n // distribute shards among threads equally, thus reducing contention due to read-locking.\n let current_index = current_index().unwrap_or(0);\n let shard_index = current_index & (self.shards.len() - 1);\n\n match self.shards[shard_index].lock.try_read() {\n Ok(guard) => Ok(ShardedLockReadGuard {\n lock: self,\n _guard: guard,\n _marker: PhantomData,\n }),\n Err(TryLockError::Poisoned(err)) => {\n let guard = ShardedLockReadGuard {\n lock: self,\n _guard: err.into_inner(),\n _marker: PhantomData,\n };\n Err(TryLockError::Poisoned(PoisonError::new(guard)))\n }\n Err(TryLockError::WouldBlock) => Err(TryLockError::WouldBlock),\n }\n }\n\n /// Locks with shared read access, blocking the current thread until it can be acquired.\n ///\n /// The calling thread will be blocked until there are no more writers which hold the lock.\n /// There may be other readers currently inside the lock when this method returns. This method\n /// does not provide any guarantees with respect to the ordering of whether contentious readers\n /// or writers will acquire the lock first.\n ///\n /// Returns a guard which will release the shared access when dropped.\n ///\n /// # Errors\n ///\n /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write\n /// operation panics.\n ///\n /// # Panics\n ///\n /// This method might panic when called if the lock is already held by the current thread.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::ShardedLock;\n /// use std::sync::Arc;\n /// use std::thread;\n ///\n /// let lock = Arc::new(ShardedLock::new(1));\n /// let c_lock = lock.clone();\n ///\n /// let n = lock.read().unwrap();\n /// assert_eq!(*n, 1);\n ///\n /// thread::spawn(move || {\n /// let r = c_lock.read();\n /// assert!(r.is_ok());\n /// }).join().unwrap();\n /// ```\n pub fn read(&self) -> LockResult> {\n // Take the current thread index and map it to a shard index. Thread indices will tend to\n // distribute shards among threads equally, thus reducing contention due to read-locking.\n let current_index = current_index().unwrap_or(0);\n let shard_index = current_index & (self.shards.len() - 1);\n\n match self.shards[shard_index].lock.read() {\n Ok(guard) => Ok(ShardedLockReadGuard {\n lock: self,\n _guard: guard,\n _marker: PhantomData,\n }),\n Err(err) => Err(PoisonError::new(ShardedLockReadGuard {\n lock: self,\n _guard: err.into_inner(),\n _marker: PhantomData,\n })),\n }\n }\n\n /// Attempts to acquire this lock with exclusive write access.\n ///\n /// If the access could not be granted at this time, an error is returned. Otherwise, a guard\n /// is returned which will release the exclusive access when it is dropped. This method does\n /// not provide any guarantees with respect to the ordering of whether contentious readers or\n /// writers will acquire the lock first.\n ///\n /// # Errors\n ///\n /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write\n /// operation panics.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::ShardedLock;\n ///\n /// let lock = ShardedLock::new(1);\n ///\n /// let n = lock.read().unwrap();\n /// assert_eq!(*n, 1);\n ///\n /// assert!(lock.try_write().is_err());\n /// ```\n pub fn try_write(&self) -> TryLockResult> {\n let mut poisoned = false;\n let mut blocked = None;\n\n // Write-lock each shard in succession.\n for (i, shard) in self.shards.iter().enumerate() {\n let guard = match shard.lock.try_write() {\n Ok(guard) => guard,\n Err(TryLockError::Poisoned(err)) => {\n poisoned = true;\n err.into_inner()\n }\n Err(TryLockError::WouldBlock) => {\n blocked = Some(i);\n break;\n }\n };\n\n // Store the guard into the shard.\n unsafe {\n let guard: RwLockWriteGuard<'static, ()> = mem::transmute(guard);\n let dest: *mut _ = shard.write_guard.get();\n *dest = Some(guard);\n }\n }\n\n if let Some(i) = blocked {\n // Unlock the shards in reverse order of locking.\n for shard in self.shards[0..i].iter().rev() {\n unsafe {\n let dest: *mut _ = shard.write_guard.get();\n let guard = (*dest).take();\n drop(guard);\n }\n }\n Err(TryLockError::WouldBlock)\n } else if poisoned {\n let guard = ShardedLockWriteGuard {\n lock: self,\n _marker: PhantomData,\n };\n Err(TryLockError::Poisoned(PoisonError::new(guard)))\n } else {\n Ok(ShardedLockWriteGuard {\n lock: self,\n _marker: PhantomData,\n })\n }\n }\n\n /// Locks with exclusive write access, blocking the current thread until it can be acquired.\n ///\n /// The calling thread will be blocked until there are no more writers which hold the lock.\n /// There may be other readers currently inside the lock when this method returns. This method\n /// does not provide any guarantees with respect to the ordering of whether contentious readers\n /// or writers will acquire the lock first.\n ///\n /// Returns a guard which will release the exclusive access when dropped.\n ///\n /// # Errors\n ///\n /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write\n /// operation panics.\n ///\n /// # Panics\n ///\n /// This method might panic when called if the lock is already held by the current thread.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::ShardedLock;\n ///\n /// let lock = ShardedLock::new(1);\n ///\n /// let mut n = lock.write().unwrap();\n /// *n = 2;\n ///\n /// assert!(lock.try_read().is_err());\n /// ```\n pub fn write(&self) -> LockResult> {\n let mut poisoned = false;\n\n // Write-lock each shard in succession.\n for shard in self.shards.iter() {\n let guard = match shard.lock.write() {\n Ok(guard) => guard,\n Err(err) => {\n poisoned = true;\n err.into_inner()\n }\n };\n\n // Store the guard into the shard.\n unsafe {\n let guard: RwLockWriteGuard<'_, ()> = guard;\n let guard: RwLockWriteGuard<'static, ()> = mem::transmute(guard);\n let dest: *mut _ = shard.write_guard.get();\n *dest = Some(guard);\n }\n }\n\n if poisoned {\n Err(PoisonError::new(ShardedLockWriteGuard {\n lock: self,\n _marker: PhantomData,\n }))\n } else {\n Ok(ShardedLockWriteGuard {\n lock: self,\n _marker: PhantomData,\n })\n }\n }\n}\n\nimpl fmt::Debug for ShardedLock {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match self.try_read() {\n Ok(guard) => f\n .debug_struct(\"ShardedLock\")\n .field(\"data\", &&*guard)\n .finish(),\n Err(TryLockError::Poisoned(err)) => f\n .debug_struct(\"ShardedLock\")\n .field(\"data\", &&**err.get_ref())\n .finish(),\n Err(TryLockError::WouldBlock) => {\n struct LockedPlaceholder;\n impl fmt::Debug for LockedPlaceholder {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.write_str(\"\")\n }\n }\n f.debug_struct(\"ShardedLock\")\n .field(\"data\", &LockedPlaceholder)\n .finish()\n }\n }\n }\n}\n\nimpl Default for ShardedLock {\n fn default() -> Self {\n Self::new(Default::default())\n }\n}\n\nimpl From for ShardedLock {\n fn from(t: T) -> Self {\n Self::new(t)\n }\n}\n\n/// A guard used to release the shared read access of a [`ShardedLock`] when dropped.\n#[clippy::has_significant_drop]\npub struct ShardedLockReadGuard<'a, T: ?Sized> {\n lock: &'a ShardedLock,\n _guard: RwLockReadGuard<'a, ()>,\n _marker: PhantomData>,\n}\n\nunsafe impl Sync for ShardedLockReadGuard<'_, T> {}\n\nimpl Deref for ShardedLockReadGuard<'_, T> {\n type Target = T;\n\n fn deref(&self) -> &T {\n unsafe { &*self.lock.value.get() }\n }\n}\n\nimpl fmt::Debug for ShardedLockReadGuard<'_, T> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"ShardedLockReadGuard\")\n .field(\"lock\", &self.lock)\n .finish()\n }\n}\n\nimpl fmt::Display for ShardedLockReadGuard<'_, T> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n (**self).fmt(f)\n }\n}\n\n/// A guard used to release the exclusive write access of a [`ShardedLock`] when dropped.\n#[clippy::has_significant_drop]\npub struct ShardedLockWriteGuard<'a, T: ?Sized> {\n lock: &'a ShardedLock,\n _marker: PhantomData>,\n}\n\nunsafe impl Sync for ShardedLockWriteGuard<'_, T> {}\n\nimpl Drop for ShardedLockWriteGuard<'_, T> {\n fn drop(&mut self) {\n // Unlock the shards in reverse order of locking.\n for shard in self.lock.shards.iter().rev() {\n unsafe {\n let dest: *mut _ = shard.write_guard.get();\n let guard = (*dest).take();\n drop(guard);\n }\n }\n }\n}\n\nimpl fmt::Debug for ShardedLockWriteGuard<'_, T> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_struct(\"ShardedLockWriteGuard\")\n .field(\"lock\", &self.lock)\n .finish()\n }\n}\n\nimpl fmt::Display for ShardedLockWriteGuard<'_, T> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n (**self).fmt(f)\n }\n}\n\nimpl Deref for ShardedLockWriteGuard<'_, T> {\n type Target = T;\n\n fn deref(&self) -> &T {\n unsafe { &*self.lock.value.get() }\n }\n}\n\nimpl DerefMut for ShardedLockWriteGuard<'_, T> {\n fn deref_mut(&mut self) -> &mut T {\n unsafe { &mut *self.lock.value.get() }\n }\n}\n\n/// Returns a `usize` that identifies the current thread.\n///\n/// Each thread is associated with an 'index'. While there are no particular guarantees, indices\n/// usually tend to be consecutive numbers between 0 and the number of running threads.\n///\n/// Since this function accesses TLS, `None` might be returned if the current thread's TLS is\n/// tearing down.\n#[inline]\nfn current_index() -> Option {\n REGISTRATION.try_with(|reg| reg.index).ok()\n}\n\n/// The global registry keeping track of registered threads and indices.\nstruct ThreadIndices {\n /// Mapping from `ThreadId` to thread index.\n mapping: HashMap,\n\n /// A list of free indices.\n free_list: Vec,\n\n /// The next index to allocate if the free list is empty.\n next_index: usize,\n}\n\nfn thread_indices() -> &'static Mutex {\n static THREAD_INDICES: OnceLock> = OnceLock::new();\n fn init() -> Mutex {\n Mutex::new(ThreadIndices {\n mapping: HashMap::new(),\n free_list: Vec::new(),\n next_index: 0,\n })\n }\n THREAD_INDICES.get_or_init(init)\n}\n\n/// A registration of a thread with an index.\n///\n/// When dropped, unregisters the thread and frees the reserved index.\nstruct Registration {\n index: usize,\n thread_id: ThreadId,\n}\n\nimpl Drop for Registration {\n fn drop(&mut self) {\n let mut indices = thread_indices().lock().unwrap();\n indices.mapping.remove(&self.thread_id);\n indices.free_list.push(self.index);\n }\n}\n\nstd::thread_local! {\n static REGISTRATION: Registration = {\n let thread_id = thread::current().id();\n let mut indices = thread_indices().lock().unwrap();\n\n let index = match indices.free_list.pop() {\n Some(i) => i,\n None => {\n let i = indices.next_index;\n indices.next_index += 1;\n i\n }\n };\n indices.mapping.insert(thread_id, index);\n\n Registration {\n index,\n thread_id,\n }\n };\n}\n" }, { "path": "crossbeam-utils/src/sync/wait_group.rs", "content": "use core::{fmt, mem::ManuallyDrop};\n\nuse crate::primitive::sync::{\n Arc, Condvar, Mutex,\n atomic::{AtomicUsize, Ordering},\n};\n\n/// Enables threads to synchronize the beginning or end of some computation.\n///\n/// # Wait groups vs barriers\n///\n/// `WaitGroup` is very similar to [`Barrier`], but there are a few differences:\n///\n/// * [`Barrier`] needs to know the number of threads at construction, while `WaitGroup` is cloned to\n/// register more threads.\n///\n/// * A [`Barrier`] can be reused even after all threads have synchronized, while a `WaitGroup`\n/// synchronizes threads only once.\n///\n/// * All threads wait for others to reach the [`Barrier`]. With `WaitGroup`, each thread can choose\n/// to either wait for other threads or to continue without blocking.\n///\n/// # Examples\n///\n/// ```\n/// use crossbeam_utils::sync::WaitGroup;\n/// use std::thread;\n///\n/// // Create a new wait group.\n/// let wg = WaitGroup::new();\n///\n/// for _ in 0..4 {\n/// // Create another reference to the wait group.\n/// let wg = wg.clone();\n///\n/// thread::spawn(move || {\n/// // Do some work.\n///\n/// // Drop the reference to the wait group.\n/// drop(wg);\n/// });\n/// }\n///\n/// // Block until all threads have finished their work.\n/// wg.wait();\n/// # if cfg!(miri) { std::thread::sleep(std::time::Duration::from_millis(500)); } // wait for background threads closed: https://github.com/rust-lang/miri/issues/1371\n/// ```\n///\n/// [`Barrier`]: std::sync::Barrier\npub struct WaitGroup {\n inner: Arc,\n}\n\n/// Inner state of a `WaitGroup`.\nstruct Inner {\n cvar: Condvar,\n lock: Mutex<()>,\n count: AtomicUsize,\n}\n\nimpl Default for WaitGroup {\n fn default() -> Self {\n Self {\n inner: Arc::new(Inner {\n cvar: Condvar::new(),\n lock: Mutex::new(()),\n count: AtomicUsize::new(1),\n }),\n }\n }\n}\n\nimpl WaitGroup {\n /// Creates a new wait group and returns the single reference to it.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::WaitGroup;\n ///\n /// let wg = WaitGroup::new();\n /// ```\n pub fn new() -> Self {\n Self::default()\n }\n\n /// Drops this reference and waits until all other references are dropped.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::sync::WaitGroup;\n /// use std::thread;\n ///\n /// let wg = WaitGroup::new();\n ///\n /// # let t =\n /// thread::spawn({\n /// let wg = wg.clone();\n /// move || {\n /// // Block until both threads have reached `wait()`.\n /// wg.wait();\n /// }\n /// });\n ///\n /// // Block until both threads have reached `wait()`.\n /// wg.wait();\n /// # t.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n /// ```\n pub fn wait(self) {\n // SAFETY: this is equivalent to let Self { inner } = self, without calling our Drop.\n let inner = unsafe {\n let slf = ManuallyDrop::new(self);\n core::ptr::read(&slf.inner)\n };\n\n if inner.count.fetch_sub(1, Ordering::AcqRel) == 1 {\n // Acquire lock after updating count, see below.\n drop(inner.lock.lock().unwrap());\n inner.cvar.notify_all();\n return;\n }\n\n // We check the counter while holding the lock, and notifiers acquire\n // the lock between updating the counter and notifying, ensuring we\n // can not miss the notification.\n let mut guard = inner.lock.lock().unwrap();\n while inner.count.load(Ordering::Acquire) != 0 {\n guard = inner.cvar.wait(guard).unwrap();\n }\n }\n}\n\nimpl Drop for WaitGroup {\n fn drop(&mut self) {\n if self.inner.count.fetch_sub(1, Ordering::Release) == 1 {\n // Acquire lock after updating count, see wait().\n drop(self.inner.lock.lock().unwrap());\n self.inner.cvar.notify_all();\n }\n }\n}\n\nimpl Clone for WaitGroup {\n fn clone(&self) -> Self {\n self.inner.count.fetch_add(1, Ordering::Relaxed);\n Self {\n inner: self.inner.clone(),\n }\n }\n}\n\nimpl fmt::Debug for WaitGroup {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n let count = self.inner.count.load(Ordering::Relaxed);\n f.debug_struct(\"WaitGroup\").field(\"count\", &count).finish()\n }\n}\n" }, { "path": "crossbeam-utils/src/thread.rs", "content": "//! Threads that can borrow variables from the stack.\n//!\n//! Create a scope when spawned threads need to access variables on the stack:\n//!\n//! ```\n//! use crossbeam_utils::thread;\n//!\n//! let people = vec![\n//! \"Alice\".to_string(),\n//! \"Bob\".to_string(),\n//! \"Carol\".to_string(),\n//! ];\n//!\n//! thread::scope(|s| {\n//! for person in &people {\n//! s.spawn(move |_| {\n//! println!(\"Hello, {}!\", person);\n//! });\n//! }\n//! }).unwrap();\n//! ```\n//!\n//! # Why scoped threads?\n//!\n//! Suppose we wanted to re-write the previous example using plain threads:\n//!\n//! ```compile_fail,E0597\n//! use std::thread;\n//!\n//! let people = vec![\n//! \"Alice\".to_string(),\n//! \"Bob\".to_string(),\n//! \"Carol\".to_string(),\n//! ];\n//!\n//! let mut threads = Vec::new();\n//!\n//! for person in &people {\n//! threads.push(thread::spawn(move || {\n//! println!(\"Hello, {}!\", person);\n//! }));\n//! }\n//!\n//! for thread in threads {\n//! thread.join().unwrap();\n//! }\n//! ```\n//!\n//! This doesn't work because the borrow checker complains about `people` not living long enough:\n//!\n//! ```text\n//! error[E0597]: `people` does not live long enough\n//! --> src/main.rs:12:20\n//! |\n//! 12 | for person in &people {\n//! | ^^^^^^ borrowed value does not live long enough\n//! ...\n//! 21 | }\n//! | - borrowed value only lives until here\n//! |\n//! = note: borrowed value must be valid for the static lifetime...\n//! ```\n//!\n//! The problem here is that spawned threads are not allowed to borrow variables on stack because\n//! the compiler cannot prove they will be joined before `people` is destroyed.\n//!\n//! Scoped threads are a mechanism to guarantee to the compiler that spawned threads will be joined\n//! before the scope ends.\n//!\n//! # How scoped threads work\n//!\n//! If a variable is borrowed by a thread, the thread must complete before the variable is\n//! destroyed. Threads spawned using [`std::thread::spawn`] can only borrow variables with the\n//! `'static` lifetime because the borrow checker cannot be sure when the thread will complete.\n//!\n//! A scope creates a clear boundary between variables outside the scope and threads inside the\n//! scope. Whenever a scope spawns a thread, it promises to join the thread before the scope ends.\n//! This way we guarantee to the borrow checker that scoped threads only live within the scope and\n//! can safely access variables outside it.\n//!\n//! # Nesting scoped threads\n//!\n//! Sometimes scoped threads need to spawn more threads within the same scope. This is a little\n//! tricky because argument `s` lives *inside* the invocation of `thread::scope()` and as such\n//! cannot be borrowed by scoped threads:\n//!\n//! ```compile_fail,E0521\n//! use crossbeam_utils::thread;\n//!\n//! thread::scope(|s| {\n//! s.spawn(|_| {\n//! // Not going to compile because we're trying to borrow `s`,\n//! // which lives *inside* the scope! :(\n//! s.spawn(|_| println!(\"nested thread\"));\n//! });\n//! });\n//! ```\n//!\n//! Fortunately, there is a solution. Every scoped thread is passed a reference to its scope as an\n//! argument, which can be used for spawning nested threads:\n//!\n//! ```\n//! use crossbeam_utils::thread;\n//!\n//! thread::scope(|s| {\n//! // Note the `|s|` here.\n//! s.spawn(|s| {\n//! // Yay, this works because we're using a fresh argument `s`! :)\n//! s.spawn(|_| println!(\"nested thread\"));\n//! });\n//! }).unwrap();\n//! ```\n\nuse alloc::{boxed::Box, string::String, sync::Arc, vec::Vec};\nuse core::{fmt, marker::PhantomData, mem};\nuse std::{io, panic, sync::Mutex, thread};\n\nuse crate::sync::WaitGroup;\n\ntype SharedVec = Arc>>;\ntype SharedOption = Arc>>;\n\n/// Creates a new scope for spawning threads.\n///\n/// All child threads that haven't been manually joined will be automatically joined just before\n/// this function invocation ends. If all joined threads have successfully completed, `Ok` is\n/// returned with the return value of `f`. If any of the joined threads has panicked, an `Err` is\n/// returned containing errors from panicked threads. Note that if panics are implemented by\n/// aborting the process, no error is returned; see the notes of [std::panic::catch_unwind].\n///\n/// **Note:** Since Rust 1.63, this function is soft-deprecated in favor of the more efficient [`std::thread::scope`].\n///\n/// # Examples\n///\n/// ```\n/// use crossbeam_utils::thread;\n///\n/// let var = vec![1, 2, 3];\n///\n/// thread::scope(|s| {\n/// s.spawn(|_| {\n/// println!(\"A child thread borrowing `var`: {:?}\", var);\n/// });\n/// }).unwrap();\n/// ```\npub fn scope<'env, F, R>(f: F) -> thread::Result\nwhere\n F: FnOnce(&Scope<'env>) -> R,\n{\n struct AbortOnPanic;\n impl Drop for AbortOnPanic {\n fn drop(&mut self) {\n if thread::panicking() {\n std::process::abort();\n }\n }\n }\n\n let wg = WaitGroup::new();\n let scope = Scope::<'env> {\n handles: SharedVec::default(),\n wait_group: wg.clone(),\n _marker: PhantomData,\n };\n\n // Execute the scoped function, but catch any panics.\n let result = panic::catch_unwind(panic::AssertUnwindSafe(|| f(&scope)));\n\n // If an unwinding panic occurs before all threads are joined\n // promote it to an aborting panic to prevent any threads from escaping the scope.\n let guard = AbortOnPanic;\n\n // Wait until all nested scopes are dropped.\n drop(scope.wait_group);\n wg.wait();\n\n // Join all remaining spawned threads.\n let panics: Vec<_> = scope\n .handles\n .lock()\n .unwrap()\n // Filter handles that haven't been joined, join them, and collect errors.\n .drain(..)\n .filter_map(|handle| handle.lock().unwrap().take())\n .filter_map(|handle| handle.join().err())\n .collect();\n\n mem::forget(guard);\n\n // If `f` has panicked, resume unwinding.\n // If any of the child threads have panicked, return the panic errors.\n // Otherwise, everything is OK and return the result of `f`.\n match result {\n Err(err) => panic::resume_unwind(err),\n Ok(res) => {\n if panics.is_empty() {\n Ok(res)\n } else {\n Err(Box::new(panics))\n }\n }\n }\n}\n\n/// A scope for spawning threads.\npub struct Scope<'env> {\n /// The list of the thread join handles.\n handles: SharedVec>>,\n\n /// Used to wait until all subscopes all dropped.\n wait_group: WaitGroup,\n\n /// Borrows data with invariant lifetime `'env`.\n _marker: PhantomData<&'env mut &'env ()>,\n}\n\nunsafe impl Sync for Scope<'_> {}\n\nimpl<'env> Scope<'env> {\n /// Spawns a scoped thread.\n ///\n /// This method is similar to the [`spawn`] function in Rust's standard library. The difference\n /// is that this thread is scoped, meaning it's guaranteed to terminate before the scope exits,\n /// allowing it to reference variables outside the scope.\n ///\n /// The scoped thread is passed a reference to this scope as an argument, which can be used for\n /// spawning nested threads.\n ///\n /// The returned [handle](ScopedJoinHandle) can be used to manually\n /// [join](ScopedJoinHandle::join) the thread before the scope exits.\n ///\n /// This will create a thread using default parameters of [`ScopedThreadBuilder`], if you want to specify the\n /// stack size or the name of the thread, use this API instead.\n ///\n /// [`spawn`]: std::thread::spawn\n ///\n /// # Panics\n ///\n /// Panics if the OS fails to create a thread; use [`ScopedThreadBuilder::spawn`]\n /// to recover from such errors.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::thread;\n ///\n /// thread::scope(|s| {\n /// let handle = s.spawn(|_| {\n /// println!(\"A child thread is running\");\n /// 42\n /// });\n ///\n /// // Join the thread and retrieve its result.\n /// let res = handle.join().unwrap();\n /// assert_eq!(res, 42);\n /// }).unwrap();\n /// ```\n pub fn spawn<'scope, F, T>(&'scope self, f: F) -> ScopedJoinHandle<'scope, T>\n where\n F: FnOnce(&Scope<'env>) -> T,\n F: Send + 'env,\n T: Send + 'env,\n {\n self.builder()\n .spawn(f)\n .expect(\"failed to spawn scoped thread\")\n }\n\n /// Creates a builder that can configure a thread before spawning.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::thread;\n ///\n /// thread::scope(|s| {\n /// s.builder()\n /// .spawn(|_| println!(\"A child thread is running\"))\n /// .unwrap();\n /// }).unwrap();\n /// ```\n pub fn builder<'scope>(&'scope self) -> ScopedThreadBuilder<'scope, 'env> {\n ScopedThreadBuilder {\n scope: self,\n builder: thread::Builder::new(),\n }\n }\n}\n\nimpl fmt::Debug for Scope<'_> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"Scope { .. }\")\n }\n}\n\n/// Configures the properties of a new thread.\n///\n/// The two configurable properties are:\n///\n/// - [`name`]: Specifies an [associated name for the thread][naming-threads].\n/// - [`stack_size`]: Specifies the [desired stack size for the thread][stack-size].\n///\n/// The [`spawn`] method will take ownership of the builder and return an [`io::Result`] of the\n/// thread handle with the given configuration.\n///\n/// The [`Scope::spawn`] method uses a builder with default configuration and unwraps its return\n/// value. You may want to use this builder when you want to recover from a failure to launch a\n/// thread.\n///\n/// # Examples\n///\n/// ```\n/// use crossbeam_utils::thread;\n///\n/// thread::scope(|s| {\n/// s.builder()\n/// .spawn(|_| println!(\"Running a child thread\"))\n/// .unwrap();\n/// }).unwrap();\n/// ```\n///\n/// [`name`]: ScopedThreadBuilder::name\n/// [`stack_size`]: ScopedThreadBuilder::stack_size\n/// [`spawn`]: ScopedThreadBuilder::spawn\n/// [`io::Result`]: std::io::Result\n/// [naming-threads]: std::thread#naming-threads\n/// [stack-size]: std::thread#stack-size\n#[must_use = \"must eventually spawn the thread\"]\n#[derive(Debug)]\npub struct ScopedThreadBuilder<'scope, 'env> {\n scope: &'scope Scope<'env>,\n builder: thread::Builder,\n}\n\nimpl<'scope, 'env> ScopedThreadBuilder<'scope, 'env> {\n /// Sets the name for the new thread.\n ///\n /// The name must not contain null bytes (`\\0`).\n ///\n /// For more information about named threads, see [here][naming-threads].\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::thread;\n /// use std::thread::current;\n ///\n /// thread::scope(|s| {\n /// s.builder()\n /// .name(\"my thread\".to_string())\n /// .spawn(|_| assert_eq!(current().name(), Some(\"my thread\")))\n /// .unwrap();\n /// }).unwrap();\n /// ```\n ///\n /// [naming-threads]: std::thread#naming-threads\n pub fn name(mut self, name: String) -> Self {\n self.builder = self.builder.name(name);\n self\n }\n\n /// Sets the size of the stack for the new thread.\n ///\n /// The stack size is measured in bytes.\n ///\n /// For more information about the stack size for threads, see [here][stack-size].\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::thread;\n ///\n /// thread::scope(|s| {\n /// s.builder()\n /// .stack_size(32 * 1024)\n /// .spawn(|_| println!(\"Running a child thread\"))\n /// .unwrap();\n /// }).unwrap();\n /// ```\n ///\n /// [stack-size]: std::thread#stack-size\n pub fn stack_size(mut self, size: usize) -> Self {\n self.builder = self.builder.stack_size(size);\n self\n }\n\n /// Spawns a scoped thread with this configuration.\n ///\n /// The scoped thread is passed a reference to this scope as an argument, which can be used for\n /// spawning nested threads.\n ///\n /// The returned handle can be used to manually join the thread before the scope exits.\n ///\n /// # Errors\n ///\n /// Unlike the [`Scope::spawn`] method, this method yields an\n /// [`io::Result`] to capture any failure to create the thread at\n /// the OS level.\n ///\n /// [`io::Result`]: std::io::Result\n ///\n /// # Panics\n ///\n /// Panics if a thread name was set and it contained null bytes.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::thread;\n ///\n /// thread::scope(|s| {\n /// let handle = s.builder()\n /// .spawn(|_| {\n /// println!(\"A child thread is running\");\n /// 42\n /// })\n /// .unwrap();\n ///\n /// // Join the thread and retrieve its result.\n /// let res = handle.join().unwrap();\n /// assert_eq!(res, 42);\n /// }).unwrap();\n /// ```\n pub fn spawn(self, f: F) -> io::Result>\n where\n F: FnOnce(&Scope<'env>) -> T,\n F: Send + 'env,\n T: Send + 'env,\n {\n // The result of `f` will be stored here.\n let result = SharedOption::default();\n\n // Spawn the thread and grab its join handle and thread handle.\n let (handle, thread) = {\n let result = Arc::clone(&result);\n\n // A clone of the scope that will be moved into the new thread.\n let scope = Scope::<'env> {\n handles: Arc::clone(&self.scope.handles),\n wait_group: self.scope.wait_group.clone(),\n _marker: PhantomData,\n };\n\n // Spawn the thread.\n let handle = {\n let closure = move || {\n // Make sure the scope is inside the closure with the proper `'env` lifetime.\n let scope: Scope<'env> = scope;\n\n // Run the closure.\n let res = f(&scope);\n\n // Store the result if the closure didn't panic.\n *result.lock().unwrap() = Some(res);\n };\n\n // Allocate `closure` on the heap and erase the `'env` bound.\n let closure: Box = Box::new(closure);\n let closure: Box =\n unsafe { mem::transmute(closure) };\n\n // Finally, spawn the closure.\n self.builder.spawn(closure)?\n };\n\n let thread = handle.thread().clone();\n let handle = Arc::new(Mutex::new(Some(handle)));\n (handle, thread)\n };\n\n // Add the handle to the shared list of join handles.\n self.scope.handles.lock().unwrap().push(Arc::clone(&handle));\n\n Ok(ScopedJoinHandle {\n handle,\n result,\n thread,\n _marker: PhantomData,\n })\n }\n}\n\nunsafe impl Send for ScopedJoinHandle<'_, T> {}\nunsafe impl Sync for ScopedJoinHandle<'_, T> {}\n\n/// A handle that can be used to join its scoped thread.\n///\n/// This struct is created by the [`Scope::spawn`] method and the\n/// [`ScopedThreadBuilder::spawn`] method.\npub struct ScopedJoinHandle<'scope, T> {\n /// A join handle to the spawned thread.\n handle: SharedOption>,\n\n /// Holds the result of the inner closure.\n result: SharedOption,\n\n /// A handle to the spawned thread.\n thread: thread::Thread,\n\n /// Borrows the parent scope with lifetime `'scope`.\n _marker: PhantomData<&'scope ()>,\n}\n\nimpl ScopedJoinHandle<'_, T> {\n /// Waits for the thread to finish and returns its result.\n ///\n /// If the child thread panics, an error is returned. Note that if panics are implemented by\n /// aborting the process, no error is returned; see the notes of [std::panic::catch_unwind].\n ///\n /// # Panics\n ///\n /// This function may panic on some platforms if a thread attempts to join itself or otherwise\n /// may create a deadlock with joining threads.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::thread;\n ///\n /// thread::scope(|s| {\n /// let handle1 = s.spawn(|_| println!(\"I'm a happy thread :)\"));\n /// let handle2 = s.spawn(|_| panic!(\"I'm a sad thread :(\"));\n ///\n /// // Join the first thread and verify that it succeeded.\n /// let res = handle1.join();\n /// assert!(res.is_ok());\n ///\n /// // Join the second thread and verify that it panicked.\n /// let res = handle2.join();\n /// assert!(res.is_err());\n /// }).unwrap();\n /// ```\n pub fn join(self) -> thread::Result {\n // Take out the handle. The handle will surely be available because the root scope waits\n // for nested scopes before joining remaining threads.\n let handle = self.handle.lock().unwrap().take().unwrap();\n\n // Join the thread and then take the result out of its inner closure.\n handle\n .join()\n .map(|()| self.result.lock().unwrap().take().unwrap())\n }\n\n /// Returns a handle to the underlying thread.\n ///\n /// # Examples\n ///\n /// ```\n /// use crossbeam_utils::thread;\n ///\n /// thread::scope(|s| {\n /// let handle = s.spawn(|_| println!(\"A child thread is running\"));\n /// println!(\"The child thread ID: {:?}\", handle.thread().id());\n /// }).unwrap();\n /// ```\n pub fn thread(&self) -> &thread::Thread {\n &self.thread\n }\n}\n\n/// Unix-specific extensions.\n#[cfg(unix)]\npub mod unix {\n use std::os::unix::thread::JoinHandleExt as _;\n #[doc(no_inline)]\n pub use std::os::unix::thread::RawPthread;\n\n use super::ScopedJoinHandle;\n\n mod sealed {\n pub trait Sealed {}\n }\n\n /// Unix-specific extensions to [`ScopedJoinHandle`].\n pub trait JoinHandleExt: sealed::Sealed {\n /// Extracts the raw pthread_t without taking ownership\n fn as_pthread_t(&self) -> RawPthread;\n\n /// Consumes the thread, returning the raw pthread_t\n ///\n /// This function **transfers ownership** of the underlying pthread_t to\n /// the caller. Callers are then the unique owners of the pthread_t and\n /// must either detach or join the pthread_t once it's no longer needed.\n fn into_pthread_t(self) -> RawPthread;\n }\n\n impl sealed::Sealed for ScopedJoinHandle<'_, T> {}\n impl JoinHandleExt for ScopedJoinHandle<'_, T> {\n fn as_pthread_t(&self) -> RawPthread {\n // Borrow the handle. The handle will surely be available because the root scope waits\n // for nested scopes before joining remaining threads.\n let handle = self.handle.lock().unwrap();\n handle.as_ref().unwrap().as_pthread_t()\n }\n fn into_pthread_t(self) -> RawPthread {\n self.as_pthread_t()\n }\n }\n}\n/// Windows-specific extensions.\n#[cfg(windows)]\nmod windows {\n use std::os::windows::io::{AsRawHandle, IntoRawHandle, RawHandle};\n\n use super::ScopedJoinHandle;\n\n impl AsRawHandle for ScopedJoinHandle<'_, T> {\n fn as_raw_handle(&self) -> RawHandle {\n // Borrow the handle. The handle will surely be available because the root scope waits\n // for nested scopes before joining remaining threads.\n let handle = self.handle.lock().unwrap();\n handle.as_ref().unwrap().as_raw_handle()\n }\n }\n\n impl IntoRawHandle for ScopedJoinHandle<'_, T> {\n fn into_raw_handle(self) -> RawHandle {\n self.as_raw_handle()\n }\n }\n}\n\nimpl fmt::Debug for ScopedJoinHandle<'_, T> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.pad(\"ScopedJoinHandle { .. }\")\n }\n}\n" }, { "path": "crossbeam-utils/tests/atomic_cell.rs", "content": "use std::{\n mem,\n sync::atomic::{AtomicUsize, Ordering::SeqCst},\n};\n\nuse crossbeam_utils::atomic::AtomicCell;\n\n// Always use fallback for now on environments that do not support inline assembly.\nfn always_use_fallback() -> bool {\n atomic_maybe_uninit::cfg_has_atomic_cas! {\n cfg!(any(\n miri,\n crossbeam_loom,\n crossbeam_atomic_cell_force_fallback,\n ))\n }\n atomic_maybe_uninit::cfg_no_atomic_cas! { true }\n}\n\n#[test]\nfn is_lock_free() {\n let always_use_fallback = always_use_fallback();\n\n struct UsizeWrap(#[allow(dead_code)] usize);\n struct U8Wrap(#[allow(dead_code)] bool);\n struct I16Wrap(#[allow(dead_code)] i16);\n #[repr(align(8))]\n struct U64Align8(#[allow(dead_code)] u64);\n\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n assert_eq!(\n AtomicCell::::is_lock_free(),\n !always_use_fallback\n );\n\n assert!(AtomicCell::<()>::is_lock_free());\n\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n assert_eq!(AtomicCell::::is_lock_free(), !always_use_fallback);\n\n // Sizes of both types must be equal, and the alignment of `u64` must be greater or equal than\n // that of `AtomicU64`. In i686-unknown-linux-gnu, the alignment of `u64` is `4` and alignment\n // of `AtomicU64` is `8`, so `AtomicCell` is not lock-free.\n assert_eq!(\n AtomicCell::::is_lock_free(),\n cfg!(target_has_atomic = \"64\") && mem::align_of::() == 8 && !always_use_fallback\n );\n assert_eq!(mem::size_of::(), 8);\n assert_eq!(mem::align_of::(), 8);\n assert_eq!(\n AtomicCell::::is_lock_free(),\n cfg!(target_has_atomic = \"64\") && !always_use_fallback\n );\n\n atomic_maybe_uninit::cfg_has_atomic_128! {\n assert_eq!(\n AtomicCell::::is_lock_free(),\n mem::align_of::() == 16 && !always_use_fallback\n );\n };\n atomic_maybe_uninit::cfg_no_atomic_128! {\n assert!(!AtomicCell::::is_lock_free());\n };\n}\n\n#[test]\nfn drops_unit() {\n static CNT: AtomicUsize = AtomicUsize::new(0);\n CNT.store(0, SeqCst);\n\n #[derive(Debug, PartialEq, Eq)]\n struct Foo();\n\n impl Foo {\n fn new() -> Self {\n CNT.fetch_add(1, SeqCst);\n Self()\n }\n }\n\n impl Drop for Foo {\n fn drop(&mut self) {\n CNT.fetch_sub(1, SeqCst);\n }\n }\n\n impl Default for Foo {\n fn default() -> Self {\n Self::new()\n }\n }\n\n let a = AtomicCell::new(Foo::new());\n\n assert_eq!(a.swap(Foo::new()), Foo::new());\n assert_eq!(CNT.load(SeqCst), 1);\n\n a.store(Foo::new());\n assert_eq!(CNT.load(SeqCst), 1);\n\n assert_eq!(a.swap(Foo::default()), Foo::new());\n assert_eq!(CNT.load(SeqCst), 1);\n\n drop(a);\n assert_eq!(CNT.load(SeqCst), 0);\n}\n\n#[test]\nfn drops_u8() {\n static CNT: AtomicUsize = AtomicUsize::new(0);\n CNT.store(0, SeqCst);\n\n #[derive(Debug, PartialEq, Eq)]\n struct Foo(u8);\n\n impl Foo {\n fn new(val: u8) -> Self {\n CNT.fetch_add(1, SeqCst);\n Self(val)\n }\n }\n\n impl Drop for Foo {\n fn drop(&mut self) {\n CNT.fetch_sub(1, SeqCst);\n }\n }\n\n impl Default for Foo {\n fn default() -> Self {\n Self::new(0)\n }\n }\n\n let a = AtomicCell::new(Foo::new(5));\n\n assert_eq!(a.swap(Foo::new(6)), Foo::new(5));\n assert_eq!(a.swap(Foo::new(1)), Foo::new(6));\n assert_eq!(CNT.load(SeqCst), 1);\n\n a.store(Foo::new(2));\n assert_eq!(CNT.load(SeqCst), 1);\n\n assert_eq!(a.swap(Foo::default()), Foo::new(2));\n assert_eq!(CNT.load(SeqCst), 1);\n\n assert_eq!(a.swap(Foo::default()), Foo::new(0));\n assert_eq!(CNT.load(SeqCst), 1);\n\n drop(a);\n assert_eq!(CNT.load(SeqCst), 0);\n}\n\n#[test]\nfn drops_usize() {\n static CNT: AtomicUsize = AtomicUsize::new(0);\n CNT.store(0, SeqCst);\n\n #[derive(Debug, PartialEq, Eq)]\n struct Foo(usize);\n\n impl Foo {\n fn new(val: usize) -> Self {\n CNT.fetch_add(1, SeqCst);\n Self(val)\n }\n }\n\n impl Drop for Foo {\n fn drop(&mut self) {\n CNT.fetch_sub(1, SeqCst);\n }\n }\n\n impl Default for Foo {\n fn default() -> Self {\n Self::new(0)\n }\n }\n\n let a = AtomicCell::new(Foo::new(5));\n\n assert_eq!(a.swap(Foo::new(6)), Foo::new(5));\n assert_eq!(a.swap(Foo::new(1)), Foo::new(6));\n assert_eq!(CNT.load(SeqCst), 1);\n\n a.store(Foo::new(2));\n assert_eq!(CNT.load(SeqCst), 1);\n\n assert_eq!(a.swap(Foo::default()), Foo::new(2));\n assert_eq!(CNT.load(SeqCst), 1);\n\n assert_eq!(a.swap(Foo::default()), Foo::new(0));\n assert_eq!(CNT.load(SeqCst), 1);\n\n drop(a);\n assert_eq!(CNT.load(SeqCst), 0);\n}\n\n#[test]\nfn modular_u8() {\n #[derive(Clone, Copy, Eq, Debug, Default)]\n struct Foo(u8);\n\n impl PartialEq for Foo {\n fn eq(&self, other: &Self) -> bool {\n self.0 % 5 == other.0 % 5\n }\n }\n\n let a = AtomicCell::new(Foo(1));\n\n assert_eq!(a.load(), Foo(1));\n assert_eq!(a.swap(Foo(2)), Foo(11));\n assert_eq!(a.load(), Foo(52));\n\n a.store(Foo(0));\n assert_eq!(a.compare_exchange(Foo(0), Foo(5)), Ok(Foo(100)));\n assert_eq!(a.load().0, 5);\n assert_eq!(a.compare_exchange(Foo(10), Foo(15)), Ok(Foo(100)));\n assert_eq!(a.load().0, 15);\n}\n\n#[test]\nfn modular_usize() {\n #[derive(Clone, Copy, Eq, Debug, Default)]\n struct Foo(usize);\n\n impl PartialEq for Foo {\n fn eq(&self, other: &Self) -> bool {\n self.0 % 5 == other.0 % 5\n }\n }\n\n let a = AtomicCell::new(Foo(1));\n\n assert_eq!(a.load(), Foo(1));\n assert_eq!(a.swap(Foo(2)), Foo(11));\n assert_eq!(a.load(), Foo(52));\n\n a.store(Foo(0));\n assert_eq!(a.compare_exchange(Foo(0), Foo(5)), Ok(Foo(100)));\n assert_eq!(a.load().0, 5);\n assert_eq!(a.compare_exchange(Foo(10), Foo(15)), Ok(Foo(100)));\n assert_eq!(a.load().0, 15);\n}\n\n#[test]\nfn garbage_padding() {\n #[derive(Copy, Clone, Eq, PartialEq)]\n struct Object {\n a: i64,\n b: i32,\n }\n\n let cell = AtomicCell::new(Object { a: 0, b: 0 });\n let _garbage = [0xfe, 0xfe, 0xfe, 0xfe, 0xfe]; // Needed\n let next = Object { a: 0, b: 0 };\n\n let prev = cell.load();\n assert!(cell.compare_exchange(prev, next).is_ok());\n println!();\n}\n\n#[test]\nfn const_atomic_cell() {\n const _IS_LOCK_FREE_U: bool = AtomicCell::::is_lock_free();\n const _IS_LOCK_FREE_I: bool = AtomicCell::::is_lock_free();\n static INTO_INNER: usize = {\n let v = AtomicCell::new(!0);\n v.into_inner()\n };\n #[rustversion::since(1.83)]\n static _AS_PTR: AtomicCell = {\n let v = AtomicCell::new(!0);\n let _p = v.as_ptr();\n v\n };\n static CELL: AtomicCell = AtomicCell::new(0);\n\n CELL.store(1);\n assert_eq!(CELL.load(), 1);\n assert_eq!(INTO_INNER, !0);\n}\n\n// https://github.com/crossbeam-rs/crossbeam/pull/767\nmacro_rules! test_arithmetic {\n ($test_name:ident, $ty:ident) => {\n #[test]\n fn $test_name() {\n let a: AtomicCell<$ty> = AtomicCell::new(7);\n\n assert_eq!(a.fetch_add(3), 7);\n assert_eq!(a.load(), 10);\n\n assert_eq!(a.fetch_sub(3), 10);\n assert_eq!(a.load(), 7);\n\n assert_eq!(a.fetch_and(3), 7);\n assert_eq!(a.load(), 3);\n\n assert_eq!(a.fetch_or(16), 3);\n assert_eq!(a.load(), 19);\n\n assert_eq!(a.fetch_xor(2), 19);\n assert_eq!(a.load(), 17);\n\n assert_eq!(a.fetch_max(18), 17);\n assert_eq!(a.load(), 18);\n\n assert_eq!(a.fetch_min(17), 18);\n assert_eq!(a.load(), 17);\n\n assert_eq!(a.fetch_nand(7), 17);\n assert_eq!(a.load(), !(17 & 7));\n }\n };\n}\ntest_arithmetic!(arithmetic_u8, u8);\ntest_arithmetic!(arithmetic_i8, i8);\ntest_arithmetic!(arithmetic_u16, u16);\ntest_arithmetic!(arithmetic_i16, i16);\ntest_arithmetic!(arithmetic_u32, u32);\ntest_arithmetic!(arithmetic_i32, i32);\ntest_arithmetic!(arithmetic_u64, u64);\ntest_arithmetic!(arithmetic_i64, i64);\ntest_arithmetic!(arithmetic_u128, u128);\ntest_arithmetic!(arithmetic_i128, i128);\n\n// https://github.com/crossbeam-rs/crossbeam/issues/748\n#[test]\nfn issue_748() {\n #[allow(dead_code)]\n #[repr(align(8))]\n #[derive(Debug, Clone, Copy, PartialEq, Eq)]\n enum Test {\n Field(u32),\n FieldLess,\n }\n\n assert_eq!(mem::size_of::(), 8);\n assert_eq!(\n AtomicCell::::is_lock_free(),\n cfg!(target_has_atomic = \"64\") && !always_use_fallback()\n );\n let x = AtomicCell::new(Test::FieldLess);\n assert_eq!(x.load(), Test::FieldLess);\n}\n\n// https://github.com/crossbeam-rs/crossbeam/issues/833\n#[test]\nfn issue_833() {\n use std::{\n num::NonZeroU128,\n sync::atomic::{AtomicBool, Ordering},\n thread,\n };\n\n const N: usize = if cfg!(miri) { 10_000 } else { 1_000_000 };\n\n #[allow(dead_code)]\n enum Enum {\n NeverConstructed,\n Cell(AtomicCell),\n }\n\n static STATIC: Enum = Enum::Cell(AtomicCell::new(match NonZeroU128::new(1) {\n Some(nonzero) => nonzero,\n None => unreachable!(),\n }));\n static FINISHED: AtomicBool = AtomicBool::new(false);\n\n let handle = thread::spawn(|| {\n let cell = match &STATIC {\n Enum::NeverConstructed => unreachable!(),\n Enum::Cell(cell) => cell,\n };\n let x = NonZeroU128::new(0xFFFF_FFFF_FFFF_FFFF_0000_0000_0000_0000).unwrap();\n let y = NonZeroU128::new(0x0000_0000_0000_0000_FFFF_FFFF_FFFF_FFFF).unwrap();\n while !FINISHED.load(Ordering::Relaxed) {\n cell.store(x);\n cell.store(y);\n }\n });\n\n for _ in 0..N {\n if let Enum::NeverConstructed = STATIC {\n unreachable!(\":(\");\n }\n }\n\n FINISHED.store(true, Ordering::Relaxed);\n handle.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n}\n" }, { "path": "crossbeam-utils/tests/cache_padded.rs", "content": "use std::{cell::Cell, mem};\n\nuse crossbeam_utils::CachePadded;\n\n#[test]\nfn default() {\n let x: CachePadded = Default::default();\n assert_eq!(*x, 0);\n}\n\n#[test]\nfn store_u64() {\n let x: CachePadded = CachePadded::new(17);\n assert_eq!(*x, 17);\n}\n\n#[test]\nfn store_pair() {\n let x: CachePadded<(u64, u64)> = CachePadded::new((17, 37));\n assert_eq!(x.0, 17);\n assert_eq!(x.1, 37);\n}\n\n#[test]\nfn distance() {\n let arr = [CachePadded::new(17u8), CachePadded::new(37u8)];\n let a = &*arr[0] as *const u8;\n let b = &*arr[1] as *const u8;\n let align = mem::align_of::>();\n assert!(align >= 32);\n assert_eq!(unsafe { a.add(align) }, b);\n}\n\n#[test]\nfn different_sizes() {\n CachePadded::new(17u8);\n CachePadded::new(17u16);\n CachePadded::new(17u32);\n CachePadded::new([17u64; 0]);\n CachePadded::new([17u64; 1]);\n CachePadded::new([17u64; 2]);\n CachePadded::new([17u64; 3]);\n CachePadded::new([17u64; 4]);\n CachePadded::new([17u64; 5]);\n CachePadded::new([17u64; 6]);\n CachePadded::new([17u64; 7]);\n CachePadded::new([17u64; 8]);\n}\n\n#[test]\nfn large() {\n let a = [17u64; 9];\n let b = CachePadded::new(a);\n assert!(mem::size_of_val(&a) <= mem::size_of_val(&b));\n}\n\n#[test]\nfn debug() {\n assert_eq!(\n format!(\"{:?}\", CachePadded::new(17u64)),\n \"CachePadded { value: 17 }\"\n );\n}\n\n#[test]\nfn drops() {\n let count = Cell::new(0);\n\n struct Foo<'a>(&'a Cell);\n\n impl Drop for Foo<'_> {\n fn drop(&mut self) {\n self.0.set(self.0.get() + 1);\n }\n }\n\n let a = CachePadded::new(Foo(&count));\n let b = CachePadded::new(Foo(&count));\n\n assert_eq!(count.get(), 0);\n drop(a);\n assert_eq!(count.get(), 1);\n drop(b);\n assert_eq!(count.get(), 2);\n}\n\n#[allow(clippy::clone_on_copy)] // This is intentional.\n#[test]\nfn clone() {\n let a = CachePadded::new(17);\n let b = a.clone();\n assert_eq!(*a, *b);\n}\n\n#[test]\nfn runs_custom_clone() {\n let count = Cell::new(0);\n\n struct Foo<'a>(&'a Cell);\n\n impl Clone for Foo<'_> {\n fn clone(&self) -> Self {\n self.0.set(self.0.get() + 1);\n Self(self.0)\n }\n }\n\n let a = CachePadded::new(Foo(&count));\n let _ = a.clone();\n\n assert_eq!(count.get(), 1);\n}\n" }, { "path": "crossbeam-utils/tests/parker.rs", "content": "use std::{thread::sleep, time::Duration};\n\nuse crossbeam_utils::{\n sync::{Parker, UnparkReason},\n thread,\n};\n\n#[test]\nfn park_timeout_unpark_before() {\n let p = Parker::new();\n for _ in 0..10 {\n p.unparker().unpark();\n assert_eq!(\n p.park_timeout(Duration::from_millis(u32::MAX as u64)),\n UnparkReason::Unparked,\n );\n }\n}\n\n#[test]\nfn park_timeout_unpark_not_called() {\n let p = Parker::new();\n for _ in 0..10 {\n assert_eq!(\n p.park_timeout(Duration::from_millis(10)),\n UnparkReason::Timeout,\n );\n }\n}\n\n#[test]\nfn park_timeout_unpark_called_other_thread() {\n for _ in 0..10 {\n let p = Parker::new();\n let u = p.unparker().clone();\n\n thread::scope(|scope| {\n scope.spawn(move |_| {\n sleep(Duration::from_millis(50));\n u.unpark();\n });\n\n assert_eq!(\n p.park_timeout(Duration::from_millis(u32::MAX as u64)),\n UnparkReason::Unparked,\n );\n })\n .unwrap();\n }\n}\n" }, { "path": "crossbeam-utils/tests/sharded_lock.rs", "content": "use std::{\n sync::{\n Arc, TryLockError,\n atomic::{AtomicUsize, Ordering},\n mpsc::channel,\n },\n thread,\n};\n\nuse crossbeam_utils::sync::ShardedLock;\n\n#[derive(Eq, PartialEq, Debug)]\nstruct NonCopy(i32);\n\n#[test]\nfn smoke() {\n let l = ShardedLock::new(());\n drop(l.read().unwrap());\n drop(l.write().unwrap());\n drop((l.read().unwrap(), l.read().unwrap()));\n drop(l.write().unwrap());\n}\n\n#[test]\nfn frob() {\n const N: u32 = 10;\n const M: usize = if cfg!(miri) { 50 } else { 1000 };\n\n let r = Arc::new(ShardedLock::new(()));\n\n let (tx, rx) = channel::<()>();\n for _ in 0..N {\n let tx = tx.clone();\n let r = r.clone();\n thread::spawn(move || {\n let mut rng = fastrand::Rng::new();\n for _ in 0..M {\n if rng.u32(0..N) == 0 {\n drop(r.write().unwrap());\n } else {\n drop(r.read().unwrap());\n }\n }\n drop(tx);\n });\n }\n drop(tx);\n let _ = rx.recv();\n}\n\n#[test]\nfn arc_poison_wr() {\n let arc = Arc::new(ShardedLock::new(1));\n let arc2 = arc.clone();\n let _: Result<(), _> = thread::spawn(move || {\n let _lock = arc2.write().unwrap();\n panic!();\n })\n .join();\n assert!(arc.read().is_err());\n}\n\n#[test]\nfn arc_poison_ww() {\n let arc = Arc::new(ShardedLock::new(1));\n assert!(!arc.is_poisoned());\n let arc2 = arc.clone();\n let _: Result<(), _> = thread::spawn(move || {\n let _lock = arc2.write().unwrap();\n panic!();\n })\n .join();\n assert!(arc.write().is_err());\n assert!(arc.is_poisoned());\n}\n\n#[test]\nfn arc_no_poison_rr() {\n let arc = Arc::new(ShardedLock::new(1));\n let arc2 = arc.clone();\n let _: Result<(), _> = thread::spawn(move || {\n let _lock = arc2.read().unwrap();\n panic!();\n })\n .join();\n let lock = arc.read().unwrap();\n assert_eq!(*lock, 1);\n}\n#[test]\nfn arc_no_poison_sl() {\n let arc = Arc::new(ShardedLock::new(1));\n let arc2 = arc.clone();\n let _: Result<(), _> = thread::spawn(move || {\n let _lock = arc2.read().unwrap();\n panic!()\n })\n .join();\n let lock = arc.write().unwrap();\n assert_eq!(*lock, 1);\n}\n\n#[test]\nfn arc() {\n let arc = Arc::new(ShardedLock::new(0));\n let arc2 = arc.clone();\n let (tx, rx) = channel();\n\n thread::spawn(move || {\n let mut lock = arc2.write().unwrap();\n for _ in 0..10 {\n let tmp = *lock;\n *lock = -1;\n thread::yield_now();\n *lock = tmp + 1;\n }\n tx.send(()).unwrap();\n });\n\n // Readers try to catch the writer in the act\n let mut children = Vec::new();\n for _ in 0..5 {\n let arc3 = arc.clone();\n children.push(thread::spawn(move || {\n let lock = arc3.read().unwrap();\n assert!(*lock >= 0);\n }));\n }\n\n // Wait for children to pass their asserts\n for r in children {\n assert!(r.join().is_ok());\n }\n\n // Wait for writer to finish\n rx.recv().unwrap();\n let lock = arc.read().unwrap();\n assert_eq!(*lock, 10);\n}\n\n#[test]\nfn arc_access_in_unwind() {\n let arc = Arc::new(ShardedLock::new(1));\n let arc2 = arc.clone();\n let _: Result<(), _> = thread::spawn(move || {\n struct Unwinder {\n i: Arc>,\n }\n impl Drop for Unwinder {\n fn drop(&mut self) {\n let mut lock = self.i.write().unwrap();\n *lock += 1;\n }\n }\n let _u = Unwinder { i: arc2 };\n panic!();\n })\n .join();\n let lock = arc.read().unwrap();\n assert_eq!(*lock, 2);\n}\n\n#[test]\nfn unsized_type() {\n let sl: &ShardedLock<[i32]> = &ShardedLock::new([1, 2, 3]);\n {\n let b = &mut *sl.write().unwrap();\n b[0] = 4;\n b[2] = 5;\n }\n let comp: &[i32] = &[4, 2, 5];\n assert_eq!(&*sl.read().unwrap(), comp);\n}\n\n#[test]\nfn try_write() {\n let lock = ShardedLock::new(0isize);\n let read_guard = lock.read().unwrap();\n\n let write_result = lock.try_write();\n match write_result {\n Err(TryLockError::WouldBlock) => (),\n Ok(_) => panic!(\"try_write should not succeed while read_guard is in scope\"),\n Err(_) => panic!(\"unexpected error\"),\n }\n\n drop(read_guard);\n}\n\n#[test]\nfn test_into_inner() {\n let m = ShardedLock::new(NonCopy(10));\n assert_eq!(m.into_inner().unwrap(), NonCopy(10));\n}\n\n#[test]\nfn test_into_inner_drop() {\n struct Foo(Arc);\n impl Drop for Foo {\n fn drop(&mut self) {\n self.0.fetch_add(1, Ordering::SeqCst);\n }\n }\n let num_drops = Arc::new(AtomicUsize::new(0));\n let m = ShardedLock::new(Foo(num_drops.clone()));\n assert_eq!(num_drops.load(Ordering::SeqCst), 0);\n {\n let _inner = m.into_inner().unwrap();\n assert_eq!(num_drops.load(Ordering::SeqCst), 0);\n }\n assert_eq!(num_drops.load(Ordering::SeqCst), 1);\n}\n\n#[test]\nfn test_into_inner_poison() {\n let m = Arc::new(ShardedLock::new(NonCopy(10)));\n let m2 = m.clone();\n let _: Result<(), _> = thread::spawn(move || {\n let _lock = m2.write().unwrap();\n panic!(\"test panic in inner thread to poison ShardedLock\");\n })\n .join();\n\n assert!(m.is_poisoned());\n match Arc::try_unwrap(m).unwrap().into_inner() {\n Err(e) => assert_eq!(e.into_inner(), NonCopy(10)),\n Ok(x) => panic!(\"into_inner of poisoned ShardedLock is Ok: {:?}\", x),\n }\n}\n\n#[test]\nfn test_get_mut() {\n let mut m = ShardedLock::new(NonCopy(10));\n *m.get_mut().unwrap() = NonCopy(20);\n assert_eq!(m.into_inner().unwrap(), NonCopy(20));\n}\n\n#[test]\nfn test_get_mut_poison() {\n let m = Arc::new(ShardedLock::new(NonCopy(10)));\n let m2 = m.clone();\n let _: Result<(), _> = thread::spawn(move || {\n let _lock = m2.write().unwrap();\n panic!(\"test panic in inner thread to poison ShardedLock\");\n })\n .join();\n\n assert!(m.is_poisoned());\n match Arc::try_unwrap(m).unwrap().get_mut() {\n Err(e) => assert_eq!(*e.into_inner(), NonCopy(10)),\n Ok(x) => panic!(\"get_mut of poisoned ShardedLock is Ok: {:?}\", x),\n }\n}\n" }, { "path": "crossbeam-utils/tests/thread.rs", "content": "use std::{\n any::Any,\n sync::atomic::{AtomicUsize, Ordering},\n thread::sleep,\n time::Duration,\n};\n\nuse crossbeam_utils::thread;\n\nconst THREADS: usize = 10;\nconst SMALL_STACK_SIZE: usize = 20;\n\n#[test]\nfn join() {\n let counter = AtomicUsize::new(0);\n thread::scope(|scope| {\n let handle = scope.spawn(|_| {\n counter.store(1, Ordering::Relaxed);\n });\n assert!(handle.join().is_ok());\n\n let panic_handle = scope.spawn(|_| {\n panic!(\"\\\"My honey is running out!\\\", said Pooh.\");\n });\n assert!(panic_handle.join().is_err());\n })\n .unwrap();\n\n // There should be sufficient synchronization.\n assert_eq!(1, counter.load(Ordering::Relaxed));\n}\n\n#[test]\nfn counter() {\n let counter = AtomicUsize::new(0);\n thread::scope(|scope| {\n for _ in 0..THREADS {\n scope.spawn(|_| {\n counter.fetch_add(1, Ordering::Relaxed);\n });\n }\n })\n .unwrap();\n\n assert_eq!(THREADS, counter.load(Ordering::Relaxed));\n}\n\n#[test]\nfn counter_builder() {\n let counter = AtomicUsize::new(0);\n thread::scope(|scope| {\n for i in 0..THREADS {\n scope\n .builder()\n .name(format!(\"child-{}\", i))\n .stack_size(SMALL_STACK_SIZE)\n .spawn(|_| {\n counter.fetch_add(1, Ordering::Relaxed);\n })\n .unwrap();\n }\n })\n .unwrap();\n\n assert_eq!(THREADS, counter.load(Ordering::Relaxed));\n}\n\n#[test]\nfn counter_panic() {\n let counter = AtomicUsize::new(0);\n let result = thread::scope(|scope| {\n scope.spawn(|_| {\n panic!(\"\\\"My honey is running out!\\\", said Pooh.\");\n });\n sleep(Duration::from_millis(100));\n\n for _ in 0..THREADS {\n scope.spawn(|_| {\n counter.fetch_add(1, Ordering::Relaxed);\n });\n }\n });\n\n assert_eq!(THREADS, counter.load(Ordering::Relaxed));\n assert!(result.is_err());\n}\n\n#[test]\nfn panic_twice() {\n let result = thread::scope(|scope| {\n scope.spawn(|_| {\n sleep(Duration::from_millis(500));\n panic!(\"thread #1\");\n });\n scope.spawn(|_| {\n panic!(\"thread #2\");\n });\n });\n\n let err = result.unwrap_err();\n let vec = err\n .downcast_ref::>>()\n .unwrap();\n assert_eq!(2, vec.len());\n\n let first = vec[0].downcast_ref::<&str>().unwrap();\n let second = vec[1].downcast_ref::<&str>().unwrap();\n assert_eq!(\"thread #1\", *first);\n assert_eq!(\"thread #2\", *second)\n}\n\n#[test]\nfn panic_many() {\n let result = thread::scope(|scope| {\n scope.spawn(|_| panic!(\"deliberate panic #1\"));\n scope.spawn(|_| panic!(\"deliberate panic #2\"));\n scope.spawn(|_| panic!(\"deliberate panic #3\"));\n });\n\n let err = result.unwrap_err();\n let vec = err\n .downcast_ref::>>()\n .unwrap();\n assert_eq!(3, vec.len());\n\n for panic in vec.iter() {\n let panic = panic.downcast_ref::<&str>().unwrap();\n assert!(\n *panic == \"deliberate panic #1\"\n || *panic == \"deliberate panic #2\"\n || *panic == \"deliberate panic #3\"\n );\n }\n}\n\n#[test]\nfn nesting() {\n let var = \"foo\".to_string();\n\n struct Wrapper<'a> {\n var: &'a String,\n }\n\n impl<'a> Wrapper<'a> {\n fn recurse(&'a self, scope: &thread::Scope<'a>, depth: usize) {\n assert_eq!(self.var, \"foo\");\n\n if depth > 0 {\n scope.spawn(move |scope| {\n self.recurse(scope, depth - 1);\n });\n }\n }\n }\n\n let wrapper = Wrapper { var: &var };\n\n thread::scope(|scope| {\n scope.spawn(|scope| {\n scope.spawn(|scope| {\n wrapper.recurse(scope, 5);\n });\n });\n })\n .unwrap();\n}\n\n#[test]\nfn join_nested() {\n thread::scope(|scope| {\n scope.spawn(|scope| {\n let handle = scope.spawn(|_| 7);\n\n sleep(Duration::from_millis(200));\n handle.join().unwrap();\n });\n\n sleep(Duration::from_millis(100));\n })\n .unwrap();\n}\n\n#[test]\nfn scope_returns_ok() {\n let result = thread::scope(|scope| scope.spawn(|_| 1234).join().unwrap()).unwrap();\n assert_eq!(result, 1234);\n}\n\n#[cfg(unix)]\n#[test]\nfn as_pthread_t() {\n use thread::unix::JoinHandleExt;\n thread::scope(|scope| {\n let handle = scope.spawn(|_scope| {\n sleep(Duration::from_millis(100));\n 42\n });\n let _pthread_t = handle.as_pthread_t();\n handle.join().unwrap();\n })\n .unwrap();\n}\n\n#[cfg(windows)]\n#[test]\nfn as_raw_handle() {\n use std::os::windows::io::AsRawHandle;\n thread::scope(|scope| {\n let handle = scope.spawn(|_scope| {\n sleep(Duration::from_millis(100));\n 42\n });\n let _raw_handle = handle.as_raw_handle();\n handle.join().unwrap();\n })\n .unwrap();\n}\n" }, { "path": "crossbeam-utils/tests/wait_group.rs", "content": "use std::{sync::mpsc, thread, time::Duration};\n\nuse crossbeam_utils::sync::WaitGroup;\n\nconst THREADS: usize = 10;\n\n#[test]\nfn wait() {\n let wg = WaitGroup::new();\n let (tx, rx) = mpsc::channel();\n\n for _ in 0..THREADS {\n let wg = wg.clone();\n let tx = tx.clone();\n\n thread::spawn(move || {\n wg.wait();\n tx.send(()).unwrap();\n });\n }\n\n thread::sleep(Duration::from_millis(100));\n\n // At this point, all spawned threads should be blocked, so we shouldn't get anything from the\n // channel.\n assert!(rx.try_recv().is_err());\n\n wg.wait();\n\n // Now, the wait group is cleared and we should receive messages.\n for _ in 0..THREADS {\n rx.recv().unwrap();\n }\n}\n\n#[test]\nfn wait_and_drop() {\n let wg = WaitGroup::new();\n let wg2 = WaitGroup::new();\n let (tx, rx) = mpsc::channel();\n\n for _ in 0..THREADS {\n let wg = wg.clone();\n let wg2 = wg2.clone();\n let tx = tx.clone();\n\n thread::spawn(move || {\n wg2.wait();\n tx.send(()).unwrap();\n drop(wg);\n });\n }\n\n // At this point, no thread has gotten past `wg2.wait()`, so we shouldn't get anything from the\n // channel.\n assert!(rx.try_recv().is_err());\n drop(wg2);\n\n wg.wait();\n\n // Now, the wait group is cleared and we should receive messages.\n for _ in 0..THREADS {\n rx.try_recv().unwrap();\n }\n}\n" }, { "path": "no_atomic.rs", "content": "// This file is @generated by no_atomic.sh.\n// It is not intended for manual editing.\n\nconst NO_ATOMIC: &[&str] = &[\n \"armv4t-none-eabi\",\n \"armv5te-none-eabi\",\n \"bpfeb-unknown-none\",\n \"bpfel-unknown-none\",\n \"mipsel-sony-psx\",\n \"msp430-none-elf\",\n \"thumbv4t-none-eabi\",\n \"thumbv5te-none-eabi\",\n];\n" }, { "path": "src/lib.rs", "content": "//! Tools for concurrent programming.\n//!\n//! ## Atomics\n//!\n//! * [`AtomicCell`], a thread-safe mutable memory location.\n//! * [`AtomicConsume`], for reading from primitive atomic types with \"consume\" ordering.\n//!\n//! ## Data structures\n//!\n//! * [`deque`], work-stealing deques for building task schedulers.\n//! * [`ArrayQueue`], a bounded MPMC queue that allocates a fixed-capacity buffer on construction.\n//! * [`SegQueue`], an unbounded MPMC queue that allocates small buffers, segments, on demand.\n//!\n//! ## Memory management\n//!\n//! * [`epoch`], an epoch-based garbage collector.\n//!\n//! ## Thread synchronization\n//!\n//! * [`channel`], multi-producer multi-consumer channels for message passing.\n//! * [`Parker`], a thread parking primitive.\n//! * [`ShardedLock`], a sharded reader-writer lock with fast concurrent reads.\n//! * [`WaitGroup`], for synchronizing the beginning or end of some computation.\n//!\n//! ## Utilities\n//!\n//! * [`Backoff`], for exponential backoff in spin loops.\n//! * [`CachePadded`], for padding and aligning a value to the length of a cache line.\n//! * [`scope`], for spawning threads that borrow local variables from the stack.\n//!\n//! [`AtomicCell`]: atomic::AtomicCell\n//! [`AtomicConsume`]: atomic::AtomicConsume\n//! [`ArrayQueue`]: queue::ArrayQueue\n//! [`SegQueue`]: queue::SegQueue\n//! [`Parker`]: sync::Parker\n//! [`ShardedLock`]: sync::ShardedLock\n//! [`WaitGroup`]: sync::WaitGroup\n//! [`Backoff`]: utils::Backoff\n//! [`CachePadded`]: utils::CachePadded\n\n#![no_std]\n#![doc(test(\n no_crate_inject,\n attr(allow(dead_code, unused_assignments, unused_variables))\n))]\n#![warn(\n missing_docs,\n unsafe_op_in_unsafe_fn,\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core\n)]\n\n#[cfg(feature = \"std\")]\nextern crate std;\n\npub use crossbeam_utils::atomic;\n\npub mod utils {\n //! Miscellaneous utilities.\n //!\n //! * [`Backoff`], for exponential backoff in spin loops.\n //! * [`CachePadded`], for padding and aligning a value to the length of a cache line.\n\n pub use crossbeam_utils::{Backoff, CachePadded};\n}\n\n#[cfg(feature = \"std\")]\n#[cfg(not(crossbeam_loom))]\npub use crossbeam_utils::thread::{self, scope};\n#[cfg(feature = \"std\")]\n#[doc(inline)]\npub use {\n crossbeam_channel as channel, crossbeam_channel::select, crossbeam_deque as deque,\n crossbeam_utils::sync,\n};\n#[cfg(feature = \"alloc\")]\n#[doc(inline)]\npub use {crossbeam_epoch as epoch, crossbeam_queue as queue};\n" }, { "path": "tests/subcrates.rs", "content": "//! Makes sure subcrates are properly re-exported.\n\nuse crossbeam::select;\n\n#[test]\nfn channel() {\n let (s, r) = crossbeam::channel::bounded(1);\n\n select! {\n send(s, 0) -> res => res.unwrap(),\n recv(r) -> res => assert!(res.is_ok()),\n }\n}\n\n#[test]\nfn deque() {\n let w = crossbeam::deque::Worker::new_fifo();\n w.push(1);\n let _ = w.pop();\n}\n\n#[test]\nfn epoch() {\n crossbeam::epoch::pin();\n}\n\n#[test]\nfn queue() {\n let a = crossbeam::queue::ArrayQueue::new(10);\n let _ = a.push(1);\n let _ = a.pop();\n}\n\n#[test]\nfn utils() {\n crossbeam::utils::CachePadded::new(7);\n\n crossbeam::scope(|scope| {\n scope.spawn(|_| ());\n })\n .unwrap();\n\n crossbeam::thread::scope(|scope| {\n scope.spawn(|_| ());\n })\n .unwrap();\n}\n" }, { "path": "tools/publish.sh", "content": "#!/bin/bash\nset -euo pipefail\nIFS=$'\\n\\t'\ncd \"$(dirname \"$0\")\"/..\n\n# Publish a new release.\n#\n# USAGE:\n# ./tools/publish.sh \n\nbail() {\n echo >&2 \"error: $*\"\n exit 1\n}\n\ncrate=\"${1:?}\"\nversion=\"${2:?}\"\nversion=\"${version#v}\"\ntag=\"${crate}-${version}\"\nif [[ ! \"${version}\" =~ ^[0-9]+\\.[0-9]+\\.[0-9]+(-[0-9A-Za-z\\.-]+)?(\\+[0-9A-Za-z\\.-]+)?$ ]]; then\n bail \"invalid version format '${version}'\"\nfi\nif [[ $# -gt 2 ]]; then\n bail \"invalid argument '$3'\"\nfi\n\n# Make sure there is no uncommitted change.\ngit diff --exit-code\ngit diff --exit-code --staged\n\n# Make sure the same release has not been created in the past.\nif gh release view \"${tag}\" &>/dev/null; then\n bail \"tag '${tag}' has already been created and pushed\"\nfi\n\nif ! git branch | grep -q '\\* master'; then\n bail \"current branch is not 'master'\"\nfi\n\ngit tag \"${tag}\"\n\n(\n if [[ \"${crate}\" != \"crossbeam\" ]]; then\n cd \"${crate}\"\n fi\n cargo +stable publish\n)\n\ngit push origin --tags\n" } ]