Repository: orxfun/orx-parallel Branch: main Commit: 5c2da2eb39ca Files: 341 Total size: 1.1 MB Directory structure: gitextract_lozyb1bf/ ├── .github/ │ ├── FUNDING.yml │ └── workflows/ │ └── ci.yml ├── .gitignore ├── .scripts/ │ ├── run_benchmark.sh │ └── run_benchmarks.sh ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── benches/ │ ├── chain3_collect_map.rs │ ├── chain4_collect_map.rs │ ├── chain_collect_map.rs │ ├── collect_filter.rs │ ├── collect_filtermap.rs │ ├── collect_flatmap.rs │ ├── collect_iter_into_par.rs │ ├── collect_long_chain.rs │ ├── collect_map.rs │ ├── collect_map_filter.rs │ ├── collect_map_filter_hash_set.rs │ ├── count_filtermap.rs │ ├── count_flatmap.rs │ ├── count_map.rs │ ├── count_map_filter.rs │ ├── drain_vec_collect_map_filter.rs │ ├── find.rs │ ├── find_any.rs │ ├── find_flatmap.rs │ ├── find_iter_into_par.rs │ ├── find_map_filter.rs │ ├── mut_for_each_iter.rs │ ├── mut_for_each_slice.rs │ ├── rec_iter_map_collect.rs │ ├── rec_iter_map_sum.rs │ ├── reduce.rs │ ├── reduce_iter_into_par.rs │ ├── reduce_long_chain.rs │ ├── reduce_map.rs │ ├── reduce_map_filter.rs │ ├── result_collect_map.rs │ ├── result_reduce_map.rs │ ├── results/ │ │ └── benchmark-results.xlsx │ ├── sum.rs │ ├── sum_filtermap.rs │ ├── sum_flatmap.rs │ ├── sum_map_filter.rs │ ├── t_par_merge_sorted.rs │ ├── t_seq_merge_sorted.rs │ ├── vec_deque_collect_map_filter.rs │ └── vec_deque_collect_map_filter_owned.rs ├── docs/ │ └── using.md ├── examples/ │ ├── benchmark_collect.rs │ ├── benchmark_find.rs │ ├── benchmark_find_any.rs │ ├── benchmark_heterogeneous.rs │ ├── benchmark_pools.rs │ ├── benchmark_reduce.rs │ ├── collection_of_results.rs │ ├── function_composition_with_mut_using.rs │ ├── map_while.rs │ ├── max_num_threads_config.rs │ ├── mutable_par_iter.rs │ ├── par_merge_sorted.rs │ ├── parallelization_on_tree/ │ │ ├── collection_on_entire_tree.rs │ │ ├── main.rs │ │ ├── node.rs │ │ ├── reduction_on_entire_tree.rs │ │ ├── reduction_on_subset_of_tree.rs │ │ ├── run_utils.rs │ │ └── tree.rs │ ├── using_for_each.rs │ ├── using_map.rs │ ├── using_metrics.rs │ ├── using_random_walk.rs │ └── utils/ │ ├── benchmark_utils.rs │ └── mod.rs ├── src/ │ ├── collect_into/ │ │ ├── collect.rs │ │ ├── fixed_vec.rs │ │ ├── mod.rs │ │ ├── par_collect_into.rs │ │ ├── split_vec.rs │ │ ├── utils.rs │ │ └── vec.rs │ ├── computational_variants/ │ │ ├── fallible_option.rs │ │ ├── fallible_result/ │ │ │ ├── map_result.rs │ │ │ ├── mod.rs │ │ │ ├── par_result.rs │ │ │ └── xap_result.rs │ │ ├── map.rs │ │ ├── mod.rs │ │ ├── par.rs │ │ ├── tests/ │ │ │ ├── copied.rs │ │ │ ├── count.rs │ │ │ ├── enumerate.rs │ │ │ ├── fallible_option.rs │ │ │ ├── fallible_result.rs │ │ │ ├── flatten.rs │ │ │ ├── for_each.rs │ │ │ ├── inspect.rs │ │ │ ├── iter_consuming.rs │ │ │ ├── iter_ref.rs │ │ │ ├── map/ │ │ │ │ ├── collect.rs │ │ │ │ ├── find.rs │ │ │ │ ├── mod.rs │ │ │ │ └── reduce.rs │ │ │ ├── min_max.rs │ │ │ ├── mod.rs │ │ │ ├── range.rs │ │ │ ├── slice.rs │ │ │ ├── sum.rs │ │ │ ├── vectors.rs │ │ │ └── xap/ │ │ │ ├── collect.rs │ │ │ ├── find.rs │ │ │ ├── mod.rs │ │ │ └── reduce.rs │ │ └── xap.rs │ ├── default_fns.rs │ ├── enumerate/ │ │ └── mod.rs │ ├── env.rs │ ├── executor/ │ │ ├── computation_kind.rs │ │ ├── executor_with_diagnostics/ │ │ │ ├── mod.rs │ │ │ ├── parallel_executor.rs │ │ │ ├── shared_state.rs │ │ │ └── thread_executor.rs │ │ ├── fixed_chunk_executor/ │ │ │ ├── chunk_size.rs │ │ │ ├── mod.rs │ │ │ ├── parallel_executor.rs │ │ │ └── thread_executor.rs │ │ ├── mod.rs │ │ ├── parallel_compute/ │ │ │ ├── collect_arbitrary.rs │ │ │ ├── collect_ordered.rs │ │ │ ├── mod.rs │ │ │ ├── next.rs │ │ │ ├── next_any.rs │ │ │ └── reduce.rs │ │ ├── parallel_executor.rs │ │ ├── thread_compute/ │ │ │ ├── collect_arbitrary.rs │ │ │ ├── collect_ordered.rs │ │ │ ├── mod.rs │ │ │ ├── next.rs │ │ │ ├── next_any.rs │ │ │ └── reduce.rs │ │ └── thread_executor.rs │ ├── experiment/ │ │ ├── algorithms/ │ │ │ ├── merge_sorted_slices/ │ │ │ │ ├── mod.rs │ │ │ │ ├── par.rs │ │ │ │ ├── seq.rs │ │ │ │ └── tests/ │ │ │ │ ├── inputs.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── par.rs │ │ │ │ └── seq.rs │ │ │ └── mod.rs │ │ ├── data_structures/ │ │ │ ├── mod.rs │ │ │ ├── slice.rs │ │ │ ├── slice_dst.rs │ │ │ ├── slice_iter_ptr.rs │ │ │ ├── slice_iter_ptr_dst.rs │ │ │ ├── slice_iter_ptr_src.rs │ │ │ ├── slice_src.rs │ │ │ └── tests/ │ │ │ ├── mod.rs │ │ │ └── slice.rs │ │ └── mod.rs │ ├── generic_iterator/ │ │ ├── collect.rs │ │ ├── early_exit.rs │ │ ├── iter.rs │ │ ├── mod.rs │ │ ├── reduce.rs │ │ └── transformations.rs │ ├── generic_values/ │ │ ├── fallible_iterators/ │ │ │ ├── mod.rs │ │ │ └── result_of_iter.rs │ │ ├── mod.rs │ │ ├── option.rs │ │ ├── option_result.rs │ │ ├── result.rs │ │ ├── runner_results/ │ │ │ ├── collect_arbitrary.rs │ │ │ ├── collect_ordered.rs │ │ │ ├── collect_sequential.rs │ │ │ ├── fallibility.rs │ │ │ ├── mod.rs │ │ │ ├── next.rs │ │ │ ├── reduce.rs │ │ │ └── stop.rs │ │ ├── transformable_values.rs │ │ ├── values.rs │ │ ├── vector.rs │ │ ├── vector_result.rs │ │ ├── whilst_atom.rs │ │ ├── whilst_atom_result.rs │ │ ├── whilst_iterators/ │ │ │ ├── mod.rs │ │ │ ├── whilst_atom_flat_map.rs │ │ │ └── whilst_option_flat_map.rs │ │ ├── whilst_option.rs │ │ ├── whilst_option_result.rs │ │ ├── whilst_vector.rs │ │ └── whilst_vector_result.rs │ ├── heap_sort.rs │ ├── into_par_iter.rs │ ├── iter/ │ │ ├── mod.rs │ │ ├── recursive/ │ │ │ ├── into_par_rec_iter.rs │ │ │ ├── mod.rs │ │ │ └── rec_par_iter.rs │ │ └── special_iterators.rs │ ├── iter_into_par_iter.rs │ ├── lib.rs │ ├── par_iter.rs │ ├── par_iter_option.rs │ ├── par_iter_result.rs │ ├── par_thread_pool.rs │ ├── parallel_drainable.rs │ ├── parallelizable.rs │ ├── parallelizable_collection.rs │ ├── parallelizable_collection_mut.rs │ ├── parameters/ │ │ ├── chunk_size.rs │ │ ├── iteration_order.rs │ │ ├── mod.rs │ │ ├── num_threads.rs │ │ └── params.rs │ ├── runner/ │ │ ├── computation_kind.rs │ │ ├── implementations/ │ │ │ ├── mod.rs │ │ │ ├── pond.rs │ │ │ ├── poolite.rs │ │ │ ├── rayon_core.rs │ │ │ ├── runner_with_pool.rs │ │ │ ├── scoped_pool.rs │ │ │ ├── scoped_threadpool.rs │ │ │ ├── sequential.rs │ │ │ ├── std_runner.rs │ │ │ ├── tests/ │ │ │ │ ├── mod.rs │ │ │ │ ├── pond.rs │ │ │ │ ├── poolite.rs │ │ │ │ ├── rayon_core.rs │ │ │ │ ├── scoped_pool.rs │ │ │ │ ├── scoped_threadpool.rs │ │ │ │ ├── sequential.rs │ │ │ │ ├── std.rs │ │ │ │ ├── utils.rs │ │ │ │ └── yastl.rs │ │ │ └── yastl.rs │ │ ├── mod.rs │ │ ├── num_spawned.rs │ │ └── parallel_runner.rs │ ├── special_type_sets/ │ │ ├── mod.rs │ │ └── sum.rs │ ├── test_utils.rs │ ├── using/ │ │ ├── collect_into/ │ │ │ ├── collect.rs │ │ │ ├── fixed_vec.rs │ │ │ ├── mod.rs │ │ │ ├── split_vec.rs │ │ │ ├── u_par_collect_into.rs │ │ │ └── vec.rs │ │ ├── computational_variants/ │ │ │ ├── mod.rs │ │ │ ├── tests/ │ │ │ │ ├── copied.rs │ │ │ │ ├── count.rs │ │ │ │ ├── fallible_option.rs │ │ │ │ ├── fallible_result.rs │ │ │ │ ├── flatten.rs │ │ │ │ ├── for_each.rs │ │ │ │ ├── inspect.rs │ │ │ │ ├── iter_consuming.rs │ │ │ │ ├── iter_ref.rs │ │ │ │ ├── map/ │ │ │ │ │ ├── collect.rs │ │ │ │ │ ├── find.rs │ │ │ │ │ ├── mod.rs │ │ │ │ │ └── reduce.rs │ │ │ │ ├── min_max.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── range.rs │ │ │ │ ├── slice.rs │ │ │ │ ├── sum.rs │ │ │ │ ├── utils.rs │ │ │ │ ├── vectors.rs │ │ │ │ └── xap/ │ │ │ │ ├── collect.rs │ │ │ │ ├── find.rs │ │ │ │ ├── mod.rs │ │ │ │ └── reduce.rs │ │ │ ├── u_fallible_option.rs │ │ │ ├── u_fallible_result/ │ │ │ │ ├── mod.rs │ │ │ │ ├── u_map_result.rs │ │ │ │ ├── u_par_result.rs │ │ │ │ └── u_xap_result.rs │ │ │ ├── u_map.rs │ │ │ ├── u_par.rs │ │ │ └── u_xap.rs │ │ ├── executor/ │ │ │ ├── mod.rs │ │ │ ├── parallel_compute/ │ │ │ │ ├── collect_arbitrary.rs │ │ │ │ ├── collect_ordered.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── next.rs │ │ │ │ ├── next_any.rs │ │ │ │ └── reduce.rs │ │ │ └── thread_compute/ │ │ │ ├── collect_arbitrary.rs │ │ │ ├── collect_ordered.rs │ │ │ ├── mod.rs │ │ │ ├── next.rs │ │ │ ├── next_any.rs │ │ │ └── reduce.rs │ │ ├── mod.rs │ │ ├── u_par_iter.rs │ │ ├── u_par_iter_option.rs │ │ ├── u_par_iter_result.rs │ │ └── using_variants.rs │ └── value_variants/ │ ├── whilst_iterators/ │ │ ├── whilst_atom_flat_map.rs │ │ └── whilst_option_flat_map.rs │ └── whilst_vector.rs └── tests/ ├── chain.rs ├── into_par.rs ├── iter_into_par.rs ├── map_while_ok_collect/ │ ├── from_map.rs │ ├── from_par.rs │ ├── from_xap_chain.rs │ ├── from_xap_filter.rs │ ├── from_xap_filter_map.rs │ ├── from_xap_flat_map.rs │ └── mod.rs ├── map_while_ok_collect_arbitrary/ │ ├── from_map.rs │ ├── from_par.rs │ ├── from_xap_chain.rs │ ├── from_xap_filter.rs │ ├── from_xap_filter_map.rs │ ├── from_xap_flat_map.rs │ ├── mod.rs │ └── utils.rs ├── map_while_ok_reduce/ │ ├── from_map.rs │ ├── from_par.rs │ ├── from_xap_chain.rs │ ├── from_xap_filter.rs │ ├── from_xap_filter_map.rs │ ├── from_xap_flat_map.rs │ └── mod.rs ├── mut_iter.rs ├── parallel_drainable.rs ├── parallelizable.rs ├── parallelizable_collection.rs ├── test_groups.rs ├── trait_bounds.rs ├── using/ │ ├── mod.rs │ └── rng.rs └── whilst/ ├── collect.rs ├── collect_arbitrary.rs ├── find.rs ├── mod.rs └── reduce.rs ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/FUNDING.yml ================================================ github: [orxfun] ================================================ FILE: .github/workflows/ci.yml ================================================ # Updated workflow (based on .github/workflows/ci.yml at ref d04ad7abce0c67776a7189001f706535e3e5c402) name: Rust on: push: branches: [ "main" ] pull_request: branches: [ "main" ] env: CARGO_TERM_COLOR: always jobs: build: runs-on: ubuntu-latest strategy: matrix: toolchain: ["stable"] features: ["", "--all-features", "--no-default-features"] steps: - uses: actions/checkout@v4 - name: Cache cargo registry & target uses: actions/cache@v4 with: path: | ~/.cargo/registry ~/.cargo/git target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-stable-${{ matrix.features || 'default' }} - name: Install toolchain uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.toolchain }} - name: Install 32bit target (for builds if needed) run: rustup target add i686-unknown-linux-musl - name: Install wasm target run: rustup target add wasm32-unknown-unknown - name: Install miri run: rustup component add --toolchain nightly-x86_64-unknown-linux-gnu miri - name: Install no-std-check run: cargo install cargo-no-std-check - name: Build run: cargo build --verbose ${{ matrix.features }} - name: Build-wasm run: cargo build --verbose --target wasm32-unknown-unknown ${{ matrix.features }} - name: Test run: cargo test --verbose ${{ matrix.features }} - name: Check-wasm run: cargo check --verbose --target wasm32-unknown-unknown ${{ matrix.features }} - name: Clippy run: cargo clippy ${{ matrix.features }} -- -D warnings --verbose - name: Miri run: cargo +nightly miri test --lib --bins --tests --verbose ${{ matrix.features }} test-32: name: Test (i686) runs-on: ubuntu-latest needs: build env: CARGO_TERM_COLOR: always steps: - uses: actions/checkout@v4 - name: Cache cargo registry & target uses: actions/cache@v4 with: path: | ~/.cargo/registry ~/.cargo/git target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-stable-i686 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master with: toolchain: stable - name: Set up QEMU for multi-arch Docker uses: docker/setup-qemu-action@v2 - name: Install Docker Buildx uses: docker/setup-buildx-action@v2 - name: Install cross run: cargo install --locked cross || true - name: Run tests for i686 target via cross run: | # Run unit tests under QEMU inside Docker so i686 test binaries are executed reliably. # Uses the glibc i686 target by default. Change to i686-unknown-linux-musl if you require musl. cross test --target i686-unknown-linux-gnu --verbose ================================================ FILE: .gitignore ================================================ # Generated by Cargo # will have compiled files and executables debug/ target/ .vscode/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk # MSVC Windows builds of rustc generate these, which store debugging information *.pdb benches/results/*.txt ================================================ FILE: .scripts/run_benchmark.sh ================================================ original_bench=find_iter_into_par bench=$1 sed -i "s/$original_bench/$bench/g" Cargo.toml rm -f benches/results/$bench.txt cargo bench --all-features >> benches/results/$bench.txt sed -i "s/$bench/$original_bench/g" Cargo.toml ================================================ FILE: .scripts/run_benchmarks.sh ================================================ allBenches=( chain_collect_map chain3_collect_map chain4_collect_map collect_filter collect_filtermap collect_flatmap collect_iter_into_par collect_long_chain collect_map_filter_hash_set collect_map_filter collect_map count_filtermap count_flatmap count_map_filter count_map drain_vec_collect_map_filter find_any find_flatmap find_iter_into_par find_map_filter find mut_for_each_iter mut_for_each_slice reduce_iter_into_par reduce_long_chain reduce_map_filter reduce_map reduce result_collect_map result_reduce_map sum_filtermap sum_flatmap sum_map_filter sum vec_deque_collect_map_filter vec_deque_collect_map_filter_owned ) counter = 0 for t in ${allBenches[@]}; do counter=$((counter+1)) echo -e "\n\n" echo ------------- $counter : $t ---------------------------------- ./.scripts/run_benchmark.sh $t done ================================================ FILE: Cargo.toml ================================================ [package] name = "orx-parallel" version = "3.4.0" edition = "2024" authors = ["orxfun "] readme = "README.md" description = "High performance, configurable and expressive parallel computation library." license = "MIT OR Apache-2.0" repository = "https://github.com/orxfun/orx-parallel/" keywords = ["parallel", "concurrency", "performance", "thread", "iterator"] categories = ["concurrency", "algorithms"] [dependencies] orx-pinned-vec = { version = "3.21.0", default-features = false } orx-fixed-vec = { version = "3.22.0", default-features = false } orx-split-vec = { version = "3.22.0", default-features = false } orx-concurrent-iter = { version = "3.3.0", default-features = false } orx-concurrent-bag = { version = "3.4.0", default-features = false } orx-concurrent-ordered-bag = { version = "3.4.0", default-features = false } orx-pinned-concurrent-col = { version = "2.18.0", default-features = false } orx-iterable = { version = "1.3.0", default-features = false } orx-priority-queue = { version = "1.7.0", default-features = false } orx-pseudo-default = { version = "2.1.0", default-features = false } orx-concurrent-recursive-iter = { version = "2.0.0", default-features = false } # optional: generic iterator rayon = { version = "1.11.0", optional = true, default-features = false } # optional: thread pool pond = { version = "0.3.1", optional = true, default-features = false } poolite = { version = "0.7.1", optional = true, default-features = false } rayon-core = { version = "1.13.0", optional = true, default-features = false } scoped-pool = { version = "1.0.0", optional = true, default-features = false } scoped_threadpool = { version = "0.1.9", optional = true, default-features = false } yastl = { version = "0.1.2", optional = true, default-features = false } [dev-dependencies] chrono = "0.4.42" clap = { version = "4.5.50", features = ["derive"] } criterion = { version = "0.8", features = ["html_reports"] } orx-concurrent-option = { version = "1.5.0", default-features = false } orx-concurrent-vec = "3.10.0" rand = "0.9.2" rand_chacha = "0.9" rayon = "1.11.0" test-case = "3.3.1" orx-criterion = { git = "https://github.com/orxfun/orx-criterion" } [[bench]] name = "find_iter_into_par" harness = false [package.metadata.docs.rs] all-features = true [features] default = ["std"] std = [] generic_iterator = ["rayon"] experiment = [] ================================================ FILE: LICENSE-APACHE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS ================================================ FILE: LICENSE-MIT ================================================ MIT License Copyright (c) 2024 Ugur Arikan Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # orx-parallel [![orx-parallel crate](https://img.shields.io/crates/v/orx-parallel.svg)](https://crates.io/crates/orx-parallel) [![orx-parallel crate](https://img.shields.io/crates/d/orx-parallel.svg)](https://crates.io/crates/orx-parallel) [![orx-parallel documentation](https://docs.rs/orx-parallel/badge.svg)](https://docs.rs/orx-parallel) [High performance](#performance-and-benchmarks), [configurable](#configurable) and [expressive](#parallel-computation-by-iterators) parallel computation library. * [Parallel Computation by Iterators](#parallel-computation-by-iterators) * [Parallelizable Collections](#parallelizable-collections) * [Parallelization over Nonlinear Data Structures](#parallelization-over-nonlinear-data-structures) * [Performance and Benchmarks](#performance-and-benchmarks) * [Fallible Parallel Iterators](#fallible-parallel-iterators) * [Using Mutable Variables](#using-mutable-variables) * [Configurations](#configurations) * [Runner: Pools and Executors](#runner-pools-and-executors) * [Contributing](#contributing) ## Parallel Computation by Iterators Parallel computation is defined using the parallel iterator trait [`ParIter`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIter.html). The goal is to convert an expressive sequential program into an efficient parallel program only by replacing `iter` with `par`; and `into_iter` with `into_par`. The following is a naive [traveling salesperson](https://en.wikipedia.org/wiki/Travelling_salesman_problem) algorithm which randomly generates sequences and picks the one with the minimum duration as the best tour. The example demonstrates chaining of very common and useful `map`, `filter` and `reduce` (`min_by_key`) operations. Notice that the only difference between the sequential and parallel programs is the `par()` call. ```rust use orx_parallel::*; use rand::prelude::*; struct Tour(Vec); impl Tour { fn random(n: usize) -> Self { let mut cities: Vec<_> = (0..n).collect(); cities.shuffle(&mut rand::rng()); Self(cities) } fn not_in_standard_order(&self) -> bool { self.0.iter().enumerate().any(|(i, c)| i != *c) } fn duration(&self) -> usize { let mut total = 0; let links = self.0.iter().zip(self.0.iter().skip(1)); for (a, b) in links { total += (*a as i64 - *b as i64).abs() as usize; } total } } let num_tours = 1_000_000; let num_cities = 10; // sequential let best_tour = (0..num_tours) .map(|_| Tour::random(num_cities)) .filter(|t| t.not_in_standard_order()) .min_by_key(|t| t.duration()) .unwrap(); // parallel let best_tour = (0..num_tours) .par() // parallelized !! .map(|_| Tour::random(num_cities)) .filter(|t| t.not_in_standard_order()) .min_by_key(|t| t.duration()) .unwrap(); ``` ## Parallelizable Collections Inputs that can be used in parallel computations can be categorized in three groups: * i. directly parallelizable collections * ii. parallelization of any iterator * iii. parallelization of any collection ### i. Directly Parallelizable Collections These are collections which are parallelized by utilizing their specific structure to achieve high performance. This crate provides direct implementations of std collections; the table below lists the most recent table of direct implementations. | Type | Over References
`&T` | Over Mut References
`&mut T>` | Over Owned Values
` T` | |:--|:-:|:-:|:-:| | `v: Vec` | `v.par()` | `v.par_mut()` | `v.into_par()` | | `v: VecDeque` | `v.par()` | | `v.into_par()` | | `s: &[T]` | `s.par()`
`s.into_par()` | | | | `s: &mut [T]` | | `s.into_par()` | | | `r: Range`| | | `r.par()`
`r.into_par()` | Implementations of custom collections belong to their respective crates as they most likely require access to internals. Currently, the following collections are known to allow parallel computation using this crate: │ [SplitVec](https://crates.io/crates/orx-split-vec) │ [FixedVec](https://crates.io/crates/orx-fixed-vec) │ [LinkedList](https://crates.io/crates/orx-linked-list) │ [Tree](https://crates.io/crates/orx-tree) │ [ImpVec](https://crates.io/crates/orx-imp-vec) │ Since these implementations are particularly optimized for the collection type, it is preferable to start defining parallel computation from the collection whenever available. In other words, for a direclty parallelizable collection `col`, * `col.par().map(_).filter(_).reduce(_)` is a better approach than * `col.iter().iter_into_par().map(_).filter(_).reduce(_)`, which will be explained in the next subsection. > **extensibility**: Note that any input collection or generator that implements [`IntoConcurrentIter`](https://docs.rs/orx-concurrent-iter/latest/orx_concurrent_iter/trait.IntoConcurrentIter.html) automatically implements [`IntoParIter`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.IntoParIter.html). Therefore, a new collection can be parallelized provided that its concurrent iterator is implemented. In addition, there exist the following special parallel iterators that can be directly created from the collection. | Type | Method | Definition | |---|---|---| | `v: Vec` | [`v.par_drain(range)`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParallelDrainableOverSlice.html) | Parallel counterpart of `v.drain(range)` | ### ii. Parallelization of Any Iterator Any arbitrary sequential [Iterator](https://doc.rust-lang.org/std/iter/trait.Iterator.html) implements [`IterIntoParIter`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.IterIntoParIter.html) trait and can be converted into a parallel iterator using the `iter_into_par` method. As demonstrated below, item type of the Iterator can as well be a mutable reference. ```rust use orx_parallel::*; use std::collections::HashMap; let mut map: HashMap<_, _> = (0..1024).map(|x| (x.to_string(), x)).collect(); let par = map.values_mut().iter_into_par(); // mutable parallel iterator from Iterator par.filter(|x| **x != 42).for_each(|x| *x *= 0); assert_eq!(map.values().iter_into_par().sum(), 42); // parallel iterator from Iterator ``` This is powerful since it allows to parallelize all iterables, including pretty much every collection and more. On the other hand, due to being a generic implementation without collection specific optimizations, parallelized computation might underperform its sequential counterpart if the work to be done on each input element is insignificant. For instance, `i` being an arbitrary iterator of numbers, `i.sum()` will most likely be faster than `i.iter_into_par().sum()`. This being said, `ParIter` takes advantage of certain optimizations, such as buffering and chunk size optimization, in order to improve performance. Therefore, whenever the computation on the iterator elements is more involved than just returning them or adding numbers, we can benefit from parallelization. The respective section of [benchmarks](#parallelization-of-arbitrary-iterators) present significant improvements achieved consistently. ### iii. Parallelization of Any Collection Lastly, consider a collection which does not provide a direct concurrent iterator implementation. This might be our custom collection, say `MyCollection`; or an external collection without a concurrent iterator implementation, such as the `HashSet`. There are two methods to parallelize computations over such collections: * (ii) parallelize using the collection's iterator, or * (i) collect the elements in a vector and then parallelize work over the vector. The following table demonstrates these methods for the `HashSet`; however, they are applicable to any collection with `iter` and `into_iter` methods. | Type | Method | Over References
`&T` | Over Owned Values
`T` | |:--|:-:|---|---| | `h: HashSet` | ii | `h.iter()`
  `.iter_into_par()` | `h.into_iter()`
  `.iter_into_par()` | | | i | `h.iter()`
  `.collect::>()`
  `.par()` | `h.into_iter()`
  `.collect::>()`
  `.into_par()` | Note that each approach can be more efficient in different scenarios. For large elements, (ii) might be preferred to avoid allocation of the vector. For insignificant tasks to be performed on each element, (i) might be preferred to take full benefit of vector-specific optimizations. ## Parallelization over Nonlinear Data Structures [IntoParIterRec](https://docs.rs/orx-parallel/latest/orx_parallel/trait.IntoParIterRec.html) trait can be used to create a **parallel recursive iterator** over an initial set of elements which is useful when working with non-linear data structures such as **trees** and **graphs**. Consider, for instance, a tree which is defined by the following node struct: ```rust ignore pub struct Node { pub data: T, pub children: Vec>, } ``` Assume that we want to map all the data with `map: impl Fn(T) -> u64` and compute the sum of mapped values of all nodes descending from a `root: &Node`. We can express this computation and execute in parallel with the following: ```rust ignore fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } [root].into_par_rec(extend).map(map).sum() ``` Instead of `into_par`, we use `into_par_rec` and provide `extend` function as its argument. This function defines the recursive extension of the parallel iterator such that every time we process a `node` we first add its children to the `queue`. [`Queue`](https://docs.rs/orx-concurrent-recursive-iter/latest/orx_concurrent_recursive_iter/struct.Queue.html) is the queue of elements to be processed and it exposes two growth methods to define the recursive extension: `push` and `extend`. Although we create the parallel iterator differently, we get a `ParIter`. Therefore, we have access to all features of a regular parallel iterator. For instance, assume we want to filter nodes first. Further, instead of summing up the mapped values, we need to collect them in a vector. We can express this computation just as we would do on a linear data structure: ```rust ignore [root].into_par_rec(extend).filter(filter).map(map).collect() ``` For more details, you may see the [parallelization_on_tree](https://github.com/orxfun/orx-parallel/blob/main/examples/parallelization_on_tree) example. ## Performance and Benchmarks *Please also see [impact of ChunkSize on performance](#impact-of-chunksize-on-performance) section.* You may find some sample parallel programs in [examples](https://github.com/orxfun/orx-parallel/blob/main/examples) directory. These examples allow to express parallel computations as iterator method compositions and run quick experiments with different approaches. Examples use `GenericIterator`. As the name suggests, it is a generalization of sequential iterator, rayon's parallel iterator and orx-parallel's parallel iterator, and hence, allows for convenient experiments. You may play with the code, update the tested computations and run these examples by including **generic_iterator** feature, such as: `cargo run --release --features generic_iterator --example benchmark_collect -- --len 123456 --num-repetitions 10` Actual benchmark files are located in [benches](https://github.com/orxfun/orx-parallel/blob/main/benches) directory. Tables below report average execution times in microseconds. The numbers in parentheses represent the ratio of execution time to that of sequential computation which is used as the baseline (1.00). Parallelized executions of all benchmarks are carried out with default settings. Computations are separated into three categories with respect to how the iterator is consumed: collect, reduce and early-exit. Further, two additional categories are created to test parallelization of arbitrary iterators ([ii](#ii-parallelization-of-any-iterator)) and flexibility in composition of computations. ### Collect In this group of benchmarks, outputs of parallel computations are collected into vectors. Details of the iterator chains and tested functions can be found in the respective benchmark files (you may use the link in the **file** column). > **(s)** Outputs can also be collected into a [`SplitVec`](https://crates.io/crates/orx-split-vec), which can provide further improvements by avoiding memory copies. Note that a split vector provides constant time random access; and despite the fact that it is split to fragments, it asymptotically inherits advantages of contiguous vectors. |file|computation|sequential|rayon|orx-parallel|orx-parallel (s)| |---|---|---:|---:|---:|---:| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/collect_filter.rs)|`.filter(_).collect()`|2.74 (1.00)|12.14 (4.43)|**1.80 (0.66)**|1.87 (0.68)| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/collect_filtermap.rs)|`.filter_map(_).collect()`|6.96 (1.00)|13.28 (1.91)|3.51 (0.50)|**3.35 (0.48)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/collect_flatmap.rs)|`.flat_map(_).collect()`|77.93 (1.00)|239.83 (3.08)|31.73 (0.41)|**23.79 (0.31)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/collect_map_filter.rs)|`.map(_).filter(_).collect()`|19.24 (1.00)|9.99 (0.52)|6.21 (0.32)|**5.98 (0.31)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/collect_map.rs)|`.map(_).collect()`|18.08 (1.00)|7.98 (0.44)|**5.28 (0.29)**|6.09 (0.34)| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/drain_vec_collect_map_filter.rs)|`.map(_).filter(_).collect()` [☆][draining_iterator]|19.41 (1.00)|7.54 (0.39)|5.90 (0.30)|**5.77 (0.30)**| [draining_iterator]: ## "parallel draining iterator" ### Reduce In this group, instead of collecting outputs, the results are reduced to a single value. Some common reductions are `sum`, `count`, `min`, etc. |file|computation|sequential|rayon|orx-parallel| |---|---|---:|---:|---:| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/reduce_map_filter.rs)|`.map(_).filter(_).reduce(_)`|14.15 (1.00)|7.55 (0.53)|**3.86 (0.27)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/reduce_map.rs)|`.map(_).reduce(_)`|13.81 (1.00)|6.25 (0.45)|**4.15 (0.30)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/reduce.rs)|`.reduce(_)`|0.97 (1.00)|10.58 (10.91)|**0.90 (0.93)**| ### Find Here, computations that allow for *early exit* or *short-circuit* are investigated. As an example, experiments on `find` method are presented; methods such as `find_any`, `any` or `all` lead to similar results. |file|computation|sequential|rayon|orx-parallel| |---|---|---:|---:|---:| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/)|`.flat_map(_).find(_)`|160.24 (1.00)|127.37 (0.79)|**27.66 (0.17)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/)|`.map(_).filter(_).find(_)`|43.01 (1.00)|11.14 (0.26)|**8.61 (0.20)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/)|`.find(_)`|2.94 (1.00)|12.85 (4.37)|**1.54 (0.52)**| ### Parallelization of Arbitrary Iterators As discussed in [ii](#ii-parallelization-of-any-iterator), parallelization of regular iterators is a powerful feature. The benchmarks in this category demonstrate that improvements can be achieved provided that the computation on elements is not insignificant. |file|computation|sequential|rayon|orx-parallel| |---|---|---:|---:|---:| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/collect_long_chain.rs)|`…long_chain.collect()`|19.72 (1.00)|32.54 (1.65)|**6.12 (0.31)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/reduce_iter_into_par.rs)|`.map(_).filter(_).reduce(_)`|15.17 (1.00)|118.28 (7.80)|**4.98 (0.33)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/)|`.map(_).filter(_).find(_)`|42.58 (1.00)|63.60 (1.49)|**7.98 (0.19)**| ### Parallel Mutable Iterators In this group, we investigate the performance of parallel computation which mutates the input elements. In the benchmarks, we filter elements and update the ones which satisfy the given criterion within the `for_each` call. |file|computation|sequential|rayon|orx-parallel| |---|---|---:|---:|---:| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/mut_for_each_slice.rs)|`slice.par_mut().filter(_).for_each(_)`|62.61 (1.00)|14.08 (0.22)|**8.45 (0.13)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/mut_for_each_iter.rs)|`iter.iter_into_par().filter(_).for_each(_)`|77.63 (1.00)|78.69 (1.01)|**10.03 (0.13)**| ### Composition In the final category of benchmarks, impact of long chains of transformations on computation time is tested. You may see such example long chains in the benchmark computations below, where `long_chain` is a shorthand for `.map(map1).filter(filter1).map(map2).filter(filter2).map(map3).map(map4).filter(filter4)`. Notice that the caller can actually shorten the chains by composing some of them. An obvious one is the `.map(map3).map(map4)` call which could have been one call like `map(map3-then-map4)`. However, this is not always possible as the computation might be conditionally built up in stages. Further, breaking transformations into smaller pieces help in achieving more descriptive computation definitions. The results suggest that the functions are efficiently composed by the parallel iterator. |file|computation|sequential|rayon|orx-parallel| |---|---|---:|---:|---:| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/collect_long_chain.rs)|`…long_chain.collect()`|14.27 (1.00)|6.33 (0.44)|**3.80 (0.27)**| |[⇨](https://github.com/orxfun/orx-parallel/blob/main/benches/reduce_long_chain.rs)|`…long_chain.reduce(_)`|15.08 (1.00)|6.10 (0.40)|**4.03 (0.27)**| ## Fallible Parallel Iterators We enjoy rust's [`?`](https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator) operator when working with fallible computations. It allows us to focus on and code only the success path. Failure at any step of the computation leads to a short-circuit and immediately returns from the function. ```rust fn try_to_parse() -> Result { let x: i32 = "123".parse()?; // x = 123 let y: i32 = "24a".parse()?; // returns an Err() immediately Ok(x + y) // Doesn't run. } ``` However, we do not have this convenience while working with iterators. `collect` is the only exception. Normally, it allows us to pick the container to collect the items into. ```rust let into_vec: Vec = (0..10).collect(); let into_set: std::collections::HashSet = (0..10).collect(); ``` But it also does something exceptional when the item type is a result: * The first computation below is similar to above, it simply collects each element to the container which is defined as a vector. * The second computation; however, is fundamentally different. It collects elements iff all elements are of the Ok variant. Further, it short-circuits the computation as soon as an Err is observed. This is exactly how the `?` operator behaves. ```rust let into_vec_of_results: Vec> = (0..10).map(|x| Ok(x)).collect(); let into_result_of_vec: Result, char> = (0..10).map(|x| Ok(x)).collect(); ``` Although convenient, change in the behavior of the collect computation might be considered *unexpected*, at least for me. Further, we do have not short-circuiting methods for computations other than collect. For instance, it is not as convenient to compute the sum of numbers of an iterator provided that all elements are of the Ok variant, and receive the error otherwise. In general, the requirement to early exit in fallible computation is common and important both for performance and convenience reasons. For parallel computation, this crate proposes to explicitly transform an iterator with fallible elements into a fallible parallel iterator. ```rust use orx_parallel::*; use std::num::ParseIntError; let collect: Result, ParseIntError> = vec!["7", "2", "34"] .into_par() .map(|x| x.parse::()) .into_fallible_result() // <-- explicit transformation to fallible iterator .collect(); ``` Currently, there exist two fallible parallel iterators [`ParIterResult`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIterResult.html) and [`ParIterOption`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIterOption.html). The transformation is as follows: | Regular Iterator | Transformation Method| Fallible Iterator | | --- | --- | --- | | `ParIter>` | `into_fallible_result()` | `ParIterResult` | | `ParIter>` | `into_fallible_option()` | `ParIterOption` | After converting into a fallible iterator, each chaining transformation is based on the success item type. Similar to `?` operator, this allows us to focus on the success path while any error case will be handled by early returning from the iterator with the error. ```rust use orx_parallel::*; use std::num::ParseIntError; let sum: Result = vec!["7", "2", "34"] .into_par() .map(|x| x.parse::()) // Item = Result .into_fallible_result() // we are only working with success type after this point .map(|x| x + 1) .filter(|x| x % 2 == 0) .flat_map(|x| [x, x + 1, x + 2]) .sum(); // returns Result, rather than i32 assert_eq!(sum, Ok(27)); let sum: Result = vec!["7", "!!!", "34"] .into_par() .map(|x| x.parse::()) .into_fallible_result() .map(|x| x + 1) .filter(|x| x % 2 == 0) .flat_map(|x| [x, x + 1, x + 2]) .sum(); assert!(sum.is_err()); ``` As demonstrated above, not only `collect` but all computation methods return a `Result`. To summarize: * We can use all iterator methods with fallible iterators as well. * The transformations are based on the success type. All computations return a `Result`: * if all computations succeed, it is `Ok` of the value that an infallible iterator would return; * it is the first discovered `Err` if any of the computations fails. * Finally, all computations immediately return in case of an error. Optional fallible iterator behaves exactly the same, except that `None` is treated as the failure case. ## Using Mutable Variables Iterator methods allow us to define expressive computations using closures. These closures are often `FnMut` for sequential iterators allowing to mutably capture variables from the scope. It is clear that this is not possible for parallel iterators as it would lead to race condition when multiple threads simultaneously try to access the captured mutable variable. Therefore, parallel counterpart of the iterator methods often accept closures implementing `Fn`. However, it is necessary to have mutable variables for certain programs. A very common example is computations requiring random number generators which are stateful and can create random numbers only with a mutable reference. **using** transformation aims to provide a general and safe solution to this problem as follows: * One mutable variable per thread; hence, no race conditions. * The mutable variable is explicitly and mutably available to all iterator methods. The following two examples demonstrate the idea and usage: * [`using`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIter.html#tymethod.using) takes a closure with thread index as the argument, describing how the mutable variable should be created for each thread. * [`using_clone`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIter.html#tymethod.using_clone), on the other hand, takes the value to be used as the mutable variable and shares a clone of it with each thread (just a shorthand for `using(|_| sender.clone())`). In either case, there will exactly be `n` mutable variables created provided that the parallel computation uses `n` threads. ```rust ignore input .into_par() .using(|t_idx| ChaCha20Rng::seed_from_u64(42 * t_idx as u64)) // <-- explicit using .map(|_, i| fibonacci((i % 50) + 1) % 10) // rng: &mut ChaCha20Rng .filter(|rng, _: &u64| rng.random_bool(0.4)) // is accessible for .map(|rng, i: u64| rng.random_range(0..i)) // all iterator methods .sum() let (sender, receiver) = channel(); ``` ```rust ignore let (sender, receiver) = channel(); (0..5) .into_par() .using_clone(sender) .for_each(|s, x| s.send(x).unwrap()); let mut res: Vec<_> = receiver.iter().collect(); ``` Further details can be found in [using.md](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). ## Configurations ### Configuration per Computation Each parallel computation is governed by two main straightforward parameters. [`NumThreads`](https://docs.rs/orx-parallel/latest/orx_parallel/enum.NumThreads.html) is the degree of parallelization. This is a *capacity parameter* used to limit the resources that can be used by the computation. * `Auto`: All available threads can be used, but not necessarily. * `Max(n)`: The computation can spawn at most n threads. * `Max(1)`: Falls back to sequential execution on the main thread. [`ChunkSize`](https://docs.rs/orx-parallel/latest/orx_parallel/enum.ChunkSize.html) represents the number of elements a parallel worker will pull and process every time it becomes idle. This is an *optimization parameter* that can be tuned to balance the overhead of parallelization and cost of heterogeneity of tasks. * `Auto`: Let the parallel executor dynamically decide, achieves high performance in general and can be used unless we have useful computation specific knowledge. * `Exact(c)`: Chunks will have c elements; gives complete control to the caller. Useful when we have a very good knowledge or want to tune the computation for certain data. * `Min(c)`: Every chunk will have at least c elements. Parallel executor; however, might decide to pull more if each computation is handled very fast. See also the last parameter [`IterationOrder`](https://docs.rs/orx-parallel/latest/orx_parallel/enum.IterationOrder.html) with variants `Ordered` (default) and `Arbitrary` which is another useful optimization parameter for specific use cases. When omitted, `NumThreads::Auto` and `ChunkSize::Auto` will be used. Configuring parallel computation is **straightforward** and **specific to computation** rather than through a global setting. ```rust use orx_parallel::*; use std::num::NonZeroUsize; let n = 1024; _ = (0..n).par().sum(); // NumThreads::Auto & ChunkSize::Auto _ = (0..n).par().num_threads(4).sum(); // <= 4 threads _ = (0..n).par().num_threads(1).sum(); // sequential _ = (0..n).par().num_threads(0).sum(); // shorthand for NumThreads::Auto _ = (0..n).par().chunk_size(64).sum(); // chunks of exactly 64 elements let c = ChunkSize::Min(NonZeroUsize::new(16).unwrap()); _ = (0..n).par().chunk_size(c).sum(); // chunks of at least 16 elements _ = (0..n).par().num_threads(4).chunk_size(16).sum(); // set both params ``` Note that `NumThreads::Max(1)` executes the computation sequentially. This gives the consumer, who actually executes the defined computation, complete control to: * execute in parallel with the given configuration, or * execute sequentially, or * execute in parallel with any number of threads that it decides. This is guaranteed by the fact that both consuming computation calls and configuration methods require ownership (`self`) of the iterator. ### Global Configuration Additionally, maximum number of threads that can be used by parallel computations can be globally bounded by the environment variable `ORX_PARALLEL_MAX_NUM_THREADS`. Please see the corresponding [example](https://github.com/orxfun/orx-parallel/blob/main/examples/max_num_threads_config.rs) for details. ### Impact of `ChunkSize` on Performance The impact of the chunk size on performance might be significant. Our objective is to minimize the sum of two computational costs: * parallelization overhead => it gets smaller as chunk size gets greater * cost of heterogeneity => it gets larger as chunk size gets greater Parallelization overhead can further be divided into two: * concurrent state update: This often corresponds to one atomic update per chunk. It may be significant if our computation is very small such as `input.par().sum()`. Otherwise, cost of atomic update could be negligible. * false sharing: This is relevant only if we are writing results. For instance, when we are one-to-one mapping an input and collecting the results such as `input.par().map(|x| x.to_string()).collect()`, or if are writing with mut references such as `input.par().for_each(|x| *x += 1)`. Here, the performance might suffer from false sharing when the `chunk size × size of output item` is not large enough. You may also see [false sharing](https://docs.rs/orx-concurrent-bag/latest/orx_concurrent_bag/#false-sharing) section for `ConcurrentBag`. In either case, when computation on each item is sufficiently long, parallelization overhead is negligible. Here, we want to make sure that we do not have heterogeneity cost. Therefore, a safe chunk size choice would be one, `par.chunk_size(1)`. Otherwise, our choice depends on the use case. As a rule of thumb, we want a chunk size that is **just large enough** to mitigate the parallelization overhead but not larger so that we do not suffer from heterogeneity. The default configuration `par.chunk_size(ChunkSize::Auto)` or `par.chunk_size(0)` uses a heuristic to solve this tradeoff. A difficult case for the current version is when the tasks are significantly heterogeneous (see the [discussion](https://github.com/orxfun/orx-parallel/discussions/26) for future improvements). As described above, the **best way to deal with heterogeneity** is to have `par.chunk_size(1)`. You may of course test larger chunk sizes to optimize the computation for your data. ## Runner: Pools and Executors This crate defines parallel computation by combining two basic components. **Pulling inputs** * Pulling inputs in parallel is achieved through [`ConcurrentIter`](https://crates.io/crates/orx-concurrent-iter). Concurrent iterator implementations are lock-free, efficient and support pull-by-chunks optimization to reduce the parallelization overhead. A thread can pull any number of inputs from the concurrent iterator every time it becomes idle. This provides the means to dynamically decide on the chunk sizes. * Furthermore, this allows to reduce the overhead of defining creating tasks. To illustrate, provided that the computation will be handled by `n` threads, a closure holding a reference to the input concurrent iterator is defined to represent the computation. This same closure is passed to `n` threads; i.e., `n` spawn calls are made. Each of these threads keep pulling elements from the input until the computation is completed, without requiring to define another task. **Writing outputs** * When we collect results, writing outputs is handled using lock-free containers such as [`ConcurrentBag`](https://crates.io/crates/orx-concurrent-bag) and [`ConcurrentOrderedBag`](https://crates.io/crates/orx-concurrent-ordered-bag) which aim for high performance collection of results. There are two main decisions to be taken while executing these components: * how many threads do we use? * what is the chunk size; i.e., how many input items does a thread pull each time? A [`ParallelRunner`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParallelRunner) is a combination of a `ParThreadPool` and a `ParallelExecutor` that are responsible for these decisions, respectively. ### ParThreadPool: number of threads [`ParThreadPool`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParThreadPool) trait generalizes thread pools that can be used for parallel computations. This allows the parallel computation to be generic over thread pools. When not explicitly set, [`DefaultPool`](https://docs.rs/orx-parallel/latest/orx_parallel/type.DefaultPool) is used: * When **std** feature is enabled, default pool is the [`StdDefaultPool`](https://docs.rs/orx-parallel/latest/orx_parallel/struct.StdDefaultPool). In other words, all available native threads can be used by the parallel computation. This number can globally bounded by "ORX_PARALLEL_MAX_NUM_THREADS" environment variable when set. * When working in a **no-std** environment, default pool is the [`SequentialPool`](https://docs.rs/orx-parallel/latest/orx_parallel/struct.SequentialPool). As the name suggests, this pool executes the parallel computation sequentially on the main thread. It can be considered as a placeholder to be overwritten by `with_pool` or `with_runner` methods to achieve parallelism. *Note that thread pool defines the resource, or upper bound. This upper bound can further be bounded by the [`num_threads`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIter.html#tymethod.num_threads) configuration. Finally, parallel executor might choose not to use all available threads if it decides that the computation is small enough.* To overwrite the defaults and explicitly set the thread pool to be used for the computation, [`with_pool`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIter.html#tymethod.with_pool) or [`with_runner`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIter.html#tymethod.with_runner) methods are used. ```rust use orx_parallel::*; let inputs: Vec<_> = (0..42).collect(); // uses the DefaultPool // assuming "std" enabled, StdDefaultPool will be used; i.e., native threads let sum = inputs.par().sum(); // equivalent to: #[cfg(feature = "std")] { let sum2 = inputs.par().with_pool(StdDefaultPool::default()).sum(); assert_eq!(sum, sum2); } #[cfg(not(miri))] #[cfg(feature = "scoped_threadpool")] { let mut pool = scoped_threadpool::Pool::new(8); // uses the scoped_threadpool::Pool created with 8 threads let sum2 = inputs.par().with_pool(&mut pool).sum(); assert_eq!(sum, sum2); } #[cfg(not(miri))] #[cfg(feature = "rayon-core")] { let pool = rayon_core::ThreadPoolBuilder::new() .num_threads(8) .build() .unwrap(); // uses the rayon-core::ThreadPool created with 8 threads let sum2 = inputs.par().with_pool(&pool).sum(); assert_eq!(sum, sum2); } #[cfg(not(miri))] #[cfg(feature = "yastl")] { let pool = YastlPool::new(8); // uses the yastl::Pool created with 8 threads let sum2 = inputs.par().with_pool(&pool).sum(); assert_eq!(sum, sum2); } ``` `ParThreadPool` implementations of several thread pools are provided in this crate as optional features (see [features](#features) section). Provided that the pool supports scoped computations, it is trivial to implement this trait in most cases (see [implementations](https://github.com/orxfun/orx-parallel/tree/main/src/runner/implementations) for examples). In most of the cases, *rayon-core*, *scoped_threadpool* and *scoped_pool* perform better than others, and get close to native threads performance with `StdDefaultPool`. Since parallel computations are generic over the thread pools, performances can be conveniently compared for specific use cases. Such an example benchmark can be found in [collect_filter_map](https://github.com/orxfun/orx-parallel/blob/main/benches/collect_filter_map.rs) file. To have quick tests, you may also use the example [benchmark_pools](https://github.com/orxfun/orx-parallel/blob/main/examples/benchmark_pools.rs). ### ParallelExecutor: chunk size Once thread pool provides the computation resources, it is [`ParallelExecutor`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParallelExecutor)'s task to distribute work to available threads. As mentioned above, all threads receive exactly the same closure. This closure continues to pull elements from the input concurrent iterator and operate on the inputs until all elements are processed. The critical decision that parallel executor makes is the chunk size. Depending on the state of the computation, it can dynamically decide on number of elements to pull from the input iterator. The tradeoff it tries to solve is as follows: * the larger the chunk size, * the smaller the parallelization overhead; but also * the larger the risk of imbalance in cases of heterogeneity. ## Features * **std**: This is a **no-std** crate while *std* is included as a default feature. Please use `--no-default-features` flag for no-std use cases. **std** feature enables `StdDefaultPool` as the default thread provider which uses native threads. * **rayon-core**: This feature enables using `rayon_core::ThreadPool` for parallel computations. * **scoped_threadpool**: This feature enables using `scoped_threadpool::Pool`. * **scoped-pool**: This feature enables using `scoped-pool::Pool`. * **yastl**: This feature enables using `yastl::Pool`. * **pond**: This feature enables using `pond::Pool`. * **poolite**: This feature enables using `poolite::Pool`. ## Contributing Contributions are welcome! Please open an [issue](https://github.com/orxfun/orx-parallel/issues/new) or create a PR, * if you notice an error, * have a question or think something could be improved, * have an input collection or generator that needs to be parallelized, * want to use a particular thread pool with parallel iterators, * having trouble representing a particular parallel computation with parallel iterators, * or anything else:) Finally, feel free to contact [me](mailto:orx.ugur.arikan@gmail.com) if you are interested in optimization of the parallel runner to further improve performance, *by maybe dynamic optimization of chunk size decisions with respect to online collection and analysis of metrics*. ## License Dual-licensed under [Apache 2.0](LICENSE-APACHE) or [MIT](LICENSE-MIT). ================================================ FILE: benches/chain3_collect_map.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Vec { inputs.iter().chain(inputs).chain(inputs).map(map).collect() } fn rayon(inputs: &[usize]) -> Vec { use rayon::iter::ParallelIterator; inputs .into_par_iter() .chain(inputs) .chain(inputs) .map(map) .collect() } fn orx_into_vec(inputs: &[usize]) -> Vec { inputs .into_par() .chain(inputs) .chain(inputs) .map(map) .collect() } fn orx_into_split_vec(inputs: &[usize]) -> SplitVec { inputs .into_par() .chain(inputs) .chain(inputs) .map(map) .collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("chain3_collect_map"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); let mut sorted = seq(&input); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&input)); b.iter(|| orx_into_split_vec(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/chain4_collect_map.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Vec { inputs .iter() .chain(inputs) .chain(inputs) .chain(inputs) .map(map) .collect() } fn rayon(inputs: &[usize]) -> Vec { use rayon::iter::ParallelIterator; inputs .into_par_iter() .chain(inputs) .chain(inputs) .chain(inputs) .map(map) .collect() } fn orx_into_vec(inputs: &[usize]) -> Vec { inputs .into_par() .chain(inputs) .chain(inputs) .chain(inputs) .map(map) .collect() } fn orx_into_split_vec(inputs: &[usize]) -> SplitVec { inputs .into_par() .chain(inputs) .chain(inputs) .chain(inputs) .map(map) .collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("chain4_collect_map"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); let mut sorted = seq(&input); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&input)); b.iter(|| orx_into_split_vec(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/chain_collect_map.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Vec { inputs.iter().chain(inputs).map(map).collect() } fn rayon(inputs: &[usize]) -> Vec { use rayon::iter::ParallelIterator; inputs.into_par_iter().chain(inputs).map(map).collect() } fn orx_into_vec(inputs: &[usize]) -> Vec { inputs.into_par().chain(inputs).map(map).collect() } fn orx_into_split_vec(inputs: &[usize]) -> SplitVec { inputs.into_par().chain(inputs).map(map).collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("chain_collect_map"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); let mut sorted = seq(&input); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&input)); b.iter(|| orx_into_split_vec(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/collect_filter.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn filter(output: &&Output) -> bool { let last_char = output.name.chars().last().unwrap(); let last_digit: u32 = last_char.to_string().parse().unwrap(); last_digit < 4 } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .map(|x| to_output(&x)) .collect() } fn seq(inputs: &[Output]) -> Vec<&Output> { inputs.iter().filter(filter).collect() } fn rayon(inputs: &[Output]) -> Vec<&Output> { use rayon::iter::ParallelIterator; inputs.into_par_iter().filter(filter).collect() } fn orx_into_vec(inputs: &[Output]) -> Vec<&Output> { inputs.into_par().filter(filter).collect() } fn orx_into_split_vec(inputs: &[Output]) -> SplitVec<&Output> { inputs.into_par().filter(filter).collect() } #[allow(dead_code)] fn orx_into_vec_with(inputs: &[Output], pool: P) -> Vec<&Output> { inputs.into_par().with_pool(pool).filter(filter).collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("collect_filter"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); let mut sorted = seq(&input); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&input)); b.iter(|| orx_into_split_vec(black_box(&input))) }); #[cfg(feature = "rayon-core")] group.bench_with_input( BenchmarkId::new("orx-vec (rayon-core::ThreadPool)", n), n, |b, _| { let pool = rayon_core::ThreadPoolBuilder::new() .num_threads(32) .build() .unwrap(); assert_eq!(&expected, &orx_into_vec_with(&input, &pool)); b.iter(|| orx_into_vec_with(black_box(&input), &pool)) }, ); #[cfg(feature = "scoped-pool")] group.bench_with_input( BenchmarkId::new("orx-vec (scoped-pool::Pool)", n), n, |b, _| { let pool = scoped_pool::Pool::new(32); assert_eq!(&expected, &orx_into_vec_with(&input, &pool)); b.iter(|| orx_into_vec_with(black_box(&input), &pool)) }, ); #[cfg(feature = "scoped_threadpool")] group.bench_with_input( BenchmarkId::new("orx-vec (scoped_threadpool::Pool)", n), n, |b, _| { let pool = || scoped_threadpool::Pool::new(32); assert_eq!(&expected, &orx_into_vec_with(&input, pool())); b.iter(|| orx_into_vec_with(black_box(&input), pool())) }, ); #[cfg(feature = "yastl")] group.bench_with_input(BenchmarkId::new("orx-vec (yastl::Pool)", n), n, |b, _| { let pool = YastlPool::new(32); assert_eq!(&expected, &orx_into_vec_with(&input, &pool)); b.iter(|| orx_into_vec_with(black_box(&input), &pool)) }); #[cfg(feature = "pond")] group.bench_with_input(BenchmarkId::new("orx-vec (pond::Pool)", n), n, |b, _| { let pool = || PondPool::new_threads_unbounded(32); assert_eq!(&expected, &orx_into_vec_with(&input, pool())); b.iter(|| orx_into_vec_with(black_box(&input), pool())) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/collect_filtermap.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn filter_map(idx: &usize) -> Option { idx.is_multiple_of(3).then(|| to_output(idx)) } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Vec { inputs.iter().filter_map(filter_map).collect() } fn rayon(inputs: &[usize]) -> Vec { use rayon::iter::ParallelIterator; inputs.into_par_iter().filter_map(filter_map).collect() } fn orx_into_vec(inputs: &[usize]) -> Vec { inputs.into_par().filter_map(filter_map).collect() } fn orx_into_split_vec(inputs: &[usize]) -> SplitVec { inputs.into_par().filter_map(filter_map).collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("collect_filtermap"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); let mut sorted = seq(&input); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&input)); b.iter(|| orx_into_split_vec(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/collect_flatmap.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn flat_map(idx: &usize) -> Vec { (0..4).map(|i| to_output(&(idx + i))).collect::>() } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Vec { inputs.iter().flat_map(flat_map).collect() } fn rayon(inputs: &[usize]) -> Vec { use rayon::iter::ParallelIterator; inputs.into_par_iter().flat_map(flat_map).collect() } fn orx_into_vec(inputs: &[usize]) -> Vec { inputs.into_par().flat_map(flat_map).collect() } fn orx_sorted_into_vec(inputs: &[usize]) -> SplitVec { inputs.into_par().flat_map(flat_map).collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("collect_flatmap"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); let mut sorted = seq(&input); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_sorted_into_vec(&input)); b.iter(|| orx_sorted_into_vec(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/collect_iter_into_par.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::ParallelBridge; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn filter(output: &Output) -> bool { let last_char = output.name.chars().last().unwrap(); let last_digit: u32 = last_char.to_string().parse().unwrap(); last_digit < 4 } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Vec { inputs.iter().map(map).filter(filter).collect() } fn rayon(inputs: &[usize]) -> Vec { use rayon::iter::ParallelIterator; inputs.iter().par_bridge().map(map).filter(filter).collect() } fn orx_into_vec(inputs: &[usize]) -> Vec { inputs .iter() .iter_into_par() .map(map) .filter(filter) .collect() } fn orx_into_split_vec(inputs: &[usize]) -> SplitVec { inputs .iter() .iter_into_par() .map(map) .filter(filter) .collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("collect_iter_into_par"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { // TODO: for some reason rayon does not return an ordered collection, create an issue // assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&input)); b.iter(|| orx_into_split_vec(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/collect_long_chain.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 29; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Out1 { name: String, } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Out2 { name: String, number: u32, } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Out3 { out2: Out2, fib: u32, } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Out4 { a: String, b: char, fib: u32, } fn map1(idx: &usize) -> Out1 { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32 % FIB_UPPER_BOUND)); let name = format!("{}-fib-{}", prefix, fib); Out1 { name } } fn filter1(output: &Out1) -> bool { let last_char = output.name.chars().last().unwrap(); let last_digit: u32 = last_char.to_string().parse().unwrap(); last_digit < 4 } fn map2(input: Out1) -> Out2 { let number = (FIB_UPPER_BOUND + input.name.len() as u32).saturating_sub(10); let number = fibonacci(&(number & FIB_UPPER_BOUND)); Out2 { name: number.to_string(), number, } } fn filter2(output: &Out2) -> bool { output.number.is_multiple_of(2) || output.name.contains('0') } fn map3(input: Out2) -> Out3 { let fib = fibonacci(&input.number); Out3 { out2: input, fib } } fn map4(input: Out3) -> Out4 { let a = format!("{}!", input.out2.name); let b = input.out2.name.chars().next().unwrap(); let fib = fibonacci(&((input.out2.number * 7) % FIB_UPPER_BOUND)); Out4 { a, b, fib } } fn filter4(output: &Out4) -> bool { output.a.len() == 5 || output.b == 'x' || output.fib % 2 == 1 } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Vec { inputs .iter() .map(map1) .filter(filter1) .map(map2) .filter(filter2) .map(map3) .map(map4) .filter(filter4) .collect() } fn rayon(inputs: &[usize]) -> Vec { use rayon::iter::ParallelIterator; inputs .into_par_iter() .map(map1) .filter(filter1) .map(map2) .filter(filter2) .map(map3) .map(map4) .filter(filter4) .collect() } fn orx_into_vec(inputs: &[usize]) -> Vec { inputs .into_par() .map(map1) .filter(filter1) .map(map2) .filter(filter2) .map(map3) .map(map4) .filter(filter4) .collect() } fn orx_into_split_vec(inputs: &[usize]) -> SplitVec { inputs .into_par() .map(map1) .filter(filter1) .map(map2) .filter(filter2) .map(map3) .map(map4) .filter(filter4) .collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("collect_long_chain"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&input)); b.iter(|| orx_into_split_vec(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/collect_map.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Vec { inputs.iter().map(map).collect() } fn rayon(inputs: &[usize]) -> Vec { use rayon::iter::ParallelIterator; inputs.into_par_iter().map(map).collect() } fn orx_into_vec(inputs: &[usize]) -> Vec { inputs.into_par().map(map).collect() } fn orx_into_split_vec(inputs: &[usize]) -> SplitVec { inputs.into_par().map(map).collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("collect_map"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); let mut sorted = seq(&input); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&input)); b.iter(|| orx_into_split_vec(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/collect_map_filter.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn filter(output: &Output) -> bool { let last_char = output.name.chars().last().unwrap(); let last_digit: u32 = last_char.to_string().parse().unwrap(); last_digit < 4 } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Vec { inputs.iter().map(map).filter(filter).collect() } fn rayon(inputs: &[usize]) -> Vec { use rayon::iter::ParallelIterator; inputs.into_par_iter().map(map).filter(filter).collect() } fn orx_into_vec(inputs: &[usize]) -> Vec { inputs.into_par().map(map).filter(filter).collect() } fn orx_into_split_vec(inputs: &[usize]) -> SplitVec { inputs.into_par().map(map).filter(filter).collect() } #[allow(dead_code)] fn orx_into_vec_with(inputs: &[usize], pool: P) -> Vec { inputs .into_par() .with_pool(pool) .map(map) .filter(filter) .collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("collect_map_filter"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); let mut sorted = seq(&input); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&input)); b.iter(|| orx_into_split_vec(black_box(&input))) }); #[cfg(feature = "rayon-core")] group.bench_with_input( BenchmarkId::new("orx-into-vec (rayon-core::ThreadPool)", n), n, |b, _| { let pool = rayon_core::ThreadPoolBuilder::new() .num_threads(32) .build() .unwrap(); assert_eq!(&expected, &orx_into_vec_with(&input, &pool)); b.iter(|| orx_into_vec_with(black_box(&input), &pool)) }, ); #[cfg(feature = "scoped-pool")] group.bench_with_input( BenchmarkId::new("orx-into-vec (scoped-pool::Pool)", n), n, |b, _| { let pool = scoped_pool::Pool::new(32); assert_eq!(&expected, &orx_into_vec_with(&input, &pool)); b.iter(|| orx_into_vec_with(black_box(&input), &pool)) }, ); #[cfg(feature = "scoped_threadpool")] group.bench_with_input( BenchmarkId::new("orx-into-vec (scoped_threadpool::Pool)", n), n, |b, _| { let pool = || scoped_threadpool::Pool::new(32); assert_eq!(&expected, &orx_into_vec_with(&input, pool())); b.iter(|| orx_into_vec_with(black_box(&input), pool())) }, ); #[cfg(feature = "yastl")] group.bench_with_input( BenchmarkId::new("orx-into-vec (yastl::Pool)", n), n, |b, _| { let pool = YastlPool::new(32); assert_eq!(&expected, &orx_into_vec_with(&input, &pool)); b.iter(|| orx_into_vec_with(black_box(&input), &pool)) }, ); #[cfg(feature = "pond")] group.bench_with_input( BenchmarkId::new("orx-into-vec (pond::Pool)", n), n, |b, _| { let pool = || PondPool::new_threads_unbounded(32); assert_eq!(&expected, &orx_into_vec_with(&input, pool())); b.iter(|| orx_into_vec_with(black_box(&input), pool())) }, ); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/collect_map_filter_hash_set.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::collections::HashSet; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn filter(output: &Output) -> bool { let last_char = output.name.chars().last().unwrap(); let last_digit: u32 = last_char.to_string().parse().unwrap(); last_digit < 4 } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> HashSet { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &HashSet) -> Vec { inputs.iter().map(map).filter(filter).collect() } fn rayon(inputs: &HashSet) -> Vec { use rayon::iter::ParallelIterator; inputs.into_par_iter().map(map).filter(filter).collect() } fn orx_through_vec(inputs: &HashSet) -> Vec { let inputs: Vec<_> = inputs.iter().collect(); inputs.into_par().map(map).filter(filter).collect() } fn orx_through_iter(inputs: &HashSet) -> Vec { inputs .iter() .iter_into_par() .map(map) .filter(filter) .collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("collect_map_filter_hash_set"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); let mut sorted = seq(&input); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-through-vec", n), n, |b, _| { assert_eq!(&expected, &orx_through_vec(&input)); b.iter(|| orx_through_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-through-iter", n), n, |b, _| { assert_eq!(&expected, &orx_through_iter(&input)); b.iter(|| orx_through_iter(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/count_filtermap.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn filter_map(idx: &usize) -> Option { idx.is_multiple_of(3).then(|| to_output(idx)) } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> usize { inputs.iter().filter_map(filter_map).count() } fn rayon(inputs: &[usize]) -> usize { use rayon::iter::ParallelIterator; inputs.into_par_iter().filter_map(filter_map).count() } fn orx(inputs: &[usize]) -> usize { inputs.into_par().filter_map(filter_map).count() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("count_filtermap"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/count_flatmap.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = true; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn flat_map(idx: &usize) -> Vec { (0..4).map(|i| to_output(&(idx + i))).collect::>() } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> usize { inputs.iter().flat_map(flat_map).count() } fn rayon(inputs: &[usize]) -> usize { use rayon::iter::ParallelIterator; inputs.into_par_iter().flat_map(flat_map).count() } fn orx(inputs: &[usize]) -> usize { inputs.into_par().flat_map(flat_map).count() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("count_flatmap"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/count_map.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> usize { inputs.iter().map(map).count() } fn rayon(inputs: &[usize]) -> usize { use rayon::iter::ParallelIterator; inputs.into_par_iter().map(map).count() } fn orx(inputs: &[usize]) -> usize { inputs.into_par().map(map).count() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("count_map"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/count_map_filter.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = true; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn filter(output: &Output) -> bool { let last_char = output.name.chars().last().unwrap(); let last_digit: u32 = last_char.to_string().parse().unwrap(); last_digit < 4 } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> usize { inputs.iter().map(map).filter(filter).count() } fn rayon(inputs: &[usize]) -> usize { use rayon::iter::ParallelIterator; inputs.into_par_iter().map(map).filter(filter).count() } fn orx(inputs: &[usize]) -> usize { inputs.into_par().map(map).filter(filter).count() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("count_map_filter"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/drain_vec_collect_map_filter.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: usize) -> Output { let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn filter(output: &Output) -> bool { let last_char = output.name.chars().last().unwrap(); let last_digit: u32 = last_char.to_string().parse().unwrap(); last_digit < 4 } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(input: &mut Vec) -> Vec { input.drain(..).map(map).filter(filter).collect() } fn rayon(input: &mut Vec) -> Vec { use rayon::iter::ParallelIterator; rayon::iter::ParallelDrainRange::par_drain(input, ..) .map(map) .filter(filter) .collect() } fn orx_into_vec(input: &mut Vec) -> Vec { input.par_drain(..).map(map).filter(filter).collect() } fn orx_into_split_vec(input: &mut Vec) -> SplitVec { input.par_drain(..).map(map).filter(filter).collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("drain_vec_collect_map_filter"); for n in &treatments { let input = inputs(*n); let expected = seq(&mut input.clone()); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&mut input.clone())); b.iter(|| seq(black_box(&mut input.clone()))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&mut input.clone())); b.iter(|| rayon(black_box(&mut input.clone()))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&mut input.clone())); b.iter(|| orx_into_vec(black_box(&mut input.clone()))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&mut input.clone())); b.iter(|| orx_into_split_vec(black_box(&mut input.clone()))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/find.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const N: usize = 65_536 * 4; const N_EARLY: usize = 1000; const N_MIDDLE: usize = 65_536 * 2; const N_LATE: usize = 65_536 * 4 - 10; const N_NEVER: usize = usize::MAX; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { id: String, name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } let id = idx.to_string(); Output { id, name, numbers } } fn get_find(n: usize) -> impl Fn(&Output) -> bool { move |a: &Output| a.id.parse::().unwrap() == n } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .map(|x| to_output(&x)) .collect() } fn seq(inputs: &[Output], find: impl Fn(&Output) -> bool) -> Option<&Output> { inputs.iter().find(|x| find(x)) } fn rayon(inputs: &[Output], find: impl Fn(&Output) -> bool + Send + Sync) -> Option<&Output> { use rayon::iter::ParallelIterator; inputs.into_par_iter().find_first(|x| find(x)) } fn orx(inputs: &[Output], find: impl Fn(&Output) -> bool + Send + Sync) -> Option<&Output> { inputs.into_par().find(|x| find(x)) } fn run(c: &mut Criterion) { let treatments = [N_EARLY, N_MIDDLE, N_LATE, N_NEVER]; let mut group = c.benchmark_group("find"); for n_when in &treatments { let find = get_find(*n_when); let input = inputs(N); let expected = seq(&input, &find); let n_when = match *n_when { N_EARLY => "find-early", N_MIDDLE => "find-in-the-middle", N_LATE => "find-late", N_NEVER => "find-never", _ => panic!("unhandled n-when"), }; group.bench_with_input(BenchmarkId::new("seq", n_when), n_when, |b, _| { assert_eq!(&expected, &seq(&input, &find)); b.iter(|| seq(black_box(&input), &find)) }); group.bench_with_input(BenchmarkId::new("rayon", n_when), n_when, |b, _| { assert_eq!(&expected, &rayon(&input, &find)); b.iter(|| rayon(black_box(&input), &find)) }); group.bench_with_input(BenchmarkId::new("orx", n_when), n_when, |b, _| { assert_eq!(&expected, &orx(&input, &find)); b.iter(|| orx(black_box(&input), &find)) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/find_any.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const N: usize = 65_536 * 4; const N_EARLY: usize = 1000; const N_MIDDLE: usize = 65_536 * 2; const N_LATE: usize = 65_536 * 4 - 10; const N_NEVER: usize = usize::MAX; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { id: String, name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } let id = idx.to_string(); Output { id, name, numbers } } fn get_find(n: usize) -> impl Fn(&Output) -> bool { move |a: &Output| a.id.parse::().unwrap() == n } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .map(|x| to_output(&x)) .collect() } fn seq(inputs: &[Output], find: impl Fn(&Output) -> bool) -> Option<&Output> { inputs.iter().find(|x| find(x)) } fn rayon(inputs: &[Output], find: impl Fn(&Output) -> bool + Send + Sync) -> Option<&Output> { use rayon::iter::ParallelIterator; inputs.into_par_iter().find_any(|x| find(x)) } fn orx(inputs: &[Output], find: impl Fn(&Output) -> bool + Send + Sync) -> Option<&Output> { inputs .into_par() .iteration_order(IterationOrder::Arbitrary) .find(|x| find(x)) } fn run(c: &mut Criterion) { let treatments = [N_EARLY, N_MIDDLE, N_LATE, N_NEVER]; let mut group = c.benchmark_group("find_any"); for n_when in &treatments { let find = get_find(*n_when); let input = inputs(N); let expected = seq(&input, &find); let n_when = match *n_when { N_EARLY => "find-early", N_MIDDLE => "find-in-the-middle", N_LATE => "find-late", N_NEVER => "find-never", _ => panic!("unhandled n-when"), }; group.bench_with_input(BenchmarkId::new("seq", n_when), n_when, |b, _| { assert_eq!(&expected, &seq(&input, &find)); b.iter(|| seq(black_box(&input), &find)) }); group.bench_with_input(BenchmarkId::new("rayon", n_when), n_when, |b, _| { assert_eq!(&expected, &rayon(&input, &find)); b.iter(|| rayon(black_box(&input), &find)) }); group.bench_with_input(BenchmarkId::new("orx", n_when), n_when, |b, _| { assert_eq!(&expected, &orx(&input, &find)); b.iter(|| orx(black_box(&input), &find)) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/find_flatmap.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const N: usize = 65_536 * 4; const N_EARLY: usize = 1000; const N_MIDDLE: usize = 65_536 * 2; const N_LATE: usize = 65_536 * 4 - 10; const N_NEVER: usize = usize::MAX; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { id: String, name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn flat_map(idx: &usize) -> Vec { (0..4).map(|i| to_output(&(idx + i))).collect::>() } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } let id = idx.to_string(); Output { id, name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn get_find(n: usize) -> impl Fn(&Output) -> bool { move |a: &Output| a.id.parse::().unwrap() == n } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize], find: impl Fn(&Output) -> bool + Send + Sync) -> Option { inputs.iter().flat_map(flat_map).find(find) } fn rayon(inputs: &[usize], find: impl Fn(&Output) -> bool + Send + Sync) -> Option { use rayon::iter::ParallelIterator; inputs.into_par_iter().flat_map(flat_map).find_first(find) } fn orx(inputs: &[usize], find: impl Fn(&Output) -> bool + Send + Sync) -> Option { inputs.into_par().flat_map(flat_map).find(&find) } fn run(c: &mut Criterion) { let treatments = [N_EARLY, N_MIDDLE, N_LATE, N_NEVER]; let mut group = c.benchmark_group("find_flatmap"); for n_when in &treatments { let find = get_find(*n_when); let input = inputs(N); let expected = seq(&input, &find); let n_when = match *n_when { N_EARLY => "find-early", N_MIDDLE => "find-in-the-middle", N_LATE => "find-late", N_NEVER => "find-never", _ => panic!("unhandled n-when"), }; group.bench_with_input(BenchmarkId::new("seq", n_when), n_when, |b, _| { assert_eq!(&expected, &seq(&input, &find)); b.iter(|| seq(black_box(&input), &find)) }); group.bench_with_input(BenchmarkId::new("rayon", n_when), n_when, |b, _| { assert_eq!(&expected, &rayon(&input, &find)); b.iter(|| rayon(black_box(&input), &find)) }); group.bench_with_input(BenchmarkId::new("orx", n_when), n_when, |b, _| { assert_eq!(&expected, &orx(&input, &find)); b.iter(|| orx(black_box(&input), &find)) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/find_iter_into_par.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::ParallelBridge; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const N: usize = 65_536 * 4; const N_EARLY: usize = 1000; const N_MIDDLE: usize = 65_536 * 2; const N_LATE: usize = 65_536 * 4 - 10; const N_NEVER: usize = usize::MAX; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { id: String, name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } let id = idx.to_string(); Output { id, name, numbers } } fn get_find(n: usize) -> impl Fn(&Output) -> bool { move |a: &Output| a.id.parse::().unwrap() == n } fn filter(a: &Output) -> bool { !a.name.ends_with('1') } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize], find: impl Fn(&Output) -> bool) -> Option { inputs.iter().map(map).filter(filter).find(find) } fn rayon(inputs: &[usize], find: impl Fn(&Output) -> bool + Send + Sync) -> Option { use rayon::iter::ParallelIterator; inputs .iter() .par_bridge() .map(map) .filter(filter) .find_first(find) } fn orx(inputs: &[usize], find: impl Fn(&Output) -> bool + Send + Sync) -> Option { inputs .iter() .iter_into_par() .map(map) .filter(filter) .find(&find) } fn run(c: &mut Criterion) { let treatments = [N_EARLY, N_MIDDLE, N_LATE, N_NEVER]; let mut group = c.benchmark_group("find_iter_into_par"); for n_when in &treatments { let find = get_find(*n_when); let input = inputs(N); let expected = seq(&input, &find); let n_when = match *n_when { N_EARLY => "find-early", N_MIDDLE => "find-in-the-middle", N_LATE => "find-late", N_NEVER => "find-never", _ => panic!("unhandled n-when"), }; group.bench_with_input(BenchmarkId::new("seq", n_when), n_when, |b, _| { assert_eq!(&expected, &seq(&input, &find)); b.iter(|| seq(black_box(&input), &find)) }); group.bench_with_input(BenchmarkId::new("rayon", n_when), n_when, |b, _| { assert_eq!(&expected, &rayon(&input, &find)); b.iter(|| rayon(black_box(&input), &find)) }); group.bench_with_input(BenchmarkId::new("orx", n_when), n_when, |b, _| { assert_eq!(&expected, &orx(&input, &find)); b.iter(|| orx(black_box(&input), &find)) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/find_map_filter.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const N: usize = 65_536 * 4; const N_EARLY: usize = 1000; const N_MIDDLE: usize = 65_536 * 2; const N_LATE: usize = 65_536 * 4 - 10; const N_NEVER: usize = usize::MAX; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { id: String, name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } let id = idx.to_string(); Output { id, name, numbers } } fn get_find(n: usize) -> impl Fn(&Output) -> bool { move |a: &Output| a.id.parse::().unwrap() == n } fn filter(a: &Output) -> bool { !a.name.ends_with('1') } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize], find: impl Fn(&Output) -> bool) -> Option { inputs.iter().map(map).filter(filter).find(find) } fn rayon(inputs: &[usize], find: impl Fn(&Output) -> bool + Send + Sync) -> Option { use rayon::iter::ParallelIterator; inputs .into_par_iter() .map(map) .filter(filter) .find_first(find) } fn orx(inputs: &[usize], find: impl Fn(&Output) -> bool + Send + Sync) -> Option { inputs.into_par().map(map).filter(filter).find(&find) } fn run(c: &mut Criterion) { let treatments = [N_EARLY, N_MIDDLE, N_LATE, N_NEVER]; let mut group = c.benchmark_group("find_map_filter"); for n_when in &treatments { let find = get_find(*n_when); let input = inputs(N); let expected = seq(&input, &find); let n_when = match *n_when { N_EARLY => "find-early", N_MIDDLE => "find-in-the-middle", N_LATE => "find-late", N_NEVER => "find-never", _ => panic!("unhandled n-when"), }; group.bench_with_input(BenchmarkId::new("seq", n_when), n_when, |b, _| { assert_eq!(&expected, &seq(&input, &find)); b.iter(|| seq(black_box(&input), &find)) }); group.bench_with_input(BenchmarkId::new("rayon", n_when), n_when, |b, _| { assert_eq!(&expected, &rayon(&input, &find)); b.iter(|| rayon(black_box(&input), &find)) }); group.bench_with_input(BenchmarkId::new("orx", n_when), n_when, |b, _| { assert_eq!(&expected, &orx(&input, &find)); b.iter(|| orx(black_box(&input), &find)) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/mut_for_each_iter.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use std::collections::HashMap; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] struct Data { name: String, number: usize, } fn to_output(idx: usize) -> Data { let name = idx.to_string(); let number = idx; Data { name, number } } fn inputs(len: usize) -> HashMap { (0..len).map(|i| (i, to_output(i))).collect() } fn fibonacci(n: usize) -> usize { let mut a = 0; let mut b = 1; for _ in 0..n { let c = a + b; a = b; b = c; } a } fn filter(data: &&mut Data) -> bool { !data.name.starts_with('3') } fn update(data: &mut Data) { for _ in 0..50 { let increment = fibonacci(data.number % 100) % 10; data.number += increment } } fn seq<'a>(inputs: impl Iterator) { inputs.filter(filter).for_each(update); } fn rayon<'a>(inputs: impl Iterator + Send) { use rayon::iter::{ParallelBridge, ParallelIterator}; inputs.par_bridge().filter(filter).for_each(update); } fn orx<'a>(inputs: impl Iterator) { use orx_parallel::*; inputs.iter_into_par().filter(filter).for_each(update); } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("mut_for_each_iter"); for n in &treatments { let input = inputs(*n); let mut expected = input.clone(); seq(expected.values_mut()); let expected = expected; group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { let mut input = input.clone(); seq(input.values_mut()); assert_eq!(expected, input); b.iter(|| seq(input.values_mut())) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { let mut input = input.clone(); rayon(input.values_mut()); assert_eq!(expected, input); b.iter(|| rayon(input.values_mut())) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { let mut input = input.clone(); orx(input.values_mut()); assert_eq!(expected, input); b.iter(|| orx(input.values_mut())) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/mut_for_each_slice.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use std::hint::black_box; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] struct Data { name: String, number: usize, } fn to_output(idx: usize) -> Data { let name = idx.to_string(); let number = idx; Data { name, number } } fn inputs(len: usize) -> Vec { (0..len).map(to_output).collect() } fn fibonacci(n: usize) -> usize { let mut a = 0; let mut b = 1; for _ in 0..n { let c = a + b; a = b; b = c; } a } fn filter(data: &&mut Data) -> bool { !data.name.starts_with('3') } fn update(data: &mut Data) { for _ in 0..50 { let increment = fibonacci(data.number % 100) % 10; data.number += increment } } fn seq(inputs: &mut [Data]) { inputs.iter_mut().filter(filter).for_each(update); } fn rayon(inputs: &mut [Data]) { use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; inputs.par_iter_mut().filter(filter).for_each(update); } fn orx(inputs: &mut [Data]) { use orx_parallel::*; inputs.into_par().filter(filter).for_each(update); } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("mut_for_each_slice"); for n in &treatments { let input = inputs(*n); let mut expected = input.clone(); seq(&mut expected); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { let mut input = input.clone(); seq(&mut input); assert_eq!(expected, input); b.iter(|| seq(black_box(&mut input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { let mut input = input.clone(); seq(&mut input); assert_eq!(expected, input); b.iter(|| rayon(black_box(&mut input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { let mut input = input.clone(); seq(&mut input); assert_eq!(expected, input); b.iter(|| orx(black_box(&mut input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/rec_iter_map_collect.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_concurrent_recursive_iter::Queue; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use std::hint::black_box; fn fibonacci(n: u64, work: usize) -> u64 { (7..(work + 7)) .map(|j| { let n = black_box((n + j as u64) % 100); let mut a = 0; let mut b = 1; for _ in 0..n { let c = a + b; a = b; b = c; } a }) .sum() } struct Node { value: Vec, children: Vec, } impl Node { fn new(mut n: u32, rng: &mut impl Rng) -> Self { let mut children = Vec::new(); if n < 5 { for _ in 0..n { children.push(Node::new(0, rng)); } } else { while n > 0 { let n2 = rng.random_range(0..=n); children.push(Node::new(n2, rng)); n -= n2; } } Self { value: (0..rng.random_range(1..500)) .map(|_| rng.random_range(0..40)) .collect(), children, } } fn seq_num_nodes(&self) -> usize { 1 + self .children .iter() .map(|node| node.seq_num_nodes()) .sum::() } fn seq(&self, work: usize, numbers: &mut Vec) { numbers.extend(self.value.iter().map(|x| fibonacci(*x, work))); for c in &self.children { c.seq(work, numbers); } } } // alternatives fn seq(roots: &[Node], work: usize) -> Vec { let mut result = vec![]; for root in roots { root.seq(work, &mut result); } result } fn orx_lazy_unknown_chunk1024(roots: &[Node], work: usize) -> SplitVec { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } roots .into_par_rec(extend) .chunk_size(1024) .flat_map(|x| x.value.iter().map(|x| fibonacci(*x, work))) .collect() } fn orx_lazy_exact(roots: &[Node], work: usize, num_nodes: usize) -> SplitVec { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } roots .into_par_rec_exact(extend, num_nodes) .flat_map(|x| x.value.iter().map(|x| fibonacci(*x, work))) .collect() } fn orx_linearized(roots: &[Node], work: usize) -> SplitVec { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } roots .into_par_rec(extend) .linearize() .flat_map(|x| x.value.iter().map(|x| fibonacci(*x, work))) .collect() } fn run(c: &mut Criterion) { let treatments = [1, 10, 25]; let mut group = c.benchmark_group("rec_iter_map_collect"); let mut rng = ChaCha8Rng::seed_from_u64(42); let roots = vec![ Node::new(5000, &mut rng), Node::new(2000, &mut rng), Node::new(4000, &mut rng), ]; let num_nodes: usize = roots.iter().map(|x| x.seq_num_nodes()).sum(); for work in &treatments { let mut expected = seq(&roots, *work); expected.sort(); group.bench_with_input(BenchmarkId::new("seq", work), work, |b, _| { let mut result = seq(&roots, *work); result.sort(); assert_eq!(&expected, &result); b.iter(|| seq(&roots, *work)) }); group.bench_with_input(BenchmarkId::new("orx_lazy_exact", work), work, |b, _| { let mut result = orx_lazy_exact(&roots, *work, num_nodes).to_vec(); result.sort(); assert_eq!(&expected, &result); b.iter(|| orx_lazy_exact(&roots, *work, num_nodes)) }); group.bench_with_input( BenchmarkId::new("orx_lazy_unknown_chunk1024", work), work, |b, _| { let mut result = orx_lazy_unknown_chunk1024(&roots, *work).to_vec(); result.sort(); assert_eq!(&expected, &result); b.iter(|| orx_lazy_unknown_chunk1024(&roots, *work)) }, ); group.bench_with_input(BenchmarkId::new("orx_linearized", work), work, |b, _| { let mut result = orx_linearized(&roots, *work).to_vec(); result.sort(); assert_eq!(&expected, &result); b.iter(|| orx_linearized(&roots, *work)) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/rec_iter_map_sum.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_concurrent_recursive_iter::Queue; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use std::{ hint::black_box, sync::atomic::{AtomicU64, Ordering}, }; fn fibonacci(n: u64, work: usize) -> u64 { (7..(work + 7)) .map(|j| { let n = black_box((n + j as u64) % 100); let mut a = 0; let mut b = 1; for _ in 0..n { let c = a + b; a = b; b = c; } a }) .sum() } struct Node { value: Vec, children: Vec, } impl Node { fn new(mut n: u32, rng: &mut impl Rng) -> Self { let mut children = Vec::new(); if n < 5 { for _ in 0..n { children.push(Node::new(0, rng)); } } else { while n > 0 { let n2 = rng.random_range(0..=n); children.push(Node::new(n2, rng)); n -= n2; } } Self { value: (0..rng.random_range(1..500)) .map(|_| rng.random_range(0..40)) .collect(), children, } } fn seq_num_nodes(&self) -> usize { 1 + self .children .iter() .map(|node| node.seq_num_nodes()) .sum::() } fn seq_sum_fib(&self, work: usize) -> u64 { self.value.iter().map(|x| fibonacci(*x, work)).sum::() + self .children .iter() .map(|x| x.seq_sum_fib(work)) .sum::() } } // alternatives fn seq(roots: &[Node], work: usize) -> u64 { roots.iter().map(|x| x.seq_sum_fib(work)).sum() } fn rayon(roots: &[Node], work: usize) -> u64 { use rayon::iter::*; fn process_node<'scope>( work: usize, sum: &'scope AtomicU64, node: &'scope Node, s: &rayon::Scope<'scope>, ) { for child in &node.children { s.spawn(move |s| { process_node(work, sum, child, s); }); } let val = node.value.par_iter().map(|x| fibonacci(*x, work)).sum(); sum.fetch_add(val, Ordering::Relaxed); } let sum = AtomicU64::new(0); rayon::in_place_scope(|s| { for root in roots { process_node(work, &sum, root, s); } }); sum.into_inner() } fn orx_lazy_unknown_chunk1024(roots: &[Node], work: usize) -> u64 { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } roots .into_par_rec(extend) .chunk_size(1024) .map(|x| x.value.iter().map(|x| fibonacci(*x, work)).sum::()) .sum() } fn orx_lazy_exact(roots: &[Node], work: usize, num_nodes: usize) -> u64 { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } roots .into_par_rec_exact(extend, num_nodes) .map(|x| x.value.iter().map(|x| fibonacci(*x, work)).sum::()) .sum() } fn orx_lazy_exact_flat_map(roots: &[Node], work: usize, num_nodes: usize) -> u64 { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } roots .into_par_rec_exact(extend, num_nodes) .flat_map(|x| x.value.iter().map(|x| fibonacci(*x, work))) .sum() } fn orx_linearized(roots: &[Node], work: usize) -> u64 { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } roots .into_par_rec(extend) .linearize() .map(|x| x.value.iter().map(|x| fibonacci(*x, work)).sum::()) .sum() } fn run(c: &mut Criterion) { let treatments = [1, 10, 25]; let mut group = c.benchmark_group("rec_iter_map_sum"); let mut rng = ChaCha8Rng::seed_from_u64(42); let roots = vec![ Node::new(5000, &mut rng), Node::new(2000, &mut rng), Node::new(4000, &mut rng), ]; let num_nodes: usize = roots.iter().map(|x| x.seq_num_nodes()).sum(); for work in &treatments { let expected = seq(&roots, *work); group.bench_with_input(BenchmarkId::new("seq", work), work, |b, _| { assert_eq!(&expected, &seq(&roots, *work)); b.iter(|| seq(&roots, *work)) }); group.bench_with_input(BenchmarkId::new("rayon", work), work, |b, _| { assert_eq!(&expected, &rayon(&roots, *work)); b.iter(|| rayon(&roots, *work)) }); group.bench_with_input( BenchmarkId::new("orx_lazy_unknown_chunk1024", work), work, |b, _| { assert_eq!(&expected, &orx_lazy_unknown_chunk1024(&roots, *work)); b.iter(|| orx_lazy_unknown_chunk1024(&roots, *work)) }, ); group.bench_with_input(BenchmarkId::new("orx_lazy_exact", work), work, |b, _| { assert_eq!(&expected, &orx_lazy_exact(&roots, *work, num_nodes)); b.iter(|| orx_lazy_exact(&roots, *work, num_nodes)) }); group.bench_with_input( BenchmarkId::new("orx_lazy_exact_flat_map", work), work, |b, _| { assert_eq!( &expected, &orx_lazy_exact_flat_map(&roots, *work, num_nodes) ); b.iter(|| orx_lazy_exact_flat_map(&roots, *work, num_nodes)) }, ); group.bench_with_input(BenchmarkId::new("orx_linearized", work), work, |b, _| { assert_eq!(&expected, &orx_linearized(&roots, *work)); b.iter(|| orx_linearized(&roots, *work)) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/reduce.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn reduce<'a>(a: &'a Output, b: &'a Output) -> &'a Output { match a.name < b.name { true => a, false => b, } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .map(|x| to_output(&x)) .collect() } fn seq(inputs: &[Output]) -> Option<&Output> { inputs.iter().reduce(reduce) } fn rayon(inputs: &[Output]) -> Option<&Output> { use rayon::iter::ParallelIterator; inputs.into_par_iter().reduce_with(reduce) } fn orx(inputs: &[Output]) -> Option<&Output> { inputs.into_par().reduce(reduce) } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("reduce"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/reduce_iter_into_par.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::ParallelBridge; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn filter(a: &Output) -> bool { !a.name.ends_with('1') } fn reduce(a: Output, b: Output) -> Output { match a.name < b.name { true => a, false => b, } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Option { inputs.iter().map(map).filter(filter).reduce(reduce) } fn rayon(inputs: &[usize]) -> Option { use rayon::iter::ParallelIterator; inputs .iter() .map(map) .filter(filter) .par_bridge() .reduce_with(reduce) } fn orx(inputs: &[usize]) -> Option { inputs .iter() .iter_into_par() .map(map) .filter(filter) .reduce(reduce) } #[allow(dead_code)] fn orx_with(inputs: &[usize], pool: P) -> Option { inputs .iter() .iter_into_par() .with_pool(pool) .map(map) .filter(filter) .reduce(reduce) } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("reduce_iter_into_par"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); #[cfg(feature = "rayon-core")] group.bench_with_input( BenchmarkId::new("orx (rayon-core::ThreadPool)", n), n, |b, _| { let pool = rayon_core::ThreadPoolBuilder::new() .num_threads(32) .build() .unwrap(); assert_eq!(&expected, &orx_with(&input, &pool)); b.iter(|| orx_with(black_box(&input), &pool)) }, ); #[cfg(feature = "scoped-pool")] group.bench_with_input(BenchmarkId::new("orx (scoped-pool::Pool)", n), n, |b, _| { let pool = scoped_pool::Pool::new(32); assert_eq!(&expected, &orx_with(&input, &pool)); b.iter(|| orx_with(black_box(&input), &pool)) }); #[cfg(feature = "scoped_threadpool")] group.bench_with_input( BenchmarkId::new("orx (scoped_threadpool::Pool)", n), n, |b, _| { let pool = || scoped_threadpool::Pool::new(32); assert_eq!(&expected, &orx_with(&input, pool())); b.iter(|| orx_with(black_box(&input), pool())) }, ); #[cfg(feature = "yastl")] group.bench_with_input(BenchmarkId::new("orx (yastl::Pool)", n), n, |b, _| { let pool = YastlPool::new(32); assert_eq!(&expected, &orx_with(&input, &pool)); b.iter(|| orx_with(black_box(&input), &pool)) }); #[cfg(feature = "pond")] group.bench_with_input(BenchmarkId::new("orx (pond::Pool)", n), n, |b, _| { let pool = || PondPool::new_threads_unbounded(32); assert_eq!(&expected, &orx_with(&input, pool())); b.iter(|| orx_with(black_box(&input), pool())) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/reduce_long_chain.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 29; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Out1 { name: String, } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Out2 { name: String, number: u32, } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Out3 { out2: Out2, fib: u32, } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Out4 { a: String, b: char, fib: u32, } fn map1(idx: &usize) -> Out1 { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32 % FIB_UPPER_BOUND)); let name = format!("{}-fib-{}", prefix, fib); Out1 { name } } fn filter1(output: &Out1) -> bool { let last_char = output.name.chars().last().unwrap(); let last_digit: u32 = last_char.to_string().parse().unwrap(); last_digit < 4 } fn map2(input: Out1) -> Out2 { let number = (FIB_UPPER_BOUND + input.name.len() as u32).saturating_sub(10); let number = fibonacci(&(number & FIB_UPPER_BOUND)); Out2 { name: number.to_string(), number, } } fn filter2(output: &Out2) -> bool { output.number.is_multiple_of(2) || output.name.contains('0') } fn map3(input: Out2) -> Out3 { let fib = fibonacci(&input.number); Out3 { out2: input, fib } } fn map4(input: Out3) -> Out4 { let a = format!("{}!", input.out2.name); let b = input.out2.name.chars().next().unwrap(); let fib = fibonacci(&((input.out2.number * 7) % FIB_UPPER_BOUND)); Out4 { a, b, fib } } fn filter4(output: &Out4) -> bool { output.a.len() == 5 || output.b == 'x' || output.fib % 2 == 1 } fn reduce(x: Out4, y: Out4) -> Out4 { let a = match x.fib.is_multiple_of(2) { true => x.a, false => y.a, }; let b = match y.fib.is_multiple_of(2) { true => x.b, false => y.b, }; let fib = x.fib + y.fib; Out4 { a, b, fib } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Option { inputs .iter() .map(map1) .filter(filter1) .map(map2) .filter(filter2) .map(map3) .map(map4) .filter(filter4) .reduce(reduce) } fn rayon(inputs: &[usize]) -> Option { use rayon::iter::ParallelIterator; inputs .into_par_iter() .map(map1) .filter(filter1) .map(map2) .filter(filter2) .map(map3) .map(map4) .filter(filter4) .reduce_with(reduce) } fn orx(inputs: &[usize]) -> Option { inputs .into_par() .map(map1) .filter(filter1) .map(map2) .filter(filter2) .map(map3) .map(map4) .filter(filter4) .reduce(reduce) } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("collect_long_chain"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/reduce_map.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn reduce(a: Output, b: Output) -> Output { match a.name < b.name { true => a, false => b, } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Option { inputs.iter().map(map).reduce(reduce) } fn rayon(inputs: &[usize]) -> Option { use rayon::iter::ParallelIterator; inputs.into_par_iter().map(map).reduce_with(reduce) } fn orx(inputs: &[usize]) -> Option { inputs.into_par().map(map).reduce(reduce) } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("reduce_map"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/reduce_map_filter.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn filter(a: &Output) -> bool { !a.name.ends_with('1') } fn reduce(a: Output, b: Output) -> Output { match a.name < b.name { true => a, false => b, } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> Option { inputs.iter().map(map).filter(filter).reduce(reduce) } fn rayon(inputs: &[usize]) -> Option { use rayon::iter::ParallelIterator; inputs .into_par_iter() .map(map) .filter(filter) .reduce_with(reduce) } fn orx(inputs: &[usize]) -> Option { inputs.into_par().map(map).filter(filter).reduce(reduce) } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("reduce_map_filter"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/result_collect_map.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use std::hint::black_box; use std::num::ParseIntError; type Err = ParseIntError; const TEST_LARGE_OUTPUT: bool = false; const N: usize = 65_536 * 4; const N_EARLY: usize = 1000; const N_MIDDLE: usize = 65_536 * 2; const N_LATE: usize = 65_536 * 4 - 10; const N_NEVER: usize = usize::MAX; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Input { id: String, name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn to_input(idx: &usize) -> Input { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } let id = idx.to_string(); Input { id, name, numbers } } fn to_bad_input() -> Input { Input { id: "xyz".to_string(), name: "xyz".to_string(), numbers: Default::default(), } } fn map_input_to_result(input: &Input) -> Result { match input.id.parse::() { Ok(_) => Ok(input.id.clone()), Err(e) => Err(e), } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize, idx_error: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|i| match i == idx_error { true => to_bad_input(), false => { let x = rng.random_range(0..FIB_UPPER_BOUND) as usize; to_input(&x) } }) .collect() } fn seq(inputs: &[Input], map: impl Fn(&Input) -> Result) -> Result, Err> { inputs.iter().map(map).collect() } fn rayon( inputs: &[Input], map: impl Fn(&Input) -> Result + Sync + Send, ) -> Result, Err> { use rayon::iter::*; inputs.into_par_iter().map(map).collect() } fn orx( inputs: &[Input], map: impl Fn(&Input) -> Result + Sync + Clone, ) -> Result, Err> { use orx_parallel::*; inputs.into_par().map(map).into_fallible_result().collect() } fn orx_arbitrary( inputs: &[Input], map: impl Fn(&Input) -> Result + Sync + Clone, ) -> Result, Err> { use orx_parallel::*; inputs .into_par() .iteration_order(IterationOrder::Arbitrary) .map(map) .into_fallible_result() .collect() } fn run(c: &mut Criterion) { let treatments = [N_EARLY, N_MIDDLE, N_LATE, N_NEVER]; let mut group = c.benchmark_group("result_collect_map"); for n_when in &treatments { let input = inputs(N, *n_when); let expected = seq(&input, map_input_to_result); let n_when = match *n_when { N_EARLY => "error-early", N_MIDDLE => "error-in-the-middle", N_LATE => "error-late", N_NEVER => "error-never", _ => panic!("unhandled n-when"), }; group.bench_with_input(BenchmarkId::new("seq", n_when), n_when, |b, _| { assert_eq!(&expected, &seq(&input, map_input_to_result)); b.iter(|| seq(black_box(&input), map_input_to_result)) }); group.bench_with_input(BenchmarkId::new("rayon", n_when), n_when, |b, _| { assert_eq!(&expected, &rayon(&input, map_input_to_result)); b.iter(|| rayon(black_box(&input), map_input_to_result)) }); group.bench_with_input(BenchmarkId::new("orx", n_when), n_when, |b, _| { assert_eq!(&expected, &orx(&input, map_input_to_result)); b.iter(|| orx(black_box(&input), map_input_to_result)) }); group.bench_with_input(BenchmarkId::new("orx_arbitrary", n_when), n_when, |b, _| { assert_eq!( expected.as_ref().map(|x| x.len()), orx_arbitrary(&input, map_input_to_result) .as_ref() .map(|x| x.len()) ); b.iter(|| orx_arbitrary(black_box(&input), map_input_to_result)) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/result_reduce_map.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_concurrent_option::{ConcurrentOption, IntoOption}; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use std::hint::black_box; use std::num::ParseIntError; type Err = ParseIntError; const TEST_LARGE_OUTPUT: bool = false; const N: usize = 65_536 * 4; const N_EARLY: usize = 1000; const N_MIDDLE: usize = 65_536 * 2; const N_LATE: usize = 65_536 * 4 - 10; const N_NEVER: usize = usize::MAX; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Input { id: String, name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn to_input(idx: &usize) -> Input { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } let id = idx.to_string(); Input { id, name, numbers } } fn to_bad_input() -> Input { Input { id: "xyz".to_string(), name: "xyz".to_string(), numbers: Default::default(), } } fn map_input_to_result(input: &Input) -> Result { match input.id.parse::() { Ok(_) => Ok(input.id.clone()), Err(e) => Err(e), } } fn map_to_number(a: String) -> u32 { let num = a.parse::().unwrap(); fibonacci(&(num % 72)) } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize, idx_error: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|i| match i == idx_error { true => to_bad_input(), false => { let x = rng.random_range(0..FIB_UPPER_BOUND) as usize; to_input(&x) } }) .collect() } fn seq(inputs: &[Input], map: impl Fn(&Input) -> Result) -> Result, Err> { let mut error = None; let sum = inputs .iter() .map(map) .inspect(|x| { if let Err(e) = x { error = Some(e.clone()); } }) .take_while(|x| x.is_ok()) .map(|x| map_to_number(x.unwrap())) .reduce(|a, b| a + b); match error { Some(error) => Err(error), None => Ok(sum), } } fn rayon( inputs: &[Input], map: impl Fn(&Input) -> Result + Sync + Send, ) -> Result, Err> { use rayon::iter::*; let error = ConcurrentOption::none(); let sum = inputs .into_par_iter() .map(map) .inspect(|x| { if let Err(e) = x { error.set_some(e.clone()); } }) .map(|x| x.ok()) .while_some() .map(map_to_number) .reduce_with(|a, b| a + b); match error.into_option() { Some(error) => Err(error), None => Ok(sum), } } fn orx( inputs: &[Input], map: impl Fn(&Input) -> Result + Sync + Clone, ) -> Result, Err> { use orx_parallel::*; inputs .into_par() .map(map) .into_fallible_result() .map(map_to_number) .reduce(|a, b| a + b) } fn orx_arbitrary( inputs: &[Input], map: impl Fn(&Input) -> Result + Sync + Clone, ) -> Result, Err> { use orx_parallel::*; inputs .into_par() .iteration_order(IterationOrder::Arbitrary) .map(map) .into_fallible_result() .map(map_to_number) .reduce(|a, b| a + b) } fn run(c: &mut Criterion) { let treatments = [N_EARLY, N_MIDDLE, N_LATE, N_NEVER]; let mut group = c.benchmark_group("result_reduce_map"); for n_when in &treatments { let input = inputs(N, *n_when); let expected = seq(&input, map_input_to_result); let n_when = match *n_when { N_EARLY => "error-early", N_MIDDLE => "error-in-the-middle", N_LATE => "error-late", N_NEVER => "error-never", _ => panic!("unhandled n-when"), }; group.bench_with_input(BenchmarkId::new("seq", n_when), n_when, |b, _| { assert_eq!(&expected, &seq(&input, map_input_to_result)); b.iter(|| seq(black_box(&input), map_input_to_result)) }); group.bench_with_input(BenchmarkId::new("rayon", n_when), n_when, |b, _| { assert_eq!(&expected, &rayon(&input, map_input_to_result)); b.iter(|| rayon(black_box(&input), map_input_to_result)) }); group.bench_with_input(BenchmarkId::new("orx", n_when), n_when, |b, _| { assert_eq!(&expected, &orx(&input, map_input_to_result)); b.iter(|| orx(black_box(&input), map_input_to_result)) }); group.bench_with_input(BenchmarkId::new("orx_arbitrary", n_when), n_when, |b, _| { assert_eq!(&expected, &orx_arbitrary(&input, map_input_to_result)); b.iter(|| orx_arbitrary(black_box(&input), map_input_to_result)) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/sum.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; fn to_output(idx: &usize) -> u32 { let idx = *idx; fibonacci(&(idx as u32)) } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .map(|x| to_output(&x)) .collect() } fn seq(inputs: &[u32]) -> u32 { inputs.iter().sum() } fn rayon(inputs: &[u32]) -> u32 { use rayon::iter::ParallelIterator; inputs.into_par_iter().sum() } fn orx(inputs: &[u32]) -> u32 { inputs.into_par().sum() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("sum"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/sum_filtermap.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn filter_map(idx: &usize) -> Option { idx.is_multiple_of(3) .then(|| to_output(idx)) .map(|x| x.name.len() as u64) } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> u64 { inputs.iter().filter_map(filter_map).sum() } fn rayon(inputs: &[usize]) -> u64 { use rayon::iter::ParallelIterator; inputs.into_par_iter().filter_map(filter_map).sum() } fn orx(inputs: &[usize]) -> u64 { inputs.into_par().filter_map(filter_map).sum() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("sum_filtermap"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/sum_flatmap.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 9562; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn flat_map(idx: &usize) -> Vec { (0..4) .map(|i| to_output(&(idx + i))) .map(|x| x.name.len() as u64) .collect::>() } fn to_output(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> u64 { inputs.iter().flat_map(flat_map).sum() } fn rayon(inputs: &[usize]) -> u64 { use rayon::iter::ParallelIterator; inputs.into_par_iter().flat_map(flat_map).sum() } fn orx(inputs: &[usize]) -> u64 { inputs.into_par().flat_map(flat_map).sum() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("sum_flatmap"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/sum_map_filter.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::hint::black_box; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 201; fn map(idx: &usize) -> u32 { let idx = *idx; fibonacci(&(idx as u32)) } fn filter(output: &u32) -> bool { !output.is_multiple_of(3) } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> Vec { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &[usize]) -> u32 { inputs.iter().map(map).filter(filter).sum() } fn rayon(inputs: &[usize]) -> u32 { use rayon::iter::ParallelIterator; inputs.into_par_iter().map(map).filter(filter).sum() } fn orx(inputs: &[usize]) -> u32 { inputs.into_par().map(map).filter(filter).sum() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("sum_map_filter"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); group.bench_with_input(BenchmarkId::new("seq", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx", n), n, |b, _| { assert_eq!(&expected, &orx(&input)); b.iter(|| orx(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/t_par_merge_sorted.rs ================================================ use criterion::{Criterion, criterion_group, criterion_main}; #[cfg(feature = "experiment")] mod inner { use criterion::Criterion; use orx_criterion::{Experiment, Factors}; use orx_parallel::DefaultRunner; use orx_parallel::experiment::algorithms::merge_sorted_slices::par::{ ParamsParMergeSortedSlices, PivotSearch, par_merge, }; use orx_parallel::experiment::algorithms::merge_sorted_slices::seq::{ ParamsSeqMergeSortedSlices, StreakSearch, }; use orx_parallel::experiment::data_structures::slice_dst::SliceDst; use orx_parallel::experiment::data_structures::slice_src::SliceSrc; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use std::cell::UnsafeCell; type X = usize; fn elem(i: usize) -> X { i } #[inline(always)] fn is_leq(a: &X, b: &X) -> bool { a < b } fn new_vec(len: usize, elem: impl Fn(usize) -> T, sort_kind: SortKind) -> Vec { let mut vec: Vec<_> = (0..len).map(elem).collect(); match sort_kind { SortKind::Sorted => vec.sort(), SortKind::Mixed => { let num_shuffles = 10 * len; let mut rng = ChaCha8Rng::seed_from_u64(42); for _ in 0..num_shuffles { let i = rng.random_range(0..len); let j = rng.random_range(0..len); vec.swap(i, j); } } } vec } fn split_to_sorted_vecs(vec: &[T], split_kind: SplitKind) -> (Vec, Vec) { split_at(vec, split_kind.split_point(vec.len())) } fn split_at(vec: &[T], split_at: usize) -> (Vec, Vec) { let (left, right) = vec.split_at(split_at); let mut left = left.to_vec(); let mut right = right.to_vec(); left.sort(); right.sort(); (left, right) } // treatments #[derive(Clone, Copy, Debug)] #[allow(dead_code)] enum SortKind { Sorted, Mixed, } #[derive(Clone, Copy, Debug)] #[allow(dead_code)] enum SplitKind { MoreInLeft, MoreInRight, Middle, } impl SplitKind { fn split_point(&self, len: usize) -> usize { match self { Self::MoreInLeft => len * 3 / 4, Self::MoreInRight => len / 4, Self::Middle => len / 2, } } } struct Input { left: Vec, right: Vec, target: UnsafeCell>, } impl Drop for Input { fn drop(&mut self) { unsafe { let target = &mut *self.target.get(); target.set_len(self.left.len() + self.right.len()); self.left.set_len(0); self.right.set_len(0); } } } struct MergeData { e: usize, sort: SortKind, split: SplitKind, } impl Factors for MergeData { fn factor_names() -> Vec<&'static str> { vec!["e (len=2^e)", "sort", "split"] } fn factor_names_short() -> Vec<&'static str> { vec!["e", "so", "sp"] } fn factor_levels(&self) -> Vec { vec![ self.e.to_string(), format!("{:?}", self.sort), format!("{:?}", self.split), ] } fn factor_levels_short(&self) -> Vec { vec![ self.e.to_string(), match self.sort { SortKind::Sorted => "T", SortKind::Mixed => "F", } .to_string(), match self.split { SplitKind::Middle => "M", SplitKind::MoreInLeft => "L", SplitKind::MoreInRight => "R", } .to_string(), ] } } impl MergeData { fn all() -> Vec { let mut all = vec![]; let e = [15, 20]; let sort = [SortKind::Mixed]; let split = [SplitKind::Middle]; for e in e { for sort in sort { for split in split { all.push(MergeData { e, sort, split }); } } } all } } // factors struct Params(ParamsParMergeSortedSlices); impl Factors for Params { fn factor_names() -> Vec<&'static str> { vec![ "streak_search", "pivot_search", "put_large_to_left", "min_split_len", "chunk_size", "num_threads", ] } fn factor_names_short() -> Vec<&'static str> { vec!["ss", "ps", "ll", "min", "ch", "nt"] } fn factor_levels(&self) -> Vec { vec![ format!("{:?}", self.0.seq_params.streak_search), format!("{:?}", self.0.pivot_search), self.0.put_large_to_left.to_string(), self.0.min_split_len.to_string(), self.0.chunk_size.to_string(), self.0.num_threads.to_string(), ] } fn factor_levels_short(&self) -> Vec { vec![ match self.0.seq_params.streak_search { StreakSearch::None => "X", StreakSearch::Linear => "L", StreakSearch::Binary => "B", } .to_string(), match self.0.pivot_search { PivotSearch::Linear => "L", PivotSearch::Binary => "B", } .to_string(), match self.0.put_large_to_left { true => "T", false => "F", } .to_string(), self.0.min_split_len.to_string(), self.0.chunk_size.to_string(), self.0.num_threads.to_string(), ] } } impl Params { fn all() -> Vec { let mut all = vec![]; let put_large_to_left = [false, true]; let min_split_len = [1024]; let streak_search = [StreakSearch::None, StreakSearch::Linear]; let pivot_search = [PivotSearch::Linear, PivotSearch::Binary]; let chunk_size = [1, 1024]; let num_threads = [1, 8]; for put_large_to_left in put_large_to_left[..1].to_vec() { for min_split_len in min_split_len[..1].to_vec() { for streak_search in streak_search[..1].to_vec() { for pivot_search in pivot_search[..1].to_vec() { for chunk_size in chunk_size[..1].to_vec() { for num_threads in num_threads { all.push(Self(ParamsParMergeSortedSlices { seq_params: ParamsSeqMergeSortedSlices { streak_search, put_large_to_left, }, put_large_to_left, min_split_len, pivot_search, num_threads, chunk_size, })); } } } } } } all } } // exp struct TuneExperiment; impl Experiment for TuneExperiment { type InputFactors = MergeData; type AlgFactors = Params; type Input = Input; type Output = (); fn input(&mut self, treatment: &Self::InputFactors) -> Self::Input { let len = 1 << treatment.e; let vec = new_vec(len, elem, treatment.sort); let (left, right) = split_to_sorted_vecs(&vec, treatment.split); let target = Vec::with_capacity(vec.len()).into(); Input { left, right, target, } } fn execute(&mut self, variant: &Self::AlgFactors, input: &Self::Input) -> Self::Output { let target = unsafe { &mut *input.target.get() }; let target = SliceDst::from_vec(target); let left = SliceSrc::from_slice(input.left.as_slice()); let right = SliceSrc::from_slice(input.right.as_slice()); let params = variant.0; par_merge( is_leq, left, right, target, ¶ms, DefaultRunner::default(), ); } } pub fn run(c: &mut Criterion) { let treatments = MergeData::all(); let variants = Params::all(); TuneExperiment.bench(c, "t_seq_merge_sorted", &treatments, &variants); } } #[cfg(feature = "experiment")] fn run(c: &mut Criterion) { inner::run(c); } #[cfg(not(feature = "experiment"))] fn run(_: &mut Criterion) { panic!("REQUIRES FEATURE: experiment"); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/t_seq_merge_sorted.rs ================================================ use criterion::{Criterion, criterion_group, criterion_main}; #[cfg(feature = "experiment")] mod inner { use criterion::Criterion; use orx_criterion::{Experiment, Factors}; use orx_parallel::experiment::algorithms::merge_sorted_slices::seq::seq_merge; use orx_parallel::experiment::algorithms::merge_sorted_slices::seq::{ ParamsSeqMergeSortedSlices, StreakSearch, }; use orx_parallel::experiment::data_structures::slice_dst::SliceDst; use orx_parallel::experiment::data_structures::slice_src::SliceSrc; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use std::cell::UnsafeCell; type X = usize; fn elem(i: usize) -> X { i } #[inline(always)] fn is_leq(a: &X, b: &X) -> bool { a < b } fn new_vec(len: usize, elem: impl Fn(usize) -> T, sort_kind: SortKind) -> Vec { let mut vec: Vec<_> = (0..len).map(elem).collect(); match sort_kind { SortKind::Sorted => vec.sort(), SortKind::Mixed => { let num_shuffles = 10 * len; let mut rng = ChaCha8Rng::seed_from_u64(42); for _ in 0..num_shuffles { let i = rng.random_range(0..len); let j = rng.random_range(0..len); vec.swap(i, j); } } } vec } fn split_to_sorted_vecs(vec: &[T], split_kind: SplitKind) -> (Vec, Vec) { split_at(vec, split_kind.split_point(vec.len())) } fn split_at(vec: &[T], split_at: usize) -> (Vec, Vec) { let (left, right) = vec.split_at(split_at); let mut left = left.to_vec(); let mut right = right.to_vec(); left.sort(); right.sort(); (left, right) } // treatments #[derive(Clone, Copy, Debug)] enum SortKind { Sorted, Mixed, } #[derive(Clone, Copy, Debug)] enum SplitKind { MoreInLeft, MoreInRight, Middle, } impl SplitKind { fn split_point(&self, len: usize) -> usize { match self { Self::MoreInLeft => len * 3 / 4, Self::MoreInRight => len / 4, Self::Middle => len / 2, } } } struct Input { left: Vec, right: Vec, target: UnsafeCell>, } impl Drop for Input { fn drop(&mut self) { unsafe { let target = &mut *self.target.get(); target.set_len(self.left.len() + self.right.len()); self.left.set_len(0); self.right.set_len(0); } } } struct MergeData { e: usize, sort: SortKind, split: SplitKind, } impl Factors for MergeData { fn factor_names() -> Vec<&'static str> { vec!["e (len=2^e)", "sort", "split"] } fn factor_names_short() -> Vec<&'static str> { vec!["e", "so", "sp"] } fn factor_levels(&self) -> Vec { vec![ self.e.to_string(), format!("{:?}", self.sort), format!("{:?}", self.split), ] } fn factor_levels_short(&self) -> Vec { vec![ self.e.to_string(), match self.sort { SortKind::Sorted => "T", SortKind::Mixed => "F", } .to_string(), match self.split { SplitKind::Middle => "M", SplitKind::MoreInLeft => "L", SplitKind::MoreInRight => "R", } .to_string(), ] } } impl MergeData { fn all() -> Vec { let mut all = vec![]; let e = [15, 20, 22, 25]; let sort = [SortKind::Mixed, SortKind::Sorted]; let split = [ SplitKind::Middle, SplitKind::MoreInLeft, SplitKind::MoreInRight, ]; for e in e { for sort in sort { for split in split { all.push(MergeData { e, sort, split }); } } } all } } // factors struct Params(ParamsSeqMergeSortedSlices); impl Factors for Params { fn factor_names() -> Vec<&'static str> { vec!["streak_search", "put_large_to_left"] } fn factor_names_short() -> Vec<&'static str> { vec!["nt", "ss"] } fn factor_levels(&self) -> Vec { vec![ format!("{:?}", self.0.streak_search), self.0.put_large_to_left.to_string(), ] } fn factor_levels_short(&self) -> Vec { vec![ match self.0.streak_search { StreakSearch::None => "X", StreakSearch::Linear => "L", StreakSearch::Binary => "B", } .to_string(), match self.0.put_large_to_left { true => "T", false => "F", } .to_string(), ] } } impl Params { fn all() -> Vec { let mut all = vec![]; let put_large_to_left = [false, true]; let streak_search = [ StreakSearch::None, StreakSearch::Linear, StreakSearch::Binary, ]; for put_large_to_left in put_large_to_left { for streak_search in streak_search.iter().cloned() { all.push(Self(ParamsSeqMergeSortedSlices { streak_search, put_large_to_left, })); } } all } } // exp struct TuneExperiment; impl Experiment for TuneExperiment { type InputFactors = MergeData; type AlgFactors = Params; type Input = Input; type Output = (); fn input(&mut self, treatment: &Self::InputFactors) -> Self::Input { let len = 1 << treatment.e; let vec = new_vec(len, elem, treatment.sort); let (left, right) = split_to_sorted_vecs(&vec, treatment.split); let target = Vec::with_capacity(vec.len()).into(); Input { left, right, target, } } fn execute(&mut self, variant: &Self::AlgFactors, input: &Self::Input) -> Self::Output { let target = unsafe { &mut *input.target.get() }; let target = SliceDst::from_vec(target); let left = SliceSrc::from_slice(input.left.as_slice()); let right = SliceSrc::from_slice(input.right.as_slice()); let params = variant.0; seq_merge(is_leq, left, right, target, ¶ms); } } pub fn run(c: &mut Criterion) { let treatments = MergeData::all(); let variants = Params::all(); TuneExperiment.bench(c, "t_seq_merge_sorted", &treatments, &variants); } } #[cfg(feature = "experiment")] fn run(c: &mut Criterion) { inner::run(c); } #[cfg(not(feature = "experiment"))] fn run(_: &mut Criterion) { panic!("REQUIRES FEATURE: experiment"); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/vec_deque_collect_map_filter.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::collections::VecDeque; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: &usize) -> Output { let idx = *idx; let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn filter(output: &Output) -> bool { let last_char = output.name.chars().last().unwrap(); let last_digit: u32 = last_char.to_string().parse().unwrap(); last_digit < 4 } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> VecDeque { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: &VecDeque) -> Vec { inputs.iter().map(map).filter(filter).collect() } fn rayon(inputs: &VecDeque) -> Vec { use rayon::iter::ParallelIterator; inputs.into_par_iter().map(map).filter(filter).collect() } fn orx_into_vec(inputs: &VecDeque) -> Vec { inputs.into_par().map(map).filter(filter).collect() } fn orx_into_split_vec(inputs: &VecDeque) -> SplitVec { inputs.into_par().map(map).filter(filter).collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("VecDeque: collect_map_filter"); for n in &treatments { let input = inputs(*n); let expected = seq(&input); let mut sorted = seq(&input); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(&input)); b.iter(|| seq(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(&input)); b.iter(|| rayon(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(&input)); b.iter(|| orx_into_vec(black_box(&input))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(&input)); b.iter(|| orx_into_split_vec(black_box(&input))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: benches/vec_deque_collect_map_filter_owned.rs ================================================ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use orx_parallel::*; use orx_split_vec::SplitVec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::IntoParallelIterator; use std::collections::VecDeque; use std::hint::black_box; const TEST_LARGE_OUTPUT: bool = false; const LARGE_OUTPUT_LEN: usize = match TEST_LARGE_OUTPUT { true => 64, false => 0, }; const SEED: u64 = 5426; const FIB_UPPER_BOUND: u32 = 201; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct Output { name: String, numbers: [i64; LARGE_OUTPUT_LEN], } fn map(idx: usize) -> Output { let prefix = match idx % 7 { 0 => "zero-", 3 => "three-", _ => "sth-", }; let fib = fibonacci(&(idx as u32)); let name = format!("{}-fib-{}", prefix, fib); let mut numbers = [0i64; LARGE_OUTPUT_LEN]; for (i, x) in numbers.iter_mut().enumerate() { *x = match (idx * 7 + i) % 3 { 0 => idx as i64 + i as i64, _ => idx as i64 - i as i64, }; } Output { name, numbers } } fn filter(output: &Output) -> bool { let last_char = output.name.chars().last().unwrap(); let last_digit: u32 = last_char.to_string().parse().unwrap(); last_digit < 4 } fn fibonacci(n: &u32) -> u32 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn inputs(len: usize) -> VecDeque { let mut rng = ChaCha8Rng::seed_from_u64(SEED); (0..len) .map(|_| rng.random_range(0..FIB_UPPER_BOUND) as usize) .collect() } fn seq(inputs: VecDeque) -> Vec { inputs.into_iter().map(map).filter(filter).collect() } fn rayon(inputs: VecDeque) -> Vec { use rayon::iter::ParallelIterator; inputs.into_par_iter().map(map).filter(filter).collect() } fn orx_into_vec(inputs: VecDeque) -> Vec { inputs.into_par().map(map).filter(filter).collect() } fn orx_into_split_vec(inputs: VecDeque) -> SplitVec { inputs.into_par().map(map).filter(filter).collect() } fn run(c: &mut Criterion) { let treatments = [65_536 * 2]; let mut group = c.benchmark_group("VecDeque-owned: collect_map_filter"); for n in &treatments { let input = inputs(*n); let input = || input.clone(); let expected = seq(input()); let mut sorted = seq(input()); sorted.sort(); group.bench_with_input(BenchmarkId::new("seq-into-vec", n), n, |b, _| { assert_eq!(&expected, &seq(input())); b.iter(|| seq(black_box(input()))) }); group.bench_with_input(BenchmarkId::new("rayon-into-vec", n), n, |b, _| { assert_eq!(&expected, &rayon(input())); b.iter(|| rayon(black_box(input()))) }); group.bench_with_input(BenchmarkId::new("orx-into-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_vec(input())); b.iter(|| orx_into_vec(black_box(input()))) }); group.bench_with_input(BenchmarkId::new("orx-into-split-vec", n), n, |b, _| { assert_eq!(&expected, &orx_into_split_vec(input())); b.iter(|| orx_into_split_vec(black_box(input()))) }); } group.finish(); } criterion_group!(benches, run); criterion_main!(benches); ================================================ FILE: docs/using.md ================================================ # Using Transformation [`ParIter`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIter.html) trait is designed to be as close as possible to the `Iterator` api both due to the fact that: * iterator api is familiar and awesome, and * the goal is to parallelize the computation conveniently by changing `iter` with `par`; or `into_iter` with `into_par`. However, there are certain differences in terms of mutability of the closures, mainly to prevent race conditions. ## The limitation For instance, the following is the `map` signature of the sequential `Iterator`: ```rust fn map(self, f: F) -> Map where F: FnMut(Self::Item) -> B; ``` while, the map signature of the parallel `ParIter` uses `Fn` rather than `FnMut`: ```rust fn map(self, f: F) -> impl ParIter where F: Fn(Self::Item) -> B; ``` It might be obvious but to clarify why parallel map can only accept `Fn`, assume that `f` captures a mutable counter, and increments the counter every time it is called. * This is perfectly fine in a sequential iterator. The counter will be incremented one at a time. * However, in parallel computation, multiple threads will be trying to increment this mutable counter at the same time leading to a race condition. Limiting `f` to be `Fn`, we guarantee that our computation is free of race condition. ## A common use case, random number generators Consider that we have a computation that requires random numbers. For brevity, random number generators can be considered as iterators returning a series of random numbers as we request. Therefore, it keeps its current position and generates random numbers via a mutable reference. Consider the following example, where we take a set of `positions` and take 100 random steps to left or to right starting from each one of them. Then, we check the final positions. In order to take the random step, we need a mutable `rng`. Since the sequential iterator's `map` method allows for `FnMut`: * we can use `.map(|position| random_walk(&mut rng, position, 100))`, * which matches the signature `F: FnMut(i64) -> i64`, * while the captured mutable `rng` reference is the reason of the closure being `FnMut` and it is conveniently abstracted away. ```rust use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha20Rng; fn random_walk(rng: &mut impl Rng, position: i64, num_steps: usize) -> i64 { (0..num_steps).fold(position, |p, _| random_step(rng, p)) } fn random_step(rng: &mut impl Rng, position: i64) -> i64 { match rng.random_bool(0.5) { true => position + 1, // to right false => position - 1, // to left } } fn input_positions() -> Vec { (-10_000..=10_000).collect() } fn sequential() { let positions = input_positions(); let sum_initial_positions = positions.iter().sum::(); println!("sum_initial_positions = {sum_initial_positions}"); let mut rng = ChaCha20Rng::seed_from_u64(42); let final_positions: Vec<_> = positions .iter() .copied() .map(|position| random_walk(&mut rng, position, 100)) .collect(); let sum_final_positions = final_positions.iter().sum::(); println!("sum_final_positions = {sum_final_positions}"); } ``` However, we cannot do the same with a parallel iterator. The following will not compile: ```rust fn parallel() { let positions = input_positions(); let sum_initial_positions = positions.iter().sum::(); println!("sum_initial_positions = {sum_initial_positions}"); let mut rng = ChaCha20Rng::seed_from_u64(42); let final_positions: Vec<_> = positions .par() // <-- parallel computation .copied() .map(|position| random_walk(&mut rng, position, 100)) // <-- does not compile!! .collect(); let sum_final_positions = final_positions.iter().sum::(); println!("sum_final_positions = {sum_final_positions}"); } ``` And it should not compile. If it did, multiple threads would call `random_bool` on the same `rng` at the same time, which would lead to the race condition. ## Solution `using` an explicit mutable variable [`ParIter`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIter.html) does not allow the parallel computation defined above; however, [`ParIterUsing`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIterUsing.html) enables it safely with the following approach: * No race conditions are allowed; therefore, there cannot be one mutable **variable** that all threads accesses. * Instead, all threads have their own mutable **variable**. * Since the computation within the thread is sequential, there cannot be any race condition and we can freely mutate the **variable**. * This **variable** is explicitly defined by one of the two methods which transforms the `ParIter` into `ParIterUsing`. * [`using`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIter.html#tymethod.using): method takes a closure with signature `F: FnMut(usize) -> U`. It takes the index of the spawned thread as input and creates an instance of the variable of type `U`. * [`using_clone`](https://docs.rs/orx-parallel/latest/orx_parallel/trait.ParIter.html#tymethod.using_clone) instead takes a cloneable value of type `U` and provides one clone of it to each of the threads. We can consider `par.using_clone(value)` as a shorthand for `par.using(|_thread_idx| value.clone())`. Provided that the parallel computation is executed with `N` threads, then **exactly** `N` different instances of `U` will be created and send to each thread. Then, a mutable reference to this variable will be available to all of the parallel iterator methods. For instance, the signature of `map` method of the `ParIterUsing` is as follows: ```rust fn map(self, f: F) -> impl ParIter where F: Fn(&mut U, Self::Item) -> B; ``` Notice that the closure is safely `Fn` as it does not mutate any captured variable. On the other hand, it explicitly takes a mutable reference to a value of `U`. This allows us to represent the computation above and execute it in parallel as follows: ```rust fn parallel() { let positions = input_positions(); let sum_initial_positions = positions.iter().sum::(); println!("sum_initial_positions = {sum_initial_positions}"); let final_positions: Vec<_> = positions .par() // <-- parallel computation .copied() .using(|t_idx| ChaCha20Rng::seed_from_u64(42 * t_idx as u64)) // <-- explicit using .map(|rng, position| random_walk(rng, position, 100)) // <-- safe access to mutable rng .collect(); let sum_final_positions = final_positions.iter().sum::(); println!("sum_final_positions = {sum_final_positions}"); } ``` There are two important differences here. 1. With the following line, we are expressing that each thread will create a random number generator (rng). Further, we state that the seed of the rng will be `42 * t_idx` which guarantees that no two threads will use the same sequence of random numbers (important when it matters). ```rust .using(|t_idx| ChaCha20Rng::seed_from_u64(42 * t_idx as u64)) ``` 2. Next, in the `map` call, we have access to a mutable reference to the used variable. This variable is the `rng` created for that specific thread. ```rust .map(|rng, position| random_walk(rng, position, 100)) // rng: &mut ChaCha20Rng ``` ## Generalization Notice that once the safety measures are defined by `ParIterUsing`, it is not different to implement `map`, `filter` or `for_each`, etc. Therefore, all these methods have access to a mutable reference of `U`. The following example demonstrates some of them, all with safe access to the mutable variable `rng`. ```rust let input: Vec = (1..N).collect(); let some_counter = AtomicU64::new(0); input .into_par() .using(|thread_idx| ChaCha20Rng::seed_from_u64(thread_idx as u64 * 10)) .map(|_, i| fibonacci((i % 50) + 1) % 100) .filter(|rng, _| rng.random_bool(0.4)) .flat_map(|rng, i| [rng.random_range(0..i), rng.random_range(0..i)]) .inspect(|rng, i| { if *i < 42 && rng.random_bool(0.2) { some_counter.fetch_add(*i, Ordering::Relaxed); } }) .sum() ``` ## Examples - Channels Random number generator is one of the common use cases that is important for a certain class of algorithms. However, there are other use cases where access to mutable variable is useful. `rayon` allows such computations with `map_with` and `for_each_with` methods, and channels are used as examples in the corresponding documentations. The following example is taken from these documentations and converted to `using` transformation: ```rust use orx_parallel::*; use std::sync::mpsc::channel; let (sender, receiver) = channel(); (0..5) .into_par() .using_clone(sender) .for_each(|s, x| s.send(x).unwrap()); let mut res: Vec<_> = receiver.iter().collect(); res.sort(); assert_eq!(&res[..], &[0, 1, 2, 3, 4]) ``` ## Examples - Metrics Another potential use case is to be able to collect certain metrics about the parallel execution, which is often not trivial. Revisiting the safety notes above, we should be able to collect metrics through mutation per each thread which would give us insights about the parallel execution. To achieve this, in addition to `using`, we need some `unsafe` help with interior mutability. You may see the corresponding example file here: [using_metrics](https://github.com/orxfun/orx-parallel/blob/main/examples/using_metrics.rs). ```rust use orx_parallel::*; use std::cell::UnsafeCell; const N: u64 = 10_000_000; const MAX_NUM_THREADS: usize = 8; // just some work fn fibonacci(n: u64) -> u64 { let mut a = 0; let mut b = 1; for _ in 0..n { let c = a + b; a = b; b = c; } a } #[derive(Default, Debug)] struct ThreadMetrics { thread_idx: usize, num_items_handled: usize, handled_42: bool, num_filtered_out: usize, } struct ThreadMetricsWriter<'a> { metrics_ref: &'a mut ThreadMetrics, } struct ComputationMetrics { thread_metrics: UnsafeCell<[ThreadMetrics; MAX_NUM_THREADS]>, } impl ComputationMetrics { fn new() -> Self { let mut thread_metrics: [ThreadMetrics; MAX_NUM_THREADS] = Default::default(); for i in 0..MAX_NUM_THREADS { thread_metrics[i].thread_idx = i; } Self { thread_metrics: UnsafeCell::new(thread_metrics), } } } impl ComputationMetrics { unsafe fn create_for_thread<'a>(&mut self, thread_idx: usize) -> ThreadMetricsWriter<'a> { // SAFETY: here we create a mutable variable to the thread_idx-th metrics // * If we call this method multiple times with the same index, // we create multiple mutable references to the same ThreadMetrics, // which would lead to a race condition. // * We must make sure that `create_for_thread` is called only once per thread. // * If we use `create_for_thread` within the `using` call to create mutable values // used by the threads, we are certain that the parallel computation // will only call this method once per thread; hence, it will not // cause the race condition. // * On the other hand, we must ensure that we do not call this method // externally. let array = unsafe { &mut *self.thread_metrics.get() }; ThreadMetricsWriter { metrics_ref: &mut array[thread_idx], } } } fn main() { let mut metrics = ComputationMetrics::new(); let input: Vec = (0..N).collect(); let sum = input .par() // SAFETY: we do not call `create_for_thread` externally; // it is safe if it is called only by the parallel computation. .using(|t| unsafe { metrics.create_for_thread(t) }) .map(|m: &mut ThreadMetricsWriter<'_>, i| { // collect some useful metrics m.metrics_ref.num_items_handled += 1; m.metrics_ref.handled_42 |= *i == 42; // actual work fibonacci((*i % 50) + 1) % 100 }) .filter(|m, i| { let is_even = i % 2 == 0; if !is_even { m.metrics_ref.num_filtered_out += 1; } is_even }) .num_threads(MAX_NUM_THREADS) .sum(); println!("\nINPUT-LEN = {N}"); println!("SUM = {sum}"); println!("\n\n"); println!("COLLECTED METRICS PER THREAD"); for metrics in metrics.thread_metrics.get_mut().iter() { println!("* {metrics:?}"); } let total_by_metrics: usize = metrics .thread_metrics .get_mut() .iter() .map(|x| x.num_items_handled) .sum(); println!("\n-> total num_items_handled by collected metrics: {total_by_metrics:?}\n"); assert_eq!(N as usize, total_by_metrics); } ``` Note that creating a thread metrics writer with the `ComputationMetrics::create_for_thread` method is `unsafe` as we can create multiple of them for the same thread metrics. However, parallel execution will not do this: it guarantees that the method will be called once per thread; hence, each call with a different thread index. Once we define how to create the thread metrics writer with `using(|t| unsafe { metrics.create_for_thread(t) })`, the writer is then conveniently available to each of the parallel iterator methods. We can safely use it within each of the closures to collect information; or simply omit when not required. At the end of the computation, the collected metrics will be available by the computation metrics, `metrics`. The output of the program is as follows: ```bash INPUT-LEN = 10000000 SUM = 162400000 COLLECTED METRICS PER THREAD * ThreadMetrics { thread_idx: 0, num_items_handled: 1310720, handled_42: true, num_filtered_out: 891288 } * ThreadMetrics { thread_idx: 1, num_items_handled: 1310720, handled_42: false, num_filtered_out: 891290 } * ThreadMetrics { thread_idx: 2, num_items_handled: 1310720, handled_42: false, num_filtered_out: 891290 } * ThreadMetrics { thread_idx: 3, num_items_handled: 1310720, handled_42: false, num_filtered_out: 891290 } * ThreadMetrics { thread_idx: 4, num_items_handled: 1087104, handled_42: false, num_filtered_out: 739231 } * ThreadMetrics { thread_idx: 5, num_items_handled: 1310720, handled_42: false, num_filtered_out: 891290 } * ThreadMetrics { thread_idx: 6, num_items_handled: 1048576, handled_42: false, num_filtered_out: 713032 } * ThreadMetrics { thread_idx: 7, num_items_handled: 1310720, handled_42: false, num_filtered_out: 891289 } -> total num_items_handled by collected metrics: 10000000 ``` ================================================ FILE: examples/benchmark_collect.rs ================================================ mod utils; #[cfg(not(feature = "generic_iterator"))] fn main() { panic!( r#" REQUIRES FEATURE: generic_iterator To view the arguments: cargo run --release --features generic_iterator --example benchmark_collect -- --help To run with default arguments: cargo run --release --features generic_iterator --example benchmark_collect To run with desired arguments: cargo run --release --features generic_iterator --example benchmark_collect -- --len 123456 --num-repetitions 10 Play with the transformations inside the compute method to test out different computations. "# ); } #[cfg(feature = "generic_iterator")] fn main() { use clap::Parser; use orx_parallel::{IntoParIter, ParIter, generic_iterator::GenericIterator}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use utils::timed_collect_all; #[derive(Parser, Debug)] struct Args { /// Number of items in the input iterator. #[arg(long, default_value_t = 100000)] len: usize, /// Number of repetitions to measure time; total time will be reported. #[arg(long, default_value_t = 100)] num_repetitions: usize, } fn compute( iter: GenericIterator< usize, impl Iterator, impl ParallelIterator, impl ParIter, >, ) -> Vec { iter.map(|x| x.to_string()) .filter_map(|x| (!x.starts_with('1')).then_some(x)) .flat_map(|x| [format!("{}!", &x), x]) .filter(|x| !x.starts_with('2')) .filter_map(|x| x.parse::().ok()) .map(|x| x.to_string()) .collect_vec() } let args = Args::parse(); let input = move || (0..args.len as usize).collect::>(); let expected_output = compute(GenericIterator::sequential(input().into_iter())); let computations: Vec<(&str, Box Vec>)> = vec![ ( "sequential", Box::new(move || compute(GenericIterator::sequential(input().into_iter()))), ), ( "rayon", Box::new(move || compute(GenericIterator::rayon(input().into_par_iter()))), ), ( "orx", Box::new(move || compute(GenericIterator::orx(input().into_par()))), ), ]; timed_collect_all( "benchmark_collect", args.num_repetitions, &expected_output, &computations, ); } ================================================ FILE: examples/benchmark_find.rs ================================================ mod utils; #[cfg(not(feature = "generic_iterator"))] fn main() { panic!( r#" REQUIRES FEATURE: generic_iterator To view the arguments: cargo run --release --features generic_iterator --example benchmark_find -- --help To run with default arguments: cargo run --release --features generic_iterator --example benchmark_find To run with desired arguments: cargo run --release --features generic_iterator --example benchmark_find -- --len 123456 --num-repetitions 10 Play with the transformations inside the compute method to test out different computations. "# ); } #[cfg(feature = "generic_iterator")] fn main() { use clap::Parser; use orx_parallel::{IntoParIter, ParIter, generic_iterator::GenericIterator}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use std::fmt::Display; use utils::timed_reduce_all; #[derive(Parser, Debug)] struct Args { /// Number of items in the input iterator. #[arg(long, default_value_t = 100000)] len: usize, /// Number of repetitions to measure time; total time will be reported. #[arg(long, default_value_t = 100)] num_repetitions: usize, } #[derive(Parser, Debug, Clone, Copy)] enum FindWhen { Early, Middle, Late, Never, } impl Display for FindWhen { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { FindWhen::Early => write!(f, "at the BEGINNING of the iterator"), FindWhen::Middle => write!(f, "in the MIDDLE of the iterator"), FindWhen::Late => write!(f, "at the END of the iterator"), FindWhen::Never => write!(f, "is NOT in the iterator"), } } } fn get_find(n: usize, find_when: FindWhen) -> impl Fn(&String) -> bool + Send + Sync + Clone { move |x| match find_when { FindWhen::Early => x.starts_with("3"), FindWhen::Middle => { let position = n / 2; x == &position.to_string() } FindWhen::Late => { let position = n.saturating_sub(1); x == &position.to_string() } FindWhen::Never => x.starts_with("x"), } } fn compute( find: impl Fn(&String) -> bool + Send + Sync + Clone, iter: GenericIterator< usize, impl Iterator, impl ParallelIterator, impl ParIter, >, ) -> String { iter.map(|x| x.to_string()) .filter_map(|x| (!x.starts_with('1')).then_some(x)) .flat_map(|x| [format!("{}!", &x), x]) .filter(|x| !x.starts_with('2')) .filter_map(|x| (!x.ends_with("!")).then_some(x)) .find(find) .unwrap_or_default() } let args = Args::parse(); let find_when = [ FindWhen::Early, FindWhen::Middle, FindWhen::Late, FindWhen::Never, ]; let input = move || (0..args.len as usize).collect::>(); for when in find_when { let find = move || get_find(args.len, when); let expected_output = compute(find(), GenericIterator::sequential(input().into_iter())); let computations: Vec<(&str, Box String>)> = vec![ ( "sequential", Box::new(move || compute(find(), GenericIterator::sequential(input().into_iter()))), ), ( "rayon", Box::new(move || compute(find(), GenericIterator::rayon(input().into_par_iter()))), ), ( "orx", Box::new(move || compute(find(), GenericIterator::orx(input().into_par()))), ), ]; timed_reduce_all( &format!("find item that is {}", when), args.num_repetitions, Some(expected_output), &computations, ); } } ================================================ FILE: examples/benchmark_find_any.rs ================================================ mod utils; #[cfg(not(feature = "generic_iterator"))] fn main() { panic!( r#" REQUIRES FEATURE: generic_iterator To view the arguments: cargo run --release --features generic_iterator --example benchmark_find_any -- --help To run with default arguments: cargo run --release --features generic_iterator --example benchmark_find_any To run with desired arguments: cargo run --release --features generic_iterator --example benchmark_find_any -- --len 123456 --num-repetitions 10 Play with the transformations inside the compute method to test out different computations. "# ); } #[cfg(feature = "generic_iterator")] fn main() { use clap::Parser; use orx_parallel::{IntoParIter, ParIter, generic_iterator::GenericIterator}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use std::fmt::Display; use utils::timed_reduce_all; #[derive(Parser, Debug)] struct Args { /// Number of items in the input iterator. #[arg(long, default_value_t = 100000)] len: usize, /// Number of repetitions to measure time; total time will be reported. #[arg(long, default_value_t = 100)] num_repetitions: usize, } #[derive(Parser, Debug, Clone, Copy)] enum FindWhen { Early, Middle, Late, Never, } impl Display for FindWhen { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { FindWhen::Early => write!(f, "at the BEGINNING of the iterator"), FindWhen::Middle => write!(f, "in the MIDDLE of the iterator"), FindWhen::Late => write!(f, "at the END of the iterator"), FindWhen::Never => write!(f, "is NOT in the iterator"), } } } fn get_find(n: usize, find_when: FindWhen) -> impl Fn(&String) -> bool + Send + Sync + Clone { move |x| match find_when { FindWhen::Early => x.starts_with("3"), FindWhen::Middle => match x.parse::() { Ok(number) => number > n / 2, _ => false, }, FindWhen::Late => match x.parse::() { Ok(number) => number > n * 10 / 9, _ => false, }, FindWhen::Never => x.starts_with("x"), } } fn compute( find: impl Fn(&String) -> bool + Send + Sync + Clone, iter: GenericIterator< usize, impl Iterator, impl ParallelIterator, impl ParIter, >, ) -> String { iter.map(|x| x.to_string()) .filter_map(|x| (!x.starts_with('1')).then_some(x)) .flat_map(|x| [format!("{}!", &x), x]) .filter(|x| !x.starts_with('2')) .filter_map(|x| (!x.ends_with("!")).then_some(x)) .find_any(find) .unwrap_or_default() } let args = Args::parse(); let find_when = [ FindWhen::Early, FindWhen::Middle, FindWhen::Late, FindWhen::Never, ]; let input = move || (0..args.len as usize).collect::>(); for when in find_when { let find = move || get_find(args.len, when); let computations: Vec<(&str, Box String>)> = vec![ ( "sequential", Box::new(move || compute(find(), GenericIterator::sequential(input().into_iter()))), ), ( "rayon", Box::new(move || compute(find(), GenericIterator::rayon(input().into_par_iter()))), ), ( "orx", Box::new(move || compute(find(), GenericIterator::orx(input().into_par()))), ), ]; timed_reduce_all( &format!("find any of the items that is {}", when), args.num_repetitions, None, &computations, ); } } ================================================ FILE: examples/benchmark_heterogeneous.rs ================================================ mod utils; #[cfg(not(feature = "generic_iterator"))] fn main() { panic!( r#" REQUIRES FEATURE: generic_iterator To view the arguments: cargo run --release --features generic_iterator --example benchmark_heterogeneous -- --help To run with default arguments: cargo run --release --features generic_iterator --example benchmark_heterogeneous To run with desired arguments: cargo run --release --features generic_iterator --example benchmark_heterogeneous -- --len 123456 --num-repetitions 10 Play with the transformations inside the compute method to test out different computations. "# ); } #[cfg(feature = "generic_iterator")] fn main() { use clap::Parser; use orx_parallel::{IntoParIter, ParIter, generic_iterator::GenericIterator}; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use utils::timed_reduce_all; #[derive(Parser, Debug)] struct Args { /// Number of items in the input iterator. #[arg(long, default_value_t = 2000)] len: usize, /// Number of repetitions to measure time; total time will be reported. #[arg(long, default_value_t = 100)] num_repetitions: usize, } fn fibonacci(n: &u64) -> u64 { let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } fn heterogeneous_computation(i: usize) -> u64 { let mut rng = ChaCha8Rng::seed_from_u64(i as u64); for _ in 0..10 * i { let _: u32 = rng.random(); } let n = match rng.random_bool(0.75) { true => rng.random_range(1..100), false => rng.random_range(10000..20000), }; fibonacci(&n) } fn compute( iter: GenericIterator< usize, impl Iterator, impl ParallelIterator, impl ParIter, >, ) -> u64 { iter.map(heterogeneous_computation).max().unwrap() } let args = Args::parse(); let input = move || (0..args.len as usize).collect::>(); let expected_output = compute(GenericIterator::sequential(input().into_iter())); let computations: Vec<(&str, Box u64>)> = vec![ ( "sequential", Box::new(move || compute(GenericIterator::sequential(input().into_iter()))), ), ( "rayon", Box::new(move || compute(GenericIterator::rayon(input().into_par_iter()))), ), ( "orx", Box::new(move || compute(GenericIterator::orx(input().into_par()))), ), ]; timed_reduce_all( "benchmark_heterogeneous", args.num_repetitions, Some(expected_output), &computations, ); } ================================================ FILE: examples/benchmark_pools.rs ================================================ // cargo run --all-features --release --example benchmark_pools // to run with all options: // // output: // // Args { pool_type: All, num_threads: 16, len: 100000, num_repetitions: 1000 } // Std => 15.912437916s // Sequential => 46.194610858s // Pond => 42.560279289s // Poolite => 21.422590826s // RayonCore => 16.227641997s // ScopedPool => 15.958834105s // ScopedThreadPool => 17.228307255s // Yastl => 43.914882593s // cargo run --all-features --release --example benchmark_pools -- --pool-type scoped-pool // to run only using scoped-pool // // output: // // Args { pool_type: ScopedPool, num_threads: 16, len: 100000, num_repetitions: 1000 } // ScopedPool => 16.640308686s // cargo run --all-features --release --example benchmark_pools -- --pool-type rayon-core --len 1000 --num-repetitions 10000 // to run only using rayon-core ThreadPool, with 10000 repetitions for input size of 1000 // // output: // // Args { pool_type: RayonCore, num_threads: 16, len: 1000, num_repetitions: 10000 } // RayonCore => 6.950370104s mod utils; fn main() { #[cfg(feature = "std")] #[cfg(feature = "pond")] #[cfg(feature = "poolite")] #[cfg(feature = "rayon-core")] #[cfg(feature = "scoped-pool")] #[cfg(feature = "scoped_threadpool")] #[cfg(feature = "yastl")] { use clap::Parser; use orx_parallel::runner::ParallelRunner; use orx_parallel::*; use std::hint::black_box; use std::num::NonZeroUsize; use std::time::SystemTime; #[derive(Parser, Debug)] struct Args { /// Type of the thread pool to be used for computations. #[arg(long, default_value_t, value_enum)] pool_type: PoolType, /// Number of threads. #[arg(long, default_value_t = NonZeroUsize::new(16).unwrap())] num_threads: NonZeroUsize, /// Number of items in the input iterator. #[arg(long, default_value_t = 100000)] len: usize, /// Number of repetitions to measure time; total time will be reported. #[arg(long, default_value_t = 100)] num_repetitions: usize, } #[derive(clap::ValueEnum, Clone, Copy, Default, Debug)] enum PoolType { Std, Sequential, Pond, Poolite, RayonCore, ScopedPool, ScopedThreadPool, Yastl, #[default] All, } impl PoolType { fn run_single(self, nt: usize, reps: usize, input: &[usize], expected: &[String]) { let now = SystemTime::now(); let result = match self { Self::Std => run_std(nt, reps, input), Self::Sequential => run_sequential(nt, reps, input), Self::Pond => run_pond(nt, reps, input), Self::Poolite => run_poolite(nt, reps, input), Self::RayonCore => run_rayon_core(nt, reps, input), Self::ScopedPool => run_scoped_pool(nt, reps, input), Self::ScopedThreadPool => run_scoped_threadpool(nt, reps, input), Self::Yastl => run_yastl(nt, reps, input), Self::All => panic!("all is handled by run_all"), }; let elapsed = now.elapsed().unwrap(); println!("{self:?} => {elapsed:?}"); assert_eq!(expected, result); } fn run_all(nt: usize, reps: usize, input: &[usize], expected: &[String]) { Self::Std.run_single(nt, reps, input, expected); Self::Sequential.run_single(nt, reps, input, expected); Self::Pond.run_single(nt, reps, input, expected); Self::Poolite.run_single(nt, reps, input, expected); Self::RayonCore.run_single(nt, reps, input, expected); Self::ScopedPool.run_single(nt, reps, input, expected); Self::ScopedThreadPool.run_single(nt, reps, input, expected); Self::Yastl.run_single(nt, reps, input, expected); } fn run(self, nt: usize, reps: usize, input: &[usize], expected: &[String]) { match self { Self::All => Self::run_all(nt, reps, input, expected), _ => self.run_single(nt, reps, input, expected), } } } fn run_with_runner( mut runner: R, num_threads: usize, num_repetitions: usize, input: &[usize], ) -> Vec { let mut dummy = vec![]; let mut result = vec![]; for i in 0..num_repetitions { result = black_box( input .par() .num_threads(num_threads) .with_runner(&mut runner) .flat_map(|x| { [ *x, fibonacci(x % 10), fibonacci(x % 21), fibonacci(x % 17), fibonacci(x % 33), fibonacci(x % 21), ] }) .map(|x| 3 * x) .filter(|x| !(100..150).contains(x)) .map(|x| x.to_string()) .collect(), ); if i < num_repetitions.min(result.len()) { dummy.push(result[i].clone()) }; } for i in 0..dummy.len() { assert_eq!(&dummy[i], &result[i]); } result } fn fibonacci(n: usize) -> usize { let mut a = 0; let mut b = 1; for _ in 0..n { let c = a + b; a = b; b = c; } a } fn run_std(num_threads: usize, num_repetitions: usize, input: &[usize]) -> Vec { let mut runner = DefaultRunner::default(); // StdRunner run_with_runner(&mut runner, num_threads, num_repetitions, input) } fn run_sequential( num_threads: usize, num_repetitions: usize, input: &[usize], ) -> Vec { let mut runner = RunnerWithPool::from(SequentialPool); run_with_runner(&mut runner, num_threads, num_repetitions, input) } fn run_pond(num_threads: usize, num_repetitions: usize, input: &[usize]) -> Vec { let mut pool = PondPool::new_threads_unbounded(num_threads); let mut runner = RunnerWithPool::from(&mut pool); run_with_runner(&mut runner, num_threads, num_repetitions, input) } fn run_poolite(num_threads: usize, num_repetitions: usize, input: &[usize]) -> Vec { let pool = poolite::Pool::with_builder( poolite::Builder::new().min(num_threads).max(num_threads), ) .unwrap(); let mut runner = RunnerWithPool::from(&pool); run_with_runner(&mut runner, num_threads, num_repetitions, input) } fn run_rayon_core( num_threads: usize, num_repetitions: usize, input: &[usize], ) -> Vec { let pool = rayon_core::ThreadPoolBuilder::new() .num_threads(num_threads) .build() .unwrap(); let mut runner = RunnerWithPool::from(&pool); run_with_runner(&mut runner, num_threads, num_repetitions, input) } fn run_scoped_pool( num_threads: usize, num_repetitions: usize, input: &[usize], ) -> Vec { let pool = scoped_pool::Pool::new(num_threads); let mut runner = RunnerWithPool::from(&pool); run_with_runner(&mut runner, num_threads, num_repetitions, input) } fn run_scoped_threadpool( num_threads: usize, num_repetitions: usize, input: &[usize], ) -> Vec { let mut pool = scoped_threadpool::Pool::new(num_threads as u32); let mut runner = RunnerWithPool::from(&mut pool); run_with_runner(&mut runner, num_threads, num_repetitions, input) } fn run_yastl(num_threads: usize, num_repetitions: usize, input: &[usize]) -> Vec { let pool = YastlPool::new(num_threads); let mut runner = RunnerWithPool::from(&pool); run_with_runner(&mut runner, num_threads, num_repetitions, input) } let args = Args::parse(); println!("{args:?}"); let input: Vec<_> = (0..args.len as usize).collect::>(); let expected: Vec<_> = input .iter() .flat_map(|x| { [ *x, fibonacci(x % 10), fibonacci(x % 21), fibonacci(x % 17), fibonacci(x % 33), fibonacci(x % 21), ] }) .map(|x| 3 * x) .filter(|x| !(100..150).contains(x)) .map(|x| x.to_string()) .collect(); args.pool_type.run( args.num_threads.into(), args.num_repetitions, &input, &expected, ); } } ================================================ FILE: examples/benchmark_reduce.rs ================================================ mod utils; #[cfg(not(feature = "generic_iterator"))] fn main() { panic!( r#" REQUIRES FEATURE: generic_iterator To view the arguments: cargo run --release --features generic_iterator --example benchmark_reduce -- --help To run with default arguments: cargo run --release --features generic_iterator --example benchmark_reduce To run with desired arguments: cargo run --release --features generic_iterator --example benchmark_reduce -- --len 123456 --num-repetitions 10 Play with the transformations inside the compute method to test out different computations. "# ); } #[cfg(feature = "generic_iterator")] fn main() { use clap::Parser; use orx_parallel::{IntoParIter, ParIter, generic_iterator::GenericIterator}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use utils::timed_reduce_all; #[derive(Parser, Debug)] struct Args { /// Number of items in the input iterator. #[arg(long, default_value_t = 100000)] len: usize, /// Number of repetitions to measure time; total time will be reported. #[arg(long, default_value_t = 100)] num_repetitions: usize, } fn reduce(a: usize, b: usize) -> usize { (a + b).saturating_sub(1) } fn compute( iter: GenericIterator< usize, impl Iterator, impl ParallelIterator, impl ParIter, >, ) -> usize { iter.map(|x| x.to_string()) .filter_map(|x| (!x.starts_with('1')).then_some(x)) .flat_map(|x| [format!("{}!", &x), x]) .filter(|x| !x.starts_with('2')) .filter_map(|x| x.parse::().ok()) .map(|x| x.to_string().len()) .reduce(reduce) .unwrap_or(0) } let args = Args::parse(); let input = move || (0..args.len as usize).collect::>(); let expected_output = compute(GenericIterator::sequential(input().into_iter())); let computations: Vec<(&str, Box usize>)> = vec![ ( "sequential", Box::new(move || compute(GenericIterator::sequential(input().into_iter()))), ), ( "rayon", Box::new(move || compute(GenericIterator::rayon(input().into_par_iter()))), ), ( "orx", Box::new(move || compute(GenericIterator::orx(input().into_par()))), ), ]; timed_reduce_all( "benchmark_reduce", args.num_repetitions, Some(expected_output), &computations, ); } ================================================ FILE: examples/collection_of_results.rs ================================================ use orx_concurrent_option::ConcurrentOption; use orx_parallel::*; const N: usize = 10_000; const IDX_BAD_INPUT: [usize; 4] = [1900, 4156, 6777, 5663]; const ITERATION_ORDERS: [IterationOrder; 2] = [IterationOrder::Ordered, IterationOrder::Arbitrary]; fn good_input() -> Vec { (0..N).map(|x| x.to_string()).collect() } fn bad_input() -> Vec { (0..N) .map(|x| match IDX_BAD_INPUT.contains(&x) { true => format!("{x}!"), false => x.to_string(), }) .collect() } // demonstrates an attempt to handle fallible parallel iteration manually, not so convenient fn collection_of_results_good_case() { for iteration_order in ITERATION_ORDERS { let error = ConcurrentOption::none(); let mut output: Vec<_> = good_input() .par() .iteration_order(iteration_order) .map(|x| match x.parse::() { Ok(x) => Some(x), Err(e) => { _ = error.set_some(e); None } }) .take_while(|x| x.is_some()) .map(|x| x.unwrap()) .collect(); if iteration_order == IterationOrder::Arbitrary { output.sort(); } assert!(error.is_none()); // since no error, whilst will always be true and all results will be collected regardless of order assert_eq!(output, (0..N).collect::>()); } } // demonstrates an attempt to handle fallible parallel iteration manually, not so convenient fn collection_of_results_bad_case() { for iteration_order in ITERATION_ORDERS { let error = ConcurrentOption::none(); let mut output: Vec<_> = bad_input() .par() .iteration_order(iteration_order) .map(|x| match x.parse::() { Ok(x) => Some(x), Err(e) => { _ = error.set_some(e); None } }) .take_while(|x| x.is_some()) .map(|x| x.unwrap()) .collect(); if iteration_order == IterationOrder::Arbitrary { output.sort(); } assert_eq!( error.map(|x| x.to_string()), Some("invalid digit found in string".to_string()) ); match iteration_order { IterationOrder::Ordered => { // guaranteed to stop at the first error assert_eq!(output, (0..IDX_BAD_INPUT[0]).collect::>()) } IterationOrder::Arbitrary => { // guaranteed to stop at any of the errors and we can take more // but everything before the first error are guaranteed to be in for i in 0..IDX_BAD_INPUT[0] { assert!(output.contains(&i)); } } } } } // using fallible parallel iterator instead, which directly returns the Result fn collect_result() { let output: Result, _> = good_input() .par() .map(|x| x.parse::()) .into_fallible_result() .collect(); assert_eq!( output.map_err(|x| x.to_string()), Ok((0..N).collect::>()) ); let output: Result, _> = bad_input() .par() .map(|x| x.parse::()) .into_fallible_result() .collect(); assert_eq!( output.map_err(|x| x.to_string()), Err("invalid digit found in string".to_string()) ); } fn main() { collection_of_results_good_case(); collection_of_results_bad_case(); collect_result(); } ================================================ FILE: examples/function_composition_with_mut_using.rs ================================================ use clap::Parser; use core::cell::UnsafeCell; /* This example demonstrates potential issues found by miri with different versions of actually dong exactly the same thing: * We compose functions which use a mutable reference to a variable. * These functions access the value sequentially. We do not have a race condition. * However, due to complexity of function composition with mutable lifetimes, we are unable to represent the correctness of the composition with mutable references. * Miri remarks the problem since multiple closures hold a mutable reference to the same value. This is correct and a potential race-condition risk in multi-threaded scenarios. * On the other hand, using transformation guarantees to send one clone of the used value to each one of the thread which is then used sequentially. This example demonstrates and tests different approaches which would avoid having multiple mutable references and fix the Miri error. The versions implemented and tested are as follows: cargo +nightly miri run --example function_composition_with_mut_using -- --version mut-ref-unsafe-lt1 > fails cargo +nightly miri run --example function_composition_with_mut_using -- --version mut-ref-unsafe-lt2 > fails cargo +nightly miri run --example function_composition_with_mut_using -- --version unsafe-cell-on-reduce > fails cargo +nightly miri run --example function_composition_with_mut_using -- --version unsafe-cell-on-all > passes cargo +nightly miri run --example function_composition_with_mut_using -- --version clone > passes cargo +nightly miri run --example function_composition_with_mut_using -- --version raw-ptr-all > passes */ // # FAIL - mut ref & unsafe - 1 fn compose_mut_ref_unsafe_1( xap1: X, reduce1: R, ) -> impl FnOnce(&mut U, T) -> Option where X: Fn(&mut U, T) -> I + Clone, R: Fn(&mut U, T, T) -> T + Clone, I: IntoIterator + Default, { move |u: &mut U, x: T| { let xap1 = xap1.clone(); let u2 = unsafe { &mut *{ let p: *mut U = u; p } }; let first = xap1(u, x); first.into_iter().reduce(|x, y| reduce1(u2, x, y)) } } fn test_mut_ref_unsafe_1() { let xap1 = |u: &mut String, x: i32| { u.push('1'); [x, x + 2] }; let reduce1 = |u: &mut String, x: i32, y: i32| { u.push('2'); x + y }; let composed = compose_mut_ref_unsafe_1(xap1, reduce1); let mut s = String::new(); let result = composed(&mut s, 12); dbg!(result, &s); assert_eq!(result, Some(26)); } // # FAIL - mut ref & unsafe - 2 fn compose_mut_ref_unsafe_2( map1: M, xap1: X, reduce1: R, ) -> impl FnOnce(&mut U, T) -> Option where M: Fn(&mut U, T) -> T + Clone, X: Fn(&mut U, T) -> I + Clone, I: IntoIterator + Default, R: Fn(&mut U, T, T) -> T + Clone, U: 'static, { let xap_map = move |u: &mut U, x: T| { let u2 = unsafe { &mut *{ let p: *mut U = u; p } }; let first = xap1(u, x); first.into_iter().map(move |x| map1(u2, x)) }; move |u: &mut U, x: T| { let values = xap_map(u, x); values.reduce(|x, y| reduce1(u, x, y)) } } fn test_mut_ref_unsafe_2() { let map1 = |u: &mut String, x: i32| { u.push('0'); x + 1 }; let xap1 = |u: &mut String, x: i32| { u.push('1'); [x, x + 2] }; let reduce1 = |u: &mut String, x: i32, y: i32| { u.push('2'); x + y }; let composed = compose_mut_ref_unsafe_2(map1, xap1, reduce1); let mut s = String::new(); let result = composed(&mut s, 12); dbg!(result, &s); assert_eq!(result, Some(28)); } // # FAIL - UnsafeCell reduce fn compose_unsafe_cell_on_reduce( map1: M, xap1: X, reduce1: R, ) -> impl FnOnce(&UnsafeCell, T) -> Option where M: Fn(&mut U, T) -> T + Clone, X: Fn(&mut U, T) -> I + Clone, I: IntoIterator + Default, R: Fn(&mut U, T, T) -> T + Clone, U: 'static, { let xap_map = move |u: &mut U, x: T| { let u2 = unsafe { &mut *{ let p: *mut U = u; p } }; let first = xap1(u, x); first.into_iter().map(move |x| map1(u2, x)) }; move |u: &UnsafeCell, x: T| { let values = xap_map(unsafe { &mut *u.get() }, x); values.reduce(|x, y| reduce1(unsafe { &mut *u.get() }, x, y)) } } fn test_unsafe_cell_on_reduce() { let map1 = |u: &mut String, x: i32| { u.push('0'); x + 1 }; let xap1 = |u: &mut String, x: i32| { u.push('1'); [x, x + 2] }; let reduce1 = |u: &mut String, x: i32, y: i32| { u.push('2'); x + y }; let composed = compose_unsafe_cell_on_reduce(map1, xap1, reduce1); let s = UnsafeCell::new(String::new()); let result = composed(&s, 12); dbg!(result, &s); assert_eq!(result, Some(28)); } // # PASS - UnsafeCell everything fn compose_unsafe_cell_on_all( map1: M, xap1: X, reduce1: R, ) -> impl FnOnce(&UnsafeCell, T) -> Option where M: Fn(&mut U, T) -> T + Clone, X: Fn(&mut U, T) -> I + Clone, I: IntoIterator + Default, R: Fn(&mut U, T, T) -> T + Clone, U: 'static, { let map2 = move |u: &UnsafeCell, x: T| map1(unsafe { &mut *u.get() }, x); let xap_map = move |u: &UnsafeCell, x: T| { let u2 = unsafe { &*{ let p: *const UnsafeCell<_> = u; p } }; let first = xap1(unsafe { &mut *u.get() }, x); first.into_iter().map(move |x| map2(u2, x)) }; move |u: &UnsafeCell, x: T| { let values = xap_map(u, x); values.reduce(|x, y| reduce1(unsafe { &mut *u.get() }, x, y)) } } fn test_unsafe_cell_on_all() { let map1 = |u: &mut String, x: i32| { u.push('0'); x + 1 }; let xap1 = |u: &mut String, x: i32| { u.push('1'); [x, x + 2] }; let reduce1 = |u: &mut String, x: i32, y: i32| { u.push('2'); x + y }; let composed = compose_unsafe_cell_on_all(map1, xap1, reduce1); let s = UnsafeCell::new(String::new()); let result = composed(&s, 12); dbg!(result, &s); assert_eq!(result, Some(28)); } // # PASS - clone fn compose_clone(xap1: X, reduce1: R) -> impl FnOnce(&mut U, T) -> Option where U: Clone, X: Fn(&mut U, T) -> I + Clone, R: Fn(&mut U, T, T) -> T + Clone, I: IntoIterator + Default, { move |u: &mut U, x: T| { let xap1 = xap1.clone(); let u2 = &mut u.clone(); let first = xap1(u, x); first.into_iter().reduce(|x, y| reduce1(u2, x, y)) } } fn test_clone() { let xap1 = |u: &mut String, x: i32| { u.push('1'); [x, x + 2] }; let reduce1 = |u: &mut String, x: i32, y: i32| { u.push('2'); x + y }; let composed = compose_clone(xap1, reduce1); let mut s = String::new(); let result = composed(&mut s, 12); dbg!(result, &s); assert_eq!(result, Some(26)); } // # PASS - raw-ptr fn compose_raw_ptr_all( map1: M, xap1: X, reduce1: R, ) -> impl FnOnce(*mut U, T) -> Option where M: Fn(&mut U, T) -> T + Clone, X: Fn(&mut U, T) -> I + Clone, I: IntoIterator + Default, R: Fn(&mut U, T, T) -> T + Clone, U: 'static, { let map2 = move |u: *mut U, x: T| map1(unsafe { &mut *u }, x); let xap_map = move |u: *mut U, x: T| { let first = xap1(unsafe { &mut *u }, x); first.into_iter().map(move |x| map2(unsafe { &mut *u }, x)) }; move |u: *mut U, x: T| { let values = xap_map(u, x); values.reduce(|x, y| reduce1(unsafe { &mut *u }, x, y)) } } fn test_raw_ptr_all() { let map1 = |u: &mut String, x: i32| { u.push('0'); x + 1 }; let xap1 = |u: &mut String, x: i32| { u.push('1'); [x, x + 2] }; let reduce1 = |u: &mut String, x: i32, y: i32| { u.push('2'); x + y }; let composed = compose_raw_ptr_all(map1, xap1, reduce1); let mut s = String::new(); let result = composed(&mut s as *mut String, 12); dbg!(result, &s); assert_eq!(result, Some(28)); } #[derive(clap::ValueEnum, Clone, Copy, Debug)] enum Version { MutRefUnsafeLt1, MutRefUnsafeLt2, UnsafeCellOnReduce, UnsafeCellOnAll, Clone, RawPtrAll, } #[derive(Parser, Debug)] struct Args { /// Version of function composition to run and test with miri. #[arg(long, value_enum)] version: Version, } fn main() { let args = Args::parse(); println!("{args:?}"); match args.version { Version::MutRefUnsafeLt1 => test_mut_ref_unsafe_1(), Version::MutRefUnsafeLt2 => test_mut_ref_unsafe_2(), Version::UnsafeCellOnReduce => test_unsafe_cell_on_reduce(), Version::UnsafeCellOnAll => test_unsafe_cell_on_all(), Version::Clone => test_clone(), Version::RawPtrAll => test_raw_ptr_all(), } } // mut-ref-unsafe-lt1, mut-ref-unsafe-lt2, unsafe-cell-on-reduce, unsafe-cell-on-all, clone, raw-ptr-all ================================================ FILE: examples/map_while.rs ================================================ use orx_parallel::*; const N: usize = 10_000; const IDX_BAD_INPUT: [usize; 4] = [1900, 4156, 6777, 5663]; fn good_input() -> Vec { (0..N).map(|x| x.to_string()).collect() } fn bad_input() -> Vec { (0..N) .map(|x| match IDX_BAD_INPUT.contains(&x) { true => format!("{x}!"), false => x.to_string(), }) .collect() } fn main() { let result: Vec<_> = good_input() .par() .map_while(|x| x.parse::().ok()) .collect(); assert_eq!(result, (0..N).collect::>()); let result: Vec<_> = bad_input() .par() .map_while(|x| x.parse::().ok()) // maps until 1900 .collect(); assert_eq!(result, (0..IDX_BAD_INPUT[0]).collect::>()); } ================================================ FILE: examples/max_num_threads_config.rs ================================================ /* 1. to run the computation without any limits on max number of threads: cargo run --release --example max_num_threads_config OR ORX_PARALLEL_MAX_NUM_THREADS=0 cargo run --release --example max_num_threads_config 2. to allow parallel computation at most 4 threads: ORX_PARALLEL_MAX_NUM_THREADS=4 cargo run --release --example max_num_threads_config */ use orx_parallel::*; fn fib(n: &u64) -> u64 { // just some work let mut a = 0; let mut b = 1; for _ in 0..*n { let c = a + b; a = b; b = c; } a } // A: what should name of the variable be? const MAX_NUM_THREADS_ENV_VARIABLE: &str = "ORX_PARALLEL_MAX_NUM_THREADS"; fn max_num_threads_by_env_variable() -> Option { match std::env::var(MAX_NUM_THREADS_ENV_VARIABLE) { Ok(s) => match s.parse::() { Ok(0) => None, Ok(x) => Some(x), Err(_e) => None, }, Err(_e) => None, } } fn main() { match max_num_threads_by_env_variable() { Some(x) => { println!( "Environment variable {MAX_NUM_THREADS_ENV_VARIABLE} is set to {x}\n -> this will be the hard limit on the maximum number of threads that can be used by parallelization" ) } None => { println!( "Environment variable {MAX_NUM_THREADS_ENV_VARIABLE} is not set\n -> all available threads might be used" ) } } let n = 1 << 31; let input = 0..n; // default -> might use all threads let sum = input.par().map(|i| fib(&(i as u64 % 42)) % 42).sum(); println!("sum = {sum}"); } ================================================ FILE: examples/mutable_par_iter.rs ================================================ use orx_parallel::{ IntoParIter, IterIntoParIter, ParIter, ParallelizableCollection, ParallelizableCollectionMut, }; use std::collections::HashMap; const N: usize = 1_000_000; fn mut_slice_into_par() { // We create a mutable `slice` as our source: `&mut [T]` // // `&mut [T]` implements `IntoConcurrentIter` // => `&mut [T]` auto-implements `IntoParIter` // Therefore, we can call `slice.into_par()` to create a parallel // iterator yielding mutable references consuming the `slice`. let mut vec: Vec<_> = (0..N).collect(); let slice = vec.as_mut_slice(); let par = slice.into_par(); // IntoParIter on &mut [T] par.filter(|x| **x != 42).for_each(|x| *x *= 0); let sum = vec.par().sum(); assert_eq!(sum, 42); } fn vec_par_mut() { // Here, we directly use the `vec` as our source: `Vec` // // `Vec` implements `ConcurrentCollectionMut` // => `Vec` auto-implements `ParallelizableCollectionMut` // Therefore, we can call `vec.par_mut()` to create a parallel // iterator yielding mutable references, using a mutable reference to `vec`. let mut vec: Vec<_> = (0..N).collect(); let par = vec.par_mut(); par.filter(|x| **x != 42).for_each(|x| *x *= 0); let sum = vec.par().sum(); assert_eq!(sum, 42); } fn iter_mut_into_par() { // Finally, here we convert any mutable iterator into a parallel iterator. // // `Iterator` implements `IterIntoConcurrentIter`. // => `Iterator` auto-implements `IterIntoParIter` // Therefore, we can call `iter.iter_into_par()` to create a parallel // iterator yielding mutable references. let mut map: HashMap<_, _> = (0..N).map(|x| (10 * x, x)).collect(); let iter = map.values_mut(); let par = iter.iter_into_par(); par.filter(|x| **x != 42).for_each(|x| *x *= 0); let sum = map.values().iter_into_par().sum(); assert_eq!(sum, 42); } fn main() { mut_slice_into_par(); vec_par_mut(); iter_mut_into_par(); } ================================================ FILE: examples/par_merge_sorted.rs ================================================ #[cfg(not(feature = "experiment"))] fn main() { panic!("REQUIRES FEATURE: experiment"); } #[cfg(feature = "experiment")] fn main() { use clap::Parser; use orx_parallel::{ DefaultRunner, experiment::{ algorithms::merge_sorted_slices::{ par::{ParamsParMergeSortedSlices, PivotSearch, par_merge}, seq::{ParamsSeqMergeSortedSlices, StreakSearch, seq_merge}, }, data_structures::{slice_dst::SliceDst, slice_src::SliceSrc}, }, }; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use std::{cell::UnsafeCell, time::Instant}; type X = usize; fn elem(i: usize) -> X { i } #[inline(always)] fn is_leq(a: &X, b: &X) -> bool { a < b } fn new_vec(len: usize, elem: impl Fn(usize) -> T) -> Vec { let mut vec: Vec<_> = (0..len).map(elem).collect(); let num_shuffles = 10 * len; let mut rng = ChaCha8Rng::seed_from_u64(42); for _ in 0..num_shuffles { let i = rng.random_range(0..len); let j = rng.random_range(0..len); vec.swap(i, j); } vec } fn split_to_sorted_vecs(vec: &[T]) -> (Vec, Vec) { split_at(vec, vec.len() / 2) } fn split_at(vec: &[T], split_at: usize) -> (Vec, Vec) { let (left, right) = vec.split_at(split_at); let mut left = left.to_vec(); let mut right = right.to_vec(); left.sort(); right.sort(); (left, right) } struct Input { left: Vec, right: Vec, target: UnsafeCell>, } impl Input { fn is_target_sorted(&mut self) -> bool { let target = self.target.get_mut(); let mut sorted = target.clone(); sorted.sort(); target == &sorted } } impl Drop for Input { fn drop(&mut self) { unsafe { let target = &mut *self.target.get(); target.set_len(self.left.len() + self.right.len()); self.left.set_len(0); self.right.set_len(0); } } } #[derive(Parser, Debug)] struct Args { #[arg(long, default_value_t = false)] with_diagnostics: bool, #[arg(long, default_value_t = 10)] min_split_len_e: usize, #[arg(long, default_value_t = 8)] num_threads: usize, #[arg(long, default_value_t = 1024)] chunk_size: usize, #[arg(long, default_value_t = 23)] len_e: usize, } let args = Args::parse(); let Args { with_diagnostics, min_split_len_e, num_threads, chunk_size, len_e, } = args; let len = 1 << len_e; let min_split_len = 1 << min_split_len_e; let par = num_threads != 1; let vec = new_vec(len, elem); let (left, right) = split_to_sorted_vecs(&vec); let target = Vec::with_capacity(vec.len()).into(); let mut input = Input { left, right, target, }; let target = unsafe { &mut *input.target.get() }; let target = SliceDst::from_vec(target); let left = SliceSrc::from_slice(input.left.as_slice()); let right = SliceSrc::from_slice(input.right.as_slice()); let params = ParamsParMergeSortedSlices { seq_params: ParamsSeqMergeSortedSlices { streak_search: StreakSearch::None, put_large_to_left: true, }, pivot_search: PivotSearch::Binary, put_large_to_left: true, min_split_len, chunk_size, num_threads, }; let begin = Instant::now(); match par { true => match with_diagnostics { true => par_merge( is_leq, left, right, target, ¶ms, DefaultRunner::default().with_diagnostics(), ), false => par_merge( is_leq, left, right, target, ¶ms, DefaultRunner::default(), ), }, false => seq_merge(is_leq, left, right, target, ¶ms.seq_params), } println!("{:?}", begin.elapsed()); assert!(input.is_target_sorted()); } ================================================ FILE: examples/parallelization_on_tree/collection_on_entire_tree.rs ================================================ use crate::run_utils::timed; use orx_parallel::*; type Node = crate::node::Node; pub fn run(root: &Node) { println!("\n\n\n\n"); println!( r#"# COLLECTION ON ENTIRE TREE This example is almost the same as the "reduction" example. The only difference is that instead of computing the some of mapped values, we collect all mapped values in a vector. This demonstrates the fact that a "parallel recursive iterator" is nothing but a "parallel iterator" with access to all `ParIter` methods. In order to change the computation from reduction to collection, all we need to do is to change [root].into_par_rec(extend).map(compute).sum() into [root].into_par_rec(extend).map(compute).collect() "# ); let log = |vec: Vec| println!(" collection-len = {:?}", vec.len()); timed("sequential", || sequential(root), log); timed("orx_rec", || orx_rec(root), log); timed("orx_rec_linearized", || orx_rec_linearized(root), log); timed("orx_rec_exact", || orx_rec_exact(root), log); println!(); } /// Just a demo computation we perform for each node. fn compute(node: &Node) -> u64 { crate::run_utils::compute(node.data.parse::().unwrap()) } /// # sequential /// /// This is a recursive sequential implementation to compute and reduce values of /// all nodes descending from the root. fn sequential(root: &Node) -> Vec { fn seq_compute_node(node: &Node, result: &mut Vec) { let node_value = compute(node); result.push(node_value); for child in &node.children { seq_compute_node(child, result); } } let mut result = vec![]; seq_compute_node(root, &mut result); result } /// # orx-parallel: parallel recursive iterator with unknown length /// /// Here we parallelize by providing the `extend` function. /// /// Although we don't use it here, please consider `chunk_size` /// optimization depending on the data whenever necessary. This might /// be more important in non-linear data structures compared to linear /// due to the dynamic nature of iteration. fn orx_rec(root: &Node) -> Vec { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } [root].into_par_rec(extend).map(compute).collect() } /// # orx-parallel: parallel recursive iterator with unknown length /// /// Here we parallelize by providing the `extend` function. /// /// However, rather than parallel processing over a dynamic recursive /// input, the iterator first flattens the tasks with the `linearize` /// call and then operates on it as if it is over a linear data structure. fn orx_rec_linearized(root: &Node) -> Vec { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } [root] .into_par_rec(extend) .linearize() .map(compute) .collect() } /// # orx-parallel: parallel recursive iterator with exact length /// /// Here we parallelize by providing the `extend` function. /// Further, we precompute the total number of children and provide it while creating /// the parallel iterator. This is helpful to optimize parallel execution whenever /// it is available and cheap to compute. /// /// Good thing, we can also count the number of nodes in parallel. /// /// On the other hand, it is good not to keep the chunk size too large in a recursive /// iterator, as we limit it to 32 in the following. fn orx_rec_exact(root: &Node) -> Vec { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } let num_nodes = [root].into_par_rec(extend).count(); [root] .into_par_rec_exact(extend, num_nodes) .chunk_size(32) .map(compute) .collect() } ================================================ FILE: examples/parallelization_on_tree/main.rs ================================================ use crate::tree::Tree; use clap::Parser; use rand::SeedableRng; use rand_chacha::ChaCha8Rng; use std::sync::OnceLock; mod collection_on_entire_tree; mod node; mod reduction_on_entire_tree; mod reduction_on_subset_of_tree; mod run_utils; mod tree; #[derive(Parser, Debug)] struct Args { /// Amount of work (num times Fibonacci will be repeated). #[arg(long, default_value_t = 10)] amount_of_work: usize, } pub fn amount_of_work() -> &'static usize { static WORK: OnceLock = OnceLock::new(); WORK.get_or_init(|| Args::parse().amount_of_work) } fn main() { let num_nodes = 100_000; let out_degree = 0..100; let mut rng = ChaCha8Rng::seed_from_u64(42); let data = |idx: usize| idx.to_string(); let root = Tree::new_node(num_nodes, out_degree, data, &mut rng); println!("\nOne path from root to a leaf as an example"); let mut next_node = Some(&root); let mut i = 0; while let Some(node) = next_node { let indent: String = (0..i).map(|_| '*').collect(); println!("{indent}{node:?}"); i += 1; next_node = node.children.iter().max_by_key(|x| x.children.len()); } println!("\nTotal number of nodes = {}", root.num_nodes()); reduction_on_entire_tree::run(&root); collection_on_entire_tree::run(&root); reduction_on_subset_of_tree::run(&root); } ================================================ FILE: examples/parallelization_on_tree/node.rs ================================================ use std::fmt::Debug; pub struct Node { pub idx: usize, pub data: T, pub children: Vec>, } impl Debug for Node { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Node") .field("idx", &self.idx) .field("num_children", &self.children.len()) .field( "children_idx", &self .children .iter() .take(10) .map(|x| x.idx) .collect::>(), ) .finish() } } impl Node { pub fn num_nodes(&self) -> usize { 1 + self .children .iter() .map(|node| node.num_nodes()) .sum::() } } ================================================ FILE: examples/parallelization_on_tree/reduction_on_entire_tree.rs ================================================ use crate::run_utils::timed; use orx_parallel::*; use std::sync::atomic::{AtomicU64, Ordering}; type Node = crate::node::Node; pub fn run(root: &Node) { println!("\n\n\n\n"); println!( r#"# REDUCTION ON ENTIRE TREE This example demonstrates parallel computation over a tree. Unlike parallelization over linear data structures, we don't have access to all input elements, or say tasks, ahead of time. Instead, the new elements are added dynamically on the fly. Therefore, these iterators are called "parallel recursive iterator"s. In addition to an initial set of elements, a parallel recursive iterator is created with an "extend" function which defines the recursive behavior. In this example we create an iterator where elements are "&Node". We define the "extend" function as follows: fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) {{ queue.extend(&node.children); }} While processing a particular "node", we add all its children to the "queue". In this example, we use "queue.extend", later we will also use "queue.push". This allows to express the parallel computation as simple as over a linear data structure: [root].into_par_rec(extend).map(compute).sum() "# ); let log = |sum: u64| println!(" sum = {sum}"); timed("sequential", || sequential(root), log); // rayon miri fails with: // Undefined Behavior: trying to retag from <84156795> for SharedReadWrite permission at alloc41643328[0x8], // but that tag does not exist in the borrow stack for this location #[cfg(not(miri))] timed("rayon", || rayon(root), log); timed("orx_rec", || orx_rec(root), log); timed("orx_rec_linearized", || orx_rec_linearized(root), log); timed("orx_rec_exact", || orx_rec_exact(root), log); println!(); } /// Just a demo computation we perform for each node. fn compute(node: &Node) -> u64 { crate::run_utils::compute(node.data.parse::().unwrap()) } /// # sequential /// /// This is a recursive sequential implementation to compute and reduce values of /// all nodes descending from the root. fn sequential(root: &Node) -> u64 { fn seq_compute_node(node: &Node) -> u64 { let node_value = compute(node); let child_values = node.children.iter().map(seq_compute_node); node_value + child_values.sum::() } seq_compute_node(root) } /// # rayon: defining the computation with rayon's scoped threads. pub fn rayon(root: &Node) -> u64 { fn process_node<'scope>(sum: &'scope AtomicU64, node: &'scope Node, s: &rayon::Scope<'scope>) { for child in &node.children { s.spawn(move |s| { process_node(sum, child, s); }); } let node_value = compute(node); sum.fetch_add(node_value, Ordering::Relaxed); } let sum = AtomicU64::new(0); rayon::in_place_scope(|s| { process_node(&sum, root, s); }); sum.into_inner() } /// # orx-parallel: parallel recursive iterator with unknown length /// /// Here we parallelize by providing the `extend` function. /// /// Although we don't use it here, please consider `chunk_size` /// optimization depending on the data whenever necessary. This might /// be more important in non-linear data structures compared to linear /// due to the dynamic nature of iteration. fn orx_rec(root: &Node) -> u64 { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } [root].into_par_rec(extend).map(compute).sum() } /// # orx-parallel: parallel recursive iterator with unknown length /// /// Here we parallelize by providing the `extend` function. /// /// However, rather than parallel processing over a dynamic recursive /// input, the iterator first flattens the tasks with the `linearize` /// call and then operates on it as if it is over a linear data structure. fn orx_rec_linearized(root: &Node) -> u64 { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } [root].into_par_rec(extend).linearize().map(compute).sum() } /// # orx-parallel: parallel recursive iterator with exact length /// /// Here we parallelize by providing the `extend` function. /// Further, we precompute the total number of children and provide it while creating /// the parallel iterator. This is helpful to optimize parallel execution whenever /// it is available and cheap to compute. /// /// Good thing, we can also count the number of nodes in parallel. /// /// On the other hand, it is good not to keep the chunk size too large in a recursive /// iterator, as we limit it to 32 in the following. fn orx_rec_exact(root: &Node) -> u64 { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { queue.extend(&node.children); } let num_nodes = [root].into_par_rec(extend).count(); [root] .into_par_rec_exact(extend, num_nodes) .chunk_size(32) .map(compute) .sum() } ================================================ FILE: examples/parallelization_on_tree/reduction_on_subset_of_tree.rs ================================================ use crate::run_utils::timed; use orx_parallel::*; type Node = crate::node::Node; pub fn run(root: &Node) { println!("\n\n\n\n"); println!( r#"# REDUCTION ON SUBSET OF THE TREE In the previous examples we used "queue.extend" method to dynamically add children to the queue. However, this method requires the children to implement 'ExactSizeIterator'. When we don't have pre-allocated children, or when we apply a filter on these children, we cannot always satisfy this requirement. In these cases, * we can use `queue.push(child)` to add children one-by-one; or * we can collect children into a vec and then use `queue.extend(children_vec)` to add them together. `queue.push` approach has the following pros and cons: * (+) makes new children available as soon as available. * (+) does not require allocation. * (-) might have greater parallelization overhead. `queue.extend` approach has the following pros and cons: * (+) will have the minimum parallelization overhead. * (-) requires allocation for processing each node. These are a couple of recommendations, we can use `push` and `extend` methods in a different way to optimize our use case. "# ); println!("\n\n\n\n# REDUCTION ON SUBSET OF THE TREE"); let log = |sum: u64| println!(" sum = {sum}"); timed("sequential", || sequential(root), log); timed("push_orx_rec", || push_orx_rec(root), log); timed( "collect_extend_orx_rec", || collect_extend_orx_rec(root), log, ); println!(); } /// Just a demo computation we perform for each node. fn compute(node: &Node) -> u64 { crate::run_utils::compute(node.data.parse::().unwrap()) } fn filter(node: &&Node) -> bool { !node.data.parse::().unwrap().is_multiple_of(42) } /// # sequential /// /// This is a recursive sequential implementation to compute and reduce values of /// all nodes descending from the root. fn sequential(root: &Node) -> u64 { fn seq_compute_node(node: &Node) -> u64 { let node_value = compute(node); let child_values = node.children.iter().filter(filter).map(seq_compute_node); node_value + child_values.sum::() } seq_compute_node(root) } /// # orx-parallel: parallel recursive iterator with unknown length /// /// Since we do not know how many children to add ahead-of-time, we /// don't have an ExactSizeIterator. Therefore, instead of `queue.extend`, /// we use `queue.push` to add new children. /// /// * (+) makes new children available as soon as available. /// * (+) does not require allocation. /// * (-) might have greater parallelization overhead. fn push_orx_rec(root: &Node) -> u64 { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { for child in node.children.iter().filter(filter) { queue.push(child); } } [root].into_par_rec(extend).map(compute).sum() } /// # orx-parallel: parallel recursive iterator with unknown length /// /// Alternatively, we can collect children in a vector and then call /// `queue.extend` to add the new children. /// /// * (+) will have the minimum parallelization overhead. /// * (-) requires allocation for processing each node. fn collect_extend_orx_rec(root: &Node) -> u64 { fn extend<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { let children: Vec<_> = node.children.iter().filter(filter).collect(); queue.extend(children); } [root].into_par_rec(extend).map(compute).sum() } ================================================ FILE: examples/parallelization_on_tree/run_utils.rs ================================================ use crate::amount_of_work; use std::time::Instant; pub fn timed(name: &'static str, fun: F, log: L) where F: Fn() -> T, L: Fn(T), { println!("> {name}"); let start = Instant::now(); let result = fun(); let elapsed = start.elapsed(); println!(" elapsed = {elapsed:?}"); log(result); println!(); } /// Fibonacci as example computation on each of the node values. pub fn compute(value: u64) -> u64 { (0..*amount_of_work()) .map(|j| { let n = core::hint::black_box(value + j as u64); let mut a = 0; let mut b = 1; for _ in 0..n { let c = a + b; a = b; b = c; } a }) .sum() } ================================================ FILE: examples/parallelization_on_tree/tree.rs ================================================ use crate::node::Node; use rand::Rng; use std::{collections::HashSet, marker::PhantomData, ops::Range}; pub struct Tree(PhantomData); impl Tree { pub fn new_node( num_nodes: usize, degree: Range, data: fn(usize) -> T, rng: &mut impl Rng, ) -> Node { assert!(num_nodes >= 2); let mut leaves = vec![0]; let mut remaining: Vec<_> = (1..num_nodes).collect(); let mut edges = vec![]; let mut out_edges = vec![vec![]; num_nodes]; while !remaining.is_empty() { let leaf_idx = rng.random_range(0..leaves.len()); let leaf = leaves.remove(leaf_idx); let degree = rng.random_range(degree.clone()); match degree == 0 { true => leaves.push(leaf), false => { let children_indices: HashSet<_> = (0..degree) .map(|_| rng.random_range(0..remaining.len())) .collect(); let mut sorted: Vec<_> = children_indices.iter().copied().collect(); sorted.sort(); edges.extend(children_indices.iter().map(|c| (leaf, remaining[*c]))); out_edges[leaf] = children_indices.iter().map(|c| remaining[*c]).collect(); leaves.extend(children_indices.iter().map(|c| remaining[*c])); for idx in sorted.into_iter().rev() { remaining.remove(idx); } } } } create_node(&out_edges, 0, data) } } fn create_node(out_edges: &[Vec], idx: usize, data: fn(usize) -> T) -> Node { let children: Vec<_> = out_edges[idx] .iter() .map(|child_idx| create_node(out_edges, *child_idx, data)) .collect(); let data = data(idx); Node { idx, data, children, } } ================================================ FILE: examples/using_for_each.rs ================================================ use orx_parallel::*; use std::sync::mpsc::channel; // taken from rayon's documentation: // https://docs.rs/rayon/1.10.0/rayon/iter/trait.ParallelIterator.html#method.map fn for_each() { let (sender, receiver) = channel(); (0..5) .into_par() .using_clone(sender) .for_each(|s, x| s.send(x).unwrap()); let mut res: Vec<_> = receiver.iter().collect(); res.sort(); assert_eq!(&res[..], &[0, 1, 2, 3, 4]) } // taken from rayon's documentation: // https://docs.rs/rayon/1.10.0/rayon/iter/trait.ParallelIterator.html#method.map fn map() { let (sender, receiver) = channel(); let a: Vec<_> = (0..5) .into_par() // iterating over i32 .using_clone(sender) .map(|s, x| { s.send(x).unwrap(); // sending i32 values through the channel x // returning i32 }) .collect(); // collecting the returned values into a vector let mut b: Vec<_> = receiver .iter() // iterating over the values in the channel .collect(); // and collecting them b.sort(); assert_eq!(a, b); } fn main() { for_each(); map(); } ================================================ FILE: examples/using_map.rs ================================================ use orx_parallel::*; use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha20Rng; use std::sync::atomic::{AtomicUsize, Ordering}; const N: u64 = 100_000; // just some work fn fibonacci(n: u64) -> u64 { let mut a = 0; let mut b = 1; for _ in 0..n { let c = a + b; a = b; b = c; } a } fn using() -> u64 { let input: Vec = (1..N).collect(); input .into_par() .using(|thread_idx| ChaCha20Rng::seed_from_u64(thread_idx as u64 * 10)) .map(|_, i| fibonacci((i % 50) + 1) % 100) .filter(|rng: &mut ChaCha20Rng, _: &u64| rng.random_bool(0.4)) .map(|rng: &mut ChaCha20Rng, i: u64| rng.random_range(0..i)) .sum() } fn using_clone() -> u64 { let rng = ChaCha20Rng::seed_from_u64(42); let input: Vec = (1..N).collect(); input .into_par() .using_clone(rng) .map(|_, i| fibonacci((i % 50) + 1) % 100) .filter(|rng: &mut ChaCha20Rng, _: &u64| rng.random_bool(0.4)) .map(|rng: &mut ChaCha20Rng, i: u64| rng.random_range(0..i)) .sum() } fn using_clone_while_counting_clones() -> u64 { static COUNTER: AtomicUsize = AtomicUsize::new(0); struct Rng(ChaCha20Rng); unsafe impl Send for Rng {} impl Clone for Rng { fn clone(&self) -> Self { _ = COUNTER.fetch_add(1, Ordering::Relaxed); Self(self.0.clone()) } } let rng = ChaCha20Rng::seed_from_u64(42); let input: Vec = (1..N).collect(); let result = input .into_par() .num_threads(8) .using_clone(Rng(rng)) .map(|_, i| fibonacci((i % 50) + 1) % 100) .filter(|rng: &mut Rng, _: &u64| rng.0.random_bool(0.4)) .map(|rng: &mut Rng, i: u64| rng.0.random_range(0..i)) .sum(); let num_clones = COUNTER.load(Ordering::Relaxed); println!("\n> number of times RNG is cloned (= used threads) = {num_clones}"); result } fn main() { println!("\n\n\n### I. using:"); println!( "* Allows to create the random number generator (RNG) of each thread using a closure that takes the thread index." ); println!("* Hence, we can guarantee to use different RNGs for each thread."); println!("* The following is the example 'using' transformation used in this example."); println!("\n > par.using(|thread_idx| ChaCha20Rng::seed_from_u64(thread_idx as u64 * 10))"); let parallel_result = using(); println!("\n* result = {parallel_result}"); println!("\n\n\n### II. using_clone:"); println!( "* With this call we create and pass in the RNG; then each thread receives a clone of it as in its initial state." ); println!("\n > par.using_clone(my_rng)"); let parallel_result = using_clone(); println!("\n* result = {parallel_result}"); println!("\n\n\n### III. using_clone with a hook to count number of times the RNG is cloned:"); println!( "* The computation is limited to 8 threads; therefore, at most, 8 clones of the RNG must be created." ); println!( "* This example demonstrates that exactly one value is created-or-cloned per thread used." ); let parallel_result = using_clone_while_counting_clones(); println!("{parallel_result}"); println!("\n* result = {parallel_result}"); } ================================================ FILE: examples/using_metrics.rs ================================================ use orx_parallel::*; use std::cell::UnsafeCell; const N: u64 = 10_000_000; const MAX_NUM_THREADS: usize = 8; // just some work fn fibonacci(n: u64) -> u64 { let mut a = 0; let mut b = 1; for _ in 0..n { let c = a + b; a = b; b = c; } a } #[derive(Default, Debug)] struct ThreadMetrics { thread_idx: usize, num_items_handled: usize, handled_42: bool, num_filtered_out: usize, } struct ThreadMetricsWriter<'a> { metrics_ref: &'a mut ThreadMetrics, } struct ComputationMetrics { thread_metrics: UnsafeCell<[ThreadMetrics; MAX_NUM_THREADS]>, } impl ComputationMetrics { fn new() -> Self { let mut thread_metrics: [ThreadMetrics; MAX_NUM_THREADS] = Default::default(); for (i, metrics) in thread_metrics.iter_mut().enumerate().take(MAX_NUM_THREADS) { metrics.thread_idx = i; } Self { thread_metrics: UnsafeCell::new(thread_metrics), } } } unsafe impl Sync for ComputationMetrics {} impl ComputationMetrics { unsafe fn create_for_thread<'a>(&self, thread_idx: usize) -> ThreadMetricsWriter<'a> { // SAFETY: here we create a mutable variable to the thread_idx-th metrics // * If we call this method multiple times with the same index, // we create multiple mutable references to the same ThreadMetrics, // which would lead to a race condition. // * We must make sure that `create_for_thread` is called only once per thread. // * If we use `create_for_thread` within the `using` call to create mutable values // used by the threads, we are certain that the parallel computation // will only call this method once per thread; hence, it will not // cause the race condition. // * On the other hand, we must ensure that we do not call this method // externally. let array = unsafe { &mut *self.thread_metrics.get() }; ThreadMetricsWriter { metrics_ref: &mut array[thread_idx], } } } fn main() { let mut metrics = ComputationMetrics::new(); let input: Vec = (0..N).collect(); let sum = input .par() // SAFETY: we do not call `create_for_thread` externally; // it is safe if it is called only by the parallel computation. // Since we unsafely implement Sync for ComputationMetrics, // we must ensure that ComputationMetrics is not used elsewhere. .using(|t| unsafe { metrics.create_for_thread(t) }) .map(|m: &mut ThreadMetricsWriter<'_>, i| { // collect some useful metrics m.metrics_ref.num_items_handled += 1; m.metrics_ref.handled_42 |= *i == 42; // actual work fibonacci((*i % 50) + 1) % 100 }) .filter(|m, i| { let is_even = i % 2 == 0; if !is_even { m.metrics_ref.num_filtered_out += 1; } is_even }) .num_threads(MAX_NUM_THREADS) .sum(); println!("\nINPUT-LEN = {N}"); println!("SUM = {sum}"); println!("\n\n"); println!("COLLECTED METRICS PER THREAD"); for metrics in metrics.thread_metrics.get_mut().iter() { println!("* {metrics:?}"); } let total_by_metrics: usize = metrics .thread_metrics .get_mut() .iter() .map(|x| x.num_items_handled) .sum(); println!("\n-> total num_items_handled by collected metrics: {total_by_metrics:?}\n"); assert_eq!(N as usize, total_by_metrics); } ================================================ FILE: examples/using_random_walk.rs ================================================ use orx_parallel::*; use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha20Rng; fn random_walk(rng: &mut impl Rng, position: i64, num_steps: usize) -> i64 { (0..num_steps).fold(position, |p, _| random_step(rng, p)) } fn random_step(rng: &mut impl Rng, position: i64) -> i64 { match rng.random_bool(0.5) { true => position + 1, // to right false => position - 1, // to left } } fn input_positions() -> Vec { (-10_000..=10_000).collect() } fn sequential() { let positions = input_positions(); let sum_initial_positions = positions.iter().sum::(); println!("sum_initial_positions = {sum_initial_positions}"); let mut rng = ChaCha20Rng::seed_from_u64(42); let final_positions: Vec<_> = positions .iter() .copied() .map(|position| random_walk(&mut rng, position, 100)) .collect(); let sum_final_positions = final_positions.iter().sum::(); println!("sum_final_positions = {sum_final_positions}"); } fn parallel() { let positions = input_positions(); let sum_initial_positions = positions.iter().sum::(); println!("sum_initial_positions = {sum_initial_positions}"); let final_positions: Vec<_> = positions .par() .copied() .using(|t_idx| ChaCha20Rng::seed_from_u64(42 * t_idx as u64)) .map(|rng, position| random_walk(rng, position, 100)) .collect(); let sum_final_positions = final_positions.iter().sum::(); println!("sum_final_positions = {sum_final_positions}"); } fn main() { println!("\n\nSEQUENTIAL"); sequential(); println!("\n\nPARALLEL"); parallel(); } ================================================ FILE: examples/utils/benchmark_utils.rs ================================================ #![allow(dead_code)] use std::{ fmt::Debug, hint::black_box, time::{Duration, SystemTime}, }; // reduce fn timed_reduce(num_repetitions: usize, expected_output: &Option, fun: F) -> Duration where F: Fn() -> O, O: PartialEq + Debug, { if let Some(expected_output) = expected_output.as_ref() { let result = fun(); assert_eq!(&result, expected_output); } // warm up for _ in 0..10 { let _ = black_box(fun()); } // measurement let now = SystemTime::now(); for _ in 0..num_repetitions { let result = black_box(fun()); if let Some(expected_output) = expected_output.as_ref() { assert_eq!(&result, expected_output); } } now.elapsed().unwrap() } type Computation<'a, O> = (&'a str, Box O>); pub fn timed_reduce_all( benchmark_name: &str, num_repetitions: usize, expected_output: Option, computations: &[Computation<'_, O>], ) where O: PartialEq + Debug + Clone, { println!("\n{} {} {}", "#".repeat(10), benchmark_name, "#".repeat(10)); for (name, fun) in computations { let duration = timed_reduce(num_repetitions, &expected_output, fun); println!("{:>10} : {:?}", name, duration); } println!("{}\n", "#".repeat(10 + 10 + 2 + benchmark_name.len())); } // collect fn timed_collect(num_repetitions: usize, expected_output: &[O], fun: F) -> Duration where F: Fn() -> Out, Out: IntoIterator, O: PartialEq + Debug, { let result = fun(); assert_eq!(result.into_iter().collect::>(), expected_output); // warm up for _ in 0..10 { let _ = black_box(fun()); } // measurement let now = SystemTime::now(); for _ in 0..num_repetitions { let _ = black_box(fun()); } now.elapsed().unwrap() } pub fn timed_collect_all( benchmark_name: &str, num_repetitions: usize, expected_output: &[O], computations: &[Computation<'_, Out>], ) where Out: IntoIterator, O: PartialEq + Debug, { println!("\n{} {} {}", "#".repeat(10), benchmark_name, "#".repeat(10)); for (name, fun) in computations { let duration = timed_collect(num_repetitions, expected_output, fun); println!("{:>10} : {:?}", name, duration); } println!("{}\n", "#".repeat(10 + 10 + 2 + benchmark_name.len())); } ================================================ FILE: examples/utils/mod.rs ================================================ #![allow(unused_imports)] mod benchmark_utils; pub use benchmark_utils::*; ================================================ FILE: src/collect_into/collect.rs ================================================ use crate::Params; use crate::executor::parallel_compute as prc; use crate::generic_values::runner_results::{ Fallibility, Infallible, ParallelCollect, ParallelCollectArbitrary, Stop, }; use crate::runner::{NumSpawned, ParallelRunner}; use crate::{IterationOrder, generic_values::Values}; use orx_concurrent_iter::ConcurrentIter; use orx_fixed_vec::IntoConcurrentPinnedVec; pub fn map_collect_into( orchestrator: R, params: Params, iter: I, map1: M1, pinned_vec: P, ) -> (NumSpawned, P) where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, O: Send, P: IntoConcurrentPinnedVec, { match (params.is_sequential(), params.iteration_order) { (true, _) => ( NumSpawned::zero(), map_collect_into_seq(iter, map1, pinned_vec), ), #[cfg(test)] (false, IterationOrder::Arbitrary) => { prc::collect_arbitrary::m(orchestrator, params, iter, map1, pinned_vec) } (false, _) => prc::collect_ordered::m(orchestrator, params, iter, map1, pinned_vec), } } fn map_collect_into_seq(iter: I, map1: M1, mut pinned_vec: P) -> P where I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, O: Send, P: IntoConcurrentPinnedVec, { let iter = iter.into_seq_iter(); for i in iter { pinned_vec.push(map1(i)); } pinned_vec } pub fn xap_collect_into( orchestrator: R, params: Params, iter: I, xap1: X1, pinned_vec: P, ) -> (NumSpawned, P) where R: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { match (params.is_sequential(), params.iteration_order) { (true, _) => ( NumSpawned::zero(), xap_collect_into_seq(iter, xap1, pinned_vec), ), (false, IterationOrder::Arbitrary) => { let (num_threads, result) = prc::collect_arbitrary::x(orchestrator, params, iter, xap1, pinned_vec); let pinned_vec = match result { ParallelCollectArbitrary::AllOrUntilWhileCollected { pinned_vec } => pinned_vec, }; (num_threads, pinned_vec) } (false, IterationOrder::Ordered) => { let (num_threads, result) = prc::collect_ordered::x(orchestrator, params, iter, xap1, pinned_vec); let pinned_vec = match result { ParallelCollect::AllCollected { pinned_vec } => pinned_vec, ParallelCollect::StoppedByWhileCondition { pinned_vec, stopped_idx: _, } => pinned_vec, }; (num_threads, pinned_vec) } } } fn xap_collect_into_seq(iter: I, xap1: X1, mut pinned_vec: P) -> P where I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { let iter = iter.into_seq_iter(); for i in iter { let vt = xap1(i); let done = vt.push_to_pinned_vec(&mut pinned_vec); if Vo::sequential_push_to_stop(done).is_some() { break; } } pinned_vec } pub fn xap_try_collect_into( orchestrator: R, params: Params, iter: I, xap1: X1, pinned_vec: P, ) -> ( NumSpawned, Result::Error>, ) where R: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { match (params.is_sequential(), params.iteration_order) { (true, _) => ( NumSpawned::zero(), xap_try_collect_into_seq(iter, xap1, pinned_vec), ), (false, IterationOrder::Arbitrary) => { let (nt, result) = prc::collect_arbitrary::x(orchestrator, params, iter, xap1, pinned_vec); (nt, result.into_result()) } (false, IterationOrder::Ordered) => { let (nt, result) = prc::collect_ordered::x(orchestrator, params, iter, xap1, pinned_vec); (nt, result.into_result()) } } } fn xap_try_collect_into_seq( iter: I, xap1: X1, mut pinned_vec: P, ) -> Result::Error> where I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { let iter = iter.into_seq_iter(); for i in iter { let vt = xap1(i); let done = vt.push_to_pinned_vec(&mut pinned_vec); if let Some(stop) = Vo::sequential_push_to_stop(done) { match stop { Stop::DueToWhile => return Ok(pinned_vec), Stop::DueToError { error } => return Err(error), } } } Ok(pinned_vec) } ================================================ FILE: src/collect_into/fixed_vec.rs ================================================ use super::par_collect_into::ParCollectIntoCore; use crate::Params; use crate::generic_values::runner_results::{Fallibility, Infallible}; use crate::generic_values::{TransformableValues, Values}; use crate::runner::ParallelRunner; use alloc::vec::Vec; use orx_concurrent_iter::ConcurrentIter; use orx_fixed_vec::FixedVec; #[cfg(test)] use orx_pinned_vec::PinnedVec; impl ParCollectIntoCore for FixedVec where O: Send + Sync, { type BridgePinnedVec = Self; fn empty(iter_len: Option) -> Self { let vec = as ParCollectIntoCore<_>>::empty(iter_len); vec.into() } fn m_collect_into(self, orchestrator: R, params: Params, iter: I, map1: M1) -> Self where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, O: Send, { let vec = Vec::from(self); FixedVec::from(vec.m_collect_into(orchestrator, params, iter, map1)) } fn x_collect_into( self, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Self where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(I::Item) -> Vo + Sync, { let vec = Vec::from(self); FixedVec::from(vec.x_collect_into(orchestrator, params, iter, xap1)) } fn x_try_collect_into( self, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Result::Error> where R: ParallelRunner, I: ConcurrentIter, X1: Fn(I::Item) -> Vo + Sync, Vo: Values, { let vec = Vec::from(self); vec.x_try_collect_into(orchestrator, params, iter, xap1) .map(FixedVec::from) } // test #[cfg(test)] fn length(&self) -> usize { self.len() } } ================================================ FILE: src/collect_into/mod.rs ================================================ pub(crate) mod collect; mod fixed_vec; mod par_collect_into; mod split_vec; pub(crate) mod utils; mod vec; pub use par_collect_into::ParCollectInto; pub(crate) use par_collect_into::ParCollectIntoCore; ================================================ FILE: src/collect_into/par_collect_into.rs ================================================ use crate::Params; use crate::generic_values::runner_results::{Fallibility, Infallible}; use crate::generic_values::{TransformableValues, Values}; use crate::runner::ParallelRunner; use crate::using::UParCollectIntoCore; use orx_concurrent_iter::ConcurrentIter; use orx_iterable::Collection; use orx_pinned_vec::IntoConcurrentPinnedVec; pub trait ParCollectIntoCore: Collection { type BridgePinnedVec: IntoConcurrentPinnedVec; fn empty(iter_len: Option) -> Self; fn m_collect_into(self, orchestrator: R, params: Params, iter: I, map1: M1) -> Self where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync; fn x_collect_into( self, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Self where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(I::Item) -> Vo + Sync; fn x_try_collect_into( self, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Result::Error> where R: ParallelRunner, I: ConcurrentIter, X1: Fn(I::Item) -> Vo + Sync, Vo: Values, Self: Sized; // test #[cfg(test)] fn length(&self) -> usize; #[cfg(test)] fn is_empty(&self) -> bool { self.length() == 0 } #[cfg(test)] fn is_equal_to<'a>(&self, b: impl orx_iterable::Iterable) -> bool where O: PartialEq + 'a, { let mut b = b.iter(); for x in self.iter() { match b.next() { Some(y) if x != y => return false, None => return false, _ => {} } } b.next().is_none() } #[cfg(test)] fn is_equal_to_ref(&self, b: impl orx_iterable::Iterable) -> bool where O: PartialEq, { let mut b = b.iter(); for x in self.iter() { match b.next() { Some(y) if x != &y => return false, None => return false, _ => {} } } b.next().is_none() } } /// Collection types into which outputs of a parallel computations can be collected into. pub trait ParCollectInto: ParCollectIntoCore + UParCollectIntoCore {} impl ParCollectInto for C where C: ParCollectIntoCore + UParCollectIntoCore {} ================================================ FILE: src/collect_into/split_vec.rs ================================================ use super::collect::{map_collect_into, xap_collect_into, xap_try_collect_into}; use super::par_collect_into::ParCollectIntoCore; use crate::Params; use crate::collect_into::utils::split_vec_reserve; use crate::generic_values::runner_results::{Fallibility, Infallible}; use crate::generic_values::{TransformableValues, Values}; use crate::runner::ParallelRunner; use orx_concurrent_iter::ConcurrentIter; #[cfg(test)] use orx_pinned_vec::PinnedVec; use orx_split_vec::{GrowthWithConstantTimeAccess, PseudoDefault, SplitVec}; impl ParCollectIntoCore for SplitVec where O: Send + Sync, G: GrowthWithConstantTimeAccess, Self: PseudoDefault, { type BridgePinnedVec = Self; fn empty(iter_len: Option) -> Self { let mut vec = Self::pseudo_default(); split_vec_reserve(&mut vec, false, iter_len); vec } fn m_collect_into( mut self, orchestrator: R, params: Params, iter: I, map1: M1, ) -> Self where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, O: Send, { split_vec_reserve(&mut self, params.is_sequential(), iter.try_get_len()); let (_, pinned_vec) = map_collect_into(orchestrator, params, iter, map1, self); pinned_vec } fn x_collect_into( mut self, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Self where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(I::Item) -> Vo + Sync, { split_vec_reserve(&mut self, params.is_sequential(), iter.try_get_len()); let (_num_spawned, pinned_vec) = xap_collect_into(orchestrator, params, iter, xap1, self); pinned_vec } fn x_try_collect_into( mut self, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Result::Error> where R: ParallelRunner, I: ConcurrentIter, X1: Fn(I::Item) -> Vo + Sync, Vo: Values, Self: Sized, { split_vec_reserve(&mut self, params.is_sequential(), iter.try_get_len()); let (_num_spawned, result) = xap_try_collect_into(orchestrator, params, iter, xap1, self); result } // test #[cfg(test)] fn length(&self) -> usize { self.len() } } ================================================ FILE: src/collect_into/utils.rs ================================================ use alloc::vec::Vec; use orx_pinned_vec::PinnedVec; use orx_split_vec::{GrowthWithConstantTimeAccess, SplitVec}; pub fn extend_vec_from_split( mut initial_vec: Vec, collected_split_vec: SplitVec, ) -> Vec where G: GrowthWithConstantTimeAccess, { match initial_vec.len() { 0 => collected_split_vec.to_vec(), _ => { initial_vec.reserve(collected_split_vec.len()); initial_vec.extend(collected_split_vec); initial_vec } } } pub fn split_vec_reserve( split_vec: &mut SplitVec, is_sequential: bool, iter_len: Option, ) { let len_to_extend = match (is_sequential, iter_len) { (true, _) => None, // not required to concurrent reserve when seq (false, x) => x, }; match len_to_extend { None => { let capacity_bound = split_vec.capacity_bound(); split_vec.reserve_maximum_concurrent_capacity(capacity_bound) } Some(len) => split_vec.reserve_maximum_concurrent_capacity(split_vec.len() + len), }; } ================================================ FILE: src/collect_into/vec.rs ================================================ use super::par_collect_into::ParCollectIntoCore; use crate::Params; use crate::collect_into::collect::map_collect_into; use crate::collect_into::utils::extend_vec_from_split; use crate::generic_values::runner_results::{Fallibility, Infallible}; use crate::generic_values::{TransformableValues, Values}; use crate::runner::ParallelRunner; use alloc::vec::Vec; use orx_concurrent_iter::ConcurrentIter; use orx_fixed_vec::FixedVec; use orx_split_vec::SplitVec; impl ParCollectIntoCore for Vec where O: Send + Sync, { type BridgePinnedVec = FixedVec; fn empty(iter_len: Option) -> Self { match iter_len { Some(len) => Vec::with_capacity(len), None => Vec::new(), } } fn m_collect_into( mut self, orchestrator: R, params: Params, iter: I, map1: M1, ) -> Self where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, O: Send, { match iter.try_get_len() { None => { let split_vec = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let split_vec = split_vec.m_collect_into(orchestrator, params, iter, map1); extend_vec_from_split(self, split_vec) } Some(len) => { self.reserve(len); let fixed_vec = FixedVec::from(self); let (_, fixed_vec) = map_collect_into(orchestrator, params, iter, map1, fixed_vec); Vec::from(fixed_vec) } } } fn x_collect_into( self, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Self where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(I::Item) -> Vo + Sync, { let split_vec = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let split_vec = split_vec.x_collect_into(orchestrator, params, iter, xap1); extend_vec_from_split(self, split_vec) } fn x_try_collect_into( self, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Result::Error> where R: ParallelRunner, I: ConcurrentIter, X1: Fn(I::Item) -> Vo + Sync, Vo: Values, Self: Sized, { let split_vec = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let result = split_vec.x_try_collect_into(orchestrator, params, iter, xap1); result.map(|split_vec| extend_vec_from_split(self, split_vec)) } // test #[cfg(test)] fn length(&self) -> usize { self.len() } } ================================================ FILE: src/computational_variants/fallible_option.rs ================================================ use crate::{ ChunkSize, IterationOrder, NumThreads, ParCollectInto, ParIterResult, par_iter_option::{ParIterOption, ResultIntoOption}, runner::{DefaultRunner, ParallelRunner}, }; use core::marker::PhantomData; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with None. pub struct ParOption where R: ParallelRunner, F: ParIterResult, { par: F, phantom: PhantomData<(T, R)>, } impl ParOption where R: ParallelRunner, F: ParIterResult, { pub(crate) fn new(par: F) -> Self { Self { par, phantom: PhantomData, } } } impl ParIterOption for ParOption where R: ParallelRunner, F: ParIterResult, { type Item = T; // params transformations fn num_threads(self, num_threads: impl Into) -> Self { Self::new(self.par.num_threads(num_threads)) } fn chunk_size(self, chunk_size: impl Into) -> Self { Self::new(self.par.chunk_size(chunk_size)) } fn iteration_order(self, order: IterationOrder) -> Self { Self::new(self.par.iteration_order(order)) } fn with_runner( self, orchestrator: Q, ) -> impl ParIterOption { ParOption::new(self.par.with_runner(orchestrator)) } // computation transformations fn map(self, map: Map) -> impl ParIterOption where Map: Fn(Self::Item) -> Out + Sync + Clone, Out: Send, { ParOption::new(self.par.map(map)) } fn filter(self, filter: Filter) -> impl ParIterOption where Self: Sized, Filter: Fn(&Self::Item) -> bool + Sync + Clone, Self::Item: Send, { ParOption::new(self.par.filter(filter)) } fn flat_map(self, flat_map: FlatMap) -> impl ParIterOption where Self: Sized, IOut: IntoIterator, IOut::Item: Send, FlatMap: Fn(Self::Item) -> IOut + Sync + Clone, { ParOption::new(self.par.flat_map(flat_map)) } fn filter_map(self, filter_map: FilterMap) -> impl ParIterOption where Self: Sized, FilterMap: Fn(Self::Item) -> Option + Sync + Clone, Out: Send, { ParOption::new(self.par.filter_map(filter_map)) } fn inspect(self, operation: Operation) -> impl ParIterOption where Self: Sized, Operation: Fn(&Self::Item) + Sync + Clone, Self::Item: Send, { ParOption::new(self.par.inspect(operation)) } // collect fn collect_into(self, output: C) -> Option where Self::Item: Send, C: ParCollectInto, { self.par.collect_into(output).into_option() } fn collect(self) -> Option where Self::Item: Send, C: ParCollectInto, { self.par.collect().into_option() } // reduce fn reduce(self, reduce: Reduce) -> Option> where Self::Item: Send, Reduce: Fn(Self::Item, Self::Item) -> Self::Item + Sync, { self.par.reduce(reduce).into_option() } // early exit fn first(self) -> Option> where Self::Item: Send, { self.par.first().into_option() } } ================================================ FILE: src/computational_variants/fallible_result/map_result.rs ================================================ use crate::computational_variants::ParMap; use crate::executor::parallel_compute as prc; use crate::par_iter_result::{IntoResult, ParIterResult}; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::{IterationOrder, ParCollectInto, ParIter}; use core::marker::PhantomData; use orx_concurrent_iter::ConcurrentIter; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with an error. pub struct ParMapResult where R: ParallelRunner, I: ConcurrentIter, O: IntoResult, M1: Fn(I::Item) -> O + Sync, { par: ParMap, phantom: PhantomData<(T, E)>, } impl ParMapResult where R: ParallelRunner, I: ConcurrentIter, O: IntoResult, M1: Fn(I::Item) -> O + Sync, { pub(crate) fn new(par: ParMap) -> Self { Self { par, phantom: PhantomData, } } } impl ParIterResult for ParMapResult where R: ParallelRunner, I: ConcurrentIter, O: IntoResult, M1: Fn(I::Item) -> O + Sync, { type Item = T; type Err = E; type RegularItem = O; type RegularParIter = ParMap; fn con_iter_len(&self) -> Option { self.par.con_iter().try_get_len() } fn into_regular_par(self) -> Self::RegularParIter { self.par } fn from_regular_par(regular_par: Self::RegularParIter) -> Self { Self { par: regular_par, phantom: PhantomData, } } // params transformations fn with_runner( self, orchestrator: Q, ) -> impl ParIterResult { let (_, params, iter, m1) = self.par.destruct(); ParMapResult { par: ParMap::new(orchestrator, params, iter, m1), phantom: PhantomData, } } // collect fn collect_into(self, output: C) -> Result where C: ParCollectInto, Self::Item: Send, Self::Err: Send, { let (orchestrator, params, iter, m1) = self.par.destruct(); let x1 = |i: I::Item| m1(i).into_result(); output.x_try_collect_into(orchestrator, params, iter, x1) } // reduce fn reduce(self, reduce: Reduce) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, Reduce: Fn(Self::Item, Self::Item) -> Self::Item + Sync, { let (orchestrator, params, iter, m1) = self.par.destruct(); let x1 = |i: I::Item| m1(i).into_result(); prc::reduce::x(orchestrator, params, iter, x1, reduce).1 } // early exit fn first(self) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, { let (orchestrator, params, iter, m1) = self.par.destruct(); let x1 = |i: I::Item| m1(i).into_result(); match params.iteration_order { IterationOrder::Ordered => { let (_, result) = prc::next::x(orchestrator, params, iter, x1); result.map(|x| x.map(|y| y.1)) } IterationOrder::Arbitrary => { let (_, result) = prc::next_any::x(orchestrator, params, iter, x1); result } } } } ================================================ FILE: src/computational_variants/fallible_result/mod.rs ================================================ mod map_result; mod par_result; mod xap_result; pub use map_result::ParMapResult; pub use par_result::ParResult; pub use xap_result::ParXapResult; ================================================ FILE: src/computational_variants/fallible_result/par_result.rs ================================================ use crate::computational_variants::Par; use crate::executor::parallel_compute as prc; use crate::par_iter_result::{IntoResult, ParIterResult}; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::{IterationOrder, ParCollectInto, ParIter}; use core::marker::PhantomData; use orx_concurrent_iter::ConcurrentIter; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with an error. pub struct ParResult where R: ParallelRunner, I: ConcurrentIter, I::Item: IntoResult, { par: Par, phantom: PhantomData<(T, E)>, } impl ParResult where R: ParallelRunner, I: ConcurrentIter, I::Item: IntoResult, { pub(crate) fn new(par: Par) -> Self { Self { par, phantom: PhantomData, } } } impl ParIterResult for ParResult where R: ParallelRunner, I: ConcurrentIter, I::Item: IntoResult, { type Item = T; type Err = E; type RegularItem = I::Item; type RegularParIter = Par; fn con_iter_len(&self) -> Option { self.par.con_iter().try_get_len() } fn into_regular_par(self) -> Self::RegularParIter { self.par } fn from_regular_par(regular_par: Self::RegularParIter) -> Self { Self { par: regular_par, phantom: PhantomData, } } // params transformations fn with_runner( self, orchestrator: Q, ) -> impl ParIterResult { let (_, params, iter) = self.par.destruct(); ParResult { par: Par::new(orchestrator, params, iter), phantom: PhantomData, } } // collect fn collect_into(self, output: C) -> Result where C: ParCollectInto, Self::Item: Send, Self::Err: Send, { let (orchestrator, params, iter) = self.par.destruct(); let x1 = |i: I::Item| i.into_result(); output.x_try_collect_into(orchestrator, params, iter, x1) } // reduce fn reduce(self, reduce: Reduce) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, Reduce: Fn(Self::Item, Self::Item) -> Self::Item + Sync, { let (orchestrator, params, iter) = self.par.destruct(); let x1 = |i: I::Item| i.into_result(); prc::reduce::x(orchestrator, params, iter, x1, reduce).1 } // early exit fn first(self) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, { let (orchestrator, params, iter) = self.par.destruct(); let x1 = |i: I::Item| i.into_result(); match params.iteration_order { IterationOrder::Ordered => { let (_, result) = prc::next::x(orchestrator, params, iter, x1); result.map(|x| x.map(|y| y.1)) } IterationOrder::Arbitrary => { let (_, result) = prc::next_any::x(orchestrator, params, iter, x1); result } } } } ================================================ FILE: src/computational_variants/fallible_result/xap_result.rs ================================================ use crate::computational_variants::ParXap; use crate::executor::parallel_compute as prc; use crate::generic_values::TransformableValues; use crate::generic_values::runner_results::Infallible; use crate::par_iter_result::{IntoResult, ParIterResult}; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::{IterationOrder, ParCollectInto, Params}; use core::marker::PhantomData; use orx_concurrent_iter::ConcurrentIter; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with an error. pub struct ParXapResult where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, Vo::Item: IntoResult, X1: Fn(I::Item) -> Vo + Sync, { orchestrator: R, params: Params, iter: I, xap1: X1, phantom: PhantomData<(T, E)>, } impl ParXapResult where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, Vo::Item: IntoResult, X1: Fn(I::Item) -> Vo + Sync, { pub(crate) fn new(orchestrator: R, params: Params, iter: I, xap1: X1) -> Self { Self { orchestrator, params, iter, xap1, phantom: PhantomData, } } fn destruct(self) -> (R, Params, I, X1) { (self.orchestrator, self.params, self.iter, self.xap1) } } impl ParIterResult for ParXapResult where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, Vo::Item: IntoResult, X1: Fn(I::Item) -> Vo + Sync, { type Item = T; type Err = E; type RegularItem = Vo::Item; type RegularParIter = ParXap; fn con_iter_len(&self) -> Option { self.iter.try_get_len() } fn into_regular_par(self) -> Self::RegularParIter { let (orchestrator, params, iter, x1) = self.destruct(); ParXap::new(orchestrator, params, iter, x1) } fn from_regular_par(regular_par: Self::RegularParIter) -> Self { let (orchestrator, params, iter, x1) = regular_par.destruct(); Self::new(orchestrator, params, iter, x1) } // params transformations fn with_runner( self, orchestrator: Q, ) -> impl ParIterResult { let (_, params, iter, x1) = self.destruct(); ParXapResult::new(orchestrator, params, iter, x1) } // collect fn collect_into(self, output: C) -> Result where C: ParCollectInto, Self::Item: Send, Self::Err: Send, { let (orchestrator, params, iter, x1) = self.destruct(); let x1 = |i: I::Item| x1(i).map_while_ok(|x| x.into_result()); output.x_try_collect_into(orchestrator, params, iter, x1) } // reduce fn reduce(self, reduce: Reduce) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, Reduce: Fn(Self::Item, Self::Item) -> Self::Item + Sync, { let (orchestrator, params, iter, x1) = self.destruct(); let x1 = |i: I::Item| x1(i).map_while_ok(|x| x.into_result()); prc::reduce::x(orchestrator, params, iter, x1, reduce).1 } // early exit fn first(self) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, { let (orchestrator, params, iter, x1) = self.destruct(); let x1 = |i: I::Item| x1(i).map_while_ok(|x| x.into_result()); match params.iteration_order { IterationOrder::Ordered => { let (_, result) = prc::next::x(orchestrator, params, iter, x1); result.map(|x| x.map(|y| y.1)) } IterationOrder::Arbitrary => { let (_, result) = prc::next_any::x(orchestrator, params, iter, x1); result } } } } ================================================ FILE: src/computational_variants/map.rs ================================================ use super::xap::ParXap; use crate::computational_variants::fallible_result::ParMapResult; use crate::executor::parallel_compute as prc; use crate::generic_values::{Vector, WhilstAtom}; use crate::par_iter_result::IntoResult; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::{UParMap, UsingClone, UsingFun}; use crate::{ChunkSize, IterationOrder, NumThreads, ParCollectInto, ParEnumerate, ParIter, Params}; use crate::{ParIterResult, ParIterUsing}; use orx_concurrent_iter::ConcurrentIter; /// A parallel iterator that maps inputs. pub struct ParMap where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, { orchestrator: R, params: Params, iter: I, map1: M1, } impl ParMap where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, { pub(crate) fn new(orchestrator: R, params: Params, iter: I, map1: M1) -> Self { Self { orchestrator, params, iter, map1, } } pub(crate) fn destruct(self) -> (R, Params, I, M1) { (self.orchestrator, self.params, self.iter, self.map1) } } unsafe impl Send for ParMap where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, { } unsafe impl Sync for ParMap where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, { } impl ParIter for ParMap where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, { type Item = O; fn con_iter(&self) -> &impl ConcurrentIter { &self.iter } fn params(&self) -> Params { self.params } // params transformations fn num_threads(mut self, num_threads: impl Into) -> Self { self.params = self.params.with_num_threads(num_threads); self } fn chunk_size(mut self, chunk_size: impl Into) -> Self { self.params = self.params.with_chunk_size(chunk_size); self } fn iteration_order(mut self, collect: IterationOrder) -> Self { self.params = self.params.with_collect_ordering(collect); self } fn with_runner(self, orchestrator: Q) -> impl ParIter { let (_, params, iter, map) = self.destruct(); ParMap::new(orchestrator, params, iter, map) } // using transformations fn using<'using, U, F>( self, using: F, ) -> impl ParIterUsing<'using, UsingFun, R, Item = >::Item> where U: 'using, F: Fn(usize) -> U + Sync, { let using = UsingFun::new(using); let (orchestrator, params, iter, x1) = self.destruct(); let m1 = move |_: &mut U, t: I::Item| x1(t); UParMap::new(using, orchestrator, params, iter, m1) } fn using_clone( self, value: U, ) -> impl ParIterUsing<'static, UsingClone, R, Item = >::Item> where U: Clone + 'static, { let using = UsingClone::new(value); let (orchestrator, params, iter, x1) = self.destruct(); let m1 = move |_: &mut U, t: I::Item| x1(t); UParMap::new(using, orchestrator, params, iter, m1) } // computation transformations fn map(self, map: Map) -> impl ParIter where Map: Fn(Self::Item) -> Out + Sync, { let (orchestrator, params, iter, m1) = self.destruct(); let m1 = move |x| map(m1(x)); ParMap::new(orchestrator, params, iter, m1) } fn filter(self, filter: Filter) -> impl ParIter where Filter: Fn(&Self::Item) -> bool + Sync, { let (orchestrator, params, iter, m1) = self.destruct(); let x1 = move |i: I::Item| { let value = m1(i); filter(&value).then_some(value) }; ParXap::new(orchestrator, params, iter, x1) } fn flat_map(self, flat_map: FlatMap) -> impl ParIter where IOut: IntoIterator, FlatMap: Fn(Self::Item) -> IOut + Sync, { let (orchestrator, params, iter, m1) = self.destruct(); let x1 = move |i: I::Item| Vector(flat_map(m1(i))); ParXap::new(orchestrator, params, iter, x1) } fn filter_map(self, filter_map: FilterMap) -> impl ParIter where FilterMap: Fn(Self::Item) -> Option + Sync, { let (orchestrator, params, iter, m1) = self.destruct(); let x1 = move |i: I::Item| filter_map(m1(i)); ParXap::new(orchestrator, params, iter, x1) } fn take_while(self, take_while: While) -> impl ParIter where While: Fn(&Self::Item) -> bool + Sync, { let (orchestrator, params, iter, m1) = self.destruct(); let x1 = move |value: I::Item| WhilstAtom::new(m1(value), &take_while); ParXap::new(orchestrator, params, iter, x1) } fn into_fallible_result(self) -> impl ParIterResult where Self::Item: IntoResult, { ParMapResult::new(self) } // collect fn collect_into(self, output: C) -> C where C: ParCollectInto, { let (orchestrator, params, iter, m1) = self.destruct(); output.m_collect_into(orchestrator, params, iter, m1) } // reduce fn reduce(self, reduce: Reduce) -> Option where Self::Item: Send, Reduce: Fn(Self::Item, Self::Item) -> Self::Item + Sync, { let (orchestrator, params, iter, m1) = self.destruct(); prc::reduce::m(orchestrator, params, iter, m1, reduce).1 } // early exit fn first(self) -> Option where Self::Item: Send, { let (orchestrator, params, iter, m1) = self.destruct(); match params.iteration_order { IterationOrder::Ordered => prc::next::m(orchestrator, params, iter, m1).1, IterationOrder::Arbitrary => prc::next_any::m(orchestrator, params, iter, m1).1, } } } impl ParEnumerate for ParMap where R: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, { fn enumerate(self) -> impl ParIter { let (orchestrator, params, iter, m1) = self.destruct(); let x1 = move |(x, i): (usize, I::Item)| (x, m1(i)); ParMap::new(orchestrator, params, iter.enumerate(), x1) } } ================================================ FILE: src/computational_variants/mod.rs ================================================ #[cfg(test)] mod tests; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with None. pub mod fallible_option; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with an error. pub mod fallible_result; mod map; mod par; mod xap; pub use map::ParMap; pub use par::Par; pub use xap::ParXap; ================================================ FILE: src/computational_variants/par.rs ================================================ use super::{map::ParMap, xap::ParXap}; use crate::computational_variants::fallible_result::ParResult; use crate::executor::parallel_compute as prc; use crate::generic_values::{Vector, WhilstAtom}; use crate::par_iter_result::IntoResult; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::{UPar, UsingClone, UsingFun}; use crate::{ ChunkSize, IterationOrder, NumThreads, ParCollectInto, ParIter, Params, default_fns::map_self, }; use crate::{IntoParIter, ParEnumerate, ParIterResult, ParIterUsing}; use orx_concurrent_iter::chain::ChainKnownLenI; use orx_concurrent_iter::{ConcurrentIter, ExactSizeConcurrentIter}; /// A parallel iterator. pub struct Par where R: ParallelRunner, I: ConcurrentIter, { orchestrator: R, params: Params, iter: I, } impl Par where R: ParallelRunner, I: ConcurrentIter, { pub(crate) fn new(orchestrator: R, params: Params, iter: I) -> Self { Self { orchestrator, iter, params, } } pub(crate) fn destruct(self) -> (R, Params, I) { (self.orchestrator, self.params, self.iter) } pub(crate) fn orchestrator(&self) -> &R { &self.orchestrator } } unsafe impl Send for Par where R: ParallelRunner, I: ConcurrentIter, { } unsafe impl Sync for Par where R: ParallelRunner, I: ConcurrentIter, { } impl ParIter for Par where R: ParallelRunner, I: ConcurrentIter, { type Item = I::Item; fn con_iter(&self) -> &impl ConcurrentIter { &self.iter } fn params(&self) -> Params { self.params } // params transformations fn num_threads(mut self, num_threads: impl Into) -> Self { self.params = self.params.with_num_threads(num_threads); self } fn chunk_size(mut self, chunk_size: impl Into) -> Self { self.params = self.params.with_chunk_size(chunk_size); self } fn iteration_order(mut self, collect: IterationOrder) -> Self { self.params = self.params.with_collect_ordering(collect); self } fn with_runner(self, orchestrator: Q) -> impl ParIter { Par::new(orchestrator, self.params, self.iter) } // using transformations fn using<'using, U, F>( self, using: F, ) -> impl ParIterUsing<'using, UsingFun, R, Item = >::Item> where U: 'using, F: Fn(usize) -> U + Sync, { let using = UsingFun::new(using); let (orchestrator, params, iter) = self.destruct(); UPar::new(using, orchestrator, params, iter) } fn using_clone( self, value: U, ) -> impl ParIterUsing<'static, UsingClone, R, Item = >::Item> where U: Clone + 'static, { let using = UsingClone::new(value); let (orchestrator, params, iter) = self.destruct(); UPar::new(using, orchestrator, params, iter) } // computation transformations fn map(self, map: Map) -> impl ParIter where Map: Fn(Self::Item) -> Out + Sync, { let (orchestrator, params, iter) = self.destruct(); ParMap::new(orchestrator, params, iter, map) } fn filter(self, filter: Filter) -> impl ParIter where Filter: Fn(&Self::Item) -> bool + Sync, { let (orchestrator, params, iter) = self.destruct(); let x1 = move |i: Self::Item| filter(&i).then_some(i); ParXap::new(orchestrator, params, iter, x1) } fn flat_map(self, flat_map: FlatMap) -> impl ParIter where IOut: IntoIterator, FlatMap: Fn(Self::Item) -> IOut + Sync, { let (orchestrator, params, iter) = self.destruct(); let x1 = move |i: Self::Item| Vector(flat_map(i)); ParXap::new(orchestrator, params, iter, x1) } fn filter_map(self, filter_map: FilterMap) -> impl ParIter where FilterMap: Fn(Self::Item) -> Option + Sync, { let (orchestrator, params, iter) = self.destruct(); ParXap::new(orchestrator, params, iter, filter_map) } fn take_while(self, take_while: While) -> impl ParIter where While: Fn(&Self::Item) -> bool + Sync, { let (orchestrator, params, iter) = self.destruct(); let x1 = move |value: Self::Item| WhilstAtom::new(value, &take_while); ParXap::new(orchestrator, params, iter, x1) } fn into_fallible_result(self) -> impl ParIterResult where Self::Item: IntoResult, { ParResult::new(self) } // collect fn collect_into(self, output: C) -> C where C: ParCollectInto, { let (orchestrator, params, iter) = self.destruct(); output.m_collect_into(orchestrator, params, iter, map_self) } // reduce fn reduce(self, reduce: Reduce) -> Option where Self::Item: Send, Reduce: Fn(Self::Item, Self::Item) -> Self::Item + Sync, { let (orchestrator, params, iter) = self.destruct(); prc::reduce::m(orchestrator, params, iter, map_self, reduce).1 } // early exit fn first(self) -> Option { let (orchestrator, params, iter) = self.destruct(); match params.iteration_order { IterationOrder::Ordered => prc::next::m(orchestrator, params, iter, map_self).1, IterationOrder::Arbitrary => prc::next_any::m(orchestrator, params, iter, map_self).1, } } } impl Par where R: ParallelRunner, I: ConcurrentIter, { /// Creates a chain of this and `other` parallel iterators. /// /// The first iterator is required to have a known length for chaining. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec!['a', 'b', 'c']; // with exact len /// let b = vec!['d', 'e', 'f'].into_iter().filter(|x| *x != 'x'); /// /// let chain = a.into_par().chain(b.iter_into_par()); /// assert_eq!( /// chain.collect::>(), /// vec!['a', 'b', 'c', 'd', 'e', 'f'], /// ); /// ``` pub fn chain(self, other: C) -> Par, R> where I: ExactSizeConcurrentIter, C: IntoParIter, { let (orchestrator, params, iter) = self.destruct(); let iter = iter.chain(other.into_con_iter()); Par::new(orchestrator, params, iter) } } impl ParEnumerate for Par where R: ParallelRunner, I: ConcurrentIter, { fn enumerate(self) -> impl ParIter { let (orchestrator, params, iter) = self.destruct(); Par::new(orchestrator, params, iter.enumerate()) } } ================================================ FILE: src/computational_variants/tests/copied.rs ================================================ use crate::{test_utils::*, *}; use alloc::vec::Vec; use test_case::test_matrix; fn input>(n: usize) -> O { (0..n).collect() } #[test_matrix(N, NT, CHUNK)] fn copied_cloned_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input: Vec<_> = input::>(n); let expected: usize = input.iter().copied().sum(); let par = || input.par().num_threads(nt).chunk_size(chunk); let output = par().copied().sum(); assert_eq!(output, expected); let output = par().cloned().sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn copied_cloned_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let map = |x: usize| x + 1; let expected: usize = input.iter().copied().map(&map).sum(); let par = || input.par().num_threads(nt).chunk_size(chunk); let output = par().copied().map(map).sum(); assert_eq!(output, expected); let output = par().cloned().map(map).sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn copied_cloned_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let flat_map = |x: usize| [x, x + 1, x + 2]; let expected: usize = input.iter().copied().flat_map(&flat_map).sum(); let par = || input.par().num_threads(nt).chunk_size(chunk); let output = par().copied().flat_map(flat_map).sum(); assert_eq!(output, expected); let output = par().cloned().flat_map(flat_map).sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn copied_cloned_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: usize| (!x.is_multiple_of(3)).then(|| x + 1); let expected: usize = input.iter().copied().filter_map(&filter_map).sum(); let par = || input.par().num_threads(nt).chunk_size(chunk); let output = par().copied().filter_map(filter_map).sum(); assert_eq!(output, expected); let output = par().cloned().filter_map(filter_map).sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn copied_cloned_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: usize| (!x.is_multiple_of(3)).then(|| x + 1); let filter = |x: &usize| x % 3 != 1; let flat_map = |x: usize| [x, x + 1, x + 2]; let expected: usize = input .iter() .copied() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .sum(); let par = || input.par().num_threads(nt).chunk_size(chunk); let output = par() .copied() .filter_map(filter_map) .filter(filter) .flat_map(flat_map) .sum(); assert_eq!(output, expected); let output = par() .cloned() .filter_map(filter_map) .filter(filter) .flat_map(flat_map) .sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/count.rs ================================================ use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } #[test_matrix(N, NT, CHUNK)] fn count_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let expected = n; let par = input.into_par().num_threads(nt).chunk_size(chunk); let output = par.count(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn count_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let expected = n; let par = input.into_par().num_threads(nt).chunk_size(chunk); let par = par.map(|x| format!("{}!", x)); let output = par.count(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn count_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let expected = input.clone().into_iter().flat_map(&flat_map).count(); let par = input.into_par().num_threads(nt).chunk_size(chunk); let par = par.flat_map(flat_map); let output = par.count(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn count_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let expected = input.clone().into_iter().filter_map(&filter_map).count(); let par = input.into_par().num_threads(nt).chunk_size(chunk); let par = par.filter_map(filter_map); let output = par.count(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn count_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let filter = |x: &String| x.ends_with('3'); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let expected = input .clone() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .count(); let par = input.into_par().num_threads(nt).chunk_size(chunk); let par = par.filter_map(filter_map).filter(filter).flat_map(flat_map); let output = par.count(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/enumerate.rs ================================================ use std::string::ToString; use crate::{test_utils::*, *}; use alloc::vec::Vec; use test_case::test_matrix; #[test_matrix(N, NT, CHUNK)] fn enumerate_sequential(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let par = (0..n).into_par().num_threads(nt).chunk_size(chunk); let par = par.enumerate(); let vec: Vec<_> = par.collect(); vec.iter().for_each(|(idx, value)| { assert_eq!(idx, value); }); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn enumerate_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let par = (0..n).into_par().num_threads(nt).chunk_size(chunk); let par = par .enumerate() .map(|(idx, value): (usize, usize)| (idx + 1, value.pow(2))); let vec: Vec<_> = par.collect(); vec.iter().for_each(|(idx_outer, value)| { assert_eq!((*idx_outer - 1).pow(2), *value); }); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn enumerate_using(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let par = (0..n) .into_par() .num_threads(nt) .chunk_size(chunk) .enumerate() .using_clone("XyZw".to_string()); let par = par.map(|_u, (idx, value)| (idx + 1, value.pow(2))); let vec: Vec<_> = par.collect(); vec.iter().for_each(|(idx_outer, value)| { assert_eq!((idx_outer - 1).pow(2), *value); }); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/fallible_option.rs ================================================ use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } #[test_matrix(NT, CHUNK)] fn fallible_option_collect_empty(nt: &[usize], chunk: &[usize]) { let test = |_, nt, chunk| { let input = || input::>(0); let par = input() .into_iter() .map(|_| None::) .collect::>() .into_par() .num_threads(nt) .chunk_size(chunk) .into_fallible_option(); let output: Option> = par.collect(); assert_eq!(output, Some(Vec::new())); }; test_n_nt_chunk(&[0], nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn fallible_option_collect_partial_success(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .map(|x| (x != "50").then_some(x)) .into_fallible_option() .filter(|x| !x.ends_with('9')) .flat_map(|x| [format!("{x}?"), x]) .map(|x| format!("{x}!")); let output: Option> = par.collect(); assert_eq!(output, None); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn fallible_option_collect_complete_success(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let expected: Vec<_> = input() .into_iter() .filter(|x| !x.ends_with('9')) .flat_map(|x| [format!("{x}?"), x]) .map(|x| format!("{x}!")) .filter_map(Some) .collect(); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .map(|x| (x != "xyz").then_some(x)) .into_fallible_option() .filter(|x| !x.ends_with('9')) .flat_map(|x| [format!("{x}?"), x]) .map(|x| format!("{x}!")) .filter_map(Some); let output: Option> = par.collect(); assert_eq!(output, Some(expected)); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/fallible_result.rs ================================================ use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } #[derive(Debug, PartialEq, Eq)] struct MyErr(String); impl MyErr { fn new() -> Self { Self("error".to_string()) } } #[test_matrix(NT, CHUNK)] fn fallible_result_collect_empty(nt: &[usize], chunk: &[usize]) { let test = |_, nt, chunk| { let input = || input::>(0); let par = input() .into_iter() .map(|_| Err::(MyErr::new())) .collect::>() .into_par() .num_threads(nt) .chunk_size(chunk) .into_fallible_result(); let output: Result, MyErr> = par.collect(); assert_eq!(output, Ok(Vec::new())); }; test_n_nt_chunk(&[0], nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn fallible_result_collect_partial_success(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .map(|x| match x == "50" { true => Err(MyErr::new()), false => Ok(x), }) .into_fallible_result() .filter(|x| !x.ends_with('9')) .flat_map(|x| [format!("{x}?"), x]) .map(|x| format!("{x}!")); let output: Result, MyErr> = par.collect(); assert_eq!(output, Err(MyErr::new())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn fallible_result_collect_complete_success(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let expected: Vec<_> = input() .into_iter() .filter(|x| !x.ends_with('9')) .flat_map(|x| [format!("{x}?"), x]) .map(|x| format!("{x}!")) .filter_map(Some) .collect(); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .map(|x| match x == "xyz" { true => Err(MyErr::new()), false => Ok(x), }) .into_fallible_result() .filter(|x| !x.ends_with('9')) .flat_map(|x| [format!("{x}?"), x]) .map(|x| format!("{x}!")) .filter_map(Some); let output: Result, MyErr> = par.collect(); assert_eq!(output, Ok(expected)); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/flatten.rs ================================================ use crate::{test_utils::*, *}; use alloc::string::{String, ToString}; use alloc::vec::Vec; use alloc::{format, vec}; use test_case::test_matrix; #[test_matrix(N, NT, CHUNK)] fn flatten_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || { (0..n) .map(|i| [i.to_string(), (i + 1).to_string()]) .collect::>() }; let expected: Vec<_> = input().into_iter().flatten().collect(); let par = input().into_par().num_threads(nt).chunk_size(chunk); let output: Vec<_> = par.flatten().collect(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn flatten_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || (0..n).collect::>(); let map = |i: usize| vec![i.to_string(), (i + 1).to_string()]; #[allow(clippy::map_flatten)] let expected: Vec<_> = input().into_iter().map(&map).flatten().collect(); let par = input().into_par().num_threads(nt).chunk_size(chunk); let output: Vec<_> = par.map(&map).flatten().collect(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn flatten_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || (0..n).map(|i| i.to_string()).collect::>(); let filter_map = |x: String| { x.starts_with('3') .then(|| vec![x.clone(), format!("{}!", x)]) }; #[allow(clippy::map_flatten)] let expected: Vec<_> = input() .clone() .into_iter() .filter_map(&filter_map) .flatten() .collect(); let par = input().into_par().num_threads(nt).chunk_size(chunk); let output: Vec<_> = par.filter_map(filter_map).flatten().collect(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn flatten_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || (0..n).map(|i| i.to_string()).collect::>(); let filter_map = |x: String| { x.starts_with('3') .then(|| vec![x.clone(), format!("{}!", x)]) }; let filter = |x: &Vec| x.len() == 2; let map = |mut x: Vec| { x.push("lorem".to_string()); x }; #[allow(clippy::map_flatten)] let expected: Vec<_> = input() .clone() .into_iter() .filter_map(&filter_map) .filter(&filter) .map(&map) .flatten() .collect(); let par = input().into_par().num_threads(nt).chunk_size(chunk); let output: Vec<_> = par .filter_map(filter_map) .filter(filter) .map(map) .flatten() .collect(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/for_each.rs ================================================ use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_vec::ConcurrentVec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn sorted(mut x: Vec) -> Vec { x.sort(); x } #[test_matrix(N, NT, CHUNK)] fn for_each_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let par = input().into_par().num_threads(nt).chunk_size(chunk); let vec = ConcurrentVec::new(); par.for_each(|x| { _ = vec.push(x); }); assert_eq!(sorted(vec.to_vec()), sorted(input())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn for_each_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let map = |x| format!("{}!", x); let par = input().into_par().num_threads(nt).chunk_size(chunk); let par = par.map(&map); let vec = ConcurrentVec::new(); par.for_each(|x| { _ = vec.push(x); }); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().map(map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn for_each_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let par = input().into_par().num_threads(nt).chunk_size(chunk); let par = par.flat_map(flat_map); let vec = ConcurrentVec::new(); par.for_each(|x| { _ = vec.push(x); }); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().flat_map(flat_map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn for_each_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let par = input().into_par().num_threads(nt).chunk_size(chunk); let par = par.filter_map(filter_map); let vec = ConcurrentVec::new(); par.for_each(|x| { _ = vec.push(x); }); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().filter_map(filter_map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn for_each_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let filter = |x: &String| x.ends_with('3'); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let par = input().into_par().num_threads(nt).chunk_size(chunk); let par = par.filter_map(filter_map).filter(filter).flat_map(flat_map); let vec = ConcurrentVec::new(); par.for_each(|x| { _ = vec.push(x); }); assert_eq!( sorted(vec.to_vec()), sorted( input() .into_iter() .filter_map(filter_map) .filter(filter) .flat_map(flat_map) .collect() ) ); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/inspect.rs ================================================ use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_vec::ConcurrentVec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn sorted(mut x: Vec) -> Vec { x.sort(); x } #[test_matrix(N, NT, CHUNK)] fn inspect_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let par = input().into_par().num_threads(nt).chunk_size(chunk); let vec = ConcurrentVec::new(); par.inspect(|x| { _ = vec.push(x.clone()); }) .count(); assert_eq!(sorted(vec.to_vec()), sorted(input())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn inspect_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let map = |x| format!("{}!", x); let par = input().into_par().num_threads(nt).chunk_size(chunk); let par = par.map(&map); let vec = ConcurrentVec::new(); par.inspect(|x| { _ = vec.push(x.clone()); }) .count(); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().map(map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn inspect_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let par = input().into_par().num_threads(nt).chunk_size(chunk); let par = par.flat_map(flat_map); let vec = ConcurrentVec::new(); par.inspect(|x| { _ = vec.push(x.clone()); }) .count(); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().flat_map(flat_map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn inspect_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let par = input().into_par().num_threads(nt).chunk_size(chunk); let par = par.filter_map(filter_map); let vec = ConcurrentVec::new(); par.inspect(|x| { _ = vec.push(x.clone()); }) .count(); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().filter_map(filter_map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn inspect_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let filter = |x: &String| x.ends_with('3'); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let par = input().into_par().num_threads(nt).chunk_size(chunk); let par = par.filter_map(filter_map).filter(filter).flat_map(flat_map); let vec = ConcurrentVec::new(); par.inspect(|x| { _ = vec.push(x.clone()); }) .count(); assert_eq!( sorted(vec.to_vec()), sorted( input() .into_iter() .filter_map(filter_map) .filter(filter) .flat_map(flat_map) .collect() ) ); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/iter_consuming.rs ================================================ use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use orx_fixed_vec::FixedVec; use orx_iterable::Collection; use orx_split_vec::SplitVec; use test_case::test_matrix; const N_OFFSET: usize = 13; fn offset() -> Vec { vec!["x".to_string(); N_OFFSET] } fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn expected( with_offset: bool, input: &impl Collection, map: impl Fn(String) -> String, ) -> Vec { match with_offset { true => { let mut vec = offset(); vec.extend(input.iter().cloned().map(map)); vec } false => input.iter().cloned().map(map).collect(), } } // collect - empty #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0), Vec::::from_iter(offset()), SplitVec::::from_iter(offset()), FixedVec::::from_iter(offset()) ], N, NT, CHUNK) ] fn empty_collect_into(_: I, output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let vec = input::>(n); let expected = expected(!output.is_empty(), &vec, |x| x); let input = vec.into_iter().filter(|x| x.as_str() != "?"); let par = input.iter_into_par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn empty_collect(_: I, _: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto, { let test = |n, nt, chunk| { let vec = input::>(n); let expected = expected(false, &vec, |x| x); let input = vec.into_iter().filter(|x| x.as_str() != "?"); let par = input.iter_into_par().num_threads(nt).chunk_size(chunk); let output: C = par.collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } // collect - map #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0), Vec::::from_iter(offset()), SplitVec::::from_iter(offset()), FixedVec::::from_iter(offset()) ], N, NT, CHUNK) ] fn map_collect_into(_: I, output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let map = |x| format!("{}!", x); let vec = input::>(n); let expected = expected(!output.is_empty(), &vec, map); let input = vec.into_iter().filter(|x| x.as_str() != "?"); let par = input.iter_into_par().num_threads(nt).chunk_size(chunk); let output = par.map(map).collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn map_collect(_: I, _: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto, { let test = |n, nt, chunk| { let map = |x| format!("{}!", x); let vec = input::>(n); let expected = expected(false, &vec, map); let input = vec.into_iter().filter(|x| x.as_str() != "?"); let par = input.iter_into_par().num_threads(nt).chunk_size(chunk); let output: C = par.map(map).collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/iter_ref.rs ================================================ use crate::{collect_into::ParCollectIntoCore, test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use orx_fixed_vec::FixedVec; use orx_iterable::{Collection, IntoCloningIterable}; use orx_split_vec::{Doubling, Linear, PseudoDefault, SplitVec}; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn expected(input: &impl Collection, map: impl Fn(String) -> String) -> Vec { input.iter().cloned().map(map).collect() } // collect - empty #[test_matrix(N, NT, CHUNK)] fn empty_collect_into(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let vec = input::>(n); let expected = expected(&vec, |x| x); let input = vec.iter().filter(|x| x.as_str() != "?").into_iterable(); let par = input.par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(Vec::new()); assert!(output.is_equal_to_ref(&expected)); let par = input.par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(FixedVec::pseudo_default()); assert!(output.is_equal_to_ref(&expected)); let par = input.par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(SplitVec::<_, Doubling>::pseudo_default()); assert!(output.is_equal_to_ref(&expected)); let par = input.par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(SplitVec::<_, Linear>::pseudo_default()); assert!(output.is_equal_to_ref(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn empty_collect(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let vec = input::>(n); let expected = expected(&vec, |x| x); let input = vec.iter().filter(|x| x.as_str() != "?").into_iterable(); let par = input.par().num_threads(nt).chunk_size(chunk); let output: Vec<&String> = par.collect(); assert!(output.is_equal_to_ref(&expected)); let par = input.par().num_threads(nt).chunk_size(chunk); let output: FixedVec<&String> = par.collect(); assert!(output.is_equal_to_ref(&expected)); let par = input.par().num_threads(nt).chunk_size(chunk); let output: SplitVec<&String, Doubling> = par.collect(); assert!(output.is_equal_to_ref(&expected)); let par = input.par().num_threads(nt).chunk_size(chunk); let output: SplitVec<&String, Linear> = par.collect(); assert!(output.is_equal_to_ref(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } // collect - map #[test_matrix(N, NT, CHUNK)] fn map_collect_into(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let map = |x: &String| format!("{}!", x); let map2 = |x: String| format!("{}!", x); let vec = input::>(n); let expected = expected(&vec, map2); let input = vec.iter().filter(|x| x.as_str() != "?").into_iterable(); let par = input.par().num_threads(nt).chunk_size(chunk); let output: Vec = par.map(map).collect_into(vec![]); assert!(output.is_equal_to(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn map_collect(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let map = |x: &String| format!("{}!", x); let map2 = |x: String| format!("{}!", x); let vec = input::>(n); let expected = expected(&vec, map2); let input = vec.iter().filter(|x| x.as_str() != "?").into_iterable(); let par = input.par().num_threads(nt).chunk_size(chunk); let output: Vec = par.map(map).collect(); assert!(output.is_equal_to(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/map/collect.rs ================================================ use crate::collect_into::collect::map_collect_into; use crate::{IterationOrder, Params, runner::DefaultRunner}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use orx_pinned_vec::PinnedVec; use orx_split_vec::SplitVec; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn m_map_collect(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let offset = 33; let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let map = |x: String| format!("{}!", x); let mut output = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let mut expected = Vec::new(); for i in 0..offset { let value = || map(i.to_string()); output.push(value()); expected.push(value()); } expected.extend(input.clone().into_iter().map(map)); let params = Params::new(nt, chunk, ordering); let iter = input.into_con_iter(); let (_, mut output) = map_collect_into(DefaultRunner::default(), params, iter, map, output); if !params.is_sequential() && matches!(params.iteration_order, IterationOrder::Arbitrary) { expected.sort(); output.sort(); } assert_eq!(expected, output.to_vec()); } ================================================ FILE: src/computational_variants/tests/map/find.rs ================================================ use crate::{Params, default_fns::map_self, executor::parallel_compute, runner::DefaultRunner}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn m_find(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let expected = input.clone().into_iter().next(); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let output = parallel_compute::next::m(DefaultRunner::default(), params, iter, map_self).1; assert_eq!(expected, output); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn m_map_find(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let map = |x: String| format!("{}!", x); let expected = input.clone().into_iter().map(map).next(); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let output = parallel_compute::next::m(DefaultRunner::default(), params, iter, map).1; assert_eq!(expected, output); } ================================================ FILE: src/computational_variants/tests/map/mod.rs ================================================ mod collect; mod find; mod reduce; ================================================ FILE: src/computational_variants/tests/map/reduce.rs ================================================ use crate::{Params, default_fns::map_self, executor::parallel_compute, runner::DefaultRunner}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn m_reduce(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let reduce = |x: String, y: String| match x < y { true => y, false => x, }; let expected = input.clone().into_iter().reduce(reduce); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let (_, output) = parallel_compute::reduce::m(DefaultRunner::default(), params, iter, map_self, reduce); assert_eq!(expected, output); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn m_map_reduce(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let map = |x: String| format!("{}!", x); let reduce = |x: String, y: String| match x < y { true => y, false => x, }; let expected = input.clone().into_iter().map(map).reduce(reduce); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let (_, output) = parallel_compute::reduce::m(DefaultRunner::default(), params, iter, map, reduce); assert_eq!(expected, output); } ================================================ FILE: src/computational_variants/tests/min_max.rs ================================================ use crate::{test_utils::*, *}; use alloc::string::{String, ToString}; use alloc::vec::Vec; use core::cmp::Ordering; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn cmp(a: &usize, b: &usize) -> Ordering { match a < b { true => Ordering::Less, false => Ordering::Greater, } } fn key(a: &usize) -> u64 { *a as u64 + 10 } #[test_matrix(N, NT, CHUNK)] fn min_max_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || { input::>(n) .iter() .map(|x| x.parse::().expect("is-ok")) .collect::>() }; let par = || input().into_par().num_threads(nt).chunk_size(chunk); assert_eq!(par().min(), input().iter().min().copied()); assert_eq!(par().max(), input().iter().max().copied()); assert_eq!(par().min_by(cmp), input().into_iter().min_by(cmp)); assert_eq!(par().max_by(cmp), input().into_iter().max_by(cmp)); assert_eq!(par().min_by_key(key), input().into_iter().min_by_key(key)); assert_eq!(par().max_by_key(key), input().into_iter().max_by_key(key)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn min_max_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let map = |x: String| x.parse::().expect("is-ok"); let par = || { input() .into_par() .num_threads(nt) .chunk_size(chunk) .map(&map) }; assert_eq!(par().min(), input().into_iter().map(&map).min()); assert_eq!(par().max(), input().into_iter().map(&map).max()); assert_eq!(par().min_by(cmp), input().into_iter().map(&map).min_by(cmp)); assert_eq!(par().max_by(cmp), input().into_iter().map(&map).max_by(cmp)); assert_eq!( par().min_by_key(key), input().into_iter().map(&map).min_by_key(key) ); assert_eq!( par().max_by_key(key), input().into_iter().map(&map).max_by_key(key) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn min_max_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let flat_map = |x: String| { let n = x.len(); let a = x.parse::().expect("is-ok"); (0..n).map(|i| a + i).collect::>() }; let par = || { input() .into_par() .num_threads(nt) .chunk_size(chunk) .flat_map(&flat_map) }; assert_eq!(par().min(), input().into_iter().flat_map(&flat_map).min()); assert_eq!(par().max(), input().into_iter().flat_map(&flat_map).max()); assert_eq!( par().min_by(cmp), input().into_iter().flat_map(&flat_map).min_by(cmp) ); assert_eq!( par().max_by(cmp), input().into_iter().flat_map(&flat_map).max_by(cmp) ); assert_eq!( par().min_by_key(key), input().into_iter().flat_map(&flat_map).min_by_key(key) ); assert_eq!( par().max_by_key(key), input().into_iter().flat_map(&flat_map).max_by_key(key) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn min_max_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| { x.starts_with('3') .then(|| x.parse::().expect("is-ok")) }; let par = || { input() .into_par() .num_threads(nt) .chunk_size(chunk) .filter_map(&filter_map) }; assert_eq!( par().min(), input().into_iter().filter_map(&filter_map).min() ); assert_eq!( par().max(), input().into_iter().filter_map(&filter_map).max() ); assert_eq!( par().min_by(cmp), input().into_iter().filter_map(&filter_map).min_by(cmp) ); assert_eq!( par().max_by(cmp), input().into_iter().filter_map(&filter_map).max_by(cmp) ); assert_eq!( par().min_by_key(key), input().into_iter().filter_map(&filter_map).min_by_key(key) ); assert_eq!( par().max_by_key(key), input().into_iter().filter_map(&filter_map).max_by_key(key) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn min_max_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let filter = |x: &String| x.ends_with('3'); let flat_map = |x: String| { let n = x.len(); let a = x.parse::().expect("is-ok"); (0..n).map(|i| a + i).collect::>() }; let par = || { input() .into_par() .num_threads(nt) .chunk_size(chunk) .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) }; assert_eq!( par().min(), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .min() ); assert_eq!( par().max(), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .max() ); assert_eq!( par().min_by(cmp), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .min_by(cmp) ); assert_eq!( par().max_by(cmp), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .max_by(cmp) ); assert_eq!( par().min_by_key(key), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .min_by_key(key) ); assert_eq!( par().max_by_key(key), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .max_by_key(key) ); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/mod.rs ================================================ mod copied; mod count; mod enumerate; mod fallible_option; mod fallible_result; mod flatten; mod for_each; mod inspect; mod iter_consuming; mod iter_ref; mod map; mod min_max; mod range; mod slice; mod sum; mod vectors; mod xap; ================================================ FILE: src/computational_variants/tests/range.rs ================================================ use crate::{test_utils::*, *}; use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use orx_fixed_vec::FixedVec; use orx_iterable::Iterable; use orx_split_vec::SplitVec; use test_case::test_matrix; const N_OFFSET: usize = 13; fn offset() -> Vec { vec![9; N_OFFSET] } fn expected( with_offset: bool, input: impl Iterable, map: impl Fn(usize) -> T + Copy, ) -> Vec { match with_offset { true => { let mut vec: Vec<_> = offset().into_iter().map(map).collect(); vec.extend(input.iter().map(map)); vec } false => input.iter().map(map).collect(), } } // collect - empty #[test_matrix( [Vec::::new(), SplitVec::::new(), FixedVec::::new(0), Vec::::from_iter(offset()), SplitVec::::from_iter(offset()), FixedVec::::from_iter(offset()) ], N, NT, CHUNK) ] fn empty_collect_into(output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let input = 0..n; let expected = expected(!output.is_empty(), input.clone(), |x| x); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn empty_collect(_: C, n: &[usize], nt: &[usize], chunk: &[usize]) where C: ParCollectInto, { let test = |n, nt, chunk| { let input = 0..n; let expected = expected(false, input.clone(), |x| x); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output: C = par.collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } // collect - map #[test_matrix( [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn map_collect_into(output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let map = |x: usize| x.to_string(); let input = 0..n; let expected = expected(!output.is_empty(), input.clone(), map); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output = par.map(map).collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn map_collect(_: C, n: &[usize], nt: &[usize], chunk: &[usize]) where C: ParCollectInto, { let test = |n, nt, chunk| { let map = |x: usize| x.to_string(); let input = 0..n; let expected = expected(false, input.clone(), map); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output: C = par.map(map).collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/slice.rs ================================================ use crate::{collect_into::ParCollectIntoCore, test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use orx_fixed_vec::FixedVec; use orx_iterable::Collection; use orx_split_vec::{Doubling, Linear, PseudoDefault, SplitVec}; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn expected(input: &impl Collection, map: impl Fn(String) -> String) -> Vec { input.iter().cloned().map(map).collect() } // collect - empty #[test_matrix(N, NT, CHUNK)] fn empty_collect_into(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let vec = input::>(n); let input = vec.as_slice(); let expected = expected(&vec, |x| x); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(Vec::new()); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(FixedVec::pseudo_default()); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(SplitVec::<_, Doubling>::pseudo_default()); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(SplitVec::<_, Linear>::pseudo_default()); assert!(output.is_equal_to_ref(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn empty_collect(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let vec = input::>(n); let input = vec.as_slice(); let expected = expected(&vec, |x| x); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output: Vec<&String> = par.collect(); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output: FixedVec<&String> = par.collect(); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output: SplitVec<&String, Doubling> = par.collect(); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output: SplitVec<&String, Linear> = par.collect(); assert!(output.is_equal_to_ref(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } // collect - map #[test_matrix(N, NT, CHUNK)] fn map_collect_into(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let map = |x: &String| format!("{}!", x); let map2 = |x: String| format!("{}!", x); let vec = input::>(n); let input = vec.as_slice(); let expected = expected(&vec, map2); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output: Vec = par.map(map).collect_into(vec![]); assert!(output.is_equal_to(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn map_collect(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let map = |x: &String| format!("{}!", x); let map2 = |x: String| format!("{}!", x); let vec = input::>(n); let input = vec.as_slice(); let expected = expected(&vec, map2); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output: Vec = par.map(map).collect(); assert!(output.is_equal_to(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/sum.rs ================================================ use crate::{test_utils::*, *}; use alloc::string::{String, ToString}; use alloc::vec::Vec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } #[test_matrix(N, NT, CHUNK)] fn sum_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input: Vec<_> = input::>(n).iter().map(|x| x.len()).collect(); let expected: usize = input.iter().sum(); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output = par.sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn sum_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let map = |x: String| x.len(); let expected: usize = input.clone().into_iter().map(&map).sum(); let par = input.into_par().num_threads(nt).chunk_size(chunk); let par = par.map(map); let output = par.sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn sum_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let flat_map = |x: String| x.chars().map(|x| x.to_string().len()).collect::>(); let expected: usize = input.clone().into_iter().flat_map(&flat_map).sum(); let par = input.into_par().num_threads(nt).chunk_size(chunk); let par = par.flat_map(flat_map); let output = par.sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn sum_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x.len()); let expected: usize = input.clone().into_iter().filter_map(&filter_map).sum(); let par = input.into_par().num_threads(nt).chunk_size(chunk); let par = par.filter_map(filter_map); let output = par.sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn sum_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let filter = |x: &String| x.ends_with('3'); let flat_map = |x: String| x.chars().map(|x| x.to_string().len()).collect::>(); let expected: usize = input .clone() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .sum(); let par = input.into_par().num_threads(nt).chunk_size(chunk); let par = par.filter_map(filter_map).filter(filter).flat_map(flat_map); let output = par.sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/vectors.rs ================================================ use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use orx_fixed_vec::FixedVec; use orx_iterable::Collection; use orx_split_vec::SplitVec; use test_case::test_matrix; const N_OFFSET: usize = 13; fn offset() -> Vec { vec!["x".to_string(); N_OFFSET] } fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn expected( with_offset: bool, input: &impl Collection, map: impl Fn(String) -> String, ) -> Vec { match with_offset { true => { let mut vec = offset(); vec.extend(input.iter().cloned().map(map)); vec } false => input.iter().cloned().map(map).collect(), } } // collect - empty #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0), Vec::::from_iter(offset()), SplitVec::::from_iter(offset()), FixedVec::::from_iter(offset()) ], N, NT, CHUNK) ] fn empty_collect_into(_: I, output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let input = input::>(n); let expected = expected(!output.is_empty(), &input, |x| x); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output = par.collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn empty_collect(_: I, _: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto, { let test = |n, nt, chunk| { let input = input::>(n); let expected = expected(false, &input, |x| x); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output: C = par.collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } // collect - map #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0), Vec::::from_iter(offset()), SplitVec::::from_iter(offset()), FixedVec::::from_iter(offset()) ], N, NT, CHUNK) ] fn map_collect_into(_: I, output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let map = |x| format!("{}!", x); let input = input::>(n); let expected = expected(!output.is_empty(), &input, map); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output = par.map(map).collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn map_collect(_: I, _: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto, { let test = |n, nt, chunk| { let map = |x| format!("{}!", x); let input = input::>(n); let expected = expected(false, &input, map); let par = input.into_par().num_threads(nt).chunk_size(chunk); let output: C = par.map(map).collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/computational_variants/tests/xap/collect.rs ================================================ use crate::ParIter; use crate::computational_variants::ParXap; use crate::generic_values::Vector; use crate::runner::DefaultRunner; use crate::{IterationOrder, Params}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use orx_pinned_vec::PinnedVec; use orx_split_vec::SplitVec; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test] fn todo_panic_at_con_bag_new() { // TODO: this code panics when ParThreadPool::map uses ConcurrentBag::new rather than ConcurrentBag::with_fixed_capacity let n = 10; let nt = 2; let chunk = 1; let ordering = IterationOrder::Arbitrary; let offset = 33; let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let xmap = |x: String| Vector(fmap(x)); let mut output = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let mut expected = Vec::new(); for i in 0..offset { let i = i.to_string(); for x in fmap(i) { output.push(x.clone()); expected.push(x); } } expected.extend(input.clone().into_iter().flat_map(&fmap)); let params = Params::new(nt, chunk, ordering); let iter = input.into_con_iter(); let x = ParXap::new(DefaultRunner::default(), params, iter, xmap); let mut output = x.collect_into(output); if !params.is_sequential() && matches!(params.iteration_order, IterationOrder::Arbitrary) { expected.sort(); output.sort(); } assert_eq!(expected, output.to_vec()); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn x_flat_map_collect(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let offset = 33; let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let xmap = |x: String| Vector(fmap(x)); let mut output = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let mut expected = Vec::new(); for i in 0..offset { let i = i.to_string(); for x in fmap(i) { output.push(x.clone()); expected.push(x); } } expected.extend(input.clone().into_iter().flat_map(&fmap)); let params = Params::new(nt, chunk, ordering); let iter = input.into_con_iter(); let x = ParXap::new(DefaultRunner::default(), params, iter, xmap); let mut output = x.collect_into(output); if !params.is_sequential() && matches!(params.iteration_order, IterationOrder::Arbitrary) { expected.sort(); output.sort(); } assert_eq!(expected, output.to_vec()); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn x_filter_map_collect(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let offset = 33; let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| (!x.starts_with('3')).then_some(format!("{}!", x)); let xmap = |x: String| Vector(fmap(x)); let mut output = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let mut expected = Vec::new(); for i in 0..offset { let i = i.to_string(); if let Some(x) = fmap(i) { output.push(x.clone()); expected.push(x); } } expected.extend(input.clone().into_iter().flat_map(&fmap)); let params = Params::new(nt, chunk, ordering); let iter = input.into_con_iter(); let x = ParXap::new(DefaultRunner::default(), params, iter, xmap); let mut output = x.collect_into(output); if !params.is_sequential() && matches!(params.iteration_order, IterationOrder::Arbitrary) { expected.sort(); output.sort(); } assert_eq!(expected, output.to_vec()); } ================================================ FILE: src/computational_variants/tests/xap/find.rs ================================================ use crate::ParIter; use crate::Params; use crate::computational_variants::ParXap; use crate::generic_values::Vector; use crate::runner::DefaultRunner; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn x_flat_map_find(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let xmap = |x: String| Vector(fmap(x)); let expected = input.clone().into_iter().flat_map(fmap).next(); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let x = ParXap::new(DefaultRunner::default(), params, iter, xmap); let output = x.first(); assert_eq!(expected, output); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn x_filter_map_find(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| (!x.starts_with('3')).then_some(format!("{}!", x)); let xmap = |x: String| Vector(fmap(x)); let expected = input.clone().into_iter().filter_map(fmap).next(); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let x = ParXap::new(DefaultRunner::default(), params, iter, xmap); let output = x.first(); assert_eq!(expected, output); } ================================================ FILE: src/computational_variants/tests/xap/mod.rs ================================================ mod collect; mod find; mod reduce; ================================================ FILE: src/computational_variants/tests/xap/reduce.rs ================================================ use crate::ParIter; use crate::Params; use crate::computational_variants::ParXap; use crate::generic_values::Vector; use crate::runner::DefaultRunner; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn x_flat_map_reduce(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let xmap = |x: String| Vector(fmap(x)); let reduce = |x: String, y: String| match x > y { true => x, false => y, }; let expected = input.clone().into_iter().flat_map(fmap).reduce(reduce); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let x = ParXap::new(DefaultRunner::default(), params, iter, xmap); let output = x.reduce(reduce); assert_eq!(expected, output); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn x_filter_map_reduce(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| (!x.starts_with('3')).then_some(format!("{}!", x)); let xmap = |x: String| Vector(fmap(x)); let reduce = |x: String, y: String| match x > y { true => x, false => y, }; let expected = input.clone().into_iter().filter_map(fmap).reduce(reduce); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let x = ParXap::new(DefaultRunner::default(), params, iter, xmap); let output = x.reduce(reduce); assert_eq!(expected, output); } ================================================ FILE: src/computational_variants/xap.rs ================================================ use crate::computational_variants::fallible_result::ParXapResult; use crate::executor::parallel_compute as prc; use crate::generic_values::TransformableValues; use crate::generic_values::runner_results::Infallible; use crate::par_iter_result::IntoResult; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::{UParXap, UsingClone, UsingFun}; use crate::{ChunkSize, IterationOrder, NumThreads, ParCollectInto, ParIter, Params}; use crate::{ParIterResult, ParIterUsing}; use orx_concurrent_iter::ConcurrentIter; /// A parallel iterator that xaps inputs. /// /// *xap* is a generalization of one-to-one map, filter-map and flat-map operations. pub struct ParXap where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(I::Item) -> Vo + Sync, { orchestrator: R, params: Params, iter: I, xap1: X1, } impl ParXap where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(I::Item) -> Vo + Sync, { pub(crate) fn new(orchestrator: R, params: Params, iter: I, xap1: X1) -> Self { Self { orchestrator, params, iter, xap1, } } pub(crate) fn destruct(self) -> (R, Params, I, X1) { (self.orchestrator, self.params, self.iter, self.xap1) } } unsafe impl Send for ParXap where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(I::Item) -> Vo + Sync, { } unsafe impl Sync for ParXap where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(I::Item) -> Vo + Sync, { } impl ParIter for ParXap where R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(I::Item) -> Vo + Sync, { type Item = Vo::Item; fn con_iter(&self) -> &impl ConcurrentIter { &self.iter } fn params(&self) -> Params { self.params } // params transformations fn num_threads(mut self, num_threads: impl Into) -> Self { self.params = self.params.with_num_threads(num_threads); self } fn chunk_size(mut self, chunk_size: impl Into) -> Self { self.params = self.params.with_chunk_size(chunk_size); self } fn iteration_order(mut self, collect: IterationOrder) -> Self { self.params = self.params.with_collect_ordering(collect); self } fn with_runner(self, orchestrator: Q) -> impl ParIter { let (_, params, iter, x1) = self.destruct(); ParXap::new(orchestrator, params, iter, x1) } // using transformations fn using<'using, U, F>( self, using: F, ) -> impl ParIterUsing<'using, UsingFun, R, Item = >::Item> where U: 'using, F: Fn(usize) -> U + Sync, { let using = UsingFun::new(using); let (orchestrator, params, iter, x1) = self.destruct(); let m1 = move |_: *mut U, t: I::Item| x1(t); UParXap::new(using, orchestrator, params, iter, m1) } fn using_clone( self, value: U, ) -> impl ParIterUsing<'static, UsingClone, R, Item = >::Item> where U: Clone + 'static, { let using = UsingClone::new(value); let (orchestrator, params, iter, x1) = self.destruct(); let m1 = move |_: *mut U, t: I::Item| x1(t); UParXap::new(using, orchestrator, params, iter, m1) } // computation transformations fn map(self, map: Map) -> impl ParIter where Map: Fn(Self::Item) -> Out + Sync + Clone, { let (orchestrator, params, iter, x1) = self.destruct(); let x1 = move |i: I::Item| { let vo = x1(i); vo.map(map.clone()) }; ParXap::new(orchestrator, params, iter, x1) } fn filter(self, filter: Filter) -> impl ParIter where Filter: Fn(&Self::Item) -> bool + Sync + Clone, { let (orchestrator, params, iter, x1) = self.destruct(); let x1 = move |i: I::Item| { let values = x1(i); values.filter(filter.clone()) }; ParXap::new(orchestrator, params, iter, x1) } fn flat_map(self, flat_map: FlatMap) -> impl ParIter where IOut: IntoIterator, FlatMap: Fn(Self::Item) -> IOut + Sync + Clone, { let (orchestrator, params, iter, x1) = self.destruct(); let x1 = move |i: I::Item| { let vo = x1(i); vo.flat_map(flat_map.clone()) }; ParXap::new(orchestrator, params, iter, x1) } fn filter_map(self, filter_map: FilterMap) -> impl ParIter where FilterMap: Fn(Self::Item) -> Option + Sync + Clone, { let (orchestrator, params, iter, x1) = self.destruct(); let x1 = move |i: I::Item| { let vo = x1(i); vo.filter_map(filter_map.clone()) }; ParXap::new(orchestrator, params, iter, x1) } fn take_while(self, take_while: While) -> impl ParIter where While: Fn(&Self::Item) -> bool + Sync + Clone, { let (orchestrator, params, iter, x1) = self.destruct(); let x1 = move |i: I::Item| { let vo = x1(i); vo.whilst(take_while.clone()) }; ParXap::new(orchestrator, params, iter, x1) } fn into_fallible_result(self) -> impl ParIterResult where Self::Item: IntoResult, { let (orchestrator, params, iter, x1) = self.destruct(); ParXapResult::new(orchestrator, params, iter, x1) } // collect fn collect_into(self, output: C) -> C where C: ParCollectInto, { let (orchestrator, params, iter, x1) = self.destruct(); output.x_collect_into(orchestrator, params, iter, x1) } // reduce fn reduce(self, reduce: Reduce) -> Option where Self::Item: Send, Reduce: Fn(Self::Item, Self::Item) -> Self::Item + Sync, { let (orchestrator, params, iter, x1) = self.destruct(); let (_, Ok(acc)) = prc::reduce::x(orchestrator, params, iter, x1, reduce); acc } // early exit fn first(self) -> Option where Self::Item: Send, { let (orchestrator, params, iter, x1) = self.destruct(); match params.iteration_order { IterationOrder::Ordered => { let (_num_threads, Ok(result)) = prc::next::x(orchestrator, params, iter, x1); result.map(|x| x.1) } IterationOrder::Arbitrary => { let (_num_threads, Ok(result)) = prc::next_any::x(orchestrator, params, iter, x1); result } } } } ================================================ FILE: src/default_fns.rs ================================================ use core::ops::Add; #[inline(always)] pub fn map_self(input: T) -> T { input } #[inline(always)] pub fn map_count(_: T) -> usize { 1 } #[inline(always)] pub fn map_copy(x: &T) -> T { *x } #[inline(always)] pub fn map_clone(x: &T) -> T { x.clone() } #[inline(always)] pub fn reduce_sum(a: T, b: T) -> T where T: Add, { a + b } #[inline(always)] pub fn reduce_unit(_: (), _: ()) {} // using #[inline(always)] pub fn u_map_self(_: &mut U, input: T) -> T { input } #[inline(always)] pub fn u_map_copy(_: &mut U, x: &T) -> T { *x } #[inline(always)] pub fn u_map_clone(_: &mut U, x: &T) -> T { x.clone() } #[inline(always)] pub fn u_map_count(_: &mut U, _: T) -> usize { 1 } #[inline(always)] pub fn u_reduce_sum(_: &mut U, a: T, b: T) -> T where T: Add, { a + b } #[inline(always)] pub fn u_reduce_unit(_: &mut U, _: (), _: ()) {} ================================================ FILE: src/enumerate/mod.rs ================================================ use crate::{ParIter, ParallelRunner}; /// A parallel iterator with a known and fixed size, /// meaning all transformations are 1 to 1. pub trait ParEnumerate: ParIter { /// Creates an iterator which gives each value along with its index in the source collection. /// /// The iterator returned yields pairs `(i, val)`, where `i` is the index in the source collection, /// and `val` is the value returned by the iterator. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let vec = vec![26i32, -27, 5]; /// /// let max_abs = vec.into_par().enumerate().max_by_key(|(_idx, x)| x.abs()); /// assert_eq!(max_abs, Some((1, -27))); /// ``` fn enumerate(self) -> impl ParIter; } ================================================ FILE: src/env.rs ================================================ use core::num::NonZeroUsize; #[cfg(feature = "std")] const MAX_NUM_THREADS_ENV_VARIABLE: &str = "ORX_PARALLEL_MAX_NUM_THREADS"; pub fn max_num_threads_by_env_variable() -> Option { #[cfg(feature = "std")] match std::env::var(MAX_NUM_THREADS_ENV_VARIABLE) { Ok(s) => match s.parse::() { Ok(x) => NonZeroUsize::new(x), // None if 0; Some(x) if x is set to a positive bound Err(_e) => None, // not a number, ignored assuming no bound }, Err(_e) => None, // not set, no bound } #[cfg(not(feature = "std"))] None } ================================================ FILE: src/executor/computation_kind.rs ================================================ /// Computation kind. #[derive(Clone, Copy)] pub enum ComputationKind { /// Computation where outputs are collected into a collection. Collect, /// Computation where the inputs or intermediate results are reduced to a single value. Reduce, /// Computation which allows for early returns, such as `find` operation. EarlyReturn, } ================================================ FILE: src/executor/executor_with_diagnostics/mod.rs ================================================ mod parallel_executor; mod shared_state; mod thread_executor; pub use parallel_executor::ParallelExecutorWithDiagnostics; ================================================ FILE: src/executor/executor_with_diagnostics/parallel_executor.rs ================================================ use super::{ shared_state::SharedStateWithDiagnostics, thread_executor::ThreadExecutorWithDiagnostics, }; use crate::ParallelExecutor; use crate::runner::{ComputationKind, NumSpawned}; use orx_concurrent_iter::ConcurrentIter; use std::num::NonZeroUsize; /// A parallel executor which wraps another parallel executor `E` and collects diagnostics about: /// /// * how many threads are used for the parallel computation /// * how many times each thread received a tasks /// * average chunk size; i.e., average number of tasks, that each thread received /// * and finally, explicit chunk sizes for the first task assignments. /// /// The diagnostics are printed on the stdout once the parallel computation is completed. /// Therefore, this executor is suitable only for test purposed, but not for production. /// /// Any executor can be converted into executor with diagnostics. /// In the example below, executor of the default runner is converted to executor with diagnostics. /// /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // normal execution /// /// let range = 0..4096; /// let sum = range /// .par() /// .map(|x| x + 1) /// .filter(|x| x.is_multiple_of(2)) /// .sum(); /// assert_eq!(sum, 4196352); /// /// // execution with diagnostics /// /// let range = 0..4096; /// let sum = range /// .par() /// .with_runner(DefaultRunner::default().with_diagnostics()) /// .map(|x| x + 1) /// .filter(|x| x.is_multiple_of(2)) /// .sum(); /// assert_eq!(sum, 4196352); /// /// // prints diagnostics, which looks something like the following: /// // /// // - Number of threads used = 5 /// // /// // - [Thread idx]: num_calls, num_tasks, avg_chunk_size, first_chunk_sizes /// // - [0]: 25, 1600, 64, [64, 64, 64, 64, 64, 64, 64, 64, 64, 64] /// // - [1]: 26, 1664, 64, [64, 64, 64, 64, 64, 64, 64, 64, 64, 64] /// // - [2]: 13, 832, 64, [64, 64, 64, 64, 64, 64, 64, 64, 64, 64] /// // - [3]: 0, 0, 0, [] /// // - [4]: 0, 0, 0, [] /// ``` #[derive(Clone)] pub struct ParallelExecutorWithDiagnostics where E: ParallelExecutor, { executor: E, } impl ParallelExecutor for ParallelExecutorWithDiagnostics where E: ParallelExecutor, { type SharedState = SharedStateWithDiagnostics; type ThreadExecutor = ThreadExecutorWithDiagnostics; fn new( kind: ComputationKind, params: crate::Params, initial_input_len: Option, max_num_threads: NonZeroUsize, ) -> Self { let executor = E::new(kind, params, initial_input_len, max_num_threads); Self { executor } } fn new_shared_state(&self) -> Self::SharedState { let inner_state = self.executor.new_shared_state(); SharedStateWithDiagnostics::new(inner_state) } fn do_spawn_new( &self, num_spawned: NumSpawned, shared_state: &Self::SharedState, iter: &I, ) -> bool where I: ConcurrentIter, { self.executor .do_spawn_new(num_spawned, shared_state.inner(), iter) } fn new_thread_executor( &self, thread_idx: usize, shared_state: &Self::SharedState, ) -> Self::ThreadExecutor { let executor = self .executor .new_thread_executor(thread_idx, shared_state.inner()); ThreadExecutorWithDiagnostics::new(thread_idx, executor) } fn complete_task(self, shared_state: Self::SharedState) { shared_state.display(); } } ================================================ FILE: src/executor/executor_with_diagnostics/shared_state.rs ================================================ use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use std::println; pub struct SharedStateWithDiagnostics { inner: S, task_counts: ConcurrentBag<(usize, Vec)>, // (thread_idx, chunk sizes) } impl SharedStateWithDiagnostics { pub fn new(inner: S) -> Self { let tasks = ConcurrentBag::new(); Self { inner, task_counts: tasks, } } #[inline(always)] pub fn inner(&self) -> &S { &self.inner } pub fn add_task_counts_of_thread(&self, thread_idx: usize, chunk_sizes: Vec) { self.task_counts.push((thread_idx, chunk_sizes)); } pub fn display(self) { let mut task_counts = self.task_counts.into_inner().to_vec(); task_counts.sort_by_key(|x| x.0); println!("\n# Parallel Executor Diagnostics"); println!("\n- Number of threads used = {}", task_counts.len()); println!("\n- [Thread idx]: num_calls, num_tasks, avg_chunk_size, first_chunk_sizes"); for (thread_idx, task_counts) in task_counts { let total: usize = task_counts.iter().sum(); let num_calls = task_counts.len(); let avg_chunk_size = match num_calls { 0 => 0, n => total / n, }; let first_chunks: Vec<_> = task_counts.iter().copied().take(10).collect(); println!( " - [{thread_idx}]: {num_calls}, {total}, {avg_chunk_size}, {first_chunks:?}", ); } } } ================================================ FILE: src/executor/executor_with_diagnostics/thread_executor.rs ================================================ use super::shared_state::SharedStateWithDiagnostics; use crate::{ParallelExecutor, ThreadExecutor}; use alloc::vec; use alloc::vec::Vec; use orx_concurrent_iter::ConcurrentIter; pub struct ThreadExecutorWithDiagnostics where E: ParallelExecutor, { thread_idx: usize, executor: E::ThreadExecutor, task_counts: Vec, // (thread_idx, chunk_size) } impl ThreadExecutorWithDiagnostics where E: ParallelExecutor, { pub(super) fn new(thread_idx: usize, executor: E::ThreadExecutor) -> Self { Self { thread_idx, executor, task_counts: vec![], } } } impl ThreadExecutor for ThreadExecutorWithDiagnostics where E: ParallelExecutor, { type SharedState = SharedStateWithDiagnostics; fn next_chunk_size(&self, shared_state: &Self::SharedState, iter: &I) -> usize where I: ConcurrentIter, { self.executor.next_chunk_size(shared_state.inner(), iter) } fn begin_chunk(&mut self, chunk_size: usize) { self.executor.begin_chunk(chunk_size); } fn complete_chunk(&mut self, shared_state: &Self::SharedState, chunk_size: usize) { self.task_counts.push(chunk_size); self.executor .complete_chunk(shared_state.inner(), chunk_size); } fn complete_task(&mut self, shared_state: &Self::SharedState) { self.executor.complete_task(shared_state.inner()); shared_state.add_task_counts_of_thread(self.thread_idx, self.task_counts.clone()); } } ================================================ FILE: src/executor/fixed_chunk_executor/chunk_size.rs ================================================ use crate::{parameters::ChunkSize, runner::ComputationKind}; use core::num::NonZeroUsize; const MAX_CHUNK_SIZE: usize = 1 << 20; const DESIRED_MIN_CHUNK_SIZE: usize = 64; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ResolvedChunkSize { Min(usize), Exact(usize), } impl ResolvedChunkSize { pub fn new( kind: ComputationKind, initial_len: Option, max_num_threads: NonZeroUsize, chunk_size: ChunkSize, ) -> Self { match chunk_size { ChunkSize::Auto => Self::Min(auto_chunk_size(kind, initial_len, max_num_threads)), ChunkSize::Min(x) => Self::Min(min_chunk_size(initial_len, max_num_threads, x.into())), ChunkSize::Exact(x) => Self::Exact(x.into()), } } pub fn chunk_size(self) -> usize { match self { Self::Min(x) => x, Self::Exact(x) => x, } } } const fn min_required_len(kind: ComputationKind, one_round_len: usize) -> usize { match kind { ComputationKind::Collect => one_round_len * 4, ComputationKind::Reduce => one_round_len * 4, ComputationKind::EarlyReturn => one_round_len * 8, } } fn auto_chunk_size( kind: ComputationKind, initial_len: Option, max_num_threads: NonZeroUsize, ) -> usize { fn find_chunk_size(kind: ComputationKind, input_len: usize, num_threads: usize) -> usize { let mut chunk_size = MAX_CHUNK_SIZE; loop { let one_round_len = chunk_size * num_threads; // heterogeneity buffer if input_len >= min_required_len(kind, one_round_len) { break; } // len is still greater, we don't want chunk size to be too small if input_len >= one_round_len && chunk_size <= DESIRED_MIN_CHUNK_SIZE { break; } // absolute breaking condition if chunk_size == 1 { break; } chunk_size >>= 1; } chunk_size } match initial_len { None => 1, // TODO: is this a good choice? Some(0) => 1, Some(input_len) => find_chunk_size(kind, input_len, max_num_threads.into()), } } fn min_chunk_size( initial_len: Option, max_num_threads: NonZeroUsize, chunk_size: usize, ) -> usize { let max_num_threads: usize = max_num_threads.into(); match initial_len { None => chunk_size, Some(0) => 1, Some(len) => { let one_round_len = max_num_threads * chunk_size; match one_round_len > len { true => div_ceil(len, max_num_threads), false => chunk_size, } } } } const fn div_ceil(number: usize, divider: usize) -> usize { let x = number / divider; let remainder = number - x * divider; x + if remainder > 0 { 1 } else { 0 } } ================================================ FILE: src/executor/fixed_chunk_executor/mod.rs ================================================ mod chunk_size; mod parallel_executor; mod thread_executor; pub use parallel_executor::FixedChunkRunner; ================================================ FILE: src/executor/fixed_chunk_executor/parallel_executor.rs ================================================ use super::{chunk_size::ResolvedChunkSize, thread_executor::FixedChunkThreadExecutor}; use crate::runner::ComputationKind; use crate::{ executor::parallel_executor::ParallelExecutor, parameters::Params, runner::NumSpawned, }; use core::{ num::NonZeroUsize, sync::atomic::{AtomicUsize, Ordering}, }; use orx_concurrent_iter::ConcurrentIter; const LAG_PERIODICITY: usize = 4; pub struct FixedChunkRunner { initial_len: Option, resolved_chunk_size: ResolvedChunkSize, max_num_threads: usize, current_chunk_size: AtomicUsize, } impl Clone for FixedChunkRunner { fn clone(&self) -> Self { Self { initial_len: self.initial_len, resolved_chunk_size: self.resolved_chunk_size, max_num_threads: self.max_num_threads, current_chunk_size: self.current_chunk_size.load(Ordering::Relaxed).into(), } } } impl FixedChunkRunner { fn spawn_new(&self, num_spawned: usize, remaining: Option) -> bool { match (num_spawned, remaining) { (_, Some(0)) => false, (x, _) if x >= self.max_num_threads => false, _ => true, } } fn next_chunk(&self, num_spawned: usize, remaining_len: Option) -> Option { match (self.initial_len, remaining_len) { (Some(initial_len), Some(remaining_len)) => { self.next_chunk_size_known_len(num_spawned, initial_len, remaining_len) } _ => self.next_chunk_size_unknown_len(num_spawned), } } fn next_chunk_size_unknown_len(&self, num_spawned: usize) -> Option { match num_spawned { x if x >= self.max_num_threads => None, _ => Some(self.resolved_chunk_size.chunk_size()), } } fn next_chunk_size_known_len( &self, num_spawned: usize, initial_len: usize, remaining_len: usize, ) -> Option { match num_spawned { x if x >= self.max_num_threads => None, _ => match self.resolved_chunk_size { ResolvedChunkSize::Exact(x) => Some(x), ResolvedChunkSize::Min(x) => { let chunk_size = match num_spawned { 0 => x, _ => { let done = initial_len - remaining_len; let done_per_thread = done / num_spawned; let num_chunks_per_thread = (done_per_thread / x).max(1); let num_chunks_per_thread = num_chunks_per_thread.max(1); num_chunks_per_thread * x } }; Some(chunk_size) } }, } } } impl ParallelExecutor for FixedChunkRunner { type SharedState = (); type ThreadExecutor = FixedChunkThreadExecutor; fn new( kind: ComputationKind, params: Params, initial_len: Option, max_num_threads: NonZeroUsize, ) -> Self { let resolved_chunk_size = ResolvedChunkSize::new(kind, initial_len, max_num_threads, params.chunk_size); Self { initial_len, resolved_chunk_size, max_num_threads: max_num_threads.into(), current_chunk_size: resolved_chunk_size.chunk_size().into(), } } fn new_shared_state(&self) -> Self::SharedState {} fn do_spawn_new(&self, num_spawned: NumSpawned, _: &Self::SharedState, iter: &I) -> bool where I: ConcurrentIter, { let num_spawned = num_spawned.into_inner(); if num_spawned.is_multiple_of(LAG_PERIODICITY) { match self.next_chunk(num_spawned, iter.try_get_len()) { Some(c) => self.current_chunk_size.store(c, Ordering::Relaxed), None => return false, } } self.spawn_new(num_spawned, iter.try_get_len()) } fn new_thread_executor(&self, _: usize, _: &Self::SharedState) -> Self::ThreadExecutor { Self::ThreadExecutor { chunk_size: self.current_chunk_size.load(Ordering::Relaxed), } } fn complete_task(self, _: Self::SharedState) {} } ================================================ FILE: src/executor/fixed_chunk_executor/thread_executor.rs ================================================ use crate::executor::thread_executor::ThreadExecutor; use orx_concurrent_iter::ConcurrentIter; pub struct FixedChunkThreadExecutor { pub(super) chunk_size: usize, } impl ThreadExecutor for FixedChunkThreadExecutor { type SharedState = (); #[inline(always)] fn next_chunk_size(&self, _: &Self::SharedState, _: &I) -> usize where I: ConcurrentIter, { self.chunk_size } fn begin_chunk(&mut self, _: usize) {} fn complete_chunk(&mut self, _: &Self::SharedState, _: usize) {} fn complete_task(&mut self, _: &Self::SharedState) {} } ================================================ FILE: src/executor/mod.rs ================================================ #[cfg(feature = "std")] mod executor_with_diagnostics; mod fixed_chunk_executor; pub(crate) mod parallel_compute; mod parallel_executor; mod thread_compute; mod thread_executor; #[cfg(feature = "std")] pub use executor_with_diagnostics::ParallelExecutorWithDiagnostics; pub use parallel_executor::ParallelExecutor; pub use thread_executor::ThreadExecutor; /// Default parallel executor. pub type DefaultExecutor = fixed_chunk_executor::FixedChunkRunner; ================================================ FILE: src/executor/parallel_compute/collect_arbitrary.rs ================================================ use crate::Params; use crate::executor::thread_compute as th; use crate::generic_values::Values; use crate::generic_values::runner_results::ParallelCollectArbitrary; use crate::runner::{ComputationKind, NumSpawned, ParallelRunner, SharedStateOf, ThreadRunnerOf}; use orx_concurrent_bag::ConcurrentBag; use orx_concurrent_iter::ConcurrentIter; use orx_fixed_vec::IntoConcurrentPinnedVec; #[cfg(test)] pub fn m( mut orchestrator: C, params: Params, iter: I, map1: M1, pinned_vec: P, ) -> (NumSpawned, P) where C: ParallelRunner, I: ConcurrentIter, O: Send, M1: Fn(I::Item) -> O + Sync, P: IntoConcurrentPinnedVec, { let capacity_bound = pinned_vec.capacity_bound(); let offset = pinned_vec.len(); let mut bag: ConcurrentBag = pinned_vec.into(); match iter.try_get_len() { Some(iter_len) => bag.reserve_maximum_capacity(offset + iter_len), None => bag.reserve_maximum_capacity(capacity_bound), }; let thread_work = |_, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { th::collect_arbitrary::m(thread_runner, iter, state, &map1, &bag); }; let num_spawned = orchestrator.run_all(params, iter, ComputationKind::Collect, thread_work); let values = bag.into_inner(); (num_spawned, values) } pub fn x( mut orchestrator: C, params: Params, iter: I, xap1: X1, pinned_vec: P, ) -> (NumSpawned, ParallelCollectArbitrary) where C: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { let capacity_bound = pinned_vec.capacity_bound(); let offset = pinned_vec.len(); let mut bag: ConcurrentBag = pinned_vec.into(); match iter.try_get_len() { Some(iter_len) => bag.reserve_maximum_capacity(offset + iter_len), None => bag.reserve_maximum_capacity(capacity_bound), }; let thread_map = |_, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { th::collect_arbitrary::x(thread_runner, iter, state, &xap1, &bag).into_result() }; let (num_spawned, result) = orchestrator.map_all::( params, iter, ComputationKind::Collect, thread_map, ); let result = match result { Err(error) => ParallelCollectArbitrary::StoppedByError { error }, Ok(_) => ParallelCollectArbitrary::AllOrUntilWhileCollected { pinned_vec: bag.into_inner(), }, }; (num_spawned, result) } ================================================ FILE: src/executor/parallel_compute/collect_ordered.rs ================================================ use crate::Params; use crate::executor::thread_compute as th; use crate::generic_values::Values; use crate::generic_values::runner_results::{Fallibility, ParallelCollect}; use crate::runner::{ComputationKind, NumSpawned, ParallelRunner, SharedStateOf, ThreadRunnerOf}; use orx_concurrent_iter::ConcurrentIter; use orx_concurrent_ordered_bag::ConcurrentOrderedBag; use orx_fixed_vec::IntoConcurrentPinnedVec; pub fn m( mut orchestrator: C, params: Params, iter: I, map1: M1, pinned_vec: P, ) -> (NumSpawned, P) where C: ParallelRunner, I: ConcurrentIter, O: Send, M1: Fn(I::Item) -> O + Sync, P: IntoConcurrentPinnedVec, { let offset = pinned_vec.len(); let o_bag: ConcurrentOrderedBag = pinned_vec.into(); let thread_do = |_, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { th::collect_ordered::m(thread_runner, iter, state, &map1, &o_bag, offset); }; let num_spawned = orchestrator.run_all(params, iter, ComputationKind::Collect, thread_do); let values = unsafe { o_bag.into_inner().unwrap_only_if_counts_match() }; (num_spawned, values) } pub fn x( mut orchestrator: C, params: Params, iter: I, xap1: X1, pinned_vec: P, ) -> (NumSpawned, ParallelCollect) where C: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, ::Error: Send, X1: Fn(I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { let thread_map = |_, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { th::collect_ordered::x(thread_runner, iter, state, &xap1).into_result() }; let (num_spawned, result) = orchestrator.map_all::( params, iter, ComputationKind::Collect, thread_map, ); let result = match result { Err(error) => ParallelCollect::StoppedByError { error }, Ok(results) => ParallelCollect::reduce(results, pinned_vec), }; (num_spawned, result) } ================================================ FILE: src/executor/parallel_compute/mod.rs ================================================ pub(crate) mod collect_arbitrary; pub(crate) mod collect_ordered; pub(crate) mod next; pub(crate) mod next_any; pub(crate) mod reduce; ================================================ FILE: src/executor/parallel_compute/next.rs ================================================ use crate::Params; use crate::executor::thread_compute as th; use crate::generic_values::Values; use crate::generic_values::runner_results::{Fallibility, NextSuccess, NextWithIdx}; use crate::runner::{ComputationKind, NumSpawned, ParallelRunner, SharedStateOf}; use orx_concurrent_iter::ConcurrentIter; pub fn m( mut orchestrator: C, params: Params, iter: I, map1: M1, ) -> (NumSpawned, Option) where C: ParallelRunner, I: ConcurrentIter, O: Send, M1: Fn(I::Item) -> O + Sync, { let thread_map = |_, iter: &I, state: &SharedStateOf, thread_runner| { Ok(th::next::m(thread_runner, iter, state, &map1)) }; let (num_spawned, result) = orchestrator.map_infallible(params, iter, ComputationKind::Collect, thread_map); let next = match result { Ok(results) => results .into_iter() .flatten() .min_by_key(|x| x.0) .map(|x| x.1), }; (num_spawned, next) } type ResultNext = Result< Option<(usize, ::Item)>, <::Fallibility as Fallibility>::Error, >; pub fn x( mut orchestrator: C, params: Params, iter: I, xap1: X1, ) -> (NumSpawned, ResultNext) where C: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(I::Item) -> Vo + Sync, { let thread_map = |_, iter: &I, state: &SharedStateOf, th_runner| match th::next::x( th_runner, iter, state, &xap1, ) { NextWithIdx::Found { idx, value } => Ok(Some(NextSuccess::Found { idx, value })), NextWithIdx::NotFound => Ok(None), NextWithIdx::StoppedByWhileCondition { idx } => { Ok(Some(NextSuccess::StoppedByWhileCondition { idx })) } NextWithIdx::StoppedByError { error } => Err(error), }; let (num_spawned, result) = orchestrator.map_all::( params, iter, ComputationKind::Collect, thread_map, ); let next = result.map(|results| NextSuccess::reduce(results.into_iter().flatten())); (num_spawned, next) } ================================================ FILE: src/executor/parallel_compute/next_any.rs ================================================ use crate::Params; use crate::executor::thread_compute as th; use crate::generic_values::Values; use crate::generic_values::runner_results::Fallibility; use crate::runner::{ComputationKind, NumSpawned, ParallelRunner, SharedStateOf}; use orx_concurrent_iter::ConcurrentIter; pub fn m( mut orchestrator: C, params: Params, iter: I, map1: M1, ) -> (NumSpawned, Option) where C: ParallelRunner, I: ConcurrentIter, O: Send, M1: Fn(I::Item) -> O + Sync, { let thread_map = |_, iter: &I, state: &SharedStateOf, thread_runner| { Ok(th::next_any::m(thread_runner, iter, state, &map1)) }; let (num_spawned, result) = orchestrator.map_infallible(params, iter, ComputationKind::Collect, thread_map); let next = match result { Ok(results) => results.into_iter().flatten().next(), }; (num_spawned, next) } type ResultNextAny = Result::Item>, <::Fallibility as Fallibility>::Error>; pub fn x( mut orchestrator: C, params: Params, iter: I, xap1: X1, ) -> (NumSpawned, ResultNextAny) where C: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(I::Item) -> Vo + Sync, { let thread_map = |_, iter: &I, state: &SharedStateOf, th_runner| { th::next_any::x(th_runner, iter, state, &xap1) }; let (num_spawned, result) = orchestrator.map_all::( params, iter, ComputationKind::Collect, thread_map, ); let next = result.map(|results| results.into_iter().flatten().next()); (num_spawned, next) } ================================================ FILE: src/executor/parallel_compute/reduce.rs ================================================ use crate::Params; use crate::executor::thread_compute as th; use crate::generic_values::Values; use crate::generic_values::runner_results::Fallibility; use crate::runner::{ComputationKind, NumSpawned, ParallelRunner, SharedStateOf, ThreadRunnerOf}; use orx_concurrent_iter::ConcurrentIter; pub fn m( mut orchestrator: C, params: Params, iter: I, map1: M1, reduce: Red, ) -> (NumSpawned, Option) where C: ParallelRunner, I: ConcurrentIter, M1: Fn(I::Item) -> O + Sync, Red: Fn(O, O) -> O + Sync, O: Send, { let thread_map = |_, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { Ok(th::reduce::m(thread_runner, iter, state, &map1, &reduce)) }; let (num_spawned, result) = orchestrator.map_infallible(params, iter, ComputationKind::Collect, thread_map); let acc = match result { Ok(results) => results.into_iter().flatten().reduce(reduce), }; (num_spawned, acc) } type ResultReduce = Result::Item>, <::Fallibility as Fallibility>::Error>; pub fn x( mut orchestrator: C, params: Params, iter: I, xap1: X1, reduce: Red, ) -> (NumSpawned, ResultReduce) where C: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(I::Item) -> Vo + Sync, Red: Fn(Vo::Item, Vo::Item) -> Vo::Item + Sync, { let thread_map = |_, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { th::reduce::x(thread_runner, iter, state, &xap1, &reduce).into_result() }; let (num_spawned, result) = orchestrator.map_all::( params, iter, ComputationKind::Collect, thread_map, ); let acc = result.map(|results| results.into_iter().flatten().reduce(reduce)); (num_spawned, acc) } ================================================ FILE: src/executor/parallel_executor.rs ================================================ use super::thread_executor::ThreadExecutor; use crate::{ parameters::Params, runner::{ComputationKind, NumSpawned}, }; use core::num::NonZeroUsize; use orx_concurrent_iter::ConcurrentIter; /// A parallel executor which is responsible for taking a computation defined as a composition /// of iterator methods, spawns threads, shares tasks and returns the result of the parallel /// execution. pub trait ParallelExecutor: Sized + Sync + 'static + Clone { /// Data shared to the thread executors. type SharedState: Send + Sync; /// Thread executor that is responsible for executing the tasks allocated to a thread. type ThreadExecutor: ThreadExecutor; /// Creates a new parallel executor for the given computation `kind`, parallelization `params` /// and `initial_input_len`. fn new( kind: ComputationKind, params: Params, initial_input_len: Option, max_num_threads: NonZeroUsize, ) -> Self; /// Creates an initial shared state. fn new_shared_state(&self) -> Self::SharedState; /// Returns true if it is beneficial to spawn a new thread provided that: /// /// * `num_spawned` threads are already been spawned, and /// * `shared_state` is the current parallel execution state. fn do_spawn_new( &self, num_spawned: NumSpawned, shared_state: &Self::SharedState, iter: &I, ) -> bool where I: ConcurrentIter; /// Creates a new thread executor provided that the current parallel execution state is /// `shared_state`. fn new_thread_executor( &self, thread_idx: usize, shared_state: &Self::SharedState, ) -> Self::ThreadExecutor; /// Executes the finalization tasks when the entire parallel computation is completed. fn complete_task(self, shared_state: Self::SharedState); } ================================================ FILE: src/executor/thread_compute/collect_arbitrary.rs ================================================ use crate::ThreadExecutor; use crate::generic_values::Values; use crate::generic_values::runner_results::{Stop, ThreadCollectArbitrary}; use orx_concurrent_bag::ConcurrentBag; use orx_concurrent_iter::{ChunkPuller, ConcurrentIter}; use orx_fixed_vec::IntoConcurrentPinnedVec; // m #[cfg(test)] pub fn m( mut runner: C, iter: &I, shared_state: &C::SharedState, map1: &M1, bag: &ConcurrentBag, ) where C: ThreadExecutor, I: ConcurrentIter, M1: Fn(I::Item) -> O, P: IntoConcurrentPinnedVec, O: Send, { let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(value) => _ = bag.push(map1(value)), None => { if iter.is_completed_when_none_returned() { break; } } }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(chunk) => _ = bag.extend(chunk.map(&map1)), None => { if iter.is_completed_when_none_returned() { break; } } } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); } // x pub fn x( mut runner: C, iter: &I, shared_state: &C::SharedState, xap1: &X1, bag: &ConcurrentBag, ) -> ThreadCollectArbitrary where C: ThreadExecutor, I: ConcurrentIter, Vo: Values, X1: Fn(I::Item) -> Vo, P: IntoConcurrentPinnedVec, Vo::Item: Send, { let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(value) => { // TODO: possible to try to get len and bag.extend(values_vt.values()) when available, same holds for chunk below let vo = xap1(value); let done = vo.push_to_bag(bag); if let Some(stop) = Vo::arbitrary_push_to_stop(done) { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { Stop::DueToWhile => { return ThreadCollectArbitrary::StoppedByWhileCondition; } Stop::DueToError { error } => { return ThreadCollectArbitrary::StoppedByError { error }; } } } } None => { if iter.is_completed_when_none_returned() { break; } } }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(chunk) => { for value in chunk { let vo = xap1(value); let done = vo.push_to_bag(bag); if let Some(stop) = Vo::arbitrary_push_to_stop(done) { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { Stop::DueToWhile => { return ThreadCollectArbitrary::StoppedByWhileCondition; } Stop::DueToError { error } => { return ThreadCollectArbitrary::StoppedByError { error }; } } } } } None => { if iter.is_completed_when_none_returned() { break; } } } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); ThreadCollectArbitrary::AllCollected } ================================================ FILE: src/executor/thread_compute/collect_ordered.rs ================================================ use crate::ThreadExecutor; use crate::generic_values::Values; use crate::generic_values::runner_results::{StopWithIdx, ThreadCollect}; use alloc::vec::Vec; use orx_concurrent_iter::{ChunkPuller, ConcurrentIter}; use orx_concurrent_ordered_bag::ConcurrentOrderedBag; use orx_fixed_vec::IntoConcurrentPinnedVec; pub fn m( mut runner: C, iter: &I, shared_state: &C::SharedState, map1: &M1, o_bag: &ConcurrentOrderedBag, offset: usize, ) where C: ThreadExecutor, I: ConcurrentIter, M1: Fn(I::Item) -> O, P: IntoConcurrentPinnedVec, O: Send, { let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller_with_idx(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some((idx, value)) => unsafe { o_bag.set_value(offset + idx, map1(value)) }, None => { if iter.is_completed_when_none_returned() { break; } } }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull_with_idx() { Some((begin_idx, chunk)) => { let values = chunk.map(map1); unsafe { o_bag.set_values(offset + begin_idx, values) }; } None => { if iter.is_completed_when_none_returned() { break; } } } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); } pub fn x( mut runner: C, iter: &I, shared_state: &C::SharedState, xap1: &X1, ) -> ThreadCollect where C: ThreadExecutor, I: ConcurrentIter, Vo: Values, X1: Fn(I::Item) -> Vo, { let mut collected = Vec::new(); let out_vec = &mut collected; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller_with_idx(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some((idx, i)) => { let vo = xap1(i); let done = vo.push_to_vec_with_idx(idx, out_vec); if let Some(stop) = Vo::ordered_push_to_stop(done) { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { StopWithIdx::DueToWhile { idx } => { return ThreadCollect::StoppedByWhileCondition { vec: collected, stopped_idx: idx, }; } StopWithIdx::DueToError { idx: _, error } => { return ThreadCollect::StoppedByError { error }; } } } } None => { if iter.is_completed_when_none_returned() { break; } } }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull_with_idx() { Some((chunk_begin_idx, chunk)) => { for (within_chunk_idx, value) in chunk.enumerate() { let vo = xap1(value); let done = vo.push_to_vec_with_idx(chunk_begin_idx, out_vec); if let Some(stop) = Vo::ordered_push_to_stop(done) { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { StopWithIdx::DueToWhile { idx } => { return ThreadCollect::StoppedByWhileCondition { vec: collected, stopped_idx: idx + within_chunk_idx, }; } StopWithIdx::DueToError { idx: _, error } => { return ThreadCollect::StoppedByError { error }; } } } } } None => { if iter.is_completed_when_none_returned() { break; } } } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); ThreadCollect::AllCollected { vec: collected } } ================================================ FILE: src/executor/thread_compute/mod.rs ================================================ pub(super) mod collect_arbitrary; pub(super) mod collect_ordered; pub(super) mod next; pub(super) mod next_any; pub(super) mod reduce; ================================================ FILE: src/executor/thread_compute/next.rs ================================================ use crate::{ ThreadExecutor, generic_values::Values, generic_values::runner_results::{Next, NextWithIdx}, }; use orx_concurrent_iter::{ChunkPuller, ConcurrentIter}; pub fn m( mut runner: C, iter: &I, shared_state: &C::SharedState, map1: &M1, ) -> Option<(usize, O)> where C: ThreadExecutor, I: ConcurrentIter, M1: Fn(I::Item) -> O, { let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller_with_idx(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some((idx, i)) => { let first = map1(i); iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Some((idx, first)); } None => { if iter.is_completed_when_none_returned() { break; } } }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull_with_idx() { Some((idx, mut chunk)) => { if let Some(i) = chunk.next() { let first = map1(i); iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Some((idx, first)); } } None => { if iter.is_completed_when_none_returned() { break; } } } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); None } pub fn x( mut runner: C, iter: &I, shared_state: &C::SharedState, xap1: &X1, ) -> NextWithIdx where C: ThreadExecutor, I: ConcurrentIter, Vo: Values, X1: Fn(I::Item) -> Vo, { let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller_with_idx(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some((idx, i)) => { let vt = xap1(i); match vt.next() { Next::Done { value } => { if let Some(value) = value { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::Found { idx, value }; } } Next::StoppedByError { error } => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::StoppedByError { error }; } Next::StoppedByWhileCondition => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::StoppedByWhileCondition { idx }; } } } None => { if iter.is_completed_when_none_returned() { break; } } }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull_with_idx() { Some((idx, chunk)) => { for i in chunk { let vt = xap1(i); match vt.next() { Next::Done { value } => { if let Some(value) = value { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::Found { idx, value }; } } Next::StoppedByError { error } => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::StoppedByError { error }; } Next::StoppedByWhileCondition => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::StoppedByWhileCondition { idx }; } } } } None => { if iter.is_completed_when_none_returned() { break; } } } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); NextWithIdx::NotFound } ================================================ FILE: src/executor/thread_compute/next_any.rs ================================================ use crate::{ ThreadExecutor, generic_values::Values, generic_values::runner_results::{Fallibility, Next}, }; use orx_concurrent_iter::{ChunkPuller, ConcurrentIter}; pub fn m( mut runner: C, iter: &I, shared_state: &C::SharedState, map1: &M1, ) -> Option where C: ThreadExecutor, I: ConcurrentIter, O: Send, M1: Fn(I::Item) -> O, { let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(i) => { let first = map1(i); iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Some(first); } None => { if iter.is_completed_when_none_returned() { break; } } }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(mut chunk) => { if let Some(i) = chunk.next() { let first = map1(i); iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Some(first); } } None => { if iter.is_completed_when_none_returned() { break; } } } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); None } pub fn x( mut runner: C, iter: &I, shared_state: &C::SharedState, xap1: &X1, ) -> Result, ::Error> where C: ThreadExecutor, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(I::Item) -> Vo, { let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(i) => { let vt = xap1(i); match vt.next() { Next::Done { value } => { if let Some(value) = value { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Ok(Some(value)); } } Next::StoppedByError { error } => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Err(error); } Next::StoppedByWhileCondition => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Ok(None); } } } None => { if iter.is_completed_when_none_returned() { break; } } }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(chunk) => { for i in chunk { let vt = xap1(i); match vt.next() { Next::Done { value } => { if let Some(value) = value { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Ok(Some(value)); } } Next::StoppedByError { error } => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Err(error); } Next::StoppedByWhileCondition => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Ok(None); } } } } None => { if iter.is_completed_when_none_returned() { break; } } } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); Ok(None) } ================================================ FILE: src/executor/thread_compute/reduce.rs ================================================ use crate::{ ThreadExecutor, generic_values::{ Values, runner_results::{Reduce, StopReduce}, }, }; use orx_concurrent_iter::{ChunkPuller, ConcurrentIter}; // m pub fn m( mut runner: C, iter: &I, shared_state: &C::SharedState, map1: &M1, reduce: &Red, ) -> Option where C: ThreadExecutor, I: ConcurrentIter, M1: Fn(I::Item) -> O, Red: Fn(O, O) -> O, { let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); let mut acc = None; loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(i) => { let y = map1(i); acc = match acc { Some(x) => Some(reduce(x, y)), None => Some(y), }; } None => { if iter.is_completed_when_none_returned() { break; } } }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(chunk) => { // println!("chunk = {}", chunk.len()); let res = chunk.map(map1).reduce(reduce); acc = match acc { Some(x) => match res { Some(y) => Some(reduce(x, y)), None => Some(x), }, None => res, }; } None => { if iter.is_completed_when_none_returned() { break; } } } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); acc } // x pub fn x( mut runner: C, iter: &I, shared_state: &C::SharedState, xap1: &X1, reduce: &Red, ) -> Reduce where C: ThreadExecutor, I: ConcurrentIter, Vo: Values, X1: Fn(I::Item) -> Vo, Red: Fn(Vo::Item, Vo::Item) -> Vo::Item, { let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); let mut acc = None; loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(i) => { let vo = xap1(i); let reduce = vo.acc_reduce(acc, reduce); acc = match Vo::reduce_to_stop(reduce) { Ok(acc) => acc, Err(stop) => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { StopReduce::DueToWhile { acc } => { return Reduce::StoppedByWhileCondition { acc }; } StopReduce::DueToError { error } => { return Reduce::StoppedByError { error }; } } } }; } None => { if iter.is_completed_when_none_returned() { break; } } }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(chunk) => { for i in chunk { let vo = xap1(i); let reduce = vo.acc_reduce(acc, reduce); acc = match Vo::reduce_to_stop(reduce) { Ok(acc) => acc, Err(stop) => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { StopReduce::DueToWhile { acc } => { return Reduce::StoppedByWhileCondition { acc }; } StopReduce::DueToError { error } => { return Reduce::StoppedByError { error }; } } } }; } } None => { if iter.is_completed_when_none_returned() { break; } } } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); Reduce::Done { acc } } ================================================ FILE: src/executor/thread_executor.rs ================================================ use orx_concurrent_iter::ConcurrentIter; /// Thread executor responsible for executing the tasks assigned to the thread by the /// parallel executor. pub trait ThreadExecutor: Sized { /// Type of the shared state among threads. type SharedState; /// Returns the next chunks size to be pulled from the input `iter` for the given /// current `shared_state`. fn next_chunk_size(&self, shared_state: &Self::SharedState, iter: &I) -> usize where I: ConcurrentIter; /// Hook that will be called before starting to execute the chunk of the given `chunk_size`. fn begin_chunk(&mut self, chunk_size: usize); /// Hook that will be called after completing the chunk of the given `chunk_size`. /// The `shared_state` is also provided so that it can be updated to send information to the /// parallel executor and other thread executors. fn complete_chunk(&mut self, shared_state: &Self::SharedState, chunk_size: usize); /// Hook that will be called after completing the task. /// The `shared_state` is also provided so that it can be updated to send information to the /// parallel executor and other thread executors. fn complete_task(&mut self, shared_state: &Self::SharedState); } ================================================ FILE: src/experiment/algorithms/merge_sorted_slices/mod.rs ================================================ #[cfg(test)] mod tests; /// Sequential variant for merging two sorted slices into one sorted slice. pub mod seq; /// Parallel variant for merging two sorted slices into one sorted slice. pub mod par; ================================================ FILE: src/experiment/algorithms/merge_sorted_slices/par.rs ================================================ use crate::experiment::algorithms::merge_sorted_slices::seq::{ ParamsSeqMergeSortedSlices, bin_search_idx, seq_merge_unchecked, }; use crate::experiment::data_structures::{slice_dst::SliceDst, slice_src::SliceSrc}; use crate::{IntoParIterRec, ParIter, ParallelRunner}; use core::cmp::Ordering; use orx_concurrent_recursive_iter::Queue; /// Determines how to search the pivot for splitting the slices. #[derive(Clone, Copy, Debug)] pub enum PivotSearch { /// We search the pivot position by a linear search. Linear, /// We search the pivot position by a binary search since both source slices are sorted. Binary, } /// Parameters of the sequential algorithm for merging two sorted slices into one sorted slice. #[derive(Clone, Copy, Debug)] pub struct ParamsParMergeSortedSlices { /// Parameters of sequential merging. pub seq_params: ParamsSeqMergeSortedSlices, /// When true, the algorithm always puts the larger slice to the left; /// otherwise to the right. pub put_large_to_left: bool, /// Determines how to search the pivot for splitting the slices. pub pivot_search: PivotSearch, /// Number of threads. pub num_threads: usize, /// Chunk size to be used by parallelization. pub chunk_size: usize, /// Minimum length of a slice to be split into two tasks. pub min_split_len: usize, } struct Task<'a, T> { left: SliceSrc<'a, T>, right: SliceSrc<'a, T>, target: SliceDst<'a, T>, } unsafe impl<'a, T> Send for Task<'a, T> {} impl<'a, T> Task<'a, T> { /// Clones the task. /// /// # SAFETY /// /// The purpose of destination slice `target` is to mutate the underlying memory. /// Therefore, cloning task is marked as unsafe. /// /// - (i) assuming the clone will be used to mutate the memory, caller must /// ensure that `&self.target` will not be used. unsafe fn clone(&self) -> Self { Self { left: self.left.clone(), right: self.right.clone(), target: unsafe { self.target.clone() }, } } } /// # Panics /// /// - (i) if `target.len()` is not equal to `left.len() + right.len()` /// - (ii) if any pair of of `left`, `right` or `target` are overlapping. pub fn par_merge<'a, T, F, R: ParallelRunner>( is_leq: F, left: SliceSrc<'a, T>, right: SliceSrc<'a, T>, target: SliceDst<'a, T>, params: &ParamsParMergeSortedSlices, runner: R, ) where T: Send + Sync, F: Fn(&T, &T) -> bool + Sync, { assert_eq!(target.len(), left.len() + right.len()); assert!(target.core().is_non_overlapping(&left.core())); assert!(target.core().is_non_overlapping(&right.core())); assert!(left.core().is_non_overlapping(&right.core())); let initial_task = [Task { left, right, target, }]; let handle_extend = |task: &Task<'a, T>, queue: &Queue<'_, Task<'a, T>>| { // SAFETY: req't (i) and (ii) are satisfied by panic conditions unsafe { handle_extend(&is_leq, params, task, queue) } }; initial_task .into_par_rec(handle_extend) .with_runner(runner) .num_threads(params.num_threads) .chunk_size(params.chunk_size) .for_each(|_| {}); } /// # SAFETY /// /// - (i) `target.len()` must equal `left.len() + right.len()` /// - (ii) no pair of `left`, `right` and `target` can be overlapping. unsafe fn handle_extend<'a, T, F>( is_leq: F, params: &ParamsParMergeSortedSlices, task: &Task<'a, T>, queue: &Queue<'_, Task<'a, T>>, ) where T: Send + Sync, F: Fn(&T, &T) -> bool, { let min_split_len = core::cmp::min(params.min_split_len, 3); // SAFETY: this method both handles and extends the queue; which will be // visited only once; hence, the reference `task` will not be used to // mutate the underlying memory, satisfying condition (i). let task = unsafe { task.clone() }; let (mut left, mut right, target) = (task.left, task.right, task.target); match (left.len(), right.len()) { (x, y) if x < min_split_len || y < min_split_len => { // SAFETY: req't (i) & (ii) are satisfied by conditions (i) & (ii) unsafe { seq_merge_unchecked(is_leq, left, right, target, ¶ms.seq_params) }; } _ => { let is_large_on_left = left.len() >= right.len(); if is_large_on_left != params.put_large_to_left { (left, right) = (right, left); } let position = left.len() / 2; // SAFETY: position <= self.len() let [left_left, left_right] = unsafe { left.split_at_unchecked(position) }; // SAFETY: since left.len() >= 2, then left_right.len() > 0 let pivot = unsafe { left_right.first_unchecked() }; let pos_right = match params.pivot_search { PivotSearch::Linear => right .values() .position(|r| is_leq(pivot, r)) .unwrap_or(right.len()), PivotSearch::Binary => bin_search_idx(right.as_slice().binary_search_by(|r| { match is_leq(r, pivot) { true => Ordering::Less, false => Ordering::Greater, } })), }; // SAFETY: (i) pos_right <= right.len() is satisfied by the expression declaring pos_right let [right_left, right_right] = unsafe { right.split_at_unchecked(pos_right) }; let target_left_len = left_left.len() + right_left.len(); // SAFETY: (i) target_left_len <= target.len() is satisfied by cond'n (i) target.len() == left.len() + right.len() let [target_left, target_right] = unsafe { target.split_at_unchecked(target_left_len) }; let task_left = Task { left: left_left, right: right_left, target: target_left, }; let task_right = Task { left: left_right, right: right_right, target: target_right, }; queue.extend([task_left, task_right]); } } } ================================================ FILE: src/experiment/algorithms/merge_sorted_slices/seq.rs ================================================ use crate::experiment::data_structures::{slice_dst::SliceDst, slice_src::SliceSrc}; use core::cmp::Ordering; pub(super) fn bin_search_idx(idx: Result) -> usize { match idx { Ok(x) => x, Err(x) => x, } } /// Determines the streak search. /// /// Assume at an intermediate step of the algorithm, current value of left slice is /// less than current value of right slice. /// /// In this case, we will copy the current element of the left slice to the current /// position of the target slice. /// /// However, if next `n` elements of the left slice are all smaller than the current /// value of the right slice, we can copy all `n` elements at once. The subslice of /// this `n` elements is called the streak. #[derive(Clone, Copy, Debug)] pub enum StreakSearch { /// We don't search for a streak; we copy elements one by one. None, /// We search the streak by a linear search. Linear, /// We search the streak by a binary search since both source slices are sorted. Binary, } /// Parameters of the sequential algorithm for merging two sorted slices into one sorted slice. #[derive(Clone, Copy, Debug)] pub struct ParamsSeqMergeSortedSlices { /// Streak search method. pub streak_search: StreakSearch, /// When true, the algorithm always puts the larger slice to the left; /// otherwise to the right. pub put_large_to_left: bool, } /// # Panics /// /// - (i) if `target.len()` is not equal to `left.len() + right.len()` /// - (ii) if any pair of of `left`, `right` or `target` are overlapping. pub fn seq_merge<'a, T: 'a, F>( is_leq: F, left: SliceSrc<'a, T>, right: SliceSrc<'a, T>, target: SliceDst<'a, T>, params: &ParamsSeqMergeSortedSlices, ) where F: Fn(&T, &T) -> bool, { assert_eq!(target.len(), left.len() + right.len()); assert!(target.core().is_non_overlapping(&left.core())); assert!(target.core().is_non_overlapping(&right.core())); assert!(left.core().is_non_overlapping(&right.core())); // SAFETY: safety requirements are satisfied by panic conditions (i) and (ii) unsafe { seq_merge_unchecked(is_leq, left, right, target, params) } } /// # SAFETY /// /// - (i) `target.len()` must equal `left.len() + right.len()` /// - (ii) no pair of `left`, `right` and `target` can be overlapping. pub unsafe fn seq_merge_unchecked<'a, T: 'a, F>( is_leq: F, left: SliceSrc<'a, T>, right: SliceSrc<'a, T>, target: SliceDst<'a, T>, params: &ParamsSeqMergeSortedSlices, ) where F: Fn(&T, &T) -> bool, { // SAFETY: safety requirements are satisfied by safety conditions (i) and (ii) unsafe { match params.streak_search { StreakSearch::None => { seq_merge_streak_none(is_leq, left, right, target, params.put_large_to_left) } StreakSearch::Linear => { seq_merge_streak_linear(is_leq, left, right, target, params.put_large_to_left) } StreakSearch::Binary => { seq_merge_streak_binary(is_leq, left, right, target, params.put_large_to_left) } } } } /// # SAFETY /// /// - (i) `target.len()` must equal `left.len() + right.len()` /// - (ii) no pair of `left`, `right` and `target` can be overlapping. unsafe fn seq_merge_streak_none<'a, T: 'a, F>( is_leq: F, mut left: SliceSrc<'a, T>, mut right: SliceSrc<'a, T>, target: SliceDst<'a, T>, put_large_to_left: bool, ) where F: Fn(&T, &T) -> bool, { let is_large_on_left = left.len() >= right.len(); if is_large_on_left != put_large_to_left { (left, right) = (right, left); } let mut left = left.into_ptr_iter(); let mut right = right.into_ptr_iter(); let mut dst = target.into_ptr_iter(); match (left.current(), right.current()) { (Some(mut l), Some(mut r)) => { loop { match is_leq(l, r) { true => { // SAFETY: left still has at least one elem `l`, so must `dst` unsafe { dst.write_one_from(&mut left) }; match left.current() { Some(x) => l = x, None => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut right) }; break; } } } false => { // SAFETY: right still has at least one elem `r`, so must `dst` unsafe { dst.write_one_from(&mut right) }; match right.current() { Some(x) => r = x, None => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut left) }; break; } } } } } } (None, None) => {} (None, _) => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut right) }; } (_, None) => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut left) }; } } } /// # SAFETY /// /// - (i) `target.len()` must equal `left.len() + right.len()` /// - (ii) no pair of `left`, `right` and `target` can be overlapping. unsafe fn seq_merge_streak_linear<'a, T: 'a, F>( is_leq: F, mut left: SliceSrc<'a, T>, mut right: SliceSrc<'a, T>, target: SliceDst<'a, T>, put_large_to_left: bool, ) where F: Fn(&T, &T) -> bool, { let is_large_on_left = left.len() >= right.len(); if is_large_on_left != put_large_to_left { (left, right) = (right, left); } let mut left = left.into_ptr_iter(); let mut right = right.into_ptr_iter(); let mut dst = target.into_ptr_iter(); match (left.current(), right.current()) { (Some(mut l), Some(mut r)) => { loop { match is_leq(l, r) { true => { let count = match left.values().position(|x| !is_leq(x, r)) { Some(idx_bigger) => idx_bigger, None => left.len(), }; // SAFETY: left still has at least `count` elements, so must `dst` unsafe { dst.write_many_from(&mut left, count) }; match left.current() { Some(x) => l = x, None => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut right) }; break; } } } false => { let count = match right.values().position(|x| !is_leq(x, l)) { Some(idx_bigger) => idx_bigger, None => right.len(), }; // SAFETY: right still has at least `count` elements, so must `dst` unsafe { dst.write_many_from(&mut right, count) }; match right.current() { Some(x) => r = x, None => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut left) }; break; } } } } } } (None, _) => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut right) }; } (_, None) => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut left) }; } } } /// # SAFETY /// /// - (i) `target.len()` must equal `left.len() + right.len()` /// - (ii) no pair of `left`, `right` and `target` can be overlapping. unsafe fn seq_merge_streak_binary<'a, T: 'a, F>( is_leq: F, mut left: SliceSrc<'a, T>, mut right: SliceSrc<'a, T>, target: SliceDst<'a, T>, put_large_to_left: bool, ) where F: Fn(&T, &T) -> bool, { let is_large_on_left = left.len() >= right.len(); if is_large_on_left != put_large_to_left { (left, right) = (right, left); } let mut left = left.into_ptr_iter(); let mut right = right.into_ptr_iter(); let mut dst = target.into_ptr_iter(); match (left.current(), right.current()) { (Some(mut l), Some(mut r)) => { loop { match is_leq(l, r) { true => { let count = bin_search_idx(left.as_slice().binary_search_by(|x| { match is_leq(x, r) { true => Ordering::Less, false => Ordering::Greater, } })); // SAFETY: left still has at least `count` elements, so must `dst` unsafe { dst.write_many_from(&mut left, count) }; match left.current() { Some(x) => l = x, None => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut right) }; break; } } } false => { let count = bin_search_idx(right.as_slice().binary_search_by(|x| { match is_leq(x, l) { true => Ordering::Less, false => Ordering::Greater, } })); // SAFETY: right still has at least `count` elements, so must `dst` unsafe { dst.write_many_from(&mut right, count) }; match right.current() { Some(x) => r = x, None => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut left) }; break; } } } } } } (None, _) => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut right) }; } (_, None) => { // SAFETY: target (i) and (ii) are satisfied by conditions (i) and (ii) unsafe { dst.write_rest_from(&mut left) }; } } } ================================================ FILE: src/experiment/algorithms/merge_sorted_slices/tests/inputs.rs ================================================ use alloc::string::{String, ToString}; use alloc::vec::Vec; use rand::prelude::*; use rand_chacha::ChaCha8Rng; #[derive(Clone, Copy)] pub enum SortKind { Sorted, ReverseSorted, Mixed, } pub fn sorted_slices( left_len: usize, total_len: usize, sort: SortKind, ) -> (Vec, Vec, Vec) { let mut all: Vec<_> = (0..total_len).map(|x| x.to_string()).collect(); match sort { SortKind::Sorted => all.sort(), SortKind::ReverseSorted => { all.sort(); all.reverse(); } SortKind::Mixed => { let num_shuffles = 10 * total_len; let mut rng = ChaCha8Rng::seed_from_u64(42); for _ in 0..num_shuffles { let i = rng.random_range(0..total_len); let j = rng.random_range(0..total_len); all.swap(i, j); } } } let (left, right) = all.split_at(left_len); let mut left = left.to_vec(); let mut right = right.to_vec(); left.sort(); right.sort(); (all, left, right) } ================================================ FILE: src/experiment/algorithms/merge_sorted_slices/tests/mod.rs ================================================ mod inputs; mod par; mod seq; ================================================ FILE: src/experiment/algorithms/merge_sorted_slices/tests/par.rs ================================================ use super::inputs::{SortKind, sorted_slices}; use crate::experiment::algorithms::merge_sorted_slices::par::{ ParamsParMergeSortedSlices, PivotSearch, par_merge, }; use crate::experiment::algorithms::merge_sorted_slices::seq::{ ParamsSeqMergeSortedSlices, StreakSearch, }; use crate::experiment::data_structures::slice_dst::SliceDst; use crate::experiment::data_structures::slice_src::SliceSrc; use crate::{DefaultPool, DefaultRunner, RunnerWithPool}; use alloc::string::String; use alloc::vec::Vec; use test_case::test_matrix; fn runner() -> RunnerWithPool { DefaultRunner::default() } const PARAMS: &[ParamsSeqMergeSortedSlices] = &[ ParamsSeqMergeSortedSlices { streak_search: StreakSearch::None, put_large_to_left: false, }, ParamsSeqMergeSortedSlices { streak_search: StreakSearch::Linear, put_large_to_left: false, }, ParamsSeqMergeSortedSlices { streak_search: StreakSearch::Binary, put_large_to_left: false, }, ParamsSeqMergeSortedSlices { streak_search: StreakSearch::None, put_large_to_left: true, }, ParamsSeqMergeSortedSlices { streak_search: StreakSearch::Linear, put_large_to_left: true, }, ParamsSeqMergeSortedSlices { streak_search: StreakSearch::Binary, put_large_to_left: true, }, ]; #[test_matrix( [(0, 0), (0, 5), (5, 5), (4, 20), (10, 20), (14, 20)], [SortKind::Sorted, SortKind::ReverseSorted, SortKind::Mixed], [PARAMS[0],PARAMS[1],PARAMS[2],PARAMS[3],PARAMS[4],PARAMS[5]], [PivotSearch::Linear, PivotSearch::Binary], [0, 1, 64]) ] fn merge_sorted_slices_par( (left_len, total_len): (usize, usize), sort: SortKind, seq_params: ParamsSeqMergeSortedSlices, pivot_search: PivotSearch, chunk_size: usize, ) { let params = ParamsParMergeSortedSlices { seq_params: seq_params, chunk_size, num_threads: 4, pivot_search, put_large_to_left: seq_params.put_large_to_left, min_split_len: 3, }; run((left_len, total_len), sort, params); } #[test_matrix( [(0, 0), (0, 5), (5, 5), (4, 20), (10, 20), (14, 20)]) ] fn merge_sorted_slices_par_single_thread((left_len, total_len): (usize, usize)) { let params = ParamsParMergeSortedSlices { seq_params: PARAMS[5], chunk_size: 1, num_threads: 1, pivot_search: PivotSearch::Binary, put_large_to_left: PARAMS[5].put_large_to_left, min_split_len: 3, }; run((left_len, total_len), SortKind::Mixed, params); } #[cfg(not(miri))] #[test_matrix( [1<<15], [0, 1, 64], [4, 16] )] fn merge_sorted_slices_par_large(len: usize, num_threads: usize, chunk_size: usize) { let params = ParamsParMergeSortedSlices { seq_params: ParamsSeqMergeSortedSlices { streak_search: StreakSearch::Linear, put_large_to_left: true, }, pivot_search: PivotSearch::Binary, put_large_to_left: true, min_split_len: 3, chunk_size, num_threads, }; run((len / 2, len), SortKind::Mixed, params); } fn run((left_len, total_len): (usize, usize), sort: SortKind, params: ParamsParMergeSortedSlices) { let (mut expected, mut left, mut right) = sorted_slices(left_len, total_len, sort); let mut result = Vec::::with_capacity(total_len); par_merge( |a, b| a < b, SliceSrc::from_slice(left.as_slice()), SliceSrc::from_slice(right.as_slice()), SliceDst::from_vec(&mut result), ¶ms, runner(), ); // all elements of left & right are moved to result unsafe { result.set_len(left.len() + right.len()); left.set_len(0); right.set_len(0); } expected.sort(); assert_eq!(result, expected); } ================================================ FILE: src/experiment/algorithms/merge_sorted_slices/tests/seq.rs ================================================ use super::inputs::{SortKind, sorted_slices}; use crate::experiment::algorithms::merge_sorted_slices::seq::{ ParamsSeqMergeSortedSlices, StreakSearch, seq_merge, }; use crate::experiment::data_structures::slice_dst::SliceDst; use crate::experiment::data_structures::slice_src::SliceSrc; use alloc::string::String; use alloc::vec::Vec; use test_case::test_matrix; const PARAMS: &[ParamsSeqMergeSortedSlices] = &[ ParamsSeqMergeSortedSlices { streak_search: StreakSearch::None, put_large_to_left: false, }, ParamsSeqMergeSortedSlices { streak_search: StreakSearch::Linear, put_large_to_left: false, }, ParamsSeqMergeSortedSlices { streak_search: StreakSearch::Binary, put_large_to_left: false, }, ParamsSeqMergeSortedSlices { streak_search: StreakSearch::None, put_large_to_left: true, }, ParamsSeqMergeSortedSlices { streak_search: StreakSearch::Linear, put_large_to_left: true, }, ParamsSeqMergeSortedSlices { streak_search: StreakSearch::Binary, put_large_to_left: true, }, ]; #[test_matrix( [(0, 0), (0, 5), (5, 5), (4, 20), (10, 20), (14, 20)], [SortKind::Sorted, SortKind::ReverseSorted, SortKind::Mixed], [PARAMS[0],PARAMS[1],PARAMS[2],PARAMS[3],PARAMS[4],PARAMS[5]]) ] fn merge_sorted_slices_seq( (left_len, total_len): (usize, usize), sort: SortKind, params: ParamsSeqMergeSortedSlices, ) { let (mut expected, mut left, mut right) = sorted_slices(left_len, total_len, sort); let mut result = Vec::::with_capacity(total_len); seq_merge( |a, b| a < b, SliceSrc::from_slice(left.as_slice()), SliceSrc::from_slice(right.as_slice()), SliceDst::from_vec(&mut result), ¶ms, ); // all elements of left & right are moved to result unsafe { result.set_len(left.len() + right.len()); left.set_len(0); right.set_len(0); } expected.sort(); assert_eq!(result, expected); } ================================================ FILE: src/experiment/algorithms/mod.rs ================================================ /// Merge two sorted slices into one sorted slice. pub mod merge_sorted_slices; ================================================ FILE: src/experiment/data_structures/mod.rs ================================================ #[cfg(test)] mod tests; /// A raw slice of contiguous data. pub mod slice; /// A raw slice of contiguous data with un-initialized values. pub mod slice_dst; /// Core structure for iterators over contiguous slices of data. pub mod slice_iter_ptr; /// Iterator over a slice of data that will be completely filled with values before the iterator is consumed. pub mod slice_iter_ptr_dst; /// Iterator over a slice of data that will be completely copied to another slice before the iterator is consumed. pub mod slice_iter_ptr_src; /// A raw slice of contiguous data with initialized values. pub mod slice_src; ================================================ FILE: src/experiment/data_structures/slice.rs ================================================ use core::{marker::PhantomData, ptr::slice_from_raw_parts}; /// A raw slice of contiguous data. pub struct Slice<'a, T: 'a> { raw: *const [T], phantom: PhantomData<&'a ()>, } impl<'a, T: 'a> Clone for Slice<'a, T> { fn clone(&self) -> Self { Self { raw: self.raw, phantom: PhantomData, } } } impl<'a, T: 'a> From<&[T]> for Slice<'a, T> { fn from(value: &[T]) -> Self { Self::new(value.as_ptr(), value.len()) } } impl<'a, T: 'a> Slice<'a, T> { /// Creates a new raw slice. #[inline(always)] pub fn new(data: *const T, len: usize) -> Self { let raw = slice_from_raw_parts(data, len); let phantom = PhantomData; Self { raw, phantom } } /// Destructs the slice wrapper and returns the underlying raw slice /// that it is created with. pub fn destruct(self) -> *const [T] { self.raw } /// Creates two slices from this slice: /// /// - first slice for positions [0..position) /// - second slice for positions [position..] /// /// # SAFETY /// /// - (i) `position` must be less than or equal to `self.len()` pub unsafe fn split_at_unchecked(self, position: usize) -> [Self; 2] { let len_right = self.len() - position; let ptr_left = self.raw as *const T; let left = Self::new(ptr_left, position); // SAFETY: ptr_right is within bounds due to condition (i) let ptr_right = unsafe { ptr_left.add(position) }; let right = Self::new(ptr_right, len_right); [left, right] } #[inline(always)] pub(super) fn data(&self) -> *const T { self.raw as *const T } /// Returns the length of the slice. #[inline(always)] pub fn len(&self) -> usize { self.raw.len() } /// Returns true if the slice is empty. pub fn is_empty(&self) -> bool { self.raw.is_empty() } /// Returns a reference to the first element of the slice. /// /// # SAFETY /// /// - (i) this must have a positive `len` /// - (ii) the first element must be initialized #[inline(always)] pub unsafe fn first_unchecked(&self) -> &'a T { unsafe { &*(self.raw as *const T) } } /// # SAFETY /// /// - (i) `self` and `src` must have the same lengths. /// - (ii) `self` and `src` must not be overlapping. pub unsafe fn copy_from_nonoverlapping(&self, src: &Self) { debug_assert_eq!(self.len(), src.len()); // SAFETY: (i) within bounds and (ii) slices do not overlap let dst = self.raw as *mut T; unsafe { dst.copy_from_nonoverlapping(src.raw as *const T, self.len()) }; } /// # SAFETY /// /// - (i) all values must be initialized. /// - (ii) the values must not be mutated before the returned slice is dropped. pub unsafe fn as_slice(&self) -> &'a [T] { unsafe { &*self.raw } } } /// A struct holding a reference to a slice, hiding its unsafe methods, /// allowing only safe methods. pub struct SliceSafe<'c, 'a, T: 'a>(&'c Slice<'a, T>); impl<'c, 'a, T: 'a> SliceSafe<'c, 'a, T> { /// Creates a safe wrapper over the `slice`. pub fn new(slice: &'c Slice<'a, T>) -> Self { Self(slice) } } impl<'c, 'a, T: 'a> From<&'c Slice<'a, T>> for SliceSafe<'c, 'a, T> { fn from(value: &'c Slice<'a, T>) -> Self { SliceSafe::new(value) } } impl<'c, 'a, T: 'a> SliceSafe<'c, 'a, T> { /// Returns true if slices `self` and `other` are non-overlapping. pub fn is_non_overlapping(&self, other: &Self) -> bool { match (self.0.len(), other.0.len()) { (0, _) | (_, 0) => true, (n, m) => { let (left, right) = match self.0.data() >= other.0.data() { true => (unsafe { other.0.data().add(m - 1) }, self.0.data()), false => (unsafe { self.0.data().add(n - 1) }, other.0.data()), }; left < right } } } } ================================================ FILE: src/experiment/data_structures/slice_dst.rs ================================================ use crate::experiment::data_structures::slice::{Slice, SliceSafe}; use crate::experiment::data_structures::slice_iter_ptr_dst::SliceIterPtrDst; use alloc::vec::Vec; /// A raw slice of contiguous data with un-initialized values. /// /// # SAFETY /// /// While constructing this slice, we must guarantee that none of the elements of it /// is initialized since they will be overwritten. /// /// This is a write-only slice. /// The caller must make sure that there is no other concurrent reads or writes to this slice. pub struct SliceDst<'a, T>(Slice<'a, T>); impl<'a, T> SliceDst<'a, T> { /// Destructs the slice wrapper and returns the underlying raw slice /// that it is created with. pub fn destruct(self) -> *const [T] { self.0.destruct() } /// Clones the destination slice. /// /// # SAFETY /// /// The purpose of destination slice is to mutate the underlying memory. /// Therefore, cloning is marked as unsafe. /// /// - (i) assuming the clone will be used to mutate the memory, caller must /// ensure that `&self` will not be used. pub unsafe fn clone(&self) -> Self { Self(self.0.clone()) } /// Creates a new slice of un-initialized values. /// /// # SAFETY /// /// - (i) `data` to `data+len` must be contiguous memory of un-initialized elements pub unsafe fn new(data: *const T, len: usize) -> Self { Self(Slice::new(data, len)) } /// Creates a new slice for the entire capacity of the vector. /// /// # Panics /// /// - (i) if `vec.len()` is not zero. /// /// # SAFETY /// /// This slice cannot outlive the `vec` it is created for due to the lifetime relation. pub fn from_vec(vec: &'a mut Vec) -> Self { assert!(vec.is_empty()); // SAFETY: constructing with contiguous un-initialized elements unsafe { Self::new(vec.as_ptr(), vec.capacity()) } } /// Length of the slice. #[inline(always)] #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> usize { self.0.len() } /// Returns a safe wrapper over this slice. #[inline(always)] pub fn core(&self) -> SliceSafe<'_, 'a, T> { self.into() } /// Converts the destination slice into a destination iterator. pub fn into_ptr_iter(self) -> SliceIterPtrDst<'a, T> { SliceIterPtrDst::new(self) } /// Creates two slices from this slice: /// /// - first slice for positions [0..position) /// - second slice for positions [position..] /// /// # SAFETY /// /// - (i) `position` must be less than or equal to `self.len()` pub unsafe fn split_at_unchecked(self, position: usize) -> [Self; 2] { // SAFETY: req't (i) is satisfied by cond'n (i) unsafe { self.0.split_at_unchecked(position) }.map(Self) } } impl<'c, 'a, T: 'a> From<&'c SliceDst<'a, T>> for SliceSafe<'c, 'a, T> { fn from(value: &'c SliceDst<'a, T>) -> Self { SliceSafe::new(&value.0) } } ================================================ FILE: src/experiment/data_structures/slice_iter_ptr.rs ================================================ use core::marker::PhantomData; /// Core structure for iterators over contiguous slices of data. pub struct SliceIterPtr<'a, T: 'a> { data: *const T, exclusive_end: *const T, phantom: PhantomData<&'a ()>, } impl Default for SliceIterPtr<'_, T> { fn default() -> Self { Self { data: core::ptr::null(), exclusive_end: core::ptr::null(), phantom: PhantomData, } } } impl<'a, T: 'a> SliceIterPtr<'a, T> { /// Creates a new iterator for `n` elements starting from the given `ptr`. /// /// # SAFETY /// /// - (i) either `ptr` is not-null or `n` is zero. pub unsafe fn new(ptr: *const T, n: usize) -> Self { let exclusive_end = unsafe { ptr.add(n) }; Self { data: ptr, exclusive_end, phantom: PhantomData, } } /// Returns true if the end of the slice is reached. #[inline(always)] pub fn is_finished(&self) -> bool { self.data == self.exclusive_end } /// Returns the remaining number of elements on the slice to be /// iterated. #[inline(always)] fn remaining(&self) -> usize { unsafe { self.exclusive_end.offset_from(self.data) as usize } } // non-progressing methods /// Returns the current pointer. #[inline(always)] pub fn peek(&self) -> *const T { self.data } /// Returns a reference to the current element. /// Returns None if the iterator `is_finished`. /// /// # SAFETY /// /// Bounds check is applied. However, the following safety /// requirement must be satisfied. /// /// - (i) the element must be initialized. #[inline(always)] pub unsafe fn current(&self) -> Option<&'a T> { match !self.is_finished() { // SAFETY: the value is initialized. true => Some(unsafe { &*self.data }), false => None, } } /// Returns a reference to the current element. /// Returns None if the iterator `is_finished`. /// /// # SAFETY /// /// - (i) the element must be initialized. /// - (ii) the iterator cannot be `is_finished`; otherwise, we /// will have an UB due to dereferencing an invalid pointer. #[inline(always)] pub unsafe fn current_unchecked(&self) -> &'a T { debug_assert!(!self.is_finished()); unsafe { &*self.data } } // progressing methods /// Returns the current pointer and progresses the iterator to the next. /// /// # SAFETY /// /// - (i) the iterator cannot be `is_finished`; otherwise, the /// obtained pointer does not belong to the slice the iterator /// is created for. #[inline(always)] pub unsafe fn next_unchecked(&mut self) -> *const T { debug_assert!(!self.is_finished()); let value = self.data; self.data = unsafe { self.data.add(1) }; value } /// Returns the current pointer and progresses by `count` elements. /// /// # SAFETY /// /// - (i) the iterator must have at least `count` more elements; i.e., /// `self.remaining() >= count`. pub unsafe fn next_n_unchecked(&mut self, count: usize) -> *const T { debug_assert!(self.remaining() >= count); let value = self.data; // SAFETY: by cond'n (i), data + count is within bounds self.data = unsafe { self.data.add(count) }; value } /// Brings the iterator to the end, skipping the remaining positions. pub fn jump_to_end(&mut self) { self.data = self.exclusive_end } } impl<'a, T: 'a> Iterator for SliceIterPtr<'a, T> { type Item = *const T; #[inline(always)] fn next(&mut self) -> Option { match !self.is_finished() { true => Some(unsafe { self.next_unchecked() }), false => None, } } #[inline(always)] fn size_hint(&self) -> (usize, Option) { (self.remaining(), Some(self.remaining())) } } impl<'a, T: 'a> ExactSizeIterator for SliceIterPtr<'a, T> { #[inline(always)] fn len(&self) -> usize { self.remaining() } } ================================================ FILE: src/experiment/data_structures/slice_iter_ptr_dst.rs ================================================ use crate::experiment::data_structures::slice_iter_ptr_src::SliceIterPtrSrc; use crate::experiment::data_structures::{slice_dst::SliceDst, slice_iter_ptr::SliceIterPtr}; /// Iterator over a slice of data that will be completely filled with values /// before the iterator is consumed. /// /// The slice might initially be uninitialized, but will completely be initialized. pub struct SliceIterPtrDst<'a, T: 'a>(SliceIterPtr<'a, T>); impl Default for SliceIterPtrDst<'_, T> { fn default() -> Self { Self(Default::default()) } } impl<'a, T: 'a> SliceIterPtrDst<'a, T> { /// # SAFETY /// /// Since `slice: SliceDst` satisfies that positions of it are not initialized, /// we satisfy the construction condition for this iterator. pub fn new(slice: SliceDst<'a, T>) -> Self { let raw = slice.destruct(); // SAFETY: requirement satisfied by `SliceDst` Self(unsafe { SliceIterPtr::new(raw as *const T, raw.len()) }) } /// Returns true if the end of the slice is reached. #[inline(always)] pub fn is_finished(&self) -> bool { self.0.is_finished() } /// Returns the number of remaining positions. #[inline(always)] #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> usize { self.0.len() } /// Pulls the next element from `src`, writes it to next position of `self`. /// /// Progresses both `self` and `src` by one element. /// /// # SAFETY /// /// - (i) `src` cannot be `is_finished` /// - (ii) `self` cannot be `is_finished` /// - (iii) `src` and `self` cannot be overlapping #[inline(always)] pub unsafe fn write_one_from(&mut self, src: &mut SliceIterPtrSrc<'a, T>) { debug_assert!(!self.is_finished() && !src.is_finished()); // SAFETY: satisfied by (i) let src = unsafe { src.next_unchecked() }; // SAFETY: satisfied by (ii) let dst = unsafe { self.0.next_unchecked() } as *mut T; // SAFETY: satisfied by (iii) unsafe { dst.copy_from_nonoverlapping(src, 1) }; } /// Pulls the next `count` elements from `src`, writes them to next `count` /// positions of `self`. /// /// Progresses both `self` and `src` by `count` positions. /// /// # SAFETY /// /// - (i) `src` must have at least `count` positions. /// - (ii) `self` must have at least `count` positions. /// - (iii) `src` and `self` cannot be overlapping pub unsafe fn write_many_from(&mut self, src: &mut SliceIterPtrSrc<'a, T>, count: usize) { debug_assert!(self.len() >= count && src.len() >= count); // SAFETY: satisfied by (i) let src = unsafe { src.next_n_unchecked(count) }; // SAFETY: satisfied by (ii) let dst = unsafe { self.0.next_n_unchecked(count) } as *mut T; // SAFETY: satisfied by (iii) unsafe { dst.copy_from_nonoverlapping(src, count) }; } /// Pulls all remaining elements from `src`, writes them to remaining /// positions of `self`. /// /// Progresses both `self` and `src` to the end. /// /// # SAFETY /// /// - (i) `src` and `self` mut have equal number of remaining elements /// - (ii) `src` and `self` cannot be overlapping pub unsafe fn write_rest_from(&mut self, src: &mut SliceIterPtrSrc<'a, T>) { debug_assert_eq!(self.len(), src.len()); if let Some(src) = src.next() { let count = self.len(); // SAFETY: having same lengths with src by (i), self cannot be finished let dst = unsafe { self.0.next_unchecked() } as *mut T; // SAFETY: satisfied by (ii) unsafe { dst.copy_from_nonoverlapping(src, count) }; } self.0.jump_to_end(); src.jump_to_end(); } } ================================================ FILE: src/experiment/data_structures/slice_iter_ptr_src.rs ================================================ use crate::experiment::data_structures::{slice_iter_ptr::SliceIterPtr, slice_src::SliceSrc}; use core::slice::from_raw_parts; /// Iterator over a slice of data that will be completely copied to another slice /// before the iterator is consumed. /// /// # SAFETY /// /// While constructing this iterator, we must guarantee that all elements of it /// are initialized since it will be used as source of values. pub struct SliceIterPtrSrc<'a, T: 'a>(SliceIterPtr<'a, T>); impl Default for SliceIterPtrSrc<'_, T> { fn default() -> Self { Self(Default::default()) } } impl<'a, T: 'a> SliceIterPtrSrc<'a, T> { /// # SAFETY /// /// Since `slice: SliceSrc` satisfies that all elements of it are initialized, /// we satisfy the construction condition for this iterator. pub fn new(slice: SliceSrc<'a, T>) -> Self { let raw = slice.destruct(); // SAFETY: requirement satisfied by `SliceSrc` Self(unsafe { SliceIterPtr::new(raw as *const T, raw.len()) }) } /// Returns true if the end of the slice is reached. #[inline(always)] pub fn is_finished(&self) -> bool { self.0.is_finished() } /// Returns a reference to the current element. /// Returns None if the iterator `is_finished`. #[inline(always)] pub fn current(&self) -> Option<&'a T> { // SAFETY: all elements are initialized unsafe { self.0.current() } } /// Returns the current pointer and progresses the iterator to the next. /// /// # SAFETY /// /// - (i) the iterator cannot be `is_finished`; otherwise, the /// obtained pointer does not belong to the slice the iterator /// is created for. #[inline(always)] pub unsafe fn next_unchecked(&mut self) -> *const T { // SAFETY: matching req't and cond'n (i) unsafe { self.0.next_unchecked() } } /// Returns the current pointer and progresses by `count` elements. /// /// # SAFETY /// /// - (i) the iterator must have at least `count` more elements; i.e., /// `self.remaining() >= count`. pub unsafe fn next_n_unchecked(&mut self, count: usize) -> *const T { // SAFETY: matching req't and cond'n (i) unsafe { self.0.next_n_unchecked(count) } } /// Brings the iterator to the end, skipping the remaining positions. pub(super) fn jump_to_end(&mut self) { self.0.jump_to_end(); } /// Creates an iterator over references to values of the remaining elements /// of this iterator. pub fn values(&self) -> core::slice::Iter<'a, T> { self.as_slice().iter() } /// Creates a slice over references to values of the remaining elements /// of this iterator. pub fn as_slice(&self) -> &'a [T] { let ptr = self.0.peek(); let n = self.len(); // SAFETY: SliceIterPtrSrc guarantees initialized values unsafe { from_raw_parts(ptr, n) } } } impl<'a, T> Iterator for SliceIterPtrSrc<'a, T> { type Item = *const T; /// Returns the current pointer and progresses the iterator to the next; /// returns None if the iterator `is_finished`. #[inline(always)] fn next(&mut self) -> Option { self.0.next() } #[inline(always)] fn size_hint(&self) -> (usize, Option) { let len = self.0.len(); (len, Some(len)) } } impl<'a, T> ExactSizeIterator for SliceIterPtrSrc<'a, T> { #[inline(always)] fn len(&self) -> usize { self.0.len() } } ================================================ FILE: src/experiment/data_structures/slice_src.rs ================================================ use crate::experiment::data_structures::{ slice::{Slice, SliceSafe}, slice_iter_ptr_src::SliceIterPtrSrc, }; /// A raw slice of contiguous data with initialized values. /// /// # SAFETY /// /// While constructing this slice, we must guarantee that all elements of it /// are initialized since it will be used as source of values. /// /// This is a read-only slice. /// The caller must make sure that there is no concurrent write to this slice. pub struct SliceSrc<'a, T>(Slice<'a, T>); impl<'a, T> Clone for SliceSrc<'a, T> { fn clone(&self) -> Self { Self(self.0.clone()) } } impl<'a, T> SliceSrc<'a, T> { /// Destructs the slice wrapper and returns the underlying raw slice /// that it is created with. pub fn destruct(self) -> *const [T] { self.0.destruct() } /// Creates the source slice from the given `slice`. /// /// # SAFETY /// /// The `slice` guarantees that all elements are initialized. /// /// Further, this slice cannot outlive the `slice` it is created for due to the lifetime relation. pub fn from_slice(slice: &'a [T]) -> Self { Self(Slice::new(slice.as_ptr(), slice.len())) } /// Length of the slice. #[inline(always)] #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> usize { self.0.len() } /// Returns a safe wrapper over this slice. #[inline(always)] pub fn core(&self) -> SliceSafe<'_, 'a, T> { self.into() } /// Returns a reference to the first element of the slice. /// /// # SAFETY /// /// - (i) this must have a positive `len` #[inline(always)] pub unsafe fn first_unchecked(&self) -> &'a T { // SAFETY: req't (i) is satisfied by cond'n (i); // req't (ii) is satisfied by SliceSrc construction. unsafe { self.0.first_unchecked() } } /// Converts the source slice into a source iterator. pub fn into_ptr_iter(self) -> SliceIterPtrSrc<'a, T> { SliceIterPtrSrc::new(self) } /// Creates two slices from this slice: /// /// - first slice for positions [0..position) /// - second slice for positions [position..] /// /// # SAFETY /// /// - (i) `position` must be less than or equal to `self.len()` pub unsafe fn split_at_unchecked(self, position: usize) -> [Self; 2] { // SAFETY: req't (i) is satisfied by cond'n (i) unsafe { self.0.split_at_unchecked(position) }.map(Self) } /// Returns the data as a slice. pub fn as_slice(&self) -> &[T] { // SAFETY: (i) all values are initialized by construction, // (ii) is satisfied by construction by shared reference of the data. unsafe { self.0.as_slice() } } /// Creates an iterator over references to values of the slice. pub fn values(&'a self) -> core::slice::Iter<'a, T> { self.as_slice().iter() } } impl<'c, 'a, T: 'a> From<&'c SliceSrc<'a, T>> for SliceSafe<'c, 'a, T> { fn from(value: &'c SliceSrc<'a, T>) -> Self { SliceSafe::new(&value.0) } } ================================================ FILE: src/experiment/data_structures/tests/mod.rs ================================================ mod slice; ================================================ FILE: src/experiment/data_structures/tests/slice.rs ================================================ use crate::experiment::data_structures::slice::{Slice, SliceSafe}; use alloc::vec::Vec; use std::string::ToString; use test_case::test_matrix; #[test] fn slice_overlap() { let a = [1, 2, 3, 4, 5, 6]; let b = [7, 8]; let assert_no_overlap = |x: &[i32], y: &[i32]| { let [x, y] = [x, y].map(Slice::from); let [x, y] = [&x, &y].map(SliceSafe::from); assert!(x.is_non_overlapping(&y)); assert!(y.is_non_overlapping(&x)); }; let assert_overlap = |x: &[i32], y: &[i32]| { let [x, y] = [x, y].map(Slice::from); let [x, y] = [&x, &y].map(SliceSafe::from); assert!(!x.is_non_overlapping(&y)); assert!(!y.is_non_overlapping(&x)); }; assert_no_overlap(&a[..], &b[..]); assert_no_overlap(&a[0..2], &a[4..]); assert_no_overlap(&a[1..3], &a[3..6]); assert_no_overlap(&a[0..0], &a[..]); assert_overlap(&a[..], &a[..]); assert_overlap(&a[0..2], &a[1..3]); assert_overlap(&a[0..2], &a[1..2]); } #[test_matrix([0, 1, 2, 3, 6])] fn slice_split_unchecked(len: usize) { let vec: Vec<_> = (1..(1 + len)).map(|x| x.to_string()).collect(); for i in 0..=vec.len() { let s = Slice::from(vec.as_slice()); // SAFETY: (i) i <= s.len() let [l, r] = unsafe { s.split_at_unchecked(i) }; // SAFETY: all initialized & no mutation let left = unsafe { l.as_slice() }.to_vec(); let right = unsafe { r.as_slice() }.to_vec(); assert_eq!(&left, &vec[0..i]); assert_eq!(&right, &vec[i..]); } } #[test_matrix([1, 6])] fn slice_first_unchecked(len: usize) { let vec: Vec<_> = (1..(1 + len)).map(|x| x.to_string()).collect(); let s = Slice::from(vec.as_slice()); // SAFETY: s.len() > 0 assert_eq!(unsafe { s.first_unchecked() }, &"1".to_string()); } ================================================ FILE: src/experiment/mod.rs ================================================ /// Data structures used for internal algorithms. pub mod data_structures; /// Algorithms. pub mod algorithms; ================================================ FILE: src/generic_iterator/collect.rs ================================================ use super::iter::GenericIterator; use crate::ParIter; use alloc::vec::Vec; impl GenericIterator where T: Send + Sync, S: Iterator, R: rayon::iter::ParallelIterator, O: ParIter, { /// Collects the elements of the iterator into a vector. /// /// See [`collect`] for details of the general collect method. /// /// [`collect`]: crate::ParIter::collect pub fn collect_vec(self) -> Vec { match self { GenericIterator::Sequential(x) => x.collect(), GenericIterator::Rayon(x) => x.collect(), GenericIterator::Orx(x) => x.collect(), } } } ================================================ FILE: src/generic_iterator/early_exit.rs ================================================ use super::iter::GenericIterator; use crate::{IterationOrder, ParIter}; impl GenericIterator where T: Send + Sync, S: Iterator, R: rayon::iter::ParallelIterator, O: ParIter, { /// Find computation for the generic iterator. /// /// See [`find`] for details. /// /// [`find`]: crate::ParIter::find pub fn find(self, predicate: Predicate) -> Option where Predicate: Fn(&T) -> bool + Send + Sync + Clone, { match self { GenericIterator::Sequential(mut x) => x.find(predicate), GenericIterator::Rayon(x) => x.find_first(predicate), GenericIterator::Orx(x) => x.find(predicate), } } /// Find-any computation for the generic iterator. /// /// See [`first`] for details. /// /// [`first`]: crate::ParIter::first pub fn find_any(self, predicate: Predicate) -> Option where Predicate: Fn(&T) -> bool + Send + Sync + Clone, { match self { GenericIterator::Sequential(mut x) => x.find(predicate), GenericIterator::Rayon(x) => x.find_any(predicate), GenericIterator::Orx(x) => x.iteration_order(IterationOrder::Arbitrary).find(predicate), } } } ================================================ FILE: src/generic_iterator/iter.rs ================================================ use crate::ParIter; /// An iterator that generalizes over: /// /// * sequential iterators, /// * rayon's parallel iterators, and /// * orx-parallel's parallel iterators. /// /// This is particularly useful for enabling a convenient way to run experiments /// using these different computation approaches. pub enum GenericIterator where T: Send + Sync, S: Iterator, R: rayon::iter::ParallelIterator, O: ParIter, { /// Sequential, or regular, iterator. Sequential(S), /// rayon's parallel iterator. Rayon(R), /// orx-parallel's parallel iterator. Orx(O), } impl GenericIterator, crate::iter::ParEmpty> where T: Send + Sync, S: Iterator, { /// Creates the generic iterator from sequential iterator variant. pub fn sequential(iter: S) -> Self { Self::Sequential(iter) } } impl GenericIterator, R, crate::iter::ParEmpty> where T: Send + Sync, R: rayon::iter::ParallelIterator, { /// Creates the generic iterator from rayon iterator variant. pub fn rayon(iter: R) -> Self { Self::Rayon(iter) } } impl GenericIterator, rayon::iter::Empty, O> where T: Send + Sync, O: ParIter, { /// Creates the generic iterator from orx-parallel iterator variant. pub fn orx(iter: O) -> Self { Self::Orx(iter) } } ================================================ FILE: src/generic_iterator/mod.rs ================================================ mod collect; mod early_exit; mod iter; mod reduce; mod transformations; pub use iter::GenericIterator; ================================================ FILE: src/generic_iterator/reduce.rs ================================================ use super::iter::GenericIterator; use crate::ParIter; use core::cmp::Ordering; impl GenericIterator where T: Send + Sync, S: Iterator, R: rayon::iter::ParallelIterator, O: ParIter, { /// Reduction for the generic iterator. /// /// See [`reduce`] for details. /// /// [`reduce`]: crate::ParIter::reduce pub fn reduce(self, reduce: Reduce) -> Option where Reduce: Fn(T, T) -> T + Send + Sync, { match self { GenericIterator::Sequential(x) => x.reduce(reduce), GenericIterator::Rayon(x) => x.reduce_with(reduce), GenericIterator::Orx(x) => x.reduce(reduce), } } /// Count reduction for the generic iterator. /// /// See [`count`] for details. /// /// [`count`]: crate::ParIter::count pub fn count(self) -> usize { match self { GenericIterator::Sequential(x) => x.count(), GenericIterator::Rayon(x) => x.count(), GenericIterator::Orx(x) => x.count(), } } /// For-each iteration for the generic iterator. /// /// See [`for_each`] for details. /// /// [`for_each`]: crate::ParIter::for_each pub fn for_each(self, operation: Operation) where Operation: Fn(T) + Sync + Send, { match self { GenericIterator::Sequential(x) => x.for_each(operation), GenericIterator::Rayon(x) => x.for_each(operation), GenericIterator::Orx(x) => x.for_each(operation), } } /// Max reduction for the generic iterator. /// /// See [`max`] for details. /// /// [`max`]: crate::ParIter::max pub fn max(self) -> Option where T: Ord, { match self { GenericIterator::Sequential(x) => x.max(), GenericIterator::Rayon(x) => x.max(), GenericIterator::Orx(x) => x.max(), } } /// Max-by reduction for the generic iterator. /// /// See [`max_by`] for details. /// /// [`max_by`]: crate::ParIter::max_by pub fn max_by(self, compare: Compare) -> Option where Compare: Fn(&T, &T) -> Ordering + Sync + Send, { match self { GenericIterator::Sequential(x) => x.max_by(compare), GenericIterator::Rayon(x) => x.max_by(compare), GenericIterator::Orx(x) => x.max_by(compare), } } /// Max-by-key reduction for the generic iterator. /// /// See [`max_by_key`] for details. /// /// [`max_by_key`]: crate::ParIter::max_by_key pub fn max_by_key(self, key: GetKey) -> Option where Key: Ord + Send, GetKey: Fn(&T) -> Key + Sync + Send, { match self { GenericIterator::Sequential(x) => x.max_by_key(key), GenericIterator::Rayon(x) => x.max_by_key(key), GenericIterator::Orx(x) => x.max_by_key(key), } } /// Min reduction for the generic iterator. /// /// See [`min`] for details. /// /// [`min`]: crate::ParIter::min pub fn min(self) -> Option where T: Ord, { match self { GenericIterator::Sequential(x) => x.min(), GenericIterator::Rayon(x) => x.min(), GenericIterator::Orx(x) => x.min(), } } /// Min-by reduction for the generic iterator. /// /// See [`min_by`] for details. /// /// [`min_by`]: crate::ParIter::min_by pub fn min_by(self, compare: Compare) -> Option where Compare: Fn(&T, &T) -> Ordering + Sync + Send, { match self { GenericIterator::Sequential(x) => x.min_by(compare), GenericIterator::Rayon(x) => x.min_by(compare), GenericIterator::Orx(x) => x.min_by(compare), } } /// Min-by-key reduction for the generic iterator. /// /// See [`min_by_key`] for details. /// /// [`min_by_key`]: crate::ParIter::min_by_key pub fn min_by_key(self, key: GetKey) -> Option where Key: Ord + Send, GetKey: Fn(&T) -> Key + Sync + Send, { match self { GenericIterator::Sequential(x) => x.min_by_key(key), GenericIterator::Rayon(x) => x.min_by_key(key), GenericIterator::Orx(x) => x.min_by_key(key), } } /// Sum reduction for the generic iterator. /// /// See [`sum`] for details. /// /// [`sum`]: crate::ParIter::sum pub fn sum(self) -> T where T: crate::special_type_sets::Sum + core::iter::Sum, { match self { GenericIterator::Sequential(x) => x.sum(), GenericIterator::Rayon(x) => x.sum(), GenericIterator::Orx(x) => x.sum(), } } } ================================================ FILE: src/generic_iterator/transformations.rs ================================================ use super::iter::GenericIterator; use crate::ParIter; impl GenericIterator where T: Send + Sync, S: Iterator, R: rayon::iter::ParallelIterator, O: ParIter, { // computation transformations /// Map transformation for the generic iterator. /// /// See [`map`] for details. /// /// [`map`]: crate::ParIter::map pub fn map( self, map: Map, ) -> GenericIterator< Out, impl Iterator, impl rayon::iter::ParallelIterator, impl ParIter, > where Out: Send + Sync, Map: Fn(T) -> Out + Send + Sync + Clone, { match self { GenericIterator::Sequential(x) => GenericIterator::Sequential(x.map(map)), GenericIterator::Rayon(x) => GenericIterator::Rayon(x.map(map)), GenericIterator::Orx(x) => GenericIterator::Orx(x.map(map)), } } /// Filter transformation for the generic iterator. /// /// See [`filter`] for details. /// /// [`filter`]: crate::ParIter::filter pub fn filter( self, filter: Filter, ) -> GenericIterator< T, impl Iterator, impl rayon::iter::ParallelIterator, impl ParIter, > where Filter: Fn(&T) -> bool + Send + Sync + Clone, { match self { GenericIterator::Sequential(x) => GenericIterator::Sequential(x.filter(filter)), GenericIterator::Rayon(x) => GenericIterator::Rayon(x.filter(filter)), GenericIterator::Orx(x) => GenericIterator::Orx(x.filter(filter)), } } /// Flat-map transformation for the generic iterator. /// /// See [`flat_map`] for details. /// /// [`flat_map`]: crate::ParIter::flat_map pub fn flat_map( self, flat_map: FlatMap, ) -> GenericIterator< ::Item, impl Iterator::Item>, impl rayon::iter::ParallelIterator::Item>, impl ParIter::Item>, > where IOut: IntoIterator + Send + Sync + rayon::iter::IntoParallelIterator::Item>, ::IntoIter: Send + Sync, ::Item: Send + Sync, FlatMap: Fn(T) -> IOut + Send + Sync + Clone, { match self { GenericIterator::Sequential(x) => GenericIterator::Sequential(x.flat_map(flat_map)), GenericIterator::Rayon(x) => GenericIterator::Rayon(x.flat_map(flat_map)), GenericIterator::Orx(x) => GenericIterator::Orx(x.flat_map(flat_map)), } } /// Filter-map transformation for the generic iterator. /// /// See [`filter_map`] for details. /// /// [`filter_map`]: crate::ParIter::filter_map pub fn filter_map( self, filter_map: FilterMap, ) -> GenericIterator< Out, impl Iterator, impl rayon::iter::ParallelIterator, impl ParIter, > where Out: Send + Sync, FilterMap: Fn(T) -> Option + Send + Sync + Clone, { match self { GenericIterator::Sequential(x) => GenericIterator::Sequential(x.filter_map(filter_map)), GenericIterator::Rayon(x) => GenericIterator::Rayon(x.filter_map(filter_map)), GenericIterator::Orx(x) => GenericIterator::Orx(x.filter_map(filter_map)), } } /// Inspect transformation for the generic iterator. /// /// See [`inspect`] for details. /// /// [`inspect`]: crate::ParIter::inspect pub fn inspect( self, operation: Operation, ) -> GenericIterator< T, impl Iterator, impl rayon::iter::ParallelIterator, impl ParIter, > where Operation: Fn(&T) + Sync + Send + Clone, { match self { GenericIterator::Sequential(x) => GenericIterator::Sequential(x.inspect(operation)), GenericIterator::Rayon(x) => GenericIterator::Rayon(x.inspect(operation)), GenericIterator::Orx(x) => GenericIterator::Orx(x.inspect(operation)), } } // special item transformations /// Flatten transformation for the generic iterator. /// /// See [`flatten`] for details. /// /// [`flatten`]: crate::ParIter::flatten pub fn flatten( self, ) -> GenericIterator< ::Item, impl Iterator::Item>, impl rayon::iter::ParallelIterator::Item>, impl ParIter::Item>, > where T: IntoIterator + rayon::iter::IntoParallelIterator::Item>, ::IntoIter: Send + Sync, ::Item: Send + Sync, R: Send + Sync, Self: Send + Sync, { match self { GenericIterator::Sequential(x) => GenericIterator::Sequential(x.flatten()), GenericIterator::Rayon(x) => GenericIterator::Rayon(x.flatten()), GenericIterator::Orx(x) => GenericIterator::Orx(x.flatten()), } } } // special item transformations impl<'a, T, S, R, O> GenericIterator<&'a T, S, R, O> where &'a T: Send + Sync, S: Iterator, R: rayon::iter::ParallelIterator, O: ParIter, { /// Copied transformation for the generic iterator. /// /// See [`copied`] for details. /// /// [`copied`]: crate::ParIter::copied pub fn copied( self, ) -> GenericIterator< T, impl Iterator + use<'a, T, S, R, O>, impl rayon::iter::ParallelIterator + use<'a, T, S, R, O>, impl ParIter + use<'a, T, S, R, O>, > where T: Copy + Send + Sync + 'a, { match self { GenericIterator::Sequential(x) => GenericIterator::Sequential(x.copied()), GenericIterator::Rayon(x) => GenericIterator::Rayon(x.copied()), GenericIterator::Orx(x) => GenericIterator::Orx(x.copied()), } } /// Cloned transformation for the generic iterator. /// /// See [`cloned`] for details. /// /// [`cloned`]: crate::ParIter::cloned pub fn cloned( self, ) -> GenericIterator< T, impl Iterator + use<'a, T, S, R, O>, impl rayon::iter::ParallelIterator + use<'a, T, S, R, O>, impl ParIter + use<'a, T, S, R, O>, > where T: Clone + Send + Sync + 'a, { match self { GenericIterator::Sequential(x) => GenericIterator::Sequential(x.cloned()), GenericIterator::Rayon(x) => GenericIterator::Rayon(x.cloned()), GenericIterator::Orx(x) => GenericIterator::Orx(x.cloned()), } } } ================================================ FILE: src/generic_values/fallible_iterators/mod.rs ================================================ mod result_of_iter; pub use result_of_iter::ResultOfIter; ================================================ FILE: src/generic_values/fallible_iterators/result_of_iter.rs ================================================ pub struct ResultOfIter where I: Iterator, { iter: Option, error: Option, } impl ResultOfIter where I: Iterator, { pub fn ok(iter: I) -> Self { Self { iter: Some(iter), error: None, } } pub fn err(error: E) -> Self { Self { iter: None, error: Some(error), } } } impl Iterator for ResultOfIter where I: Iterator, { type Item = Result; #[inline] fn next(&mut self) -> Option { match self.iter.as_mut() { Some(iter) => iter.next().map(|x| Ok(x)), None => self.error.take().map(|e| Err(e)), } } } ================================================ FILE: src/generic_values/mod.rs ================================================ pub mod fallible_iterators; mod option; mod option_result; mod result; pub(crate) mod runner_results; mod transformable_values; mod values; mod vector; mod vector_result; mod whilst_atom; mod whilst_atom_result; mod whilst_iterators; mod whilst_option; mod whilst_option_result; mod whilst_vector; mod whilst_vector_result; pub use transformable_values::TransformableValues; pub use values::Values; pub use vector::Vector; pub use vector_result::VectorResult; pub use whilst_atom::WhilstAtom; pub use whilst_option::WhilstOption; pub use whilst_vector::WhilstVector; ================================================ FILE: src/generic_values/option.rs ================================================ use super::{TransformableValues, Vector}; use crate::generic_values::Values; use crate::generic_values::{ option_result::OptionResult, runner_results::{ ArbitraryPush, Fallible, Infallible, Next, OrderedPush, Reduce, SequentialPush, }, whilst_option::WhilstOption, }; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_pinned_vec::{IntoConcurrentPinnedVec, PinnedVec}; impl Values for Option { type Item = T; type Fallibility = Infallible; #[inline(always)] fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { if let Some(x) = self { vector.push(x) } SequentialPush::Done } #[inline(always)] fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { if let Some(x) = self { vec.push((idx, x)); } OrderedPush::Done } #[inline(always)] fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { if let Some(x) = self { bag.push(x); } ArbitraryPush::Done } #[inline(always)] fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { Reduce::Done { acc: match (acc, self) { (Some(x), Some(y)) => Some(reduce(x, y)), (Some(x), None) => Some(x), (None, Some(y)) => Some(y), (None, None) => None, }, } } #[inline(always)] fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { Reduce::Done { acc: match (acc, self) { (Some(x), Some(y)) => Some(reduce(u, x, y)), (Some(x), None) => Some(x), (None, Some(y)) => Some(y), (None, None) => None, }, } } fn next(self) -> Next { Next::Done { value: self } } } impl TransformableValues for Option { #[inline(always)] fn map( self, map: M, ) -> impl TransformableValues where M: Fn(Self::Item) -> O, { self.map(map) } #[inline(always)] fn filter( self, filter: F, ) -> impl TransformableValues where F: Fn(&Self::Item) -> bool, { self.filter(filter) } #[inline(always)] fn flat_map( self, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(Self::Item) -> Vo, { Vector(self.into_iter().flat_map(flat_map)) } #[inline(always)] fn filter_map( self, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(Self::Item) -> Option, { match self { Some(x) => filter_map(x), _ => None, } } fn whilst( self, whilst: impl Fn(&Self::Item) -> bool, ) -> impl TransformableValues where Self: Sized, { match self { Some(x) => match whilst(&x) { true => WhilstOption::ContinueSome(x), false => WhilstOption::Stop, }, _ => WhilstOption::ContinueNone, } } fn map_while_ok(self, map_res: Mr) -> impl Values> where Mr: Fn(Self::Item) -> Result, E: Send, { let opt_res = self.map(map_res); OptionResult(opt_res) } #[inline(always)] fn u_map( self, u: *mut U, map: M, ) -> impl TransformableValues where M: Fn(*mut U, Self::Item) -> O, { self.map(|x| map(u, x)) } #[inline(always)] fn u_filter( self, u: *mut U, filter: F, ) -> impl TransformableValues where F: Fn(*mut U, &Self::Item) -> bool, { self.filter(|x| filter(u, x)) } fn u_flat_map( self, u: *mut U, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(*mut U, Self::Item) -> Vo, { Vector(self.into_iter().flat_map(move |x| flat_map(u, x))) } fn u_filter_map( self, u: *mut U, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(*mut U, Self::Item) -> Option, { match self { Some(x) => filter_map(u, x), _ => None, } } } ================================================ FILE: src/generic_values/option_result.rs ================================================ use crate::generic_values::Values; use crate::generic_values::runner_results::{ ArbitraryPush, Fallible, Next, OrderedPush, Reduce, SequentialPush, }; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_pinned_vec::{IntoConcurrentPinnedVec, PinnedVec}; pub struct OptionResult(pub(crate) Option>) where E: Send; impl Values for OptionResult where E: Send, { type Item = T; type Fallibility = Fallible; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { match self.0 { Some(Ok(x)) => { vector.push(x); SequentialPush::Done } Some(Err(error)) => SequentialPush::StoppedByError { error }, None => SequentialPush::Done, } } fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { match self.0 { Some(Ok(x)) => { vec.push((idx, x)); OrderedPush::Done } Some(Err(error)) => OrderedPush::StoppedByError { idx, error }, None => OrderedPush::Done, } } fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { match self.0 { Some(Ok(x)) => { _ = bag.push(x); ArbitraryPush::Done } Some(Err(error)) => ArbitraryPush::StoppedByError { error }, None => ArbitraryPush::Done, } } fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { match self.0 { Some(Ok(x)) => Reduce::Done { acc: Some(match acc { Some(y) => reduce(y, x), None => x, }), }, None => Reduce::Done { acc }, Some(Err(error)) => Reduce::StoppedByError { error }, } } fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { match self.0 { Some(Ok(x)) => Reduce::Done { acc: Some(match acc { Some(y) => reduce(u, y, x), None => x, }), }, None => Reduce::Done { acc }, Some(Err(error)) => Reduce::StoppedByError { error }, } } fn next(self) -> Next { match self.0 { Some(Ok(value)) => Next::Done { value: Some(value) }, None => Next::Done { value: None }, Some(Err(error)) => Next::StoppedByError { error }, } } } ================================================ FILE: src/generic_values/result.rs ================================================ use crate::generic_values::Values; use crate::generic_values::runner_results::{ ArbitraryPush, Fallible, Next, OrderedPush, Reduce, SequentialPush, }; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_pinned_vec::{IntoConcurrentPinnedVec, PinnedVec}; /// Represents scalar value for early stopping error cases: /// /// * Whenever computation creates an error at any point, all computed values are irrelevant, /// the only relevant value is the created error. /// * Computed values are relevant iff entire inputs result in an Ok variant. /// * Therefore, observation of an error case allows to immediately stop computation. impl Values for Result where E: Send, { type Item = T; type Fallibility = Fallible; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { match self { Ok(x) => { vector.push(x); SequentialPush::Done } Err(error) => SequentialPush::StoppedByError { error }, } } fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { match self { Ok(x) => { vec.push((idx, x)); OrderedPush::Done } Err(error) => OrderedPush::StoppedByError { idx, error }, } } fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { match self { Ok(x) => { bag.push(x); ArbitraryPush::Done } Err(error) => ArbitraryPush::StoppedByError { error }, } } fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { match self { Ok(x) => Reduce::Done { acc: Some(match acc { Some(acc) => reduce(acc, x), None => x, }), }, Err(error) => Reduce::StoppedByError { error }, } } fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { match self { Ok(x) => Reduce::Done { acc: Some(match acc { Some(acc) => reduce(u, acc, x), None => x, }), }, Err(error) => Reduce::StoppedByError { error }, } } fn next(self) -> Next { match self { Ok(x) => Next::Done { value: Some(x) }, Err(error) => Next::StoppedByError { error }, } } } ================================================ FILE: src/generic_values/runner_results/collect_arbitrary.rs ================================================ use crate::generic_values::{Values, runner_results::Fallibility}; use orx_fixed_vec::IntoConcurrentPinnedVec; pub enum ArbitraryPush { Done, StoppedByWhileCondition, StoppedByError { error: F::Error }, } pub enum ThreadCollectArbitrary where F: Fallibility, { AllCollected, StoppedByWhileCondition, StoppedByError { error: F::Error }, } impl ThreadCollectArbitrary { pub fn into_result(self) -> Result<(), F::Error> { match self { Self::StoppedByError { error } => Err(error), _ => Ok(()), } } } impl core::fmt::Debug for ThreadCollectArbitrary { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Self::AllCollected => write!(f, "AllCollected"), Self::StoppedByWhileCondition => write!(f, "StoppedByWhileCondition"), Self::StoppedByError { error: _ } => f.debug_struct("StoppedByError").finish(), } } } pub enum ParallelCollectArbitrary where V: Values, P: IntoConcurrentPinnedVec, { AllOrUntilWhileCollected { pinned_vec: P, }, StoppedByError { error: ::Error, }, } impl core::fmt::Debug for ParallelCollectArbitrary where V: Values, P: IntoConcurrentPinnedVec, { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Self::AllOrUntilWhileCollected { pinned_vec } => f .debug_struct("AllCollected") .field("pinned_vec.len()", &pinned_vec.len()) .finish(), Self::StoppedByError { error: _ } => f.debug_struct("StoppedByError").finish(), } } } impl ParallelCollectArbitrary where V: Values, P: IntoConcurrentPinnedVec, { pub fn into_result(self) -> Result::Error> { match self { Self::AllOrUntilWhileCollected { pinned_vec } => Ok(pinned_vec), Self::StoppedByError { error } => Err(error), } } } ================================================ FILE: src/generic_values/runner_results/collect_ordered.rs ================================================ use crate::{ generic_values::{Values, runner_results::Fallibility}, heap_sort::heap_sort_into, }; use alloc::vec::Vec; use core::fmt::Debug; use orx_fixed_vec::IntoConcurrentPinnedVec; pub enum OrderedPush { Done, StoppedByWhileCondition { idx: usize }, StoppedByError { idx: usize, error: F::Error }, } pub enum ThreadCollect where V: Values, { AllCollected { vec: Vec<(usize, V::Item)>, }, StoppedByWhileCondition { vec: Vec<(usize, V::Item)>, stopped_idx: usize, }, StoppedByError { error: ::Error, }, } impl Debug for ThreadCollect { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Self::AllCollected { vec } => f .debug_struct("AllCollected") .field("vec-len", &vec.len()) .finish(), Self::StoppedByWhileCondition { vec, stopped_idx } => f .debug_struct("StoppedByWhileCondition") .field("vec-len", &vec.len()) .field("stopped_idx", stopped_idx) .finish(), Self::StoppedByError { error: _ } => f.debug_struct("StoppedByError").finish(), } } } impl ThreadCollect { pub fn into_result(self) -> Result::Error> { match self { Self::StoppedByError { error } => Err(error), _ => Ok(self), } } } pub enum ParallelCollect where V: Values, P: IntoConcurrentPinnedVec, { AllCollected { pinned_vec: P, }, StoppedByWhileCondition { pinned_vec: P, stopped_idx: usize, }, StoppedByError { error: ::Error, }, } impl core::fmt::Debug for ParallelCollect where V: Values, P: IntoConcurrentPinnedVec, { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Self::AllCollected { pinned_vec } => f .debug_struct("AllCollected") .field("pinned_vec.len()", &pinned_vec.len()) .finish(), Self::StoppedByWhileCondition { pinned_vec, stopped_idx, } => f .debug_struct("StoppedByWhileCondition") .field("pinned_vec.len()", &pinned_vec.len()) .field("stopped_idx", stopped_idx) .finish(), Self::StoppedByError { error: _ } => f.debug_struct("StoppedByError").finish(), } } } impl ParallelCollect where V: Values, P: IntoConcurrentPinnedVec, { pub fn reduce(results: Vec>, mut pinned_vec: P) -> Self { let mut vectors = Vec::with_capacity(results.len()); let mut min_stopped_idx = None; for x in results { match x { ThreadCollect::AllCollected { vec } => vectors.push(vec), ThreadCollect::StoppedByWhileCondition { vec, stopped_idx } => { min_stopped_idx = match min_stopped_idx { Some(x) => Some(core::cmp::min(x, stopped_idx)), None => Some(stopped_idx), }; vectors.push(vec); } ThreadCollect::StoppedByError { error } => return Self::StoppedByError { error }, } } heap_sort_into(vectors, min_stopped_idx, &mut pinned_vec); match min_stopped_idx { Some(stopped_idx) => Self::StoppedByWhileCondition { pinned_vec, stopped_idx, }, None => Self::AllCollected { pinned_vec }, } } pub fn into_result(self) -> Result::Error> { match self { Self::AllCollected { pinned_vec } => Ok(pinned_vec), Self::StoppedByWhileCondition { pinned_vec, stopped_idx: _, } => Ok(pinned_vec), Self::StoppedByError { error } => Err(error), } } } ================================================ FILE: src/generic_values/runner_results/collect_sequential.rs ================================================ use crate::generic_values::runner_results::{ Fallibility, Fallible, Infallible, Stop, fallibility::Never, }; pub enum SequentialPush { Done, StoppedByWhileCondition, StoppedByError { error: F::Error }, } impl SequentialPush { pub fn sequential_push_to_stop(self) -> Option> { match self { SequentialPush::StoppedByWhileCondition => Some(Stop::DueToWhile), _ => None, } } } impl SequentialPush> { pub fn sequential_push_to_stop(self) -> Option> { match self { SequentialPush::Done => None, SequentialPush::StoppedByWhileCondition => Some(Stop::DueToWhile), SequentialPush::StoppedByError { error } => Some(Stop::DueToError { error }), } } } ================================================ FILE: src/generic_values/runner_results/fallibility.rs ================================================ use crate::generic_values::{ Values, runner_results::{ ArbitraryPush, OrderedPush, Reduce, SequentialPush, Stop, StopWithIdx, stop::StopReduce, }, }; use alloc::vec::Vec; use core::marker::PhantomData; pub trait Fallibility: Sized { type Error: Send; fn ordered_push_to_stop(ordered_push: OrderedPush) -> Option>; fn arbitrary_push_to_stop(arbitrary_push: ArbitraryPush) -> Option>; fn sequential_push_to_stop(sequential_push: SequentialPush) -> Option>; fn reduce_to_stop(reduce: Reduce) -> Result, StopReduce> where V: Values; fn reduce_results(results: Vec>) -> Result, Self::Error>; } pub struct Infallible; impl Fallibility for Infallible { type Error = Never; #[inline(always)] fn ordered_push_to_stop(ordered_push: OrderedPush) -> Option> { match ordered_push { OrderedPush::Done => None, OrderedPush::StoppedByWhileCondition { idx } => Some(StopWithIdx::DueToWhile { idx }), } } #[inline(always)] fn arbitrary_push_to_stop(arbitrary_push: ArbitraryPush) -> Option> { match arbitrary_push { ArbitraryPush::Done => None, ArbitraryPush::StoppedByWhileCondition => Some(Stop::DueToWhile), } } #[inline(always)] fn sequential_push_to_stop(sequential_push: SequentialPush) -> Option> { match sequential_push { SequentialPush::StoppedByWhileCondition => Some(Stop::DueToWhile), _ => None, } } #[inline(always)] fn reduce_to_stop(reduce: Reduce) -> Result, StopReduce> where V: Values, { match reduce { Reduce::Done { acc } => Ok(acc), Reduce::StoppedByWhileCondition { acc } => Err(StopReduce::DueToWhile { acc }), } } fn reduce_results(results: Vec>) -> Result, Self::Error> { Ok(results .into_iter() .map(|x| match x { Ok(x) => x, }) .collect()) } } pub struct Fallible(PhantomData); impl Fallibility for Fallible { type Error = E; #[inline(always)] fn ordered_push_to_stop(ordered_push: OrderedPush) -> Option> { match ordered_push { OrderedPush::Done => None, OrderedPush::StoppedByWhileCondition { idx } => Some(StopWithIdx::DueToWhile { idx }), OrderedPush::StoppedByError { idx, error } => { Some(StopWithIdx::DueToError { idx, error }) } } } #[inline(always)] fn arbitrary_push_to_stop(arbitrary_push: ArbitraryPush) -> Option> { match arbitrary_push { ArbitraryPush::Done => None, ArbitraryPush::StoppedByWhileCondition => Some(Stop::DueToWhile), ArbitraryPush::StoppedByError { error } => Some(Stop::DueToError { error }), } } #[inline(always)] fn sequential_push_to_stop(sequential_push: SequentialPush) -> Option> { match sequential_push { SequentialPush::Done => None, SequentialPush::StoppedByWhileCondition => Some(Stop::DueToWhile), SequentialPush::StoppedByError { error } => Some(Stop::DueToError { error }), } } #[inline(always)] fn reduce_to_stop(reduce: Reduce) -> Result, StopReduce> where V: Values, { match reduce { Reduce::Done { acc } => Ok(acc), Reduce::StoppedByWhileCondition { acc } => Err(StopReduce::DueToWhile { acc }), Reduce::StoppedByError { error } => Err(StopReduce::DueToError { error }), } } fn reduce_results(results: Vec>) -> Result, Self::Error> { let mut ok_results = Vec::with_capacity(results.len()); for result in results { match result { Ok(x) => ok_results.push(x), Err(e) => return Err(e), } } Ok(ok_results) } } pub enum Never {} ================================================ FILE: src/generic_values/runner_results/mod.rs ================================================ mod collect_arbitrary; mod collect_ordered; mod collect_sequential; mod fallibility; mod next; mod reduce; mod stop; pub use collect_arbitrary::{ArbitraryPush, ParallelCollectArbitrary, ThreadCollectArbitrary}; pub use collect_ordered::{OrderedPush, ParallelCollect, ThreadCollect}; pub use collect_sequential::SequentialPush; pub use fallibility::{Fallibility, Fallible, Infallible, Never}; pub use next::{Next, NextSuccess, NextWithIdx}; pub use reduce::Reduce; pub use stop::{Stop, StopReduce, StopWithIdx}; ================================================ FILE: src/generic_values/runner_results/next.rs ================================================ use crate::generic_values::{Values, runner_results::Fallibility}; pub enum Next { Done { value: Option, }, StoppedByWhileCondition, StoppedByError { error: ::Error, }, } pub enum NextWithIdx { Found { idx: usize, value: V::Item, }, NotFound, StoppedByWhileCondition { idx: usize, }, StoppedByError { error: ::Error, }, } pub enum NextSuccess { Found { idx: usize, value: T }, StoppedByWhileCondition { idx: usize }, } impl NextSuccess { pub fn reduce(results: impl IntoIterator) -> Option<(usize, T)> { let mut result = None; let mut idx_bound = usize::MAX; for x in results { match x { NextSuccess::Found { idx, value } if idx < idx_bound => { idx_bound = idx; result = Some((idx, value)); } NextSuccess::StoppedByWhileCondition { idx } if idx < idx_bound => { idx_bound = idx; } _ => {} } } result.and_then(|(idx, value)| match idx <= idx_bound { true => Some((idx, value)), false => None, // found value was found after stopped }) } } ================================================ FILE: src/generic_values/runner_results/reduce.rs ================================================ use crate::generic_values::{Values, runner_results::Fallibility}; pub enum Reduce { Done { acc: Option, }, StoppedByWhileCondition { acc: Option, }, StoppedByError { error: ::Error, }, } impl Reduce { pub fn into_result(self) -> Result, ::Error> { match self { Reduce::Done { acc } => Ok(acc), Reduce::StoppedByWhileCondition { acc } => Ok(acc), Reduce::StoppedByError { error } => Err(error), } } } impl core::fmt::Debug for Reduce { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Self::Done { acc: _ } => f.debug_struct("Done").finish(), Self::StoppedByWhileCondition { acc: _ } => { f.debug_struct("StoppedByWhileCondition").finish() } Self::StoppedByError { error: _ } => f.debug_struct("StoppedByError").finish(), } } } ================================================ FILE: src/generic_values/runner_results/stop.rs ================================================ use crate::generic_values::{Values, runner_results::Fallibility}; pub enum Stop { DueToWhile, DueToError { error: E }, } pub enum StopWithIdx { DueToWhile { idx: usize }, DueToError { idx: usize, error: E }, } pub enum StopReduce { DueToWhile { acc: Option, }, DueToError { error: ::Error, }, } ================================================ FILE: src/generic_values/transformable_values.rs ================================================ use crate::generic_values::{Values, runner_results::Fallible}; pub trait TransformableValues: Values { fn map( self, map: M, ) -> impl TransformableValues where M: Fn(Self::Item) -> O + Clone; fn filter( self, filter: F, ) -> impl TransformableValues where F: Fn(&Self::Item) -> bool + Clone; fn flat_map( self, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(Self::Item) -> Vo + Clone; fn filter_map( self, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(Self::Item) -> Option; fn whilst( self, whilst: impl Fn(&Self::Item) -> bool, ) -> impl TransformableValues; fn map_while_ok( self, map_res: Mr, ) -> impl Values> where Mr: Fn(Self::Item) -> Result, E: Send; fn u_map( self, u: *mut U, map: M, ) -> impl TransformableValues where M: Fn(*mut U, Self::Item) -> O; fn u_filter( self, u: *mut U, filter: F, ) -> impl TransformableValues where F: Fn(*mut U, &Self::Item) -> bool; fn u_flat_map( self, u: *mut U, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(*mut U, Self::Item) -> Vo; fn u_filter_map( self, u: *mut U, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(*mut U, Self::Item) -> Option; } ================================================ FILE: src/generic_values/values.rs ================================================ use crate::generic_values::runner_results::{ ArbitraryPush, Fallibility, Next, OrderedPush, Reduce, SequentialPush, Stop, StopReduce, StopWithIdx, }; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_fixed_vec::IntoConcurrentPinnedVec; use orx_pinned_vec::PinnedVec; pub trait Values: Sized { type Item; type Fallibility: Fallibility; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec; fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush; fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send; fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item; fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item; fn next(self) -> Next; // provided #[inline(always)] fn ordered_push_to_stop( ordered_push: OrderedPush, ) -> Option::Error>> { ::ordered_push_to_stop(ordered_push) } #[inline(always)] fn arbitrary_push_to_stop( arbitrary_push: ArbitraryPush, ) -> Option::Error>> { ::arbitrary_push_to_stop(arbitrary_push) } #[inline(always)] fn sequential_push_to_stop( sequential_push: SequentialPush, ) -> Option::Error>> { ::sequential_push_to_stop(sequential_push) } #[inline(always)] fn reduce_to_stop(reduce: Reduce) -> Result, StopReduce> { ::reduce_to_stop(reduce) } } ================================================ FILE: src/generic_values/vector.rs ================================================ use super::transformable_values::TransformableValues; use crate::generic_values::{ Values, VectorResult, WhilstAtom, WhilstVector, runner_results::{ ArbitraryPush, Fallible, Infallible, Next, OrderedPush, Reduce, SequentialPush, }, }; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_fixed_vec::IntoConcurrentPinnedVec; use orx_pinned_vec::PinnedVec; pub struct Vector(pub I) where I: IntoIterator; impl Values for Vector where I: IntoIterator, { type Item = I::Item; type Fallibility = Infallible; #[inline(always)] fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { for x in self.0 { vector.push(x); } SequentialPush::Done } #[inline(always)] fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { for x in self.0 { vec.push((idx, x)); } OrderedPush::Done } #[inline(always)] fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { for x in self.0 { bag.push(x); } ArbitraryPush::Done } #[inline(always)] fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { let reduced = self.0.into_iter().reduce(&reduce); Reduce::Done { acc: match (acc, reduced) { (Some(x), Some(y)) => Some(reduce(x, y)), (Some(x), None) => Some(x), (None, Some(y)) => Some(y), (None, None) => None, }, } } fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { let reduced = self.0.into_iter().reduce(|a, b| reduce(u, a, b)); Reduce::Done { acc: match (acc, reduced) { (Some(x), Some(y)) => Some(reduce(u, x, y)), (Some(x), None) => Some(x), (None, Some(y)) => Some(y), (None, None) => None, }, } } fn next(self) -> Next { Next::Done { value: self.0.into_iter().next(), } } } impl TransformableValues for Vector where I: IntoIterator, { #[inline(always)] fn map( self, map: M, ) -> impl TransformableValues where M: Fn(Self::Item) -> O, { Vector(self.0.into_iter().map(map)) } #[inline(always)] fn filter( self, filter: F, ) -> impl TransformableValues where F: Fn(&Self::Item) -> bool, { Vector(self.0.into_iter().filter(filter)) } #[inline(always)] fn flat_map( self, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(Self::Item) -> Vo, { Vector(self.0.into_iter().flat_map(flat_map)) } #[inline(always)] fn filter_map( self, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(Self::Item) -> Option, { Vector(self.0.into_iter().filter_map(filter_map)) } fn whilst( self, whilst: impl Fn(&Self::Item) -> bool, ) -> impl TransformableValues where Self: Sized, { let iter = self.0.into_iter().map(move |x| match whilst(&x) { true => WhilstAtom::Continue(x), false => WhilstAtom::Stop, }); WhilstVector(iter) } fn map_while_ok(self, map_res: Mr) -> impl Values> where Mr: Fn(Self::Item) -> Result, E: Send, { let iter_res = self.0.into_iter().map(map_res); VectorResult(iter_res) } fn u_map( self, u: *mut U, map: M, ) -> impl TransformableValues where M: Fn(*mut U, Self::Item) -> O, { Vector(self.0.into_iter().map(move |x| map(u, x))) } fn u_filter( self, u: *mut U, filter: F, ) -> impl TransformableValues where F: Fn(*mut U, &Self::Item) -> bool, { Vector(self.0.into_iter().filter(move |x| filter(u, x))) } fn u_flat_map( self, u: *mut U, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(*mut U, Self::Item) -> Vo, { Vector(self.0.into_iter().flat_map(move |x| flat_map(u, x))) } fn u_filter_map( self, u: *mut U, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(*mut U, Self::Item) -> Option, { Vector(self.0.into_iter().filter_map(move |x| filter_map(u, x))) } } ================================================ FILE: src/generic_values/vector_result.rs ================================================ use crate::generic_values::Values; use crate::generic_values::runner_results::{ ArbitraryPush, Fallible, Next, OrderedPush, Reduce, SequentialPush, }; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_fixed_vec::IntoConcurrentPinnedVec; use orx_pinned_vec::PinnedVec; pub struct VectorResult(pub(crate) I) where I: IntoIterator>, E: Send; impl Values for VectorResult where I: IntoIterator>, E: Send, { type Item = T; type Fallibility = Fallible; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { for x in self.0 { match x { Ok(x) => vector.push(x), Err(error) => return SequentialPush::StoppedByError { error }, } } SequentialPush::Done } fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { for x in self.0 { match x { Ok(x) => vec.push((idx, x)), Err(error) => return OrderedPush::StoppedByError { idx, error }, } } OrderedPush::Done } fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { for x in self.0 { match x { Ok(x) => _ = bag.push(x), Err(error) => return ArbitraryPush::StoppedByError { error }, } } ArbitraryPush::Done } fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { let mut iter = self.0.into_iter(); let mut acc = match acc { Some(x) => x, None => { let first = iter.next(); match first { None => return Reduce::Done { acc: None }, // empty iterator but not stopped, acc is None Some(x) => match x { Ok(x) => x, Err(error) => return Reduce::StoppedByError { error }, // first element is stop, acc is None }, } } }; for x in iter { match x { Ok(x) => acc = reduce(acc, x), Err(error) => return Reduce::StoppedByError { error }, } } Reduce::Done { acc: Some(acc) } } fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { let mut iter = self.0.into_iter(); let mut acc = match acc { Some(x) => x, None => { let first = iter.next(); match first { None => return Reduce::Done { acc: None }, // empty iterator but not stopped, acc is None Some(x) => match x { Ok(x) => x, Err(error) => return Reduce::StoppedByError { error }, // first element is stop, acc is None }, } } }; for x in iter { match x { Ok(x) => acc = reduce(u, acc, x), Err(error) => return Reduce::StoppedByError { error }, } } Reduce::Done { acc: Some(acc) } } fn next(self) -> Next { match self.0.into_iter().next() { Some(x) => match x { Ok(x) => Next::Done { value: Some(x) }, Err(error) => Next::StoppedByError { error }, }, None => Next::Done { value: None }, } } } ================================================ FILE: src/generic_values/whilst_atom.rs ================================================ use crate::generic_values::runner_results::{ ArbitraryPush, Fallible, Infallible, Next, OrderedPush, Reduce, SequentialPush, }; use crate::generic_values::whilst_atom_result::WhilstAtomResult; use crate::generic_values::whilst_iterators::WhilstAtomFlatMapIter; use crate::generic_values::{TransformableValues, Values, WhilstOption, WhilstVector}; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_pinned_vec::{IntoConcurrentPinnedVec, PinnedVec}; pub enum WhilstAtom { Continue(T), Stop, } impl WhilstAtom { #[inline(always)] pub fn new(value: T, whilst: impl Fn(&T) -> bool) -> Self { match whilst(&value) { true => Self::Continue(value), false => Self::Stop, } } } impl Values for WhilstAtom { type Item = T; type Fallibility = Infallible; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { match self { Self::Continue(x) => { vector.push(x); SequentialPush::Done } Self::Stop => SequentialPush::StoppedByWhileCondition, } } fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { match self { Self::Continue(x) => { vec.push((idx, x)); OrderedPush::Done } Self::Stop => OrderedPush::StoppedByWhileCondition { idx }, } } fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { match self { Self::Continue(x) => { bag.push(x); ArbitraryPush::Done } Self::Stop => ArbitraryPush::StoppedByWhileCondition, } } fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { match self { Self::Continue(x) => Reduce::Done { acc: Some(match acc { Some(acc) => reduce(acc, x), None => x, }), }, Self::Stop => Reduce::StoppedByWhileCondition { acc }, } } fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { match self { Self::Continue(x) => Reduce::Done { acc: Some(match acc { Some(acc) => reduce(u, acc, x), None => x, }), }, Self::Stop => Reduce::StoppedByWhileCondition { acc }, } } fn next(self) -> Next { match self { Self::Continue(x) => Next::Done { value: Some(x) }, Self::Stop => Next::StoppedByWhileCondition, } } } impl TransformableValues for WhilstAtom { fn map( self, map: M, ) -> impl TransformableValues where M: Fn(Self::Item) -> O + Clone, { match self { Self::Continue(x) => WhilstAtom::Continue(map(x)), Self::Stop => WhilstAtom::Stop, } } fn filter( self, filter: F, ) -> impl TransformableValues where F: Fn(&Self::Item) -> bool + Clone, { match self { Self::Continue(x) => match filter(&x) { true => WhilstOption::ContinueSome(x), false => WhilstOption::ContinueNone, }, Self::Stop => WhilstOption::Stop, } } fn flat_map( self, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(Self::Item) -> Vo + Clone, { let iter = WhilstAtomFlatMapIter::from_atom(self, &flat_map); WhilstVector(iter) } fn filter_map( self, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(Self::Item) -> Option, { match self { Self::Continue(x) => match filter_map(x) { Some(x) => WhilstOption::ContinueSome(x), None => WhilstOption::ContinueNone, }, Self::Stop => WhilstOption::Stop, } } fn whilst( self, whilst: impl Fn(&Self::Item) -> bool, ) -> impl TransformableValues where Self: Sized, { match self { Self::Continue(x) => match whilst(&x) { true => Self::Continue(x), false => Self::Stop, }, Self::Stop => Self::Stop, } } fn map_while_ok(self, map_res: Mr) -> impl Values> where Mr: Fn(Self::Item) -> Result, E: Send, { match self { Self::Continue(x) => match map_res(x) { Ok(x) => WhilstAtomResult::ContinueOk(x), Err(e) => WhilstAtomResult::StopErr(e), }, Self::Stop => WhilstAtomResult::StopWhile, } } fn u_map( self, u: *mut U, map: M, ) -> impl TransformableValues where M: Fn(*mut U, Self::Item) -> O, { match self { Self::Continue(x) => WhilstAtom::Continue(map(u, x)), Self::Stop => WhilstAtom::Stop, } } fn u_filter( self, u: *mut U, filter: F, ) -> impl TransformableValues where F: Fn(*mut U, &Self::Item) -> bool, { match self { Self::Continue(x) => match filter(u, &x) { true => WhilstOption::ContinueSome(x), false => WhilstOption::ContinueNone, }, Self::Stop => WhilstOption::Stop, } } fn u_flat_map( self, u: *mut U, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(*mut U, Self::Item) -> Vo, { let iter = WhilstAtomFlatMapIter::u_from_atom(u, self, &flat_map); WhilstVector(iter) } fn u_filter_map( self, u: *mut U, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(*mut U, Self::Item) -> Option, { match self { Self::Continue(x) => match filter_map(u, x) { Some(x) => WhilstOption::ContinueSome(x), None => WhilstOption::ContinueNone, }, Self::Stop => WhilstOption::Stop, } } } ================================================ FILE: src/generic_values/whilst_atom_result.rs ================================================ use crate::generic_values::Values; use crate::generic_values::runner_results::{ ArbitraryPush, Fallible, Next, OrderedPush, Reduce, SequentialPush, }; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_pinned_vec::{IntoConcurrentPinnedVec, PinnedVec}; pub enum WhilstAtomResult where E: Send, { ContinueOk(T), StopErr(E), StopWhile, } impl Values for WhilstAtomResult where E: Send, { type Item = T; type Fallibility = Fallible; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { match self { Self::ContinueOk(x) => { vector.push(x); SequentialPush::Done } Self::StopErr(error) => SequentialPush::StoppedByError { error }, Self::StopWhile => SequentialPush::StoppedByWhileCondition, } } fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { match self { Self::ContinueOk(x) => { vec.push((idx, x)); OrderedPush::Done } Self::StopErr(error) => OrderedPush::StoppedByError { idx, error }, Self::StopWhile => OrderedPush::StoppedByWhileCondition { idx }, } } fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { match self { Self::ContinueOk(x) => { bag.push(x); ArbitraryPush::Done } Self::StopErr(error) => ArbitraryPush::StoppedByError { error }, Self::StopWhile => ArbitraryPush::StoppedByWhileCondition, } } fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { match self { Self::ContinueOk(x) => Reduce::Done { acc: Some(match acc { Some(acc) => reduce(acc, x), None => x, }), }, Self::StopErr(error) => Reduce::StoppedByError { error }, Self::StopWhile => Reduce::StoppedByWhileCondition { acc }, } } fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { match self { Self::ContinueOk(x) => Reduce::Done { acc: Some(match acc { Some(acc) => reduce(u, acc, x), None => x, }), }, Self::StopErr(error) => Reduce::StoppedByError { error }, Self::StopWhile => Reduce::StoppedByWhileCondition { acc }, } } fn next(self) -> Next { match self { Self::ContinueOk(x) => Next::Done { value: Some(x) }, Self::StopErr(error) => Next::StoppedByError { error }, Self::StopWhile => Next::StoppedByWhileCondition, } } } ================================================ FILE: src/generic_values/whilst_iterators/mod.rs ================================================ mod whilst_atom_flat_map; mod whilst_option_flat_map; pub use whilst_atom_flat_map::WhilstAtomFlatMapIter; pub use whilst_option_flat_map::WhilstOptionFlatMapIter; ================================================ FILE: src/generic_values/whilst_iterators/whilst_atom_flat_map.rs ================================================ use crate::generic_values::whilst_atom::WhilstAtom; pub struct WhilstAtomFlatMapIter where Vo: IntoIterator, { current_iter: WhilstAtom, } impl WhilstAtomFlatMapIter where Vo: IntoIterator, { pub fn from_atom(atom: WhilstAtom, flat_map: Fm) -> Self where Fm: Fn(T) -> Vo, { let current_iter = match atom { WhilstAtom::Continue(x) => WhilstAtom::Continue(flat_map(x).into_iter()), WhilstAtom::Stop => WhilstAtom::Stop, }; Self { current_iter } } pub fn u_from_atom(u: *mut U, atom: WhilstAtom, flat_map: Fm) -> Self where Fm: Fn(*mut U, T) -> Vo, { let current_iter = match atom { WhilstAtom::Continue(x) => WhilstAtom::Continue(flat_map(u, x).into_iter()), WhilstAtom::Stop => WhilstAtom::Stop, }; Self { current_iter } } } impl Iterator for WhilstAtomFlatMapIter where Vo: IntoIterator, { type Item = WhilstAtom; fn next(&mut self) -> Option { match &mut self.current_iter { WhilstAtom::Continue(x) => x.next().map(WhilstAtom::Continue), // None if flat-map iterator is consumed WhilstAtom::Stop => Some(WhilstAtom::Stop), // input is Stop } } } ================================================ FILE: src/generic_values/whilst_iterators/whilst_option_flat_map.rs ================================================ use crate::generic_values::{WhilstAtom, WhilstOption}; pub struct WhilstOptionFlatMapIter where Vo: IntoIterator, { current_iter: WhilstOption, } impl WhilstOptionFlatMapIter where Vo: IntoIterator, { pub fn from_option(atom: WhilstOption, flat_map: Fm) -> Self where Fm: Fn(T) -> Vo, { let current_iter = match atom { WhilstOption::ContinueSome(x) => WhilstOption::ContinueSome(flat_map(x).into_iter()), WhilstOption::ContinueNone => WhilstOption::ContinueNone, WhilstOption::Stop => WhilstOption::Stop, }; Self { current_iter } } pub fn u_from_option(u: *mut U, atom: WhilstOption, flat_map: Fm) -> Self where Fm: Fn(*mut U, T) -> Vo, { let current_iter = match atom { WhilstOption::ContinueSome(x) => WhilstOption::ContinueSome(flat_map(u, x).into_iter()), WhilstOption::ContinueNone => WhilstOption::ContinueNone, WhilstOption::Stop => WhilstOption::Stop, }; Self { current_iter } } } impl Iterator for WhilstOptionFlatMapIter where Vo: IntoIterator, { type Item = WhilstAtom; fn next(&mut self) -> Option { match &mut self.current_iter { WhilstOption::ContinueSome(x) => x.next().map(WhilstAtom::Continue), // None if flat-map iterator is consumed WhilstOption::ContinueNone => None, // flat-map is created on None => empty iterator WhilstOption::Stop => Some(WhilstAtom::Stop), // input is Stop } } } ================================================ FILE: src/generic_values/whilst_option.rs ================================================ use crate::generic_values::runner_results::{ ArbitraryPush, Fallible, Infallible, Next, OrderedPush, Reduce, SequentialPush, }; use crate::generic_values::whilst_iterators::WhilstOptionFlatMapIter; use crate::generic_values::whilst_option_result::WhilstOptionResult; use crate::generic_values::{TransformableValues, Values, WhilstVector}; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_pinned_vec::{IntoConcurrentPinnedVec, PinnedVec}; pub enum WhilstOption { ContinueSome(T), ContinueNone, Stop, } impl Values for WhilstOption { type Item = T; type Fallibility = Infallible; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { match self { Self::ContinueSome(x) => { vector.push(x); SequentialPush::Done } Self::ContinueNone => SequentialPush::Done, Self::Stop => SequentialPush::StoppedByWhileCondition, } } fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { match self { Self::ContinueSome(x) => { vec.push((idx, x)); OrderedPush::Done } Self::ContinueNone => OrderedPush::Done, Self::Stop => OrderedPush::StoppedByWhileCondition { idx }, } } fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { match self { Self::ContinueSome(x) => { bag.push(x); ArbitraryPush::Done } Self::ContinueNone => ArbitraryPush::Done, Self::Stop => ArbitraryPush::StoppedByWhileCondition, } } fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { match self { Self::ContinueSome(x) => Reduce::Done { acc: Some(match acc { Some(acc) => reduce(acc, x), None => x, }), }, Self::ContinueNone => Reduce::Done { acc }, Self::Stop => Reduce::StoppedByWhileCondition { acc }, } } fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { match self { Self::ContinueSome(x) => Reduce::Done { acc: Some(match acc { Some(acc) => reduce(u, acc, x), None => x, }), }, Self::ContinueNone => Reduce::Done { acc }, Self::Stop => Reduce::StoppedByWhileCondition { acc }, } } fn next(self) -> Next { match self { Self::ContinueSome(x) => Next::Done { value: Some(x) }, Self::ContinueNone => Next::Done { value: None }, Self::Stop => Next::StoppedByWhileCondition, } } } impl TransformableValues for WhilstOption { fn map( self, map: M, ) -> impl TransformableValues where M: Fn(Self::Item) -> O + Clone, { match self { Self::ContinueSome(x) => WhilstOption::ContinueSome(map(x)), Self::ContinueNone => WhilstOption::ContinueNone, Self::Stop => WhilstOption::Stop, } } fn filter( self, filter: F, ) -> impl TransformableValues where F: Fn(&Self::Item) -> bool + Clone, { match self { Self::ContinueSome(x) => match filter(&x) { true => Self::ContinueSome(x), false => Self::ContinueNone, }, Self::ContinueNone => Self::ContinueNone, Self::Stop => Self::Stop, } } fn flat_map( self, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(Self::Item) -> Vo + Clone, { let iter = WhilstOptionFlatMapIter::from_option(self, &flat_map); WhilstVector(iter) } fn filter_map( self, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(Self::Item) -> Option, { match self { Self::ContinueSome(x) => match filter_map(x) { Some(x) => WhilstOption::ContinueSome(x), None => WhilstOption::ContinueNone, }, Self::ContinueNone => WhilstOption::ContinueNone, Self::Stop => WhilstOption::Stop, } } fn whilst( self, whilst: impl Fn(&Self::Item) -> bool, ) -> impl TransformableValues where Self: Sized, { match self { Self::ContinueSome(x) => match whilst(&x) { true => Self::ContinueSome(x), false => Self::Stop, }, Self::ContinueNone => Self::ContinueNone, Self::Stop => Self::Stop, } } fn map_while_ok(self, map_res: Mr) -> impl Values> where Mr: Fn(Self::Item) -> Result, E: Send, { match self { Self::ContinueSome(x) => match map_res(x) { Ok(x) => WhilstOptionResult::ContinueSomeOk(x), Err(e) => WhilstOptionResult::StopErr(e), }, Self::ContinueNone => WhilstOptionResult::ContinueNone, Self::Stop => WhilstOptionResult::StopWhile, } } fn u_map( self, u: *mut U, map: M, ) -> impl TransformableValues where M: Fn(*mut U, Self::Item) -> O, { match self { Self::ContinueSome(x) => WhilstOption::ContinueSome(map(u, x)), Self::ContinueNone => WhilstOption::ContinueNone, Self::Stop => WhilstOption::Stop, } } fn u_filter( self, u: *mut U, filter: F, ) -> impl TransformableValues where F: Fn(*mut U, &Self::Item) -> bool, { match self { Self::ContinueSome(x) => match filter(u, &x) { true => Self::ContinueSome(x), false => Self::ContinueNone, }, Self::ContinueNone => Self::ContinueNone, Self::Stop => Self::Stop, } } fn u_flat_map( self, u: *mut U, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(*mut U, Self::Item) -> Vo, { let iter = WhilstOptionFlatMapIter::u_from_option(u, self, &flat_map); WhilstVector(iter) } fn u_filter_map( self, u: *mut U, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(*mut U, Self::Item) -> Option, { match self { Self::ContinueSome(x) => match filter_map(u, x) { Some(x) => WhilstOption::ContinueSome(x), None => WhilstOption::ContinueNone, }, Self::ContinueNone => WhilstOption::ContinueNone, Self::Stop => WhilstOption::Stop, } } } ================================================ FILE: src/generic_values/whilst_option_result.rs ================================================ use crate::generic_values::Values; use crate::generic_values::runner_results::{ ArbitraryPush, Fallible, Next, OrderedPush, Reduce, SequentialPush, }; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_pinned_vec::{IntoConcurrentPinnedVec, PinnedVec}; pub enum WhilstOptionResult where E: Send, { ContinueSomeOk(T), ContinueNone, StopErr(E), StopWhile, } impl Values for WhilstOptionResult where E: Send, { type Item = T; type Fallibility = Fallible; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { match self { Self::ContinueSomeOk(x) => { vector.push(x); SequentialPush::Done } Self::ContinueNone => SequentialPush::Done, Self::StopErr(error) => SequentialPush::StoppedByError { error }, Self::StopWhile => SequentialPush::StoppedByWhileCondition, } } fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { match self { Self::ContinueSomeOk(x) => { vec.push((idx, x)); OrderedPush::Done } Self::ContinueNone => OrderedPush::Done, Self::StopErr(error) => OrderedPush::StoppedByError { idx, error }, Self::StopWhile => OrderedPush::StoppedByWhileCondition { idx }, } } fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { match self { Self::ContinueSomeOk(x) => { bag.push(x); ArbitraryPush::Done } Self::ContinueNone => ArbitraryPush::Done, Self::StopErr(error) => ArbitraryPush::StoppedByError { error }, Self::StopWhile => ArbitraryPush::StoppedByWhileCondition, } } fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { match self { Self::ContinueSomeOk(x) => Reduce::Done { acc: Some(match acc { Some(acc) => reduce(acc, x), None => x, }), }, Self::ContinueNone => Reduce::Done { acc }, Self::StopErr(error) => Reduce::StoppedByError { error }, Self::StopWhile => Reduce::StoppedByWhileCondition { acc }, } } fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { match self { Self::ContinueSomeOk(x) => Reduce::Done { acc: Some(match acc { Some(acc) => reduce(u, acc, x), None => x, }), }, Self::ContinueNone => Reduce::Done { acc }, Self::StopErr(error) => Reduce::StoppedByError { error }, Self::StopWhile => Reduce::StoppedByWhileCondition { acc }, } } fn next(self) -> Next { match self { Self::ContinueSomeOk(x) => Next::Done { value: Some(x) }, Self::ContinueNone => Next::Done { value: None }, Self::StopErr(error) => Next::StoppedByError { error }, Self::StopWhile => Next::StoppedByWhileCondition, } } } ================================================ FILE: src/generic_values/whilst_vector.rs ================================================ use super::transformable_values::TransformableValues; use crate::generic_values::{ Values, WhilstAtom, runner_results::{ ArbitraryPush, Fallible, Infallible, Next, OrderedPush, Reduce, SequentialPush, }, whilst_iterators::WhilstAtomFlatMapIter, whilst_vector_result::WhilstVectorResult, }; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_fixed_vec::IntoConcurrentPinnedVec; use orx_pinned_vec::PinnedVec; pub struct WhilstVector(pub(crate) I) where I: IntoIterator>; impl Values for WhilstVector where I: IntoIterator>, { type Item = T; type Fallibility = Infallible; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { for x in self.0 { match x { WhilstAtom::Continue(x) => vector.push(x), WhilstAtom::Stop => return SequentialPush::StoppedByWhileCondition, } } SequentialPush::Done } fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { for x in self.0 { match x { WhilstAtom::Continue(x) => vec.push((idx, x)), WhilstAtom::Stop => return OrderedPush::StoppedByWhileCondition { idx }, } } OrderedPush::Done } fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { for x in self.0 { match x { WhilstAtom::Continue(x) => _ = bag.push(x), WhilstAtom::Stop => return ArbitraryPush::StoppedByWhileCondition, } } ArbitraryPush::Done } fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { let mut iter = self.0.into_iter(); let mut acc = match acc { Some(x) => x, None => { let first = iter.next(); match first { None => return Reduce::Done { acc: None }, // empty iterator but not stopped, acc is None Some(x) => match x { WhilstAtom::Continue(x) => x, WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: None }, // first element is stop, acc is None }, } } }; for x in iter { match x { WhilstAtom::Continue(x) => acc = reduce(acc, x), WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: Some(acc) }, } } Reduce::Done { acc: Some(acc) } } fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { let mut iter = self.0.into_iter(); let mut acc = match acc { Some(x) => x, None => { let first = iter.next(); match first { None => return Reduce::Done { acc: None }, // empty iterator but not stopped, acc is None Some(x) => match x { WhilstAtom::Continue(x) => x, WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: None }, // first element is stop, acc is None }, } } }; for x in iter { match x { WhilstAtom::Continue(x) => acc = reduce(u, acc, x), WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: Some(acc) }, } } Reduce::Done { acc: Some(acc) } } fn next(self) -> Next { match self.0.into_iter().next() { Some(x) => match x { WhilstAtom::Continue(x) => Next::Done { value: Some(x) }, WhilstAtom::Stop => Next::StoppedByWhileCondition, }, None => Next::Done { value: None }, } } } impl TransformableValues for WhilstVector where I: IntoIterator>, { fn map( self, map: M, ) -> impl TransformableValues where M: Fn(Self::Item) -> O, { let iter = self.0.into_iter().map(move |x| match x { WhilstAtom::Continue(x) => WhilstAtom::Continue(map(x)), WhilstAtom::Stop => WhilstAtom::Stop, }); WhilstVector(iter) } fn filter( self, filter: F, ) -> impl TransformableValues where F: Fn(&Self::Item) -> bool + Clone, { let iter = self.0.into_iter().filter_map(move |x| match x { WhilstAtom::Continue(x) => match filter(&x) { true => Some(WhilstAtom::Continue(x)), false => None, }, WhilstAtom::Stop => Some(WhilstAtom::Stop), }); WhilstVector(iter) } fn flat_map( self, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(Self::Item) -> Vo, { let iter = self .0 .into_iter() .flat_map(move |atom| WhilstAtomFlatMapIter::from_atom(atom, &flat_map)); WhilstVector(iter) } fn filter_map( self, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(Self::Item) -> Option, { let iter = self.0.into_iter().filter_map(move |x| match x { WhilstAtom::Continue(x) => filter_map(x).map(WhilstAtom::Continue), WhilstAtom::Stop => Some(WhilstAtom::Stop), }); WhilstVector(iter) } fn whilst( self, whilst: impl Fn(&Self::Item) -> bool, ) -> impl TransformableValues where Self: Sized, { let iter = self.0.into_iter().map(move |x| match x { WhilstAtom::Continue(x) => match whilst(&x) { true => WhilstAtom::Continue(x), false => WhilstAtom::Stop, }, WhilstAtom::Stop => WhilstAtom::Stop, }); WhilstVector(iter) } fn map_while_ok(self, map_res: Mr) -> impl Values> where Mr: Fn(Self::Item) -> Result, E: Send, { let iter = self.0.into_iter().map(move |x| match x { WhilstAtom::Continue(x) => WhilstAtom::Continue(map_res(x)), WhilstAtom::Stop => WhilstAtom::Stop, }); WhilstVectorResult(iter) } fn u_map( self, u: *mut U, map: M, ) -> impl TransformableValues where M: Fn(*mut U, Self::Item) -> O, { let iter = self.0.into_iter().map(move |x| match x { WhilstAtom::Continue(x) => WhilstAtom::Continue(map(u, x)), WhilstAtom::Stop => WhilstAtom::Stop, }); WhilstVector(iter) } fn u_filter( self, u: *mut U, filter: F, ) -> impl TransformableValues where F: Fn(*mut U, &Self::Item) -> bool, { let iter = self.0.into_iter().filter_map(move |x| match x { WhilstAtom::Continue(x) => match filter(u, &x) { true => Some(WhilstAtom::Continue(x)), false => None, }, WhilstAtom::Stop => Some(WhilstAtom::Stop), }); WhilstVector(iter) } fn u_flat_map( self, u: *mut U, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(*mut U, Self::Item) -> Vo, { let iter = self .0 .into_iter() .flat_map(move |atom| WhilstAtomFlatMapIter::u_from_atom(u, atom, &flat_map)); WhilstVector(iter) } fn u_filter_map( self, u: *mut U, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(*mut U, Self::Item) -> Option, { let iter = self.0.into_iter().filter_map(move |x| match x { WhilstAtom::Continue(x) => filter_map(u, x).map(WhilstAtom::Continue), WhilstAtom::Stop => Some(WhilstAtom::Stop), }); WhilstVector(iter) } } ================================================ FILE: src/generic_values/whilst_vector_result.rs ================================================ use crate::generic_values::{ Values, WhilstAtom, runner_results::{ArbitraryPush, Fallible, Next, OrderedPush, Reduce, SequentialPush}, }; use alloc::vec::Vec; use orx_concurrent_bag::ConcurrentBag; use orx_fixed_vec::IntoConcurrentPinnedVec; use orx_pinned_vec::PinnedVec; pub struct WhilstVectorResult(pub(crate) I) where I: IntoIterator>>, E: Send; impl Values for WhilstVectorResult where I: IntoIterator>>, E: Send, { type Item = T; type Fallibility = Fallible; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { for x in self.0 { match x { WhilstAtom::Continue(Ok(x)) => vector.push(x), WhilstAtom::Continue(Err(error)) => { return SequentialPush::StoppedByError { error }; } WhilstAtom::Stop => return SequentialPush::StoppedByWhileCondition, } } SequentialPush::Done } fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { for x in self.0 { match x { WhilstAtom::Continue(Ok(x)) => vec.push((idx, x)), WhilstAtom::Continue(Err(error)) => { return OrderedPush::StoppedByError { idx, error }; } WhilstAtom::Stop => return OrderedPush::StoppedByWhileCondition { idx }, } } OrderedPush::Done } fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { for x in self.0 { match x { WhilstAtom::Continue(Ok(x)) => _ = bag.push(x), WhilstAtom::Continue(Err(error)) => return ArbitraryPush::StoppedByError { error }, WhilstAtom::Stop => return ArbitraryPush::StoppedByWhileCondition, } } ArbitraryPush::Done } fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { let mut iter = self.0.into_iter(); let mut acc = match acc { Some(x) => x, None => { let first = iter.next(); match first { None => return Reduce::Done { acc: None }, // empty iterator but not stopped, acc is None Some(first) => match first { WhilstAtom::Continue(Ok(x)) => x, WhilstAtom::Continue(Err(error)) => { return Reduce::StoppedByError { error }; } WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: None }, // first element is stop, acc is None }, } } }; for x in iter { match x { WhilstAtom::Continue(Ok(x)) => acc = reduce(acc, x), WhilstAtom::Continue(Err(error)) => return Reduce::StoppedByError { error }, WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: Some(acc) }, } } Reduce::Done { acc: Some(acc) } } fn u_acc_reduce(self, u: *mut U, acc: Option, reduce: X) -> Reduce where X: Fn(*mut U, Self::Item, Self::Item) -> Self::Item, { let mut iter = self.0.into_iter(); let mut acc = match acc { Some(x) => x, None => { let first = iter.next(); match first { None => return Reduce::Done { acc: None }, // empty iterator but not stopped, acc is None Some(first) => match first { WhilstAtom::Continue(Ok(x)) => x, WhilstAtom::Continue(Err(error)) => { return Reduce::StoppedByError { error }; } WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: None }, // first element is stop, acc is None }, } } }; for x in iter { match x { WhilstAtom::Continue(Ok(x)) => acc = reduce(u, acc, x), WhilstAtom::Continue(Err(error)) => return Reduce::StoppedByError { error }, WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: Some(acc) }, } } Reduce::Done { acc: Some(acc) } } fn next(self) -> Next { match self.0.into_iter().next() { Some(x) => match x { WhilstAtom::Continue(Ok(x)) => Next::Done { value: Some(x) }, WhilstAtom::Continue(Err(error)) => Next::StoppedByError { error }, WhilstAtom::Stop => Next::StoppedByWhileCondition, }, None => Next::Done { value: None }, } } } ================================================ FILE: src/heap_sort.rs ================================================ use alloc::vec; use alloc::vec::Vec; use orx_pinned_vec::PinnedVec; use orx_priority_queue::{BinaryHeap, PriorityQueue}; pub fn heap_sort_into( mut vectors: Vec>, max_idx_inc: Option, output: &mut P, ) where P: PinnedVec, { let mut queue = BinaryHeap::with_capacity(vectors.len()); let mut indices = vec![0; vectors.len()]; let max_idx_or_inf = max_idx_inc.unwrap_or(usize::MAX); for (v, vec) in vectors.iter().enumerate() { if let Some(x) = vec.get(indices[v]) && x.0 <= max_idx_or_inf { queue.push(v, x.0); } } let mut curr_v = queue.pop_node(); match max_idx_inc { None => { while let Some(v) = curr_v { let idx = indices[v]; indices[v] += 1; curr_v = match vectors[v].get(indices[v]) { Some(x) => Some(queue.push_then_pop(v, x.0).0), None => queue.pop_node(), }; let ptr = vectors[v].as_ptr(); output.push(unsafe { ptr.add(idx).read().1 }); } } Some(max_idx) => { while let Some(v) = curr_v { let idx = indices[v]; indices[v] += 1; curr_v = match vectors[v].get(indices[v]) { Some(x) if x.0 <= max_idx => Some(queue.push_then_pop(v, x.0).0), _ => queue.pop_node(), }; let ptr = vectors[v].as_ptr(); output.push(unsafe { ptr.add(idx).read().1 }); } } } for vec in vectors.iter_mut() { // SAFETY: this prevents to drop the elements which are already moved to pinned_vec // allocation within vec.capacity() will still be reclaimed; however, as uninitialized memory unsafe { vec.set_len(0) }; } } ================================================ FILE: src/into_par_iter.rs ================================================ use crate::{Params, computational_variants::Par, runner::DefaultRunner}; use orx_concurrent_iter::{ConcurrentIter, IntoConcurrentIter}; /// Trait to convert a source (collection or generator) into a parallel iterator; i.e., [`ParIter`], /// using its [`into_par`] method. /// /// It can be considered as the *concurrent counterpart* of the [`IntoIterator`] trait. /// /// Note that every [`IntoConcurrentIter`] type automatically implements [`IntoParIter`]. /// /// [`into_par`]: crate::IntoParIter::into_par /// [`IntoConcurrentIter`]: orx_concurrent_iter::IntoConcurrentIter /// [`ParIter`]: crate::ParIter /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // Vec: IntoParIter /// let vec = vec![1, 2, 3, 4]; /// assert_eq!(vec.into_par().max(), Some(4)); /// /// // Range: IntoParIter /// let range = 1..5; /// assert_eq!(range.into_par().max(), Some(4)); /// ``` pub trait IntoParIter: IntoConcurrentIter { /// Trait to convert a source (collection or generator) into a parallel iterator; i.e., [`ParIter`], /// using its [`into_par`] method. /// /// It can be considered as the *concurrent counterpart* of the [`IntoIterator`] trait. /// /// [`into_par`]: crate::IntoParIter::into_par /// [`ParIter`]: crate::ParIter /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // Vec: IntoParIter /// let vec = vec![1, 2, 3, 4]; /// assert_eq!(vec.into_par().max(), Some(4)); /// /// // Range: IntoParIter /// let range = 1..5; /// assert_eq!(range.into_par().max(), Some(4)); /// ``` fn into_par(self) -> Par; } impl IntoParIter for I where I: IntoConcurrentIter, { fn into_par(self) -> Par { Par::new(Default::default(), Params::default(), self.into_con_iter()) } } impl IntoConcurrentIter for Par { type Item = I::Item; type IntoIter = I; fn into_con_iter(self) -> Self::IntoIter { let (_, _, iter) = self.destruct(); iter } } ================================================ FILE: src/iter/mod.rs ================================================ mod recursive; mod special_iterators; pub use recursive::IntoParIterRec; pub use special_iterators::{ParEmpty, empty}; ================================================ FILE: src/iter/recursive/into_par_rec_iter.rs ================================================ use crate::{DefaultRunner, Params, computational_variants::Par}; use orx_concurrent_recursive_iter::{ConcurrentRecursiveIter, Queue}; // unknown size /// Trait to convert an iterator into a recursive parallel iterator together with the `extend` method. /// Recursive iterators are most useful for defining parallel computations over non-linear data structures /// such as trees or graphs. /// /// Created parallel iterator is a regular parallel iterator; i.e., we have access to all [`ParIter`] features. /// /// It is recursive due to the extension. The recursive parallel iterator will yield /// * all initial elements contained in this iterator, /// * all elements dynamically added to the queue with the `extend` method while processing the elements. /// /// You may read more about the [`ConcurrentRecursiveIter`]. /// /// [`ParIter`]: crate::ParIter pub trait IntoParIterRec where Self: IntoIterator, Self::Item: Send, { /// Converts this iterator into a recursive parallel iterator together with the `extend` method. /// Recursive iterators are most useful for defining parallel computations over non-linear data structures /// such as trees or graphs. /// /// Created parallel iterator is a regular parallel iterator; i.e., we have access to all [`ParIter`] features. /// /// It is recursive due to the extension. The recursive parallel iterator will yield /// * all initial elements contained in this iterator, /// * all elements dynamically added to the queue with the `extend` method while processing the elements. /// /// You may read more about the [`ConcurrentRecursiveIter`]. /// /// The `extend` function defines the recursive expansion behavior. It takes two arguments: /// * `element: &Self::Item` is the item being processed. /// * `queue: Queue` is the queue of remaining elements/tasks which exposes two methods: /// * `push(item)` allows us to add one item to the queue, /// * `extend(items)` allows us to add all of the items to the queue. Here `items` must have a known /// size (`ExactSizeIterator`). /// /// Adding children one-by-one with `push` or all together with `extend` might be the extreme options. /// Actually, any intermediate approach is also possible. For instance, we can choose to `extend` in /// chunks of say 50 tasks. If the item happens to create 140 children, we can handle this with four /// `extend` calls. /// /// Using either of the methods might be beneficial for different use cases. /// /// Pushing children one by one makes the new task available for other threads as fast as possible. Further, /// when we don't know the exact number of children ahead of time, and we don't want to use heap allocation /// to store the children in a vec before adding them to the queue just to make it sized, we can add the /// elements one-by-one with the `queue.push(item)` method. On the other hand, this approach will have more /// parallelization overhead. /// /// When we extending children all at once using `queue.extend(items)`, we minimize the parallelization overhead /// for adding tasks to the queue. On the other hand, the children will be available only when writing of all /// children to the queue is complete which might cause idleness when tasks are scarce. Still, the recommendation /// is to try to `extend` first whenever possible due to the following: (i) if we extend with a lot of children, /// the tasks will not be scarce; (ii) and if we extend with only a few of items, the delay of making the tasks /// available for other threads will be short. /// /// The decision is use-case specific and best to benchmark for the specific input. /// /// This crate makes use of the [`ConcurrentRecursiveIter`] for this computation and provides three ways to execute /// this computation in parallel. /// /// ## A. Recursive Iterator with Exact Length /// /// If we know, or if it is possible and sufficiently cheap to find out, the exact length of the iterator, /// it is recommended to work with exact length recursive iterator. Note that the exact length of an /// iterator is the total of all elements that will be created. This gives the parallel executor /// opportunity to optimize the chunk sizes. /// /// We can use `initial_elements.into_par_rec_exact(extend, count)` to create the iterator with exact length. /// /// ## B. Recursive Iterator with Unknown Length /// /// If we cannot know or it is expensive to know the exact length of the iterator ahead of time, we can /// still create a recursive parallel iterator. In these cases; however, it is recommended to provide /// chunk size explicitly depending on the number of threads that will be used and any estimate on the exact /// length. /// /// We can use `initial_elements.into_par_rec(extend)` to create the iterator without length information. /// /// ## C. Linearization /// /// Even with exact length, a recursive parallel iterator is much more dynamic than a flat parallel /// iterator. This dynamic nature of shrinking and growing concurrently requires a greater parallelization /// overhead. An alternative approach is to eagerly discover all tasks and then perform the parallel /// computation over the flattened input of tasks using [`linearize`] transformation. /// /// We can use `initial_elements.into_par_rec(extend).linearize()` to create the flattened iterator. /// /// [`ParIter`]: crate::ParIter /// [`ConcurrentRecursiveIter`]: orx_concurrent_recursive_iter::ConcurrentRecursiveIter /// [`linearize`]: crate::computational_variants::Par::linearize /// /// ## Examples /// /// In the following example we perform some parallel computations over a tree. /// It demonstrates that a "recursive parallel iterator" is just a parallel iterator with /// access to all [`ParIter`] methods. /// Once we create the recursive parallel iterator with the `extend` definition, we can use it as /// a regular parallel iterator. /// /// Unfortunately, the example requires a long set up for completeness. Note that the relevant /// code blocks begin after line `// parallel reduction`. /// /// ``` /// use orx_parallel::*; /// use rand::{Rng, SeedableRng}; /// use rand_chacha::ChaCha8Rng; /// use std::{collections::HashSet, ops::Range}; /// /// pub struct Node { /// pub idx: usize, /// pub data: T, /// pub children: Vec>, /// } /// /// impl Node { /// fn create_node(out_edges: &[Vec], idx: usize, data: fn(usize) -> T) -> Node { /// Node { /// idx, /// data: data(idx), /// children: out_edges[idx] /// .iter() /// .map(|child_idx| Self::create_node(out_edges, *child_idx, data)) /// .collect(), /// } /// } /// /// pub fn new_tree( /// num_nodes: usize, /// degree: Range, /// data: fn(usize) -> T, /// rng: &mut impl Rng, /// ) -> Node { /// assert!(num_nodes >= 2); /// /// let mut leaves = vec![0]; /// let mut remaining: Vec<_> = (1..num_nodes).collect(); /// let mut edges = vec![]; /// let mut out_edges = vec![vec![]; num_nodes]; /// /// while !remaining.is_empty() { /// let leaf_idx = rng.random_range(0..leaves.len()); /// let leaf = leaves.remove(leaf_idx); /// /// let degree = rng.random_range(degree.clone()); /// match degree == 0 { /// true => leaves.push(leaf), /// false => { /// let children_indices: HashSet<_> = (0..degree) /// .map(|_| rng.random_range(0..remaining.len())) /// .collect(); /// /// let mut sorted: Vec<_> = children_indices.iter().copied().collect(); /// sorted.sort(); /// /// edges.extend(children_indices.iter().map(|c| (leaf, remaining[*c]))); /// out_edges[leaf] = children_indices.iter().map(|c| remaining[*c]).collect(); /// leaves.extend(children_indices.iter().map(|c| remaining[*c])); /// /// for idx in sorted.into_iter().rev() { /// remaining.remove(idx); /// } /// } /// } /// } /// /// Self::create_node(&out_edges, 0, data) /// } /// } /// /// let num_nodes = 1_000; /// let out_degree = 0..100; /// let mut rng = ChaCha8Rng::seed_from_u64(42); /// let data = |idx: usize| idx.to_string(); /// let root = Node::new_tree(num_nodes, out_degree, data, &mut rng); /// /// let compute = |node: &Node| node.data.parse::().unwrap(); /// /// // parallel reduction /// /// fn extend<'a, T: Sync>(node: &&'a Node, queue: &Queue<&'a Node>) { /// queue.extend(&node.children); /// } /// /// let sum = [&root].into_par_rec(extend).map(compute).sum(); /// assert_eq!(sum, 499500); /// /// // or any parallel computation such as map->filter->collect /// /// let result: Vec<_> = [&root] /// .into_par_rec(extend) /// .map(compute) /// .filter(|x| x.is_multiple_of(7)) /// .collect(); /// assert_eq!(result.len(), 143); /// /// // or filter during extension /// fn extend_filtered<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { /// for child in &node.children { /// if child.idx != 42 { /// queue.push(child); /// } /// } /// } /// /// let sum = [&root].into_par_rec(extend_filtered).map(compute).sum(); /// ``` fn into_par_rec( self, extend: E, ) -> Par, DefaultRunner> where E: Fn(&Self::Item, &Queue) + Sync; /// Converts this iterator into a recursive parallel iterator together with the `extend` method. /// Recursive iterators are most useful for defining parallel computations over non-linear data structures /// such as trees or graphs. /// /// Created parallel iterator is a regular parallel iterator; i.e., we have access to all [`ParIter`] features. /// /// It is recursive due to the extension. The recursive parallel iterator will yield /// * all initial elements contained in this iterator, /// * all elements dynamically added to the queue with the `extend` method while processing the elements. /// /// You may read more about the [`ConcurrentRecursiveIter`]. /// /// The `extend` function defines the recursive expansion behavior. It takes two arguments: /// * `element: &Self::Item` is the item being processed. /// * `queue: Queue` is the queue of remaining elements/tasks which exposes two methods: /// * `push(item)` allows us to add one item to the queue, /// * `extend(items)` allows us to add all of the items to the queue. Here `items` must have a known /// size (`ExactSizeIterator`). /// /// Adding children one-by-one with `push` or all together with `extend` might be the extreme options. /// Actually, any intermediate approach is also possible. For instance, we can choose to `extend` in /// chunks of say 50 tasks. If the item happens to create 140 children, we can handle this with four /// `extend` calls. /// /// Using either of the methods might be beneficial for different use cases. /// /// Pushing children one by one makes the new task available for other threads as fast as possible. Further, /// when we don't know the exact number of children ahead of time, and we don't want to use heap allocation /// to store the children in a vec before adding them to the queue just to make it sized, we can add the /// elements one-by-one with the `queue.push(item)` method. On the other hand, this approach will have more /// parallelization overhead. /// /// When we extending children all at once using `queue.extend(items)`, we minimize the parallelization overhead /// for adding tasks to the queue. On the other hand, the children will be available only when writing of all /// children to the queue is complete which might cause idleness when tasks are scarce. Still, the recommendation /// is to try to `extend` first whenever possible due to the following: (i) if we extend with a lot of children, /// the tasks will not be scarce; (ii) and if we extend with only a few of items, the delay of making the tasks /// available for other threads will be short. /// /// The decision is use-case specific and best to benchmark for the specific input. /// /// This crate makes use of the [`ConcurrentRecursiveIter`] for this computation and provides three ways to execute /// this computation in parallel. /// /// ## A. Recursive Iterator with Exact Length /// /// If we know, or if it is possible and sufficiently cheap to find out, the exact length of the iterator, /// it is recommended to work with exact length recursive iterator. Note that the exact length of an /// iterator is the total of all elements that will be created. This gives the parallel executor /// opportunity to optimize the chunk sizes. /// /// We can use `initial_elements.into_par_rec_exact(extend, count)` to create the iterator with exact length. /// /// ## B. Recursive Iterator with Unknown Length /// /// If we cannot know or it is expensive to know the exact length of the iterator ahead of time, we can /// still create a recursive parallel iterator. In these cases; however, it is recommended to provide /// chunk size explicitly depending on the number of threads that will be used and any estimate on the exact /// length. /// /// We can use `initial_elements.into_par_rec(extend)` to create the iterator without length information. /// /// ## C. Linearization /// /// Even with exact length, a recursive parallel iterator is much more dynamic than a flat parallel /// iterator. This dynamic nature of shrinking and growing concurrently requires a greater parallelization /// overhead. An alternative approach is to eagerly discover all tasks and then perform the parallel /// computation over the flattened input of tasks using [`linearize`] transformation. /// /// We can use `initial_elements.into_par_rec(extend).linearize()` to create the flattened iterator. /// /// [`ParIter`]: crate::ParIter /// [`ConcurrentRecursiveIter`]: orx_concurrent_recursive_iter::ConcurrentRecursiveIter /// [`linearize`]: crate::computational_variants::Par::linearize /// /// ## Examples /// /// In the following example we perform some parallel computations over a tree. /// It demonstrates that a "recursive parallel iterator" is just a parallel iterator with /// access to all [`ParIter`] methods. /// Once we create the recursive parallel iterator with the `extend` definition, we can use it as /// a regular parallel iterator. /// /// Unfortunately, the example requires a long set up for completeness. Note that the relevant /// code blocks begin after line `// parallel reduction`. /// /// ``` /// use orx_parallel::*; /// use rand::{Rng, SeedableRng}; /// use rand_chacha::ChaCha8Rng; /// use std::{collections::HashSet, ops::Range}; /// /// pub struct Node { /// pub idx: usize, /// pub data: T, /// pub children: Vec>, /// } /// /// impl Node { /// fn create_node(out_edges: &[Vec], idx: usize, data: fn(usize) -> T) -> Node { /// Node { /// idx, /// data: data(idx), /// children: out_edges[idx] /// .iter() /// .map(|child_idx| Self::create_node(out_edges, *child_idx, data)) /// .collect(), /// } /// } /// /// pub fn new_tree( /// num_nodes: usize, /// degree: Range, /// data: fn(usize) -> T, /// rng: &mut impl Rng, /// ) -> Node { /// assert!(num_nodes >= 2); /// /// let mut leaves = vec![0]; /// let mut remaining: Vec<_> = (1..num_nodes).collect(); /// let mut edges = vec![]; /// let mut out_edges = vec![vec![]; num_nodes]; /// /// while !remaining.is_empty() { /// let leaf_idx = rng.random_range(0..leaves.len()); /// let leaf = leaves.remove(leaf_idx); /// /// let degree = rng.random_range(degree.clone()); /// match degree == 0 { /// true => leaves.push(leaf), /// false => { /// let children_indices: HashSet<_> = (0..degree) /// .map(|_| rng.random_range(0..remaining.len())) /// .collect(); /// /// let mut sorted: Vec<_> = children_indices.iter().copied().collect(); /// sorted.sort(); /// /// edges.extend(children_indices.iter().map(|c| (leaf, remaining[*c]))); /// out_edges[leaf] = children_indices.iter().map(|c| remaining[*c]).collect(); /// leaves.extend(children_indices.iter().map(|c| remaining[*c])); /// /// for idx in sorted.into_iter().rev() { /// remaining.remove(idx); /// } /// } /// } /// } /// /// Self::create_node(&out_edges, 0, data) /// } /// } /// /// let num_nodes = 1_000; /// let out_degree = 0..100; /// let mut rng = ChaCha8Rng::seed_from_u64(42); /// let data = |idx: usize| idx.to_string(); /// let root = Node::new_tree(num_nodes, out_degree, data, &mut rng); /// /// let compute = |node: &Node| node.data.parse::().unwrap(); /// /// // parallel reduction /// /// fn extend<'a, T: Sync>(node: &&'a Node, queue: &Queue<&'a Node>) { /// queue.extend(&node.children); /// } /// /// let sum = [&root].into_par_rec(extend).map(compute).sum(); /// assert_eq!(sum, 499500); /// /// // or any parallel computation such as map->filter->collect /// /// let result: Vec<_> = [&root] /// .into_par_rec(extend) /// .map(compute) /// .filter(|x| x.is_multiple_of(7)) /// .collect(); /// assert_eq!(result.len(), 143); /// /// // or filter during extension /// fn extend_filtered<'a>(node: &&'a Node, queue: &Queue<&'a Node>) { /// for child in &node.children { /// if child.idx != 42 { /// queue.push(child); /// } /// } /// } /// /// let sum = [&root].into_par_rec(extend_filtered).map(compute).sum(); /// ``` fn into_par_rec_exact( self, extend: E, exact_len: usize, ) -> Par, DefaultRunner> where E: Fn(&Self::Item, &Queue) + Sync; } impl IntoParIterRec for X where X: IntoIterator, X::Item: Send, { fn into_par_rec( self, extend: E, ) -> Par, DefaultRunner> where E: Fn(&Self::Item, &Queue) + Sync, { let con_rec_iter = ConcurrentRecursiveIter::new(self, extend); Par::new(DefaultRunner::default(), Params::default(), con_rec_iter) } fn into_par_rec_exact( self, extend: E, exact_len: usize, ) -> Par, DefaultRunner> where E: Fn(&Self::Item, &Queue) + Sync, { let con_rec_iter = ConcurrentRecursiveIter::new_exact(self, extend, exact_len); Par::new(DefaultRunner::default(), Params::default(), con_rec_iter) } } ================================================ FILE: src/iter/recursive/mod.rs ================================================ mod into_par_rec_iter; mod rec_par_iter; pub use into_par_rec_iter::IntoParIterRec; ================================================ FILE: src/iter/recursive/rec_par_iter.rs ================================================ use crate::{ ParIter, ParallelRunner, computational_variants::{Par, ParMap, ParXap}, generic_values::{TransformableValues, runner_results::Infallible}, }; use alloc::vec::Vec; use orx_concurrent_iter::{IntoConcurrentIter, implementations::ConIterVec}; use orx_concurrent_recursive_iter::{ConcurrentRecursiveIter, Queue}; type Rec = ConcurrentRecursiveIter; impl Par, R> where T: Send + Sync, E: Fn(&T, &Queue) + Sync, R: ParallelRunner + Clone, { /// Even with exact length, a recursive parallel iterator is much more dynamic than a flat parallel /// iterator. This dynamic nature of shrinking and growing concurrently requires a greater parallelization /// overhead. An alternative approach is to eagerly discover all tasks and then perform the parallel /// computation over the flattened input of tasks. /// /// The `linearize` approach works in two parallelization phases: /// * first phase to linearize the inputs in parallel over the non-linear data, and /// * second phase to perform the computation in parallel over the linear data. /// /// See [`into_par_rec`] and [`into_par_rec_exact`] for examples. /// /// [`into_par_rec`]: crate::IntoParIterRec::into_par_rec /// [`into_par_rec_exact`]: crate::IntoParIterRec::into_par_rec_exact pub fn linearize(self) -> Par, R> { let params = self.params(); let orchestrator = self.orchestrator().clone(); let items: Vec<_> = self.collect(); let iter = items.into_con_iter(); Par::new(orchestrator, params, iter) } } impl ParMap, O, M1, R> where T: Send + Sync, E: Fn(&T, &Queue) + Sync, R: ParallelRunner + Clone, M1: Fn(T) -> O + Sync, { /// Even with exact length, a recursive parallel iterator is much more dynamic than a flat parallel /// iterator. This dynamic nature of shrinking and growing concurrently requires a greater parallelization /// overhead. An alternative approach is to eagerly discover all tasks and then perform the parallel /// computation over the flattened input of tasks. /// /// The `linearize` approach works in two parallelization phases: /// * first phase to linearize the inputs in parallel over the non-linear data, and /// * second phase to perform the computation in parallel over the linear data. /// /// See [`into_par_rec`] and [`into_par_rec_exact`] for examples. /// /// [`into_par_rec`]: crate::IntoParIterRec::into_par_rec /// [`into_par_rec_exact`]: crate::IntoParIterRec::into_par_rec_exact pub fn linearize(self) -> ParMap, O, M1, R> { let (orchestrator, params, iter, map1) = self.destruct(); let par = Par::new(orchestrator.clone(), params, iter); let items: Vec<_> = par.collect(); let iter = items.into_con_iter(); ParMap::new(orchestrator, params, iter, map1) } } impl ParXap, Vo, X1, R> where T: Send + Sync, E: Fn(&T, &Queue) + Sync, R: ParallelRunner + Clone, X1: Fn(T) -> Vo + Sync, Vo: TransformableValues, { /// Even with exact length, a recursive parallel iterator is much more dynamic than a flat parallel /// iterator. This dynamic nature of shrinking and growing concurrently requires a greater parallelization /// overhead. An alternative approach is to eagerly discover all tasks and then perform the parallel /// computation over the flattened input of tasks. /// /// The `linearize` approach works in two parallelization phases: /// * first phase to linearize the inputs in parallel over the non-linear data, and /// * second phase to perform the computation in parallel over the linear data. /// /// See [`into_par_rec`] and [`into_par_rec_exact`] for examples. /// /// [`into_par_rec`]: crate::IntoParIterRec::into_par_rec /// [`into_par_rec_exact`]: crate::IntoParIterRec::into_par_rec_exact pub fn linearize(self) -> ParXap, Vo, X1, R> { let (orchestrator, params, iter, xap1) = self.destruct(); let par = Par::new(orchestrator.clone(), params, iter); let items: Vec<_> = par.collect(); let iter = items.into_con_iter(); ParXap::new(orchestrator, params, iter, xap1) } } ================================================ FILE: src/iter/special_iterators.rs ================================================ use crate::{computational_variants::Par, runner::DefaultRunner}; use orx_concurrent_iter::implementations::ConIterEmpty; /// An empty parallel iterator which does not yield any elements. pub type ParEmpty = Par, R>; /// Creates an empty parallel iterator which does not yield any elements. pub fn empty() -> ParEmpty { ParEmpty::new(Default::default(), Default::default(), Default::default()) } ================================================ FILE: src/iter_into_par_iter.rs ================================================ use crate::{Params, computational_variants::Par, runner::DefaultRunner}; use orx_concurrent_iter::{IterIntoConcurrentIter, implementations::ConIterOfIter}; /// Any regular iterator implements [`IterIntoParIter`] trait allowing them to be used /// as a parallel iterator; i.e., [`ParIter`], by calling [`iter_into_par`]. /// /// Pulling of elements from the iterator are synchronized and safely shared to threads. /// /// Therefore, converting an iterator into a parallel iterator is most useful whenever /// the work to be done on each element is a larger task than just yielding elements by the /// underlying collection or generator. /// /// Note that every [`IterIntoConcurrentIter`] type automatically implements [`IterIntoParIter`]. /// /// [`iter_into_par`]: crate::IterIntoParIter::iter_into_par /// [`ParIter`]: crate::ParIter /// [`IterIntoConcurrentIter`]: orx_concurrent_iter::IterIntoConcurrentIter /// /// # Examples /// /// In the following example, an arbitrary iterator is converted into a parallel iterator /// and shared with multiple threads as a shared reference. /// /// ``` /// use orx_parallel::*; /// /// let data: Vec<_> = (0..100).map(|x| x.to_string()).collect(); /// /// // an arbitrary iterator /// let iter = data /// .into_iter() /// .filter(|x| !x.starts_with('3')) /// .map(|x| format!("{x}!")); /// /// // convert arbitrary iterator into ParIter /// let par_iter = iter.iter_into_par(); /// let num_characters = par_iter.map(|x| x.len()).sum(); /// /// assert_eq!(num_characters, 258); /// ``` /// /// Similarly, in the following example, computation over elements of a generic /// iterator are distributed into multiple threads. /// /// ``` /// use orx_parallel::*; /// /// let data: Vec<_> = (0..123).collect(); /// /// // arbitrary iterator /// let iter = data.iter().filter(|x| *x % 2 == 0).map(|x| x.to_string()); /// /// // parallel computation /// let sum_evens = iter /// .iter_into_par() /// .map(|x| x.parse::().unwrap()) /// .sum(); /// /// assert_eq!(sum_evens, 3782); /// ``` pub trait IterIntoParIter: Iterator { /// Any regular iterator implements [`IterIntoParIter`] trait allowing them to be used /// as a parallel iterator; i.e., [`ParIter`], by calling [`iter_into_par`]. /// /// Pulling of elements from the iterator are synchronized and safely shared to threads. /// /// Therefore, converting an iterator into a parallel iterator is most useful whenever /// the work to be done on each element is a larger task than just yielding elements by the /// underlying collection or generator. /// /// Note that every [`IterIntoConcurrentIter`] type automatically implements [`IterIntoParIter`]. /// /// [`iter_into_par`]: crate::IterIntoParIter::iter_into_par /// [`ParIter`]: crate::ParIter /// [`IterIntoConcurrentIter`]: orx_concurrent_iter::IterIntoConcurrentIter /// /// # Examples /// /// In the following example, an arbitrary iterator is converted into a parallel iterator /// and shared with multiple threads as a shared reference. /// /// ``` /// use orx_parallel::*; /// /// let data: Vec<_> = (0..100).map(|x| x.to_string()).collect(); /// /// // an arbitrary iterator /// let iter = data /// .into_iter() /// .filter(|x| !x.starts_with('3')) /// .map(|x| format!("{x}!")); /// /// // convert arbitrary iterator into ParIter /// let par_iter = iter.iter_into_par(); /// let num_characters = par_iter.map(|x| x.len()).sum(); /// /// assert_eq!(num_characters, 258); /// ``` /// /// Similarly, in the following example, computation over elements of a generic /// iterator are distributed into multiple threads. /// /// ``` /// use orx_parallel::*; /// /// let data: Vec<_> = (0..123).collect(); /// /// // arbitrary iterator /// let iter = data.iter().filter(|x| *x % 2 == 0).map(|x| x.to_string()); /// /// // parallel computation /// let sum_evens = iter /// .iter_into_par() /// .map(|x| x.parse::().unwrap()) /// .sum(); /// /// assert_eq!(sum_evens, 3782); /// ``` fn iter_into_par(self) -> Par, DefaultRunner> where Self: Sized, Self::Item: Send; } impl IterIntoParIter for I where I: Iterator, I::Item: Send + Sync, { fn iter_into_par(self) -> Par, DefaultRunner> { Par::new( Default::default(), Params::default(), self.iter_into_con_iter(), ) } } ================================================ FILE: src/lib.rs ================================================ #![doc = include_str!("../README.md")] #![warn( missing_docs, clippy::unwrap_in_result, clippy::unwrap_used, clippy::panic, clippy::panic_in_result_fn, clippy::float_cmp, clippy::float_cmp_const, clippy::missing_panics_doc, clippy::todo )] #![no_std] extern crate alloc; #[cfg(any(test, feature = "std"))] extern crate std; mod collect_into; /// Module containing variants of parallel iterators. pub mod computational_variants; mod default_fns; mod enumerate; mod env; /// Module defining the parallel runner trait and the default parallel runner. pub mod executor; mod generic_values; mod heap_sort; mod into_par_iter; /// Module for creating special iterators. pub mod iter; mod iter_into_par_iter; mod par_iter; mod par_iter_option; mod par_iter_result; mod par_thread_pool; mod parallel_drainable; mod parallelizable; mod parallelizable_collection; mod parallelizable_collection_mut; mod parameters; /// ParallelRunner for parallel execution and managing threads. pub mod runner; mod special_type_sets; /// Module defining parallel iterators with mutable access to values distributed to each thread. pub mod using; /// Module defining the GenericIterator which is a generalization over /// sequential iterator, rayon's parallel iterator and orx-parallel's /// parallel iterator. /// This is particularly useful for running experiments and comparing /// results of computations with different methods. #[cfg(feature = "generic_iterator")] pub mod generic_iterator; /// Module containing experiments for new algorithms, computations or factorial analysis. #[cfg(feature = "experiment")] pub mod experiment; /// Module with utility methods for testing. #[cfg(test)] mod test_utils; // re-export pub use orx_concurrent_recursive_iter::Queue; // export pub use collect_into::ParCollectInto; pub use enumerate::ParEnumerate; pub use executor::{DefaultExecutor, ParallelExecutor, ThreadExecutor}; pub use into_par_iter::IntoParIter; pub use iter::IntoParIterRec; pub use iter_into_par_iter::IterIntoParIter; pub use par_iter::ParIter; pub use par_iter_option::ParIterOption; pub use par_iter_result::ParIterResult; pub use par_thread_pool::ParThreadPool; pub use parallel_drainable::ParallelDrainableOverSlice; pub use parallelizable::Parallelizable; pub use parallelizable_collection::ParallelizableCollection; pub use parallelizable_collection_mut::ParallelizableCollectionMut; pub use parameters::{ChunkSize, IterationOrder, NumThreads, Params}; pub use runner::{DefaultPool, DefaultRunner, ParallelRunner, RunnerWithPool, SequentialPool}; pub use special_type_sets::Sum; pub use using::ParIterOptionUsing; pub use using::ParIterResultUsing; pub use using::ParIterUsing; #[cfg(feature = "std")] pub use executor::ParallelExecutorWithDiagnostics; #[cfg(feature = "pond")] pub use runner::PondPool; #[cfg(feature = "std")] pub use runner::StdDefaultPool; #[cfg(feature = "yastl")] pub use runner::YastlPool; ================================================ FILE: src/par_iter.rs ================================================ use crate::computational_variants::fallible_option::ParOption; use crate::par_iter_option::{IntoOption, ParIterOption}; use crate::par_iter_result::IntoResult; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::{UsingClone, UsingFun}; use crate::{ParIterResult, ParThreadPool, RunnerWithPool}; use crate::{ ParIterUsing, Params, collect_into::ParCollectInto, default_fns::{map_clone, map_copy, map_count, reduce_sum, reduce_unit}, parameters::{ChunkSize, IterationOrder, NumThreads}, special_type_sets::Sum, }; use core::cmp::Ordering; use orx_concurrent_iter::ConcurrentIter; /// Parallel iterator. pub trait ParIter: Sized + Send + Sync where R: ParallelRunner, { /// Element type of the parallel iterator. type Item; /// Returns a reference to the input concurrent iterator. fn con_iter(&self) -> &impl ConcurrentIter; /// Parameters of the parallel iterator. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// use std::num::NonZero; /// /// let vec = vec![1, 2, 3, 4]; /// /// assert_eq!( /// vec.par().params(), /// Params::new(NumThreads::Auto, ChunkSize::Auto, IterationOrder::Ordered) /// ); /// /// assert_eq!( /// vec.par().num_threads(0).chunk_size(0).params(), /// Params::new(NumThreads::Auto, ChunkSize::Auto, IterationOrder::Ordered) /// ); /// /// assert_eq!( /// vec.par().num_threads(1).params(), /// Params::new( /// NumThreads::Max(NonZero::new(1).unwrap()), /// ChunkSize::Auto, /// IterationOrder::Ordered /// ) /// ); /// /// assert_eq!( /// vec.par().num_threads(4).chunk_size(64).params(), /// Params::new( /// NumThreads::Max(NonZero::new(4).unwrap()), /// ChunkSize::Exact(NonZero::new(64).unwrap()), /// IterationOrder::Ordered /// ) /// ); /// /// assert_eq!( /// vec.par() /// .num_threads(8) /// .chunk_size(ChunkSize::Min(NonZero::new(16).unwrap())) /// .iteration_order(IterationOrder::Arbitrary) /// .params(), /// Params::new( /// NumThreads::Max(NonZero::new(8).unwrap()), /// ChunkSize::Min(NonZero::new(16).unwrap()), /// IterationOrder::Arbitrary /// ) /// ); /// ``` fn params(&self) -> Params; // params transformations /// Sets the number of threads to be used in the parallel execution. /// Integers can be used as the argument with the following mapping: /// /// * `0` -> `NumThreads::Auto` /// * `1` -> `NumThreads::sequential()` /// * `n > 0` -> `NumThreads::Max(n)` /// /// See [`NumThreads`] for details. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// use std::num::NonZero; /// /// let vec = vec![1, 2, 3, 4]; /// /// // all available threads can be used /// /// assert_eq!( /// vec.par().params(), /// Params::new(NumThreads::Auto, ChunkSize::Auto, IterationOrder::Ordered) /// ); /// /// assert_eq!( /// vec.par().num_threads(0).chunk_size(0).params(), /// Params::new(NumThreads::Auto, ChunkSize::Auto, IterationOrder::Ordered) /// ); /// /// // computation will be executed sequentially on the main thread, no parallelization /// /// assert_eq!( /// vec.par().num_threads(1).params(), /// Params::new( /// NumThreads::Max(NonZero::new(1).unwrap()), /// ChunkSize::Auto, /// IterationOrder::Ordered /// ) /// ); /// /// // maximum 4 threads can be used /// assert_eq!( /// vec.par().num_threads(4).chunk_size(64).params(), /// Params::new( /// NumThreads::Max(NonZero::new(4).unwrap()), /// ChunkSize::Exact(NonZero::new(64).unwrap()), /// IterationOrder::Ordered /// ) /// ); /// /// // maximum 8 threads can be used /// assert_eq!( /// vec.par() /// .num_threads(8) /// .chunk_size(ChunkSize::Min(NonZero::new(16).unwrap())) /// .iteration_order(IterationOrder::Arbitrary) /// .params(), /// Params::new( /// NumThreads::Max(NonZero::new(8).unwrap()), /// ChunkSize::Min(NonZero::new(16).unwrap()), /// IterationOrder::Arbitrary /// ) /// ); /// ``` fn num_threads(self, num_threads: impl Into) -> Self; /// Sets the number of elements to be pulled from the concurrent iterator during the /// parallel execution. When integers are used as argument, the following mapping applies: /// /// * `0` -> `ChunkSize::Auto` /// * `n > 0` -> `ChunkSize::Exact(n)` /// /// Please use the default enum constructor for creating `ChunkSize::Min` variant. /// /// See [`ChunkSize`] for details. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// use std::num::NonZero; /// /// let vec = vec![1, 2, 3, 4]; /// /// // chunk sizes will be dynamically decided by the parallel runner /// /// assert_eq!( /// vec.par().params(), /// Params::new(NumThreads::Auto, ChunkSize::Auto, IterationOrder::Ordered) /// ); /// /// assert_eq!( /// vec.par().num_threads(0).chunk_size(0).params(), /// Params::new(NumThreads::Auto, ChunkSize::Auto, IterationOrder::Ordered) /// ); /// /// assert_eq!( /// vec.par().num_threads(1).params(), /// Params::new( /// NumThreads::Max(NonZero::new(1).unwrap()), /// ChunkSize::Auto, /// IterationOrder::Ordered /// ) /// ); /// /// // chunk size will always be 64, parallel runner cannot change /// /// assert_eq!( /// vec.par().num_threads(4).chunk_size(64).params(), /// Params::new( /// NumThreads::Max(NonZero::new(4).unwrap()), /// ChunkSize::Exact(NonZero::new(64).unwrap()), /// IterationOrder::Ordered /// ) /// ); /// /// // minimum chunk size will be 16, but can be dynamically increased by the parallel runner /// /// assert_eq!( /// vec.par() /// .num_threads(8) /// .chunk_size(ChunkSize::Min(NonZero::new(16).unwrap())) /// .iteration_order(IterationOrder::Arbitrary) /// .params(), /// Params::new( /// NumThreads::Max(NonZero::new(8).unwrap()), /// ChunkSize::Min(NonZero::new(16).unwrap()), /// IterationOrder::Arbitrary /// ) /// ); /// ``` fn chunk_size(self, chunk_size: impl Into) -> Self; /// Sets the iteration order of the parallel computation. /// /// See [`IterationOrder`] for details. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let vec = vec![1, 2, 3, 4]; /// /// // results are collected in order consistent to the input order, /// // or find returns the first element satisfying the predicate /// /// assert_eq!( /// vec.par().params(), /// Params::new(NumThreads::Auto, ChunkSize::Auto, IterationOrder::Ordered) /// ); /// /// assert_eq!( /// vec.par().iteration_order(IterationOrder::Ordered).params(), /// Params::new(NumThreads::Auto, ChunkSize::Auto, IterationOrder::Ordered) /// ); /// /// // results might be collected in arbitrary order /// // or find returns the any of the elements satisfying the predicate /// /// assert_eq!( /// vec.par().iteration_order(IterationOrder::Arbitrary).params(), /// Params::new(NumThreads::Auto, ChunkSize::Auto, IterationOrder::Arbitrary) /// ); /// ``` fn iteration_order(self, order: IterationOrder) -> Self; /// Rather than the [`DefaultRunner`], uses the parallel runner `Q` which implements [`ParallelRunner`]. /// /// See also [`with_pool`]. /// /// Parallel runner of each computation can be independently specified using `with_runner`. /// /// When not specified the default runner is used, which is: /// * [`RunnerWithPool`] with [`StdDefaultPool`] when "std" feature is enabled, /// * [`RunnerWithPool`] with [`SequentialPool`] when "std" feature is disabled. /// /// Note that [`StdDefaultPool`] uses standard native threads. /// /// When working in a no-std environment, the default runner falls back to sequential. /// In this case, `RunnerWithPool` using a particular thread pool must be passed in using `with_runner` /// transformation to achieve parallel computation. /// /// [`RunnerWithPool`]: crate::RunnerWithPool /// [`StdDefaultPool`]: crate::StdDefaultPool /// [`SequentialPool`]: crate::SequentialPool /// [`with_pool`]: crate::ParIter::with_pool /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let inputs: Vec<_> = (0..42).collect(); /// /// // uses the DefaultRunner /// // assuming "std" enabled, RunnerWithPool will be used; i.e., native threads /// let sum = inputs.par().sum(); /// /// // equivalent to: /// #[cfg(feature = "std")] /// { /// let sum2 = inputs.par().with_runner(RunnerWithPool::from(StdDefaultPool::default())).sum(); /// assert_eq!(sum, sum2); /// } /// /// #[cfg(not(miri))] /// #[cfg(feature = "scoped_threadpool")] /// { /// let mut pool = scoped_threadpool::Pool::new(8); /// /// // uses the scoped_threadpool::Pool created with 8 threads /// let sum2 = inputs.par().with_runner(RunnerWithPool::from(&mut pool)).sum(); /// assert_eq!(sum, sum2); /// } /// /// #[cfg(not(miri))] /// #[cfg(feature = "rayon-core")] /// { /// let pool = rayon_core::ThreadPoolBuilder::new() /// .num_threads(8) /// .build() /// .unwrap(); /// /// // uses the rayon-core::ThreadPool created with 8 threads /// let sum2 = inputs.par().with_runner(RunnerWithPool::from(&pool)).sum(); /// assert_eq!(sum, sum2); /// } /// ``` fn with_runner(self, runner: Q) -> impl ParIter; /// Rather than [`DefaultPool`], uses the parallel runner with the given `pool` implementing /// [`ParThreadPool`]. /// /// See also [`with_runner`]. /// /// Thread pool of each computation can be independently specified using `with_pool`. /// /// When not specified the default pool is used, which is: /// * [`StdDefaultPool`] when "std" feature is enabled, /// * [`SequentialPool`] when "std" feature is disabled. /// /// Note that [`StdDefaultPool`] uses standard native threads. /// /// When working in a no-std environment, the default pool falls back to sequential. /// In this case, a thread pool must be passed in using `with_pool` transformation to achieve parallel computation. /// /// Note that if a thread pool, say `pool`, is of a type that implements [`ParThreadPool`]; then: /// * `with_pool` can be called with owned value `with_pool(pool)` for all implementors; but also, /// * with a shared reference `with_pool(&pool)` for most of the implementations (eg: rayon-core, yastl), and /// * with a mutable reference `with_pool(&mut pool)` for others (eg: scoped_threadpool). /// /// [`DefaultPool`]: crate::DefaultPool /// [`RunnerWithPool`]: crate::RunnerWithPool /// [`StdDefaultPool`]: crate::StdDefaultPool /// [`SequentialPool`]: crate::SequentialPool /// [`with_runner`]: crate::ParIter::with_runner /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let inputs: Vec<_> = (0..42).collect(); /// /// // uses the DefaultPool /// // assuming "std" enabled, StdDefaultPool will be used; i.e., native threads /// let sum = inputs.par().sum(); /// /// // equivalent to: /// #[cfg(feature = "std")] /// { /// let sum2 = inputs.par().with_pool(StdDefaultPool::default()).sum(); /// assert_eq!(sum, sum2); /// } /// /// #[cfg(not(miri))] /// #[cfg(feature = "scoped_threadpool")] /// { /// let mut pool = scoped_threadpool::Pool::new(8); /// /// // uses the scoped_threadpool::Pool created with 8 threads /// let sum2 = inputs.par().with_pool(&mut pool).sum(); /// assert_eq!(sum, sum2); /// } /// /// #[cfg(not(miri))] /// #[cfg(feature = "rayon-core")] /// { /// let pool = rayon_core::ThreadPoolBuilder::new() /// .num_threads(8) /// .build() /// .unwrap(); /// /// // uses the rayon-core::ThreadPool created with 8 threads /// let sum2 = inputs.par().with_pool(&pool).sum(); /// assert_eq!(sum, sum2); /// } /// ``` fn with_pool( self, pool: P, ) -> impl ParIter, Item = Self::Item> { let runner = RunnerWithPool::from(pool).with_executor::(); self.with_runner(runner) } // using transformations /// Converts the [`ParIter`] into [`ParIterUsing`] which will have access to a mutable reference of the /// used variable throughout the computation. /// /// Note that each used thread will obtain exactly one instance of the variable. /// /// The signature of the `using` closure is `(thread_idx: usize) -> U` which will create an instance of /// `U` with respect to the `thread_idx`. The `thread_idx` is the order of the spawned thread; i.e., /// if the parallel computation uses 8 threads, the thread indices will be 0, 1, ..., 7. /// /// Details of the **using transformation** can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). /// /// # Examples /// /// ## Example 1: Channels /// /// The following example is taken from rayon's `for_each_with` documentation and converted to using transformation: /// /// ```ignore /// use orx_parallel::*; /// use std::sync::mpsc::channel; /// /// let (sender, receiver) = channel(); /// /// (0..5) /// .into_par() /// .using(|_thread_idx| sender.clone()) /// .for_each(|s, x| s.send(x).unwrap()); /// /// let mut res: Vec<_> = receiver.iter().collect(); /// /// res.sort(); /// /// assert_eq!(&res[..], &[0, 1, 2, 3, 4]) /// ``` /// /// ## Example 2: Random Number Generator /// /// Random number generator is one of the common use cases that is important for a certain class of algorithms. /// /// The following example demonstrates how to safely generate random numbers through mutable references within /// a parallel computation. /// /// Notice the differences between sequential and parallel computation. /// * In sequential computation, a mutable reference to `rng` is captured, while in parallel computation, we /// explicitly define that we will be `using` a random number generator. /// * Parallel iterator does not mutable capture any variable from the scope; however, using transformation /// converts the `ParIter` into `ParIterUsing` which allows mutable access within all iterator methods. /// /// ``` /// use orx_parallel::*; /// use rand::{Rng, SeedableRng}; /// use rand_chacha::ChaCha20Rng; /// /// fn random_walk(rng: &mut impl Rng, position: i64, num_steps: usize) -> i64 { /// (0..num_steps).fold(position, |p, _| random_step(rng, p)) /// } /// /// fn random_step(rng: &mut impl Rng, position: i64) -> i64 { /// match rng.random_bool(0.5) { /// true => position + 1, // to right /// false => position - 1, // to left /// } /// } /// /// fn input_positions() -> Vec { /// (-100..=100).collect() /// } /// /// fn sequential() { /// let positions = input_positions(); /// /// let mut rng = ChaCha20Rng::seed_from_u64(42); /// let final_positions: Vec<_> = positions /// .iter() /// .copied() /// .map(|position| random_walk(&mut rng, position, 10)) /// .collect(); /// let sum_final_positions = final_positions.iter().sum::(); /// } /// /// fn parallel() { /// let positions = input_positions(); /// /// let final_positions: Vec<_> = positions /// .par() /// .copied() /// .using(|t_idx| ChaCha20Rng::seed_from_u64(42 * t_idx as u64)) /// .map(|rng, position| random_walk(rng, position, 10)) /// .collect(); /// let sum_final_positions = final_positions.iter().sum::(); /// } /// /// sequential(); /// parallel(); /// ``` /// /// ## Example 3: Metrics Collection /// /// The following example demonstrates how to collect metrics about a parallel computation with `using` transformation and /// some `unsafe` help with interior mutability. /// /// ``` /// use orx_parallel::*; /// use std::cell::UnsafeCell; /// /// const N: u64 = 1_000; /// const MAX_NUM_THREADS: usize = 4; /// /// // just some work /// fn fibonacci(n: u64) -> u64 { /// let mut a = 0; /// let mut b = 1; /// for _ in 0..n { /// let c = a + b; /// a = b; /// b = c; /// } /// a /// } /// /// #[derive(Default, Debug)] /// struct ThreadMetrics { /// thread_idx: usize, /// num_items_handled: usize, /// handled_42: bool, /// num_filtered_out: usize, /// } /// /// struct ThreadMetricsWriter<'a> { /// metrics_ref: &'a mut ThreadMetrics, /// } /// /// struct ComputationMetrics { /// thread_metrics: UnsafeCell<[ThreadMetrics; MAX_NUM_THREADS]>, /// } /// unsafe impl Sync for ComputationMetrics {} /// impl ComputationMetrics { /// fn new() -> Self { /// let mut thread_metrics: [ThreadMetrics; MAX_NUM_THREADS] = Default::default(); /// for i in 0..MAX_NUM_THREADS { /// thread_metrics[i].thread_idx = i; /// } /// Self { /// thread_metrics: UnsafeCell::new(thread_metrics), /// } /// } /// } /// /// impl ComputationMetrics { /// unsafe fn create_for_thread<'a>(&self, thread_idx: usize) -> ThreadMetricsWriter<'a> { /// // SAFETY: here we create a mutable variable to the thread_idx-th metrics /// // * If we call this method multiple times with the same index, /// // we create multiple mutable references to the same ThreadMetrics, /// // which would lead to a race condition. /// // * We must make sure that `create_for_thread` is called only once per thread. /// // * If we use `create_for_thread` within the `using` call to create mutable values /// // used by the threads, we are certain that the parallel computation /// // will only call this method once per thread; hence, it will not /// // cause the race condition. /// // * On the other hand, we must ensure that we do not call this method /// // externally. /// let array = unsafe { &mut *self.thread_metrics.get() }; /// ThreadMetricsWriter { /// metrics_ref: &mut array[thread_idx], /// } /// } /// } /// /// let mut metrics = ComputationMetrics::new(); /// /// let input: Vec = (0..N).collect(); /// /// let sum = input /// .par() /// // SAFETY: we do not call `create_for_thread` externally; /// // it is safe if it is called only by the parallel computation. /// // Since we unsafely implement Sync for ComputationMetrics, /// // we must ensure that ComputationMetrics is not used elsewhere. /// .using(|t| unsafe { metrics.create_for_thread(t) }) /// .map(|m: &mut ThreadMetricsWriter<'_>, i| { /// // collect some useful metrics /// m.metrics_ref.num_items_handled += 1; /// m.metrics_ref.handled_42 |= *i == 42; /// /// // actual work /// fibonacci((*i % 20) + 1) % 100 /// }) /// .filter(|m, i| { /// let is_even = i % 2 == 0; /// /// if !is_even { /// m.metrics_ref.num_filtered_out += 1; /// } /// /// is_even /// }) /// .num_threads(MAX_NUM_THREADS) /// .sum(); /// /// assert_eq!(sum, 9100); /// /// let total_by_metrics: usize = metrics /// .thread_metrics /// .get_mut() /// .iter() /// .map(|x| x.num_items_handled) /// .sum(); /// /// assert_eq!(N as usize, total_by_metrics); /// ``` /// fn using<'using, U, F>( self, using: F, ) -> impl ParIterUsing<'using, UsingFun, R, Item = >::Item> where U: 'using, F: Fn(usize) -> U + Sync; /// Converts the [`ParIter`] into [`ParIterUsing`] which will have access to a mutable reference of the /// used variable throughout the computation. /// /// Note that each used thread will obtain exactly one instance of the variable. /// /// Each used thread receives a clone of the provided `value`. /// Note that, `using_clone(value)` can be considered as a shorthand for `using(|_thread_idx| value.clone())`. /// /// Please see [`using`] for examples. /// /// Details of the **using transformation** can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). /// /// [`using`]: crate::ParIter::using fn using_clone( self, value: U, ) -> impl ParIterUsing<'static, UsingClone, R, Item = >::Item> where U: Clone + 'static; // transformations into fallible computations /// Transforms a parallel iterator where elements are of the result type; i.e., `ParIter>`, /// into fallible parallel iterator with item type `T` and error type `E`; i.e., into `ParIterResult`. /// /// `ParIterResult` is also a parallel iterator; however, with methods specialized for handling fallible computations /// as follows: /// /// * All of its methods are based on the success path with item type of `T`. /// * However, computations short-circuit and immediately return the observed error if any of the items /// is of the `Err` variant of the result enum. /// /// See [`ParIterResult`] for details. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// /// let result_doubled: Result, _> = ["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::()) // ParIter with Item=Result /// .into_fallible_result() // ParIterResult with Item=i32 and Err=ParseIntError /// .map(|x| x * 2) // methods focus on the success path with Item=i32 /// .collect(); // methods return Result<_, Err> /// // where the Ok variant depends on the computation /// /// assert_eq!(result_doubled, Ok(vec![2, 4, 6])); /// /// // at least one fails /// /// let result_doubled: Result, _> = ["1", "x!", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .map(|x| x * 2) /// .collect(); /// /// assert!(result_doubled.is_err()); /// ``` fn into_fallible_result(self) -> impl ParIterResult where Self::Item: IntoResult; /// Transforms a parallel iterator where elements are of the option type; i.e., `ParIter>`, /// into fallible parallel iterator with item type `T`; i.e., into `ParIterOption`. /// /// `ParIterOption` is also a parallel iterator; however, with methods specialized for handling fallible computations /// as follows: /// /// * All of its methods are based on the success path with item type of `T`. /// * However, computations short-circuit and immediately return None if any of the items /// is of the `None` variant of the option enum. /// /// See [`ParIterResult`] for details. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// /// let result_doubled: Option> = ["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) // ParIter with Item=Option /// .into_fallible_option() // ParIterOption with Item=i32 /// .map(|x| x * 2) // methods focus on the success path with Item=i32 /// .collect(); // methods return Option /// // where T depends on the computation /// /// assert_eq!(result_doubled, Some(vec![2, 4, 6])); /// /// // at least one fails /// /// let result_doubled: Option> = ["1", "x!", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .map(|x| x * 2) /// .collect(); /// /// assert_eq!(result_doubled, None); /// ``` fn into_fallible_option(self) -> impl ParIterOption where Self::Item: IntoOption, { ParOption::new( self.map(|x| x.into_result_with_unit_err()) .into_fallible_result(), ) } // computation transformations /// Takes a closure `map` and creates a parallel iterator which calls that closure on each element. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = [1, 2, 3]; /// /// let iter = a.into_par().map(|x| 2 * x); /// /// let b: Vec<_> = iter.collect(); /// assert_eq!(b, &[2, 4, 6]); /// ``` fn map(self, map: Map) -> impl ParIter where Map: Fn(Self::Item) -> Out + Sync + Clone; /// Creates an iterator which uses a closure `filter` to determine if an element should be yielded. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = [1, 2, 3]; /// /// let iter = a.into_par().filter(|x| *x % 2 == 1).copied(); /// /// let b: Vec<_> = iter.collect(); /// assert_eq!(b, &[1, 3]); /// ``` fn filter(self, filter: Filter) -> impl ParIter where Filter: Fn(&Self::Item) -> bool + Sync + Clone; /// Creates an iterator that works like map, but flattens nested structure. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let words = ["alpha", "beta", "gamma"]; /// /// // chars() returns an iterator /// let all_chars: Vec<_> = words.into_par().flat_map(|s| s.chars()).collect(); /// /// let merged: String = all_chars.iter().collect(); /// assert_eq!(merged, "alphabetagamma"); /// ``` fn flat_map(self, flat_map: FlatMap) -> impl ParIter where IOut: IntoIterator, FlatMap: Fn(Self::Item) -> IOut + Sync + Clone; /// Creates an iterator that both filters and maps. /// /// The returned iterator yields only the values for which the supplied closure `filter_map` returns `Some(value)`. /// /// `filter_map` can be used to make chains of `filter` and `map` more concise. /// The example below shows how a `map().filter().map()` can be shortened to a single call to `filter_map`. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = ["1", "two", "NaN", "four", "5"]; /// /// let numbers: Vec<_> = a /// .into_par() /// .filter_map(|s| s.parse::().ok()) /// .collect(); /// /// assert_eq!(numbers, [1, 5]); /// ``` fn filter_map(self, filter_map: FilterMap) -> impl ParIter where FilterMap: Fn(Self::Item) -> Option + Sync + Clone; /// Does something with each element of an iterator, passing the value on. /// /// When using iterators, you’ll often chain several of them together. /// While working on such code, you might want to check out what’s happening at various parts in the pipeline. /// To do that, insert a call to `inspect()`. /// /// It’s more common for `inspect()` to be used as a debugging tool than to exist in your final code, /// but applications may find it useful in certain situations when errors need to be logged before being discarded. /// /// It is often convenient to use thread-safe collections such as [`ConcurrentBag`] and /// [`ConcurrentVec`](https://crates.io/crates/orx-concurrent-vec) to /// collect some intermediate values during parallel execution for further inspection. /// The following example demonstrates such a use case. /// /// [`ConcurrentBag`]: orx_concurrent_bag::ConcurrentBag /// /// # Examples /// /// ``` /// use orx_parallel::*; /// use orx_concurrent_bag::*; /// /// let a = vec![1, 4, 2, 3]; /// /// // let's add some inspect() calls to investigate what's happening /// // - log some events /// // - use a concurrent bag to collect and investigate numbers contributing to the sum /// let bag = ConcurrentBag::new(); /// /// let sum = a /// .par() /// .copied() /// .inspect(|x| println!("about to filter: {x}")) /// .filter(|x| x % 2 == 0) /// .inspect(|x| { /// bag.push(*x); /// println!("made it through filter: {x}"); /// }) /// .sum(); /// println!("{sum}"); /// /// let mut values_made_through = bag.into_inner(); /// values_made_through.sort(); /// assert_eq!(values_made_through, [2, 4]); /// ``` /// /// This will print: /// /// ```console /// about to filter: 1 /// about to filter: 4 /// made it through filter: 4 /// about to filter: 2 /// made it through filter: 2 /// about to filter: 3 /// 6 /// ``` fn inspect(self, operation: Operation) -> impl ParIter where Operation: Fn(&Self::Item) + Sync + Clone, { let map = move |x| { operation(&x); x }; self.map(map) } // while transformations /// Creates an iterator that yields elements based on the predicate `take_while`. /// /// It will call this closure on each element of the iterator, and yield elements while it returns true. /// /// After false is returned, take_while’s job is over, and the rest of the elements are ignored. /// /// Unlike regular sequential iterators, the result is not always deterministic due to parallel execution. /// /// However, as demonstrated in the example below, `collect` with ordered execution makes sure that all /// elements before the predicate returns false are collected in the correct order; and hence, the result /// is deterministic. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let iter = (0..10_000).par().take_while(|x| *x != 5_000); /// let b: Vec<_> = iter.collect(); /// /// assert_eq!(b, (0..5_000).collect::>()); /// ``` fn take_while(self, take_while: While) -> impl ParIter where While: Fn(&Self::Item) -> bool + Sync + Clone; /// Creates an iterator that both yields elements based on the predicate `map_while` and maps. /// /// `map_while` takes a closure as an argument. /// It will call this closure on each element of the iterator, and yield elements while it returns Some(_). /// /// After None is returned, map_while job is over, and the rest of the elements are ignored. /// /// Unlike regular sequential iterators, the result is not always deterministic due to parallel execution. /// /// However, as demonstrated in the example below, `collect` with ordered execution makes sure that all /// elements before the predicate returns None are collected in the correct order; and hence, the result /// is deterministic. /// /// ``` /// use orx_parallel::*; /// /// let iter = (0..10_000) /// .par() /// .map(|x| x as i32 - 5_000) // -5_000..10_000 /// .map_while(|x| 16i32.checked_div(x)); /// let b: Vec<_> = iter.collect(); /// /// assert_eq!(b, (-5_000..0).map(|x| 16 / x).collect::>()); /// ``` fn map_while(self, map_while: MapWhile) -> impl ParIter where MapWhile: Fn(Self::Item) -> Option + Sync + Clone, { self.map(map_while).take_while(|x| x.is_some()).map(|x| { // SAFETY: since x passed the whilst(is-some) check, unwrap_unchecked unsafe { x.unwrap_unchecked() } }) } // special item transformations /// Creates an iterator which copies all of its elements. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![1, 2, 3]; /// /// let v_copied: Vec<_> = a.par().copied().collect(); /// /// // copied is the same as .map(|&x| x) /// let v_map: Vec<_> = a.par().map(|&x| x).collect(); /// /// assert_eq!(v_copied, vec![1, 2, 3]); /// assert_eq!(v_map, vec![1, 2, 3]); /// ``` fn copied<'a, T>(self) -> impl ParIter where T: 'a + Copy, Self: ParIter, { self.map(map_copy) } /// Creates an iterator which clones all of its elements. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec<_> = [1, 2, 3].map(|x| x.to_string()).into_iter().collect(); /// /// let v_cloned: Vec<_> = a.par().cloned().collect(); /// /// // cloned is the same as .map(|x| x.clone()) /// let v_map: Vec<_> = a.par().map(|x| x.clone()).collect(); /// /// assert_eq!( /// v_cloned, /// vec![String::from("1"), String::from("2"), String::from("3")] /// ); /// assert_eq!( /// v_map, /// vec![String::from("1"), String::from("2"), String::from("3")] /// ); /// ``` fn cloned<'a, T>(self) -> impl ParIter where T: 'a + Clone, Self: ParIter, { self.map(map_clone) } /// Creates an iterator that flattens nested structure. /// /// This is useful when you have an iterator of iterators or an iterator of things that can be /// turned into iterators and you want to remove one level of indirection. /// /// # Examples /// /// Basic usage. /// /// ``` /// use orx_parallel::*; /// /// let data = vec![vec![1, 2, 3, 4], vec![5, 6]]; /// let flattened = data.into_par().flatten().collect::>(); /// assert_eq!(flattened, &[1, 2, 3, 4, 5, 6]); /// ``` /// /// Mapping and then flattening: /// /// ``` /// use orx_parallel::*; /// /// let words = vec!["alpha", "beta", "gamma"]; /// /// // chars() returns an iterator /// let all_characters: Vec<_> = words.par().map(|s| s.chars()).flatten().collect(); /// let merged: String = all_characters.into_iter().collect(); /// assert_eq!(merged, "alphabetagamma"); /// ``` /// /// But actually, you can write this in terms of `flat_map`, /// which is preferable in this case since it conveys intent more clearly: /// /// ``` /// use orx_parallel::*; /// /// let words = vec!["alpha", "beta", "gamma"]; /// /// // chars() returns an iterator /// let all_characters: Vec<_> = words.par().flat_map(|s| s.chars()).collect(); /// let merged: String = all_characters.into_iter().collect(); /// assert_eq!(merged, "alphabetagamma"); /// ``` fn flatten(self) -> impl ParIter::Item> where Self::Item: IntoIterator, { let map = |e: Self::Item| e.into_iter(); self.flat_map(map) } // collect /// Collects all the items from an iterator into a collection. /// /// This is useful when you already have a collection and want to add the iterator items to it. /// /// The collection is passed in as owned value, and returned back with the additional elements. /// /// All collections implementing [`ParCollectInto`] can be used to collect into. /// /// [`ParCollectInto`]: crate::ParCollectInto /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![1, 2, 3]; /// /// let vec: Vec = vec![0, 1]; /// let vec = a.par().map(|&x| x * 2).collect_into(vec); /// let vec = a.par().map(|&x| x * 10).collect_into(vec); /// /// assert_eq!(vec, vec![0, 1, 2, 4, 6, 10, 20, 30]); /// ``` fn collect_into(self, output: C) -> C where C: ParCollectInto; /// Transforms an iterator into a collection. /// /// Similar to [`Iterator::collect`], the type annotation on the left-hand-side determines /// the type of the result collection; or turbofish annotation can be used. /// /// All collections implementing [`ParCollectInto`] can be used to collect into. /// /// [`ParCollectInto`]: crate::ParCollectInto /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![1, 2, 3]; /// /// let doubled: Vec = a.par().map(|&x| x * 2).collect(); /// /// assert_eq!(vec![2, 4, 6], doubled); /// ``` fn collect(self) -> C where C: ParCollectInto, { let output = C::empty(self.con_iter().try_get_len()); self.collect_into(output) } // reduce /// Reduces the elements to a single one, by repeatedly applying a reducing operation. /// /// If the iterator is empty, returns `None`; otherwise, returns the result of the reduction. /// /// The `reduce` function is a closure with two arguments: an ‘accumulator’, and an element. /// /// # Example /// /// ``` /// use orx_parallel::*; /// /// let inputs = 1..10; /// let reduced: usize = inputs.par().reduce(|acc, e| acc + e).unwrap_or(0); /// assert_eq!(reduced, 45); /// ``` fn reduce(self, reduce: Reduce) -> Option where Self::Item: Send, Reduce: Fn(Self::Item, Self::Item) -> Self::Item + Sync; /// Tests if every element of the iterator matches a predicate. /// /// `all` takes a `predicate` that returns true or false. /// It applies this closure to each element of the iterator, /// and if they all return true, then so does `all`. /// If any of them returns false, it returns false. /// /// `all` is short-circuiting; in other words, it will stop processing as soon as it finds a false, /// given that no matter what else happens, the result will also be false. /// /// An empty iterator returns true. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let mut a = vec![1, 2, 3]; /// assert!(a.par().all(|x| **x > 0)); /// assert!(!a.par().all(|x| **x > 2)); /// /// a.clear(); /// assert!(a.par().all(|x| **x > 2)); // empty iterator /// ``` fn all(self, predicate: Predicate) -> bool where Self::Item: Send, Predicate: Fn(&Self::Item) -> bool + Sync, { let violates = |x: &Self::Item| !predicate(x); self.find(violates).is_none() } /// Tests if any element of the iterator matches a predicate. /// /// `any` takes a `predicate` that returns true or false. /// It applies this closure to each element of the iterator, /// and if any of the elements returns true, then so does `any`. /// If all of them return false, it returns false. /// /// `any` is short-circuiting; in other words, it will stop processing as soon as it finds a true, /// given that no matter what else happens, the result will also be true. /// /// An empty iterator returns false. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let mut a = vec![1, 2, 3]; /// assert!(a.par().any(|x| **x > 0)); /// assert!(!a.par().any(|x| **x > 5)); /// /// a.clear(); /// assert!(!a.par().any(|x| **x > 0)); // empty iterator /// ``` fn any(self, predicate: Predicate) -> bool where Self::Item: Send, Predicate: Fn(&Self::Item) -> bool + Sync, { self.find(predicate).is_some() } /// Consumes the iterator, counting the number of iterations and returning it. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![1, 2, 3]; /// assert_eq!(a.par().filter(|x| **x >= 2).count(), 2); /// ``` fn count(self) -> usize { self.map(map_count).reduce(reduce_sum).unwrap_or(0) } /// Calls a closure on each element of an iterator. /// /// # Examples /// /// Basic usage: /// /// ``` /// use orx_parallel::*; /// use std::sync::mpsc::channel; /// /// let (tx, rx) = channel(); /// (0..5) /// .par() /// .map(|x| x * 2 + 1) /// .for_each(move |x| tx.send(x).unwrap()); /// /// let mut v: Vec<_> = rx.iter().collect(); /// v.sort(); // order can be mixed, since messages will be sent in parallel /// assert_eq!(v, vec![1, 3, 5, 7, 9]); /// ``` /// /// Note that since parallel iterators cannot be used within the `for` loop as regular iterators, /// `for_each` provides a way to perform arbitrary for loops on parallel iterators. /// In the following example, we log every element that satisfies a predicate in parallel. /// /// ``` /// use orx_parallel::*; /// /// (0..5) /// .par() /// .flat_map(|x| x * 100..x * 110) /// .filter(|&x| x % 3 == 0) /// .for_each(|x| println!("{x}")); /// ``` fn for_each(self, operation: Operation) where Operation: Fn(Self::Item) + Sync, { let map = |x| operation(x); let _ = self.map(map).reduce(reduce_unit); } /// Returns the maximum element of an iterator. /// /// If the iterator is empty, None is returned. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![1, 2, 3]; /// let b: Vec = Vec::new(); /// /// assert_eq!(a.par().max(), Some(&3)); /// assert_eq!(b.par().max(), None); /// ``` fn max(self) -> Option where Self::Item: Ord + Send, { self.reduce(Ord::max) } /// Returns the element that gives the maximum value with respect to the specified `compare` function. /// /// If the iterator is empty, None is returned. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![-3_i32, 0, 1, 5, -10]; /// assert_eq!(*a.par().max_by(|x, y| x.cmp(y)).unwrap(), 5); /// ``` fn max_by(self, compare: Compare) -> Option where Self::Item: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |x, y| match compare(&x, &y) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns the element that gives the maximum value from the specified function. /// /// If the iterator is empty, None is returned. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![-3_i32, 0, 1, 5, -10]; /// assert_eq!(*a.par().max_by_key(|x| x.abs()).unwrap(), -10); /// ``` fn max_by_key(self, key: GetKey) -> Option where Self::Item: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |x, y| match key(&x).cmp(&key(&y)) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns the minimum element of an iterator. /// /// If the iterator is empty, None is returned. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![1, 2, 3]; /// let b: Vec = Vec::new(); /// /// assert_eq!(a.par().min(), Some(&1)); /// assert_eq!(b.par().min(), None); /// ``` fn min(self) -> Option where Self::Item: Ord + Send, { self.reduce(Ord::min) } /// Returns the element that gives the minimum value with respect to the specified `compare` function. /// /// If the iterator is empty, None is returned. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![-3_i32, 0, 1, 5, -10]; /// assert_eq!(*a.par().min_by(|x, y| x.cmp(y)).unwrap(), -10); /// ``` fn min_by(self, compare: Compare) -> Option where Self::Item: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |x, y| match compare(&x, &y) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Returns the element that gives the minimum value from the specified function. /// /// If the iterator is empty, None is returned. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![-3_i32, 0, 1, 5, -10]; /// assert_eq!(*a.par().min_by_key(|x| x.abs()).unwrap(), 0); /// ``` fn min_by_key(self, get_key: GetKey) -> Option where Self::Item: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |x, y| match get_key(&x).cmp(&get_key(&y)) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Sums the elements of an iterator. /// /// Takes each element, adds them together, and returns the result. /// /// An empty iterator returns the additive identity (“zero”) of the type, which is 0 for integers and -0.0 for floats. /// /// `sum` can be used to sum any type implementing [`Sum`]. /// /// [`Sum`]: crate::Sum /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![1, 2, 3]; /// let sum: i32 = a.par().sum(); /// /// assert_eq!(sum, 6); /// ``` fn sum(self) -> Out where Self::Item: Sum, Out: Send, { self.map(Self::Item::map) .reduce(Self::Item::reduce) .unwrap_or(Self::Item::zero()) } // early exit /// Returns the first (or any) element of the iterator; returns None if it is empty. /// /// * first element is returned if default iteration order `IterationOrder::Ordered` is used, /// * any element is returned if `IterationOrder::Arbitrary` is set. /// /// # Examples /// /// The following example demonstrates the usage of first with default `Ordered` iteration. /// This guarantees that the first element with respect to position in the input sequence /// is returned. /// /// ``` /// use orx_parallel::*; /// /// let a: Vec = vec![]; /// assert_eq!(a.par().copied().first(), None); /// /// let a = vec![1, 2, 3]; /// assert_eq!(a.par().copied().first(), Some(1)); /// /// let a = 1..10_000; /// assert_eq!(a.par().filter(|x| x % 3421 == 0).first(), Some(3421)); /// assert_eq!(a.par().filter(|x| x % 12345 == 0).first(), None); /// /// // or equivalently, /// assert_eq!(a.par().find(|x| x % 3421 == 0), Some(3421)); /// ``` /// /// When the order is set to `Arbitrary`, `first` might return any of the elements, /// whichever is visited first depending on the parallel execution. /// /// ``` /// use orx_parallel::*; /// /// let a = 1..10_000; /// /// // might return either of 3421 or 2*3421 /// let any = a.par().iteration_order(IterationOrder::Arbitrary).filter(|x| x % 3421 == 0).first().unwrap(); /// assert!([3421, 2 * 3421].contains(&any)); /// /// // or equivalently, /// let any = a.par().iteration_order(IterationOrder::Arbitrary).find(|x| x % 3421 == 0).unwrap(); /// assert!([3421, 2 * 3421].contains(&any)); /// ``` fn first(self) -> Option where Self::Item: Send; /// Searches for an element of an iterator that satisfies a `predicate`. /// /// Depending on the set iteration order of the parallel iterator, returns /// /// * first element satisfying the `predicate` if default iteration order `IterationOrder::Ordered` is used, /// * any element satisfying the `predicate` if `IterationOrder::Arbitrary` is set. /// /// `find` takes a closure that returns true or false. /// It applies this closure to each element of the iterator, /// and returns `Some(x)` where `x` is the first element that returns true. /// If they all return false, it returns None. /// /// `find` is short-circuiting; in other words, it will stop processing as soon as the closure returns true. /// /// `par_iter.find(predicate)` can also be considered as a shorthand for `par_iter.filter(predicate).first()`. /// /// # Examples /// /// The following example demonstrates the usage of first with default `Ordered` iteration. /// This guarantees that the first element with respect to position in the input sequence /// is returned. /// /// ``` /// use orx_parallel::*; /// /// let a = 1..10_000; /// assert_eq!(a.par().find(|x| x % 12345 == 0), None); /// assert_eq!(a.par().find(|x| x % 3421 == 0), Some(3421)); /// ``` /// /// When the order is set to `Arbitrary`, `find` might return any of the elements satisfying the predicate, /// whichever is found first depending on the parallel execution. /// /// ``` /// use orx_parallel::*; /// /// let a = 1..10_000; /// /// // might return either of 3421 or 2*3421 /// let any = a.par().iteration_order(IterationOrder::Arbitrary).find(|x| x % 3421 == 0).unwrap(); /// assert!([3421, 2 * 3421].contains(&any)); /// ``` fn find(self, predicate: Predicate) -> Option where Self::Item: Send, Predicate: Fn(&Self::Item) -> bool + Sync, { self.filter(&predicate).first() } } ================================================ FILE: src/par_iter_option.rs ================================================ use crate::default_fns::{map_count, reduce_sum, reduce_unit}; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::{ ChunkSize, IterationOrder, NumThreads, ParCollectInto, ParThreadPool, RunnerWithPool, Sum, }; use core::cmp::Ordering; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with None. /// /// # Examples /// /// To demonstrate the difference of fallible iterator's behavior, consider the following simple example. /// We parse a series of strings into integers. /// We try this twice: /// * in the first one, all inputs are good, hence, we obtain Some of parsed numbers, /// * in the second one, the value in the middle is faulty, we expect the computation to fail. /// /// In the following, we try to achieve this both with a regular parallel iterator ([`ParIter`]) and a fallible /// parallel iterator, `ParIterOption` in this case. /// /// You may notice the following differences: /// * In the regular iterator, it is not very convenient to keep both the resulting numbers and a potential error. /// Here, we make use of `filter_map`. /// * On the other hand, the `collect` method of the fallible iterator directly returns an `Option` of the computation /// which is either Some of all parsed numbers or None if any computation fails. /// * Also importantly note that the regular iterator will try to parse all the strings, regardless of how many times /// the parsing fails. /// * Fallible iterator, on the other hand, stops immediately after observing the first None and short circuits the /// computation. /// /// ``` /// use orx_parallel::*; /// /// let expected_results = [Some((0..100).collect::>()), None]; /// /// for expected in expected_results { /// let expected_some = expected.is_some(); /// let mut inputs: Vec<_> = (0..100).map(|x| x.to_string()).collect(); /// if !expected_some { /// inputs.insert(50, "x".to_string()); // plant an error case /// } /// /// // regular parallel iterator /// let results = inputs.par().map(|x| x.parse::().ok()); /// let numbers: Vec<_> = results.filter_map(|x| x).collect(); /// if expected_some { /// assert_eq!(&expected, &Some(numbers)); /// } else { /// // otherwise, numbers contains some numbers, but we are not sure /// // if the computation completely succeeded or not /// } /// /// // fallible parallel iterator /// let results = inputs.par().map(|x| x.parse::().ok()); /// let result: Option> = results.into_fallible_option().collect(); /// assert_eq!(&expected, &result); /// } /// ``` /// /// These differences are not specific to `collect`; all fallible iterator methods return an option. /// The following demonstrate reduction examples, where the result is either the reduced value if the entire computation /// succeeds, or None. /// /// ``` /// use orx_parallel::*; /// /// for will_fail in [false, true] { /// let mut inputs: Vec<_> = (0..100).map(|x| x.to_string()).collect(); /// if will_fail { /// inputs.insert(50, "x".to_string()); // plant an error case /// } /// /// // sum /// let results = inputs.par().map(|x| x.parse::().ok()); /// let result: Option = results.into_fallible_option().sum(); /// match will_fail { /// true => assert_eq!(result, None), /// false => assert_eq!(result, Some(4950)), /// } /// /// // max /// let results = inputs.par().map(|x| x.parse::().ok()); /// let result: Option> = results.into_fallible_option().max(); /// match will_fail { /// true => assert_eq!(result, None), /// false => assert_eq!(result, Some(Some(99))), /// } /// } /// ``` /// /// Finally, similar to regular iterators, a fallible parallel iterator can be tranformed using iterator methods. /// However, the transformation is on the success path, the failure case of None always short circuits and returns None. /// /// ``` /// use orx_parallel::*; /// /// for will_fail in [false, true] { /// let mut inputs: Vec<_> = (0..100).map(|x| x.to_string()).collect(); /// if will_fail { /// inputs.insert(50, "x".to_string()); // plant an error case /// } /// /// // fallible iter /// let results = inputs.par().map(|x| x.parse::().ok()); /// let fallible = results.into_fallible_option(); /// /// // transformations /// /// let result: Option = fallible /// .filter(|x| x % 2 == 1) // Item: u32 /// .map(|x| 3 * x) // Item: u32 /// .filter_map(|x| (x % 10 != 0).then_some(x)) // Item: u32 /// .flat_map(|x| [x.to_string(), (10 * x).to_string()]) // Item: String /// .map(|x| x.len()) // Item: usize /// .sum(); /// /// match will_fail { /// true => assert_eq!(result, None), /// false => assert_eq!(result, Some(312)), /// } /// } /// ``` /// /// [`ParIter`]: crate::ParIter pub trait ParIterOption where R: ParallelRunner, { /// Type of the success element, to be received as the Some variant iff the entire computation succeeds. type Item; // params transformations /// Sets the number of threads to be used in the parallel execution. /// Integers can be used as the argument with the following mapping: /// /// * `0` -> `NumThreads::Auto` /// * `1` -> `NumThreads::sequential()` /// * `n > 0` -> `NumThreads::Max(n)` /// /// See [`NumThreads`] and [`crate::ParIter::num_threads`] for details. fn num_threads(self, num_threads: impl Into) -> Self; /// Sets the number of elements to be pulled from the concurrent iterator during the /// parallel execution. When integers are used as argument, the following mapping applies: /// /// * `0` -> `ChunkSize::Auto` /// * `n > 0` -> `ChunkSize::Exact(n)` /// /// Please use the default enum constructor for creating `ChunkSize::Min` variant. /// /// See [`ChunkSize`] and [`crate::ParIter::chunk_size`] for details. fn chunk_size(self, chunk_size: impl Into) -> Self; /// Sets the iteration order of the parallel computation. /// /// See [`IterationOrder`] and [`crate::ParIter::iteration_order`] for details. fn iteration_order(self, order: IterationOrder) -> Self; /// Rather than the [`DefaultRunner`], uses the parallel runner `Q` which implements [`ParallelRunner`]. /// /// See [`ParIter::with_runner`] for details. /// /// [`DefaultRunner`]: crate::DefaultRunner /// [`ParIter::with_runner`]: crate::ParIter::with_runner fn with_runner( self, orchestrator: Q, ) -> impl ParIterOption; /// Rather than [`DefaultPool`], uses the parallel runner with the given `pool` implementing /// [`ParThreadPool`]. /// /// See [`ParIter::with_pool`] for details. /// /// [`DefaultPool`]: crate::DefaultPool /// [`ParIter::with_pool`]: crate::ParIter::with_pool fn with_pool( self, pool: P, ) -> impl ParIterOption, Item = Self::Item> where Self: Sized, { let runner = RunnerWithPool::from(pool).with_executor::(); self.with_runner(runner) } // computation transformations /// Takes a closure `map` and creates a parallel iterator which calls that closure on each element. /// /// Transformation is only for the success path where all elements are of the `Some` variant. /// Any observation of a `None` case short-circuits the computation and immediately returns None. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// let a: Vec> = vec![Some(1), Some(2), Some(3)]; /// let iter = a.into_par().into_fallible_option().map(|x| 2 * x); /// /// let b: Option> = iter.collect(); /// assert_eq!(b, Some(vec![2, 4, 6])); /// /// // at least one fails /// let a = vec![Some(1), None, Some(3)]; /// let iter = a.into_par().into_fallible_option().map(|x| 2 * x); /// /// let b: Option> = iter.collect(); /// assert_eq!(b, None); /// ``` fn map(self, map: Map) -> impl ParIterOption where Self: Sized, Map: Fn(Self::Item) -> Out + Sync + Clone, Out: Send; /// Creates an iterator which uses a closure `filter` to determine if an element should be yielded. /// /// Transformation is only for the success path where all elements are of the `Some` variant. /// Any observation of a `None` case short-circuits the computation and immediately returns None. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// let a: Vec> = vec![Some(1), Some(2), Some(3)]; /// let iter = a.into_par().into_fallible_option().filter(|x| x % 2 == 1); /// /// let b = iter.sum(); /// assert_eq!(b, Some(1 + 3)); /// /// // at least one fails /// let a = vec![Some(1), None, Some(3)]; /// let iter = a.into_par().into_fallible_option().filter(|x| x % 2 == 1); /// /// let b = iter.sum(); /// assert_eq!(b, None); /// ``` fn filter(self, filter: Filter) -> impl ParIterOption where Self: Sized, Filter: Fn(&Self::Item) -> bool + Sync + Clone, Self::Item: Send; /// Creates an iterator that works like map, but flattens nested structure. /// /// Transformation is only for the success path where all elements are of the `Some` variant. /// Any observation of a `None` case short-circuits the computation and immediately returns None. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// let words: Vec> = vec![Some("alpha"), Some("beta"), Some("gamma")]; /// /// let all_chars: Option> = words /// .into_par() /// .into_fallible_option() /// .flat_map(|s| s.chars()) // chars() returns an iterator /// .collect(); /// /// let merged: Option = all_chars.map(|chars| chars.iter().collect()); /// assert_eq!(merged, Some("alphabetagamma".to_string())); /// /// // at least one fails /// let words: Vec> = vec![Some("alpha"), Some("beta"), None, Some("gamma")]; /// /// let all_chars: Option> = words /// .into_par() /// .into_fallible_option() /// .flat_map(|s| s.chars()) // chars() returns an iterator /// .collect(); /// /// let merged: Option = all_chars.map(|chars| chars.iter().collect()); /// assert_eq!(merged, None); /// ``` fn flat_map(self, flat_map: FlatMap) -> impl ParIterOption where Self: Sized, IOut: IntoIterator, IOut::Item: Send, FlatMap: Fn(Self::Item) -> IOut + Sync + Clone; /// Creates an iterator that both filters and maps. /// /// The returned iterator yields only the values for which the supplied closure `filter_map` returns `Some(value)`. /// /// `filter_map` can be used to make chains of `filter` and `map` more concise. /// The example below shows how a `map().filter().map()` can be shortened to a single call to `filter_map`. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// let a: Vec> = vec![Some("1"), Some("two"), Some("NaN"), Some("four"), Some("5")]; /// /// let numbers: Option> = a /// .into_par() /// .into_fallible_option() /// .filter_map(|s| s.parse::().ok()) /// .collect(); /// /// assert_eq!(numbers, Some(vec![1, 5])); /// /// // at least one fails /// let a: Vec> = vec![Some("1"), Some("two"), None, Some("four"), Some("5")]; /// /// let numbers: Option> = a /// .into_par() /// .into_fallible_option() /// .filter_map(|s| s.parse::().ok()) /// .collect(); /// /// assert_eq!(numbers, None); /// ``` fn filter_map(self, filter_map: FilterMap) -> impl ParIterOption where Self: Sized, FilterMap: Fn(Self::Item) -> Option + Sync + Clone, Out: Send; /// Does something with each successful element of an iterator, passing the value on, provided that all elements are of Some variant; /// short-circuits and returns None otherwise. /// /// When using iterators, you’ll often chain several of them together. /// While working on such code, you might want to check out what’s happening at various parts in the pipeline. /// To do that, insert a call to `inspect()`. /// /// It’s more common for `inspect()` to be used as a debugging tool than to exist in your final code, /// but applications may find it useful in certain situations when errors need to be logged before being discarded. /// /// It is often convenient to use thread-safe collections such as [`ConcurrentBag`] and /// [`ConcurrentVec`](https://crates.io/crates/orx-concurrent-vec) to /// collect some intermediate values during parallel execution for further inspection. /// The following example demonstrates such a use case. /// /// [`ConcurrentBag`]: orx_concurrent_bag::ConcurrentBag /// /// ``` /// use orx_parallel::*; /// use orx_concurrent_bag::*; /// use std::num::ParseIntError; /// /// // all succeeds /// let a: Vec> = ["1", "4", "2", "3"] /// .into_iter() /// .map(|x| x.parse::().ok()) /// .collect(); /// /// // let's add some inspect() calls to investigate what's happening /// // - log some events /// // - use a concurrent bag to collect and investigate numbers contributing to the sum /// let bag = ConcurrentBag::new(); /// /// let sum = a /// .par() /// .cloned() /// .into_fallible_option() /// .inspect(|x| println!("about to filter: {x}")) /// .filter(|x| x % 2 == 0) /// .inspect(|x| { /// bag.push(*x); /// println!("made it through filter: {x}"); /// }) /// .sum(); /// assert_eq!(sum, Some(4 + 2)); /// /// let mut values_made_through = bag.into_inner(); /// values_made_through.sort(); /// assert_eq!(values_made_through, [2, 4]); /// /// // at least one fails /// let a: Vec> = ["1", "4", "x", "3"] /// .into_iter() /// .map(|x| x.parse::().ok()) /// .collect(); /// /// // let's add some inspect() calls to investigate what's happening /// // - log some events /// // - use a concurrent bag to collect and investigate numbers contributing to the sum /// let bag = ConcurrentBag::new(); /// /// let sum = a /// .par() /// .cloned() /// .into_fallible_option() /// .inspect(|x| println!("about to filter: {x}")) /// .filter(|x| x % 2 == 0) /// .inspect(|x| { /// bag.push(*x); /// println!("made it through filter: {x}"); /// }) /// .sum(); /// assert_eq!(sum, None); /// ``` fn inspect(self, operation: Operation) -> impl ParIterOption where Self: Sized, Operation: Fn(&Self::Item) + Sync + Clone, Self::Item: Send; // collect /// Collects all the items from an iterator into a collection iff all elements are of Some variant. /// Early exits and returns None if any of the elements is None. /// /// This is useful when you already have a collection and want to add the iterator items to it. /// /// The collection is passed in as owned value, and returned back with the additional elements. /// /// All collections implementing [`ParCollectInto`] can be used to collect into. /// /// [`ParCollectInto`]: crate::ParCollectInto /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let vec: Vec = vec![0, 1]; /// /// // all succeeds /// let result = ["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .map(|x| x * 10) /// .collect_into(vec); /// assert_eq!(result, Some(vec![0, 1, 10, 20, 30])); /// /// let vec = result.unwrap(); /// /// // at least one fails /// /// let result = ["1", "x!", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .map(|x| x * 10) /// .collect_into(vec); /// assert_eq!(result, None); /// ``` fn collect_into(self, output: C) -> Option where Self::Item: Send, C: ParCollectInto; /// Transforms an iterator into a collection iff all elements are of Ok variant. /// Early exits and returns the error if any of the elements is an Err. /// /// Similar to [`Iterator::collect`], the type annotation on the left-hand-side determines /// the type of the result collection; or turbofish annotation can be used. /// /// All collections implementing [`ParCollectInto`] can be used to collect into. /// /// [`ParCollectInto`]: crate::ParCollectInto /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// /// let result_doubled: Option> = ["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .map(|x| x * 2) /// .collect(); /// /// assert_eq!(result_doubled, Some(vec![2, 4, 6])); /// /// // at least one fails /// /// let result_doubled: Option> = ["1", "x!", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .map(|x| x * 2) /// .collect(); /// /// assert_eq!(result_doubled, None); /// ``` fn collect(self) -> Option where Self::Item: Send, C: ParCollectInto; // reduce /// Reduces the elements to a single one, by repeatedly applying a reducing operation. /// Early exits and returns None if any of the elements is None. /// /// If the iterator is empty, returns `Some(None)`; otherwise, returns `Some` of result of the reduction. /// /// The `reduce` function is a closure with two arguments: an ‘accumulator’, and an element. /// /// # Example /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// let reduced: Option> = (1..10) /// .par() /// .map(|x| 100u32.checked_div(x as u32)) /// .into_fallible_option() /// .reduce(|acc, e| acc + e); /// assert_eq!(reduced, Some(Some(281))); /// /// // all succeeds - empty iterator /// let reduced: Option> = (1..1) /// .par() /// .map(|x| 100u32.checked_div(x as u32)) /// .into_fallible_option() /// .reduce(|acc, e| acc + e); /// assert_eq!(reduced, Some(None)); /// /// // at least one fails /// let reduced: Option> = (0..10) /// .par() /// .map(|x| 100u32.checked_div(x as u32)) /// .into_fallible_option() /// .reduce(|acc, e| acc + e); /// assert_eq!(reduced, None); /// ``` fn reduce(self, reduce: Reduce) -> Option> where Self::Item: Send, Reduce: Fn(Self::Item, Self::Item) -> Self::Item + Sync; /// Tests if every element of the iterator matches a predicate. /// Early exits and returns None if any of the elements is None. /// /// `all` takes a `predicate` that returns true or false. /// It applies this closure to each Ok element of the iterator, /// and if they all return true, then so does `all`. /// If any of them returns false, it returns false. /// /// Note that `all` computation is itself also short-circuiting; in other words, it will stop processing as soon as it finds a false, /// given that no matter what else happens, the result will also be false. /// /// Therefore, in case the fallible iterator contains both a None element and a Some element which violates the `predicate`, /// the result is **not deterministic**. It might be the `None` if it is observed first; or `Some(false)` if the violation is observed first. /// /// On the other hand, when it returns `Some(true)`, we are certain that all elements are of `Some` variant and all satisfy the `predicate`. /// /// An empty iterator returns `Some(true)`. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all Some /// let result = vec!["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .all(|x| *x > 0); /// assert_eq!(result, Some(true)); /// /// let result = vec!["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .all(|x| *x > 1); /// assert_eq!(result, Some(false)); /// /// let result = Vec::<&str>::new() /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .all(|x| *x > 1); /// assert_eq!(result, Some(true)); // empty iterator /// /// // at least one None /// let result = vec!["1", "x!", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .all(|x| *x > 0); /// assert_eq!(result, None); /// ``` fn all(self, predicate: Predicate) -> Option where Self: Sized, Self::Item: Send, Predicate: Fn(&Self::Item) -> bool + Sync, { let violates = |x: &Self::Item| !predicate(x); self.find(violates).map(|x| x.is_none()) } /// Tests if any element of the iterator matches a predicate. /// Early exits and returns None if any of the elements is None. /// /// `any` takes a `predicate` that returns true or false. /// It applies this closure to each element of the iterator, /// and if any of the elements returns true, then so does `any`. /// If all of them return false, it returns false. /// /// Note that `any` computation is itself also short-circuiting; in other words, it will stop processing as soon as it finds a true, /// given that no matter what else happens, the result will also be true. /// /// Therefore, in case the fallible iterator contains both a None element and a Some element which satisfies the `predicate`, /// the result is **not deterministic**. It might be the `None` if it is observed first; or `Some(true)` if element satisfying the predicate /// is observed first. /// /// On the other hand, when it returns `Some(false)`, we are certain that all elements are of `Some` variant and none of them satisfies the `predicate`. /// /// An empty iterator returns `Some(false)`. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all Some /// let result = vec!["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .any(|x| *x > 1); /// assert_eq!(result, Some(true)); /// /// let result = vec!["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .any(|x| *x > 3); /// assert_eq!(result, Some(false)); /// /// let result = Vec::<&str>::new() /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .any(|x| *x > 1); /// assert_eq!(result, Some(false)); // empty iterator /// /// // at least one None /// let result = vec!["1", "x!", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .any(|x| *x > 5); /// assert_eq!(result, None); /// ``` fn any(self, predicate: Predicate) -> Option where Self: Sized, Self::Item: Send, Predicate: Fn(&Self::Item) -> bool + Sync, { self.find(predicate).map(|x| x.is_some()) } /// Consumes the iterator, counting the number of iterations and returning it. /// Early exits and returns None if any of the elements is None. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all Some /// let result = vec!["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .filter(|x| *x >= 2) /// .count(); /// assert_eq!(result, Some(2)); /// /// // at least one None /// let result = vec!["x!", "2", "3"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .filter(|x| *x >= 2) /// .count(); /// assert_eq!(result, None); /// ``` fn count(self) -> Option where Self: Sized, { self.map(map_count) .reduce(reduce_sum) .map(|x| x.unwrap_or(0)) } /// Calls a closure on each element of an iterator, and returns `Ok(())` if all elements succeed. /// Early exits and returns None if any of the elements is None. /// /// # Examples /// /// Basic usage: /// /// ``` /// use orx_parallel::*; /// use std::sync::mpsc::channel; /// /// // all Some /// let (tx, rx) = channel(); /// let result = vec!["0", "1", "2", "3", "4"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .map(|x| x * 2 + 1) /// .for_each(move |x| tx.send(x).unwrap()); /// /// assert_eq!(result, Some(())); /// /// let mut v: Vec<_> = rx.iter().collect(); /// v.sort(); // order can be mixed, since messages will be sent in parallel /// assert_eq!(v, vec![1, 3, 5, 7, 9]); /// /// // at least one None /// let (tx, _rx) = channel(); /// let result = vec!["0", "1", "2", "x!", "4"] /// .into_par() /// .map(|x| x.parse::().ok()) /// .into_fallible_option() /// .map(|x| x * 2 + 1) /// .for_each(move |x| tx.send(x).unwrap()); /// /// assert_eq!(result, None); /// ``` fn for_each(self, operation: Operation) -> Option<()> where Self: Sized, Operation: Fn(Self::Item) + Sync, { let map = |x| operation(x); self.map(map).reduce(reduce_unit).map(|_| ()) } /// Returns Some of maximum element of an iterator if all elements succeed. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![Some(1), Some(2), Some(3)]; /// assert_eq!(a.par().copied().into_fallible_option().max(), Some(Some(3))); /// /// let b: Vec> = vec![]; /// assert_eq!(b.par().copied().into_fallible_option().max(), Some(None)); /// /// let c = vec![Some(1), Some(2), None]; /// assert_eq!(c.par().copied().into_fallible_option().max(), None); /// ``` fn max(self) -> Option> where Self: Sized, Self::Item: Ord + Send, { self.reduce(Ord::max) } /// Returns the element that gives the maximum value with respect to the specified `compare` function. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![Some(1), Some(2), Some(3)]; /// assert_eq!( /// a.par() /// .copied() /// .into_fallible_option() /// .max_by(|a, b| a.cmp(b)), /// Some(Some(3)) /// ); /// /// let b: Vec> = vec![]; /// assert_eq!( /// b.par() /// .copied() /// .into_fallible_option() /// .max_by(|a, b| a.cmp(b)), /// Some(None) /// ); /// /// let c: Vec> = vec![Some(1), Some(2), None]; /// assert_eq!( /// c.par() /// .copied() /// .into_fallible_option() /// .max_by(|a, b| a.cmp(b)), /// None /// ); /// ``` fn max_by(self, compare: Compare) -> Option> where Self: Sized, Self::Item: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |x, y| match compare(&x, &y) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns the element that gives the maximum value from the specified function. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![Some(-1), Some(2), Some(-3)]; /// assert_eq!( /// a.par() /// .copied() /// .into_fallible_option() /// .max_by_key(|x| x.abs()), /// Some(Some(-3)) /// ); /// /// let b: Vec> = vec![]; /// assert_eq!( /// b.par() /// .copied() /// .into_fallible_option() /// .max_by_key(|x| x.abs()), /// Some(None) /// ); /// /// let c: Vec> = vec![Some(1), Some(2), None]; /// assert_eq!( /// c.par() /// .copied() /// .into_fallible_option() /// .max_by_key(|x| x.abs()), /// None /// ); /// ``` fn max_by_key(self, key: GetKey) -> Option> where Self: Sized, Self::Item: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |x, y| match key(&x).cmp(&key(&y)) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns Some of minimum element of an iterator if all elements succeed. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a = vec![Some(1), Some(2), Some(3)]; /// assert_eq!(a.par().copied().into_fallible_option().min(), Some(Some(1))); /// /// let b: Vec> = vec![]; /// assert_eq!(b.par().copied().into_fallible_option().min(), Some(None)); /// /// let c = vec![Some(1), Some(2), None]; /// assert_eq!(c.par().copied().into_fallible_option().min(), None); /// ``` fn min(self) -> Option> where Self: Sized, Self::Item: Ord + Send, { self.reduce(Ord::min) } /// Returns the element that gives the minimum value with respect to the specified `compare` function. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![Some(1), Some(2), Some(3)]; /// assert_eq!( /// a.par() /// .copied() /// .into_fallible_option() /// .min_by(|a, b| a.cmp(b)), /// Some(Some(1)) /// ); /// /// let b: Vec> = vec![]; /// assert_eq!( /// b.par() /// .copied() /// .into_fallible_option() /// .min_by(|a, b| a.cmp(b)), /// Some(None) /// ); /// /// let c: Vec> = vec![Some(1), Some(2), None]; /// assert_eq!( /// c.par() /// .copied() /// .into_fallible_option() /// .min_by(|a, b| a.cmp(b)), /// None /// ); /// ``` fn min_by(self, compare: Compare) -> Option> where Self: Sized, Self::Item: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |x, y| match compare(&x, &y) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Returns the element that gives the minimum value from the specified function. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![Some(-1), Some(2), Some(-3)]; /// assert_eq!( /// a.par() /// .copied() /// .into_fallible_option() /// .min_by_key(|x| x.abs()), /// Some(Some(-1)) /// ); /// /// let b: Vec> = vec![]; /// assert_eq!( /// b.par() /// .copied() /// .into_fallible_option() /// .min_by_key(|x| x.abs()), /// Some(None) /// ); /// /// let c: Vec> = vec![Some(1), Some(2), None]; /// assert_eq!( /// c.par() /// .copied() /// .into_fallible_option() /// .min_by_key(|x| x.abs()), /// None /// ); /// ``` fn min_by_key(self, get_key: GetKey) -> Option> where Self: Sized, Self::Item: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |x, y| match get_key(&x).cmp(&get_key(&y)) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Sums the elements of an iterator. /// Early exits and returns None if any of the elements is None. /// /// If the iterator is empty, returns zero; otherwise, returns `Some` of the sum. /// /// # Example /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// let reduced: Option = (1..10) /// .par() /// .map(|x| 100u32.checked_div(x as u32)) /// .into_fallible_option() /// .sum(); /// assert_eq!(reduced, Some(281)); /// /// // all succeeds - empty iterator /// let reduced: Option = (1..1) /// .par() /// .map(|x| 100u32.checked_div(x as u32)) /// .into_fallible_option() /// .sum(); /// assert_eq!(reduced, Some(0)); /// /// // at least one fails /// let reduced: Option = (0..10) /// .par() /// .map(|x| 100u32.checked_div(x as u32)) /// .into_fallible_option() /// .sum(); /// assert_eq!(reduced, None); /// ``` fn sum(self) -> Option where Self: Sized, Self::Item: Sum, Out: Send, { self.map(Self::Item::map) .reduce(Self::Item::reduce) .map(|x| x.unwrap_or(Self::Item::zero())) } // early exit /// Returns the first (or any) element of the iterator. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if a None element is observed first. /// /// * first element is returned if default iteration order `IterationOrder::Ordered` is used, /// * any element is returned if `IterationOrder::Arbitrary` is set. /// /// Note that `find` itself is short-circuiting in addition to fallible computation. /// Therefore, in case the fallible iterator contains both a None and a Some element, /// the result is **not deterministic**: /// * it might be the `None` if it is observed first; /// * or `Some(element)` if the Some element is observed first. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![]; /// assert_eq!(a.par().copied().into_fallible_option().first(), Some(None)); /// /// let a: Vec> = vec![Some(1), Some(2), Some(3)]; /// assert_eq!( /// a.par().copied().into_fallible_option().first(), /// Some(Some(1)) /// ); /// /// let a: Vec> = vec![Some(1), None, Some(3)]; /// let result = a.par().copied().into_fallible_option().first(); /// // depends on whichever is observed first in parallel execution /// assert!(result == Some(Some(1)) || result == None); /// ``` fn first(self) -> Option> where Self::Item: Send; /// Returns the first (or any) element of the iterator that satisfies the `predicate`. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if a None element is observed first. /// /// * first element is returned if default iteration order `IterationOrder::Ordered` is used, /// * any element is returned if `IterationOrder::Arbitrary` is set. /// /// Note that `find` itself is short-circuiting in addition to fallible computation. /// Therefore, in case the fallible iterator contains both a None and a Some element, /// the result is **not deterministic**: /// * it might be the `None` if it is observed first; /// * or `Some(element)` if the Some element satisfying the predicate is observed first. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![]; /// assert_eq!( /// a.par().copied().into_fallible_option().find(|x| *x > 2), /// Some(None) /// ); /// /// let a: Vec> = vec![Some(1), Some(2), Some(3)]; /// assert_eq!( /// a.par().copied().into_fallible_option().find(|x| *x > 2), /// Some(Some(3)) /// ); /// /// let a: Vec> = vec![Some(1), None, Some(3)]; /// let result = a.par().copied().into_fallible_option().find(|x| *x > 2); /// // depends on whichever is observed first in parallel execution /// assert!(result == Some(Some(3)) || result == None); /// ``` fn find(self, predicate: Predicate) -> Option> where Self: Sized, Self::Item: Send, Predicate: Fn(&Self::Item) -> bool + Sync, { self.filter(&predicate).first() } } pub trait IntoOption { fn into_option(self) -> Option; fn into_result_with_unit_err(self) -> Result; } impl IntoOption for Option { #[inline(always)] fn into_option(self) -> Option { self } #[inline(always)] fn into_result_with_unit_err(self) -> Result { match self { Some(x) => Ok(x), None => Err(()), } } } pub(crate) trait ResultIntoOption { fn into_option(self) -> Option; } impl ResultIntoOption for Result { #[inline(always)] fn into_option(self) -> Option { self.ok() } } ================================================ FILE: src/par_iter_result.rs ================================================ use crate::default_fns::{map_count, reduce_sum, reduce_unit}; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::{ChunkSize, IterationOrder, NumThreads, ParThreadPool, RunnerWithPool, Sum}; use crate::{ParCollectInto, ParIter, generic_values::fallible_iterators::ResultOfIter}; use core::cmp::Ordering; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with an error. /// /// # Examples /// /// To demonstrate the difference of fallible iterator's behavior, consider the following simple example. /// We parse a series of strings into integers. /// We try this twice: /// * in the first one, all inputs are good, hence, we obtain Ok of parsed numbers, /// * in the second one, the value in the middle is faulty, we expect the computation to fail. /// /// In the following, we try to achieve this both with a regular parallel iterator ([`ParIter`]) and a fallible /// parallel iterator, `ParIterResult` in this case. /// /// You may notice the following differences: /// * In the regular iterator, it is not very convenient to keep both the resulting numbers and a potential error. /// Here, we make use of `filter_map` and simply ignore the error. /// * On the other hand, the `collect` method of the fallible iterator directly returns a `Result` of the computation /// which is either Ok of all parsed numbers or the error. /// * Also importantly note that the regular iterator will try to parse all the strings, regardless of how many times /// the parsing fails. /// * Fallible iterator, on the other hand, stops immediately after observing the first error and short circuits the /// computation. /// /// ``` /// use orx_parallel::*; /// use std::num::{IntErrorKind, ParseIntError}; /// /// let expected_results = [ /// Ok((0..100).collect::>()), /// Err(IntErrorKind::InvalidDigit), /// ]; /// /// for expected in expected_results { /// let expected_ok = expected.is_ok(); /// let mut inputs: Vec<_> = (0..100).map(|x| x.to_string()).collect(); /// if !expected_ok { /// inputs.insert(50, "x".to_string()); // plant an error case /// } /// /// // regular parallel iterator /// let results = inputs.par().map(|x| x.parse::()); /// let numbers: Vec<_> = results.filter_map(|x| x.ok()).collect(); /// if expected_ok { /// assert_eq!(&expected, &Ok(numbers)); /// } else { /// // we lost the error /// } /// /// // fallible parallel iterator /// let results = inputs.par().map(|x| x.parse::()); /// let result: Result, ParseIntError> = results.into_fallible_result().collect(); /// assert_eq!(&expected, &result.map_err(|x| x.kind().clone())); /// } /// ``` /// /// These differences are not specific to `collect`; all fallible iterator methods return a result. /// The following demonstrate reduction examples, where the result is either the reduced value if the entire computation /// succeeds, or the error. /// /// ``` /// use orx_parallel::*; /// use std::num::ParseIntError; /// /// for will_fail in [false, true] { /// let mut inputs: Vec<_> = (0..100).map(|x| x.to_string()).collect(); /// if will_fail { /// inputs.insert(50, "x".to_string()); // plant an error case /// } /// /// // sum /// let results = inputs.par().map(|x| x.parse::()); /// let result: Result = results.into_fallible_result().sum(); /// match will_fail { /// true => assert!(result.is_err()), /// false => assert_eq!(result, Ok(4950)), /// } /// /// // max /// let results = inputs.par().map(|x| x.parse::()); /// let result: Result, ParseIntError> = results.into_fallible_result().max(); /// match will_fail { /// true => assert!(result.is_err()), /// false => assert_eq!(result, Ok(Some(99))), /// } /// } /// ``` /// /// Finally, similar to regular iterators, a fallible parallel iterator can be tranformed using iterator methods. /// However, the transformation is on the success path, the error case always short circuits and returns the error. /// Notice in the following example that the success type keeps changing through transformations while the error type /// remains the same. /// /// ``` /// use orx_parallel::*; /// use std::num::ParseIntError; /// /// for will_fail in [false, true] { /// let mut inputs: Vec<_> = (0..100).map(|x| x.to_string()).collect(); /// if will_fail { /// inputs.insert(50, "x".to_string()); // plant an error case /// } /// /// // fallible iter /// let results = inputs.par().map(|x| x.parse::()); /// let fallible = results.into_fallible_result(); // Ok: u32, Error: ParseIntError /// /// // transformations /// /// let result: Result = fallible /// .filter(|x| x % 2 == 1) // Ok: u32, Error: ParseIntError /// .map(|x| 3 * x) // Ok: u32, Error: ParseIntError /// .filter_map(|x| (x % 10 != 0).then_some(x)) // Ok: u32, Error: ParseIntError /// .flat_map(|x| [x.to_string(), (10 * x).to_string()]) // Ok: String, Error: ParseIntError /// .map(|x| x.len()) // Ok: usize, Error: ParseIntError /// .sum(); /// /// match will_fail { /// true => assert!(result.is_err()), /// false => assert_eq!(result, Ok(312)), /// } /// } /// ``` /// /// [`ParIter`]: crate::ParIter pub trait ParIterResult where R: ParallelRunner, { /// Type of the Ok element, to be received as the Ok variant iff the entire computation succeeds. type Item; /// Type of the Err element, to be received if any of the computations fails. type Err; /// Element type of the regular parallel iterator this fallible iterator can be converted to, simply `Result`. type RegularItem: IntoResult; /// Regular parallel iterator this fallible iterator can be converted into. type RegularParIter: ParIter; /// Returns a reference to the input concurrent iterator. fn con_iter_len(&self) -> Option; /// Converts this fallible iterator into a regular parallel iterator; i.e., [`ParIter`], with `Item = Result`. fn into_regular_par(self) -> Self::RegularParIter; /// Converts the `regular_par` iterator with `Item = Result` into fallible result iterator. fn from_regular_par(regular_par: Self::RegularParIter) -> Self; // params transformations /// Sets the number of threads to be used in the parallel execution. /// Integers can be used as the argument with the following mapping: /// /// * `0` -> `NumThreads::Auto` /// * `1` -> `NumThreads::sequential()` /// * `n > 0` -> `NumThreads::Max(n)` /// /// See [`NumThreads`] and [`ParIter::num_threads`] for details. fn num_threads(self, num_threads: impl Into) -> Self where Self: Sized, { Self::from_regular_par(self.into_regular_par().num_threads(num_threads)) } /// Sets the number of elements to be pulled from the concurrent iterator during the /// parallel execution. When integers are used as argument, the following mapping applies: /// /// * `0` -> `ChunkSize::Auto` /// * `n > 0` -> `ChunkSize::Exact(n)` /// /// Please use the default enum constructor for creating `ChunkSize::Min` variant. /// /// See [`ChunkSize`] and [`ParIter::chunk_size`] for details. fn chunk_size(self, chunk_size: impl Into) -> Self where Self: Sized, { Self::from_regular_par(self.into_regular_par().chunk_size(chunk_size)) } /// Sets the iteration order of the parallel computation. /// /// See [`IterationOrder`] and [`ParIter::iteration_order`] for details. fn iteration_order(self, order: IterationOrder) -> Self where Self: Sized, { Self::from_regular_par(self.into_regular_par().iteration_order(order)) } /// Rather than the [`DefaultRunner`], uses the parallel runner `Q` which implements [`ParallelRunner`]. /// /// See [`ParIter::with_runner`] for details. /// /// [`DefaultRunner`]: crate::DefaultRunner /// [`ParIter::with_runner`]: crate::ParIter::with_runner fn with_runner( self, orchestrator: Q, ) -> impl ParIterResult; /// Rather than [`DefaultPool`], uses the parallel runner with the given `pool` implementing /// [`ParThreadPool`]. /// /// See [`ParIter::with_pool`] for details. /// /// [`DefaultPool`]: crate::DefaultPool /// [`ParIter::with_pool`]: crate::ParIter::with_pool fn with_pool( self, pool: P, ) -> impl ParIterResult, Item = Self::Item, Err = Self::Err> where Self: Sized, { let runner = RunnerWithPool::from(pool).with_executor::(); self.with_runner(runner) } // computation transformations /// Takes a closure `map` and creates a parallel iterator which calls that closure on each element. /// /// Transformation is only for the success path where all elements are of the `Ok` variant. /// Any observation of an `Err` case short-circuits the computation and immediately returns the observed error. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// let a: Vec> = vec![Ok(1), Ok(2), Ok(3)]; /// let iter = a.into_par().into_fallible_result().map(|x| 2 * x); /// /// let b: Result, _> = iter.collect(); /// assert_eq!(b, Ok(vec![2, 4, 6])); /// /// // at least one fails /// let a = vec![Ok(1), Err('x'), Ok(3)]; /// let iter = a.into_par().into_fallible_result().map(|x| 2 * x); /// /// let b: Result, _> = iter.collect(); /// assert_eq!(b, Err('x')); /// ``` fn map(self, map: Map) -> impl ParIterResult where Self: Sized, Map: Fn(Self::Item) -> Out + Sync + Clone, Out: Send, { let par = self.into_regular_par(); let map = par.map(move |x| x.into_result().map(map.clone())); map.into_fallible_result() } /// Creates an iterator which uses a closure `filter` to determine if an element should be yielded. /// /// Transformation is only for the success path where all elements are of the `Ok` variant. /// Any observation of an `Err` case short-circuits the computation and immediately returns the observed error. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// let a: Vec> = vec![Ok(1), Ok(2), Ok(3)]; /// let iter = a.into_par().into_fallible_result().filter(|x| x % 2 == 1); /// /// let b = iter.sum(); /// assert_eq!(b, Ok(1 + 3)); /// /// // at least one fails /// let a = vec![Ok(1), Err('x'), Ok(3)]; /// let iter = a.into_par().into_fallible_result().filter(|x| x % 2 == 1); /// /// let b = iter.sum(); /// assert_eq!(b, Err('x')); /// ``` fn filter( self, filter: Filter, ) -> impl ParIterResult where Self: Sized, Filter: Fn(&Self::Item) -> bool + Sync + Clone, Self::Item: Send, { let par = self.into_regular_par(); let filter_map = par.filter_map(move |x| match x.into_result() { Ok(x) => match filter(&x) { true => Some(Ok(x)), false => None, }, Err(e) => Some(Err(e)), }); filter_map.into_fallible_result() } /// Creates an iterator that works like map, but flattens nested structure. /// /// Transformation is only for the success path where all elements are of the `Ok` variant. /// Any observation of an `Err` case short-circuits the computation and immediately returns the observed error. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// let words: Vec> = vec![Ok("alpha"), Ok("beta"), Ok("gamma")]; /// /// let all_chars: Result, _> = words /// .into_par() /// .into_fallible_result() /// .flat_map(|s| s.chars()) // chars() returns an iterator /// .collect(); /// /// let merged: Result = all_chars.map(|chars| chars.iter().collect()); /// assert_eq!(merged, Ok("alphabetagamma".to_string())); /// /// // at least one fails /// let words: Vec> = vec![Ok("alpha"), Ok("beta"), Err('x'), Ok("gamma")]; /// /// let all_chars: Result, _> = words /// .into_par() /// .into_fallible_result() /// .flat_map(|s| s.chars()) // chars() returns an iterator /// .collect(); /// /// let merged: Result = all_chars.map(|chars| chars.iter().collect()); /// assert_eq!(merged, Err('x')); /// ``` fn flat_map( self, flat_map: FlatMap, ) -> impl ParIterResult where Self: Sized, IOut: IntoIterator, IOut::Item: Send, FlatMap: Fn(Self::Item) -> IOut + Sync + Clone, { let par = self.into_regular_par(); let map = par.flat_map(move |x| match x.into_result() { Ok(x) => ResultOfIter::ok(flat_map(x).into_iter()), Err(e) => ResultOfIter::err(e), }); map.into_fallible_result() } /// Creates an iterator that both filters and maps. /// /// The returned iterator yields only the values for which the supplied closure `filter_map` returns `Some(value)`. /// /// `filter_map` can be used to make chains of `filter` and `map` more concise. /// The example below shows how a `map().filter().map()` can be shortened to a single call to `filter_map`. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// let a: Vec> = vec![Ok("1"), Ok("two"), Ok("NaN"), Ok("four"), Ok("5")]; /// /// let numbers: Result, char> = a /// .into_par() /// .into_fallible_result() /// .filter_map(|s| s.parse::().ok()) /// .collect(); /// /// assert_eq!(numbers, Ok(vec![1, 5])); /// /// // at least one fails /// let a: Vec> = vec![Ok("1"), Ok("two"), Err('x'), Ok("four"), Ok("5")]; /// /// let numbers: Result, char> = a /// .into_par() /// .into_fallible_result() /// .filter_map(|s| s.parse::().ok()) /// .collect(); /// /// assert_eq!(numbers, Err('x')); /// ``` fn filter_map( self, filter_map: FilterMap, ) -> impl ParIterResult where Self: Sized, FilterMap: Fn(Self::Item) -> Option + Sync + Clone, Out: Send, { let par = self.into_regular_par(); let filter_map = par.filter_map(move |x| match x.into_result() { Ok(x) => filter_map(x).map(|x| Ok(x)), Err(e) => Some(Err(e)), }); filter_map.into_fallible_result() } /// Does something with each successful element of an iterator, passing the value on, provided that all elements are of Ok variant; /// short-circuits and returns the error otherwise. /// /// When using iterators, you’ll often chain several of them together. /// While working on such code, you might want to check out what’s happening at various parts in the pipeline. /// To do that, insert a call to `inspect()`. /// /// It’s more common for `inspect()` to be used as a debugging tool than to exist in your final code, /// but applications may find it useful in certain situations when errors need to be logged before being discarded. /// /// It is often convenient to use thread-safe collections such as [`ConcurrentBag`] and /// [`ConcurrentVec`](https://crates.io/crates/orx-concurrent-vec) to /// collect some intermediate values during parallel execution for further inspection. /// The following example demonstrates such a use case. /// /// [`ConcurrentBag`]: orx_concurrent_bag::ConcurrentBag /// /// ``` /// use orx_parallel::*; /// use orx_concurrent_bag::*; /// use std::num::ParseIntError; /// /// // all succeeds /// let a: Vec> = ["1", "4", "2", "3"] /// .into_iter() /// .map(|x| x.parse::()) /// .collect(); /// /// // let's add some inspect() calls to investigate what's happening /// // - log some events /// // - use a concurrent bag to collect and investigate numbers contributing to the sum /// let bag = ConcurrentBag::new(); /// /// let sum = a /// .par() /// .cloned() /// .into_fallible_result() /// .inspect(|x| println!("about to filter: {x}")) /// .filter(|x| x % 2 == 0) /// .inspect(|x| { /// bag.push(*x); /// println!("made it through filter: {x}"); /// }) /// .sum(); /// assert_eq!(sum, Ok(4 + 2)); /// /// let mut values_made_through = bag.into_inner(); /// values_made_through.sort(); /// assert_eq!(values_made_through, [2, 4]); /// /// // at least one fails /// let a: Vec> = ["1", "4", "x", "3"] /// .into_iter() /// .map(|x| x.parse::()) /// .collect(); /// /// // let's add some inspect() calls to investigate what's happening /// // - log some events /// // - use a concurrent bag to collect and investigate numbers contributing to the sum /// let bag = ConcurrentBag::new(); /// /// let sum = a /// .par() /// .cloned() /// .into_fallible_result() /// .inspect(|x| println!("about to filter: {x}")) /// .filter(|x| x % 2 == 0) /// .inspect(|x| { /// bag.push(*x); /// println!("made it through filter: {x}"); /// }) /// .sum(); /// assert!(sum.is_err()); /// ``` fn inspect( self, operation: Operation, ) -> impl ParIterResult where Self: Sized, Operation: Fn(&Self::Item) + Sync + Clone, Self::Item: Send, { let map = move |x| { operation(&x); x }; self.map(map) } // collect /// Collects all the items from an iterator into a collection iff all elements are of Ok variant. /// Early exits and returns the error if any of the elements is an Err. /// /// This is useful when you already have a collection and want to add the iterator items to it. /// /// The collection is passed in as owned value, and returned back with the additional elements. /// /// All collections implementing [`ParCollectInto`] can be used to collect into. /// /// [`ParCollectInto`]: crate::ParCollectInto /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let vec: Vec = vec![0, 1]; /// /// // all succeeds /// let result = ["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .map(|x| x * 10) /// .collect_into(vec); /// /// assert!(result.is_ok()); /// let vec = result.unwrap(); /// assert_eq!(vec, vec![0, 1, 10, 20, 30]); /// /// // at least one fails /// /// let result = ["1", "x!", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .map(|x| x * 10) /// .collect_into(vec); /// assert!(result.is_err()); /// ``` fn collect_into(self, output: C) -> Result where C: ParCollectInto, Self::Item: Send, Self::Err: Send; /// Transforms an iterator into a collection iff all elements are of Ok variant. /// Early exits and returns the error if any of the elements is an Err. /// /// Similar to [`Iterator::collect`], the type annotation on the left-hand-side determines /// the type of the result collection; or turbofish annotation can be used. /// /// All collections implementing [`ParCollectInto`] can be used to collect into. /// /// [`ParCollectInto`]: crate::ParCollectInto /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all succeeds /// /// let result_doubled: Result, _> = ["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .map(|x| x * 2) /// .collect(); /// /// assert_eq!(result_doubled, Ok(vec![2, 4, 6])); /// /// // at least one fails /// /// let result_doubled: Result, _> = ["1", "x!", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .map(|x| x * 2) /// .collect(); /// /// assert!(result_doubled.is_err()); /// ``` fn collect(self) -> Result where Self: Sized, Self::Item: Send, Self::Err: Send, C: ParCollectInto, { let output = C::empty(self.con_iter_len()); self.collect_into(output) } // reduce /// Reduces the elements to a single one, by repeatedly applying a reducing operation. /// Early exits and returns the error if any of the elements is an Err. /// /// If the iterator is empty, returns `Ok(None)`; otherwise, returns `Ok` of result of the reduction. /// /// The `reduce` function is a closure with two arguments: an ‘accumulator’, and an element. /// /// # Example /// /// ``` /// use orx_parallel::*; /// /// fn safe_div(a: u32, b: u32) -> Result { /// match b { /// 0 => Err('!'), /// b => Ok(a / b), /// } /// } /// /// // all succeeds /// let reduced: Result, char> = (1..10) /// .par() /// .map(|x| safe_div(100, x as u32)) /// .into_fallible_result() /// .reduce(|acc, e| acc + e); /// assert_eq!(reduced, Ok(Some(281))); /// /// // all succeeds - empty iterator /// let reduced: Result, char> = (1..1) /// .par() /// .map(|x| safe_div(100, x as u32)) /// .into_fallible_result() /// .reduce(|acc, e| acc + e); /// assert_eq!(reduced, Ok(None)); /// /// // at least one fails /// let reduced: Result, char> = (0..10) /// .par() /// .map(|x| safe_div(100, x as u32)) /// .into_fallible_result() /// .reduce(|acc, e| acc + e); /// assert_eq!(reduced, Err('!')); /// ``` fn reduce(self, reduce: Reduce) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, Reduce: Fn(Self::Item, Self::Item) -> Self::Item + Sync; /// Tests if every element of the iterator matches a predicate. /// Early exits and returns the error if any of the elements is an Err. /// /// `all` takes a `predicate` that returns true or false. /// It applies this closure to each Ok element of the iterator, /// and if they all return true, then so does `all`. /// If any of them returns false, it returns false. /// /// Note that `all` computation is itself also short-circuiting; in other words, it will stop processing as soon as it finds a false, /// given that no matter what else happens, the result will also be false. /// /// Therefore, in case the fallible iterator contains both an Err element and an Ok element which violates the `predicate`, /// the result is **not deterministic**. It might be the `Err` if it is observed first; or `Ok(false)` if the violation is observed first. /// /// On the other hand, when it returns `Ok(true)`, we are certain that all elements are of `Ok` variant and all satisfy the `predicate`. /// /// An empty iterator returns `Ok(true)`. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all Ok /// let result = vec!["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .all(|x| *x > 0); /// assert_eq!(result, Ok(true)); /// /// let result = vec!["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .all(|x| *x > 1); /// assert_eq!(result, Ok(false)); /// /// let result = Vec::<&str>::new() /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .all(|x| *x > 1); /// assert_eq!(result, Ok(true)); // empty iterator /// /// // at least one Err /// let result = vec!["1", "x!", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .all(|x| *x > 0); /// assert!(result.is_err()); /// ``` fn all(self, predicate: Predicate) -> Result where Self: Sized, Self::Item: Send, Self::Err: Send, Predicate: Fn(&Self::Item) -> bool + Sync, { let violates = |x: &Self::Item| !predicate(x); self.find(violates).map(|x| x.is_none()) } /// Tests if any element of the iterator matches a predicate. /// Early exits and returns the error if any of the elements is an Err. /// /// `any` takes a `predicate` that returns true or false. /// It applies this closure to each element of the iterator, /// and if any of the elements returns true, then so does `any`. /// If all of them return false, it returns false. /// /// Note that `any` computation is itself also short-circuiting; in other words, it will stop processing as soon as it finds a true, /// given that no matter what else happens, the result will also be true. /// /// Therefore, in case the fallible iterator contains both an Err element and an Ok element which satisfies the `predicate`, /// the result is **not deterministic**. It might be the `Err` if it is observed first; or `Ok(true)` if element satisfying the predicate /// is observed first. /// /// On the other hand, when it returns `Ok(false)`, we are certain that all elements are of `Ok` variant and none of them satisfies the `predicate`. /// /// An empty iterator returns `Ok(false)`. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all Ok /// let result = vec!["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .any(|x| *x > 1); /// assert_eq!(result, Ok(true)); /// /// let result = vec!["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .any(|x| *x > 3); /// assert_eq!(result, Ok(false)); /// /// let result = Vec::<&str>::new() /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .any(|x| *x > 1); /// assert_eq!(result, Ok(false)); // empty iterator /// /// // at least one Err /// let result = vec!["1", "x!", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .any(|x| *x > 5); /// assert!(result.is_err()); /// ``` fn any(self, predicate: Predicate) -> Result where Self: Sized, Self::Item: Send, Self::Err: Send, Predicate: Fn(&Self::Item) -> bool + Sync, { self.find(predicate).map(|x| x.is_some()) } /// Consumes the iterator, counting the number of iterations and returning it. /// Early exits and returns the error if any of the elements is an Err. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // all Ok /// let result = vec!["1", "2", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .filter(|x| *x >= 2) /// .count(); /// assert_eq!(result, Ok(2)); /// /// // at least one Err /// let result = vec!["x!", "2", "3"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .filter(|x| *x >= 2) /// .count(); /// assert!(result.is_err()); /// ``` fn count(self) -> Result where Self: Sized, Self::Err: Send, { self.map(map_count) .reduce(reduce_sum) .map(|x| x.unwrap_or(0)) } /// Calls a closure on each element of an iterator, and returns `Ok(())` if all elements succeed. /// Early exits and returns the error if any of the elements is an Err. /// /// # Examples /// /// Basic usage: /// /// ``` /// use orx_parallel::*; /// use std::sync::mpsc::channel; /// /// // all Ok /// let (tx, rx) = channel(); /// let result = vec!["0", "1", "2", "3", "4"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .map(|x| x * 2 + 1) /// .for_each(move |x| tx.send(x).unwrap()); /// /// assert_eq!(result, Ok(())); /// /// let mut v: Vec<_> = rx.iter().collect(); /// v.sort(); // order can be mixed, since messages will be sent in parallel /// assert_eq!(v, vec![1, 3, 5, 7, 9]); /// /// // at least one Err /// let (tx, _rx) = channel(); /// let result = vec!["0", "1", "2", "x!", "4"] /// .into_par() /// .map(|x| x.parse::()) /// .into_fallible_result() /// .map(|x| x * 2 + 1) /// .for_each(move |x| tx.send(x).unwrap()); /// /// assert!(result.is_err()); /// ``` fn for_each(self, operation: Operation) -> Result<(), Self::Err> where Self: Sized, Self::Err: Send, Operation: Fn(Self::Item) + Sync, { let map = |x| operation(x); self.map(map).reduce(reduce_unit).map(|_| ()) } /// Returns Ok of maximum element of an iterator if all elements succeed. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![Ok(1), Ok(2), Ok(3)]; /// assert_eq!(a.par().copied().into_fallible_result().max(), Ok(Some(3))); /// /// let b: Vec> = vec![]; /// assert_eq!(b.par().copied().into_fallible_result().max(), Ok(None)); /// /// let c: Vec> = vec![Ok(1), Ok(2), Err('x')]; /// assert_eq!(c.par().copied().into_fallible_result().max(), Err('x')); /// ``` fn max(self) -> Result, Self::Err> where Self: Sized, Self::Err: Send, Self::Item: Ord + Send, { self.reduce(Ord::max) } /// Returns the element that gives the maximum value with respect to the specified `compare` function. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![Ok(1), Ok(2), Ok(3)]; /// assert_eq!( /// a.par() /// .copied() /// .into_fallible_result() /// .max_by(|a, b| a.cmp(b)), /// Ok(Some(3)) /// ); /// /// let b: Vec> = vec![]; /// assert_eq!( /// b.par() /// .copied() /// .into_fallible_result() /// .max_by(|a, b| a.cmp(b)), /// Ok(None) /// ); /// /// let c: Vec> = vec![Ok(1), Ok(2), Err('x')]; /// assert_eq!( /// c.par() /// .copied() /// .into_fallible_result() /// .max_by(|a, b| a.cmp(b)), /// Err('x') /// ); /// ``` fn max_by(self, compare: Compare) -> Result, Self::Err> where Self: Sized, Self::Item: Send, Self::Err: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |x, y| match compare(&x, &y) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns the element that gives the maximum value from the specified function. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![Ok(-1), Ok(2), Ok(-3)]; /// assert_eq!( /// a.par() /// .copied() /// .into_fallible_result() /// .max_by_key(|x| x.abs()), /// Ok(Some(-3)) /// ); /// /// let b: Vec> = vec![]; /// assert_eq!( /// b.par() /// .copied() /// .into_fallible_result() /// .max_by_key(|x| x.abs()), /// Ok(None) /// ); /// /// let c: Vec> = vec![Ok(1), Ok(2), Err('x')]; /// assert_eq!( /// c.par() /// .copied() /// .into_fallible_result() /// .max_by_key(|x| x.abs()), /// Err('x') /// ); /// ``` fn max_by_key(self, key: GetKey) -> Result, Self::Err> where Self: Sized, Self::Item: Send, Self::Err: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |x, y| match key(&x).cmp(&key(&y)) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns Ok of minimum element of an iterator if all elements succeed. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![Ok(1), Ok(2), Ok(3)]; /// assert_eq!(a.par().copied().into_fallible_result().min(), Ok(Some(1))); /// /// let b: Vec> = vec![]; /// assert_eq!(b.par().copied().into_fallible_result().min(), Ok(None)); /// /// let c: Vec> = vec![Ok(1), Ok(2), Err('x')]; /// assert_eq!(c.par().copied().into_fallible_result().min(), Err('x')); /// ``` fn min(self) -> Result, Self::Err> where Self: Sized, Self::Item: Ord + Send, Self::Err: Send, { self.reduce(Ord::min) } /// Returns the element that gives the maximum value with respect to the specified `compare` function. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![Ok(1), Ok(2), Ok(3)]; /// assert_eq!( /// a.par() /// .copied() /// .into_fallible_result() /// .min_by(|a, b| a.cmp(b)), /// Ok(Some(1)) /// ); /// /// let b: Vec> = vec![]; /// assert_eq!( /// b.par() /// .copied() /// .into_fallible_result() /// .min_by(|a, b| a.cmp(b)), /// Ok(None) /// ); /// /// let c: Vec> = vec![Ok(1), Ok(2), Err('x')]; /// assert_eq!( /// c.par() /// .copied() /// .into_fallible_result() /// .min_by(|a, b| a.cmp(b)), /// Err('x') /// ); /// ``` fn min_by(self, compare: Compare) -> Result, Self::Err> where Self: Sized, Self::Item: Send, Self::Err: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |x, y| match compare(&x, &y) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Returns the element that gives the minimum value from the specified function. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![Ok(-1), Ok(2), Ok(-3)]; /// assert_eq!( /// a.par() /// .copied() /// .into_fallible_result() /// .min_by_key(|x| x.abs()), /// Ok(Some(-1)) /// ); /// /// let b: Vec> = vec![]; /// assert_eq!( /// b.par() /// .copied() /// .into_fallible_result() /// .min_by_key(|x| x.abs()), /// Ok(None) /// ); /// /// let c: Vec> = vec![Ok(1), Ok(2), Err('x')]; /// assert_eq!( /// c.par() /// .copied() /// .into_fallible_result() /// .min_by_key(|x| x.abs()), /// Err('x') /// ); /// ``` fn min_by_key(self, get_key: GetKey) -> Result, Self::Err> where Self: Sized, Self::Item: Send, Self::Err: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |x, y| match get_key(&x).cmp(&get_key(&y)) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Sums the elements of an iterator. /// Early exits and returns the error if any of the elements is an Err. /// /// If the iterator is empty, returns zero; otherwise, returns `Ok` of the sum. /// /// # Example /// /// ``` /// use orx_parallel::*; /// /// fn safe_div(a: u32, b: u32) -> Result { /// match b { /// 0 => Err('!'), /// b => Ok(a / b), /// } /// } /// /// // all succeeds /// let reduced: Result = (1..10) /// .par() /// .map(|x| safe_div(100, x as u32)) /// .into_fallible_result() /// .sum(); /// assert_eq!(reduced, Ok(281)); /// /// // all succeeds - empty iterator /// let reduced: Result = (1..1) /// .par() /// .map(|x| safe_div(100, x as u32)) /// .into_fallible_result() /// .sum(); /// assert_eq!(reduced, Ok(0)); /// /// // at least one fails /// let reduced: Result = (0..10) /// .par() /// .map(|x| safe_div(100, x as u32)) /// .into_fallible_result() /// .sum(); /// assert_eq!(reduced, Err('!')); /// ``` fn sum(self) -> Result where Self: Sized, Self::Item: Sum, Self::Err: Send, Out: Send, { self.map(Self::Item::map) .reduce(Self::Item::reduce) .map(|x| x.unwrap_or(Self::Item::zero())) } // early exit /// Returns the first (or any) element of the iterator. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if an Err element is observed first. /// /// * first element is returned if default iteration order `IterationOrder::Ordered` is used, /// * any element is returned if `IterationOrder::Arbitrary` is set. /// /// Note that `find` itself is short-circuiting in addition to fallible computation. /// Therefore, in case the fallible iterator contains both an Err and an Ok element, /// the result is **not deterministic**: /// * it might be the `Err` if it is observed first; /// * or `Ok(element)` if the Ok element is observed first. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![]; /// assert_eq!(a.par().copied().into_fallible_result().first(), Ok(None)); /// /// let a: Vec> = vec![Ok(1), Ok(2), Ok(3)]; /// assert_eq!(a.par().copied().into_fallible_result().first(), Ok(Some(1))); /// /// let a: Vec> = vec![Ok(1), Err('x'), Ok(3)]; /// let result = a.par().copied().into_fallible_result().first(); /// // depends on whichever is observed first in parallel execution /// assert!(result == Ok(Some(1)) || result == Err('x')); /// ``` fn first(self) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send; /// Returns the first (or any) element of the iterator that satisfies the `predicate`. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if an Err element is observed first. /// /// * first element is returned if default iteration order `IterationOrder::Ordered` is used, /// * any element is returned if `IterationOrder::Arbitrary` is set. /// /// Note that `find` itself is short-circuiting in addition to fallible computation. /// Therefore, in case the fallible iterator contains both an Err and an Ok element, /// the result is **not deterministic**: /// * it might be the `Err` if it is observed first; /// * or `Ok(element)` if the Ok element satisfying the predicate is observed first. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let a: Vec> = vec![]; /// assert_eq!( /// a.par().copied().into_fallible_result().find(|x| *x > 2), /// Ok(None) /// ); /// /// let a: Vec> = vec![Ok(1), Ok(2), Ok(3)]; /// assert_eq!( /// a.par().copied().into_fallible_result().find(|x| *x > 2), /// Ok(Some(3)) /// ); /// /// let a: Vec> = vec![Ok(1), Err('x'), Ok(3)]; /// let result = a.par().copied().into_fallible_result().find(|x| *x > 2); /// // depends on whichever is observed first in parallel execution /// assert!(result == Ok(Some(3)) || result == Err('x')); /// ``` fn find(self, predicate: Predicate) -> Result, Self::Err> where Self: Sized, Self::Item: Send, Self::Err: Send, Predicate: Fn(&Self::Item) -> bool + Sync, { self.filter(&predicate).first() } } pub trait IntoResult { fn into_result(self) -> Result; } impl IntoResult for Result { #[inline(always)] fn into_result(self) -> Result { self } } ================================================ FILE: src/par_thread_pool.rs ================================================ use crate::{generic_values::runner_results::Fallibility, runner::NumSpawned}; use alloc::vec::Vec; use core::num::NonZeroUsize; use orx_concurrent_bag::ConcurrentBag; /// A thread pool that can be used for parallel computation. /// /// orx_parallel abstracts away the thread pool implementation and can work with different /// thread pool implementations. /// /// Parallel computation will not use any threads outside the pool. /// Default std thread pool assumes all OS threads are available in the pool. /// /// # Examples /// /// ## Default std pool /// /// **requires std feature** /// /// Default parallel runner spawns scoped threads using `std::thread::scope`. /// /// ``` /// use orx_parallel::*; /// /// let sum = (0..1000).par().sum(); /// assert_eq!(sum, 1000 * 999 / 2); /// /// // this is equivalent to /// let sum = (0..1000).par().with_runner(DefaultRunner::default()).sum(); /// assert_eq!(sum, 1000 * 999 / 2); /// ``` /// /// ## Rayon thread pool /// /// **requires rayon feature** /// /// The following example demonstrate using a rayon thread pool as the thread provider of /// the parallel computation. /// /// ``` /// use orx_parallel::*; /// /// #[cfg(not(miri))] /// #[cfg(feature = "rayon-core")] /// { /// let pool = rayon::ThreadPoolBuilder::new() /// .num_threads(4) /// .build() /// .unwrap(); /// /// // creating a runner for the computation /// let runner = RunnerWithPool::from(&pool); /// let sum = (0..1000).par().with_runner(runner).sum(); /// assert_eq!(sum, 1000 * 999 / 2); /// /// // or reuse a runner multiple times (identical under the hood) /// let mut runner = RunnerWithPool::from(&pool); /// let sum = (0..1000).par().with_runner(&mut runner).sum(); /// assert_eq!(sum, 1000 * 999 / 2); /// } /// ``` /// /// Note that since rayon::ThreadPool::scope only requires a shared reference `&self`, /// we can concurrently create as many runners as we want from the same thread pool and use them concurrently. /// /// ## Scoped thread pool /// /// **requires scoped_threadpool feature** /// /// The following example demonstrate using a scoped_threadpool thread pool as the thread provider of /// the parallel computation. /// /// ``` /// use orx_parallel::*; /// /// #[cfg(not(miri))] /// #[cfg(feature = "scoped_threadpool")] /// { /// // creating a runner for the computation /// let mut pool = scoped_threadpool::Pool::new(4); /// let runner = RunnerWithPool::from(&mut pool); /// let sum = (0..1000).par().with_runner(runner).sum(); /// assert_eq!(sum, 1000 * 999 / 2); /// /// // or reuse a runner multiple times (identical under the hood) /// let mut pool = scoped_threadpool::Pool::new(4); /// let mut runner = RunnerWithPool::from(&mut pool); /// let sum = (0..1000).par().with_runner(&mut runner).sum(); /// assert_eq!(sum, 1000 * 999 / 2); /// } /// ``` /// /// Since scoped_thread_pool::Pool::scoped requires an exclusive reference `&mut self`, /// we can create one runner from a pool at a time, note use of `&mut pool` in runner creation. pub trait ParThreadPool { /// Scope type of the thread pool. type ScopeRef<'s, 'env, 'scope> where 'scope: 's, 'env: 'scope + 's; /// Executes the `work` within scope `s`. fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env; /// Executes the scoped computation `f`. fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(Self::ScopeRef<'s, 'env, 'scope>) + Send; /// Returns the maximum number of threads available in the pool. fn max_num_threads(&self) -> NonZeroUsize; } // derived pub trait ParThreadPoolCompute: ParThreadPool { fn map_in_pool( &mut self, do_spawn: S, thread_map: M, max_num_threads: NonZeroUsize, ) -> (NumSpawned, Result, F::Error>) where F: Fallibility, S: Fn(NumSpawned) -> bool + Sync, M: Fn(NumSpawned) -> Result + Sync, T: Send, F::Error: Send, { let thread_map = &thread_map; let mut nt = NumSpawned::zero(); let thread_results = ConcurrentBag::with_fixed_capacity(max_num_threads.into()); let bag = &thread_results; self.scoped_computation(|s| { while do_spawn(nt) { let num_spawned = nt; nt.increment(); let work = move || { bag.push(thread_map(num_spawned)); }; Self::run_in_scope(&s, work); } }); let thread_results: Vec<_> = thread_results.into_inner().into(); let result = F::reduce_results(thread_results); (nt, result) } fn run_in_pool(&mut self, do_spawn: S, thread_do: F) -> NumSpawned where S: Fn(NumSpawned) -> bool + Sync, F: Fn(NumSpawned) + Sync, { let thread_do = &thread_do; let mut nt = NumSpawned::zero(); self.scoped_computation(|s| { while do_spawn(nt) { let num_spawned = nt; nt.increment(); let work = move || thread_do(num_spawned); Self::run_in_scope(&s, work); } }); nt } } impl ParThreadPoolCompute for X {} ================================================ FILE: src/parallel_drainable.rs ================================================ use crate::{Params, computational_variants::Par, runner::DefaultRunner}; use core::ops::RangeBounds; use orx_concurrent_iter::ConcurrentDrainableOverSlice; /// A type which can create a parallel draining iterator over any of its sub-slices. /// /// * Created draining iterator takes out and returns all elements of the slice defined by the `range`. /// * The slice defined by this `range` will be removed from the original collection. /// /// If the iterator is dropped before being fully consumed, it drops the remaining removed elements. /// /// If the complete range is provided (`..` or `0..self.len()`), self will remain empty. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let mut v = vec![1, 2, 3]; /// let u: Vec<_> = v.par_drain(1..).num_threads(2).collect(); /// /// assert_eq!(v, &[1]); /// assert_eq!(u, &[2, 3]); /// ``` pub trait ParallelDrainableOverSlice: ConcurrentDrainableOverSlice { /// Creates a parallel draining iterator over the slice defined by the given `range`. /// /// * Created draining iterator takes out and returns all elements of the slice defined by the `range`. /// * The slice defined by this `range` will be removed from the original collection. /// /// If the iterator is dropped before being fully consumed, it drops the remaining removed elements. /// /// If the complete range is provided (`..` or `0..self.len()`), self will remain empty. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// let mut v = vec![1, 2, 3]; /// let u: Vec<_> = v.par_drain(1..).num_threads(2).collect(); /// /// assert_eq!(v, &[1]); /// assert_eq!(u, &[2, 3]); /// ``` fn par_drain( &mut self, range: R, ) -> Par<::DrainingIter<'_>, DefaultRunner> where R: RangeBounds, { Par::new(Default::default(), Params::default(), self.con_drain(range)) } } impl ParallelDrainableOverSlice for I where I: ConcurrentDrainableOverSlice {} ================================================ FILE: src/parallelizable.rs ================================================ use crate::{computational_variants::Par, parameters::Params, runner::DefaultRunner}; use orx_concurrent_iter::ConcurrentIterable; /// `Parallelizable` types are those from which parallel iterators can be created /// **multiple times** using the [`par`] method, since this method call does not consume the source. /// /// This trait can be considered as the *parallel counterpart* of the [`Iterable`] trait. /// /// Note that every [`ConcurrentIterable`] type automatically implements [`Parallelizable`]. /// /// [`par`]: crate::Parallelizable::par /// [`Iterable`]: orx_iterable::Iterable /// [`ConcurrentIterable`]: orx_concurrent_iter::ConcurrentIterable /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // Vec: Parallelizable /// let vec = vec![1, 2, 3, 4]; /// assert_eq!(vec.par().sum(), 10); /// assert_eq!(vec.par().max(), Some(&4)); /// /// // &[T]: Parallelizable /// let slice = vec.as_slice(); /// assert_eq!(slice.par().sum(), 10); /// assert_eq!(slice.par().max(), Some(&4)); /// /// // Range: Parallelizable /// let range = 1..5; /// assert_eq!(range.par().sum(), 10); /// assert_eq!(range.par().max(), Some(4)); /// ``` pub trait Parallelizable: ConcurrentIterable { /// `Parallelizable` types are those from which parallel iterators can be created /// **multiple times** using the [`par`] method, since this method call does not consume the source. /// /// This trait can be considered as the *parallel counterpart* of the [`Iterable`] trait. /// /// [`par`]: crate::Parallelizable::par /// [`Iterable`]: orx_iterable::Iterable /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // Vec: Parallelizable /// let vec = vec![1, 2, 3, 4]; /// assert_eq!(vec.par().sum(), 10); /// assert_eq!(vec.par().max(), Some(&4)); /// /// // &[T]: Parallelizable /// let slice = vec.as_slice(); /// assert_eq!(slice.par().sum(), 10); /// assert_eq!(slice.par().max(), Some(&4)); /// /// // Range: Parallelizable /// let range = 1..5; /// assert_eq!(range.par().sum(), 10); /// assert_eq!(range.par().max(), Some(4)); /// ``` fn par(&self) -> Par<::Iter, DefaultRunner> { Par::new(Default::default(), Params::default(), self.con_iter()) } } impl Parallelizable for I where I: ConcurrentIterable {} ================================================ FILE: src/parallelizable_collection.rs ================================================ use crate::{Params, computational_variants::Par, runner::DefaultRunner}; use orx_concurrent_iter::{ConcurrentCollection, ConcurrentIterable}; /// A type implementing [`ParallelizableCollection`] is a collection owning the elements such that /// /// * if the elements are of type `T`, /// * non-consuming [`par`] method can be called **multiple times** to create parallel /// iterators; i.e., [`ParIter`], yielding references to the elements `&T`; and /// * consuming [`into_par`] method can be called once to create a parallel iterator yielding /// owned elements `T`. /// /// This trait can be considered as the *concurrent counterpart* of the [`Collection`] trait. /// /// Note that every [`ConcurrentCollection`] type automatically implements [`ParallelizableCollection`]. /// /// [`Collection`]: orx_iterable::Collection /// [`ConcurrentCollection`]: orx_concurrent_iter::ConcurrentCollection /// [`par`]: crate::ParallelizableCollection::par /// [`into_par`]: crate::IntoParIter::into_par /// [`ParIter`]: crate::ParIter /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // Vec: ParallelizableCollection /// let vec = vec![1, 2, 3, 4]; /// /// // non-consuming iteration over references /// assert_eq!(vec.par().sum(), 10); /// assert_eq!(vec.par().min(), Some(&1)); /// assert_eq!(vec.par().max(), Some(&4)); /// /// // consuming iteration over owned values /// assert_eq!(vec.into_par().max(), Some(4)); /// ``` pub trait ParallelizableCollection: ConcurrentCollection { /// A type implementing [`ParallelizableCollection`] is a collection owning the elements such that /// /// * if the elements are of type `T`, /// * non-consuming [`par`] method can be called **multiple times** to create parallel /// iterators; i.e., [`ParIter`], yielding references to the elements `&T`; and /// * consuming [`into_par`] method can be called once to create a parallel iterator yielding /// owned elements `T`. /// /// This trait can be considered as the *concurrent counterpart* of the [`Collection`] trait. /// /// Note that every [`ConcurrentCollection`] type automatically implements [`ParallelizableCollection`]. /// /// [`con_iter`]: orx_concurrent_iter::ConcurrentCollection::con_iter /// [`Collection`]: orx_iterable::Collection /// [`ConcurrentIter`]: orx_concurrent_iter::ConcurrentIter /// [`par`]: crate::ParallelizableCollection::par /// [`into_par`]: crate::IntoParIter::into_par /// [`ParIter`]: crate::ParIter /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // Vec: ParallelizableCollection /// let vec = vec![1, 2, 3, 4]; /// /// // non-consuming iteration over references /// assert_eq!(vec.par().sum(), 10); /// assert_eq!(vec.par().min(), Some(&1)); /// assert_eq!(vec.par().max(), Some(&4)); /// /// // consuming iteration over owned values /// assert_eq!(vec.into_par().max(), Some(4)); /// ``` fn par( &self, ) -> Par< <::Iterable<'_> as ConcurrentIterable>::Iter, DefaultRunner, > { Par::new(Default::default(), Params::default(), self.con_iter()) } } impl ParallelizableCollection for X where X: ConcurrentCollection {} ================================================ FILE: src/parallelizable_collection_mut.rs ================================================ use crate::{ ParIter, ParallelizableCollection, Params, computational_variants::Par, runner::DefaultRunner, }; use orx_concurrent_iter::ConcurrentCollectionMut; /// A type implementing [`ParallelizableCollectionMut`] is a collection owning the elements such that /// /// * if the elements are of type `T`, /// * non-consuming [`par`] method can be called **multiple times** to create parallel /// iterators; i.e., [`ParIter`], yielding references to the elements `&T`; and /// * non-consuming [`par_mut`] method can be called **multiple times** to create parallel /// iterators yielding mutable references to the elements `&mut T`; and /// * consuming [`into_par`] method can be called once to create a parallel iterator yielding /// owned elements `T`. /// /// This trait can be considered as the *concurrent counterpart* of the [`CollectionMut`] trait. /// /// Note that every [`ConcurrentCollectionMut`] type automatically implements [`ParallelizableCollectionMut`]. /// /// [`CollectionMut`]: orx_iterable::CollectionMut /// [`ConcurrentCollectionMut`]: orx_concurrent_iter::ConcurrentCollectionMut /// [`par`]: crate::ParallelizableCollection::par /// [`par_mut`]: crate::ParallelizableCollectionMut::par_mut /// [`into_par`]: crate::IntoParIter::into_par /// [`ParIter`]: crate::ParIter /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // Vec: ParallelizableCollectionMut /// let mut vec = vec![1, 2, 3, 4]; /// /// // non-consuming iteration over references /// assert_eq!(vec.par().sum(), 10); /// assert_eq!(vec.par().min(), Some(&1)); /// assert_eq!(vec.par().max(), Some(&4)); /// /// // non-consuming mutable iteration /// vec.par_mut().filter(|x| **x >= 3).for_each(|x| *x += 10); /// assert_eq!(&vec, &[1, 2, 13, 14]); /// /// // consuming iteration over owned values /// assert_eq!(vec.into_par().max(), Some(14)); /// ``` pub trait ParallelizableCollectionMut: ConcurrentCollectionMut + ParallelizableCollection { /// Creates a parallel iterator over mutable references of the collection's elements. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // Vec: ParallelizableCollectionMut /// let mut vec = vec![1, 2, 3, 4]; /// /// vec.par_mut().filter(|x| **x >= 3).for_each(|x| *x += 10); /// /// assert_eq!(&vec, &[1, 2, 13, 14]); /// ``` fn par_mut(&mut self) -> impl ParIter { Par::new(Default::default(), Params::default(), self.con_iter_mut()) } } impl ParallelizableCollectionMut for X where X: ConcurrentCollectionMut + ParallelizableCollection {} ================================================ FILE: src/parameters/chunk_size.rs ================================================ use core::num::NonZeroUsize; /// `ChunkSize` represents the batch size of elements each thread will pull from the main iterator once it becomes idle again. /// It is possible to define a minimum or exact chunk size. /// /// # Rules of Thumb / Guidelines /// /// The objective of this parameter is to balance the overhead of parallelization and cost of heterogeneity of tasks. /// /// In order to illustrate, assume that there exist 8 elements to process, or 8 jobs to execute, and we will use 2 threads for this computation. /// Two extreme strategies can be defined as follows. /// /// * **Perfect Sharing of Tasks** /// * Setting chunk size to 4 provides a perfect division of tasks in terms of quantity. /// Each thread will retrieve 4 elements at once in one pull and process them. /// This *one pull* per thread can be considered as the parallelization overhead and this is the best/minimum we can achieve. /// * Drawback of this approach, on the other hand, is observed when the execution time of each job is significantly different; i.e., when we have heterogeneous tasks. /// * Assume, for instance, that the first element requires 7 units of time while all remaining elements require 1 unit of time. /// * Roughly, the parallel execution with a chunk size of 4 would complete in 10 units of time, which is the execution time of the first thread (7 + 3*1). /// * The second thread will complete its 4 tasks in 4 units of time and will remain idle for 6 units of time. /// * **Perfect Handling of Heterogeneity** /// * Setting chunk size to 1 provides a perfect way to deal with heterogeneous tasks, minimizing the idle time of threads. /// Each thread will retrieve elements one by one whenever they become idle. /// * Considering the heterogeneous example above, the parallel execution with a chunk size of 1 would complete around 7 units of time. /// * This is again the execution time of the first thread, which will only execute the first element. /// * The second thread will execute the remaining 7 elements, again in 7 units in time. /// * None of the threads will be idle, which is the best we can achieve. /// * Drawback of this approach is the parallelization overhead due to *pull*s. /// * Chunk size being 1, this setting will lead to a total of 8 pull operations (1 pull by the first thread, 7 pulls by the second thread). /// * This leads to the maximum/worst parallelization overhead in this scenario. /// /// The objective then is to find a chunk size which is: /// * large enough that total time spent for the pulls is insignificant, while /// * small enough not to suffer from the impact of heterogeneity. /// /// Note that this decision is data dependent, and hence, can be tuned for the input when the operation is extremely time-critical. /// /// In these cases, the following rule of thumb helps to find a good chunk size. /// We can set the chunk size to the smallest value which would make the overhead of pulls insignificant: /// * The larger each individual task, the less significant the parallelization overhead. A small chunk size would do. /// * The smaller each individual task, the more significant the parallelization overhead. We require a larger chunk size while being careful not to suffer from idle times of threads due to heterogeneity. /// /// In general, it is recommended to set this parameter to its default value, `ChunkSize::Auto`. /// This library will try to solve the tradeoff explained above depending on the input data to minimize execution time and idle thread time. /// /// For more critical operations, this `ChunkSize::Exact` and `ChunkSize::Min` options can be used to tune the execution for the class of the relevant input data. #[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] pub enum ChunkSize { /// The objective of `ChunkSize` parameter is to balance the overhead of parallelization and cost of heterogeneity of tasks. /// /// When `ChunkSize::Auto` is used, this library will try to solve the tradeoff explained above depending on the input data to minimize execution time and idle thread time. #[default] Auto, /// This variant defines a minimum chunk size, say `min_chunk_size`. /// Each time a thread completes a task and becomes idle, it will pull at least `min_chunk_size` elements from the input source. /// Parallel execution is allowed to and might decide to pull more elements depending on characteristics of the inputs and used number of threads. Min(NonZeroUsize), /// This variant defines an exact chunk size, say `exact_chunk_size`. /// Each time a thread completes a task and becomes idle, it will pull exactly `exact_chunk_size` elements from the input source. Exact(NonZeroUsize), } impl From for ChunkSize { /// Converts the nonnegative integer to chunk size as follows: /// /// * 0 is converted to `ChunkSize::Auto`, /// * `n` is converted to `ChunkSize::Exact(n)` where `n > 0`. fn from(value: usize) -> Self { match value { 0 => Self::Auto, n => Self::Exact(NonZeroUsize::new(n).expect("is positive")), } } } ================================================ FILE: src/parameters/iteration_order.rs ================================================ /// Order of parallel iteration, which might be: /// * in the order of the input as in regular sequential iterators, or /// * arbitrary. /// /// This is important for certain computations: /// /// * `collect` will return exactly the same result of its sequential counterpart /// when `Ordered` is used. However, the elements might (but not necessarily) /// be in arbitrary order when `Arbitrary` is used. /// * `first` returns the first element of the iterator when `Ordered`, might /// return any element when `Arbitrary`. /// * `find` returns the first element satisfying the predicate when `Ordered`, /// might return any element satisfying the predicate when `Arbitrary` /// (sometimes this method is called `find_any`). /// /// [`collect`]: crate::ParIter::collect /// [`first`]: crate::ParIter::first /// [`find`]: crate::ParIter::find #[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] pub enum IterationOrder { /// The iteration is allowed to be in arbitrary order when it might improve performance, /// but not necessarily. Arbitrary, /// ***Default ordering***. /// /// The iteration will be in an order consistent with the input of the collection, /// and hence, the outputs will always be equivalent to the sequential counterpart. #[default] Ordered, } ================================================ FILE: src/parameters/mod.rs ================================================ mod chunk_size; mod iteration_order; mod num_threads; mod params; pub use chunk_size::ChunkSize; pub use iteration_order::IterationOrder; pub use num_threads::NumThreads; pub use params::Params; ================================================ FILE: src/parameters/num_threads.rs ================================================ use core::num::NonZeroUsize; /// `NumThreads` represents the degree of parallelization. It is possible to define an upper bound on the number of threads to be used for the parallel computation. /// When set to **1**, the computation will be executed sequentially without any overhead. /// In this sense, parallel iterators defined in this crate are a union of sequential and parallel execution. /// /// # Rules of Thumb / Guidelines /// /// It is recommended to set this parameter to its default value, `NumThreads::Auto`. /// This setting assumes that it can use all available threads; however, the computation will spawn new threads only when required. /// In other words, when we can dynamically decide that the task is not large enough to justify spawning a new thread, the parallel execution will avoid it. /// /// A special case is `NumThreads::Max(NonZeroUsize::new(1).unwrap())`, or equivalently `NumThreads::sequential()`. /// This will lead to a sequential execution of the defined computation on the main thread. /// Both in terms of used resources and computation time, this mode is not similar but **identical** to a sequential execution using the regular sequential `Iterator`s. /// /// Lastly, `NumThreads::Max(t)` where `t >= 2` can be used in the following scenarios: /// * We have a strict limit on the resources that we can use for this computation, even if the hardware has more resources. /// Parallel execution will ensure that `t` will never be exceeded. /// * We have a computation which is extremely time-critical and our benchmarks show that `t` outperforms the `NumThreads::Auto` on the corresponding system. #[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] pub enum NumThreads { /// This setting assumes that it can use all available threads; however, the computation will spawn new threads only when required. /// In other words, when we can dynamically decide that the task is not large enough to justify spawning a new thread, the parallel execution will avoid it. #[default] Auto, /// Limits the maximum number of threads that can be used by the parallel execution. /// /// A special case is `NumThreads::Max(NonZeroUsize::new(1).unwrap())`, or equivalently `NumThreads::sequential()`. /// This will lead to a sequential execution of the defined computation on the main thread. /// Both in terms of used resources and computation time, this mode is not similar but **identical** to a sequential execution using the regular sequential `Iterator`s. /// /// Lastly, `NumThreads::Max(t)` where `t >= 2` can be used in the following scenarios: /// * We have a strict limit on the resources that we can use for this computation, even if the hardware has more resources. /// Parallel execution will ensure that `t` will never be exceeded. /// * We have a computation which is extremely time-critical and our benchmarks show that `t` outperforms the `NumThreads::Auto` on the corresponding system. Max(NonZeroUsize), } const ONE: NonZeroUsize = NonZeroUsize::new(1).expect("seq=1 is positive"); impl From for NumThreads { /// Converts the nonnegative integer to number of threads as follows: /// /// * 0 is converted to `NumThreads::Auto`, /// * `n` is converted to `NumThreads::Max(n)` where `n > 0`. fn from(value: usize) -> Self { match value { 0 => Self::Auto, n => Self::Max(NonZeroUsize::new(n).expect("must be positive")), } } } impl NumThreads { /// Equivalent to `NumThreads::Max(NonZeroUsize::new(1).unwrap())`. /// /// This will lead to a sequential execution of the defined computation on the main thread. /// Both in terms of used resources and computation time, this mode is not similar but **identical** to a sequential execution using the regular sequential `Iterator`s. pub const fn sequential() -> Self { NumThreads::Max(ONE) } /// Returns true if number of threads is set to 1. /// /// Note that in this case the computation will be executed sequentially using regular iterators. pub fn is_sequential(self) -> bool { matches!(self, Self::Max(n) if n == ONE) } } ================================================ FILE: src/parameters/params.rs ================================================ use super::{chunk_size::ChunkSize, iteration_order::IterationOrder, num_threads::NumThreads}; /// Parameters of a parallel computation. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Params { /// Number of threads to be used in the parallel computation. /// /// See [`NumThreads`] for details. pub num_threads: NumThreads, /// Chunk size to be used in the parallel computation. /// /// See [`ChunkSize`] for details. pub chunk_size: ChunkSize, /// Ordering of outputs of the parallel computation that is important when the outputs /// are collected into a collection. /// /// See [`IterationOrder`] for details. pub iteration_order: IterationOrder, } impl Params { /// Crates parallel computation parameters for the given configurations. pub fn new( num_threads: impl Into, chunk_size: impl Into, iteration_order: IterationOrder, ) -> Self { Self { num_threads: num_threads.into(), chunk_size: chunk_size.into(), iteration_order, } } /// Returns true if number of threads is set to 1. /// /// Note that in this case the computation will be executed sequentially using regular iterators. pub fn is_sequential(self) -> bool { self.num_threads.is_sequential() } // helpers pub(crate) fn with_num_threads(self, num_threads: impl Into) -> Self { Self { num_threads: num_threads.into(), chunk_size: self.chunk_size, iteration_order: self.iteration_order, } } pub(crate) fn with_chunk_size(self, chunk_size: impl Into) -> Self { Self { num_threads: self.num_threads, chunk_size: chunk_size.into(), iteration_order: self.iteration_order, } } pub(crate) fn with_collect_ordering(self, iteration_order: IterationOrder) -> Self { Self { num_threads: self.num_threads, chunk_size: self.chunk_size, iteration_order, } } } ================================================ FILE: src/runner/computation_kind.rs ================================================ /// Computation kind. #[derive(Clone, Copy)] pub enum ComputationKind { /// Computation where outputs are collected into a collection. Collect, /// Computation where the inputs or intermediate results are reduced to a single value. Reduce, /// Computation which allows for early returns, such as `find` operation. EarlyReturn, } ================================================ FILE: src/runner/implementations/mod.rs ================================================ #[cfg(test)] mod tests; mod runner_with_pool; pub use runner_with_pool::RunnerWithPool; mod sequential; pub use sequential::SequentialPool; #[cfg(feature = "std")] mod std_runner; #[cfg(feature = "std")] pub use std_runner::StdDefaultPool; #[cfg(feature = "pond")] mod pond; #[cfg(feature = "pond")] pub use pond::PondPool; #[cfg(feature = "poolite")] mod poolite; #[cfg(feature = "rayon-core")] mod rayon_core; #[cfg(feature = "scoped-pool")] mod scoped_pool; #[cfg(feature = "scoped_threadpool")] mod scoped_threadpool; #[cfg(feature = "yastl")] mod yastl; #[cfg(feature = "yastl")] pub use yastl::YastlPool; ================================================ FILE: src/runner/implementations/pond.rs ================================================ use crate::par_thread_pool::ParThreadPool; use core::num::NonZeroUsize; use pond::{Pool, Scope}; /// A wrapper for `pond::Pool` and number of threads it was built with. /// /// NOTE: The reason why `pond::Pool` does not directly implement `ParThreadPool` /// is simply to be able to provide `max_num_threads` which is the argument used /// to create the pool with. /// /// Following constructor of the `pond::Pool` is made available to `PondPool`: /// * [`PondPool::new_threads_unbounded`] pub struct PondPool(Pool, NonZeroUsize); impl PondPool { /// Spawn a number of threads. The pool's queue of pending jobs is limited. /// The backlog is unbounded as in unbounded. pub fn new_threads_unbounded(num_threads: usize) -> Self { let num_threads = num_threads.min(1); let pool = Pool::new_threads_unbounded(num_threads); #[allow(clippy::missing_panics_doc)] Self(pool, NonZeroUsize::new(num_threads).expect(">0")) } /// Reference to wrapped `pond::Pool`. pub fn inner(&self) -> &Pool { &self.0 } /// Mutable reference to wrapped `pond::Pool`. pub fn inner_mut(&mut self) -> &mut Pool { &mut self.0 } /// Returns the wrapped `pond::Pool`. pub fn into_inner(self) -> Pool { self.0 } } impl ParThreadPool for PondPool { type ScopeRef<'s, 'env, 'scope> = Scope<'env, 'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.execute(work); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(Scope<'env, 'scope>) + Send, { self.0.scoped(f) } fn max_num_threads(&self) -> NonZeroUsize { self.1 } } impl ParThreadPool for &mut PondPool { type ScopeRef<'s, 'env, 'scope> = Scope<'env, 'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.execute(work); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(Scope<'env, 'scope>) + Send, { self.0.scoped(f) } fn max_num_threads(&self) -> NonZeroUsize { self.1 } } ================================================ FILE: src/runner/implementations/poolite.rs ================================================ use crate::par_thread_pool::ParThreadPool; use core::num::NonZeroUsize; use poolite::{Pool, Scoped}; impl ParThreadPool for Pool { type ScopeRef<'s, 'env, 'scope> = &'s Scoped<'env, 'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.push(work); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&'s Scoped<'env, 'scope>) + Send, { self.scoped(f); } fn max_num_threads(&self) -> NonZeroUsize { NonZeroUsize::new(self.threads_future().max(1)).expect(">0") } } impl ParThreadPool for &Pool { type ScopeRef<'s, 'env, 'scope> = &'s Scoped<'env, 'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.push(work); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&'s Scoped<'env, 'scope>) + Send, { self.scoped(f); } fn max_num_threads(&self) -> NonZeroUsize { NonZeroUsize::new(self.threads_future().max(1)).expect(">0") } } ================================================ FILE: src/runner/implementations/rayon_core.rs ================================================ use crate::par_thread_pool::ParThreadPool; use core::num::NonZeroUsize; use rayon_core::ThreadPool; impl ParThreadPool for ThreadPool { type ScopeRef<'s, 'env, 'scope> = &'s rayon_core::Scope<'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.spawn(move |_| work()); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&'s rayon_core::Scope<'scope>) + Send, { self.scope(f) } fn max_num_threads(&self) -> NonZeroUsize { NonZeroUsize::new(self.current_num_threads().max(1)).expect(">0") } } impl ParThreadPool for &rayon_core::ThreadPool { type ScopeRef<'s, 'env, 'scope> = &'s rayon_core::Scope<'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.spawn(move |_| work()); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&'s rayon_core::Scope<'scope>) + Send, { self.scope(f) } fn max_num_threads(&self) -> NonZeroUsize { NonZeroUsize::new(self.current_num_threads().max(1)).expect(">0") } } ================================================ FILE: src/runner/implementations/runner_with_pool.rs ================================================ #[cfg(feature = "std")] use crate::executor::ParallelExecutorWithDiagnostics; use crate::{DefaultExecutor, ParThreadPool, ParallelExecutor, runner::ParallelRunner}; use core::marker::PhantomData; /// Parallel runner with a given pool of type `P` and parallel executor of `R`. /// /// A `RunnerWithPool` can always be created from owned `pool` implementing [`ParThreadPool`], but also from /// * `&pool` in most cases, /// * `&mut pool` in others. /// /// Note that default parallel runner; i.e., [`DefaultRunner`] is: /// * `RunnerWithPool` when "std" feature is enabled, /// * `RunnerWithPool` when "std" feature is disabled. /// /// [`DefaultRunner`]: crate::DefaultRunner /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // parallel computation generic over parallel runner; and hence, the thread pool /// fn run_with_runner(runner: R, input: &[usize]) -> Vec { /// input /// .par() /// .with_runner(runner) /// .flat_map(|x| [*x, 2 * x, x / 7]) /// .map(|x| x.to_string()) /// .collect() /// } /// /// let vec: Vec<_> = (0..42).collect(); /// let input = vec.as_slice(); /// /// // runs sequentially on the main thread /// let runner = RunnerWithPool::from(SequentialPool); /// let expected = run_with_runner(runner, input); /// /// // uses native threads /// #[cfg(feature = "std")] /// { /// let runner = RunnerWithPool::from(StdDefaultPool::default()); /// let result = run_with_runner(runner, input); /// assert_eq!(&expected, &result); /// } /// /// // uses rayon-core ThreadPool with 8 threads /// #[cfg(not(miri))] /// #[cfg(feature = "rayon-core")] /// { /// let pool = rayon_core::ThreadPoolBuilder::new() /// .num_threads(8) /// .build() /// .unwrap(); /// let result = run_with_runner(RunnerWithPool::from(&pool), input); /// assert_eq!(&expected, &result); /// } /// /// // uses scoped-pool Pool with 8 threads /// #[cfg(not(miri))] /// #[cfg(feature = "scoped-pool")] /// { /// let pool = scoped_pool::Pool::new(8); /// let result = run_with_runner(RunnerWithPool::from(&pool), input); /// assert_eq!(&expected, &result); /// } /// /// // uses scoped_threadpool Pool with 8 threads /// #[cfg(not(miri))] /// #[cfg(feature = "scoped_threadpool")] /// { /// let mut pool = scoped_threadpool::Pool::new(8); /// let result = run_with_runner(RunnerWithPool::from(&mut pool), input); // requires &mut pool /// assert_eq!(&expected, &result); /// } /// /// // uses yastl Pool wrapped as YastlPool with 8 threads /// #[cfg(not(miri))] /// #[cfg(feature = "yastl")] /// { /// let pool = YastlPool::new(8); /// let result = run_with_runner(RunnerWithPool::from(&pool), input); /// assert_eq!(&expected, &result); /// } /// /// // uses pond Pool wrapped as PondPool with 8 threads /// #[cfg(not(miri))] /// #[cfg(feature = "pond")] /// { /// let mut pool = PondPool::new_threads_unbounded(8); /// let result = run_with_runner(RunnerWithPool::from(&mut pool), input); // requires &mut pool /// assert_eq!(&expected, &result); /// } /// /// // uses poolite Pool with 8 threads /// #[cfg(not(miri))] /// #[cfg(feature = "poolite")] /// { /// let pool = poolite::Pool::with_builder(poolite::Builder::new().min(8).max(8)).unwrap(); /// let result = run_with_runner(RunnerWithPool::from(&pool), input); /// assert_eq!(&expected, &result); /// } /// ``` #[derive(Clone)] pub struct RunnerWithPool where P: ParThreadPool, R: ParallelExecutor, { pool: P, runner: PhantomData, } impl Default for RunnerWithPool where P: ParThreadPool + Default, R: ParallelExecutor, { fn default() -> Self { Self { pool: Default::default(), runner: PhantomData, } } } impl From

for RunnerWithPool { fn from(pool: P) -> Self { Self { pool, runner: PhantomData, } } } impl RunnerWithPool where P: ParThreadPool, R: ParallelExecutor, { /// Converts the runner into the wrapped underlying pool. /// /// Note that a `RunnerWithPool` can always be created from owned `pool`, but also from /// * `&pool` in most cases, /// * `&mut pool` in others. /// /// This function is only relevant when the runner is created from owned pool, in which case /// `into_inner_pool` can be used to get back ownership of the pool. /// /// # Example /// /// The following example demonstrates the use case for rayon-core thread pool; however, it /// holds for all thread pool implementations. /// /// ``` /// use orx_parallel::*; /// /// let vec: Vec<_> = (0..42).collect(); /// let input = vec.as_slice(); /// /// #[cfg(not(miri))] /// #[cfg(feature = "rayon-core")] /// { /// let pool = rayon_core::ThreadPoolBuilder::new() /// .num_threads(8) /// .build() /// .unwrap(); /// /// // create runner owning the pool /// let mut runner = RunnerWithPool::from(pool); /// /// // use runner, and hence the pool, in parallel computations /// let sum = input.par().with_runner(&mut runner).sum(); /// let max = input.par().with_runner(&mut runner).max(); /// let txt: Vec<_> = input /// .par() /// .with_runner(&mut runner) /// .map(|x| x.to_string()) /// .collect(); /// /// // get back ownership of the pool /// let pool: rayon_core::ThreadPool = runner.into_inner_pool(); /// } /// ``` pub fn into_inner_pool(self) -> P { self.pool } /// Converts the runner into one using the [`ParallelExecutor`] `Q` rather than `R`. pub fn with_executor(self) -> RunnerWithPool { RunnerWithPool { pool: self.pool, runner: PhantomData, } } /// Converts executor of this runner `R` into one with diagnostics; i.e.,`ParallelExecutorWithDiagnostics`. /// /// Note that [`ParallelExecutorWithDiagnostics`] prints the diagnostics on the stdout. Therefore, it must /// only be used while testing a program, not in production. /// /// # Examples /// /// ``` /// use orx_parallel::*; /// /// // normal execution /// /// let range = 0..4096; /// let sum = range /// .par() /// .map(|x| x + 1) /// .filter(|x| x.is_multiple_of(2)) /// .sum(); /// assert_eq!(sum, 4196352); /// /// // execution with diagnostics /// /// let range = 0..4096; /// let sum = range /// .par() /// .with_runner(DefaultRunner::default().with_diagnostics()) /// .map(|x| x + 1) /// .filter(|x| x.is_multiple_of(2)) /// .sum(); /// assert_eq!(sum, 4196352); /// /// // prints diagnostics, which looks something like the following: /// // /// // - Number of threads used = 5 /// // /// // - [Thread idx]: num_calls, num_tasks, avg_chunk_size, first_chunk_sizes /// // - [0]: 25, 1600, 64, [64, 64, 64, 64, 64, 64, 64, 64, 64, 64] /// // - [1]: 26, 1664, 64, [64, 64, 64, 64, 64, 64, 64, 64, 64, 64] /// // - [2]: 13, 832, 64, [64, 64, 64, 64, 64, 64, 64, 64, 64, 64] /// // - [3]: 0, 0, 0, [] /// // - [4]: 0, 0, 0, [] /// ``` #[cfg(feature = "std")] pub fn with_diagnostics(self) -> RunnerWithPool> { RunnerWithPool { pool: self.pool, runner: PhantomData, } } } impl ParallelRunner for RunnerWithPool where P: ParThreadPool, R: ParallelExecutor, { type Executor = R; type ThreadPool = P; fn thread_pool(&self) -> &Self::ThreadPool { &self.pool } fn thread_pool_mut(&mut self) -> &mut Self::ThreadPool { &mut self.pool } } ================================================ FILE: src/runner/implementations/scoped_pool.rs ================================================ use crate::par_thread_pool::ParThreadPool; use core::num::NonZeroUsize; use scoped_pool::{Pool, Scope}; impl ParThreadPool for Pool { type ScopeRef<'s, 'env, 'scope> = &'s Scope<'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.execute(work); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&Scope<'scope>) + Send, { self.scoped(f) } fn max_num_threads(&self) -> NonZeroUsize { NonZeroUsize::new(self.workers().max(1)).expect(">0") } } impl ParThreadPool for &Pool { type ScopeRef<'s, 'env, 'scope> = &'s Scope<'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.execute(work); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&Scope<'scope>) + Send, { self.scoped(f) } fn max_num_threads(&self) -> core::num::NonZeroUsize { NonZeroUsize::new(self.workers().max(1)).expect(">0") } } ================================================ FILE: src/runner/implementations/scoped_threadpool.rs ================================================ use crate::par_thread_pool::ParThreadPool; use core::num::NonZeroUsize; use scoped_threadpool::Pool; impl ParThreadPool for Pool { type ScopeRef<'s, 'env, 'scope> = &'s scoped_threadpool::Scope<'env, 'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.execute(work); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&'s scoped_threadpool::Scope<'env, 'scope>) + Send, { self.scoped(f) } fn max_num_threads(&self) -> NonZeroUsize { NonZeroUsize::new((self.thread_count() as usize).max(1)).expect(">0") } } impl ParThreadPool for &mut Pool { type ScopeRef<'s, 'env, 'scope> = &'s scoped_threadpool::Scope<'env, 'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.execute(work); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&'s scoped_threadpool::Scope<'env, 'scope>) + Send, { self.scoped(f) } fn max_num_threads(&self) -> NonZeroUsize { NonZeroUsize::new((self.thread_count() as usize).max(1)).expect(">0") } } ================================================ FILE: src/runner/implementations/sequential.rs ================================================ use crate::ParThreadPool; use core::num::NonZeroUsize; /// A 'thread pool' with [`max_num_threads`] of 1. /// All computations using this thread pool are executed sequentially by the main thread. /// /// This is the default thread pool used when "std" feature is disabled. /// Note that the thread pool to be used for a parallel computation can be set by the /// [`with_runner`] transformation separately for each parallel iterator. /// /// [`max_num_threads`]: ParThreadPool::max_num_threads /// [`with_runner`]: crate::ParIter::with_runner #[derive(Default, Clone)] pub struct SequentialPool; impl ParThreadPool for SequentialPool { type ScopeRef<'s, 'env, 'scope> = () where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(_: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { work() } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(()) + Send, { f(()) } fn max_num_threads(&self) -> NonZeroUsize { NonZeroUsize::new(1).expect(">0") } } ================================================ FILE: src/runner/implementations/std_runner.rs ================================================ use crate::par_thread_pool::ParThreadPool; use core::num::NonZeroUsize; const MAX_UNSET_NUM_THREADS: NonZeroUsize = NonZeroUsize::new(8).expect(">0"); /// Native standard thread pool. /// /// This is the default thread pool used when "std" feature is enabled. /// Note that the thread pool to be used for a parallel computation can be set by the /// [`with_runner`] transformation separately for each parallel iterator. /// /// Uses `std::thread::scope` and `scope.spawn(..)` to distribute work to threads. /// /// Value of [`max_num_threads`] is determined as the minimum of: /// /// * the available parallelism of the host obtained via `std::thread::available_parallelism()`, and /// * the upper bound set by the environment variable "ORX_PARALLEL_MAX_NUM_THREADS", when set. /// /// [`max_num_threads`]: ParThreadPool::max_num_threads /// [`with_runner`]: crate::ParIter::with_runner #[derive(Clone)] pub struct StdDefaultPool { max_num_threads: NonZeroUsize, } impl StdDefaultPool { /// By default (`StdDefaultPool::default()`), std thread pool assumes that all threads are available /// for the parallel computations. /// /// Constructing the pool with this method makes sure that parallel computations cannot use more than /// `max_num_threads` threads. pub fn with_max_num_threads(max_num_threads: NonZeroUsize) -> Self { let mut pool = Self::default(); if max_num_threads < pool.max_num_threads { pool.max_num_threads = max_num_threads; } pool } } impl Default for StdDefaultPool { fn default() -> Self { let env_max_num_threads = crate::env::max_num_threads_by_env_variable(); let ava_max_num_threads = std::thread::available_parallelism().ok(); let max_num_threads = match (env_max_num_threads, ava_max_num_threads) { (Some(env), Some(ava)) => env.min(ava), (Some(env), None) => env, (None, Some(ava)) => ava, (None, None) => MAX_UNSET_NUM_THREADS, }; Self { max_num_threads } } } impl ParThreadPool for StdDefaultPool { type ScopeRef<'s, 'env, 'scope> = &'s std::thread::Scope<'s, 'env> where 'scope: 's, 'env: 'scope + 's; fn max_num_threads(&self) -> NonZeroUsize { self.max_num_threads } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&'s std::thread::Scope<'s, 'env>) + Send, { std::thread::scope(f) } fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.spawn(work); } } impl ParThreadPool for &StdDefaultPool { type ScopeRef<'s, 'env, 'scope> = &'s std::thread::Scope<'s, 'env> where 'scope: 's, 'env: 'scope + 's; fn max_num_threads(&self) -> NonZeroUsize { self.max_num_threads } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&'s std::thread::Scope<'s, 'env>) + Send, { std::thread::scope(f) } fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.spawn(work); } } ================================================ FILE: src/runner/implementations/tests/mod.rs ================================================ #[cfg(feature = "pond")] mod pond; #[cfg(feature = "poolite")] mod poolite; #[cfg(feature = "rayon-core")] mod rayon_core; #[cfg(feature = "scoped-pool")] mod scoped_pool; #[cfg(feature = "scoped_threadpool")] mod scoped_threadpool; #[cfg(feature = "std")] mod std; #[cfg(feature = "yastl")] mod yastl; mod sequential; mod utils; use utils::run_map; ================================================ FILE: src/runner/implementations/tests/pond.rs ================================================ use super::run_map; use crate::{ IterationOrder, runner::implementations::{PondPool, RunnerWithPool}, }; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; // TODO: miri test terminates with: the main thread terminated without waiting for all remaining threads #[cfg(not(miri))] #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn pool_pond_map(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let mut pool = PondPool::new_threads_unbounded(nt); let orch: RunnerWithPool<_> = (&mut pool).into(); run_map(n, chunk, ordering, orch); } ================================================ FILE: src/runner/implementations/tests/poolite.rs ================================================ use super::run_map; use crate::{IterationOrder, runner::implementations::RunnerWithPool}; use poolite::{Builder, Pool}; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; // `poolite::Builder::new()` fails miri test #[cfg(not(miri))] #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn pool_poolite_map(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let pool = Pool::with_builder(Builder::new().max(nt).min(nt)).unwrap(); let orch: RunnerWithPool<_> = (&pool).into(); run_map(n, chunk, ordering, orch); } ================================================ FILE: src/runner/implementations/tests/rayon_core.rs ================================================ use super::run_map; use crate::{IterationOrder, runner::implementations::RunnerWithPool}; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; // TODO: rayon_core pool fails the miri test (integer-to-pointer cast crossbeam-epoch-0.9.18/src/atomic.rs:204:11) #[cfg(not(miri))] #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn pool_rayon_map(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let pool = rayon_core::ThreadPoolBuilder::new() .num_threads(nt) .build() .unwrap(); let orch: RunnerWithPool<_> = (&pool).into(); run_map(n, chunk, ordering, orch); } ================================================ FILE: src/runner/implementations/tests/scoped_pool.rs ================================================ use super::run_map; use crate::{IterationOrder, runner::implementations::RunnerWithPool}; use scoped_pool::Pool; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; // `scoped_pool::Pool::new(nt)` fails miri test #[cfg(not(miri))] #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn pool_scoped_pool_map(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let pool = Pool::new(nt); let orch = RunnerWithPool::from(&pool); run_map(n, chunk, ordering, orch); } ================================================ FILE: src/runner/implementations/tests/scoped_threadpool.rs ================================================ use super::run_map; use crate::{IterationOrder, runner::implementations::RunnerWithPool}; use scoped_threadpool::Pool; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; // TODO: miri test terminates with: the main thread terminated without waiting for all remaining threads #[cfg(not(miri))] #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn pool_scoped_threadpool_map(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let mut pool = Pool::new(nt as u32); let orch: RunnerWithPool<_> = (&mut pool).into(); run_map(n, chunk, ordering, orch); } ================================================ FILE: src/runner/implementations/tests/sequential.rs ================================================ use super::run_map; use crate::{IterationOrder, RunnerWithPool, runner::implementations::sequential::SequentialPool}; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn pool_scoped_threadpool_map(n: usize, _: usize, chunk: usize, ordering: IterationOrder) { let orch = RunnerWithPool::from(SequentialPool); run_map(n, chunk, ordering, orch); } ================================================ FILE: src/runner/implementations/tests/std.rs ================================================ use super::run_map; use crate::{ DefaultRunner, IterationOrder, RunnerWithPool, runner::implementations::std_runner::StdDefaultPool, }; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn pool_scoped_threadpool_map(n: usize, _: usize, chunk: usize, ordering: IterationOrder) { let orch = DefaultRunner::default(); run_map(n, chunk, ordering, orch); let pool = StdDefaultPool::default(); let orch: RunnerWithPool<_> = (&pool).into(); run_map(n, chunk, ordering, orch); } ================================================ FILE: src/runner/implementations/tests/utils.rs ================================================ use crate::{IntoParIter, IterationOrder, ParIter, runner::ParallelRunner}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_pinned_vec::PinnedVec; use orx_split_vec::SplitVec; pub fn run_map(n: usize, chunk: usize, ordering: IterationOrder, mut orch: impl ParallelRunner) { let offset = 33; let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let map = |x: String| format!("{}!", x); let mut output = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let mut expected = Vec::new(); for i in 0..offset { let value = || map(i.to_string()); output.push(value()); expected.push(value()); } expected.extend(input.clone().into_iter().map(map)); let mut output = input .into_par() .with_runner(&mut orch) .chunk_size(chunk) .iteration_order(ordering) .map(map) .collect_into(output); if matches!(ordering, IterationOrder::Arbitrary) { expected.sort(); output.sort(); } assert_eq!(expected, output.to_vec()); } ================================================ FILE: src/runner/implementations/tests/yastl.rs ================================================ use super::run_map; use crate::{ IterationOrder, runner::implementations::{RunnerWithPool, YastlPool}, }; use test_case::test_matrix; use yastl::ThreadConfig; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; // TODO: miri test terminates with: the main thread terminated without waiting for all remaining threads #[cfg(not(miri))] #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn pool_yastl_map(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let pool = YastlPool::new(nt); let orch: RunnerWithPool<_> = (&pool).into(); run_map(n, chunk, ordering, orch); let pool = YastlPool::with_config(nt, ThreadConfig::new()); let orch: RunnerWithPool<_> = (pool).into(); run_map(n, chunk, ordering, orch); } ================================================ FILE: src/runner/implementations/yastl.rs ================================================ use crate::ParThreadPool; use core::num::NonZeroUsize; use yastl::{Pool, Scope, ThreadConfig}; /// A wrapper for `yastl::Pool` and number of threads it was built with. /// /// NOTE: The reason why `yastl::Pool` does not directly implement `ParThreadPool` /// is simply to be able to provide `max_num_threads` which is the argument used /// to create the pool with. /// /// Two constructors of the `yastl::Pool` are made available to `YastlPool`: /// * [`YastlPool::new`] /// * [`YastlPool::with_config`] #[derive(Clone)] pub struct YastlPool(Pool, NonZeroUsize); impl YastlPool { /// Create a new Pool that will execute it's tasks on `num_threads` worker threads. pub fn new(num_threads: usize) -> Self { let num_threads = num_threads.min(1); let pool = Pool::new(num_threads); #[allow(clippy::missing_panics_doc)] Self(pool, NonZeroUsize::new(num_threads).expect(">0")) } /// Create a new Pool that will execute it's tasks on `num_threads` worker threads and /// spawn them using the given `config`. pub fn with_config(num_threads: usize, config: ThreadConfig) -> Self { let num_threads = num_threads.min(1); let pool = Pool::with_config(num_threads, config); #[allow(clippy::missing_panics_doc)] Self(pool, NonZeroUsize::new(num_threads).expect(">0")) } /// Reference to wrapped `yastl::Pool`. pub fn inner(&self) -> &Pool { &self.0 } /// Mutable reference to wrapped `yastl::Pool`. pub fn inner_mut(&mut self) -> &mut Pool { &mut self.0 } /// Returns the wrapped `yastl::Pool`. pub fn into_inner(self) -> Pool { self.0 } } impl ParThreadPool for YastlPool { type ScopeRef<'s, 'env, 'scope> = &'s Scope<'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.execute(work); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&'s Scope<'scope>) + Send, { self.0.scoped(f) } fn max_num_threads(&self) -> NonZeroUsize { self.1 } } impl ParThreadPool for &YastlPool { type ScopeRef<'s, 'env, 'scope> = &'s Scope<'scope> where 'scope: 's, 'env: 'scope + 's; fn run_in_scope<'s, 'env, 'scope, W>(s: &Self::ScopeRef<'s, 'env, 'scope>, work: W) where 'scope: 's, 'env: 'scope + 's, W: Fn() + Send + 'scope + 'env, { s.execute(work); } fn scoped_computation<'env, 'scope, F>(&'env mut self, f: F) where 'env: 'scope, for<'s> F: FnOnce(&'s Scope<'scope>) + Send, { self.0.scoped(f) } fn max_num_threads(&self) -> NonZeroUsize { self.1 } } ================================================ FILE: src/runner/mod.rs ================================================ mod computation_kind; mod implementations; mod num_spawned; mod parallel_runner; pub(crate) use parallel_runner::{SharedStateOf, ThreadRunnerOf}; pub use computation_kind::ComputationKind; pub use implementations::{RunnerWithPool, SequentialPool}; pub use num_spawned::NumSpawned; pub use parallel_runner::ParallelRunner; #[cfg(feature = "pond")] pub use implementations::PondPool; #[cfg(feature = "std")] pub use implementations::StdDefaultPool; #[cfg(feature = "yastl")] pub use implementations::YastlPool; // DEFAULT /// Default pool used by orx-parallel computations: /// /// * [`StdDefaultPool`] when "std" feature is enabled, /// * [`SequentialPool`] otherwise. #[cfg(feature = "std")] pub type DefaultPool = StdDefaultPool; /// Default pool used by orx-parallel computations: /// /// * `StdDefaultPool` when "std" feature is enabled, /// * [`SequentialPool`] otherwise. #[cfg(not(feature = "std"))] pub type DefaultPool = SequentialPool; /// Default runner used by orx-parallel computations, using the [`DefaultPool`]: /// /// * [`RunnerWithPool`] with [`StdDefaultPool`] when "std" feature is enabled, /// * [`RunnerWithPool`] with [`SequentialPool`] otherwise. pub type DefaultRunner = RunnerWithPool; ================================================ FILE: src/runner/num_spawned.rs ================================================ /// Number of spawned threads to execute a parallel computation. #[derive(Clone, Copy)] pub struct NumSpawned(usize); impl NumSpawned { /// Zero. pub fn zero() -> Self { Self(0) } /// Adds one to the spawned thread count. pub fn increment(&mut self) { self.0 += 1; } /// Converts into usize. pub fn into_inner(self) -> usize { self.0 } } impl core::ops::Rem for NumSpawned { type Output = usize; fn rem(self, rhs: Self) -> Self::Output { self.0 % rhs.0 } } ================================================ FILE: src/runner/parallel_runner.rs ================================================ use crate::{ NumThreads, ParallelExecutor, Params, generic_values::runner_results::{Fallibility, Infallible, Never}, par_thread_pool::{ParThreadPool, ParThreadPoolCompute}, runner::{ComputationKind, NumSpawned}, }; use alloc::vec::Vec; use core::num::NonZeroUsize; use orx_concurrent_iter::ConcurrentIter; /// Parallel runner defining how the threads must be spawned and job must be distributed. pub trait ParallelRunner { /// Parallel executor responsible for distribution of tasks to the threads. type Executor: ParallelExecutor; /// Thread pool responsible for providing threads to the parallel computation. type ThreadPool: ParThreadPool; /// Creates a new parallel executor for a parallel computation. fn new_executor( &self, kind: ComputationKind, params: Params, initial_input_len: Option, ) -> Self::Executor { let max_num_threads = self.max_num_threads_for_computation(params, initial_input_len); ::new(kind, params, initial_input_len, max_num_threads) } /// Reference to the underlying thread pool. fn thread_pool(&self) -> &Self::ThreadPool; /// Mutable reference to the underlying thread pool. fn thread_pool_mut(&mut self) -> &mut Self::ThreadPool; // derived /// Runs `thread_do` using threads provided by the thread pool. fn run_all( &mut self, params: Params, iter: I, kind: ComputationKind, thread_do: F, ) -> NumSpawned where I: ConcurrentIter, F: Fn(NumSpawned, &I, &SharedStateOf, ThreadRunnerOf) + Sync, { let executor = self.new_executor(kind, params, iter.try_get_len()); let state = executor.new_shared_state(); let do_spawn = |num_spawned| executor.do_spawn_new(num_spawned, &state, &iter); let work = |num_spawned: NumSpawned| { let thread_idx = num_spawned.into_inner(); thread_do( num_spawned, &iter, &state, executor.new_thread_executor(thread_idx, &state), ); }; let result = self.thread_pool_mut().run_in_pool(do_spawn, work); executor.complete_task(state); result } /// Runs `thread_map` using threads provided by the thread pool. fn map_all( &mut self, params: Params, iter: I, kind: ComputationKind, thread_map: M, ) -> (NumSpawned, Result, F::Error>) where F: Fallibility, I: ConcurrentIter, M: Fn(NumSpawned, &I, &SharedStateOf, ThreadRunnerOf) -> Result + Sync, T: Send, F::Error: Send, { let iter_len = iter.try_get_len(); let executor = self.new_executor(kind, params, iter_len); let state = executor.new_shared_state(); let do_spawn = |num_spawned| executor.do_spawn_new(num_spawned, &state, &iter); let work = |num_spawned: NumSpawned| { let thread_idx = num_spawned.into_inner(); thread_map( num_spawned, &iter, &state, executor.new_thread_executor(thread_idx, &state), ) }; let max_num_threads = self.max_num_threads_for_computation(params, iter_len); let result = self.thread_pool_mut() .map_in_pool::(do_spawn, work, max_num_threads); executor.complete_task(state); result } /// Runs infallible `thread_map` using threads provided by the thread pool. fn map_infallible( &mut self, params: Params, iter: I, kind: ComputationKind, thread_map: M, ) -> (NumSpawned, Result, Never>) where I: ConcurrentIter, M: Fn(NumSpawned, &I, &SharedStateOf, ThreadRunnerOf) -> Result + Sync, T: Send, { self.map_all::(params, iter, kind, thread_map) } /// Returns the maximum number of threads that can be used for the computation defined by /// the `params` and input `iter_len`. fn max_num_threads_for_computation( &self, params: Params, iter_len: Option, ) -> NonZeroUsize { let pool = self.thread_pool().max_num_threads(); let env = crate::env::max_num_threads_by_env_variable().unwrap_or(NonZeroUsize::MAX); let req = match (iter_len, params.num_threads) { (Some(len), NumThreads::Auto) => NonZeroUsize::new(len.max(1)).expect(">0"), (Some(len), NumThreads::Max(nt)) => NonZeroUsize::new(len.max(1)).expect(">0").min(nt), (None, NumThreads::Auto) => NonZeroUsize::MAX, (None, NumThreads::Max(nt)) => nt, }; req.min(pool.min(env)) } } pub(crate) type SharedStateOf = <::Executor as ParallelExecutor>::SharedState; pub(crate) type ThreadRunnerOf = <::Executor as ParallelExecutor>::ThreadExecutor; // auto impl for &mut pool impl ParallelRunner for &'_ mut O where O: ParallelRunner, { type Executor = O::Executor; type ThreadPool = O::ThreadPool; fn thread_pool(&self) -> &Self::ThreadPool { O::thread_pool(self) } fn thread_pool_mut(&mut self) -> &mut Self::ThreadPool { O::thread_pool_mut(self) } } ================================================ FILE: src/special_type_sets/mod.rs ================================================ mod sum; pub use sum::Sum; ================================================ FILE: src/special_type_sets/sum.rs ================================================ use core::ops::Add; /// Number that can be summed over. pub trait Sum { /// Zero. fn zero() -> Output; /// Maps the number to owned value. fn map(a: Self) -> Output; /// Maps the number to owned value. fn u_map(_: &mut U, a: Self) -> Output; /// Returns sum of `a` and `b`. fn reduce(a: Output, b: Output) -> Output; /// Returns sum of `a` and `b`. fn u_reduce(_: &mut U, a: Output, b: Output) -> Output; } impl Sum for X where X: Default + Add, { fn zero() -> X { X::default() } #[inline(always)] fn map(a: Self) -> X { a } #[inline(always)] fn u_map(_: &mut U, a: Self) -> X { a } #[inline(always)] fn reduce(a: X, b: X) -> X { a + b } #[inline(always)] fn u_reduce(_: &mut U, a: X, b: X) -> X { a + b } } impl<'a, X> Sum for &'a X where X: Default + Add + Copy, &'a X: Add<&'a X, Output = X>, { fn zero() -> X { X::default() } #[inline(always)] fn map(a: Self) -> X { *a } #[inline(always)] fn u_map(_: &mut U, a: Self) -> X { *a } #[inline(always)] fn reduce(a: X, b: X) -> X { a + b } #[inline(always)] fn u_reduce(_: &mut U, a: X, b: X) -> X { a + b } } impl<'a, X> Sum for &'a mut X where X: Default + Add + Copy, &'a X: Add<&'a X, Output = X>, { fn zero() -> X { X::default() } #[inline(always)] fn map(a: Self) -> X { *a } #[inline(always)] fn u_map(_: &mut U, a: Self) -> X { *a } #[inline(always)] fn reduce(a: X, b: X) -> X { a + b } #[inline(always)] fn u_reduce(_: &mut U, a: X, b: X) -> X { a + b } } ================================================ FILE: src/test_utils.rs ================================================ /// Input size. #[cfg(not(miri))] pub const N: &[usize] = &[8025, 42735]; /// Number of threads. #[cfg(not(miri))] pub const NT: &[usize] = &[0, 1, 2, 4]; /// Chunk size. #[cfg(not(miri))] pub const CHUNK: &[usize] = &[0, 1, 64, 1024]; /// Input size. #[cfg(miri)] pub const N: &[usize] = &[57]; /// Number of threads. #[cfg(miri)] pub const NT: &[usize] = &[1, 2]; /// Chunk size. #[cfg(miri)] pub const CHUNK: &[usize] = &[4]; /// Test all combinations of the settings with the `test` method. pub fn test_n_nt_chunk(n: &[usize], nt: &[usize], chunk: &[usize], test: T) where T: Fn(usize, usize, usize), { for n in n { for nt in nt { for chunk in chunk { test(*n, *nt, *chunk); } } } } ================================================ FILE: src/using/collect_into/collect.rs ================================================ use crate::Params; use crate::generic_values::runner_results::{ Fallibility, Infallible, ParallelCollect, ParallelCollectArbitrary, Stop, }; use crate::runner::{NumSpawned, ParallelRunner}; use crate::using::executor::parallel_compute as prc; use crate::using::using_variants::Using; use crate::{IterationOrder, generic_values::Values}; use orx_concurrent_iter::ConcurrentIter; use orx_fixed_vec::IntoConcurrentPinnedVec; pub fn map_collect_into<'using, U, R, I, O, M1, P>( using: U, orchestrator: R, params: Params, iter: I, map1: M1, pinned_vec: P, ) -> (NumSpawned, P) where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, O: Send, P: IntoConcurrentPinnedVec, { match (params.is_sequential(), params.iteration_order) { (true, _) => ( NumSpawned::zero(), map_collect_into_seq(using, iter, map1, pinned_vec), ), #[cfg(test)] (false, IterationOrder::Arbitrary) => { prc::collect_arbitrary::m(using, orchestrator, params, iter, map1, pinned_vec) } (false, _) => prc::collect_ordered::m(using, orchestrator, params, iter, map1, pinned_vec), } } fn map_collect_into_seq<'using, U, I, O, M1, P>(using: U, iter: I, map1: M1, mut pinned_vec: P) -> P where U: Using<'using>, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, O: Send, P: IntoConcurrentPinnedVec, { let mut u = using.into_inner(); let iter = iter.into_seq_iter(); for i in iter { pinned_vec.push(map1(&mut u, i)); } pinned_vec } pub fn xap_collect_into<'using, U, R, I, Vo, X1, P>( using: U, orchestrator: R, params: Params, iter: I, xap1: X1, pinned_vec: P, ) -> (NumSpawned, P) where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { match (params.is_sequential(), params.iteration_order) { (true, _) => ( NumSpawned::zero(), xap_collect_into_seq(using, iter, xap1, pinned_vec), ), (false, IterationOrder::Arbitrary) => { let (num_threads, result) = prc::collect_arbitrary::x(using, orchestrator, params, iter, xap1, pinned_vec); let pinned_vec = match result { ParallelCollectArbitrary::AllOrUntilWhileCollected { pinned_vec } => pinned_vec, }; (num_threads, pinned_vec) } (false, IterationOrder::Ordered) => { let (num_threads, result) = prc::collect_ordered::x(using, orchestrator, params, iter, xap1, pinned_vec); let pinned_vec = match result { ParallelCollect::AllCollected { pinned_vec } => pinned_vec, ParallelCollect::StoppedByWhileCondition { pinned_vec, stopped_idx: _, } => pinned_vec, }; (num_threads, pinned_vec) } } } fn xap_collect_into_seq<'using, U, I, Vo, X1, P>( using: U, iter: I, xap1: X1, mut pinned_vec: P, ) -> P where U: Using<'using>, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { let mut u = using.into_inner(); let iter = iter.into_seq_iter(); for i in iter { let vt = xap1(&mut u, i); let done = vt.push_to_pinned_vec(&mut pinned_vec); if Vo::sequential_push_to_stop(done).is_some() { break; } } pinned_vec } pub fn xap_try_collect_into<'using, U, R, I, Vo, X1, P>( using: U, orchestrator: R, params: Params, iter: I, xap1: X1, pinned_vec: P, ) -> ( NumSpawned, Result::Error>, ) where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { match (params.is_sequential(), params.iteration_order) { (true, _) => ( NumSpawned::zero(), xap_try_collect_into_seq(using, iter, xap1, pinned_vec), ), (false, IterationOrder::Arbitrary) => { let (nt, result) = prc::collect_arbitrary::x(using, orchestrator, params, iter, xap1, pinned_vec); (nt, result.into_result()) } (false, IterationOrder::Ordered) => { let (nt, result) = prc::collect_ordered::x(using, orchestrator, params, iter, xap1, pinned_vec); (nt, result.into_result()) } } } fn xap_try_collect_into_seq<'using, U, I, Vo, X1, P>( using: U, iter: I, xap1: X1, mut pinned_vec: P, ) -> Result::Error> where U: Using<'using>, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { let mut u = using.into_inner(); let iter = iter.into_seq_iter(); for i in iter { let vt = xap1(&mut u, i); let done = vt.push_to_pinned_vec(&mut pinned_vec); if let Some(stop) = Vo::sequential_push_to_stop(done) { match stop { Stop::DueToWhile => return Ok(pinned_vec), Stop::DueToError { error } => return Err(error), } } } Ok(pinned_vec) } ================================================ FILE: src/using/collect_into/fixed_vec.rs ================================================ use crate::Params; use crate::generic_values::TransformableValues; use crate::generic_values::runner_results::Infallible; use crate::runner::ParallelRunner; use crate::using::Using; use crate::using::collect_into::u_par_collect_into::UParCollectIntoCore; use alloc::vec::Vec; use orx_concurrent_iter::ConcurrentIter; use orx_fixed_vec::FixedVec; impl UParCollectIntoCore for FixedVec where O: Send + Sync, { fn u_m_collect_into<'using, U, R, I, M1>( self, using: U, orchestrator: R, params: Params, iter: I, map1: M1, ) -> Self where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { let vec = Vec::from(self); FixedVec::from(vec.u_m_collect_into(using, orchestrator, params, iter, map1)) } fn u_x_collect_into<'using, U, R, I, Vo, X1>( self, using: U, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Self where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { let vec = Vec::from(self); FixedVec::from(vec.u_x_collect_into(using, orchestrator, params, iter, xap1)) } fn u_x_try_collect_into<'using, U, R, I, Vo, X1>( self, using: U, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Result::Error> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, Vo: crate::generic_values::Values, Self: Sized, { let vec = Vec::from(self); vec.u_x_try_collect_into(using, orchestrator, params, iter, xap1) .map(FixedVec::from) } } ================================================ FILE: src/using/collect_into/mod.rs ================================================ pub(crate) mod collect; mod fixed_vec; mod split_vec; mod u_par_collect_into; mod vec; pub(crate) use u_par_collect_into::UParCollectIntoCore; ================================================ FILE: src/using/collect_into/split_vec.rs ================================================ use crate::Params; use crate::collect_into::utils::split_vec_reserve; use crate::generic_values::TransformableValues; use crate::generic_values::runner_results::Infallible; use crate::runner::ParallelRunner; use crate::using::Using; use crate::using::collect_into::collect::{ map_collect_into, xap_collect_into, xap_try_collect_into, }; use crate::using::collect_into::u_par_collect_into::UParCollectIntoCore; use orx_concurrent_iter::ConcurrentIter; use orx_split_vec::{GrowthWithConstantTimeAccess, PseudoDefault, SplitVec}; impl UParCollectIntoCore for SplitVec where O: Send + Sync, G: GrowthWithConstantTimeAccess, Self: PseudoDefault, { fn u_m_collect_into<'using, U, R, I, M1>( mut self, using: U, orchestrator: R, params: Params, iter: I, map1: M1, ) -> Self where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { split_vec_reserve(&mut self, params.is_sequential(), iter.try_get_len()); let (_, pinned_vec) = map_collect_into(using, orchestrator, params, iter, map1, self); pinned_vec } fn u_x_collect_into<'using, U, R, I, Vo, X1>( mut self, using: U, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Self where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { split_vec_reserve(&mut self, params.is_sequential(), iter.try_get_len()); let (_num_spawned, pinned_vec) = xap_collect_into(using, orchestrator, params, iter, xap1, self); pinned_vec } fn u_x_try_collect_into<'using, U, R, I, Vo, X1>( mut self, using: U, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Result::Error> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, Vo: crate::generic_values::Values, Self: Sized, { split_vec_reserve(&mut self, params.is_sequential(), iter.try_get_len()); let (_num_spawned, result) = xap_try_collect_into(using, orchestrator, params, iter, xap1, self); result } } ================================================ FILE: src/using/collect_into/u_par_collect_into.rs ================================================ use crate::Params; use crate::collect_into::ParCollectIntoCore; use crate::generic_values::runner_results::{Fallibility, Infallible}; use crate::generic_values::{TransformableValues, Values}; use crate::runner::ParallelRunner; use crate::using::using_variants::Using; use orx_concurrent_iter::ConcurrentIter; pub trait UParCollectIntoCore: ParCollectIntoCore { fn u_m_collect_into<'using, U, R, I, M1>( self, using: U, orchestrator: R, params: Params, iter: I, map1: M1, ) -> Self where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync; fn u_x_collect_into<'using, U, R, I, Vo, X1>( self, using: U, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Self where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync; fn u_x_try_collect_into<'using, U, R, I, Vo, X1>( self, using: U, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Result::Error> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, Vo: Values, Self: Sized; } ================================================ FILE: src/using/collect_into/vec.rs ================================================ use crate::Params; use crate::collect_into::utils::extend_vec_from_split; use crate::generic_values::TransformableValues; use crate::generic_values::runner_results::Infallible; use crate::runner::ParallelRunner; use crate::using::collect_into::collect::map_collect_into; use crate::using::collect_into::u_par_collect_into::UParCollectIntoCore; use crate::using::using_variants::Using; use alloc::vec::Vec; use orx_concurrent_iter::ConcurrentIter; use orx_fixed_vec::FixedVec; use orx_split_vec::SplitVec; impl UParCollectIntoCore for Vec where O: Send + Sync, { fn u_m_collect_into<'using, U, R, I, M1>( mut self, using: U, orchestrator: R, params: Params, iter: I, map1: M1, ) -> Self where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { match iter.try_get_len() { None => { let split_vec = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let split_vec = split_vec.u_m_collect_into(using, orchestrator, params, iter, map1); extend_vec_from_split(self, split_vec) } Some(len) => { self.reserve(len); let fixed_vec = FixedVec::from(self); let (_, fixed_vec) = map_collect_into(using, orchestrator, params, iter, map1, fixed_vec); Vec::from(fixed_vec) } } } fn u_x_collect_into<'using, U, R, I, Vo, X1>( self, using: U, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Self where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { let split_vec = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let split_vec = split_vec.u_x_collect_into(using, orchestrator, params, iter, xap1); extend_vec_from_split(self, split_vec) } fn u_x_try_collect_into<'using, U, R, I, Vo, X1>( self, using: U, orchestrator: R, params: Params, iter: I, xap1: X1, ) -> Result::Error> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, Vo: crate::generic_values::Values, Self: Sized, { let split_vec = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let result = split_vec.u_x_try_collect_into(using, orchestrator, params, iter, xap1); result.map(|split_vec| extend_vec_from_split(self, split_vec)) } } ================================================ FILE: src/using/computational_variants/mod.rs ================================================ #[cfg(test)] mod tests; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with None. pub mod u_fallible_option; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with an error. pub mod u_fallible_result; mod u_map; mod u_par; mod u_xap; pub use u_map::UParMap; pub use u_par::UPar; pub use u_xap::UParXap; ================================================ FILE: src/using/computational_variants/tests/copied.rs ================================================ use super::utils::{make_u_filter, make_u_map}; use crate::{test_utils::*, *}; use alloc::vec::Vec; use std::string::ToString; use test_case::test_matrix; fn input>(n: usize) -> O { (0..n).collect() } #[test_matrix(N, NT, CHUNK)] fn copied_cloned_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input: Vec<_> = input::>(n); let expected: usize = input.iter().copied().sum(); let par = || { input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()) }; let output = par().copied().sum(); assert_eq!(output, expected); let output = par().cloned().sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn copied_cloned_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let map = |x: usize| x + 1; let expected: usize = input.iter().copied().map(&map).sum(); let par = || { input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()) }; let output = par().copied().map(make_u_map(map)).sum(); assert_eq!(output, expected); let output = par().cloned().map(make_u_map(map)).sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn copied_cloned_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let flat_map = |x: usize| [x, x + 1, x + 2]; let expected: usize = input.iter().copied().flat_map(&flat_map).sum(); let par = || { input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()) }; let output = par().copied().flat_map(make_u_map(flat_map)).sum(); assert_eq!(output, expected); let output = par().cloned().flat_map(make_u_map(flat_map)).sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn copied_cloned_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: usize| (!x.is_multiple_of(3)).then_some(x + 1); let expected: usize = input.iter().copied().filter_map(&filter_map).sum(); let par = || { input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()) }; let output = par().copied().filter_map(make_u_map(filter_map)).sum(); assert_eq!(output, expected); let output = par().cloned().filter_map(make_u_map(filter_map)).sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn copied_cloned_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: usize| (!x.is_multiple_of(3)).then_some(x + 1); let filter = |x: &usize| x % 3 != 1; let flat_map = |x: usize| [x, x + 1, x + 2]; let expected: usize = input .iter() .copied() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .sum(); let par = || { input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()) }; let output = par() .copied() .filter_map(make_u_map(filter_map)) .filter(make_u_filter(&filter)) .flat_map(make_u_map(flat_map)) .sum(); assert_eq!(output, expected); let output = par() .cloned() .filter_map(make_u_map(filter_map)) .filter(make_u_filter(&filter)) .flat_map(make_u_map(flat_map)) .sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/count.rs ================================================ use super::utils::{make_u_filter, make_u_map}; use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } #[test_matrix(N, NT, CHUNK)] fn count_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let expected = n; let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.count(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn count_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let expected = n; let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.map(make_u_map(|x| format!("{}!", x))); let output = par.count(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn count_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let expected = input.clone().into_iter().flat_map(&flat_map).count(); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.flat_map(make_u_map(flat_map)); let output = par.count(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn count_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let expected = input.clone().into_iter().filter_map(&filter_map).count(); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.filter_map(make_u_map(filter_map)); let output = par.count(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn count_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let filter = |x: &String| x.ends_with('3'); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let expected = input .clone() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .count(); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par .filter_map(make_u_map(filter_map)) .filter(make_u_filter(&filter)) .flat_map(make_u_map(flat_map)); let output = par.count(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/fallible_option.rs ================================================ use crate::using::computational_variants::tests::utils::{make_u_filter, make_u_map}; use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } #[test_matrix(NT, CHUNK)] fn fallible_option_collect_empty(nt: &[usize], chunk: &[usize]) { let test = |_, nt, chunk| { let input = || input::>(0); let par = input() .into_iter() .map(|_| None::) .collect::>() .into_par() .using_clone("XyZw".to_string()) .num_threads(nt) .chunk_size(chunk) .into_fallible_option(); let output: Option> = par.collect(); assert_eq!(output, Some(Vec::new())); }; test_n_nt_chunk(&[0], nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn fallible_option_collect_partial_success(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let par = input() .into_par() .using_clone("XyZw".to_string()) .num_threads(nt) .chunk_size(chunk) .map(make_u_map(|x| (x != "50").then_some(x))) .into_fallible_option() .filter(make_u_filter(&|x: &String| !x.ends_with('9'))) .flat_map(make_u_map(|x| [format!("{x}?"), x])) .map(make_u_map(|x| format!("{x}!"))); let output: Option> = par.collect(); assert_eq!(output, None); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn fallible_option_collect_complete_success(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let expected: Vec<_> = input() .into_iter() .filter(|x| !x.ends_with('9')) .flat_map(|x| [format!("{x}?"), x]) .map(|x| format!("{x}!")) .filter_map(Some) .collect(); let par = input() .into_par() .using_clone("XyZw".to_string()) .num_threads(nt) .chunk_size(chunk) .map(make_u_map(|x| (x != "xyz").then_some(x))) .into_fallible_option() .filter(make_u_filter(&|x: &String| !x.ends_with('9'))) .flat_map(make_u_map(|x| [format!("{x}?"), x])) .map(make_u_map(|x| format!("{x}!"))) .filter_map(make_u_map(Some)); let output: Option> = par.collect(); assert_eq!(output, Some(expected)); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/fallible_result.rs ================================================ use crate::using::computational_variants::tests::utils::{make_u_filter, make_u_map}; use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } #[derive(Debug, PartialEq, Eq)] struct MyErr(String); impl MyErr { fn new() -> Self { Self("error".to_string()) } } #[test_matrix(NT, CHUNK)] fn fallible_result_collect_empty(nt: &[usize], chunk: &[usize]) { let test = |_, nt, chunk| { let input = || input::>(0); let par = input() .into_iter() .map(|_| Err::(MyErr::new())) .collect::>() .into_par() .using_clone("XyZw".to_string()) .num_threads(nt) .chunk_size(chunk) .into_fallible_result(); let output: Result, MyErr> = par.collect(); assert_eq!(output, Ok(Vec::new())); }; test_n_nt_chunk(&[0], nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn fallible_result_collect_partial_success(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let par = input() .into_par() .using_clone("XyZw".to_string()) .num_threads(nt) .chunk_size(chunk) .map(make_u_map(|x| match x == "50" { true => Err(MyErr::new()), false => Ok(x), })) .into_fallible_result() .filter(make_u_filter(&|x: &String| !x.ends_with('9'))) .flat_map(make_u_map(|x| [format!("{x}?"), x])) .map(make_u_map(|x| format!("{x}!"))); let output: Result, MyErr> = par.collect(); assert_eq!(output, Err(MyErr::new())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn fallible_result_collect_complete_success(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let expected: Vec<_> = input() .into_iter() .filter(|x| !x.ends_with('9')) .flat_map(|x| [format!("{x}?"), x]) .map(|x| format!("{x}!")) .filter_map(Some) .collect(); let par = input() .into_par() .using_clone("XyZw".to_string()) .num_threads(nt) .chunk_size(chunk) .map(make_u_map(|x| match x == "xyz" { true => Err(MyErr::new()), false => Ok(x), })) .into_fallible_result() .filter(make_u_filter(&|x: &String| !x.ends_with('9'))) .flat_map(make_u_map(|x| [format!("{x}?"), x])) .map(make_u_map(|x| format!("{x}!"))) .filter_map(make_u_map(Some)); let output: Result, MyErr> = par.collect(); assert_eq!(output, Ok(expected)); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/flatten.rs ================================================ use super::utils::{make_u_filter, make_u_map}; use crate::{test_utils::*, *}; use alloc::string::{String, ToString}; use alloc::vec::Vec; use alloc::{format, vec}; use test_case::test_matrix; #[test_matrix(N, NT, CHUNK)] fn flatten_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || { (0..n) .map(|i| [i.to_string(), (i + 1).to_string()]) .collect::>() }; let expected: Vec<_> = input().into_iter().flatten().collect(); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: Vec<_> = par.flatten().collect(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn flatten_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || (0..n).collect::>(); let map = |i: usize| vec![i.to_string(), (i + 1).to_string()]; #[allow(clippy::map_flatten)] let expected: Vec<_> = input().into_iter().map(&map).flatten().collect(); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: Vec<_> = par.map(make_u_map(map)).flatten().collect(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn flatten_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || (0..n).map(|i| i.to_string()).collect::>(); let filter_map = |x: String| { x.starts_with('3') .then(|| vec![x.clone(), format!("{}!", x)]) }; #[allow(clippy::map_flatten)] let expected: Vec<_> = input() .clone() .into_iter() .filter_map(&filter_map) .flatten() .collect(); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: Vec<_> = par.filter_map(make_u_map(filter_map)).flatten().collect(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn flatten_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || (0..n).map(|i| i.to_string()).collect::>(); let filter_map = |x: String| { x.starts_with('3') .then(|| vec![x.clone(), format!("{}!", x)]) }; let filter = |x: &Vec| x.len() == 2; let map = |mut x: Vec| { x.push("lorem".to_string()); x }; #[allow(clippy::map_flatten)] let expected: Vec<_> = input() .clone() .into_iter() .filter_map(&filter_map) .filter(&filter) .map(&map) .flatten() .collect(); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: Vec<_> = par .filter_map(make_u_map(filter_map)) .filter(make_u_filter(&filter)) .map(make_u_map(map)) .flatten() .collect(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/for_each.rs ================================================ use super::utils::{make_u_filter, make_u_map}; use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_vec::ConcurrentVec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn sorted(mut x: Vec) -> Vec { x.sort(); x } #[test_matrix(N, NT, CHUNK)] fn for_each_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let vec = ConcurrentVec::new(); par.for_each(make_u_map(|x| { _ = vec.push(x); })); assert_eq!(sorted(vec.to_vec()), sorted(input())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn for_each_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let map = |x| format!("{}!", x); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.map(make_u_map(map)); let vec = ConcurrentVec::new(); par.for_each(make_u_map(|x| { _ = vec.push(x); })); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().map(map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn for_each_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.flat_map(make_u_map(flat_map)); let vec = ConcurrentVec::new(); par.for_each(make_u_map(|x| { _ = vec.push(x); })); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().flat_map(flat_map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn for_each_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.filter_map(make_u_map(filter_map)); let vec = ConcurrentVec::new(); par.for_each(make_u_map(|x| { _ = vec.push(x); })); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().filter_map(filter_map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn for_each_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let filter = |x: &String| x.ends_with('3'); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par .filter_map(make_u_map(filter_map)) .filter(make_u_filter(&filter)) .flat_map(make_u_map(flat_map)); let vec = ConcurrentVec::new(); par.for_each(make_u_map(|x| { _ = vec.push(x); })); assert_eq!( sorted(vec.to_vec()), sorted( input() .into_iter() .filter_map(filter_map) .filter(filter) .flat_map(flat_map) .collect() ) ); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/inspect.rs ================================================ use super::utils::{make_u_filter, make_u_map}; use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_vec::ConcurrentVec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn sorted(mut x: Vec) -> Vec { x.sort(); x } #[test_matrix(N, NT, CHUNK)] fn inspect_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let vec = ConcurrentVec::new(); par.inspect(|u, x| { let u = u.as_mut_str(); u.get_mut(0..2).expect("len>1").make_ascii_uppercase(); _ = vec.push(x.clone()); }) .count(); assert_eq!(sorted(vec.to_vec()), sorted(input())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn inspect_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let map = |x| format!("{}!", x); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.map(make_u_map(map)); let vec = ConcurrentVec::new(); par.inspect(|u, x| { let u = u.as_mut_str(); u.get_mut(0..2).expect("len>1").make_ascii_uppercase(); _ = vec.push(x.clone()); }) .count(); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().map(map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn inspect_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.flat_map(make_u_map(flat_map)); let vec = ConcurrentVec::new(); par.inspect(|u, x| { let u = u.as_mut_str(); u.get_mut(0..2).expect("len>1").make_ascii_uppercase(); _ = vec.push(x.clone()); }) .count(); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().flat_map(flat_map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn inspect_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.filter_map(make_u_map(filter_map)); let vec = ConcurrentVec::new(); par.inspect(|u, x| { let u = u.as_mut_str(); u.get_mut(0..2).expect("len>1").make_ascii_uppercase(); _ = vec.push(x.clone()); }) .count(); assert_eq!( sorted(vec.to_vec()), sorted(input().into_iter().filter_map(filter_map).collect()) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn inspect_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let filter = |x: &String| x.ends_with('3'); let flat_map = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let par = input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par .filter_map(make_u_map(filter_map)) .filter(make_u_filter(&filter)) .flat_map(make_u_map(flat_map)); let vec = ConcurrentVec::new(); par.inspect(|u, x| { let u = u.as_mut_str(); u.get_mut(0..2).expect("len>1").make_ascii_uppercase(); _ = vec.push(x.clone()); }) .count(); assert_eq!( sorted(vec.to_vec()), sorted( input() .into_iter() .filter_map(filter_map) .filter(filter) .flat_map(flat_map) .collect() ) ); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/iter_consuming.rs ================================================ use super::utils::make_u_map; use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use orx_fixed_vec::FixedVec; use orx_iterable::Collection; use orx_split_vec::SplitVec; use test_case::test_matrix; const N_OFFSET: usize = 13; fn offset() -> Vec { vec!["x".to_string(); N_OFFSET] } fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn expected( with_offset: bool, input: &impl Collection, map: impl Fn(String) -> String, ) -> Vec { match with_offset { true => { let mut vec = offset(); vec.extend(input.iter().cloned().map(map)); vec } false => input.iter().cloned().map(map).collect(), } } // collect - empty #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0), Vec::::from_iter(offset()), SplitVec::::from_iter(offset()), FixedVec::::from_iter(offset()) ], N, NT, CHUNK) ] fn empty_collect_into(_: I, output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let vec = input::>(n); let expected = expected(!output.is_empty(), &vec, |x| x); let input = vec.into_iter().filter(|x| x.as_str() != "?"); let par = input .iter_into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn empty_collect(_: I, _: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto, { let test = |n, nt, chunk| { let vec = input::>(n); let expected = expected(false, &vec, |x| x); let input = vec.into_iter().filter(|x| x.as_str() != "?"); let par = input .iter_into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: C = par.collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } // collect - map #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0), Vec::::from_iter(offset()), SplitVec::::from_iter(offset()), FixedVec::::from_iter(offset()) ], N, NT, CHUNK) ] fn map_collect_into(_: I, output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let map = |x| format!("{}!", x); let vec = input::>(n); let expected = expected(!output.is_empty(), &vec, map); let input = vec.into_iter().filter(|x| x.as_str() != "?"); let par = input .iter_into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.map(make_u_map(map)).collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn map_collect(_: I, _: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto, { let test = |n, nt, chunk| { let map = |x| format!("{}!", x); let vec = input::>(n); let expected = expected(false, &vec, map); let input = vec.into_iter().filter(|x| x.as_str() != "?"); let par = input .iter_into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: C = par.map(make_u_map(map)).collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/iter_ref.rs ================================================ use super::utils::make_u_map; use crate::{collect_into::ParCollectIntoCore, test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use orx_fixed_vec::FixedVec; use orx_iterable::{Collection, IntoCloningIterable}; use orx_split_vec::{Doubling, Linear, PseudoDefault, SplitVec}; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn expected(input: &impl Collection, map: impl Fn(String) -> String) -> Vec { input.iter().cloned().map(map).collect() } // collect - empty #[test_matrix(N, NT, CHUNK)] fn empty_collect_into(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let vec = input::>(n); let expected = expected(&vec, |x| x); let input = vec.iter().filter(|x| x.as_str() != "?").into_iterable(); let par = input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(Vec::new()); assert!(output.is_equal_to_ref(&expected)); let par = input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(FixedVec::pseudo_default()); assert!(output.is_equal_to_ref(&expected)); let par = input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(SplitVec::<_, Doubling>::pseudo_default()); assert!(output.is_equal_to_ref(&expected)); let par = input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(SplitVec::<_, Linear>::pseudo_default()); assert!(output.is_equal_to_ref(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn empty_collect(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let vec = input::>(n); let expected = expected(&vec, |x| x); let input = vec.iter().filter(|x| x.as_str() != "?").into_iterable(); let par = input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: Vec<&String> = par.collect(); assert!(output.is_equal_to_ref(&expected)); let par = input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: FixedVec<&String> = par.collect(); assert!(output.is_equal_to_ref(&expected)); let par = input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: SplitVec<&String, Doubling> = par.collect(); assert!(output.is_equal_to_ref(&expected)); let par = input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: SplitVec<&String, Linear> = par.collect(); assert!(output.is_equal_to_ref(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } // collect - map #[test_matrix(N, NT, CHUNK)] fn map_collect_into(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let map = |x: &String| format!("{}!", x); let map2 = |x: String| format!("{}!", x); let vec = input::>(n); let expected = expected(&vec, map2); let input = vec.iter().filter(|x| x.as_str() != "?").into_iterable(); let par = input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: Vec = par.map(make_u_map(map)).collect_into(vec![]); assert!(output.is_equal_to(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn map_collect(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let map = |x: &String| format!("{}!", x); let map2 = |x: String| format!("{}!", x); let vec = input::>(n); let expected = expected(&vec, map2); let input = vec.iter().filter(|x| x.as_str() != "?").into_iterable(); let par = input .par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: Vec = par.map(make_u_map(map)).collect(); assert!(output.is_equal_to(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/map/collect.rs ================================================ use super::super::utils::make_u_map; use crate::using::UsingClone; use crate::using::collect_into::collect::map_collect_into; use crate::{IterationOrder, Params, runner::DefaultRunner}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use orx_pinned_vec::PinnedVec; use orx_split_vec::SplitVec; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn m_map_collect(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let offset = 33; let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let map = |x: String| format!("{}!", x); let mut output = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let mut expected = Vec::new(); for i in 0..offset { let value = || map(i.to_string()); output.push(value()); expected.push(value()); } expected.extend(input.clone().into_iter().map(map)); let params = Params::new(nt, chunk, ordering); let iter = input.into_con_iter(); let (_, mut output) = map_collect_into( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_map(map), output, ); if !params.is_sequential() && matches!(params.iteration_order, IterationOrder::Arbitrary) { expected.sort(); output.sort(); } assert_eq!(expected, output.to_vec()); } ================================================ FILE: src/using/computational_variants/tests/map/find.rs ================================================ use crate::using::UsingClone; use crate::using::computational_variants::tests::utils::make_u_map; use crate::{ Params, default_fns::map_self, runner::DefaultRunner, using::executor::parallel_compute, }; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn m_find(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let expected = input.clone().into_iter().next(); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let output = parallel_compute::next::m( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_map(map_self), ) .1; assert_eq!(expected, output); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn m_map_find(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let map = |x: String| format!("{}!", x); let expected = input.clone().into_iter().map(map).next(); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let output = parallel_compute::next::m( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_map(map), ) .1; assert_eq!(expected, output); } ================================================ FILE: src/using/computational_variants/tests/map/mod.rs ================================================ mod collect; mod find; mod reduce; ================================================ FILE: src/using/computational_variants/tests/map/reduce.rs ================================================ use crate::using::UsingClone; use crate::using::computational_variants::tests::utils::{make_u_map, make_u_reduce}; use crate::{ Params, default_fns::map_self, runner::DefaultRunner, using::executor::parallel_compute, }; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn m_reduce(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let reduce = |x: String, y: String| match x < y { true => y, false => x, }; let expected = input.clone().into_iter().reduce(reduce); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let (_, output) = parallel_compute::reduce::m( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_map(map_self), make_u_reduce(reduce), ); assert_eq!(expected, output); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn m_map_reduce(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let map = |x: String| format!("{}!", x); let reduce = |x: String, y: String| match x < y { true => y, false => x, }; let expected = input.clone().into_iter().map(map).reduce(reduce); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let (_, output) = parallel_compute::reduce::m( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_map(map), make_u_reduce(reduce), ); assert_eq!(expected, output); } ================================================ FILE: src/using/computational_variants/tests/min_max.rs ================================================ use crate::using::computational_variants::tests::utils::{make_u_filter, make_u_map}; use crate::{test_utils::*, *}; use alloc::string::{String, ToString}; use alloc::vec::Vec; use core::cmp::Ordering; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn cmp(a: &usize, b: &usize) -> Ordering { match a < b { true => Ordering::Less, false => Ordering::Greater, } } fn key(a: &usize) -> u64 { *a as u64 + 10 } #[test_matrix(N, NT, CHUNK)] fn min_max_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || { input::>(n) .iter() .map(|x| x.parse::().expect("is-ok")) .collect::>() }; let par = || { input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()) }; assert_eq!(par().min(), input().iter().min().copied()); assert_eq!(par().max(), input().iter().max().copied()); assert_eq!(par().min_by(cmp), input().into_iter().min_by(cmp)); assert_eq!(par().max_by(cmp), input().into_iter().max_by(cmp)); assert_eq!(par().min_by_key(key), input().into_iter().min_by_key(key)); assert_eq!(par().max_by_key(key), input().into_iter().max_by_key(key)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn min_max_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let map = |x: String| x.parse::().expect("is-ok"); let par = || { input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()) .map(make_u_map(map)) }; assert_eq!(par().min(), input().into_iter().map(&map).min()); assert_eq!(par().max(), input().into_iter().map(&map).max()); assert_eq!(par().min_by(cmp), input().into_iter().map(&map).min_by(cmp)); assert_eq!(par().max_by(cmp), input().into_iter().map(&map).max_by(cmp)); assert_eq!( par().min_by_key(key), input().into_iter().map(&map).min_by_key(key) ); assert_eq!( par().max_by_key(key), input().into_iter().map(&map).max_by_key(key) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn min_max_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let flat_map = |x: String| { let n = x.len(); let a = x.parse::().expect("is-ok"); (0..n).map(|i| a + i).collect::>() }; let par = || { input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()) .flat_map(make_u_map(flat_map)) }; assert_eq!(par().min(), input().into_iter().flat_map(&flat_map).min()); assert_eq!(par().max(), input().into_iter().flat_map(&flat_map).max()); assert_eq!( par().min_by(cmp), input().into_iter().flat_map(&flat_map).min_by(cmp) ); assert_eq!( par().max_by(cmp), input().into_iter().flat_map(&flat_map).max_by(cmp) ); assert_eq!( par().min_by_key(key), input().into_iter().flat_map(&flat_map).min_by_key(key) ); assert_eq!( par().max_by_key(key), input().into_iter().flat_map(&flat_map).max_by_key(key) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn min_max_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| { x.starts_with('3') .then(|| x.parse::().expect("is-ok")) }; let par = || { input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()) .filter_map(make_u_map(filter_map)) }; assert_eq!( par().min(), input().into_iter().filter_map(&filter_map).min() ); assert_eq!( par().max(), input().into_iter().filter_map(&filter_map).max() ); assert_eq!( par().min_by(cmp), input().into_iter().filter_map(&filter_map).min_by(cmp) ); assert_eq!( par().max_by(cmp), input().into_iter().filter_map(&filter_map).max_by(cmp) ); assert_eq!( par().min_by_key(key), input().into_iter().filter_map(&filter_map).min_by_key(key) ); assert_eq!( par().max_by_key(key), input().into_iter().filter_map(&filter_map).max_by_key(key) ); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn min_max_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = || input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let filter = |x: &String| x.ends_with('3'); let flat_map = |x: String| { let n = x.len(); let a = x.parse::().expect("is-ok"); (0..n).map(|i| a + i).collect::>() }; let par = || { input() .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()) .filter_map(make_u_map(filter_map)) .filter(make_u_filter(&filter)) .flat_map(make_u_map(flat_map)) }; assert_eq!( par().min(), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .min() ); assert_eq!( par().max(), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .max() ); assert_eq!( par().min_by(cmp), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .min_by(cmp) ); assert_eq!( par().max_by(cmp), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .max_by(cmp) ); assert_eq!( par().min_by_key(key), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .min_by_key(key) ); assert_eq!( par().max_by_key(key), input() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .max_by_key(key) ); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/mod.rs ================================================ mod utils; mod copied; mod count; mod fallible_option; mod fallible_result; mod flatten; mod for_each; mod inspect; mod iter_consuming; mod iter_ref; mod map; mod min_max; mod range; mod slice; mod sum; mod vectors; mod xap; ================================================ FILE: src/using/computational_variants/tests/range.rs ================================================ use crate::using::computational_variants::tests::utils::make_u_map; use crate::{test_utils::*, *}; use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use orx_fixed_vec::FixedVec; use orx_iterable::Iterable; use orx_split_vec::SplitVec; use test_case::test_matrix; const N_OFFSET: usize = 13; fn offset() -> Vec { vec![9; N_OFFSET] } fn expected( with_offset: bool, input: impl Iterable, map: impl Fn(usize) -> T + Copy, ) -> Vec { match with_offset { true => { let mut vec: Vec<_> = offset().into_iter().map(map).collect(); vec.extend(input.iter().map(map)); vec } false => input.iter().map(map).collect(), } } // collect - empty #[test_matrix( [Vec::::new(), SplitVec::::new(), FixedVec::::new(0), Vec::::from_iter(offset()), SplitVec::::from_iter(offset()), FixedVec::::from_iter(offset()) ], N, NT, CHUNK) ] fn empty_collect_into(output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let input = 0..n; let expected = expected(!output.is_empty(), input.clone(), |x| x); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn empty_collect(_: C, n: &[usize], nt: &[usize], chunk: &[usize]) where C: ParCollectInto, { let test = |n, nt, chunk| { let input = 0..n; let expected = expected(false, input.clone(), |x| x); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: C = par.collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } // collect - map #[test_matrix( [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn map_collect_into(output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let map = |x: usize| x.to_string(); let input = 0..n; let expected = expected(!output.is_empty(), input.clone(), map); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.map(make_u_map(map)).collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn map_collect(_: C, n: &[usize], nt: &[usize], chunk: &[usize]) where C: ParCollectInto, { let test = |n, nt, chunk| { let map = |x: usize| x.to_string(); let input = 0..n; let expected = expected(false, input.clone(), map); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: C = par.map(make_u_map(map)).collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/slice.rs ================================================ use crate::using::computational_variants::tests::utils::make_u_map; use crate::{collect_into::ParCollectIntoCore, test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use orx_fixed_vec::FixedVec; use orx_iterable::Collection; use orx_split_vec::{Doubling, Linear, PseudoDefault, SplitVec}; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn expected(input: &impl Collection, map: impl Fn(String) -> String) -> Vec { input.iter().cloned().map(map).collect() } // collect - empty #[test_matrix(N, NT, CHUNK)] fn empty_collect_into(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let vec = input::>(n); let input = vec.as_slice(); let expected = expected(&vec, |x| x); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(Vec::new()); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(FixedVec::pseudo_default()); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(SplitVec::<_, Doubling>::pseudo_default()); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(SplitVec::<_, Linear>::pseudo_default()); assert!(output.is_equal_to_ref(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn empty_collect(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let vec = input::>(n); let input = vec.as_slice(); let expected = expected(&vec, |x| x); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: Vec<&String> = par.collect(); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: FixedVec<&String> = par.collect(); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: SplitVec<&String, Doubling> = par.collect(); assert!(output.is_equal_to_ref(expected.as_slice())); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: SplitVec<&String, Linear> = par.collect(); assert!(output.is_equal_to_ref(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } // collect - map #[test_matrix(N, NT, CHUNK)] fn map_collect_into(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let map = |x: &String| format!("{}!", x); let map2 = |x: String| format!("{}!", x); let vec = input::>(n); let input = vec.as_slice(); let expected = expected(&vec, map2); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: Vec = par.map(make_u_map(map)).collect_into(vec![]); assert!(output.is_equal_to(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn map_collect(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let map = |x: &String| format!("{}!", x); let map2 = |x: String| format!("{}!", x); let vec = input::>(n); let input = vec.as_slice(); let expected = expected(&vec, map2); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: Vec = par.map(make_u_map(map)).collect(); assert!(output.is_equal_to(expected.as_slice())); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/sum.rs ================================================ use crate::using::computational_variants::tests::utils::{make_u_filter, make_u_map}; use crate::{test_utils::*, *}; use alloc::string::{String, ToString}; use alloc::vec::Vec; use test_case::test_matrix; fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } #[test_matrix(N, NT, CHUNK)] fn sum_empty(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input: Vec<_> = input::>(n).iter().map(|x| x.len()).collect(); let expected: usize = input.iter().sum(); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn sum_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let map = |x: String| x.len(); let expected: usize = input.clone().into_iter().map(&map).sum(); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.map(make_u_map(map)); let output = par.sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn sum_xap_flat_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let flat_map = |x: String| x.chars().map(|x| x.to_string().len()).collect::>(); let expected: usize = input.clone().into_iter().flat_map(&flat_map).sum(); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.flat_map(make_u_map(flat_map)); let output = par.sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn sum_xap_filter_map(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x.len()); let expected: usize = input.clone().into_iter().filter_map(&filter_map).sum(); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par.filter_map(make_u_map(filter_map)); let output = par.sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix(N, NT, CHUNK)] fn sum_xap_filter_xap(n: &[usize], nt: &[usize], chunk: &[usize]) { let test = |n, nt, chunk| { let input = input::>(n); let filter_map = |x: String| x.starts_with('3').then_some(x); let filter = |x: &String| x.ends_with('3'); let flat_map = |x: String| x.chars().map(|x| x.to_string().len()).collect::>(); let expected: usize = input .clone() .into_iter() .filter_map(&filter_map) .filter(&filter) .flat_map(&flat_map) .sum(); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let par = par .filter_map(make_u_map(filter_map)) .filter(make_u_filter(&filter)) .flat_map(make_u_map(flat_map)); let output = par.sum(); assert_eq!(output, expected); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/utils.rs ================================================ use std::string::String; pub(super) fn make_u_map( map: impl Fn(I) -> O + Clone, ) -> impl Fn(&mut String, I) -> O + Clone { move |u: &mut String, x: I| { let u = u.as_mut_str(); u.get_mut(0..2) .expect("must have len>1") .make_ascii_uppercase(); map(x) } } pub(super) fn make_u_xap( map: impl Fn(I) -> O + Clone, ) -> impl Fn(*mut String, I) -> O + Clone { move |u: *mut String, x: I| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; let u = u.as_mut_str(); u.get_mut(0..2) .expect("must have len>1") .make_ascii_uppercase(); map(x) } } pub(super) fn make_u_filter( filter: &impl Fn(&I) -> bool, ) -> impl Fn(&mut String, &I) -> bool + Clone { |u: &mut String, x: &I| { let u = u.as_mut_str(); u.get_mut(0..2) .expect("must have len>1") .make_ascii_uppercase(); filter(x) } } pub(super) fn make_u_reduce( reduce: impl Fn(I, I) -> I + Clone, ) -> impl Fn(&mut String, I, I) -> I + Clone { move |u: &mut String, x: I, y: I| { let u = u.as_mut_str(); u.get_mut(0..2) .expect("must have len>1") .make_ascii_uppercase(); reduce(x, y) } } ================================================ FILE: src/using/computational_variants/tests/vectors.rs ================================================ use crate::using::computational_variants::tests::utils::make_u_map; use crate::{test_utils::*, *}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use orx_fixed_vec::FixedVec; use orx_iterable::Collection; use orx_split_vec::SplitVec; use test_case::test_matrix; const N_OFFSET: usize = 13; fn offset() -> Vec { vec!["x".to_string(); N_OFFSET] } fn input>(n: usize) -> O { let elem = |x: usize| (x + 10).to_string(); (0..n).map(elem).collect() } fn expected( with_offset: bool, input: &impl Collection, map: impl Fn(String) -> String, ) -> Vec { match with_offset { true => { let mut vec = offset(); vec.extend(input.iter().cloned().map(map)); vec } false => input.iter().cloned().map(map).collect(), } } // collect - empty #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0), Vec::::from_iter(offset()), SplitVec::::from_iter(offset()), FixedVec::::from_iter(offset()) ], N, NT, CHUNK) ] fn empty_collect_into(_: I, output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let input = input::>(n); let expected = expected(!output.is_empty(), &input, |x| x); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn empty_collect(_: I, _: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto, { let test = |n, nt, chunk| { let input = input::>(n); let expected = expected(false, &input, |x| x); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: C = par.collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } // collect - map #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0), Vec::::from_iter(offset()), SplitVec::::from_iter(offset()), FixedVec::::from_iter(offset()) ], N, NT, CHUNK) ] fn map_collect_into(_: I, output: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto + Clone, { let test = |n, nt, chunk| { let map = |x| format!("{}!", x); let input = input::>(n); let expected = expected(!output.is_empty(), &input, map); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output = par.map(make_u_map(map)).collect_into(output.clone()); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } #[test_matrix( [Vec::::new()], [Vec::::new(), SplitVec::::new(), FixedVec::::new(0)], N, NT, CHUNK) ] fn map_collect(_: I, _: C, n: &[usize], nt: &[usize], chunk: &[usize]) where I: FromIterator + Collection + IntoParIter, C: ParCollectInto, { let test = |n, nt, chunk| { let map = |x| format!("{}!", x); let input = input::>(n); let expected = expected(false, &input, map); let par = input .into_par() .num_threads(nt) .chunk_size(chunk) .using_clone("XyZw".to_string()); let output: C = par.map(make_u_map(map)).collect(); assert!(output.is_equal_to(&expected)); }; test_n_nt_chunk(n, nt, chunk, test); } ================================================ FILE: src/using/computational_variants/tests/xap/collect.rs ================================================ use crate::generic_values::Vector; use crate::runner::DefaultRunner; use crate::using::UsingClone; use crate::using::computational_variants::UParXap; use crate::using::computational_variants::tests::utils::make_u_xap; use crate::using::u_par_iter::ParIterUsing; use crate::{IterationOrder, Params}; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use orx_pinned_vec::PinnedVec; use orx_split_vec::SplitVec; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test] fn todo_panic_at_con_bag_new() { // TODO: this code panics when ParThreadPool::map uses ConcurrentBag::new rather than ConcurrentBag::with_fixed_capacity let n = 10; let nt = 2; let chunk = 1; let ordering = IterationOrder::Arbitrary; let offset = 33; let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let xmap = |x: String| Vector(fmap(x)); let mut output = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let mut expected = Vec::new(); for i in 0..offset { let i = i.to_string(); for x in fmap(i) { output.push(x.clone()); expected.push(x); } } expected.extend(input.clone().into_iter().flat_map(&fmap)); let params = Params::new(nt, chunk, ordering); let iter = input.into_con_iter(); let x = UParXap::new( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_xap(xmap), ); let mut output = x.collect_into(output); if !params.is_sequential() && matches!(params.iteration_order, IterationOrder::Arbitrary) { expected.sort(); output.sort(); } assert_eq!(expected, output.to_vec()); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn x_flat_map_collect(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let offset = 33; let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let xmap = |x: String| Vector(fmap(x)); let mut output = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let mut expected = Vec::new(); for i in 0..offset { let i = i.to_string(); for x in fmap(i) { output.push(x.clone()); expected.push(x); } } expected.extend(input.clone().into_iter().flat_map(&fmap)); let params = Params::new(nt, chunk, ordering); let iter = input.into_con_iter(); let x = UParXap::new( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_xap(xmap), ); let mut output = x.collect_into(output); if !params.is_sequential() && matches!(params.iteration_order, IterationOrder::Arbitrary) { expected.sort(); output.sort(); } assert_eq!(expected, output.to_vec()); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64], [IterationOrder::Ordered, IterationOrder::Arbitrary]) ] fn x_filter_map_collect(n: usize, nt: usize, chunk: usize, ordering: IterationOrder) { let offset = 33; let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| (!x.starts_with('3')).then_some(format!("{}!", x)); let xmap = |x: String| Vector(fmap(x)); let mut output = SplitVec::with_doubling_growth_and_max_concurrent_capacity(); let mut expected = Vec::new(); for i in 0..offset { let i = i.to_string(); if let Some(x) = fmap(i) { output.push(x.clone()); expected.push(x); } } expected.extend(input.clone().into_iter().flat_map(&fmap)); let params = Params::new(nt, chunk, ordering); let iter = input.into_con_iter(); let x = UParXap::new( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_xap(xmap), ); let mut output = x.collect_into(output); if !params.is_sequential() && matches!(params.iteration_order, IterationOrder::Arbitrary) { expected.sort(); output.sort(); } assert_eq!(expected, output.to_vec()); } ================================================ FILE: src/using/computational_variants/tests/xap/find.rs ================================================ use crate::ParIterUsing; use crate::Params; use crate::generic_values::Vector; use crate::runner::DefaultRunner; use crate::using::UsingClone; use crate::using::computational_variants::UParXap; use crate::using::computational_variants::tests::utils::make_u_xap; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn x_flat_map_find(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let xmap = |x: String| Vector(fmap(x)); let expected = input.clone().into_iter().flat_map(fmap).next(); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let x = UParXap::new( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_xap(xmap), ); let output = x.first(); assert_eq!(expected, output); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn x_filter_map_find(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| (!x.starts_with('3')).then_some(format!("{}!", x)); let xmap = |x: String| Vector(fmap(x)); let expected = input.clone().into_iter().filter_map(fmap).next(); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let x = UParXap::new( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_xap(xmap), ); let output = x.first(); assert_eq!(expected, output); } ================================================ FILE: src/using/computational_variants/tests/xap/mod.rs ================================================ mod collect; mod find; mod reduce; ================================================ FILE: src/using/computational_variants/tests/xap/reduce.rs ================================================ use crate::ParIterUsing; use crate::Params; use crate::generic_values::Vector; use crate::runner::DefaultRunner; use crate::using::UsingClone; use crate::using::computational_variants::UParXap; use crate::using::computational_variants::tests::utils::make_u_reduce; use crate::using::computational_variants::tests::utils::make_u_xap; use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use orx_concurrent_iter::IntoConcurrentIter; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn x_flat_map_reduce(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| x.chars().map(|x| x.to_string()).collect::>(); let xmap = |x: String| Vector(fmap(x)); let reduce = |x: String, y: String| match x > y { true => x, false => y, }; let expected = input.clone().into_iter().flat_map(fmap).reduce(reduce); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let x = UParXap::new( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_xap(xmap), ); let output = x.reduce(make_u_reduce(reduce)); assert_eq!(expected, output); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn x_filter_map_reduce(n: usize, nt: usize, chunk: usize) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let fmap = |x: String| (!x.starts_with('3')).then_some(format!("{}!", x)); let xmap = |x: String| Vector(fmap(x)); let reduce = |x: String, y: String| match x > y { true => x, false => y, }; let expected = input.clone().into_iter().filter_map(fmap).reduce(reduce); let params = Params::new(nt, chunk, Default::default()); let iter = input.into_con_iter(); let x = UParXap::new( UsingClone::new("XyZw".to_string()), DefaultRunner::default(), params, iter, make_u_xap(xmap), ); let output = x.reduce(make_u_reduce(reduce)); assert_eq!(expected, output); } ================================================ FILE: src/using/computational_variants/u_fallible_option.rs ================================================ use crate::{ ChunkSize, IterationOrder, NumThreads, ParCollectInto, par_iter_option::ResultIntoOption, runner::{DefaultRunner, ParallelRunner}, using::{ParIterOptionUsing, ParIterResultUsing, Using}, }; use core::marker::PhantomData; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with None. pub struct UParOption<'using, U, F, T, R = DefaultRunner> where R: ParallelRunner, F: ParIterResultUsing<'using, U, R, Item = T, Err = ()>, U: Using<'using>, { par: F, phantom: PhantomData<(&'using (), U, T, R)>, } impl<'using, U, F, T, R> UParOption<'using, U, F, T, R> where R: ParallelRunner, F: ParIterResultUsing<'using, U, R, Item = T, Err = ()>, U: Using<'using>, { pub(crate) fn new(par: F) -> Self { Self { par, phantom: PhantomData, } } } impl<'using, U, F, T, R> ParIterOptionUsing<'using, U, R> for UParOption<'using, U, F, T, R> where R: ParallelRunner, F: ParIterResultUsing<'using, U, R, Item = T, Err = ()>, U: Using<'using>, { type Item = T; // params transformations fn num_threads(self, num_threads: impl Into) -> Self { Self::new(self.par.num_threads(num_threads)) } fn chunk_size(self, chunk_size: impl Into) -> Self { Self::new(self.par.chunk_size(chunk_size)) } fn iteration_order(self, order: IterationOrder) -> Self { Self::new(self.par.iteration_order(order)) } fn with_runner( self, orchestrator: Q, ) -> impl ParIterOptionUsing<'using, U, Q, Item = Self::Item> { UParOption::::new(self.par.with_runner(orchestrator)) } // computation transformations fn map(self, map: Map) -> impl ParIterOptionUsing<'using, U, R, Item = Out> where Map: Fn(&mut U::Item, Self::Item) -> Out + Sync + Clone, Out: Send, { UParOption::::new(self.par.map(map)) } fn filter( self, filter: Filter, ) -> impl ParIterOptionUsing<'using, U, R, Item = Self::Item> where Self: Sized, Filter: Fn(&mut U::Item, &Self::Item) -> bool + Sync + Clone, Self::Item: Send, { UParOption::::new(self.par.filter(filter)) } fn flat_map( self, flat_map: FlatMap, ) -> impl ParIterOptionUsing<'using, U, R, Item = IOut::Item> where Self: Sized, IOut: IntoIterator, IOut::Item: Send, FlatMap: Fn(&mut U::Item, Self::Item) -> IOut + Sync + Clone, { UParOption::::new(self.par.flat_map(flat_map)) } fn filter_map( self, filter_map: FilterMap, ) -> impl ParIterOptionUsing<'using, U, R, Item = Out> where Self: Sized, FilterMap: Fn(&mut U::Item, Self::Item) -> Option + Sync + Clone, Out: Send, { UParOption::::new(self.par.filter_map(filter_map)) } fn inspect( self, operation: Operation, ) -> impl ParIterOptionUsing<'using, U, R, Item = Self::Item> where Self: Sized, Operation: Fn(&mut U::Item, &Self::Item) + Sync + Clone, Self::Item: Send, { UParOption::::new(self.par.inspect(operation)) } // collect fn collect_into(self, output: C) -> Option where Self::Item: Send, C: ParCollectInto, { self.par.collect_into(output).into_option() } fn collect(self) -> Option where Self::Item: Send, C: ParCollectInto, { self.par.collect().into_option() } // reduce fn reduce(self, reduce: Reduce) -> Option> where Self::Item: Send, Reduce: Fn(&mut U::Item, Self::Item, Self::Item) -> Self::Item + Sync, { self.par.reduce(reduce).into_option() } // early exit fn first(self) -> Option> where Self::Item: Send, { self.par.first().into_option() } } ================================================ FILE: src/using/computational_variants/u_fallible_result/mod.rs ================================================ mod u_map_result; mod u_par_result; mod u_xap_result; pub use u_map_result::UParMapResult; pub use u_par_result::UParResult; pub use u_xap_result::UParXapResult; ================================================ FILE: src/using/computational_variants/u_fallible_result/u_map_result.rs ================================================ use crate::par_iter_result::IntoResult; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::executor::parallel_compute as prc; use crate::using::{ParIterResultUsing, UParMap, Using}; use crate::{IterationOrder, ParCollectInto, ParIterUsing}; use core::marker::PhantomData; use orx_concurrent_iter::ConcurrentIter; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with an error. pub struct UParMapResult<'using, U, I, T, E, O, M1, R = DefaultRunner> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, O: IntoResult, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { par: UParMap<'using, U, I, O, M1, R>, phantom: PhantomData<(T, E)>, } impl<'using, U, I, T, E, O, M1, R> UParMapResult<'using, U, I, T, E, O, M1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, O: IntoResult, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { pub(crate) fn new(par: UParMap<'using, U, I, O, M1, R>) -> Self { Self { par, phantom: PhantomData, } } } impl<'using, U, I, T, E, O, M1, R> ParIterResultUsing<'using, U, R> for UParMapResult<'using, U, I, T, E, O, M1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, O: IntoResult, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { type Item = T; type Err = E; type RegularItem = O; type RegularParIter = UParMap<'using, U, I, O, M1, R>; fn con_iter_len(&self) -> Option { self.par.con_iter().try_get_len() } fn into_regular_par(self) -> Self::RegularParIter { self.par } fn from_regular_par(regular_par: Self::RegularParIter) -> Self { Self { par: regular_par, phantom: PhantomData, } } // params transformations fn with_runner( self, orchestrator: Q, ) -> impl ParIterResultUsing<'using, U, Q, Item = Self::Item, Err = Self::Err> { let (using, _, params, iter, m1) = self.par.destruct(); UParMapResult { par: UParMap::new(using, orchestrator, params, iter, m1), phantom: PhantomData, } } // collect fn collect_into(self, output: C) -> Result where C: ParCollectInto, Self::Item: Send, Self::Err: Send, { let (using, orchestrator, params, iter, m1) = self.par.destruct(); let x1 = |u: *mut U::Item, i: I::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; m1(u, i).into_result() }; output.u_x_try_collect_into(using, orchestrator, params, iter, x1) } // reduce fn reduce(self, reduce: Reduce) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, Reduce: Fn(&mut U::Item, Self::Item, Self::Item) -> Self::Item + Sync, { let (using, orchestrator, params, iter, m1) = self.par.destruct(); let x1 = |u: *mut U::Item, i: I::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; m1(u, i).into_result() }; let reduce = move |u: *mut U::Item, a: Self::Item, b: Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; reduce(u, a, b) }; prc::reduce::x(using, orchestrator, params, iter, x1, reduce).1 } // early exit fn first(self) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, { let (using, orchestrator, params, iter, m1) = self.par.destruct(); let x1 = |u: *mut U::Item, i: I::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; m1(u, i).into_result() }; match params.iteration_order { IterationOrder::Ordered => { let (_, result) = prc::next::x(using, orchestrator, params, iter, x1); result.map(|x: Option<(usize, T)>| x.map(|y| y.1)) } IterationOrder::Arbitrary => { let (_, result) = prc::next_any::x(using, orchestrator, params, iter, x1); result } } } } ================================================ FILE: src/using/computational_variants/u_fallible_result/u_par_result.rs ================================================ use crate::par_iter_result::IntoResult; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::executor::parallel_compute as prc; use crate::using::{ParIterResultUsing, UPar, Using}; use crate::{IterationOrder, ParCollectInto, ParIterUsing}; use core::marker::PhantomData; use orx_concurrent_iter::ConcurrentIter; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with an error. pub struct UParResult<'using, U, I, T, E, R = DefaultRunner> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, I::Item: IntoResult, { par: UPar<'using, U, I, R>, phantom: PhantomData<(T, E)>, } impl<'using, U, I, T, E, R> UParResult<'using, U, I, T, E, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, I::Item: IntoResult, { pub(crate) fn new(par: UPar<'using, U, I, R>) -> Self { Self { par, phantom: PhantomData, } } } impl<'using, U, I, T, E, R> ParIterResultUsing<'using, U, R> for UParResult<'using, U, I, T, E, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, I::Item: IntoResult, { type Item = T; type Err = E; type RegularItem = I::Item; type RegularParIter = UPar<'using, U, I, R>; fn con_iter_len(&self) -> Option { self.par.con_iter().try_get_len() } fn into_regular_par(self) -> Self::RegularParIter { self.par } fn from_regular_par(regular_par: Self::RegularParIter) -> Self { Self { par: regular_par, phantom: PhantomData, } } // params transformations fn with_runner( self, orchestrator: Q, ) -> impl ParIterResultUsing<'using, U, Q, Item = Self::Item, Err = Self::Err> { let (using, _, params, iter) = self.par.destruct(); UParResult { par: UPar::new(using, orchestrator, params, iter), phantom: PhantomData, } } // collect fn collect_into(self, output: C) -> Result where C: ParCollectInto, Self::Item: Send, Self::Err: Send, { let (using, orchestrator, params, iter) = self.par.destruct(); let x1 = |_: *mut U::Item, i: I::Item| i.into_result(); output.u_x_try_collect_into(using, orchestrator, params, iter, x1) } // reduce fn reduce(self, reduce: Reduce) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, Reduce: Fn(&mut U::Item, Self::Item, Self::Item) -> Self::Item + Sync, { let (using, orchestrator, params, iter) = self.par.destruct(); let x1 = |_: *mut U::Item, i: I::Item| i.into_result(); let reduce = move |u: *mut U::Item, a: Self::Item, b: Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; reduce(u, a, b) }; prc::reduce::x(using, orchestrator, params, iter, x1, reduce).1 } // early exit fn first(self) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, { let (using, orchestrator, params, iter) = self.par.destruct(); let x1 = |_: *mut U::Item, i: I::Item| i.into_result(); match params.iteration_order { IterationOrder::Ordered => { let (_, result) = prc::next::x(using, orchestrator, params, iter, x1); result.map(|x: Option<(usize, T)>| x.map(|y| y.1)) } IterationOrder::Arbitrary => { let (_, result) = prc::next_any::x(using, orchestrator, params, iter, x1); result } } } } ================================================ FILE: src/using/computational_variants/u_fallible_result/u_xap_result.rs ================================================ use crate::generic_values::TransformableValues; use crate::generic_values::runner_results::Infallible; use crate::par_iter_result::IntoResult; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::executor::parallel_compute as prc; use crate::using::{ParIterResultUsing, UParXap, Using}; use crate::{IterationOrder, ParCollectInto, Params}; use core::marker::PhantomData; use orx_concurrent_iter::ConcurrentIter; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with an error. pub struct UParXapResult<'using, U, I, T, E, Vo, X1, R = DefaultRunner> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, Vo::Item: IntoResult, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { using: U, orchestrator: R, params: Params, iter: I, xap1: X1, phantom: PhantomData<(&'using (), T, E)>, } impl<'using, U, I, T, E, Vo, X1, R> UParXapResult<'using, U, I, T, E, Vo, X1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, Vo::Item: IntoResult, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { pub(crate) fn new(using: U, orchestrator: R, params: Params, iter: I, xap1: X1) -> Self { Self { using, orchestrator, params, iter, xap1, phantom: PhantomData, } } fn destruct(self) -> (U, R, Params, I, X1) { ( self.using, self.orchestrator, self.params, self.iter, self.xap1, ) } } impl<'using, U, I, T, E, Vo, X1, R> ParIterResultUsing<'using, U, R> for UParXapResult<'using, U, I, T, E, Vo, X1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, Vo::Item: IntoResult, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { type Item = T; type Err = E; type RegularItem = Vo::Item; type RegularParIter = UParXap<'using, U, I, Vo, X1, R>; fn con_iter_len(&self) -> Option { self.iter.try_get_len() } fn into_regular_par(self) -> Self::RegularParIter { let (using, orchestrator, params, iter, x1) = self.destruct(); UParXap::new(using, orchestrator, params, iter, x1) } fn from_regular_par(regular_par: Self::RegularParIter) -> Self { let (using, orchestrator, params, iter, x1) = regular_par.destruct(); Self::new(using, orchestrator, params, iter, x1) } // params transformations fn with_runner( self, orchestrator: Q, ) -> impl ParIterResultUsing<'using, U, Q, Item = Self::Item, Err = Self::Err> { let (using, _, params, iter, x1) = self.destruct(); UParXapResult::new(using, orchestrator, params, iter, x1) } // collect fn collect_into(self, output: C) -> Result where C: ParCollectInto, Self::Item: Send, Self::Err: Send, { let (using, orchestrator, params, iter, x1) = self.destruct(); let x1 = |u: *mut U::Item, i: I::Item| x1(u, i).map_while_ok(|x| x.into_result()); output.u_x_try_collect_into(using, orchestrator, params, iter, x1) } // reduce fn reduce(self, reduce: Reduce) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, Reduce: Fn(&mut U::Item, Self::Item, Self::Item) -> Self::Item + Sync, { let (using, orchestrator, params, iter, x1) = self.destruct(); let x1 = |u: *mut U::Item, i: I::Item| x1(u, i).map_while_ok(|x| x.into_result()); let reduce = move |u: *mut U::Item, a: Self::Item, b: Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; reduce(u, a, b) }; prc::reduce::x(using, orchestrator, params, iter, x1, reduce).1 } // early exit fn first(self) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, { let (using, orchestrator, params, iter, x1) = self.destruct(); let x1 = |u: *mut U::Item, i: I::Item| x1(u, i).map_while_ok(|x| x.into_result()); match params.iteration_order { IterationOrder::Ordered => { let (_, result) = prc::next::x(using, orchestrator, params, iter, x1); result.map(|x: Option<(usize, T)>| x.map(|y| y.1)) } IterationOrder::Arbitrary => { let (_, result) = prc::next_any::x(using, orchestrator, params, iter, x1); result } } } } ================================================ FILE: src/using/computational_variants/u_map.rs ================================================ use core::marker::PhantomData; use crate::ParIterUsing; use crate::generic_values::Vector; use crate::par_iter_result::IntoResult; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::ParIterResultUsing; use crate::using::computational_variants::u_fallible_result::UParMapResult; use crate::using::computational_variants::u_xap::UParXap; use crate::using::executor::parallel_compute as prc; use crate::using::using_variants::Using; use crate::{ChunkSize, IterationOrder, NumThreads, ParCollectInto, Params}; use orx_concurrent_iter::ConcurrentIter; /// A parallel iterator that maps inputs. pub struct UParMap<'using, U, I, O, M1, R = DefaultRunner> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { using: U, orchestrator: R, params: Params, iter: I, map1: M1, phantom: PhantomData<&'using ()>, } impl<'using, U, I, O, M1, R> UParMap<'using, U, I, O, M1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { pub(crate) fn new(using: U, orchestrator: R, params: Params, iter: I, map1: M1) -> Self { Self { using, orchestrator, params, iter, map1, phantom: PhantomData, } } pub(crate) fn destruct(self) -> (U, R, Params, I, M1) { ( self.using, self.orchestrator, self.params, self.iter, self.map1, ) } } unsafe impl<'using, U, I, O, M1, R> Send for UParMap<'using, U, I, O, M1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { } unsafe impl<'using, U, I, O, M1, R> Sync for UParMap<'using, U, I, O, M1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { } impl<'using, U, I, O, M1, R> ParIterUsing<'using, U, R> for UParMap<'using, U, I, O, M1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { type Item = O; fn con_iter(&self) -> &impl ConcurrentIter { &self.iter } fn params(&self) -> Params { self.params } fn num_threads(mut self, num_threads: impl Into) -> Self { self.params = self.params.with_num_threads(num_threads); self } fn chunk_size(mut self, chunk_size: impl Into) -> Self { self.params = self.params.with_chunk_size(chunk_size); self } fn iteration_order(mut self, collect: IterationOrder) -> Self { self.params = self.params.with_collect_ordering(collect); self } fn with_runner( self, orchestrator: Q, ) -> impl ParIterUsing<'using, U, Q, Item = Self::Item> { let (using, _, params, iter, x1) = self.destruct(); UParMap::new(using, orchestrator, params, iter, x1) } fn map(self, map: Map) -> impl ParIterUsing<'using, U, R, Item = Out> where Map: Fn(&mut U::Item, Self::Item) -> Out + Sync + Clone, { let (using, orchestrator, params, iter, m1) = self.destruct(); let m1 = move |u: &mut U::Item, x: I::Item| { let v1 = m1(u, x); map(u, v1) }; UParMap::new(using, orchestrator, params, iter, m1) } fn filter(self, filter: Filter) -> impl ParIterUsing<'using, U, R, Item = Self::Item> where Filter: Fn(&mut U::Item, &Self::Item) -> bool + Sync + Clone, { let (using, orchestrator, params, iter, m1) = self.destruct(); let x1 = move |u: *mut U::Item, i: I::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; let value = m1(u, i); filter(u, &value).then_some(value) }; UParXap::new(using, orchestrator, params, iter, x1) } fn flat_map( self, flat_map: FlatMap, ) -> impl ParIterUsing<'using, U, R, Item = IOut::Item> where IOut: IntoIterator, FlatMap: Fn(&mut U::Item, Self::Item) -> IOut + Sync + Clone, { let (using, orchestrator, params, iter, m1) = self.destruct(); let x1 = move |u: *mut U::Item, i: I::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; let a = m1(u, i); Vector(flat_map(u, a)) }; UParXap::new(using, orchestrator, params, iter, x1) } fn filter_map( self, filter_map: FilterMap, ) -> impl ParIterUsing<'using, U, R, Item = Out> where FilterMap: Fn(&mut U::Item, Self::Item) -> Option + Sync + Clone, { let (using, orchestrator, params, iter, m1) = self.destruct(); let x1 = move |u: *mut U::Item, i: I::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; let a = m1(u, i); filter_map(u, a) }; UParXap::new(using, orchestrator, params, iter, x1) } fn into_fallible_result( self, ) -> impl ParIterResultUsing<'using, U, R, Item = Out, Err = Err> where Self::Item: IntoResult, { UParMapResult::new(self) } fn collect_into(self, output: C) -> C where C: ParCollectInto, { let (using, orchestrator, params, iter, m1) = self.destruct(); output.u_m_collect_into(using, orchestrator, params, iter, m1) } fn reduce(self, reduce: Reduce) -> Option where Self::Item: Send, Reduce: Fn(&mut U::Item, Self::Item, Self::Item) -> Self::Item + Sync, { let (using, orchestrator, params, iter, m1) = self.destruct(); prc::reduce::m(using, orchestrator, params, iter, m1, reduce).1 } fn first(self) -> Option where Self::Item: Send, { let (using, orchestrator, params, iter, m1) = self.destruct(); match params.iteration_order { IterationOrder::Ordered => prc::next::m(using, orchestrator, params, iter, m1).1, IterationOrder::Arbitrary => prc::next_any::m(using, orchestrator, params, iter, m1).1, } } } ================================================ FILE: src/using/computational_variants/u_par.rs ================================================ use core::marker::PhantomData; use crate::ParIterUsing; use crate::default_fns::u_map_self; use crate::generic_values::Vector; use crate::par_iter_result::IntoResult; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::ParIterResultUsing; use crate::using::computational_variants::u_fallible_result::UParResult; use crate::using::computational_variants::u_map::UParMap; use crate::using::computational_variants::u_xap::UParXap; use crate::using::executor::parallel_compute as prc; use crate::using::using_variants::Using; use crate::{ChunkSize, IterationOrder, NumThreads, ParCollectInto, Params}; use orx_concurrent_iter::ConcurrentIter; /// A parallel iterator. pub struct UPar<'using, U, I, R = DefaultRunner> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, { using: U, orchestrator: R, params: Params, iter: I, phantom: PhantomData<&'using ()>, } impl<'using, U, I, R> UPar<'using, U, I, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, { pub(crate) fn new(using: U, orchestrator: R, params: Params, iter: I) -> Self { Self { using, orchestrator, params, iter, phantom: PhantomData, } } pub(crate) fn destruct(self) -> (U, R, Params, I) { (self.using, self.orchestrator, self.params, self.iter) } } unsafe impl<'using, U, I, R> Send for UPar<'using, U, I, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, { } unsafe impl<'using, U, I, R> Sync for UPar<'using, U, I, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, { } impl<'using, U, I, R> ParIterUsing<'using, U, R> for UPar<'using, U, I, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, { type Item = I::Item; fn con_iter(&self) -> &impl ConcurrentIter { &self.iter } fn params(&self) -> Params { self.params } fn num_threads(mut self, num_threads: impl Into) -> Self { self.params = self.params.with_num_threads(num_threads); self } fn chunk_size(mut self, chunk_size: impl Into) -> Self { self.params = self.params.with_chunk_size(chunk_size); self } fn iteration_order(mut self, collect: IterationOrder) -> Self { self.params = self.params.with_collect_ordering(collect); self } fn with_runner( self, orchestrator: Q, ) -> impl ParIterUsing<'using, U, Q, Item = Self::Item> { let (using, _, params, iter) = self.destruct(); UPar::new(using, orchestrator, params, iter) } fn map(self, map: Map) -> impl ParIterUsing<'using, U, R, Item = Out> where Map: Fn(&mut U::Item, Self::Item) -> Out + Sync + Clone, { let (using, orchestrator, params, iter) = self.destruct(); UParMap::new(using, orchestrator, params, iter, map) } fn filter(self, filter: Filter) -> impl ParIterUsing<'using, U, R, Item = Self::Item> where Filter: Fn(&mut U::Item, &Self::Item) -> bool + Sync + Clone, { let (using, orchestrator, params, iter) = self.destruct(); let x1 = move |u: *mut U::Item, i: Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; filter(u, &i).then_some(i) }; UParXap::new(using, orchestrator, params, iter, x1) } fn flat_map( self, flat_map: FlatMap, ) -> impl ParIterUsing<'using, U, R, Item = IOut::Item> where IOut: IntoIterator, FlatMap: Fn(&mut U::Item, Self::Item) -> IOut + Sync + Clone, { let (using, orchestrator, params, iter) = self.destruct(); let x1 = move |u: *mut U::Item, i: Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; Vector(flat_map(u, i)) }; UParXap::new(using, orchestrator, params, iter, x1) } fn filter_map( self, filter_map: FilterMap, ) -> impl ParIterUsing<'using, U, R, Item = Out> where FilterMap: Fn(&mut U::Item, Self::Item) -> Option + Sync + Clone, { let (using, orchestrator, params, iter) = self.destruct(); let x1 = move |u: *mut U::Item, i: Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; filter_map(u, i) }; UParXap::new(using, orchestrator, params, iter, x1) } fn into_fallible_result( self, ) -> impl ParIterResultUsing<'using, U, R, Item = Out, Err = Err> where Self::Item: IntoResult, { UParResult::new(self) } fn collect_into(self, output: C) -> C where C: ParCollectInto, { let (using, orchestrator, params, iter) = self.destruct(); output.u_m_collect_into(using, orchestrator, params, iter, u_map_self) } fn reduce(self, reduce: Reduce) -> Option where Self::Item: Send, Reduce: Fn(&mut U::Item, Self::Item, Self::Item) -> Self::Item + Sync, { let (using, orchestrator, params, iter) = self.destruct(); prc::reduce::m(using, orchestrator, params, iter, u_map_self, reduce).1 } fn first(self) -> Option where Self::Item: Send, { let (using, orchestrator, params, iter) = self.destruct(); match params.iteration_order { IterationOrder::Ordered => { prc::next::m(using, orchestrator, params, iter, u_map_self).1 } IterationOrder::Arbitrary => { prc::next_any::m(using, orchestrator, params, iter, u_map_self).1 } } } } ================================================ FILE: src/using/computational_variants/u_xap.rs ================================================ use core::marker::PhantomData; use crate::par_iter_result::IntoResult; use crate::using::ParIterResultUsing; use crate::using::computational_variants::u_fallible_result::UParXapResult; use crate::using::executor::parallel_compute as prc; use crate::{ ChunkSize, IterationOrder, NumThreads, ParCollectInto, ParIterUsing, Params, generic_values::{TransformableValues, runner_results::Infallible}, runner::{DefaultRunner, ParallelRunner}, using::using_variants::Using, }; use orx_concurrent_iter::ConcurrentIter; // use crate::runner::parallel_runner_compute as prc; pub struct UParXap<'using, U, I, Vo, X1, R = DefaultRunner> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { using: U, orchestrator: R, params: Params, iter: I, xap1: X1, phantom: PhantomData<&'using ()>, } impl<'using, U, I, Vo, X1, R> UParXap<'using, U, I, Vo, X1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { pub(crate) fn new(using: U, orchestrator: R, params: Params, iter: I, xap1: X1) -> Self { Self { using, orchestrator, params, iter, xap1, phantom: PhantomData, } } pub(crate) fn destruct(self) -> (U, R, Params, I, X1) { ( self.using, self.orchestrator, self.params, self.iter, self.xap1, ) } } unsafe impl<'using, U, I, Vo, X1, R> Send for UParXap<'using, U, I, Vo, X1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { } unsafe impl<'using, U, I, Vo, X1, R> Sync for UParXap<'using, U, I, Vo, X1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { } impl<'using, U, I, Vo, X1, R> ParIterUsing<'using, U, R> for UParXap<'using, U, I, Vo, X1, R> where U: Using<'using>, R: ParallelRunner, I: ConcurrentIter, Vo: TransformableValues, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { type Item = Vo::Item; fn con_iter(&self) -> &impl ConcurrentIter { &self.iter } fn params(&self) -> Params { self.params } fn num_threads(mut self, num_threads: impl Into) -> Self { self.params = self.params.with_num_threads(num_threads); self } fn chunk_size(mut self, chunk_size: impl Into) -> Self { self.params = self.params.with_chunk_size(chunk_size); self } fn iteration_order(mut self, collect: IterationOrder) -> Self { self.params = self.params.with_collect_ordering(collect); self } fn with_runner( self, orchestrator: Q, ) -> impl ParIterUsing<'using, U, Q, Item = Self::Item> { let (using, _, params, iter, x1) = self.destruct(); UParXap::new(using, orchestrator, params, iter, x1) } fn map(self, map: Map) -> impl ParIterUsing<'using, U, R, Item = Out> where Map: Fn(&mut U::Item, Self::Item) -> Out + Sync + Clone, { let (using, orchestrator, params, iter, x1) = self.destruct(); let map = move |u: *mut U::Item, i: Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; map(u, i) }; let x1 = move |u: *mut U::Item, i: I::Item| { let vo = x1(u, i); vo.u_map(u, map.clone()) }; UParXap::new(using, orchestrator, params, iter, x1) } fn filter(self, filter: Filter) -> impl ParIterUsing<'using, U, R, Item = Self::Item> where Filter: Fn(&mut U::Item, &Self::Item) -> bool + Sync + Clone, { let (using, orchestrator, params, iter, x1) = self.destruct(); let filter = move |u: *mut U::Item, i: &Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; filter(u, i) }; let x1 = move |u: *mut U::Item, i: I::Item| { let vo = x1(u, i); vo.u_filter(u, filter.clone()) }; UParXap::new(using, orchestrator, params, iter, x1) } fn flat_map( self, flat_map: FlatMap, ) -> impl ParIterUsing<'using, U, R, Item = IOut::Item> where IOut: IntoIterator, FlatMap: Fn(&mut U::Item, Self::Item) -> IOut + Sync + Clone, { let (using, orchestrator, params, iter, x1) = self.destruct(); let flat_map = move |u: *mut U::Item, i: Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; flat_map(u, i) }; let x1 = move |u: *mut U::Item, i: I::Item| { let vo = x1(u, i); vo.u_flat_map(u, flat_map.clone()) }; UParXap::new(using, orchestrator, params, iter, x1) } fn filter_map( self, filter_map: FilterMap, ) -> impl ParIterUsing<'using, U, R, Item = Out> where FilterMap: Fn(&mut U::Item, Self::Item) -> Option + Sync + Clone, { let (using, orchestrator, params, iter, x1) = self.destruct(); let filter_map = move |u: *mut U::Item, i: Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; filter_map(u, i) }; let x1 = move |u: *mut U::Item, i: I::Item| { let vo = x1(u, i); vo.u_filter_map(u, filter_map.clone()) }; UParXap::new(using, orchestrator, params, iter, x1) } fn into_fallible_result( self, ) -> impl ParIterResultUsing<'using, U, R, Item = Out, Err = Err> where Self::Item: IntoResult, { let (using, orchestrator, params, iter, x1) = self.destruct(); UParXapResult::new(using, orchestrator, params, iter, x1) } fn collect_into(self, output: C) -> C where C: ParCollectInto, { let (using, orchestrator, params, iter, x1) = self.destruct(); output.u_x_collect_into(using, orchestrator, params, iter, x1) } fn reduce(self, reduce: Reduce) -> Option where Self::Item: Send, Reduce: Fn(&mut U::Item, Self::Item, Self::Item) -> Self::Item + Sync, { let (using, orchestrator, params, iter, x1) = self.destruct(); let reduce = move |u: *mut U::Item, a: Self::Item, b: Self::Item| { // SAFETY: TODO-USING let u = unsafe { &mut *u }; reduce(u, a, b) }; let (_, Ok(acc)) = prc::reduce::x(using, orchestrator, params, iter, x1, reduce); acc } fn first(self) -> Option where Self::Item: Send, { let (using, orchestrator, params, iter, x1) = self.destruct(); match params.iteration_order { IterationOrder::Ordered => { let (_num_threads, Ok(result)) = prc::next::x(using, orchestrator, params, iter, x1); result.map(|x| x.1) } IterationOrder::Arbitrary => { let (_num_threads, Ok(result)) = prc::next_any::x(using, orchestrator, params, iter, x1); result } } } } ================================================ FILE: src/using/executor/mod.rs ================================================ pub(super) mod parallel_compute; mod thread_compute; ================================================ FILE: src/using/executor/parallel_compute/collect_arbitrary.rs ================================================ use crate::Params; use crate::generic_values::Values; use crate::generic_values::runner_results::ParallelCollectArbitrary; use crate::runner::{ComputationKind, NumSpawned, ParallelRunner, SharedStateOf, ThreadRunnerOf}; use crate::using::executor::thread_compute as th; use crate::using::using_variants::Using; use orx_concurrent_bag::ConcurrentBag; use orx_concurrent_iter::ConcurrentIter; use orx_fixed_vec::IntoConcurrentPinnedVec; #[cfg(test)] pub fn m<'using, U, C, I, O, M1, P>( using: U, mut orchestrator: C, params: Params, iter: I, map1: M1, pinned_vec: P, ) -> (NumSpawned, P) where U: Using<'using>, C: ParallelRunner, I: ConcurrentIter, O: Send, M1: Fn(&mut U::Item, I::Item) -> O + Sync, P: IntoConcurrentPinnedVec, { let capacity_bound = pinned_vec.capacity_bound(); let offset = pinned_vec.len(); let mut bag: ConcurrentBag = pinned_vec.into(); match iter.try_get_len() { Some(iter_len) => bag.reserve_maximum_capacity(offset + iter_len), None => bag.reserve_maximum_capacity(capacity_bound), }; let thread_work = |nt: NumSpawned, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { let u = using.create(nt.into_inner()); th::collect_arbitrary::m(u, thread_runner, iter, state, &map1, &bag); }; let num_spawned = orchestrator.run_all(params, iter, ComputationKind::Collect, thread_work); let values = bag.into_inner(); (num_spawned, values) } pub fn x<'using, U, C, I, Vo, X1, P>( using: U, mut orchestrator: C, params: Params, iter: I, xap1: X1, pinned_vec: P, ) -> (NumSpawned, ParallelCollectArbitrary) where U: Using<'using>, C: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { let capacity_bound = pinned_vec.capacity_bound(); let offset = pinned_vec.len(); let mut bag: ConcurrentBag = pinned_vec.into(); match iter.try_get_len() { Some(iter_len) => bag.reserve_maximum_capacity(offset + iter_len), None => bag.reserve_maximum_capacity(capacity_bound), }; let thread_map = |nt: NumSpawned, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { let u = using.create(nt.into_inner()); th::collect_arbitrary::x(u, thread_runner, iter, state, &xap1, &bag).into_result() }; let (num_spawned, result) = orchestrator.map_all::( params, iter, ComputationKind::Collect, thread_map, ); let result = match result { Err(error) => ParallelCollectArbitrary::StoppedByError { error }, Ok(_) => ParallelCollectArbitrary::AllOrUntilWhileCollected { pinned_vec: bag.into_inner(), }, }; (num_spawned, result) } ================================================ FILE: src/using/executor/parallel_compute/collect_ordered.rs ================================================ use crate::Params; use crate::generic_values::Values; use crate::generic_values::runner_results::{Fallibility, ParallelCollect}; use crate::runner::{ComputationKind, NumSpawned, ParallelRunner, SharedStateOf, ThreadRunnerOf}; use crate::using::executor::thread_compute as th; use crate::using::using_variants::Using; use orx_concurrent_iter::ConcurrentIter; use orx_concurrent_ordered_bag::ConcurrentOrderedBag; use orx_fixed_vec::IntoConcurrentPinnedVec; pub fn m<'using, U, C, I, O, M1, P>( using: U, mut orchestrator: C, params: Params, iter: I, map1: M1, pinned_vec: P, ) -> (NumSpawned, P) where U: Using<'using>, C: ParallelRunner, I: ConcurrentIter, O: Send, M1: Fn(&mut U::Item, I::Item) -> O + Sync, P: IntoConcurrentPinnedVec, { let offset = pinned_vec.len(); let o_bag: ConcurrentOrderedBag = pinned_vec.into(); let thread_do = |nt: NumSpawned, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { let u = using.create(nt.into_inner()); th::collect_ordered::m(u, thread_runner, iter, state, &map1, &o_bag, offset); }; let num_spawned = orchestrator.run_all(params, iter, ComputationKind::Collect, thread_do); let values = unsafe { o_bag.into_inner().unwrap_only_if_counts_match() }; (num_spawned, values) } pub fn x<'using, U, C, I, Vo, X1, P>( using: U, mut orchestrator: C, params: Params, iter: I, xap1: X1, pinned_vec: P, ) -> (NumSpawned, ParallelCollect) where U: Using<'using>, C: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, ::Error: Send, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, P: IntoConcurrentPinnedVec, { let thread_map = |nt: NumSpawned, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { let u = using.create(nt.into_inner()); th::collect_ordered::x(u, thread_runner, iter, state, &xap1).into_result() }; let (num_spawned, result) = orchestrator.map_all::( params, iter, ComputationKind::Collect, thread_map, ); let result = match result { Err(error) => ParallelCollect::StoppedByError { error }, Ok(results) => ParallelCollect::reduce(results, pinned_vec), }; (num_spawned, result) } ================================================ FILE: src/using/executor/parallel_compute/mod.rs ================================================ pub(crate) mod collect_arbitrary; pub(crate) mod collect_ordered; pub(crate) mod next; pub(crate) mod next_any; pub(crate) mod reduce; ================================================ FILE: src/using/executor/parallel_compute/next.rs ================================================ use crate::Params; use crate::generic_values::Values; use crate::generic_values::runner_results::{Fallibility, NextSuccess, NextWithIdx}; use crate::runner::{ComputationKind, NumSpawned, ParallelRunner, SharedStateOf}; use crate::using::executor::thread_compute as th; use crate::using::using_variants::Using; use orx_concurrent_iter::ConcurrentIter; pub fn m<'using, U, C, I, O, M1>( using: U, mut orchestrator: C, params: Params, iter: I, map1: M1, ) -> (NumSpawned, Option) where U: Using<'using>, C: ParallelRunner, I: ConcurrentIter, O: Send, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { let thread_map = |nt: NumSpawned, iter: &I, state: &SharedStateOf, thread_runner| { let u = using.create(nt.into_inner()); Ok(th::next::m(u, thread_runner, iter, state, &map1)) }; let (num_spawned, result) = orchestrator.map_infallible(params, iter, ComputationKind::Collect, thread_map); let next = match result { Ok(results) => results .into_iter() .flatten() .min_by_key(|x| x.0) .map(|x| x.1), }; (num_spawned, next) } type ResultNext = Result< Option<(usize, ::Item)>, <::Fallibility as Fallibility>::Error, >; pub fn x<'using, U, C, I, Vo, X1>( using: U, mut orchestrator: C, params: Params, iter: I, xap1: X1, ) -> (NumSpawned, ResultNext) where U: Using<'using>, C: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { let thread_map = |nt: NumSpawned, iter: &I, state: &SharedStateOf, th_runner| { let u = using.create(nt.into_inner()); match th::next::x(u, th_runner, iter, state, &xap1) { NextWithIdx::Found { idx, value } => Ok(Some(NextSuccess::Found { idx, value })), NextWithIdx::NotFound => Ok(None), NextWithIdx::StoppedByWhileCondition { idx } => { Ok(Some(NextSuccess::StoppedByWhileCondition { idx })) } NextWithIdx::StoppedByError { error } => Err(error), } }; let (num_spawned, result) = orchestrator.map_all::( params, iter, ComputationKind::Collect, thread_map, ); let next = result.map(|results| NextSuccess::reduce(results.into_iter().flatten())); (num_spawned, next) } ================================================ FILE: src/using/executor/parallel_compute/next_any.rs ================================================ use crate::Params; use crate::generic_values::Values; use crate::generic_values::runner_results::Fallibility; use crate::runner::{ComputationKind, NumSpawned, ParallelRunner, SharedStateOf}; use crate::using::executor::thread_compute as th; use crate::using::using_variants::Using; use orx_concurrent_iter::ConcurrentIter; pub fn m<'using, U, C, I, O, M1>( using: U, mut orchestrator: C, params: Params, iter: I, map1: M1, ) -> (NumSpawned, Option) where U: Using<'using>, C: ParallelRunner, I: ConcurrentIter, O: Send, M1: Fn(&mut U::Item, I::Item) -> O + Sync, { let thread_map = |nt: NumSpawned, iter: &I, state: &SharedStateOf, thread_runner| { let u = using.create(nt.into_inner()); Ok(th::next_any::m(u, thread_runner, iter, state, &map1)) }; let (num_spawned, result) = orchestrator.map_infallible(params, iter, ComputationKind::Collect, thread_map); let next = match result { Ok(results) => results.into_iter().flatten().next(), }; (num_spawned, next) } type ResultNextAny = Result::Item>, <::Fallibility as Fallibility>::Error>; pub fn x<'using, U, C, I, Vo, X1>( using: U, mut orchestrator: C, params: Params, iter: I, xap1: X1, ) -> (NumSpawned, ResultNextAny) where U: Using<'using>, C: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, { let thread_map = |nt: NumSpawned, iter: &I, state: &SharedStateOf, th_runner| { let u = using.create(nt.into_inner()); th::next_any::x(u, th_runner, iter, state, &xap1) }; let (num_spawned, result) = orchestrator.map_all::( params, iter, ComputationKind::Collect, thread_map, ); let next = result.map(|results| results.into_iter().flatten().next()); (num_spawned, next) } ================================================ FILE: src/using/executor/parallel_compute/reduce.rs ================================================ use crate::Params; use crate::generic_values::Values; use crate::generic_values::runner_results::Fallibility; use crate::runner::{ComputationKind, NumSpawned, ParallelRunner, SharedStateOf, ThreadRunnerOf}; use crate::using::executor::thread_compute as th; use crate::using::using_variants::Using; use orx_concurrent_iter::ConcurrentIter; pub fn m<'using, U, C, I, O, M1, Red>( using: U, mut orchestrator: C, params: Params, iter: I, map1: M1, reduce: Red, ) -> (NumSpawned, Option) where U: Using<'using>, C: ParallelRunner, I: ConcurrentIter, M1: Fn(&mut U::Item, I::Item) -> O + Sync, Red: Fn(&mut U::Item, O, O) -> O + Sync, O: Send, { let thread_map = |nt: NumSpawned, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { let u = using.create(nt.into_inner()); Ok(th::reduce::m(u, thread_runner, iter, state, &map1, &reduce)) }; let (num_spawned, result) = orchestrator.map_infallible(params, iter, ComputationKind::Collect, thread_map); let mut u = using.into_inner(); let acc = match result { Ok(results) => results .into_iter() .flatten() .reduce(|a, b| reduce(&mut u, a, b)), }; (num_spawned, acc) } type ResultReduce = Result::Item>, <::Fallibility as Fallibility>::Error>; pub fn x<'using, U, C, I, Vo, X1, Red>( using: U, mut orchestrator: C, params: Params, iter: I, xap1: X1, reduce: Red, ) -> (NumSpawned, ResultReduce) where U: Using<'using>, C: ParallelRunner, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(*mut U::Item, I::Item) -> Vo + Sync, Red: Fn(*mut U::Item, Vo::Item, Vo::Item) -> Vo::Item + Sync, { let thread_map = |nt: NumSpawned, iter: &I, state: &SharedStateOf, thread_runner: ThreadRunnerOf| { let u = using.create(nt.into_inner()); th::reduce::x(u, thread_runner, iter, state, &xap1, &reduce).into_result() }; let (num_spawned, result) = orchestrator.map_all::( params, iter, ComputationKind::Collect, thread_map, ); let mut u = using.into_inner(); let acc = result.map(|results| { results .into_iter() .flatten() .reduce(|a, b| reduce(&mut u, a, b)) }); (num_spawned, acc) } ================================================ FILE: src/using/executor/thread_compute/collect_arbitrary.rs ================================================ use crate::ThreadExecutor; use crate::generic_values::Values; use crate::generic_values::runner_results::{Stop, ThreadCollectArbitrary}; use orx_concurrent_bag::ConcurrentBag; use orx_concurrent_iter::{ChunkPuller, ConcurrentIter}; use orx_fixed_vec::IntoConcurrentPinnedVec; // m #[cfg(test)] pub fn m( mut using: U, mut runner: C, iter: &I, shared_state: &C::SharedState, map1: &M1, bag: &ConcurrentBag, ) where C: ThreadExecutor, I: ConcurrentIter, M1: Fn(&mut U, I::Item) -> O, P: IntoConcurrentPinnedVec, O: Send, { let u = &mut using; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(value) => _ = bag.push(map1(u, value)), None => break, }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(chunk) => _ = bag.extend(chunk.map(|x| map1(u, x))), None => break, } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); } // x pub fn x( mut using: U, mut runner: C, iter: &I, shared_state: &C::SharedState, xap1: &X1, bag: &ConcurrentBag, ) -> ThreadCollectArbitrary where C: ThreadExecutor, I: ConcurrentIter, Vo: Values, X1: Fn(*mut U, I::Item) -> Vo, P: IntoConcurrentPinnedVec, Vo::Item: Send, { let u = &mut using; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(value) => { // TODO: possible to try to get len and bag.extend(values_vt.values()) when available, same holds for chunk below let vo = xap1(u, value); let done = vo.push_to_bag(bag); if let Some(stop) = Vo::arbitrary_push_to_stop(done) { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { Stop::DueToWhile => { return ThreadCollectArbitrary::StoppedByWhileCondition; } Stop::DueToError { error } => { return ThreadCollectArbitrary::StoppedByError { error }; } } } } None => break, }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(chunk) => { for value in chunk { let vo = xap1(u, value); let done = vo.push_to_bag(bag); if let Some(stop) = Vo::arbitrary_push_to_stop(done) { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { Stop::DueToWhile => { return ThreadCollectArbitrary::StoppedByWhileCondition; } Stop::DueToError { error } => { return ThreadCollectArbitrary::StoppedByError { error }; } } } } } None => break, } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); ThreadCollectArbitrary::AllCollected } ================================================ FILE: src/using/executor/thread_compute/collect_ordered.rs ================================================ use crate::ThreadExecutor; use crate::generic_values::Values; use crate::generic_values::runner_results::{StopWithIdx, ThreadCollect}; use alloc::vec::Vec; use orx_concurrent_iter::{ChunkPuller, ConcurrentIter}; use orx_concurrent_ordered_bag::ConcurrentOrderedBag; use orx_fixed_vec::IntoConcurrentPinnedVec; pub fn m( mut using: U, mut runner: C, iter: &I, shared_state: &C::SharedState, map1: &M1, o_bag: &ConcurrentOrderedBag, offset: usize, ) where C: ThreadExecutor, I: ConcurrentIter, M1: Fn(&mut U, I::Item) -> O, P: IntoConcurrentPinnedVec, O: Send, { let u = &mut using; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller_with_idx(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some((idx, value)) => unsafe { o_bag.set_value(offset + idx, map1(u, value)) }, None => break, }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull_with_idx() { Some((begin_idx, chunk)) => { let values = chunk.map(|x| map1(u, x)); unsafe { o_bag.set_values(offset + begin_idx, values) }; } None => break, } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); } pub fn x( mut using: U, mut runner: C, iter: &I, shared_state: &C::SharedState, xap1: &X1, ) -> ThreadCollect where C: ThreadExecutor, I: ConcurrentIter, Vo: Values, X1: Fn(*mut U, I::Item) -> Vo, { let u = &mut using; let mut collected = Vec::new(); let out_vec = &mut collected; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller_with_idx(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some((idx, i)) => { let vo = xap1(u, i); let done = vo.push_to_vec_with_idx(idx, out_vec); if let Some(stop) = Vo::ordered_push_to_stop(done) { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { StopWithIdx::DueToWhile { idx } => { return ThreadCollect::StoppedByWhileCondition { vec: collected, stopped_idx: idx, }; } StopWithIdx::DueToError { idx: _, error } => { return ThreadCollect::StoppedByError { error }; } } } } None => break, }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull_with_idx() { Some((chunk_begin_idx, chunk)) => { for (within_chunk_idx, value) in chunk.enumerate() { let vo = xap1(u, value); let done = vo.push_to_vec_with_idx(chunk_begin_idx, out_vec); if let Some(stop) = Vo::ordered_push_to_stop(done) { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { StopWithIdx::DueToWhile { idx } => { return ThreadCollect::StoppedByWhileCondition { vec: collected, stopped_idx: idx + within_chunk_idx, }; } StopWithIdx::DueToError { idx: _, error } => { return ThreadCollect::StoppedByError { error }; } } } } } None => break, } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); ThreadCollect::AllCollected { vec: collected } } ================================================ FILE: src/using/executor/thread_compute/mod.rs ================================================ pub(super) mod collect_arbitrary; pub(super) mod collect_ordered; pub(super) mod next; pub(super) mod next_any; pub(super) mod reduce; ================================================ FILE: src/using/executor/thread_compute/next.rs ================================================ use crate::{ ThreadExecutor, generic_values::Values, generic_values::runner_results::{Next, NextWithIdx}, }; use orx_concurrent_iter::{ChunkPuller, ConcurrentIter}; pub fn m( mut using: U, mut runner: C, iter: &I, shared_state: &C::SharedState, map1: &M1, ) -> Option<(usize, O)> where C: ThreadExecutor, I: ConcurrentIter, M1: Fn(&mut U, I::Item) -> O, { let u = &mut using; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller_with_idx(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some((idx, i)) => { let first = map1(u, i); iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Some((idx, first)); } None => break, }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull_with_idx() { Some((idx, mut chunk)) => { if let Some(i) = chunk.next() { let first = map1(u, i); iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Some((idx, first)); } } None => break, } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); None } pub fn x( mut using: U, mut runner: C, iter: &I, shared_state: &C::SharedState, xap1: &X1, ) -> NextWithIdx where C: ThreadExecutor, I: ConcurrentIter, Vo: Values, X1: Fn(*mut U, I::Item) -> Vo, { let u = &mut using as *mut U; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller_with_idx(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some((idx, i)) => { let vt = xap1(u, i); match vt.next() { Next::Done { value } => { if let Some(value) = value { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::Found { idx, value }; } } Next::StoppedByError { error } => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::StoppedByError { error }; } Next::StoppedByWhileCondition => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::StoppedByWhileCondition { idx }; } } } None => break, }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull_with_idx() { Some((idx, chunk)) => { for i in chunk { let vt = xap1(u, i); match vt.next() { Next::Done { value } => { if let Some(value) = value { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::Found { idx, value }; } } Next::StoppedByError { error } => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::StoppedByError { error }; } Next::StoppedByWhileCondition => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return NextWithIdx::StoppedByWhileCondition { idx }; } } } } None => break, } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); NextWithIdx::NotFound } ================================================ FILE: src/using/executor/thread_compute/next_any.rs ================================================ use crate::{ ThreadExecutor, generic_values::Values, generic_values::runner_results::{Fallibility, Next}, }; use orx_concurrent_iter::{ChunkPuller, ConcurrentIter}; pub fn m( mut using: U, mut runner: C, iter: &I, shared_state: &C::SharedState, map1: &M1, ) -> Option where C: ThreadExecutor, I: ConcurrentIter, O: Send, M1: Fn(&mut U, I::Item) -> O, { let u = &mut using; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(i) => { let first = map1(u, i); iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Some(first); } None => break, }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(mut chunk) => { if let Some(i) = chunk.next() { let first = map1(u, i); iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Some(first); } } None => break, } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); None } pub fn x( mut using: U, mut runner: C, iter: &I, shared_state: &C::SharedState, xap1: &X1, ) -> Result, ::Error> where C: ThreadExecutor, I: ConcurrentIter, Vo: Values, Vo::Item: Send, X1: Fn(*mut U, I::Item) -> Vo, { let u = &mut using as *mut U; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(i) => { let vt = xap1(u, i); match vt.next() { Next::Done { value } => { if let Some(value) = value { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Ok(Some(value)); } } Next::StoppedByError { error } => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Err(error); } Next::StoppedByWhileCondition => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Ok(None); } } } None => break, }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(chunk) => { for i in chunk { let vt = xap1(u, i); match vt.next() { Next::Done { value } => { if let Some(value) = value { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Ok(Some(value)); } } Next::StoppedByError { error } => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Err(error); } Next::StoppedByWhileCondition => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); return Ok(None); } } } } None => break, } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); Ok(None) } ================================================ FILE: src/using/executor/thread_compute/reduce.rs ================================================ use crate::{ ThreadExecutor, generic_values::{ Values, runner_results::{Reduce, StopReduce}, }, }; use orx_concurrent_iter::{ChunkPuller, ConcurrentIter}; // m pub fn m( mut u: U, mut runner: C, iter: &I, shared_state: &C::SharedState, map1: &M1, reduce: &Red, ) -> Option where C: ThreadExecutor, I: ConcurrentIter, M1: Fn(&mut U, I::Item) -> O, Red: Fn(&mut U, O, O) -> O, { let u = &mut u; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); let mut acc = None; loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(i) => { let y = map1(u, i); acc = match acc { Some(x) => Some(reduce(u, x, y)), None => Some(y), }; } None => break, }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(chunk) => { let mut res = None; for x in chunk { let b = map1(u, x); res = match res { Some(a) => Some(reduce(u, a, b)), None => Some(b), } } acc = match acc { Some(x) => match res { Some(y) => Some(reduce(u, x, y)), None => Some(x), }, None => res, }; } None => break, } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); acc } // x pub fn x( mut u: U, mut runner: C, iter: &I, shared_state: &C::SharedState, xap1: &X1, reduce: &Red, ) -> Reduce where C: ThreadExecutor, I: ConcurrentIter, Vo: Values, X1: Fn(*mut U, I::Item) -> Vo, Red: Fn(*mut U, Vo::Item, Vo::Item) -> Vo::Item, { let u = &mut u as *mut U; let mut chunk_puller = iter.chunk_puller(0); let mut item_puller = iter.item_puller(); let mut acc = None; loop { let chunk_size = runner.next_chunk_size(shared_state, iter); runner.begin_chunk(chunk_size); match chunk_size { 0 | 1 => match item_puller.next() { Some(i) => { let vo = xap1(u, i); let reduce = vo.u_acc_reduce(u, acc, reduce); acc = match Vo::reduce_to_stop(reduce) { Ok(acc) => acc, Err(stop) => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { StopReduce::DueToWhile { acc } => { return Reduce::StoppedByWhileCondition { acc }; } StopReduce::DueToError { error } => { return Reduce::StoppedByError { error }; } } } }; } None => break, }, c => { if c > chunk_puller.chunk_size() { chunk_puller = iter.chunk_puller(c); } match chunk_puller.pull() { Some(chunk) => { for i in chunk { let vo = xap1(u, i); let reduce = vo.u_acc_reduce(u, acc, reduce); acc = match Vo::reduce_to_stop(reduce) { Ok(acc) => acc, Err(stop) => { iter.skip_to_end(); runner.complete_chunk(shared_state, chunk_size); runner.complete_task(shared_state); match stop { StopReduce::DueToWhile { acc } => { return Reduce::StoppedByWhileCondition { acc }; } StopReduce::DueToError { error } => { return Reduce::StoppedByError { error }; } } } }; } } None => break, } } } runner.complete_chunk(shared_state, chunk_size); } runner.complete_task(shared_state); Reduce::Done { acc } } ================================================ FILE: src/using/mod.rs ================================================ mod collect_into; mod computational_variants; mod executor; mod u_par_iter; mod u_par_iter_option; mod u_par_iter_result; mod using_variants; pub(crate) use collect_into::UParCollectIntoCore; pub(crate) use computational_variants::{UPar, UParMap, UParXap}; pub use u_par_iter::ParIterUsing; pub use u_par_iter_option::ParIterOptionUsing; pub use u_par_iter_result::ParIterResultUsing; pub use using_variants::{Using, UsingClone, UsingFun}; ================================================ FILE: src/using/u_par_iter.rs ================================================ use crate::{ ChunkSize, IterationOrder, NumThreads, ParCollectInto, Params, RunnerWithPool, Sum, par_iter_option::IntoOption, par_iter_result::IntoResult, runner::{DefaultRunner, ParallelRunner}, using::{ ParIterOptionUsing, ParIterResultUsing, computational_variants::u_fallible_option::UParOption, using_variants::Using, }, }; use crate::{ParThreadPool, default_fns::*}; use core::cmp::Ordering; use orx_concurrent_iter::ConcurrentIter; /// Parallel iterator which allows mutable access to a variable of type `U` within its iterator methods. /// /// Note that one variable will be created per thread used by the parallel computation. pub trait ParIterUsing<'using, U, R = DefaultRunner>: Sized + Send + Sync where R: ParallelRunner, U: Using<'using>, { /// Element type of the parallel iterator. type Item; /// Returns a reference to the input concurrent iterator. fn con_iter(&self) -> &impl ConcurrentIter; /// Parameters of the parallel iterator. /// /// See [crate::ParIter::params] for details. fn params(&self) -> Params; // params transformations /// Sets the number of threads to be used in the parallel execution. /// Integers can be used as the argument with the following mapping: /// /// * `0` -> `NumThreads::Auto` /// * `1` -> `NumThreads::sequential()` /// * `n > 0` -> `NumThreads::Max(n)` /// /// /// Parameters of the parallel iterator. /// /// See [crate::ParIter::num_threads] for details. fn num_threads(self, num_threads: impl Into) -> Self; /// Sets the number of elements to be pulled from the concurrent iterator during the /// parallel execution. When integers are used as argument, the following mapping applies: /// /// * `0` -> `ChunkSize::Auto` /// * `n > 0` -> `ChunkSize::Exact(n)` /// /// Please use the default enum constructor for creating `ChunkSize::Min` variant. /// /// See [crate::ParIter::chunk_size] for details. fn chunk_size(self, chunk_size: impl Into) -> Self; /// Sets the iteration order of the parallel computation. /// /// See [crate::ParIter::iteration_order] for details. fn iteration_order(self, collect: IterationOrder) -> Self; /// Rather than the [`DefaultRunner`], uses the parallel runner `Q` which implements [`ParallelRunner`]. /// /// See [`ParIter::with_runner`] for details. /// /// [`DefaultRunner`]: crate::DefaultRunner /// [`ParIter::with_runner`]: crate::ParIter::with_runner fn with_runner( self, orchestrator: Q, ) -> impl ParIterUsing<'using, U, Q, Item = Self::Item>; /// Rather than [`DefaultPool`], uses the parallel runner with the given `pool` implementing /// [`ParThreadPool`]. /// /// See [`ParIter::with_pool`] for details. /// /// [`DefaultPool`]: crate::DefaultPool /// [`ParIter::with_pool`]: crate::ParIter::with_pool fn with_pool( self, pool: P, ) -> impl ParIterUsing<'using, U, RunnerWithPool, Item = Self::Item> where Self: Sized, { let runner = RunnerWithPool::from(pool).with_executor::(); self.with_runner(runner) } // transformations into fallible computations /// Transforms a parallel iterator where elements are of the result type; i.e., `ParIterUsing>`, /// into fallible parallel iterator with item type `T` and error type `E`; i.e., into `ParIterResultUsing`. /// /// `ParIterResultUsing` is also a parallel iterator; however, with methods specialized for handling fallible computations /// as follows: /// /// * All of its methods are based on the success path with item type of `T`. /// * However, computations short-circuit and immediately return the observed error if any of the items /// is of the `Err` variant of the result enum. /// /// See [`ParIterUsing`] for details. /// /// Unlike [crate::ParIter::into_fallible_result], the methods of `ParIterResultUsing` give a mutable reference to the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn into_fallible_result(self) -> impl ParIterResultUsing<'using, U, R, Item = T, Err = E> where Self::Item: IntoResult; /// Transforms a parallel iterator where elements are of the option type; i.e., `ParIterUsing>`, /// into fallible parallel iterator with item type `T`; i.e., into `ParIterOptionUsing`. /// /// `ParIterOptionUsing` is also a parallel iterator; however, with methods specialized for handling fallible computations /// as follows: /// /// * All of its methods are based on the success path with item type of `T`. /// * However, computations short-circuit and immediately return None if any of the items /// is of the `None` variant of the option enum. /// /// See [`ParIterOptionUsing`] for details. /// /// Unlike [crate::ParIter::into_fallible_option], the methods of `ParIterOptionUsing` give a mutable reference to the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn into_fallible_option(self) -> impl ParIterOptionUsing<'using, U, R, Item = T> where Self::Item: IntoOption, { UParOption::new( self.map(|_, x| x.into_result_with_unit_err()) .into_fallible_result(), ) } // computation transformations /// Takes a closure `map` and creates a parallel iterator which calls that closure on each element. /// /// Unlike [crate::ParIter::map], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn map(self, map: Map) -> impl ParIterUsing<'using, U, R, Item = Out> where Map: Fn(&mut U::Item, Self::Item) -> Out + Sync + Clone; /// Creates an iterator which uses a closure `filter` to determine if an element should be yielded. /// /// Unlike [crate::ParIter::filter], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn filter(self, filter: Filter) -> impl ParIterUsing<'using, U, R, Item = Self::Item> where Filter: Fn(&mut U::Item, &Self::Item) -> bool + Sync + Clone; /// Creates an iterator that works like map, but flattens nested structure. /// /// Unlike [crate::ParIter::flat_map], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn flat_map( self, flat_map: FlatMap, ) -> impl ParIterUsing<'using, U, R, Item = IOut::Item> where IOut: IntoIterator, FlatMap: Fn(&mut U::Item, Self::Item) -> IOut + Sync + Clone; /// Creates an iterator that both filters and maps. /// /// The returned iterator yields only the values for which the supplied closure `filter_map` returns `Some(value)`. /// /// `filter_map` can be used to make chains of `filter` and `map` more concise. /// /// Unlike [crate::ParIter::filter_map], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn filter_map( self, filter_map: FilterMap, ) -> impl ParIterUsing<'using, U, R, Item = Out> where FilterMap: Fn(&mut U::Item, Self::Item) -> Option + Sync + Clone; /// Does something with each element of an iterator, passing the value on. /// /// Unlike [crate::ParIter::inspect], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn inspect( self, operation: Operation, ) -> impl ParIterUsing<'using, U, R, Item = Self::Item> where Operation: Fn(&mut U::Item, &Self::Item) + Sync + Clone, { let map = move |u: &mut U::Item, x: Self::Item| { operation(u, &x); x }; self.map(map) } // special item transformations /// Creates an iterator which copies all of its elements. /// /// Unlike [crate::ParIter::copied], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn copied<'a, T>(self) -> impl ParIterUsing<'using, U, R, Item = T> where T: 'a + Copy, Self: ParIterUsing<'using, U, R, Item = &'a T>, { self.map(u_map_copy) } /// Creates an iterator which clones all of its elements. /// /// Unlike [crate::ParIter::cloned], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn cloned<'a, T>(self) -> impl ParIterUsing<'using, U, R, Item = T> where T: 'a + Clone, Self: ParIterUsing<'using, U, R, Item = &'a T>, { self.map(u_map_clone) } /// Creates an iterator that flattens nested structure. /// /// Unlike [crate::ParIter::flatten], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn flatten(self) -> impl ParIterUsing<'using, U, R, Item = ::Item> where Self::Item: IntoIterator, { let map = |_: &mut U::Item, e: Self::Item| e.into_iter(); self.flat_map(map) } // collect /// Collects all the items from an iterator into a collection. /// /// Unlike [crate::ParIter::collect_into], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn collect_into(self, output: C) -> C where C: ParCollectInto; /// Transforms an iterator into a collection. /// /// Unlike [crate::ParIter::collect], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn collect(self) -> C where C: ParCollectInto, { let output = C::empty(self.con_iter().try_get_len()); self.collect_into(output) } // reduce /// Reduces the elements to a single one, by repeatedly applying a reducing operation. /// /// See the details here: [crate::ParIter::reduce]. fn reduce(self, reduce: Reduce) -> Option where Self::Item: Send, Reduce: Fn(&mut U::Item, Self::Item, Self::Item) -> Self::Item + Sync; /// Tests if every element of the iterator matches a predicate. /// /// Unlike [crate::ParIter::all], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn all(self, predicate: Predicate) -> bool where Self::Item: Send, Predicate: Fn(&mut U::Item, &Self::Item) -> bool + Sync + Clone, { let violates = |u: &mut U::Item, x: &Self::Item| !predicate(u, x); self.find(violates).is_none() } /// Tests if any element of the iterator matches a predicate. /// /// Unlike [crate::ParIter::any], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn any(self, predicate: Predicate) -> bool where Self::Item: Send, Predicate: Fn(&mut U::Item, &Self::Item) -> bool + Sync + Clone, { self.find(predicate).is_some() } /// Consumes the iterator, counting the number of iterations and returning it. /// /// See the details here: [crate::ParIter::count]. fn count(self) -> usize { self.map(u_map_count).reduce(u_reduce_sum).unwrap_or(0) } /// Calls a closure on each element of an iterator. /// /// Unlike [crate::ParIter::for_each], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn for_each(self, operation: Operation) where Operation: Fn(&mut U::Item, Self::Item) + Sync, { let map = |u: &mut U::Item, x| operation(u, x); let _ = self.map(map).reduce(u_reduce_unit); } /// Returns the maximum element of an iterator. /// /// See the details here: [crate::ParIter::max]. fn max(self) -> Option where Self::Item: Ord + Send, { self.reduce(|_, a, b| Ord::max(a, b)) } /// Returns the element that gives the maximum value with respect to the specified `compare` function. /// /// See the details here: [crate::ParIter::max_by]. fn max_by(self, compare: Compare) -> Option where Self::Item: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |_: &mut U::Item, x, y| match compare(&x, &y) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns the element that gives the maximum value from the specified function. /// /// See the details here: [crate::ParIter::max_by_key]. fn max_by_key(self, key: GetKey) -> Option where Self::Item: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |_: &mut U::Item, x, y| match key(&x).cmp(&key(&y)) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns the minimum element of an iterator. /// /// See the details here: [crate::ParIter::min]. fn min(self) -> Option where Self::Item: Ord + Send, { self.reduce(|_, a, b| Ord::min(a, b)) } /// Returns the element that gives the minimum value with respect to the specified `compare` function. /// /// See the details here: [crate::ParIter::min_by]. fn min_by(self, compare: Compare) -> Option where Self::Item: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |_: &mut U::Item, x, y| match compare(&x, &y) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Returns the element that gives the minimum value from the specified function. /// /// See the details here: [crate::ParIter::min_by_key]. fn min_by_key(self, get_key: GetKey) -> Option where Self::Item: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |_: &mut U::Item, x, y| match get_key(&x).cmp(&get_key(&y)) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Sums the elements of an iterator. /// /// See the details here: [crate::ParIter::sum]. fn sum(self) -> Out where Self::Item: Sum, Out: Send, { self.map(Self::Item::u_map) .reduce(Self::Item::u_reduce) .unwrap_or(Self::Item::zero()) } // early exit /// Returns the first (or any) element of the iterator; returns None if it is empty. /// /// * first element is returned if default iteration order `IterationOrder::Ordered` is used, /// * any element is returned if `IterationOrder::Arbitrary` is set. /// /// See the details here: [crate::ParIter::first]. fn first(self) -> Option where Self::Item: Send; /// Searches for an element of an iterator that satisfies a `predicate`. /// /// Unlike [crate::ParIter::find], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn find(self, predicate: Predicate) -> Option where Self::Item: Send, Predicate: Fn(&mut U::Item, &Self::Item) -> bool + Sync, { self.filter(&predicate).first() } } ================================================ FILE: src/using/u_par_iter_option.rs ================================================ use crate::default_fns::{u_map_count, u_reduce_sum, u_reduce_unit}; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::Using; use crate::{ ChunkSize, IterationOrder, NumThreads, ParCollectInto, ParThreadPool, RunnerWithPool, Sum, }; use core::cmp::Ordering; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with None. /// /// Unlike [crate::ParIterOption], the threads have access to a mutable reference of the used variable in each thread. /// /// Please see [`crate::ParIterUsing`] for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). pub trait ParIterOptionUsing<'using, U, R = DefaultRunner> where R: ParallelRunner, U: Using<'using>, { /// Type of the success element, to be received as the Some variant iff the entire computation succeeds. type Item; // params transformations /// Sets the number of threads to be used in the parallel execution. /// Integers can be used as the argument with the following mapping: /// /// * `0` -> `NumThreads::Auto` /// * `1` -> `NumThreads::sequential()` /// * `n > 0` -> `NumThreads::Max(n)` /// /// See [`NumThreads`] and [`crate::ParIter::num_threads`] for details. fn num_threads(self, num_threads: impl Into) -> Self; /// Sets the number of elements to be pulled from the concurrent iterator during the /// parallel execution. When integers are used as argument, the following mapping applies: /// /// * `0` -> `ChunkSize::Auto` /// * `n > 0` -> `ChunkSize::Exact(n)` /// /// Please use the default enum constructor for creating `ChunkSize::Min` variant. /// /// See [`ChunkSize`] and [`crate::ParIter::chunk_size`] for details. fn chunk_size(self, chunk_size: impl Into) -> Self; /// Sets the iteration order of the parallel computation. /// /// See [`IterationOrder`] and [`crate::ParIter::iteration_order`] for details. fn iteration_order(self, order: IterationOrder) -> Self; /// Rather than the [`DefaultRunner`], uses the parallel runner `Q` which implements [`ParallelRunner`]. /// /// See [`ParIter::with_runner`] for details. /// /// [`DefaultRunner`]: crate::DefaultRunner /// [`ParIter::with_runner`]: crate::ParIter::with_runner fn with_runner( self, orchestrator: Q, ) -> impl ParIterOptionUsing<'using, U, Q, Item = Self::Item>; /// Rather than [`DefaultPool`], uses the parallel runner with the given `pool` implementing /// [`ParThreadPool`]. /// /// See [`ParIter::with_pool`] for details. /// /// [`DefaultPool`]: crate::DefaultPool /// [`ParIter::with_pool`]: crate::ParIter::with_pool fn with_pool( self, pool: P, ) -> impl ParIterOptionUsing<'using, U, RunnerWithPool, Item = Self::Item> where Self: Sized, { let runner = RunnerWithPool::from(pool).with_executor::(); self.with_runner(runner) } // computation transformations /// Takes a closure `map` and creates a parallel iterator which calls that closure on each element. /// /// Transformation is only for the success path where all elements are of the `Some` variant. /// Any observation of a `None` case short-circuits the computation and immediately returns None. /// /// Unlike [crate::ParIterOption::map], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn map(self, map: Map) -> impl ParIterOptionUsing<'using, U, R, Item = Out> where Self: Sized, Map: Fn(&mut U::Item, Self::Item) -> Out + Sync + Clone, Out: Send; /// Creates an iterator which uses a closure `filter` to determine if an element should be yielded. /// /// Transformation is only for the success path where all elements are of the `Some` variant. /// Any observation of a `None` case short-circuits the computation and immediately returns None. /// /// Unlike [crate::ParIterOption::filter], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn filter( self, filter: Filter, ) -> impl ParIterOptionUsing<'using, U, R, Item = Self::Item> where Self: Sized, Filter: Fn(&mut U::Item, &Self::Item) -> bool + Sync + Clone, Self::Item: Send; /// Creates an iterator that works like map, but flattens nested structure. /// /// Transformation is only for the success path where all elements are of the `Some` variant. /// Any observation of a `None` case short-circuits the computation and immediately returns None. /// /// Unlike [crate::ParIterOption::flat_map], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn flat_map( self, flat_map: FlatMap, ) -> impl ParIterOptionUsing<'using, U, R, Item = IOut::Item> where Self: Sized, IOut: IntoIterator, IOut::Item: Send, FlatMap: Fn(&mut U::Item, Self::Item) -> IOut + Sync + Clone; /// Creates an iterator that both filters and maps. /// /// The returned iterator yields only the values for which the supplied closure `filter_map` returns `Some(value)`. /// /// `filter_map` can be used to make chains of `filter` and `map` more concise. /// /// Unlike [crate::ParIterOption::filter_map], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn filter_map( self, filter_map: FilterMap, ) -> impl ParIterOptionUsing<'using, U, R, Item = Out> where Self: Sized, FilterMap: Fn(&mut U::Item, Self::Item) -> Option + Sync + Clone, Out: Send; /// Does something with each successful element of an iterator, passing the value on, provided that all elements are of Some variant; /// short-circuits and returns None otherwise. /// /// Unlike [crate::ParIterOption::inspect], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn inspect( self, operation: Operation, ) -> impl ParIterOptionUsing<'using, U, R, Item = Self::Item> where Self: Sized, Operation: Fn(&mut U::Item, &Self::Item) + Sync + Clone, Self::Item: Send; // collect /// Collects all the items from an iterator into a collection iff all elements are of Some variant. /// Early exits and returns None if any of the elements is None. /// /// Unlike [crate::ParIterOption::collect_into], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn collect_into(self, output: C) -> Option where Self::Item: Send, C: ParCollectInto; /// Transforms an iterator into a collection iff all elements are of Ok variant. /// Early exits and returns the error if any of the elements is an Err. /// /// Similar to [`Iterator::collect`], the type annotation on the left-hand-side determines /// the type of the result collection; or turbofish annotation can be used. /// /// All collections implementing [`ParCollectInto`] can be used to collect into. /// /// [`ParCollectInto`]: crate::ParCollectInto /// /// Unlike [crate::ParIterOption::collect], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn collect(self) -> Option where Self::Item: Send, C: ParCollectInto; // reduce /// Reduces the elements to a single one, by repeatedly applying a reducing operation. /// Early exits and returns None if any of the elements is None. /// /// If the iterator is empty, returns `Some(None)`; otherwise, returns `Some` of result of the reduction. /// /// Unlike [crate::ParIterOption::reduce], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn reduce(self, reduce: Reduce) -> Option> where Self::Item: Send, Reduce: Fn(&mut U::Item, Self::Item, Self::Item) -> Self::Item + Sync; /// Tests if every element of the iterator matches a predicate. /// Early exits and returns None if any of the elements is None. /// /// Unlike [crate::ParIterOption::all], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn all(self, predicate: Predicate) -> Option where Self: Sized, Self::Item: Send, Predicate: Fn(&mut U::Item, &Self::Item) -> bool + Sync, { let violates = |u: &mut U::Item, x: &Self::Item| !predicate(u, x); self.find(violates).map(|x| x.is_none()) } /// Tests if any element of the iterator matches a predicate. /// Early exits and returns None if any of the elements is None. /// /// Unlike [crate::ParIterOption::any], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn any(self, predicate: Predicate) -> Option where Self: Sized, Self::Item: Send, Predicate: Fn(&mut U::Item, &Self::Item) -> bool + Sync, { self.find(predicate).map(|x| x.is_some()) } /// Consumes the iterator, counting the number of iterations and returning it. /// Early exits and returns None if any of the elements is None. /// /// See the details here: [crate::ParIterOption::count]. fn count(self) -> Option where Self: Sized, { self.map(u_map_count) .reduce(u_reduce_sum) .map(|x| x.unwrap_or(0)) } /// Calls a closure on each element of an iterator, and returns `Ok(())` if all elements succeed. /// Early exits and returns None if any of the elements is None. /// /// Unlike [crate::ParIterOption::for_each], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn for_each(self, operation: Operation) -> Option<()> where Self: Sized, Operation: Fn(&mut U::Item, Self::Item) + Sync, { let map = |u: &mut U::Item, x: Self::Item| operation(u, x); self.map(map).reduce(u_reduce_unit).map(|_| ()) } /// Returns Some of maximum element of an iterator if all elements succeed. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// See the details here: [crate::ParIterOption::max]. fn max(self) -> Option> where Self: Sized, Self::Item: Ord + Send, { self.reduce(|_, a, b| Ord::max(a, b)) } /// Returns the element that gives the maximum value with respect to the specified `compare` function. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// See the details here: [crate::ParIterOption::max_by]. fn max_by(self, compare: Compare) -> Option> where Self: Sized, Self::Item: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |_: &mut U::Item, x, y| match compare(&x, &y) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns the element that gives the maximum value from the specified function. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// See the details here: [crate::ParIterOption::max_by_key]. fn max_by_key(self, key: GetKey) -> Option> where Self: Sized, Self::Item: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |_: &mut U::Item, x, y| match key(&x).cmp(&key(&y)) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns Some of minimum element of an iterator if all elements succeed. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// See the details here: [crate::ParIterOption::min]. fn min(self) -> Option> where Self: Sized, Self::Item: Ord + Send, { self.reduce(|_, a, b| Ord::min(a, b)) } /// Returns the element that gives the minimum value with respect to the specified `compare` function. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// See the details here: [crate::ParIterOption::min_by]. fn min_by(self, compare: Compare) -> Option> where Self: Sized, Self::Item: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |_: &mut U::Item, x, y| match compare(&x, &y) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Returns the element that gives the minimum value from the specified function. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if any of the elements is None. /// /// See the details here: [crate::ParIterOption::min_by_key]. fn min_by_key(self, get_key: GetKey) -> Option> where Self: Sized, Self::Item: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |_: &mut U::Item, x, y| match get_key(&x).cmp(&get_key(&y)) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Sums the elements of an iterator. /// Early exits and returns None if any of the elements is None. /// /// If the iterator is empty, returns zero; otherwise, returns `Some` of the sum. /// /// See the details here: [crate::ParIterOption::sum]. fn sum(self) -> Option where Self: Sized, Self::Item: Sum, Out: Send, { self.map(Self::Item::u_map) .reduce(Self::Item::u_reduce) .map(|x| x.unwrap_or(Self::Item::zero())) } // early exit /// Returns the first (or any) element of the iterator. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if a None element is observed first. /// /// * first element is returned if default iteration order `IterationOrder::Ordered` is used, /// * any element is returned if `IterationOrder::Arbitrary` is set. /// /// See the details here: [crate::ParIter::first]. fn first(self) -> Option> where Self::Item: Send; /// Returns the first (or any) element of the iterator that satisfies the `predicate`. /// If the iterator is empty, `Some(None)` is returned. /// Early exits and returns None if a None element is observed first. /// /// * first element is returned if default iteration order `IterationOrder::Ordered` is used, /// * any element is returned if `IterationOrder::Arbitrary` is set. /// /// Note that `find` itself is short-circuiting in addition to fallible computation. /// Therefore, in case the fallible iterator contains both a None and a Some element, /// the result is **not deterministic**: /// * it might be the `None` if it is observed first; /// * or `Some(element)` if the Some element satisfying the predicate is observed first. /// /// Unlike [crate::ParIterOption::find], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn find(self, predicate: Predicate) -> Option> where Self: Sized, Self::Item: Send, Predicate: Fn(&mut U::Item, &Self::Item) -> bool + Sync, { self.filter(&predicate).first() } } ================================================ FILE: src/using/u_par_iter_result.rs ================================================ use crate::default_fns::{u_map_count, u_reduce_sum, u_reduce_unit}; use crate::par_iter_result::IntoResult; use crate::runner::{DefaultRunner, ParallelRunner}; use crate::using::Using; use crate::{ ChunkSize, IterationOrder, NumThreads, ParIterUsing, ParThreadPool, RunnerWithPool, Sum, }; use crate::{ParCollectInto, generic_values::fallible_iterators::ResultOfIter}; use core::cmp::Ordering; /// A parallel iterator for which the computation either completely succeeds, /// or fails and **early exits** with an error. /// /// Unlike [crate::ParIterResult], the threads have access to a mutable reference of the used variable in each thread. /// /// Please see [`crate::ParIterUsing`] for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). pub trait ParIterResultUsing<'using, U, R = DefaultRunner> where R: ParallelRunner, U: Using<'using>, { /// Type of the Ok element, to be received as the Ok variant iff the entire computation succeeds. type Item; /// Type of the Err element, to be received if any of the computations fails. type Err; /// Element type of the regular parallel iterator this fallible iterator can be converted to, simply `Result`. type RegularItem: IntoResult; /// Regular parallel iterator this fallible iterator can be converted into. type RegularParIter: ParIterUsing<'using, U, R, Item = Self::RegularItem>; /// Returns a reference to the input concurrent iterator. fn con_iter_len(&self) -> Option; /// Converts this fallible iterator into a regular parallel iterator; i.e., [`ParIter`], with `Item = Result`. fn into_regular_par(self) -> Self::RegularParIter; /// Converts the `regular_par` iterator with `Item = Result` into fallible result iterator. fn from_regular_par(regular_par: Self::RegularParIter) -> Self; // params transformations /// Sets the number of threads to be used in the parallel execution. /// Integers can be used as the argument with the following mapping: /// /// * `0` -> `NumThreads::Auto` /// * `1` -> `NumThreads::sequential()` /// * `n > 0` -> `NumThreads::Max(n)` /// /// See [`NumThreads`] and [`ParIter::num_threads`] for details. fn num_threads(self, num_threads: impl Into) -> Self where Self: Sized, { Self::from_regular_par(self.into_regular_par().num_threads(num_threads)) } /// Sets the number of elements to be pulled from the concurrent iterator during the /// parallel execution. When integers are used as argument, the following mapping applies: /// /// * `0` -> `ChunkSize::Auto` /// * `n > 0` -> `ChunkSize::Exact(n)` /// /// Please use the default enum constructor for creating `ChunkSize::Min` variant. /// /// See [`ChunkSize`] and [`ParIter::chunk_size`] for details. fn chunk_size(self, chunk_size: impl Into) -> Self where Self: Sized, { Self::from_regular_par(self.into_regular_par().chunk_size(chunk_size)) } /// Sets the iteration order of the parallel computation. /// /// See [`IterationOrder`] and [`ParIter::iteration_order`] for details. fn iteration_order(self, order: IterationOrder) -> Self where Self: Sized, { Self::from_regular_par(self.into_regular_par().iteration_order(order)) } /// Rather than the [`DefaultRunner`], uses the parallel runner `Q` which implements [`ParallelRunner`]. /// /// See [`ParIter::with_runner`] for details. /// /// [`DefaultRunner`]: crate::DefaultRunner /// [`ParIter::with_runner`]: crate::ParIter::with_runner fn with_runner( self, orchestrator: Q, ) -> impl ParIterResultUsing<'using, U, Q, Item = Self::Item, Err = Self::Err>; /// Rather than [`DefaultPool`], uses the parallel runner with the given `pool` implementing /// [`ParThreadPool`]. /// /// See [`ParIter::with_pool`] for details. /// /// [`DefaultPool`]: crate::DefaultPool /// [`ParIter::with_pool`]: crate::ParIter::with_pool fn with_pool( self, pool: P, ) -> impl ParIterResultUsing< 'using, U, RunnerWithPool, Item = Self::Item, Err = Self::Err, > where Self: Sized, { let runner = RunnerWithPool::from(pool).with_executor::(); self.with_runner(runner) } // computation transformations /// Takes a closure `map` and creates a parallel iterator which calls that closure on each element. /// /// Transformation is only for the success path where all elements are of the `Ok` variant. /// Any observation of an `Err` case short-circuits the computation and immediately returns the observed error. /// /// Unlike [crate::ParIterResult::map], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn map( self, map: Map, ) -> impl ParIterResultUsing<'using, U, R, Item = Out, Err = Self::Err> where Self: Sized, Map: Fn(&mut U::Item, Self::Item) -> Out + Sync + Clone, Out: Send, { let par = self.into_regular_par(); let map = par.map(move |u, x| x.into_result().map(|inner| map.clone()(u, inner))); map.into_fallible_result() } /// Creates an iterator which uses a closure `filter` to determine if an element should be yielded. /// /// Transformation is only for the success path where all elements are of the `Ok` variant. /// Any observation of an `Err` case short-circuits the computation and immediately returns the observed error. /// /// Unlike [crate::ParIterResult::filter], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn filter( self, filter: Filter, ) -> impl ParIterResultUsing<'using, U, R, Item = Self::Item, Err = Self::Err> where Self: Sized, Filter: Fn(&mut U::Item, &Self::Item) -> bool + Sync + Clone, Self::Item: Send, { let par = self.into_regular_par(); let filter_map = par.filter_map(move |u, x| match x.into_result() { Ok(x) => match filter(u, &x) { true => Some(Ok(x)), false => None, }, Err(e) => Some(Err(e)), }); filter_map.into_fallible_result() } /// Creates an iterator that works like map, but flattens nested structure. /// /// Transformation is only for the success path where all elements are of the `Ok` variant. /// Any observation of an `Err` case short-circuits the computation and immediately returns the observed error. /// /// Unlike [crate::ParIterResult::flat_map], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn flat_map( self, flat_map: FlatMap, ) -> impl ParIterResultUsing<'using, U, R, Item = IOut::Item, Err = Self::Err> where Self: Sized, IOut: IntoIterator, IOut::Item: Send, FlatMap: Fn(&mut U::Item, Self::Item) -> IOut + Sync + Clone, { let par = self.into_regular_par(); let map = par.flat_map(move |u, x| match x.into_result() { Ok(x) => ResultOfIter::ok(flat_map(u, x).into_iter()), Err(e) => ResultOfIter::err(e), }); map.into_fallible_result() } /// Creates an iterator that both filters and maps. /// /// The returned iterator yields only the values for which the supplied closure `filter_map` returns `Some(value)`. /// /// `filter_map` can be used to make chains of `filter` and `map` more concise. /// The example below shows how a `map().filter().map()` can be shortened to a single call to `filter_map`. /// /// Unlike [crate::ParIterResult::filter_map], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn filter_map( self, filter_map: FilterMap, ) -> impl ParIterResultUsing<'using, U, R, Item = Out, Err = Self::Err> where Self: Sized, FilterMap: Fn(&mut U::Item, Self::Item) -> Option + Sync + Clone, Out: Send, { let par = self.into_regular_par(); let filter_map = par.filter_map(move |u, x| match x.into_result() { Ok(x) => filter_map(u, x).map(|x| Ok(x)), Err(e) => Some(Err(e)), }); filter_map.into_fallible_result() } /// Does something with each successful element of an iterator, passing the value on, provided that all elements are of Ok variant; /// short-circuits and returns the error otherwise. /// /// Unlike [crate::ParIterResult::inspect], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn inspect( self, operation: Operation, ) -> impl ParIterResultUsing<'using, U, R, Item = Self::Item, Err = Self::Err> where Self: Sized, Operation: Fn(&mut U::Item, &Self::Item) + Sync + Clone, Self::Item: Send, { let map = move |u: &mut U::Item, x: Self::Item| { operation(u, &x); x }; self.map(map) } // collect /// Collects all the items from an iterator into a collection iff all elements are of Ok variant. /// Early exits and returns the error if any of the elements is an Err. /// /// Unlike [crate::ParIterResult::collect_into], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn collect_into(self, output: C) -> Result where C: ParCollectInto, Self::Item: Send, Self::Err: Send; /// Transforms an iterator into a collection iff all elements are of Ok variant. /// Early exits and returns the error if any of the elements is an Err. /// /// Similar to [`Iterator::collect`], the type annotation on the left-hand-side determines /// the type of the result collection; or turbofish annotation can be used. /// /// All collections implementing [`ParCollectInto`] can be used to collect into. /// /// [`ParCollectInto`]: crate::ParCollectInto /// /// Unlike [crate::ParIterResult::collect], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn collect(self) -> Result where Self: Sized, Self::Item: Send, Self::Err: Send, C: ParCollectInto, { let output = C::empty(self.con_iter_len()); self.collect_into(output) } // reduce /// Reduces the elements to a single one, by repeatedly applying a reducing operation. /// Early exits and returns the error if any of the elements is an Err. /// /// If the iterator is empty, returns `Ok(None)`; otherwise, returns `Ok` of result of the reduction. /// /// Unlike [crate::ParIterResult::reduce], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn reduce(self, reduce: Reduce) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send, Reduce: Fn(&mut U::Item, Self::Item, Self::Item) -> Self::Item + Sync; /// Tests if every element of the iterator matches a predicate. /// Early exits and returns the error if any of the elements is an Err. /// /// Unlike [crate::ParIterResult::all], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn all(self, predicate: Predicate) -> Result where Self: Sized, Self::Item: Send, Self::Err: Send, Predicate: Fn(&mut U::Item, &Self::Item) -> bool + Sync, { let violates = |u: &mut U::Item, x: &Self::Item| !predicate(u, x); self.find(violates).map(|x| x.is_none()) } /// Tests if any element of the iterator matches a predicate. /// Early exits and returns the error if any of the elements is an Err. /// /// Unlike [crate::ParIterResult::any], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn any(self, predicate: Predicate) -> Result where Self: Sized, Self::Item: Send, Self::Err: Send, Predicate: Fn(&mut U::Item, &Self::Item) -> bool + Sync, { self.find(predicate).map(|x| x.is_some()) } /// Consumes the iterator, counting the number of iterations and returning it. /// Early exits and returns the error if any of the elements is an Err. /// /// See the details here: [crate::ParIterResult::count]. fn count(self) -> Result where Self: Sized, Self::Err: Send, { self.map(u_map_count) .reduce(u_reduce_sum) .map(|x| x.unwrap_or(0)) } /// Calls a closure on each element of an iterator, and returns `Ok(())` if all elements succeed. /// Early exits and returns the error if any of the elements is an Err. /// /// Unlike [crate::ParIterResult::for_each], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn for_each(self, operation: Operation) -> Result<(), Self::Err> where Self: Sized, Self::Err: Send, Operation: Fn(&mut U::Item, Self::Item) + Sync, { let map = |u: &mut U::Item, x: Self::Item| operation(u, x); self.map(map).reduce(u_reduce_unit).map(|_| ()) } /// Returns Ok of maximum element of an iterator if all elements succeed. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// See the details here: [crate::ParIterResult::max]. fn max(self) -> Result, Self::Err> where Self: Sized, Self::Err: Send, Self::Item: Ord + Send, { self.reduce(|_, a, b| Ord::max(a, b)) } /// Returns the element that gives the maximum value with respect to the specified `compare` function. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// See the details here: [crate::ParIterResult::max_by]. fn max_by(self, compare: Compare) -> Result, Self::Err> where Self: Sized, Self::Item: Send, Self::Err: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |_: &mut U::Item, x, y| match compare(&x, &y) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns the element that gives the maximum value from the specified function. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// See the details here: [crate::ParIterResult::max_by_key]. fn max_by_key(self, key: GetKey) -> Result, Self::Err> where Self: Sized, Self::Item: Send, Self::Err: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |_: &mut U::Item, x, y| match key(&x).cmp(&key(&y)) { Ordering::Greater | Ordering::Equal => x, Ordering::Less => y, }; self.reduce(reduce) } /// Returns Ok of minimum element of an iterator if all elements succeed. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// See the details here: [crate::ParIterResult::min]. fn min(self) -> Result, Self::Err> where Self: Sized, Self::Item: Ord + Send, Self::Err: Send, { self.reduce(|_, a, b| Ord::min(a, b)) } /// Returns the element that gives the maximum value with respect to the specified `compare` function. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// See the details here: [crate::ParIterResult::min_by]. fn min_by(self, compare: Compare) -> Result, Self::Err> where Self: Sized, Self::Item: Send, Self::Err: Send, Compare: Fn(&Self::Item, &Self::Item) -> Ordering + Sync, { let reduce = |_: &mut U::Item, x, y| match compare(&x, &y) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Returns the element that gives the minimum value from the specified function. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if any of the elements is an Err. /// /// See the details here: [crate::ParIterResult::min_by_key]. fn min_by_key(self, get_key: GetKey) -> Result, Self::Err> where Self: Sized, Self::Item: Send, Self::Err: Send, Key: Ord, GetKey: Fn(&Self::Item) -> Key + Sync, { let reduce = |_: &mut U::Item, x, y| match get_key(&x).cmp(&get_key(&y)) { Ordering::Less | Ordering::Equal => x, Ordering::Greater => y, }; self.reduce(reduce) } /// Sums the elements of an iterator. /// Early exits and returns the error if any of the elements is an Err. /// /// If the iterator is empty, returns zero; otherwise, returns `Ok` of the sum. /// /// See the details here: [crate::ParIterResult::sum]. fn sum(self) -> Result where Self: Sized, Self::Item: Sum, Self::Err: Send, Out: Send, { self.map(Self::Item::u_map) .reduce(Self::Item::u_reduce) .map(|x| x.unwrap_or(Self::Item::zero())) } // early exit /// Returns the first (or any) element of the iterator. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if an Err element is observed first. /// /// * first element is returned if default iteration order `IterationOrder::Ordered` is used, /// * any element is returned if `IterationOrder::Arbitrary` is set. /// /// See the details here: [crate::ParIter::first]. fn first(self) -> Result, Self::Err> where Self::Item: Send, Self::Err: Send; /// Returns the first (or any) element of the iterator that satisfies the `predicate`. /// If the iterator is empty, `Ok(None)` is returned. /// Early exits and returns the error if an Err element is observed first. /// /// * first element is returned if default iteration order `IterationOrder::Ordered` is used, /// * any element is returned if `IterationOrder::Arbitrary` is set. /// /// Note that `find` itself is short-circuiting in addition to fallible computation. /// Therefore, in case the fallible iterator contains both an Err and an Ok element, /// the result is **not deterministic**: /// * it might be the `Err` if it is observed first; /// * or `Ok(element)` if the Ok element satisfying the predicate is observed first. /// /// Unlike [crate::ParIterResult::find], the closure allows access to mutable reference of the used variable. /// /// Please see [`crate::ParIter::using`] transformation for details and examples. /// /// Further documentation can be found here: [`using.md`](https://github.com/orxfun/orx-parallel/blob/main/docs/using.md). fn find(self, predicate: Predicate) -> Result, Self::Err> where Self: Sized, Self::Item: Send, Self::Err: Send, Predicate: Fn(&mut U::Item, &Self::Item) -> bool + Sync, { self.filter(&predicate).first() } } ================================================ FILE: src/using/using_variants.rs ================================================ /// A type that can [`create`] a value per thread, which will then be send to the thread, /// and used mutable by the defined computation. /// /// [`create`]: crate::using::Using::create pub trait Using<'using>: Sync { /// Item to be used mutably by each threads used in parallel computation. type Item: 'using; /// Creates an instance of the variable to be used by the `thread_idx`-th thread. fn create(&self, thread_idx: usize) -> Self::Item; /// Consumes self and creates exactly one instance of the variable. fn into_inner(self) -> Self::Item; } /// Using variant that creates instances of each thread by cloning an initial value. pub struct UsingClone(T); impl UsingClone { pub(crate) fn new(value: T) -> Self { Self(value) } } impl Using<'static> for UsingClone { type Item = T; fn create(&self, _: usize) -> T { self.0.clone() } fn into_inner(self) -> Self::Item { self.0 } } unsafe impl Sync for UsingClone {} /// Using variant that creates instances of each thread using a closure. pub struct UsingFun where F: Fn(usize) -> T + Sync, { fun: F, } impl UsingFun where F: Fn(usize) -> T + Sync, { pub(crate) fn new(fun: F) -> Self { Self { fun } } } impl<'using, F, T> Using<'using> for UsingFun where T: 'using, F: Fn(usize) -> T + Sync, { type Item = T; fn create(&self, thread_idx: usize) -> Self::Item { (self.fun)(thread_idx) } fn into_inner(self) -> Self::Item { (self.fun)(0) } } ================================================ FILE: src/value_variants/whilst_iterators/whilst_atom_flat_map.rs ================================================ use crate::generic_values::whilst_atom::WhilstAtom; pub struct WhilstAtomFlatMapIter where Vo: IntoIterator, { current_iter: WhilstAtom, } impl WhilstAtomFlatMapIter where Vo: IntoIterator, { pub fn from_atom(atom: WhilstAtom, flat_map: Fm) -> Self where Fm: Fn(T) -> Vo, { let current_iter = match atom { WhilstAtom::Continue(x) => WhilstAtom::Continue(flat_map(x).into_iter()), WhilstAtom::Stop => WhilstAtom::Stop, }; Self { current_iter } } pub fn u_from_atom(u: &mut U, atom: WhilstAtom, flat_map: Fm) -> Self where Fm: Fn(&mut U, T) -> Vo, { let current_iter = match atom { WhilstAtom::Continue(x) => WhilstAtom::Continue(flat_map(u, x).into_iter()), WhilstAtom::Stop => WhilstAtom::Stop, }; Self { current_iter } } } impl Iterator for WhilstAtomFlatMapIter where Vo: IntoIterator, { type Item = WhilstAtom; fn next(&mut self) -> Option { match &mut self.current_iter { WhilstAtom::Continue(x) => x.next().map(WhilstAtom::Continue), // None if flat-map iterator is consumed WhilstAtom::Stop => Some(WhilstAtom::Stop), // input is Stop } } } ================================================ FILE: src/value_variants/whilst_iterators/whilst_option_flat_map.rs ================================================ use crate::generic_values::{WhilstAtom, WhilstOption}; pub struct WhilstOptionFlatMapIter where Vo: IntoIterator, { current_iter: WhilstOption, } impl WhilstOptionFlatMapIter where Vo: IntoIterator, { pub fn from_option(atom: WhilstOption, flat_map: Fm) -> Self where Fm: Fn(T) -> Vo, { let current_iter = match atom { WhilstOption::ContinueSome(x) => WhilstOption::ContinueSome(flat_map(x).into_iter()), WhilstOption::ContinueNone => WhilstOption::ContinueNone, WhilstOption::Stop => WhilstOption::Stop, }; Self { current_iter } } pub fn u_from_option(u: &mut U, atom: WhilstOption, flat_map: Fm) -> Self where Fm: Fn(&mut U, T) -> Vo, { let current_iter = match atom { WhilstOption::ContinueSome(x) => WhilstOption::ContinueSome(flat_map(u, x).into_iter()), WhilstOption::ContinueNone => WhilstOption::ContinueNone, WhilstOption::Stop => WhilstOption::Stop, }; Self { current_iter } } } impl Iterator for WhilstOptionFlatMapIter where Vo: IntoIterator, { type Item = WhilstAtom; fn next(&mut self) -> Option { match &mut self.current_iter { WhilstOption::ContinueSome(x) => x.next().map(WhilstAtom::Continue), // None if flat-map iterator is consumed WhilstOption::ContinueNone => None, // flat-map is created on None => empty iterator WhilstOption::Stop => Some(WhilstAtom::Stop), // input is Stop } } } ================================================ FILE: src/value_variants/whilst_vector.rs ================================================ use super::transformable_values::TransformableValues; use crate::generic_values::{ Values, WhilstAtom, runner_results::{ ArbitraryPush, Fallible, Infallible, Next, OrderedPush, Reduce, SequentialPush, }, whilst_iterators::WhilstAtomFlatMapIter, whilst_vector_result::WhilstVectorResult, }; use orx_concurrent_bag::ConcurrentBag; use orx_fixed_vec::IntoConcurrentPinnedVec; use orx_pinned_vec::PinnedVec; pub struct WhilstVector(pub(crate) I) where I: IntoIterator>; impl Values for WhilstVector where I: IntoIterator>, { type Item = T; type Fallibility = Infallible; fn push_to_pinned_vec

(self, vector: &mut P) -> SequentialPush where P: PinnedVec, { for x in self.0 { match x { WhilstAtom::Continue(x) => vector.push(x), WhilstAtom::Stop => return SequentialPush::StoppedByWhileCondition, } } SequentialPush::Done } fn push_to_vec_with_idx( self, idx: usize, vec: &mut Vec<(usize, Self::Item)>, ) -> OrderedPush { for x in self.0 { match x { WhilstAtom::Continue(x) => vec.push((idx, x)), WhilstAtom::Stop => return OrderedPush::StoppedByWhileCondition { idx }, } } OrderedPush::Done } fn push_to_bag

(self, bag: &ConcurrentBag) -> ArbitraryPush where P: IntoConcurrentPinnedVec, Self::Item: Send, { for x in self.0 { match x { WhilstAtom::Continue(x) => _ = bag.push(x), WhilstAtom::Stop => return ArbitraryPush::StoppedByWhileCondition, } } ArbitraryPush::Done } fn acc_reduce(self, acc: Option, reduce: X) -> Reduce where X: Fn(Self::Item, Self::Item) -> Self::Item, { let mut iter = self.0.into_iter(); let mut acc = match acc { Some(x) => x, None => { let first = iter.next(); match first { None => return Reduce::Done { acc: None }, // empty iterator but not stopped, acc is None Some(x) => match x { WhilstAtom::Continue(x) => x, WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: None }, // first element is stop, acc is None }, } } }; for x in iter { match x { WhilstAtom::Continue(x) => acc = reduce(acc, x), WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: Some(acc) }, } } Reduce::Done { acc: Some(acc) } } fn u_acc_reduce(self, u: &mut U, acc: Option, reduce: X) -> Reduce where X: Fn(&mut U, Self::Item, Self::Item) -> Self::Item, { let mut iter = self.0.into_iter(); let mut acc = match acc { Some(x) => x, None => { let first = iter.next(); match first { None => return Reduce::Done { acc: None }, // empty iterator but not stopped, acc is None Some(x) => match x { WhilstAtom::Continue(x) => x, WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: None }, // first element is stop, acc is None }, } } }; for x in iter { match x { WhilstAtom::Continue(x) => acc = reduce(u, acc, x), WhilstAtom::Stop => return Reduce::StoppedByWhileCondition { acc: Some(acc) }, } } Reduce::Done { acc: Some(acc) } } fn next(self) -> Next { match self.0.into_iter().next() { Some(x) => match x { WhilstAtom::Continue(x) => Next::Done { value: Some(x) }, WhilstAtom::Stop => Next::StoppedByWhileCondition, }, None => Next::Done { value: None }, } } } impl TransformableValues for WhilstVector where I: IntoIterator>, { fn map( self, map: M, ) -> impl TransformableValues where M: Fn(Self::Item) -> O, { let iter = self.0.into_iter().map(move |x| match x { WhilstAtom::Continue(x) => WhilstAtom::Continue(map(x)), WhilstAtom::Stop => WhilstAtom::Stop, }); WhilstVector(iter) } fn filter( self, filter: F, ) -> impl TransformableValues where F: Fn(&Self::Item) -> bool + Clone, { let iter = self.0.into_iter().filter_map(move |x| match x { WhilstAtom::Continue(x) => match filter(&x) { true => Some(WhilstAtom::Continue(x)), false => None, }, WhilstAtom::Stop => Some(WhilstAtom::Stop), }); WhilstVector(iter) } fn flat_map( self, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(Self::Item) -> Vo, { let iter = self .0 .into_iter() .flat_map(move |atom| WhilstAtomFlatMapIter::from_atom(atom, &flat_map)); WhilstVector(iter) } fn filter_map( self, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(Self::Item) -> Option, { let iter = self.0.into_iter().filter_map(move |x| match x { WhilstAtom::Continue(x) => filter_map(x).map(WhilstAtom::Continue), WhilstAtom::Stop => Some(WhilstAtom::Stop), }); WhilstVector(iter) } fn whilst( self, whilst: impl Fn(&Self::Item) -> bool, ) -> impl TransformableValues where Self: Sized, { let iter = self.0.into_iter().map(move |x| match x { WhilstAtom::Continue(x) => match whilst(&x) { true => WhilstAtom::Continue(x), false => WhilstAtom::Stop, }, WhilstAtom::Stop => WhilstAtom::Stop, }); WhilstVector(iter) } fn map_while_ok(self, map_res: Mr) -> impl Values> where Mr: Fn(Self::Item) -> Result, E: Send, { let iter = self.0.into_iter().map(move |x| match x { WhilstAtom::Continue(x) => WhilstAtom::Continue(map_res(x)), WhilstAtom::Stop => WhilstAtom::Stop, }); WhilstVectorResult(iter) } fn u_map( self, u: &mut U, map: M, ) -> impl TransformableValues where M: Fn(&mut U, Self::Item) -> O, { let iter = self.0.into_iter().map(move |x| match x { WhilstAtom::Continue(x) => WhilstAtom::Continue(map(u, x)), WhilstAtom::Stop => WhilstAtom::Stop, }); WhilstVector(iter) } fn u_filter( self, u: &mut U, filter: F, ) -> impl TransformableValues where F: Fn(&mut U, &Self::Item) -> bool, { let iter = self.0.into_iter().filter_map(move |x| match x { WhilstAtom::Continue(x) => match filter(u, &x) { true => Some(WhilstAtom::Continue(x)), false => None, }, WhilstAtom::Stop => Some(WhilstAtom::Stop), }); WhilstVector(iter) } fn u_flat_map( self, u: &mut U, flat_map: Fm, ) -> impl TransformableValues where Vo: IntoIterator, Fm: Fn(&mut U, Self::Item) -> Vo, { let iter = self .0 .into_iter() .flat_map(move |atom| WhilstAtomFlatMapIter::u_from_atom(u, atom, &flat_map)); WhilstVector(iter) } fn u_filter_map( self, u: &mut U, filter_map: Fm, ) -> impl TransformableValues where Fm: Fn(&mut U, Self::Item) -> Option, { let iter = self.0.into_iter().filter_map(move |x| match x { WhilstAtom::Continue(x) => filter_map(u, x).map(WhilstAtom::Continue), WhilstAtom::Stop => Some(WhilstAtom::Stop), }); WhilstVector(iter) } } ================================================ FILE: tests/chain.rs ================================================ use orx_parallel::*; use test_case::test_matrix; #[test_matrix([193], [0, 1], [0, 47])] fn chain_known_known(n: usize, nt: usize, chunk: usize) { let a: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let b: Vec<_> = (0..n).map(|x| (n + x).to_string()).collect(); let c: Vec<_> = a .par() .num_threads(nt) .chunk_size(chunk) .chain(&b) .collect(); assert_eq!(c.len(), 2 * n); assert_eq!(c, a.iter().chain(&b).collect::>()); } #[test_matrix([193], [0, 1], [0, 47])] fn chain_known_unknown(n: usize, nt: usize, chunk: usize) { let a: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let b: Vec<_> = (0..n) .map(|x| (n + x).to_string()) .filter(|x| !x.starts_with('x')) .collect(); let c: Vec<_> = a .par() .num_threads(nt) .chunk_size(chunk) .chain(&b) .collect(); assert_eq!(c.len(), 2 * n); assert_eq!(c, a.iter().chain(&b).collect::>()); } ================================================ FILE: tests/into_par.rs ================================================ use orx_concurrent_iter::IntoConcurrentIter; use orx_parallel::IntoParIter; fn take_into_par_into_par_bounds(a: impl IntoParIter) { let _ = a.into_par(); } fn take_into_par_into_con_iter_bounds(a: impl IntoConcurrentIter) { let _ = a.into_par(); } #[test] fn vec_into_par_iter() { let vec: Vec<_> = (0..10).map(|x| x.to_string()).collect(); take_into_par_into_par_bounds::(vec.clone()); take_into_par_into_con_iter_bounds::(vec); } #[test] fn slice_into_par_iter() { let vec: Vec<_> = (0..10).map(|x| x.to_string()).collect(); let slice = vec.as_slice(); take_into_par_into_par_bounds::<&String>(slice); take_into_par_into_con_iter_bounds::<&String>(slice); } #[test] fn range_into_par_iter() { let range = 0..10; take_into_par_into_par_bounds::(range.clone()); take_into_par_into_con_iter_bounds::(range); } ================================================ FILE: tests/iter_into_par.rs ================================================ use orx_concurrent_iter::IterIntoConcurrentIter; use orx_parallel::IterIntoParIter; fn take_iter_into_par_iterator_bounds(a: I) where I: Iterator, I::Item: Send + Sync, { let _par_iter = a.iter_into_par(); } fn take_iter_into_par_iter_into_con_iter_bounds(a: I) where I: IterIntoConcurrentIter, I::Item: Send + Sync, { let _par_iter = a.iter_into_par(); } fn take_iter_into_par_iter_into_par_bounds(a: impl IterIntoParIter) { let _ = a.iter_into_par(); } #[test] fn iter_into_par_iter() { let vec: Vec<_> = (0..10).map(|x| x.to_string()).collect(); let iter = vec.iter().filter(|x| x.as_str() != "x"); take_iter_into_par_iterator_bounds(iter); let iter = vec.iter().filter(|x| x.as_str() != "x"); take_iter_into_par_iter_into_con_iter_bounds(iter); let iter = vec.iter().filter(|x| x.as_str() != "x"); take_iter_into_par_iter_into_par_bounds(iter); } ================================================ FILE: tests/map_while_ok_collect/from_map.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_map_when_ok() { let input = 1..1025; let map = |i: usize| i - 1; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .map(map) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024).collect::>()); assert_eq!(result, expected); } #[test] fn map_while_ok_from_map_when_error() { let input = 1..1025; let map = |i: usize| i - 1; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .map(map) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } ================================================ FILE: tests/map_while_ok_collect/from_par.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_par_when_ok() { let input = 0..1024; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024).collect::>()); assert_eq!(result, expected); } #[test] fn map_while_ok_from_par_when_error() { let input = 0..1024; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } ================================================ FILE: tests/map_while_ok_collect/from_xap_chain.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_xap_chain_when_ok() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let filter = |i: &usize| i < &2000; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let map_res = |i: usize| match (11300..11350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024) .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .collect::>()); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_chain_when_error() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let filter = |i: &usize| i < &2000; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .clone() .into_par() .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_chain_whilst_when_ok() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let filter = |i: &usize| i < &2000; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let map_res = |i: usize| match (11300..11350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .clone() .into_par() .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let expected = input .into_iter() .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .take_while(|i| i < &777) .map(map_res) .collect(); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_chain_whilst_when_err() { let input = 0..1024; let flat_map = |i: usize| [i, 1024 + i, 2048 + i]; let filter = |i: &usize| i < &1024; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .clone() .into_par() .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); number < 777 && is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_chain_whilst_when_err_out_of_reach() { let input = 0..1024; let flat_map = |i: usize| [i, 1024 + i, 2048 + i]; let filter = |i: &usize| i < &1024; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let is_error = |i: &usize| (800..850).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .clone() .into_par() .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let expected = input .into_iter() .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .take_while(|i| i < &777) .map(map_res) .collect(); assert_eq!(result, expected); } ================================================ FILE: tests/map_while_ok_collect/from_xap_filter.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_xap_filter_when_ok() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .filter(filter) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024).filter(filter).collect::>()); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_filter_when_error() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .filter(filter) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_filter_whilst_when_ok() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .filter(filter) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024) .filter(filter) .take_while(|i| i < &777) .collect::>()); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_filter_whilst_when_err() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .filter(filter) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); number < 777 && is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_filter_whilst_when_err_out_of_reach() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let is_error = |i: &usize| (800..850).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .filter(filter) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024) .filter(filter) .take_while(|i| i < &777) .collect::>()); assert_eq!(result, expected); } ================================================ FILE: tests/map_while_ok_collect/from_xap_filter_map.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_xap_filter_map_when_ok() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .filter_map(filter_map) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024).filter_map(filter_map).collect::>()); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_filter_map_when_error() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .filter_map(filter_map) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_filter_map_whilst_when_ok() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .filter_map(filter_map) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024) .filter_map(filter_map) .take_while(|i| i < &777) .collect::>()); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_filter_map_whilst_when_error() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .filter_map(filter_map) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); number < 777 && is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_filter_map_whilst_when_error_out_of_reach() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let is_error = |i: &usize| (800..850).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .filter_map(filter_map) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024) .filter_map(filter_map) .take_while(|i| i < &777) .collect::>()); assert_eq!(result, expected); } ================================================ FILE: tests/map_while_ok_collect/from_xap_flat_map.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_xap_flat_map_when_ok() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let map_res = |i: usize| match (11300..11350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .flat_map(flat_map) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024).flat_map(flat_map).collect::>()); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_flat_map_when_error() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .flat_map(flat_map) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_flat_map_whilst_when_ok() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .clone() .into_par() .flat_map(flat_map) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let expected = input .flat_map(flat_map) .take_while(|i| i < &777) .map(map_res) .collect(); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_flat_map_whilst_when_error() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .flat_map(flat_map) .take_while(|i| i < &2777) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); number < 2777 && is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_flat_map_whilst_when_error_out_of_reach() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let is_error = |i: &usize| (800..850).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .flat_map(flat_map) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024) .flat_map(flat_map) .take_while(|i| i < &777) .collect::>()); assert_eq!(result, expected); } ================================================ FILE: tests/map_while_ok_collect/mod.rs ================================================ mod from_map; mod from_par; mod from_xap_chain; mod from_xap_filter; mod from_xap_filter_map; mod from_xap_flat_map; ================================================ FILE: tests/map_while_ok_collect_arbitrary/from_map.rs ================================================ use orx_parallel::*; use crate::map_while_ok_collect_arbitrary::utils::sort_if_ok; #[test] fn map_while_ok_from_map_when_ok() { let input = 1..1025; let map = |i: usize| i - 1; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .map(map) .map(map_res) .into_fallible_result() .collect(); let result = sort_if_ok(result); let expected = Ok((0..1024).collect::>()); assert_eq!(result, expected); } #[test] fn map_while_ok_from_map_when_error() { let input = 1..1025; let map = |i: usize| i - 1; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .map(map) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } ================================================ FILE: tests/map_while_ok_collect_arbitrary/from_par.rs ================================================ use orx_parallel::*; use crate::map_while_ok_collect_arbitrary::utils::sort_if_ok; #[test] fn map_while_ok_from_par_when_ok() { let input = 0..1024; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .map(map_res) .into_fallible_result() .collect(); let result = sort_if_ok(result); let expected = Ok((0..1024).collect::>()); assert_eq!(result, expected); } #[test] fn map_while_ok_from_par_when_error() { let input = 0..1024; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } ================================================ FILE: tests/map_while_ok_collect_arbitrary/from_xap_chain.rs ================================================ use crate::map_while_ok_collect_arbitrary::utils::sort_if_ok; use orx_parallel::*; #[test] fn map_while_ok_from_xap_chain_when_ok() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let filter = |i: &usize| i < &2000; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let map_res = |i: usize| match (11300..11350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024) .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .collect::>()); let result = sort_if_ok(result); let expected = sort_if_ok(expected); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_chain_when_error() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let filter = |i: &usize| i < &2000; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .clone() .into_par() .iteration_order(IterationOrder::Arbitrary) .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_chain_whilst_when_ok() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let filter = |i: &usize| i < &2000; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let map_res = |i: usize| match (11300..11350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .clone() .into_par() .iteration_order(IterationOrder::Arbitrary) .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); assert!(result.is_ok()); let result = result.unwrap(); let all_satisfies_whilst = result.iter().all(|x| x < &777); assert!(all_satisfies_whilst); } #[test] fn map_while_ok_from_xap_chain_whilst_when_err() { let input = 0..1024; let flat_map = |i: usize| [i, 1024 + i, 2048 + i]; let filter = |i: &usize| i < &1024; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .clone() .into_par() .iteration_order(IterationOrder::Arbitrary) .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); number < 777 && is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_chain_whilst_when_err_out_of_reach() { let input = 0..1024; let flat_map = |i: usize| [i, 1024 + i, 2048 + i]; let filter = |i: &usize| i < &1024; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let is_error = |i: &usize| (800..850).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .clone() .into_par() .iteration_order(IterationOrder::Arbitrary) .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); assert!(result.is_ok()); let result = result.unwrap(); let all_satisfies_whilst = result.iter().all(|x| x < &777); assert!(all_satisfies_whilst); } ================================================ FILE: tests/map_while_ok_collect_arbitrary/from_xap_filter.rs ================================================ use crate::map_while_ok_collect_arbitrary::utils::sort_if_ok; use orx_parallel::*; #[test] fn map_while_ok_from_xap_filter_when_ok() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .filter(filter) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024).filter(filter).collect::>()); let result = sort_if_ok(result); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_filter_when_error() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .filter(filter) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_filter_whilst_when_ok() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .filter(filter) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); assert!(result.is_ok()); let result = result.unwrap(); let all_satisfies_whilst = result.iter().all(|x| x < &777); assert!(all_satisfies_whilst); } #[test] fn map_while_ok_from_xap_filter_whilst_when_err() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .filter(filter) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); number < 777 && is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_filter_whilst_when_err_out_of_reach() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let is_error = |i: &usize| (800..850).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .filter(filter) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); assert!(result.is_ok()); let result = result.unwrap(); let all_satisfies_whilst = result.iter().all(|x| x < &777); assert!(all_satisfies_whilst); } ================================================ FILE: tests/map_while_ok_collect_arbitrary/from_xap_filter_map.rs ================================================ use crate::map_while_ok_collect_arbitrary::utils::sort_if_ok; use orx_parallel::*; #[test] fn map_while_ok_from_xap_filter_map_when_ok() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .filter_map(filter_map) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024).filter_map(filter_map).collect::>()); let result = sort_if_ok(result); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_filter_map_when_error() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .filter_map(filter_map) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_filter_map_whilst_when_ok() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .filter_map(filter_map) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); assert!(result.is_ok()); let result = result.unwrap(); let all_satisfies_whilst = result.iter().all(|x| x < &777); assert!(all_satisfies_whilst); } #[test] fn map_while_ok_from_xap_filter_map_whilst_when_error() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .filter_map(filter_map) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); number < 777 && is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_filter_map_whilst_when_error_out_of_reach() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let is_error = |i: &usize| (800..850).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .filter_map(filter_map) .take_while(|i| i < &777) .map(map_res) .into_fallible_result() .collect(); assert!(result.is_ok()); let result = result.unwrap(); let all_satisfies_whilst = result.iter().all(|x| x < &777); assert!(all_satisfies_whilst); } ================================================ FILE: tests/map_while_ok_collect_arbitrary/from_xap_flat_map.rs ================================================ use orx_parallel::*; use crate::map_while_ok_collect_arbitrary::utils::sort_if_ok; #[test] fn map_while_ok_from_xap_flat_map_when_ok() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let map_res = |i: usize| match (11300..11350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .flat_map(flat_map) .map(map_res) .into_fallible_result() .collect(); let expected = Ok((0..1024).flat_map(flat_map).collect::>()); let result = sort_if_ok(result); let expected = sort_if_ok(expected); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_flat_map_when_error() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .flat_map(flat_map) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_flat_map_whilst_when_ok() { let input = 0..1024; let flat_map = |i: usize| [i, 1024 + i, 2048 + i]; let map_res = |i: usize| match (11300..11350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .clone() .into_par() .iteration_order(IterationOrder::Arbitrary) .flat_map(flat_map) .take_while(|i| i < &2777) .map(map_res) .into_fallible_result() .collect(); assert!(result.is_ok()); let result = result.unwrap(); let all_satisfies_whilst = result.iter().all(|x| x < &2777); assert!(all_satisfies_whilst); } #[test] fn map_while_ok_from_xap_flat_map_whilst_when_error() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .flat_map(flat_map) .take_while(|i| i < &2777) .map(map_res) .into_fallible_result() .collect(); let result = result.map_err(|e| { let number = e.parse::().unwrap(); number < 2777 && is_error(&number) }); assert_eq!(result, Err(true)); } #[test] fn map_while_ok_from_xap_flat_map_whilst_when_error_out_of_reach() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let is_error = |i: &usize| (2800..2850).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result: Result, _> = input .into_par() .iteration_order(IterationOrder::Arbitrary) .flat_map(flat_map) .take_while(|i| i < &2777) .map(map_res) .into_fallible_result() .collect(); assert!(result.is_ok()); let result = result.unwrap(); let all_satisfies_whilst = result.iter().all(|x| x < &2777); assert!(all_satisfies_whilst); } ================================================ FILE: tests/map_while_ok_collect_arbitrary/mod.rs ================================================ mod from_map; mod from_par; mod from_xap_chain; mod from_xap_filter; mod from_xap_filter_map; mod from_xap_flat_map; mod utils; ================================================ FILE: tests/map_while_ok_collect_arbitrary/utils.rs ================================================ pub fn sort_if_ok(res_vec: Result, E>) -> Result, E> { res_vec.map(|mut x| { x.sort(); x }) } ================================================ FILE: tests/map_while_ok_reduce/from_map.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_map_when_ok() { let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = (1..1025) .into_par() .map(|i| i - 1) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let expected = Ok(Some((0..1024).sum::())); assert_eq!(result, expected); let result = (1..1) .into_par() .map(|i| i - 1) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let expected = Ok(None); assert_eq!(result, expected); } #[test] fn map_while_ok_from_map_when_error() { let input = 1..1025; let map = |i: usize| i - 1; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .map(map) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } ================================================ FILE: tests/map_while_ok_reduce/from_par.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_par_when_ok() { let input = 0..1024; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let expected = Ok(Some((0..1024).sum::())); assert_eq!(result, expected); } #[test] fn map_while_ok_from_par_when_error() { let input = 0..1024; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } ================================================ FILE: tests/map_while_ok_reduce/from_xap_chain.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_xap_chain_when_ok() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let filter = |i: &usize| i < &2000; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let map_res = |i: usize| match (11300..11350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let expected = Ok((0..1024) .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .reduce(|a, b| a + b)); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_chain_when_error() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let filter = |i: &usize| i < &2000; let map = |i: usize| 2 * i; let filter_map = |i: usize| i.is_multiple_of(2).then_some(i); let map2 = |i: usize| i / 2; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .clone() .into_par() .flat_map(flat_map) .filter(filter) .map(map) .filter_map(filter_map) .map(map2) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } ================================================ FILE: tests/map_while_ok_reduce/from_xap_filter.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_xap_filter_when_ok() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .filter(filter) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let expected = Ok(Some((0..1024).filter(filter).sum::())); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_filter_when_ok_but_none() { let input = 0..1024; let filter = |i: &usize| i > &1024; let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .filter(filter) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let expected = Ok(None); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_filter_when_error() { let input = 0..1024; let filter = |i: &usize| i % 2 == 1; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .filter(filter) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } ================================================ FILE: tests/map_while_ok_reduce/from_xap_filter_map.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_xap_filter_map_when_ok() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .filter_map(filter_map) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let expected = Ok(Some((0..1024).filter_map(filter_map).sum::())); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_filter_map_when_ok_but_none() { let input = 0..1024; let filter_map = |i: usize| (i > 1024).then_some(i); let map_res = |i: usize| match (1300..1350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .filter_map(filter_map) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let expected = Ok(None); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_filter_map_when_error() { let input = 0..1024; let filter_map = |i: usize| (i % 2 == 1).then_some(i); let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .filter_map(filter_map) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } ================================================ FILE: tests/map_while_ok_reduce/from_xap_flat_map.rs ================================================ use orx_parallel::*; #[test] fn map_while_ok_from_xap_flat_map_when_ok() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let map_res = |i: usize| match (11300..11350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .flat_map(flat_map) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let expected = Ok(Some((0..1024).flat_map(flat_map).sum())); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_flat_map_when_ok_but_none() { let input = 0..1024; let flat_map = |_: usize| []; let map_res = |i: usize| match (11300..11350).contains(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .flat_map(flat_map) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let expected = Ok(None); assert_eq!(result, expected); } #[test] fn map_while_ok_from_xap_flat_map_when_error() { let input = 0..1024; let flat_map = |i: usize| [i, 1000 + i, 2000 + i]; let is_error = |i: &usize| (300..350).contains(i) || (400..450).contains(i) || (500..550).contains(i); let map_res = |i: usize| match is_error(&i) { true => Err(i.to_string()), false => Ok(i), }; let result = input .into_par() .flat_map(flat_map) .map(map_res) .into_fallible_result() .reduce(|a, b| a + b); let result = result.map_err(|e| { let number = e.parse::().unwrap(); is_error(&number) }); assert_eq!(result, Err(true)); } ================================================ FILE: tests/map_while_ok_reduce/mod.rs ================================================ mod from_map; mod from_par; mod from_xap_chain; mod from_xap_filter; mod from_xap_filter_map; mod from_xap_flat_map; ================================================ FILE: tests/mut_iter.rs ================================================ use orx_parallel::*; use std::collections::HashMap; use std::hint::black_box; use test_case::test_matrix; #[cfg(miri)] const N: [usize; 2] = [37, 125]; #[cfg(not(miri))] const N: [usize; 2] = [1025, 4735]; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] struct Data { name: String, number: usize, } fn to_output(idx: usize) -> Data { let name = idx.to_string(); let number = idx; Data { name, number } } fn filter(data: &&mut Data) -> bool { !data.name.starts_with('3') } fn update(data: &mut Data) { for _ in 0..50 { let increment = black_box(1); data.number += increment } } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn mut_iter(n: usize, nt: usize, chunk: usize) { let input: HashMap = (0..n).map(|i| (i, to_output(i))).collect(); let mut expected = input.clone(); expected.values_mut().filter(filter).for_each(update); let expected = expected; let mut input = input.clone(); input .values_mut() .iter_into_par() .num_threads(nt) .chunk_size(chunk) .filter(filter) .for_each(update); assert_eq!(expected, input); } #[test_matrix( [0, 1, N[0], N[1]], [1, 4], [1, 64]) ] fn mut_slice(n: usize, nt: usize, chunk: usize) { let input: Vec = (0..n).map(to_output).collect(); let mut expected = input.clone(); expected.iter_mut().filter(filter).for_each(update); let expected = expected; let mut input = input.clone(); input .par_mut() .num_threads(nt) .chunk_size(chunk) .filter(filter) .for_each(update); assert_eq!(expected, input); } ================================================ FILE: tests/parallel_drainable.rs ================================================ use core::ops::Range; use orx_parallel::*; use test_case::test_matrix; #[derive(Clone, Debug)] struct VecAndRange(Vec, Range); impl VecAndRange { fn new(n: usize) -> VecAndRange { let vec: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let range = match n % 4 { 0 => 0..n, 1 => 0..(n / 2), 2 => core::cmp::min(n.saturating_sub(1), 3)..n, _ => { let a = core::cmp::min(n.saturating_sub(1), 3); let b = core::cmp::min(a + 10, n); a..b } }; Self(vec, range) } } #[test_matrix( [0, 1, 2, 3, 71, 72, 73, 74], [0, 1], [0, 1] )] fn parallel_drainable_vec(n: usize, nt: usize, chunk: usize) { let vec_and_range = VecAndRange::new(n); let VecAndRange(mut vec, range) = vec_and_range; let mut vec2 = vec.clone(); let drained: Vec<_> = vec2.drain(range.clone()).collect(); let expected = (vec2, drained); let drained: Vec<_> = vec .par_drain(range) .num_threads(nt) .chunk_size(chunk) .collect(); let result = (vec, drained); assert_eq!(result, expected); } ================================================ FILE: tests/parallelizable.rs ================================================ use orx_iterable::IntoCloningIterable; use orx_parallel::Parallelizable; fn take_parallelizable(a: impl Parallelizable) { let _ = a.par(); let _ = a.par(); } #[test] fn vec_parallelizable() { let vec: Vec<_> = (0..10).map(|x| x.to_string()).collect(); take_parallelizable::<&String>(&vec); } #[test] fn slice_parallelizable() { let vec: Vec<_> = (0..10).map(|x| x.to_string()).collect(); let slice = vec.as_slice(); take_parallelizable::<&String>(slice); } #[test] fn range_parallelizable() { let range = 0..10; take_parallelizable::(range); } #[test] fn cloning_iter_parallelizable() { let vec: Vec<_> = (0..10).map(|x| x.to_string()).collect(); let iter = vec.iter().filter(|x| x.as_str() != "x"); let cloning = iter.into_iterable(); take_parallelizable(cloning); } ================================================ FILE: tests/parallelizable_collection.rs ================================================ use orx_parallel::ParallelizableCollection; fn take_parallelizable_collection(a: impl ParallelizableCollection) { let _ = a.par(); let _ = a.par(); } #[test] fn vec_parallelizable_collection() { let vec: Vec<_> = (0..10).map(|x| x.to_string()).collect(); take_parallelizable_collection::(vec); } ================================================ FILE: tests/test_groups.rs ================================================ mod map_while_ok_collect; mod map_while_ok_collect_arbitrary; mod map_while_ok_reduce; mod using; mod whilst; pub fn fibonacci(n: u64) -> u64 { let mut a = 0; let mut b = 1; for _ in 0..n { let c = a + b; a = b; b = c; } a } ================================================ FILE: tests/trait_bounds.rs ================================================ use orx_fixed_vec::FixedVec; use orx_split_vec::SplitVec; use std::collections::VecDeque; #[test] fn trait_bounds_parallelizable() { use orx_parallel::Parallelizable; fn fun(source: impl Parallelizable) { let _iter = source.par(); } fun(vec![1, 2, 3].as_slice()); fun(&vec![1, 2, 3]); fun(&VecDeque::::new()); fun(0..9); fun(&FixedVec::::new(3)); fun(&SplitVec::::new()); } #[test] fn trait_bounds_parallelizable_collection() { use orx_parallel::ParallelizableCollection; fn fun(source: impl ParallelizableCollection) { let _iter = source.par(); } fun(vec![1, 2, 3]); fun(VecDeque::::new()); fun(FixedVec::::new(3)); fun(SplitVec::::new()); } #[test] fn trait_bounds_into_par_iter() { use orx_parallel::IntoParIter; fn fun(source: impl IntoParIter) { let _iter = source.into_par(); } // owned fun(vec![1, 2, 3]); fun(VecDeque::::new()); fun(FixedVec::::new(3)); fun(SplitVec::::new()); // ref #[allow(clippy::needless_borrows_for_generic_args)] { fun(vec![1, 2, 3].as_slice()); fun(&vec![1, 2, 3]); fun(&VecDeque::::new()); fun(0..9); fun(&FixedVec::::new(3)); fun(&SplitVec::::new()); } } ================================================ FILE: tests/using/mod.rs ================================================ mod rng; ================================================ FILE: tests/using/rng.rs ================================================ use orx_parallel::*; use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha20Rng; fn random_walk(rng: &mut impl Rng, position: i64, num_steps: usize) -> i64 { (0..num_steps).fold(position, |p, _| random_step(rng, p)) } fn random_step(rng: &mut impl Rng, position: i64) -> i64 { match rng.random_bool(0.5) { true => position + 1, // to right false => position - 1, // to left } } fn input_positions() -> Vec { (-50..=50).collect() } #[test] fn using_rng_map() { let positions = input_positions(); let _ = positions.iter().sum::(); let final_positions: Vec<_> = positions .par() .copied() .using(|t_idx| ChaCha20Rng::seed_from_u64(42 * t_idx as u64)) .map(|rng, position| random_walk(rng, position, 100)) .collect(); let _ = final_positions.iter().sum::(); } #[test] fn using_rng_xap() { let positions = input_positions(); let _ = positions.iter().sum::(); let final_positions: Vec<_> = positions .par() .copied() .using(|t_idx| ChaCha20Rng::seed_from_u64(42 * t_idx as u64)) .filter(|rng, _| rng.random_bool(0.7)) .map(|rng, position| random_walk(rng, position, 100)) .collect(); let _ = final_positions.iter().sum::(); } #[cfg(not(miri))] #[test] fn using_rng_xap_long() { let positions = input_positions(); let _ = positions.iter().sum::(); let final_positions: Vec<_> = positions .par() .copied() .using(|t_idx| ChaCha20Rng::seed_from_u64(42 * t_idx as u64)) .filter(|rng, _| rng.random_bool(0.7)) .filter_map(|rng, position| rng.random_bool(0.9).then_some(position)) .filter(|rng, _| rng.random_bool(0.7)) .flat_map(|_, position| [position, position]) .map(|rng, position| random_walk(rng, position, 100)) .collect(); let _ = final_positions.iter().sum::(); } ================================================ FILE: tests/whilst/collect.rs ================================================ use crate::fibonacci; use orx_parallel::*; use std::hint::black_box; use test_case::test_case; #[test_case(0, 0, 0, 0, "0")] #[test_case(512, 4, 0, 3, "22")] #[test_case(1024, 3, 0, 3, "84")] #[test_case(1024, 4, 1, 3, "84")] #[test_case(1024, 2, 0, 2, "5")] fn par(n: usize, nt: usize, c: usize, until_num_digits: usize, until_digits: &str) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let output: Vec<_> = input .into_par() .num_threads(nt) .chunk_size(c) .take_while(|x| { let _fib = black_box(fibonacci(42)); x.len() != until_num_digits || !x.starts_with(until_digits) }) .collect(); let expected: Vec<_> = (0..n) .map(|x| x.to_string()) .take_while(|x| x.len() != until_num_digits || !x.starts_with(until_digits)) .collect(); assert_eq!(output, expected); } #[test_case(0, 0, 0, 0, "0")] #[test_case(512, 4, 0, 3, "82")] #[test_case(1024, 3, 0, 3, "84")] #[test_case(1024, 4, 1, 3, "84")] #[test_case(1024, 2, 0, 2, "8")] fn map(n: usize, nt: usize, c: usize, until_num_digits: usize, until_digits: &str) { let input = 0..n; let output: Vec<_> = input .into_par() .num_threads(nt) .chunk_size(c) .map(|x| x.to_string()) .take_while(|x| { let _fib = black_box(fibonacci(42)); x.len() != until_num_digits || !x.starts_with(until_digits) }) .collect(); let expected: Vec<_> = (0..n) .map(|x| x.to_string()) .take_while(|x| x.len() != until_num_digits || !x.starts_with(until_digits)) .collect(); assert_eq!(output, expected); } #[test_case(512, 0, 0, &["55"], &["55"])] #[test_case(512, 4, 1, &["55", "222"], &["55"])] #[test_case(512, 3, 0, &["55", "222"], &["55"])] #[test_case(512, 5, 0, &["333"], &["444"])] fn xap_filter(n: usize, nt: usize, c: usize, stop_at: &[&str], filter_out: &[&str]) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let output: Vec<_> = input .into_par() .num_threads(nt) .chunk_size(c) .filter(|x| !filter_out.contains(&x.as_str())) .take_while(|x| { let _fib = black_box(fibonacci(42)); !stop_at.contains(&x.as_str()) }) .collect(); let expected: Vec<_> = (0..n) .map(|x| x.to_string()) .filter(|x| !filter_out.contains(&x.as_str())) .take_while(|x| !stop_at.contains(&x.as_str())) .collect(); assert_eq!(output, expected); } #[test_case(512, 0, 0, &["55"], &["55"])] #[test_case(512, 4, 1, &["55", "222"], &["55"])] #[test_case(512, 3, 0, &["55", "222"], &["55"])] #[test_case(512, 5, 0, &["333"], &["444"])] fn xap_filter_map(n: usize, nt: usize, c: usize, stop_at: &[&str], filter_out: &[&str]) { let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let output: Vec<_> = input .into_par() .num_threads(nt) .chunk_size(c) .filter_map(|x| (!filter_out.contains(&x.as_str())).then_some(x)) .take_while(|x| { let _fib = black_box(fibonacci(42)); !stop_at.contains(&x.as_str()) }) .collect(); let expected: Vec<_> = (0..n) .map(|x| x.to_string()) .filter_map(|x| (!filter_out.contains(&x.as_str())).then_some(x)) .take_while(|x| !stop_at.contains(&x.as_str())) .collect(); assert_eq!(output, expected); } #[test_case(1024, 0, 0, &[555], None)] #[test_case(1024, 4, 0, &[555], Some("!"))] #[test_case(1024, 3, 0, &[555], Some("?"))] #[test_case(1024, 3, 0, &[554,555,556,557,558,559], Some("?"))] fn xap_flat_map(n: usize, nt: usize, c: usize, stop_at: &[usize], stop_at_char: Option<&str>) { let input = 0..n; let output: Vec<_> = input .into_par() .num_threads(nt) .chunk_size(c) .flat_map(|i| [i.to_string(), format!("{i}!"), format!("{i}?")]) .take_while(|x| { let _fib = black_box(fibonacci(42)); let s = match x.ends_with("!") || x.ends_with("?") { true => &x[0..(x.len() - 1)], false => x.as_str(), }; let num: usize = s.parse().unwrap(); !stop_at.contains(&num) || stop_at_char .map(|stop_at_char| !x.ends_with(stop_at_char)) .unwrap_or(false) }) .collect(); let expected: Vec<_> = (0..n) .flat_map(|i| [i.to_string(), format!("{i}!"), format!("{i}?")]) .take_while(|x| { let s = match x.ends_with("!") || x.ends_with("?") { true => &x[0..(x.len() - 1)], false => x.as_str(), }; let num: usize = s.parse().unwrap(); !stop_at.contains(&num) || stop_at_char .map(|stop_at_char| !x.ends_with(stop_at_char)) .unwrap_or(false) }) .collect(); assert_eq!(output, expected); } ================================================ FILE: tests/whilst/collect_arbitrary.rs ================================================ use crate::fibonacci; use orx_parallel::*; use std::hint::black_box; use test_case::test_case; #[test_case(512, 4, 0, 3, "22", 220)] #[test_case(1024, 3, 0, 3, "84", 840)] #[test_case(1024, 4, 1, 3, "84", 840)] #[test_case(1024, 2, 0, 2, "5", 50)] fn par(n: usize, nt: usize, c: usize, until_num_digits: usize, until_digits: &str, min_len: usize) { let whilst = |x: &String| x.len() != until_num_digits || !x.starts_with(until_digits); let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let output: Vec<_> = input .into_par() .num_threads(nt) .chunk_size(c) .iteration_order(IterationOrder::Arbitrary) .take_while(|x| { let _fib = black_box(fibonacci(42)); whilst(x) }) .collect(); assert!(output.len() >= min_len); assert!(output.iter().all(whilst)); } #[test_case(512, 4, 0, 3, "32", 320)] #[test_case(1024, 3, 0, 3, "84", 840)] #[test_case(1024, 4, 1, 3, "84", 840)] #[test_case(1024, 2, 0, 2, "8", 80)] fn map(n: usize, nt: usize, c: usize, until_num_digits: usize, until_digits: &str, min_len: usize) { let whilst = |x: &String| x.len() != until_num_digits || !x.starts_with(until_digits); let input = 0..n; let output: Vec<_> = input .into_par() .num_threads(nt) .chunk_size(c) .iteration_order(IterationOrder::Arbitrary) .map(|x| x.to_string()) .take_while(|x| { let _fib = black_box(fibonacci(42)); whilst(x) }) .collect(); assert!(output.len() >= min_len); assert!(output.iter().all(whilst)); } #[test_case(512, 0, 0, &["55"], &["55"], 511)] #[test_case(512, 4, 1, &["55", "222"], &["55"], 221)] #[test_case(512, 3, 0, &["55", "222"], &["55"], 221)] #[test_case(512, 5, 0, &["333"], &["444"], 333)] fn xap_filter( n: usize, nt: usize, c: usize, stop_at: &[&str], filter_out: &[&str], min_len: usize, ) { let whilst = |x: &String| !stop_at.contains(&x.as_str()); let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let output: Vec<_> = input .into_par() .num_threads(nt) .chunk_size(c) .iteration_order(IterationOrder::Arbitrary) .filter(|x| !filter_out.contains(&x.as_str())) .take_while(|x| { let _fib = black_box(fibonacci(42)); whilst(x) }) .collect(); assert!(output.len() >= min_len); assert!(output.iter().all(whilst)); } #[test_case(512, 0, 0, &["55"], &["55"], 511)] #[test_case(512, 4, 1, &["55", "222"], &["55"], 221)] #[test_case(512, 3, 0, &["55", "222"], &["55"], 221)] #[test_case(512, 5, 0, &["333"], &["444"], 333)] fn xap_filter_map( n: usize, nt: usize, c: usize, stop_at: &[&str], filter_out: &[&str], min_len: usize, ) { let whilst = |x: &String| !stop_at.contains(&x.as_str()); let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let output: Vec<_> = input .into_par() .num_threads(nt) .chunk_size(c) .iteration_order(IterationOrder::Arbitrary) .filter_map(|x| (!filter_out.contains(&x.as_str())).then_some(x)) .take_while(|x| { let _fib = black_box(fibonacci(42)); whilst(x) }) .collect(); assert!(output.len() >= min_len); assert!(output.iter().all(whilst)); } #[test_case(1024, 0, 0, &[555], None, 555*3)] #[test_case(1024, 4, 0, &[555], Some("!"), 555*3+1)] #[test_case(1024, 3, 0, &[555], Some("?"), 555*3+2)] #[test_case(1024, 3, 0, &[554,555,556,557,558,559], Some("?"), 554*3)] fn xap_flat_map( n: usize, nt: usize, c: usize, stop_at: &[usize], stop_at_char: Option<&str>, min_len: usize, ) { let whilst = |x: &String| { let s = match x.ends_with("!") || x.ends_with("?") { true => &x[0..(x.len() - 1)], false => x.as_str(), }; let num: usize = s.parse().unwrap(); !stop_at.contains(&num) || stop_at_char .map(|stop_at_char| !x.ends_with(stop_at_char)) .unwrap_or(false) }; let input = 0..n; let output: Vec<_> = input .into_par() .num_threads(nt) .chunk_size(c) .iteration_order(IterationOrder::Arbitrary) .flat_map(|i| [i.to_string(), format!("{i}!"), format!("{i}?")]) .take_while(|x| { let _fib = black_box(fibonacci(42)); whilst(x) }) .collect(); assert!(output.len() >= min_len); assert!(output.iter().all(whilst)); } ================================================ FILE: tests/whilst/find.rs ================================================ use crate::fibonacci; use orx_parallel::*; use std::hint::black_box; use test_case::test_case; #[test_case(511, 0, 0, &[333], &[333], None)] #[test_case(511, 0, 0, &[333], &[332], Some(332))] #[test_case(511, 0, 0, &[222, 333], &[223, 224, 225, 332], None)] #[test_case(511, 0, 0, &[171], &[172], None)] fn par(n: usize, nt: usize, c: usize, stop_at: &[usize], find: &[usize], expected: Option) { let filter = |x: &String| find.contains(&x.parse().unwrap()); let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let output = input .into_par() .num_threads(nt) .chunk_size(c) .take_while(|x| { let _fib = black_box(fibonacci(42)); let num: usize = x.parse().unwrap(); !stop_at.contains(&num) }) .find(filter) .map(|x| x.parse().unwrap()); assert_eq!(output, expected); } #[test_case(511, 0, 0, &[333], &[334], Some(334))] #[test_case(511, 0, 0, &[334], &[332], Some(332))] #[test_case(511, 0, 0, &[222, 333], &[223, 224, 225, 332], None)] #[test_case(511, 0, 0, &[170], &[172], None)] fn filter( n: usize, nt: usize, c: usize, stop_at: &[usize], find: &[usize], expected: Option, ) { let filter = |x: &String| find.contains(&x.parse().unwrap()); let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let output = input .into_par() .num_threads(nt) .chunk_size(c) .filter(|x| x.parse::().unwrap() % 2 == 0) // only evens remain .take_while(|x| { let _fib = black_box(fibonacci(42)); let num: usize = x.parse().unwrap(); !stop_at.contains(&num) }) .find(filter) .map(|x| x.parse().unwrap()); assert_eq!(output, expected); } #[test_case(511, 0, 0, &["333!"], &[333], Some("333".to_string()))] #[test_case(511, 0, 0, &["334?"], &[332], Some("332".to_string()))] #[test_case(511, 0, 0, &["222!","333"], &[223, 224, 225, 332], None)] #[test_case(511, 0, 0, &["170!"], &[170], Some("170".to_string()))] fn flat_map( n: usize, nt: usize, c: usize, stop_at: &[&str], find: &[usize], expected: Option, ) { let filter = |x: &String| { let s = match x.ends_with("!") || x.ends_with("?") { true => &x[0..(x.len() - 1)], false => x.as_str(), }; let num: usize = s.parse().unwrap(); find.contains(&num) }; let input: Vec<_> = (0..n).map(|x| x.to_string()).collect(); let output = input .into_par() .num_threads(nt) .chunk_size(c) .flat_map(|i| [i.to_string(), format!("{i}!"), format!("{i}?")]) .take_while(|x| { let _fib = black_box(fibonacci(42)); !stop_at.contains(&x.as_str()) }) .find(filter) .map(|x| x.parse().unwrap()); assert_eq!(output, expected); } ================================================ FILE: tests/whilst/mod.rs ================================================ mod collect; mod collect_arbitrary; mod find; mod reduce; ================================================ FILE: tests/whilst/reduce.rs ================================================ use crate::fibonacci; use orx_parallel::*; use std::hint::black_box; use test_case::test_case; #[test_case(511, 0, 0, 333, 332*331/2)] #[test_case(511, 0, 0, 1000, 510*509/2)] #[test_case(511, 0, 0, 222, 221*220/2)] #[test_case(511, 0, 0, 171, 170*169/2)] fn par(n: usize, nt: usize, c: usize, stop_at: usize, expected_min: usize) { let input = 0..n; let output = input .into_par() .num_threads(nt) .chunk_size(c) .take_while(|x| { let _fib = black_box(fibonacci(42)); *x != stop_at }) .reduce(|x, y| x + y); assert!(output.unwrap() >= expected_min); } #[test_case(511, 0, 0, 333, 332*331/2)] #[test_case(511, 0, 0, 1000, 510*509/2)] #[test_case(511, 0, 0, 222, 221*220/2)] #[test_case(511, 0, 0, 171, 170*169/2)] fn www_map(n: usize, nt: usize, c: usize, stop_at: usize, expected_min: usize) { let input = 0..n; let output = input .into_par() .num_threads(nt) .chunk_size(c) .map(|x| x.to_string()) .take_while(|x| { let _fib = black_box(fibonacci(42)); let x: usize = x.parse().unwrap(); x != stop_at }) .map(|x| x.parse::().unwrap()) .reduce(|x, y| x + y); assert!(output.unwrap() >= expected_min); } #[test_case(511, 0, 0, 333, 332*331/2)] #[test_case(511, 0, 0, 1000, 510*509/2)] #[test_case(511, 0, 0, 222, 221*220/2)] #[test_case(511, 0, 0, 171, 170*169/2)] fn filter(n: usize, nt: usize, c: usize, stop_at: usize, expected_min: usize) { let input = 0..n; let output = input .into_par() .num_threads(nt) .chunk_size(c) .filter(|x| x != &42) .take_while(|x| { let _fib = black_box(fibonacci(42)); *x != stop_at }) .reduce(|x, y| x + y); assert!(output.unwrap() >= expected_min - 42); } #[test_case(511, 0, 0, 333, 332*331/2)] #[test_case(511, 0, 0, 1000, 510*509/2)] #[test_case(511, 0, 0, 222, 221*220/2)] #[test_case(511, 0, 0, 171, 170*169/2)] fn flatmap(n: usize, nt: usize, c: usize, stop_at: usize, expected_min: usize) { let input = 0..n; let output = input .into_par() .num_threads(nt) .chunk_size(c) .flat_map(|x| [x, x, x]) .take_while(|x| { let _fib = black_box(fibonacci(42)); *x != stop_at }) .reduce(|x, y| x + y); assert!(output.unwrap() >= expected_min * 3); }